From 8fc469c869a725a16bfda74827b2b288c6ad02d2 Mon Sep 17 00:00:00 2001
From: Jim Bacon <jame2@ceh.ac.uk>
Date: Thu, 22 Apr 2021 10:52:46 +0100
Subject: [PATCH 01/93] MB_OVERLOAD_STRING no longer defined

Function overloading by mbstring has been removed.
---
 system/core/utf8.php | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/system/core/utf8.php b/system/core/utf8.php
index 9f20f421c7..61f6933aa9 100644
--- a/system/core/utf8.php
+++ b/system/core/utf8.php
@@ -49,7 +49,8 @@
 	);
 }
 
-if (extension_loaded('mbstring') AND (ini_get('mbstring.func_overload') & MB_OVERLOAD_STRING))
+// Function overloading was deprecated in PHP 7.2 and removed in PHP 8.0.
+if (extension_loaded('mbstring') AND defined('MB_OVERLOAD_STRING') AND (ini_get('mbstring.func_overload') & MB_OVERLOAD_STRING))
 {
 	trigger_error
 	(

From 8371b044038ae7a9779ff075a71b812699f6e173 Mon Sep 17 00:00:00 2001
From: Jim Bacon <jame2@ceh.ac.uk>
Date: Thu, 22 Apr 2021 12:19:07 +0100
Subject: [PATCH 02/93] Remove faux default from required argument

---
 system/libraries/drivers/Database/Pgsql.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/system/libraries/drivers/Database/Pgsql.php b/system/libraries/drivers/Database/Pgsql.php
index 0731e6eb1c..a6c72f3587 100644
--- a/system/libraries/drivers/Database/Pgsql.php
+++ b/system/libraries/drivers/Database/Pgsql.php
@@ -326,7 +326,7 @@ class Pgsql_Result extends Database_Result {
    * @param  boolean   return objects or arrays
    * @param  string    SQL query that was run
    */
-  public function __construct($result, $link, $object = TRUE, $sql)
+  public function __construct($result, $link, $object, $sql)
   {
     $this->link = $link;
     $this->result = $result;

From 1269780200ad5b5425bb9bfbee7a7d2953f96cae Mon Sep 17 00:00:00 2001
From: Jim Bacon <jame2@ceh.ac.uk>
Date: Thu, 22 Apr 2021 14:43:34 +0100
Subject: [PATCH 03/93] Fix for Postgres13 compatibility

The query was getting columns for the entity from  both the warehouse
schema and the information_schema. In pg13 a 'name' datatype is used rather
than varchar for system info which is an unlisted sql_type. Rather than
add that type, maybe we were processing data we didn't need so the
information_schema columns are filtered out.
This may prove to be a mistake!
---
 application/helpers/postgreSQL.php | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/application/helpers/postgreSQL.php b/application/helpers/postgreSQL.php
index a66a808758..4ab538fb6f 100644
--- a/application/helpers/postgreSQL.php
+++ b/application/helpers/postgreSQL.php
@@ -354,7 +354,8 @@ public static function list_fields($entity, $db = NULL) {
         SELECT column_name, column_default, is_nullable, data_type, udt_name,
           character_maximum_length, numeric_precision, numeric_precision_radix, numeric_scale
         FROM information_schema.columns
-        WHERE table_name = \'' . $entity . '\'
+        WHERE table_name = \'' . $entity . '\' 
+        AND table_schema != \'information_schema\'
         ORDER BY ordinal_position
       ');
 

From 3dcd7d054e79ca7de4b384c7841a7d96843bc732 Mon Sep 17 00:00:00 2001
From: Jim Bacon <jame2@ceh.ac.uk>
Date: Thu, 22 Apr 2021 15:03:49 +0100
Subject: [PATCH 04/93] docker: Update to use PHP 8.0

---
 docker/warehouse/Dockerfile | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/docker/warehouse/Dockerfile b/docker/warehouse/Dockerfile
index 961c80142f..4a435c1390 100644
--- a/docker/warehouse/Dockerfile
+++ b/docker/warehouse/Dockerfile
@@ -1,8 +1,6 @@
-# This image contains Debian's Apache httpd in conjunction with PHP7.3
-# (as mod_php) and uses mpm_prefork by default.
+# This image contains Debian's Apache httpd in conjunction with PHP8.0.
 # https://hub.docker.com/_/php
-# Currently warehouse is not compatible with PHP 7.4
-FROM php:7.3-apache
+FROM php:8.0-apache
 # Use PHP development configuration file
 RUN mv "$PHP_INI_DIR/php.ini-development" "$PHP_INI_DIR/php.ini"
 # Add a tool to assist with installing PHP extensions.

From dfed1c99ec87bb39678e8f67053aa72a6eba9d55 Mon Sep 17 00:00:00 2001
From: Jim Bacon <jame2@ceh.ac.uk>
Date: Fri, 30 Apr 2021 16:57:34 +0100
Subject: [PATCH 05/93] Fix deprecated curly brace syntax

---
 system/vendor/swift/Swift/Connection/SMTP.php | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/system/vendor/swift/Swift/Connection/SMTP.php b/system/vendor/swift/Swift/Connection/SMTP.php
index a16f0306c1..393261b8ae 100644
--- a/system/vendor/swift/Swift/Connection/SMTP.php
+++ b/system/vendor/swift/Swift/Connection/SMTP.php
@@ -253,7 +253,7 @@ public function read()
         $this->smtpErrors());
       }
       $ret .= trim($tmp) . "\r\n";
-      if ($tmp{3} == " ") break;
+      if ($tmp[3] == " ") break;
     }
     return $ret = substr($ret, 0, -2);
   }
@@ -385,7 +385,7 @@ public function runAuthenticators($user, $pass, Swift $swift)
     foreach ($this->authenticators as $name => $obj)
     {
       //Server supports this authentication mechanism
-      if (in_array($name, $this->getAttributes("AUTH")) || $name{0} == "*")
+      if (in_array($name, $this->getAttributes("AUTH")) || $name[0] == "*")
       {
         $tried++;
         if ($log->hasLevel(Swift_Log::LOG_EVERYTHING))

From 53cc9370410bb5afc4cb2c6a5381961d85f684a1 Mon Sep 17 00:00:00 2001
From: Jim Bacon <jame2@ceh.ac.uk>
Date: Tue, 4 May 2021 13:48:33 +0100
Subject: [PATCH 06/93] Fix TypeError when table has no serial column

With PHP 7.3, if $result is false, pg_fetch_array returns null.
$result is expected to be false for tables that have no serial column.
With PHP 8.0, a TypeError exception is raised.
---
 system/libraries/drivers/Database/Pgsql.php | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/system/libraries/drivers/Database/Pgsql.php b/system/libraries/drivers/Database/Pgsql.php
index a6c72f3587..3a09101cf0 100644
--- a/system/libraries/drivers/Database/Pgsql.php
+++ b/system/libraries/drivers/Database/Pgsql.php
@@ -453,10 +453,12 @@ public function insert_id()
       $ER = error_reporting(0);
 
       $result = pg_query($this->link, $query);
-      $insert_id = pg_fetch_array($result, NULL, PGSQL_ASSOC);
-
-      $this->insert_id = $insert_id['insert_id'];
-
+      // $result is false when tables have no serial column.
+      if ($result !== false) {
+        $insert_id = pg_fetch_array($result, NULL, PGSQL_ASSOC);
+        $this->insert_id = $insert_id['insert_id'];
+      }
+        
       // Reset error reporting
       error_reporting($ER);
     }

From c861692ae50148dd0bd8f786f3b30cdfee9746ba Mon Sep 17 00:00:00 2001
From: Jim Bacon <jame2@ceh.ac.uk>
Date: Tue, 4 May 2021 15:16:14 +0100
Subject: [PATCH 07/93] Fix typo

---
 modules/data_cleaner/controllers/verification_rule.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/data_cleaner/controllers/verification_rule.php b/modules/data_cleaner/controllers/verification_rule.php
index 703c204efb..8166eeb60b 100644
--- a/modules/data_cleaner/controllers/verification_rule.php
+++ b/modules/data_cleaner/controllers/verification_rule.php
@@ -127,7 +127,7 @@ private function get_server_list() {
     $response = curl_exec($session);
     if (curl_errno($session)) {
       $this->session->set_flash('flash_info', 'The list of verification rule servers could not be retrieved from the internet. ' .
-          'More information is avaailable in the server logs.');
+          'More information is available in the server logs.');
       kohana::log('error', 'Error occurred when retrieving list of verification rule servers. ' . curl_error($session));
       return array();
     }

From cbb55782f820905542d5893d2c60cfec3a8c19aa Mon Sep 17 00:00:00 2001
From: Jim Bacon <jame2@ceh.ac.uk>
Date: Wed, 30 Jun 2021 13:22:10 +0100
Subject: [PATCH 08/93] ci: Test PHP 8

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 2f67857ee5..6177d93c52 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,7 +6,7 @@ language: php
 php:
   # Test oldest and newest maintained versions.
   - '7.3'
-  # - '8.0'
+  - '8.0'
 
 env:
   # Test oldest and newest maintained versions.

From a6a300899ee8a5d26876bd904ac8366d6eb1791e Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Thu, 22 Jul 2021 18:02:31 +0100
Subject: [PATCH 09/93] Linting

---
 .../rest_api/controllers/services/rest.php    | 244 +++++++++++-------
 1 file changed, 151 insertions(+), 93 deletions(-)

diff --git a/modules/rest_api/controllers/services/rest.php b/modules/rest_api/controllers/services/rest.php
index 3124839378..29d22a01a0 100644
--- a/modules/rest_api/controllers/services/rest.php
+++ b/modules/rest_api/controllers/services/rest.php
@@ -65,9 +65,36 @@ class RestApiAbort extends Exception {}
  * Simple object to keep globally useful stuff in.
  */
 class RestObjects {
+
+  /**
+   * Database connection.
+   *
+   * @var object
+   */
   public static $db;
+
+  /**
+   * Website ID the client is authenticated as.
+   *
+   * @var int
+   */
   public static $clientWebsiteId;
+
+  /**
+   * User ID the client is authenticated as.
+   *
+   * @var int
+   */
   public static $clientUserId;
+
+  /**
+   * Name of the warehouse module handling the request.
+   *
+   * Might not be rest_api if the REST services extended by a modules's plugin
+   * file.
+   *
+   * @var string
+   */
   public static $handlerModule;
 
   /**
@@ -815,7 +842,8 @@ public function __call($name, $arguments) {
             $requestForId = $ids[0];
           }
           // When using a client system ID, we also want a project ID in most cases.
-          if (isset(RestObjects::$clientSystemId) && !in_array($name, ['projects', 'taxa'])) {
+          if (isset(RestObjects::$clientSystemId)
+              && !in_array($name, ['projects', 'taxa'])) {
             if (empty($this->request['proj_id'])) {
               // Should not have got this far - just in case.
               RestObjects::$apiResponse->fail('Bad request', 400, 'Missing proj_id parameter');
@@ -937,7 +965,7 @@ private function getRestPluginConfigs() {
    * @return string
    *   Name of the key in the current HTTP method's config to use.
    */
-  private function getPathConfigPatternMatch($methodConfig, $arguments) {
+  private function getPathConfigPatternMatch(array $methodConfig, array $arguments) {
     // Build an array to help locate the correct bit of configuration for
     // this path inside the resource config's method key.
     $searchArr = array_merge($arguments);
@@ -984,7 +1012,6 @@ private function getMethodName(array $arguments, $usingPath) {
     return $methodName;
   }
 
-
   /**
    * Check if resource allowed for project.
    *
@@ -1021,10 +1048,10 @@ private function projectsGetId($id) {
     if (!array_key_exists($id, $this->projects)) {
       RestObjects::$apiResponse->fail('No Content', 204);
     }
-    RestObjects::$apiResponse->succeed($this->projects[$id], array(
+    RestObjects::$apiResponse->succeed($this->projects[$id], [
       'columnsToUnset' => ['filter_id', 'website_id', 'sharing', 'resources'],
       'attachHref' => ['projects', 'id'],
-    ));
+    ]);
   }
 
   /**
@@ -1057,11 +1084,11 @@ private function projectsGet() {
   private function taxonObservationsGetId($id) {
     if (substr($id, 0, strlen(kohana::config('rest.user_id'))) === kohana::config('rest.user_id')) {
       $occurrence_id = substr($id, strlen(kohana::config('rest.user_id')));
-      $params = array('occurrence_id' => $occurrence_id);
+      $params = ['occurrence_id' => $occurrence_id];
     }
     else {
       // @todo What happens if system not recognised?
-      $params = array('external_key' => $id);
+      $params = ['external_key' => $id];
     }
     $params['dataset_name_attr_id'] = kohana::config('rest.dataset_name_attr_id');
 
@@ -1076,10 +1103,10 @@ private function taxonObservationsGetId($id) {
     else {
       RestObjects::$apiResponse->succeed(
         $report['content']['records'][0],
-        array(
-          'attachHref' => array('taxon-observations', 'id'),
+        [
+          'attachHref' => ['taxon-observations', 'id'],
           'columns' => $report['content']['columns'],
-        )
+        ]
       );
     }
   }
@@ -1110,7 +1137,7 @@ private function taxonObservationsGet() {
     RestObjects::$apiResponse->succeed(
       $this->listResponseStructure($report['content']['records']),
       [
-        'attachHref' => array('taxon-observations', 'id'),
+        'attachHref' => ['taxon-observations', 'id'],
         'columns' => $report['content']['columns'],
       ]
     );
@@ -1125,7 +1152,7 @@ private function taxonObservationsGet() {
    *   Unique ID for the annotations to output.
    */
   private function annotationsGetId($id) {
-    $params = array('id' => $id);
+    $params = ['id' => $id];
     $report = $this->loadReport('rest_api/filterable_annotations', $params);
     if (empty($report['content']['records'])) {
       RestObjects::$apiResponse->fail('No Content', 204);
@@ -1136,11 +1163,11 @@ private function annotationsGetId($id) {
     }
     else {
       $record = $report['content']['records'][0];
-      $record['taxonObservation'] = array(
+      $record['taxonObservation'] = [
         'id' => $record['taxon_observation_id'],
         // @todo href
-      );
-      RestObjects::$apiResponse->succeed($record, array(
+      ];
+      RestObjects::$apiResponse->succeed($record, [
         'attachHref' => [
           'annotations',
           'id',
@@ -1151,7 +1178,7 @@ private function annotationsGetId($id) {
           'taxon-observations',
         ],
         'columns' => $report['content']['columns'],
-      ));
+      ]);
     }
   }
 
@@ -1208,10 +1235,10 @@ private function annotationsGet() {
    * @todo caching option
    */
   private function taxaGetSearch() {
-    $params = array_merge(array(
+    $params = array_merge([
       'limit' => REST_API_DEFAULT_PAGE_SIZE,
       'include' => ['data', 'count', 'paging', 'columns'],
-    ), $this->request);
+    ], $this->request);
     try {
       $params['count'] = FALSE;
       $query = postgreSQL::taxonSearchQuery($params);
@@ -1242,30 +1269,30 @@ private function taxaGetSearch() {
         $result['paging'] = $this->getPagination($count);
       }
     }
-    $columns = array(
-      'taxa_taxon_list_id' => array('caption' => 'Taxa taxon list ID'),
-      'searchterm' => array('caption' => 'Search term'),
-      'highlighted' => array('caption' => 'Highlighted'),
-      'taxon' => array('caption' => 'Taxon'),
-      'authority' => array('caption' => 'Authority'),
-      'language_iso' => array('caption' => 'Language'),
-      'preferred_taxon' => array('caption' => 'Preferred name'),
-      'preferred_authority' => array('caption' => 'Preferred name authority'),
-      'default_common_name' => array('caption' => 'Common name'),
-      'taxon_group' => array('caption' => 'Taxon group'),
-      'preferred' => array('caption' => 'Preferred'),
-      'preferred_taxa_taxon_list_id' => array('caption' => 'Preferred taxa taxon list ID'),
-      'taxon_meaning_id' => array('caption' => 'Taxon meaning ID'),
-      'external_key' => array('caption' => 'External Key'),
-      'taxon_group_id' => array('caption' => 'Taxon group ID'),
-      'parent_id' => array('caption' => 'Parent taxa taxon list ID'),
-      'identification_difficulty' => array('caption' => 'Ident. difficulty'),
-      'id_diff_verification_rule_id' => array('caption' => 'Ident. difficulty verification rule ID'),
-    );
+    $columns = [
+      'taxa_taxon_list_id' => ['caption' => 'Taxa taxon list ID'],
+      'searchterm' => ['caption' => 'Search term'],
+      'highlighted' => ['caption' => 'Highlighted'],
+      'taxon' => ['caption' => 'Taxon'],
+      'authority' => ['caption' => 'Authority'],
+      'language_iso' => ['caption' => 'Language'],
+      'preferred_taxon' => ['caption' => 'Preferred name'],
+      'preferred_authority' => ['caption' => 'Preferred name authority'],
+      'default_common_name' => ['caption' => 'Common name'],
+      'taxon_group' => ['caption' => 'Taxon group'],
+      'preferred' => ['caption' => 'Preferred'],
+      'preferred_taxa_taxon_list_id' => ['caption' => 'Preferred taxa taxon list ID'],
+      'taxon_meaning_id' => ['caption' => 'Taxon meaning ID'],
+      'external_key' => ['caption' => 'External Key'],
+      'taxon_group_id' => ['caption' => 'Taxon group ID'],
+      'parent_id' => ['caption' => 'Parent taxa taxon list ID'],
+      'identification_difficulty' => ['caption' => 'Ident. difficulty'],
+      'id_diff_verification_rule_id' => ['caption' => 'Ident. difficulty verification rule ID'],
+    ];
     if (in_array('columns', $params['include'])) {
       $result['columns'] = $columns;
     }
-    $resultOptions = array('columns' => $columns);
+    $resultOptions = ['columns' => $columns];
     RestObjects::$apiResponse->succeed(
       $result,
       $resultOptions
@@ -1329,15 +1356,24 @@ private function reportsGet($featured = FALSE) {
     }
   }
 
+  /**
+   * Handler for GET reports/featured.
+   *
+   * Returns a list of the reports that have the attribute featured="true".
+   */
   private function reportsGetFeatured() {
     $this->reportsGet(TRUE);
   }
 
+  /**
+   * Handler for GET reports/path.
+   *
+   * Returns a list of the reports found within a folder path.
+   */
   private function reportsGetPath() {
     $this->reportsGet();
   }
 
-
   /**
    * Converts the segments in the URL to a full report path.
    *
@@ -1374,12 +1410,12 @@ private function getPagination($count) {
       parse_str($parts[1], $params);
     }
     else {
-      $params = array();
+      $params = [];
     }
     $params['known_count'] = $count;
-    $pagination = array(
+    $pagination = [
       'self' => "$urlPrefix$url?" . http_build_query($params),
-    );
+    ];
     $limit = empty($params['limit']) ? REST_API_DEFAULT_PAGE_SIZE : $params['limit'];
     $offset = empty($params['offset']) ? 0 : $params['offset'];
     if ($offset > 0) {
@@ -1460,7 +1496,7 @@ private function getReportOutput(array $segments) {
       }
       else {
         kohana::log('error', 'Rest API getReportOutput method retrieved invalid report response: ' .
-          var_export($report, true));
+          var_export($report, TRUE));
       }
     }
     finally {
@@ -1508,7 +1544,7 @@ private function getReportMetadataItem(array $segments, $item, $description) {
     }
     RestObjects::$apiResponse->responseTitle = ucfirst("$item for $reportFile");
     RestObjects::$apiResponse->wantIndex = TRUE;
-    RestObjects::$apiResponse->succeed(array('data' => $list), array('metadata' => array('description' => $description)));
+    RestObjects::$apiResponse->succeed(['data' => $list], ['metadata' => ['description' => $description]]);
   }
 
   /**
@@ -1547,12 +1583,14 @@ private function getReportColumns(array $segments) {
    *
    * @param array $segments
    *   URL segments.
+   * @param bool $featured
+   *   True if returning a list of the featured reports.
    */
   private function getReportHierarchy(array $segments, $featured) {
     $this->loadReportEngine();
     // @todo Cache this
     $reportHierarchy = $this->reportEngine->reportList();
-    $response = array();
+    $response = [];
     $folderReadme = '';
     if ($featured) {
       $folderReadme = kohana::lang("rest_api.reports.featured_folder_description");
@@ -1575,15 +1613,15 @@ private function getReportHierarchy(array $segments, $featured) {
     // If at the top level of the hierarchy, add a virtual featured folder
     // unless we are only showing featured reports.
     if (empty($segments) && empty($this->resourceOptions['featured'])) {
-      $reportHierarchy = array(
-        'featured' => array(
+      $reportHierarchy = [
+        'featured' => [
           'type' => 'folder',
           'description' => kohana::lang("rest_api.reports.featured_folder_description"),
-        ),
-      ) + $reportHierarchy;
+        ],
+      ] + $reportHierarchy;
     }
     if ($featured) {
-      $response = array();
+      $response = [];
       $this->getFeaturedReports($reportHierarchy, $response);
     }
     else {
@@ -1604,7 +1642,7 @@ private function getReportHierarchy(array $segments, $featured) {
     $relativePath = '/reports/' . ($relativePath ? "$relativePath/" : '');
     $description = 'A list of reports and report folders stored on the warehouse under ' .
       "the folder <em>$relativePath</em>. $folderReadme";
-    RestObjects::$apiResponse->succeed($response, array('metadata' => array('description' => $description)));
+    RestObjects::$apiResponse->succeed($response, ['metadata' => ['description' => $description]]);
   }
 
   /**
@@ -1664,9 +1702,9 @@ private function addReportLinks(array &$metadata) {
         'https://indicia-docs.readthedocs.io/en/latest/developing/reporting/standard-parameters.html';
       unset($metadata['standard_params']);
     }
-    $metadata['columns'] = array(
+    $metadata['columns'] = [
       'href' => RestObjects::$apiResponse->getUrlWithCurrentParams("reports/$metadata[path].xml/columns"),
-    );
+    ];
   }
 
   /**
@@ -1741,12 +1779,8 @@ private function checkParamDatatype($paramName, &$value, array $paramDef) {
    * Validates that the request parameters provided fullful the requirements of
    * the method being called.
    *
-   * @param string $resourceName
-   *   Name of the resource.
-   * @param string $method
-   *   Method name, e.g. GET or POST.
-   * @param bool $requestForId
-   *   ID of resource being requested if any.
+   * @param array $methodConfig
+   *  Configuration for the request.
    */
   private function validateParameters($methodConfig) {
     // Check through the known list of parameters to ensure data formats are
@@ -1812,7 +1846,7 @@ private function loadReportEngine() {
     // Should also return an object to iterate rather than loading the full
     // array.
     if (!isset($this->reportEngine)) {
-      $this->reportEngine = new ReportEngine(array(RestObjects::$clientWebsiteId));
+      $this->reportEngine = new ReportEngine([RestObjects::$clientWebsiteId]);
       // Resource configuration can provide a list of restricted reports that
       // are allowed for this client.
       if (isset($this->resourceOptions['authorise'])) {
@@ -2022,7 +2056,9 @@ private function loadFilterForProject($id) {
     }
     if (isset($this->projects[$id]['filter_id'])) {
       $filterId = $this->projects[$id]['filter_id'];
-      $filters = RestObjects::$db->select('definition')->from('filters')->where(array('id' => $filterId, 'deleted' => 'f'))
+      $filters = RestObjects::$db->select('definition')
+        ->from('filters')
+        ->where(['id' => $filterId, 'deleted' => 'f'])
         ->get()->result_array();
       if (count($filters) !== 1) {
         RestObjects::$apiResponse->fail('Internal Server Error', 500, 'Failed to find unique project filter record');
@@ -2030,7 +2066,7 @@ private function loadFilterForProject($id) {
       return json_decode($filters[0]->definition, TRUE);
     }
     else {
-      return array();
+      return [];
     }
   }
 
@@ -2049,16 +2085,16 @@ private function loadFilterForProject($id) {
   private function getPermissionsFilterDefinition() {
     $filters = RestObjects::$db->select('definition')
       ->from('filters')
-      ->join('filters_users', array(
+      ->join('filters_users', [
         'filters_users.filter_id' => 'filters.id',
-      ))
-      ->where(array(
+      ])
+      ->where([
         'filters.id' => $_GET['filter_id'],
         'filters.deleted' => 'f',
         'filters.defines_permissions' => 't',
         'filters_users.user_id' => RestObjects::$clientUserId,
         'filters_users.deleted' => 'f',
-      ))
+      ])
       ->get()->result_array();
     if (count($filters) !== 1) {
       RestObjects::$apiResponse->fail('Bad request', 400, 'Filter ID missing or not a permissions filter for the user');
@@ -2174,7 +2210,7 @@ private function authenticate() {
         $method = ucfirst($method);
         // Try this authentication method.
         if (method_exists($this, "authenticateUsing$method")) {
-          call_user_func(array($this, "authenticateUsing$method"));
+          call_user_func([$this, "authenticateUsing$method"]);
         }
         if ($this->authenticated) {
           RestObjects::$authMethod = $method;
@@ -2274,6 +2310,16 @@ private function getWebsiteByUrl($url) {
     return $website;
   }
 
+  /**
+   * Checks that the current user has website permissions.
+   *
+   * Fails if unauthorized.
+   *
+   * @param int $websiteId
+   *   Website ID to test against.
+   * @param int $userId
+   *   Warehouse user ID to test.   *
+   */
   private function checkWebsiteUser($websiteId, $userId) {
     $cache = Cache::instance();
     $cacheKey = "website-user-$websiteId-$userId";
@@ -2687,6 +2733,12 @@ private function getFiles() {
     return $files;
   }
 
+  /**
+   * Request handler for POST /rest/media-queue.
+   *
+   * Allows media to be cached on the server prior to submitting the data the
+   * media should be attached to.
+   */
   public function mediaQueuePost() {
     // Upload size.
     $ups = Kohana::config('indicia.maxUploadSize');
@@ -2719,11 +2771,13 @@ public function mediaQueuePost() {
       RestObjects::$apiResponse->fail('Bad Request', 400, json_encode($errors));
     }
     foreach ($files as $key => $file) {
-      kohana::log('debug', var_export($file, TRUE));
       $typeParts = explode('/', $file['type']);
       $fileName = uniqid('', TRUE) . '.' . $typeParts[1];
       upload::save($file, $fileName, 'upload-queue');
-      $response[$key] = ['name' => $fileName, 'tempPath' => url::base() . "upload-queue/$fileName"];
+      $response[$key] = [
+        'name' => $fileName,
+        'tempPath' => url::base() . "upload-queue/$fileName",
+      ];
     }
     RestObjects::$apiResponse->succeed($response);
   }
@@ -2885,7 +2939,7 @@ public function occurrencesDeleteId($id) {
   /**
    * API end-point to retrieve a location by ID.
    *
-   * @param integer $id
+   * @param int $id
    *   ID of the location.
    */
   public function locationsGetId($id) {
@@ -3313,12 +3367,12 @@ public function sampleAttributesPutId($id) {
   private function createAttributeTermlist(array &$item) {
     // Create a new termlist.
     $termlist = ORM::factory('termlist');
-    $termlist->set_submission_data(array(
+    $termlist->set_submission_data([
       'title' => 'Termlist for ' . $item['values']['caption'],
       'description' => 'Termlist created by the REST API for attribute ' . $item['values']['caption'],
       'website_id' => RestObjects::$clientWebsiteId,
       'deleted' => 'f',
-    ));
+    ]);
     if (!$termlist->submit()) {
       RestObjects::$apiResponse->fail('Internal Server Error', 500,
             'Error occurred creating new termlist: ' . implode("\n", $termlist->getAllErrors()));
@@ -3345,7 +3399,11 @@ private function updateAttributeTermlist(array $item) {
       ->select('tlt.id, t.term, tlt.sort_order')
       ->from('termlists_terms AS tlt')
       ->join('terms AS t', 't.id', 'tlt.term_id')
-      ->where(['tlt.deleted' => 'f', 't.deleted' => 'f', 'tlt.termlist_id' => $item['values']['termlist_id']])
+      ->where([
+        'tlt.deleted' => 'f',
+        't.deleted' => 'f',
+        'tlt.termlist_id' => $item['values']['termlist_id'],
+      ])
       ->orderby(['tlt.sort_order' => 'ASC', 't.term' => 'ASC'])
       ->get()->result();
     foreach ($existingRows as $row) {
@@ -3371,13 +3429,13 @@ private function updateAttributeTermlist(array $item) {
       else {
         // Create new term.
         $termlists_term = ORM::factory('termlists_term');
-        $termlists_term->set_submission_data(array(
+        $termlists_term->set_submission_data([
           'term:term' => $term,
           'term:fk_language:iso' => kohana::config('indicia.default_lang'),
           'sort_order' => $idx + 1,
           'termlist_id' => $item['values']['termlist_id'],
           'preferred' => 't',
-        ));
+        ]);
         if (!$termlists_term->submit()) {
           RestObjects::$apiResponse->fail('Internal Server Error', 500,
             'Error occurred creating new term: ' . implode("\n", $termlists_term->getAllErrors()));
@@ -3447,14 +3505,14 @@ public function sampleAttributesWebsitesPost() {
     }
     // Duplicate check.
     $existing = RestObjects::$db->select('aw.id')
-        ->from('sample_attributes_websites aw')
-        ->where([
-          'website_id' => $postArray['values']['website_id'],
-          'sample_attribute_id' => $postArray['values']['sample_attribute_id'],
-          'restrict_to_survey_id' => $postArray['values']['restrict_to_survey_id'],
-          'deleted' => 'f',
-        ])
-        ->get()->current();
+      ->from('sample_attributes_websites aw')
+      ->where([
+        'website_id' => $postArray['values']['website_id'],
+        'sample_attribute_id' => $postArray['values']['sample_attribute_id'],
+        'restrict_to_survey_id' => $postArray['values']['restrict_to_survey_id'],
+        'deleted' => 'f',
+      ])
+      ->get()->current();
     if ($existing) {
       $r = rest_crud::update('sample_attributes_website', $existing->id, $postArray);
       echo json_encode($r);
@@ -3597,14 +3655,14 @@ public function occurrenceAttributesWebsitesPost() {
     }
     // Duplicate check.
     $existing = RestObjects::$db->select('aw.id')
-        ->from('occurrence_attributes_websites aw')
-        ->where([
-          'website_id' => $postArray['values']['website_id'],
-          'occurrence_attribute_id' => $postArray['values']['occurrence_attribute_id'],
-          'restrict_to_survey_id' => $postArray['values']['restrict_to_survey_id'],
-          'deleted' => 'f',
-        ])
-        ->get()->current();
+      ->from('occurrence_attributes_websites aw')
+      ->where([
+        'website_id' => $postArray['values']['website_id'],
+        'occurrence_attribute_id' => $postArray['values']['occurrence_attribute_id'],
+        'restrict_to_survey_id' => $postArray['values']['restrict_to_survey_id'],
+        'deleted' => 'f',
+      ])
+      ->get()->current();
     if ($existing) {
       $r = rest_crud::update('occurrence_attributes_website', $existing->id, $postArray);
       echo json_encode($r);
@@ -3618,7 +3676,7 @@ public function occurrenceAttributesWebsitesPost() {
   }
 
   /**
-   * API end-point to PUT to an existing occurrence attributes website to update.
+   * End-point to PUT to an existing occurrence attributes website to update.
    */
   public function occurrenceAttributesWebsitesPutId($id) {
     $this->assertUserHasWebsiteAccess();

From f8f42027b36e1bd3d6b29357199211e008e9d4f4 Mon Sep 17 00:00:00 2001
From: Jim Bacon <jame2@ceh.ac.uk>
Date: Tue, 20 Jul 2021 16:42:56 +0100
Subject: [PATCH 10/93] Linting

---
 .../libraries/Indicia_ArrayDataSet.php        |  73 ++-
 .../libraries/Indicia_DatabaseTestCase.php    |  72 ++-
 modules/phpUnit/libraries/MY_Session.php      | 237 +++++----
 .../rest_api/tests/Rest_ControllerTest.php    | 490 ++++++++++++------
 .../tests/TaxonAssocations_ServicesTest.php   |  78 +--
 5 files changed, 564 insertions(+), 386 deletions(-)

diff --git a/modules/phpUnit/libraries/Indicia_ArrayDataSet.php b/modules/phpUnit/libraries/Indicia_ArrayDataSet.php
index d3f26b8406..322bb0a25a 100644
--- a/modules/phpUnit/libraries/Indicia_ArrayDataSet.php
+++ b/modules/phpUnit/libraries/Indicia_ArrayDataSet.php
@@ -9,47 +9,44 @@
 
 /**
  * Implements a dataset created from a PHP array.
- * https://phpunit.de/manual/current/en/database.html#database.available-implementations
+ * https://phpunit.de/manual/6.5/en/database.html#database.available-implementations
  */
-class Indicia_ArrayDataSet extends DbUDataSetAbstractDataSet
-{
-    /**
-     * @var array
-     */
-    protected $tables = [];
-
-    /**
-     * @param array $data
-     */
-    public function __construct(array $data)
-    {
-        foreach ($data AS $tableName => $rows) {
-            $columns = [];
-            if (isset($rows[0])) {
-                $columns = array_keys($rows[0]);
-            }
-
-            $metaData = new DbUDataSetDefaultTableMetadata($tableName, $columns);
-            $table = new DbUDataSetDefaultTable($metaData);
-
-            foreach ($rows AS $row) {
-                $table->addRow($row);
-            }
-            $this->tables[$tableName] = $table;
-        }
+class Indicia_ArrayDataSet extends DbUDataSetAbstractDataSet {
+  /**
+   * @var array
+   */
+  protected $tables = [];
+
+  /**
+   * @param array $data
+   */
+  public function __construct(array $data) {
+    foreach ($data as $tableName => $rows) {
+      $columns = [];
+      if (isset($rows[0])) {
+        $columns = array_keys($rows[0]);
+      }
+
+      $metaData = new DbUDataSetDefaultTableMetadata($tableName, $columns);
+      $table = new DbUDataSetDefaultTable($metaData);
+
+      foreach ($rows as $row) {
+        $table->addRow($row);
+      }
+      $this->tables[$tableName] = $table;
     }
+  }
 
-    protected function createIterator(bool $reverse = false): DbUDataSetITableIterator
-    {
-        return new DbUDataSetDefaultTableIterator($this->tables, $reverse);
+  protected function createIterator(bool $reverse = false): DbUDataSetITableIterator {
+    return new DbUDataSetDefaultTableIterator($this->tables, $reverse);
+  }
+
+  public function getTable(string $tableName): DbUDataSetITable {
+    if (!isset($this->tables[$tableName])) {
+      throw new InvalidArgumentException("$tableName is not a table in the current database.");
     }
 
-    public function getTable(string $tableName): DbUDataSetITable
-    {
-        if (!isset($this->tables[$tableName])) {
-            throw new InvalidArgumentException("$tableName is not a table in the current database.");
-        }
+    return $this->tables[$tableName];
+  }
 
-        return $this->tables[$tableName];
-    }
-}
\ No newline at end of file
+}
diff --git a/modules/phpUnit/libraries/Indicia_DatabaseTestCase.php b/modules/phpUnit/libraries/Indicia_DatabaseTestCase.php
index 0a2f191c71..963d4d4706 100644
--- a/modules/phpUnit/libraries/Indicia_DatabaseTestCase.php
+++ b/modules/phpUnit/libraries/Indicia_DatabaseTestCase.php
@@ -13,47 +13,58 @@
 
 /**
  * An abstract test case to efficiently make database connections.
- * https://phpunit.de/manual/current/en/database.html#database.tip-use-your-own-abstract-database-testcase
+ *
+ * https://phpunit.de/manual/6.5/en/database.html#database.tip-use-your-own-abstract-database-testcase
  */
 abstract class Indicia_DatabaseTestCase extends DbUTestCase {
   // Only instantiate pdo once for test clean-up/fixture load.
   static private $pdo = NULL;
 
-  // only instantiate PHPUnit_Extensions_Database_DB_IDatabaseConnection once per test
-  private $conn = null;
+  // Only instantiate PHPUnit_Extensions_Database_DB_IDatabaseConnection once
+  // per test.
+  private $conn = NULL;
 
   final public function getConnection() {
-    if ($this->conn === null) {
-      if (self::$pdo == null) {
+    if ($this->conn === NULL) {
+      if (self::$pdo == NULL) {
         $dsn = 'pgsql:host=127.0.0.1;dbname=indicia';
         $user = 'indicia_user';
         $pass = 'indicia_user_pass';
         self::$pdo = new PDO($dsn, $user, $pass);
-        }
+      }
       $this->conn = $this->createDefaultDBConnection(self::$pdo, 'indicia');
     }
 
     return $this->conn;
   }
-  
-  // Default implementation which does nothing
+
+  /**
+   * Default implementation which does nothing.
+   */
   public function getDataSet() {
-    return new Indicia_ArrayDataSet(array());
+    return new Indicia_ArrayDataSet([]);
   }
 
-  // Override the operation used to set up the database.
-  // The default is CLEAN_INSERT($cascadeTruncates = FALSE)
-  // We require truncates to be cascaded to prevent foreign key
-  // violations and sequences to be restarted.
+  /**
+   * Override the operation used to set up the database.
+   *
+   * The default is CLEAN_INSERT($cascadeTruncates = FALSE)
+   * We require truncates to be cascaded to prevent foreign key
+   * violations and sequences to be restarted.
+   */
   protected function getSetUpOperation() {
-     return My_Operation_Factory::RESTART_INSERT(true);
+    return My_Operation_Factory::RESTART_INSERT(TRUE);
   }
-  
-  // Override the function to create the database connection so that is uses 
-  // My_DB_DefaultDatabaseConnection.
+
+  /**
+   * Override the function to create the database connection.
+   *
+   * Use My_DB_DefaultDatabaseConnection.
+   */
   protected function createDefaultDBConnection(PDO $connection, $schema = ''): dbUDatabaseDefualtConnection {
     return new My_DB_DefaultDatabaseConnection($connection, $schema);
-  }  
+  }
+
 }
 
 /**
@@ -61,6 +72,7 @@ protected function createDefaultDBConnection(PDO $connection, $schema = ''): dbU
  * a database specific command to restart sequences.
  */
 class My_DB_MetaData_PgSQL extends DbUDatabaseMetadataPgSQL {
+
   public function getRestartCommand($table) {
     // Assumes sequence naming convention has been followed.
     $seq = $table . '_id_seq';
@@ -75,14 +87,16 @@ public function getRestartCommand($table) {
 }
 
 /**
- * Extends PHPUnit_Extensions_Database_DB_MetaData in order to replace the 
+ * Extends PHPUnit_Extensions_Database_DB_MetaData in order to replace the
  * default postgres driver with mine.
  */
 abstract class My_DB_MetaData extends DbUDatabaseMetadataAbstractMetadata {
+
   public static function createMetaData(PDO $pdo, $schema = '') {
     self::$metaDataClassMap['pgsql'] = 'My_DB_MetaData_PgSQL';
     return parent::createMetaData($pdo, $schema);
-  }    
+  }
+
 }
 
 /**
@@ -91,6 +105,7 @@ public static function createMetaData(PDO $pdo, $schema = '') {
  * b. give access to the command that will restart sequences.
  */
 class My_DB_DefaultDatabaseConnection extends dbUDatabaseDefualtConnection {
+
   public function __construct(PDO $connection, $schema = '') {
     $this->connection = $connection;
     $this->metaData   = My_DB_MetaData::createMetaData($connection, $schema);
@@ -100,12 +115,14 @@ public function __construct(PDO $connection, $schema = '') {
   public function getRestartCommand($table) {
     return $this->getMetaData()->getRestartCommand($table);
   }
+
 }
 
 /**
  * New class to add a Restart operation.
  */
-Class My_Operation_Restart implements DbUOperationOperation {
+class My_Operation_Restart implements DbUOperationOperation {
+
   public function execute(
     DbUDatabaseConnection $connection,
     DbUDatasetIDataset $dataSet
@@ -114,8 +131,9 @@ public function execute(
       $tableName = $table->getTableMetaData()->getTableName();
       $query = $connection->getRestartCommand($tableName);
       try {
-          $connection->getConnection()->query($query);
-      } catch (\Exception $e) {
+        $connection->getConnection()->query($query);
+      }
+      catch (\Exception $e) {
         if ($e instanceof PDOException) {
           throw new DbUOperationException('RESTART', $query, [], $table, $e->getMessage());
         }
@@ -123,23 +141,25 @@ public function execute(
       }
     }
   }
+
 }
 
 /**
- * Extends PHPUnit_Extensions_Database_Operation_Factory in order to add 
+ * Extends PHPUnit_Extensions_Database_Operation_Factory in order to add
  * functions that call the Restart operation.
  */
-Class My_Operation_Factory extends DbUOperationFactory {
+class My_Operation_Factory extends DbUOperationFactory {
   public static function RESTART_INSERT($cascadeTruncates = FALSE) {
     return new DbUOperationComposite([
       self::TRUNCATE($cascadeTruncates),
       self::RESTART(),
-      self::INSERT()
+      self::INSERT(),
     ]);
   }
 
   public static function RESTART() {
     return new My_Operation_Restart();
   }
+
 }
 
diff --git a/modules/phpUnit/libraries/MY_Session.php b/modules/phpUnit/libraries/MY_Session.php
index 2766863265..69c9d685db 100644
--- a/modules/phpUnit/libraries/MY_Session.php
+++ b/modules/phpUnit/libraries/MY_Session.php
@@ -1,128 +1,127 @@
-<?php defined('SYSPATH') OR die('No direct access allowed.');
+<?php
+
+defined('SYSPATH') or die('No direct access allowed.');
 
 /**
  * An alternative to the core session handler for use in unit testing.
+ *
  * Initially created just to prevent session_start() being called.
- * 
- * Since there is no browser during unit testing there is no support for 
+ * Since there is no browser during unit testing there is no support for
  * cookies. This class could overcome that.
  */
 class Session extends Session_Core {
 
-	/**
-	 * Create a new session.
-	 *
-	 * @param   array  variables to set after creation
-	 * @return  void
-	 */
-	public function create($vars = NULL)
-	{
-		// Destroy any current sessions
-		$this->destroy();
-
-		if (Session::$config['driver'] !== 'native')
-		{
-			// Set driver name
-			$driver = 'Session_'.ucfirst(Session::$config['driver']).'_Driver';
-
-			// Load the driver
-			if ( ! Kohana::auto_load($driver))
-				throw new Kohana_Exception('core.driver_not_found', Session::$config['driver'], get_class($this));
-
-			// Initialize the driver
-			Session::$driver = new $driver();
-
-			// Validate the driver
-			if ( ! (Session::$driver instanceof Session_Driver))
-				throw new Kohana_Exception('core.driver_implements', Session::$config['driver'], get_class($this), 'Session_Driver');
-
-			// Register non-native driver as the session handler
-			session_set_save_handler
-			(
-				array(Session::$driver, 'open'),
-				array(Session::$driver, 'close'),
-				array(Session::$driver, 'read'),
-				array(Session::$driver, 'write'),
-				array(Session::$driver, 'destroy'),
-				array(Session::$driver, 'gc')
-			);
-		}
-
-		// Validate the session name
-		if ( ! preg_match('~^(?=.*[a-z])[a-z0-9_]++$~iD', Session::$config['name']))
-			throw new Kohana_Exception('session.invalid_session_name', Session::$config['name']);
-
-		// Name the session, this will also be the name of the cookie
-		session_name(Session::$config['name']);
-
-		// Set the session cookie parameters
-		session_set_cookie_params
-		(
-			Session::$config['expiration'],
-			Kohana::config('cookie.path'),
-			Kohana::config('cookie.domain'),
-			Kohana::config('cookie.secure'),
-			Kohana::config('cookie.httponly')
-		);
-
-		// DO NOT start the session! phpUnit has done so.
-		// session_start();
-
-		// Put session_id in the session variable
-		$_SESSION['session_id'] = session_id();
-
-		// Set defaults
-		if ( ! isset($_SESSION['_kf_flash_']))
-		{
-			$_SESSION['total_hits'] = 0;
-			$_SESSION['_kf_flash_'] = array();
-
-			$_SESSION['user_agent'] = Kohana::$user_agent;
-			$_SESSION['ip_address'] = $this->input->ip_address();
-		}
-
-		// Set up flash variables
-		Session::$flash =& $_SESSION['_kf_flash_'];
-
-		// Increase total hits
-		$_SESSION['total_hits'] += 1;
-
-		// Validate data only on hits after one
-		if ($_SESSION['total_hits'] > 1)
-		{
-			// Validate the session
-			foreach (Session::$config['validate'] as $valid)
-			{
-				switch ($valid)
-				{
-					// Check user agent for consistency
-					case 'user_agent':
-						if ($_SESSION[$valid] !== Kohana::$user_agent)
-							return $this->create();
-					break;
-
-					// Check ip address for consistency
-					case 'ip_address':
-						if ($_SESSION[$valid] !== $this->input->$valid())
-							return $this->create();
-					break;
-
-					// Check expiration time to prevent users from manually modifying it
-					case 'expiration':
-						if (time() - $_SESSION['last_activity'] > ini_get('session.gc_maxlifetime'))
-							return $this->create();
-					break;
-				}
-			}
-		}
-
-		// Expire flash keys
-		$this->expire_flash();
-
-		// Update last activity
-		$_SESSION['last_activity'] = time();
-
-		// Set the new data
-		Session::set($vars);
-	}
+  /**
+   * Create a new session.
+   *
+   * @param array  variables to set after creation
+   *
+   * @return  void
+   */
+  public function create($vars = NULL) {
+    // Destroy any current sessions.
+    $this->destroy();
+
+    if (Session::$config['driver'] !== 'native') {
+      // Set driver name.
+      $driver = 'Session_' . ucfirst(Session::$config['driver']) . '_Driver';
+
+      // Load the driver.
+      if (!Kohana::auto_load($driver)) {
+        throw new Kohana_Exception('core.driver_not_found', Session::$config['driver'], get_class($this));
+      }
+
+      // Initialize the driver.
+      Session::$driver = new $driver();
+
+      // Validate the driver.
+      if (!(Session::$driver instanceof Session_Driver)) {
+        throw new Kohana_Exception('core.driver_implements', Session::$config['driver'], get_class($this), 'Session_Driver');
+      }
+      // Register non-native driver as the session handler.
+      session_set_save_handler(
+        [Session::$driver, 'open'],
+        [Session::$driver, 'close'],
+        [Session::$driver, 'read'],
+        [Session::$driver, 'write'],
+        [Session::$driver, 'destroy'],
+        [Session::$driver, 'gc']
+      );
+    }
+
+    // Validate the session name.
+    if (!preg_match('~^(?=.*[a-z])[a-z0-9_]++$~iD', Session::$config['name'])) {
+      throw new Kohana_Exception('session.invalid_session_name', Session::$config['name']);
+    }
+    // Name the session, this will also be the name of the cookie.
+    session_name(Session::$config['name']);
+
+    // Set the session cookie parameters.
+    session_set_cookie_params(
+      Session::$config['expiration'],
+      Kohana::config('cookie.path'),
+      Kohana::config('cookie.domain'),
+      Kohana::config('cookie.secure'),
+      Kohana::config('cookie.httponly')
+    );
+
+    // DO NOT start the session! phpUnit has done so.
+    // session_start();
+
+    // Put session_id in the session variable.
+    $_SESSION['session_id'] = session_id();
+
+    // Set defaults.
+    if (!isset($_SESSION['_kf_flash_'])) {
+      $_SESSION['total_hits'] = 0;
+      $_SESSION['_kf_flash_'] = [];
+
+      $_SESSION['user_agent'] = Kohana::$user_agent;
+      $_SESSION['ip_address'] = $this->input->ip_address();
+    }
+
+    // Set up flash variables.
+    Session::$flash =& $_SESSION['_kf_flash_'];
+
+    // Increase total hits.
+    $_SESSION['total_hits'] += 1;
+
+    // Validate data only on hits after one.
+    if ($_SESSION['total_hits'] > 1) {
+      // Validate the session.
+      foreach (Session::$config['validate'] as $valid) {
+        switch ($valid) {
+          // Check user agent for consistency.
+          case 'user_agent':
+            if ($_SESSION[$valid] !== Kohana::$user_agent) {
+              return $this->create();
+            }
+            break;
+
+          // Check ip address for consistency.
+          case 'ip_address':
+            if ($_SESSION[$valid] !== $this->input->$valid()) {
+              return $this->create();
+            }
+            break;
+
+          // Check expiration time to prevent users from manually modifying it.
+          case 'expiration':
+            if (time() - $_SESSION['last_activity'] > ini_get('session.gc_maxlifetime')) {
+              return $this->create();
+            }
+            break;
+        }
+      }
+    }
+
+    // Expire flash keys.
+    $this->expire_flash();
+
+    // Update last activity.
+    $_SESSION['last_activity'] = time();
+
+    // Set the new data.
+    Session::set($vars);
+  }
 }
\ No newline at end of file
diff --git a/modules/rest_api/tests/Rest_ControllerTest.php b/modules/rest_api/tests/Rest_ControllerTest.php
index ed0fae2a64..8d3985ad74 100644
--- a/modules/rest_api/tests/Rest_ControllerTest.php
+++ b/modules/rest_api/tests/Rest_ControllerTest.php
@@ -90,9 +90,9 @@ public function getDataSet() {
      * Create an occurrence comment for annotation testing.
      */
     $ds2 = new Indicia_ArrayDataSet(
-      array(
-        'filters' => array(
-          array(
+      [
+        'filters' => [
+          [
             'title' => 'Test filter',
             'description' => 'Filter for unit testing',
             'definition' => '{"quality":"!R"}',
@@ -101,8 +101,8 @@ public function getDataSet() {
             'created_by_id' => 1,
             'updated_on' => '2016-07-22:16:00:00',
             'updated_by_id' => 1,
-          ),
-          array(
+          ],
+          [
             'title' => 'Test user permission filter',
             'description' => 'Filter for unit testing',
             'definition' => '{"quality":"!R","occurrence_id":2}',
@@ -111,27 +111,27 @@ public function getDataSet() {
             'created_by_id' => 1,
             'updated_on' => '2016-07-22:16:00:00',
             'updated_by_id' => 1,
-          ),
-        ),
-        'filters_users' => array(
-          array(
+          ],
+        ],
+        'filters_users' => [
+          [
             'filter_id' => 2,
             'user_id' => 1,
             'created_on' => '2016-07-22:16:00:00',
             'created_by_id' => 1
-          )
-        ),
-        'occurrence_comments' => array(
-          array(
+          ],
+        ],
+        'occurrence_comments' => [
+          [
             'comment' => 'Occurrence comment for unit testing',
             'created_on' => '2016-07-22:16:00:00',
             'created_by_id' => 1,
             'updated_on' => '2016-07-22:16:00:00',
             'updated_by_id' => 1,
             'occurrence_id' => 1,
-          ),
-        ),
-      )
+          ],
+        ],
+      ]
     );
 
     $compositeDs = new DbUDataSetCompositeDataSet();
@@ -143,19 +143,19 @@ public function getDataSet() {
     $db = new Database();
     $db->update(
       'users',
-      array('password' => '18d025c6c8809e34371e2ec7d84215bd3eb6031dcd804006f4'),
-      array('id' => 1)
+      ['password' => '18d025c6c8809e34371e2ec7d84215bd3eb6031dcd804006f4'],
+      ['id' => 1]
     );
 
     return $compositeDs;
   }
 
   public static function setUpBeforeClass(): void {
-    // grab the clients registered on this system
+    // Grab the clients registered on this system.
     $clientUserIds = array_keys(Kohana::config('rest.clients'));
     $clientConfigs = array_values(Kohana::config('rest.clients'));
 
-    // just test the first client
+    // Just test the first client.
     self::$clientUserId = $clientUserIds[0];
     self::$config = $clientConfigs[0];
   }
@@ -194,8 +194,8 @@ public function testJwt() {
     $db = new Database();
     $db->update(
       'websites',
-      array('public_key' => NULL),
-      array('id' => 1)
+      ['public_key' => NULL],
+      ['id' => 1]
     );
     $cache->delete($cacheKey);
     // Make an otherwise valid call - should be unauthorised.
@@ -206,8 +206,8 @@ public function testJwt() {
     $db = new Database();
     $db->update(
       'websites',
-      array('public_key' => 'INVALID!!!'),
-      array('id' => 1)
+      ['public_key' => 'INVALID!!!'],
+      ['id' => 1]
     );
     $cache->delete($cacheKey);
     // Make an otherwise valid call - should be unauthorised.
@@ -218,8 +218,8 @@ public function testJwt() {
     $db = new Database();
     $db->update(
       'websites',
-      array('public_key' => self::$publicKey),
-      array('id' => 1)
+      ['public_key' => self::$publicKey],
+      ['id' => 1]
     );
     $cache->delete($cacheKey);
     // Make a valid call - should be authorised.
@@ -255,17 +255,20 @@ public function testJwtHeaderCaseInsensitive() {
     $response = $this->callService(
       'samples',
       FALSE,
-      ['values' => [
-        'survey_id' => 1,
-        'entered_sref' => 'SU1234',
-        'entered_sref_system' => 'OSGB',
-        'date' => '01/08/2020',
-        'comment' => 'A sample to delete',
-      ]]
+      [
+        'values' => [
+          'survey_id' => 1,
+          'entered_sref' => 'SU1234',
+          'entered_sref_system' => 'OSGB',
+          'date' => '01/08/2020',
+          'comment' => 'A sample to delete',
+        ]
+      ]
     );
     $this->assertEquals(201, $response['httpCode']);
     $id = $response['response']['values']['id'];
-    // Now GET to check values stored OK using manually set auth header in lowercase.
+    // Now GET to check values stored OK using manually set auth header in
+    // lowercase.
     $this->authMethod = 'none';
     $storedObj = $this->callService("samples/$id", FALSE, NULL, ['authorization: bearer ' . self::$jwt]);
     $this->assertResponseOk($storedObj, "/samples/$id GET");
@@ -350,8 +353,14 @@ private function putTest($table, array $exampleData, array $updateData) {
     $storedObj = $this->callService("$table/$id");
     $expectedValues = array_merge($exampleData, $updateData);
     foreach ($expectedValues as $field => $value) {
-      $this->assertTrue(isset($storedObj['response']['values'][$field]), "Stored info in $table does not include value for $field");
-      $this->assertEquals($value, $storedObj['response']['values'][$field], "Stored info in $table does not match value for $field");
+      $this->assertTrue(
+        isset($storedObj['response']['values'][$field]),
+        "Stored info in $table does not include value for $field"
+      );
+      $this->assertEquals(
+        $value, $storedObj['response']['values'][$field],
+        "Stored info in $table does not match value for $field"
+      );
     }
   }
 
@@ -384,8 +393,14 @@ private function getTest($table, $exampleData) {
     $this->assertTrue(array_key_exists('ETag', $headers), "$table GET does not return ETag.");
     $this->assertEquals($insertedRecordETag, $headers['ETag'], 'GET returns ETag which does not match expected');
     foreach ($exampleData as $field => $value) {
-      $this->assertTrue(isset($storedObj['response']['values'][$field]), "Stored info in $table does not include value for $field");
-      $this->assertEquals($exampleData[$field], $storedObj['response']['values'][$field], "Stored info in $table does not match value for $field");
+      $this->assertTrue(
+        isset($storedObj['response']['values'][$field]),
+        "Stored info in $table does not include value for $field"
+      );
+      $this->assertEquals(
+        $exampleData[$field], $storedObj['response']['values'][$field],
+        "Stored info in $table does not match value for $field"
+      );
     }
   }
 
@@ -459,7 +474,10 @@ private function getListTest($table, $exampleData) {
         break;
       }
     }
-    $this->assertFalse($found, 'POSTed survey found in filtered retrieved list using GET which it should be excluded from.');
+    $this->assertFalse(
+      $found,
+      'POSTed survey found in filtered retrieved list using GET which it should be excluded from.'
+    );
   }
 
   /**
@@ -599,13 +617,15 @@ public function testJwtSamplePostUserAuth() {
     $response = $this->callService(
       'samples',
       FALSE,
-      ['values' => [
-        'survey_id' => 1,
-        'entered_sref' => 'SU1234',
-        'entered_sref_system' => 'OSGB',
-        'date' => '01/08/2020',
-        'comment' => 'A sample comment test',
-      ]]
+      [
+        'values' => [
+          'survey_id' => 1,
+          'entered_sref' => 'SU1234',
+          'entered_sref_system' => 'OSGB',
+          'date' => '01/08/2020',
+          'comment' => 'A sample comment test',
+        ],
+      ]
     );
     $this->assertEquals(401, $response['httpCode']);
     // Grant website access.
@@ -615,20 +635,25 @@ public function testJwtSamplePostUserAuth() {
     $response = $this->callService(
       'samples',
       FALSE,
-      ['values' => [
-        'survey_id' => 1,
-        'entered_sref' => 'SU1234',
-        'entered_sref_system' => 'OSGB',
-        'date' => '01/08/2020',
-        'comment' => 'A sample comment test',
-      ]]
+      [
+        'values' => [
+          'survey_id' => 1,
+          'entered_sref' => 'SU1234',
+          'entered_sref_system' => 'OSGB',
+          'date' => '01/08/2020',
+          'comment' => 'A sample comment test',
+        ],
+      ]
     );
     $this->assertEquals(201, $response['httpCode']);
     $id = $response['response']['values']['id'];
     // Check created_by_id.
     $response = $this->callService("samples/$id");
     $this->assertEquals(200, $response['httpCode']);
-    $this->assertEquals($userId, $response['response']['values']['created_by_id'], 'Created_by_id not set correctly for sample');
+    $this->assertEquals(
+      $userId, $response['response']['values']['created_by_id'],
+      'Created_by_id not set correctly for sample'
+    );
     // Re-authenticate as user 1.
     self::$jwt = $this->getJwt(self::$privateKey, 'http://www.indicia.org.uk', 1, time() + 120);
     // They shouldn't have access.
@@ -653,9 +678,7 @@ public function testJwtSamplePostMoreTests() {
     $response = $this->callService(
       'samples',
       FALSE,
-      [
-        'values' => $data
-      ]
+      ['values' => $data]
     );
     $this->assertEquals(201, $response['httpCode']);
     $headers = $this->parseHeaders($response['headers']);
@@ -697,12 +720,10 @@ public function testJwtSamplePostMoreTests() {
     $response = $this->callService(
       'samples',
       FALSE,
-      [
-        'values' => $data
-      ]
+      ['values' => $data]
     );
     $this->assertEquals(400, $response['httpCode']);
-    // GET the posted data;
+    // GET the posted data.
     $response = $this->callService("samples/$id");
     $this->assertResponseOk($response, "/samples GET");
     $this->assertTrue(array_key_exists('comment', $response['response']['values']),
@@ -719,7 +740,7 @@ public function testJwtSamplePostMoreTests() {
       'GET samples response does not contain processed lat output.');
       $this->assertTrue(array_key_exists('lon', $response['response']['values']),
       'GET samples response does not contain processed lon output.');
-    // PUT a bad update with ID mismatch
+    // PUT a bad update with ID mismatch.
     $data = [
       'id' => $id + 1,
       'entered_sref' => 'SU121341',
@@ -843,7 +864,10 @@ public function testJwtSamplePostList() {
       FALSE,
       $data
     );
-    $this->assertEquals(400, $response['httpCode'], 'POSTing a list to normal endpoint should fail');
+    $this->assertEquals(
+      400, $response['httpCode'],
+      'POSTing a list to normal endpoint should fail'
+    );
     $response = $this->callService(
       'samples/list',
       FALSE,
@@ -851,11 +875,17 @@ public function testJwtSamplePostList() {
     );
     $this->assertEquals(201, $response['httpCode']);
     foreach ($response['response'] as $idx => $item) {
-      $this->assertTrue(is_numeric($idx), 'Response from list post should be a simple list array');
+      $this->assertTrue(
+        is_numeric($idx),
+        'Response from list post should be a simple list array'
+      );
       $id = $item['values']['id'];
       $occCount = $db->query("select count(*) from occurrences where sample_id=$id")
         ->current()->count;
-      $this->assertEquals(1, $occCount, 'No occurrence created when submitted with a sample in a list.');
+      $this->assertEquals(
+        1, $occCount,
+        'No occurrence created when submitted with a sample in a list.'
+      );
     }
   }
 
@@ -883,7 +913,10 @@ public function testJwtSamplePostExtKey() {
       FALSE,
       ['values' => $data]
     );
-    $this->assertEquals(409, $response['httpCode'], 'Duplicate external key did not return 409 Conflict response.');
+    $this->assertEquals(
+      409, $response['httpCode'],
+      'Duplicate external key did not return 409 Conflict response.'
+    );
     $this->assertArrayHasKey('duplicate_of', $response['response']);
     $this->assertArrayHasKey('id', $response['response']['duplicate_of']);
     $this->assertArrayHasKey('href', $response['response']['duplicate_of']);
@@ -895,7 +928,10 @@ public function testJwtSamplePostExtKey() {
       FALSE,
       ['values' => $data]
     );
-    $this->assertEquals(201, $response['httpCode'], 'Duplicate external key in different survey not accepted.');
+    $this->assertEquals(
+      201, $response['httpCode'],
+      'Duplicate external key in different survey not accepted.'
+    );
     // PUT with same external key should be OK.
     $response = $this->callService(
       "samples/$id",
@@ -1069,9 +1105,18 @@ public function testJwtSamplePostWithMedia() {
     $id = $response['response']['values']['id'];
     $smpMediaCount = $db->query("select count(*) from sample_media where sample_id=$id and path='$uploadedFileName'")
       ->current()->count;
-    $this->assertEquals(1, $smpMediaCount, 'No media created when submitted with a sample.');
-    $this->assertFileExists(DOCROOT . 'upload/' . $uploadedFileName, 'Uploaded media file does not exist in destination');
-    $this->assertFileExists(DOCROOT . 'upload/thumb-' . $uploadedFileName, 'Uploaded media thumbnail does not exist in destination');
+    $this->assertEquals(
+      1, $smpMediaCount, 
+      'No media created when submitted with a sample.'
+    );
+    $this->assertFileExists(
+      DOCROOT . 'upload/' . $uploadedFileName,
+      'Uploaded media file does not exist in destination'
+    );
+    $this->assertFileExists(
+      DOCROOT . 'upload/thumb-' . $uploadedFileName,
+      'Uploaded media thumbnail does not exist in destination'
+    );
     // Post a sample which refers to an incorrect file.
     $data = [
       'values' => [
@@ -1119,7 +1164,7 @@ public function testJwtSampleOccurrenceMediaPost() {
           $file,
           'image/jpg',
           basename($file)
-        )
+        ),
       ],
       [], NULL, TRUE
     );
@@ -1156,10 +1201,16 @@ public function testJwtSampleOccurrenceMediaPost() {
     $this->assertEquals(201, $response['httpCode']);
     $id = $response['response']['values']['id'];
     $occurrences = $db->query("select id from occurrences where sample_id=$id");
-    $this->assertEquals(1, count($occurrences), 'Posting a sample with occurrence did not create the occurrence');
+    $this->assertEquals(
+      1, count($occurrences),
+      'Posting a sample with occurrence did not create the occurrence'
+    );
     $occurrenceId = $occurrences->current()->id;
     $occurrences = $db->query("select id from occurrence_media where occurrence_id=$occurrenceId");
-    $this->assertEquals(1, count($occurrences), 'Posting a sample with occurrence and media did not create the media');
+    $this->assertEquals(
+      1, count($occurrences),
+      'Posting a sample with occurrence and media did not create the media'
+    );
     // Check occurrence exists.
     $response = $this->callService("occurrences/$occurrenceId");
     $this->assertResponseOk($response, "/occurrences/$occurrenceId GET");
@@ -1244,7 +1295,10 @@ public function testJwtLocationPost() {
     $db = new Database();
     $locationsWebsitesCount = $db->query("select count(*) from locations_websites where location_id=$id")
       ->current()->count;
-    $this->assertEquals(1, $locationsWebsitesCount, 'No locations_websites record created for a location POST.');
+    $this->assertEquals(
+      1, $locationsWebsitesCount,
+      'No locations_websites record created for a location POST.'
+    );
   }
 
   /**
@@ -1254,7 +1308,7 @@ public function testJwtLocationPut() {
     $this->putTest('locations', [
       'name' => 'Location test',
       'centroid_sref' => 'ST1234',
-      'centroid_sref_system' => 'OSGB'
+      'centroid_sref_system' => 'OSGB',
     ], [
       'name' => 'Location test updated',
     ]);
@@ -1267,7 +1321,7 @@ public function testJwtLocationGet() {
     $this->getTest('locations', [
       'name' => 'Location GET test',
       'centroid_sref' => 'ST1234',
-      'centroid_sref_system' => 'OSGB'
+      'centroid_sref_system' => 'OSGB',
     ]);
   }
 
@@ -1293,7 +1347,7 @@ public function testJwtLocationOptions() {
    * Test behaviour around REST support for ETags.
    */
   public function testJwtLocationETags() {
-    $this->eTagsTest('locations',  [
+    $this->eTagsTest('locations', [
       'name' => 'Location GET test',
       'centroid_sref' => 'ST1234',
       'centroid_sref_system' => 'OSGB',
@@ -1311,7 +1365,7 @@ public function testJwtSurveyPost() {
    * Test /surveys PUT behaviour.
    */
   public function testJwtSurveyPut() {
-    $this->putTest('surveys',  [
+    $this->putTest('surveys', [
       'title' => 'Test survey',
       'description' => 'A test',
     ], [
@@ -1323,7 +1377,7 @@ public function testJwtSurveyPut() {
    * A basic test of /surveys GET.
    */
   public function testJwtSurveyGet() {
-    $this->getTest('surveys',  [
+    $this->getTest('surveys', [
       'title' => 'Test survey',
       'description' => 'A test',
     ]);
@@ -1333,7 +1387,7 @@ public function testJwtSurveyGet() {
    * A basic test of /surveys GET.
    */
   public function testJwtSurveysGetList() {
-    $this->getListTest('surveys',  [
+    $this->getListTest('surveys', [
       'title' => 'Test survey ' . microtime(TRUE),
       'description' => 'A test',
     ]);
@@ -1343,7 +1397,7 @@ public function testJwtSurveysGetList() {
    * Test DELETE for a survey.
    */
   public function testJwtSurveyDelete() {
-    $this->deleteTest('surveys',  [
+    $this->deleteTest('surveys', [
       'title' => 'Test survey',
       'description' => 'A test',
     ]);
@@ -1352,7 +1406,7 @@ public function testJwtSurveyDelete() {
   public function testJwtSurveyPostPermissions() {
     $this->authMethod = 'jwtUser';
     self::$jwt = $this->getJwt(self::$privateKey, 'http://www.indicia.org.uk', 1, time() + 120);
-    $data =  [
+    $data = [
       'title' => 'Test survey',
       'description' => 'A test',
     ];
@@ -1419,7 +1473,7 @@ public function testJwtSurveyOptions() {
    * Test behaviour around REST support for ETags.
    */
   public function testJwtSurveyETags() {
-    $this->eTagsTest('surveys',  [
+    $this->eTagsTest('surveys', [
       'title' => 'Test survey',
       'description' => 'A test',
     ]);
@@ -1436,7 +1490,7 @@ public function testJwtSampleAttributePost() {
    * Test /sample_attributes PUT behaviour.
    */
   public function testJwtSampleAttributePut() {
-    $this->putTest('sample_attributes',  [
+    $this->putTest('sample_attributes', [
       'caption' => 'Test sample attribute',
       'data_type' => 'T',
     ], [
@@ -1448,7 +1502,7 @@ public function testJwtSampleAttributePut() {
    * A basic test of /sample_attributes GET.
    */
   public function testJwtSampleAttributeGet() {
-    $this->getTest('sample_attributes',  [
+    $this->getTest('sample_attributes', [
       'caption' => 'Test sample attribute',
       'data_type' => 'T',
     ]);
@@ -1458,7 +1512,7 @@ public function testJwtSampleAttributeGet() {
    * A basic test of /sample_attributes GET.
    */
   public function testJwtSampleAttributeGetList() {
-    $this->getListTest('sample_attributes',  [
+    $this->getListTest('sample_attributes', [
       'caption' => 'Test sample attribute ' . microtime(TRUE),
       'data_type' => 'T',
     ]);
@@ -1485,7 +1539,7 @@ public function testJwtSampleAttributeOptions() {
    * Test behaviour around REST support for ETags.
    */
   public function testJwtSampleAttributeETags() {
-    $this->eTagsTest('sample_attributes',  [
+    $this->eTagsTest('sample_attributes', [
       'caption' => 'Test sample attribute',
       'data_type' => 'T',
     ]);
@@ -1502,7 +1556,7 @@ public function testJwtOccurrenceAttributePost() {
    * Test /occurrence_attributes PUT behaviour.
    */
   public function testJwtOccurrenceAttributePut() {
-    $this->putTest('occurrence_attributes',  [
+    $this->putTest('occurrence_attributes', [
       'caption' => 'Test occurrence attribute',
       'data_type' => 'T',
     ], [
@@ -1514,7 +1568,7 @@ public function testJwtOccurrenceAttributePut() {
    * A basic test of /occurrence_attributes GET.
    */
   public function testJwtOccurrenceAttributeGet() {
-    $this->getTest('occurrence_attributes',  [
+    $this->getTest('occurrence_attributes', [
       'caption' => 'Test occurrence attribute',
       'data_type' => 'T',
     ]);
@@ -1551,7 +1605,7 @@ public function testJwtOccurrenceAttributeOptions() {
    * Test behaviour around REST support for ETags.
    */
   public function testJwtOccurrenceAttributeETags() {
-    $this->eTagsTest('occurrence_attributes',  [
+    $this->eTagsTest('occurrence_attributes', [
       'caption' => 'Test occurrence attribute',
       'data_type' => 'T',
     ]);
@@ -1595,7 +1649,7 @@ public function testJwtOccurrencePost() {
     ], 'taxa_taxon_list_id');
   }
 
-   /**
+  /**
    * Test /occurrences PUT in isolation.
    */
   public function testJwtOccurrencePut() {
@@ -1654,7 +1708,7 @@ public function testJwtOccurrenceOptions() {
    */
   public function testJwtOccurrenceETags() {
     $sampleId = $this->postSampleToAddOccurrencesTo();
-    $this->eTagsTest('occurrences',  [
+    $this->eTagsTest('occurrences', [
       'taxa_taxon_list_id' => 1,
       'sample_id' => $sampleId,
     ]);
@@ -1683,7 +1737,10 @@ public function testJwtOccurrencePostExtKey() {
       FALSE,
       ['values' => $data]
     );
-    $this->assertEquals(409, $response['httpCode'], 'Duplicate external key did not return 409 Conflict response.');
+    $this->assertEquals(
+      409, $response['httpCode'],
+      'Duplicate external key did not return 409 Conflict response.'
+    );
   }
 
   /**
@@ -1702,7 +1759,10 @@ public function testJwtOccurrencePostDeletedSample() {
       FALSE,
       ['values' => $data]
     );
-    $this->assertEquals(400, $response['httpCode'], 'Adding occurrence to deleted sample did not return 400 Bad request response.');
+    $this->assertEquals(
+      400, $response['httpCode'],
+      'Adding occurrence to deleted sample did not return 400 Bad request response.'
+    );
   }
 
   public function testProjects_authentication() {
@@ -1715,20 +1775,32 @@ public function testProjects_authentication() {
     // user and website authentications don't allow access to projects
     $this->authMethod = 'hmacUser';
     $response = $this->callService('projects');
-    $this->assertTrue($response['httpCode']===401, 'Invalid authentication method hmacUser for projects ' .
-        "but response still OK. Http response $response[httpCode].");
+    $this->assertTrue(
+      $response['httpCode'] === 401,
+      'Invalid authentication method hmacUser for projects ' .
+      "but response still OK. Http response $response[httpCode]."
+    );
     $this->authMethod = 'directUser';
     $response = $this->callService('projects');
-    $this->assertTrue($response['httpCode']===401, 'Invalid authentication method directUser for projects ' .
-        "but response still OK. Http response $response[httpCode].");
+    $this->assertTrue(
+      $response['httpCode'] === 401,
+      'Invalid authentication method directUser for projects ' .
+      "but response still OK. Http response $response[httpCode]."
+    );
     $this->authMethod = 'hmacWebsite';
     $response = $this->callService('projects');
-    $this->assertTrue($response['httpCode']===401, 'Invalid authentication method hmacWebsite for projects ' .
-        "but response still OK. Http response $response[httpCode].");
+    $this->assertTrue(
+      $response['httpCode'] === 401,
+      'Invalid authentication method hmacWebsite for projects ' .
+      "but response still OK. Http response $response[httpCode]."
+    );
     $this->authMethod = 'directWebsite';
     $response = $this->callService('projects');
-    $this->assertTrue($response['httpCode']===401, 'Invalid authentication method directWebsite for projects ' .
-        "but response still OK. Http response $response[httpCode].");
+    $this->assertTrue(
+      $response['httpCode'] === 401,
+      'Invalid authentication method directWebsite for projects ' .
+      "but response still OK. Http response $response[httpCode]."
+    );
 
     $this->authMethod = 'hmacClient';
   }
@@ -1742,14 +1814,23 @@ public function testProjects_get() {
     $this->assertEquals(count($viaConfig), count($response['response']['data']),
         'Incorrect number of projects returned from /projects.');
     foreach ($response['response']['data'] as $projDef) {
-      $this->assertArrayHasKey($projDef['id'], $viaConfig, "Unexpected project $projDef[id]returned by /projects.");
+      $this->assertArrayHasKey(
+        $projDef['id'], $viaConfig, 
+        "Unexpected project $projDef[id]returned by /projects."
+      );
       $this->assertEquals($viaConfig[$projDef['id']]['title'], $projDef['title'],
         "Unexpected title $projDef[title] returned for project $projDef[id] by /projects.");
       $this->assertEquals($viaConfig[$projDef['id']]['description'], $projDef['description'],
         "Unexpected description $projDef[description] returned for project $projDef[id] by /projects.");
-      // Some project keys are supposed to be removed
-      $this->assertNotContains('filter_id', $projDef, 'Project definition should not contain filter_id');
-      $this->assertNotContains('sharing', $projDef, 'Project definition should not contain sharing');
+      // Some project keys are supposed to be removed.
+      $this->assertNotContains(
+        'filter_id', $projDef,
+        'Project definition should not contain filter_id'
+      );
+      $this->assertNotContains(
+        'sharing', $projDef,
+        'Project definition should not contain sharing'
+      );
     }
   }
 
@@ -1765,17 +1846,23 @@ public function testProjects_get_id() {
       $this->assertEquals($projDef['description'], $response['response']['description'],
           "Unexpected description " . $response['response']['description'] .
           " returned for project $projDef[id] by /projects/$projDef[id].");
-      // Some project keys are supposed to be removed
-      $this->assertNotContains('filter_id', $projDef, 'Project definition should not contain filter_id');
-      $this->assertNotContains('sharing', $projDef, 'Project definition should not contain sharing');
+      // Some project keys are supposed to be removed.
+      $this->assertNotContains(
+        'filter_id', $projDef,
+        'Project definition should not contain filter_id'
+      );
+      $this->assertNotContains(
+        'sharing', $projDef,
+        'Project definition should not contain sharing'
+      );
     }
   }
 
   public function testTaxon_observations_authentication() {
     Kohana::log('debug', "Running unit test, Rest_ControllerTest::testProjects_clientAuthentication");
     $proj_id = self::$config['projects'][array_keys(self::$config['projects'])[0]]['id'];
-    $queryWithProj = array('proj_id' => $proj_id, 'edited_date_from' => '2015-01-01');
-    $query = array('edited_date_from' => '2015-01-01');
+    $queryWithProj = ['proj_id' => $proj_id, 'edited_date_from' => '2015-01-01');
+    $query = ['edited_date_from' => '2015-01-01');
 
     $this->authMethod = 'hmacClient';
     $this->checkResourceAuthentication('taxon-observations', $queryWithProj);
@@ -1785,7 +1872,7 @@ public function testTaxon_observations_authentication() {
     $this->checkResourceAuthentication('taxon-observations', $query);
     // @todo The following test needs to check filtered response rather than authentication
     $this->authMethod = 'directUser';
-    $this->checkResourceAuthentication('taxon-observations', $query + array('filter_id' => self::$userFilterId));
+    $this->checkResourceAuthentication('taxon-observations', $query + ['filter_id' => self::$userFilterId));
     $this->authMethod = 'hmacWebsite';
     $this->checkResourceAuthentication('taxon-observations', $query);
     $this->authMethod = 'directWebsite';
@@ -1800,10 +1887,10 @@ public function testTaxon_observations_get_incorrect_params() {
     $this->assertEquals(400, $response['httpCode'],
         'Requesting taxon observations without params should be a bad request');
     foreach (self::$config['projects'] as $projDef) {
-      $response = $this->callService("taxon-observations", array('proj_id' => $projDef['id']));
+      $response = $this->callService("taxon-observations", ['proj_id' => $projDef['id']));
       $this->assertEquals(400, $response['httpCode'],
           'Requesting taxon observations without edited_date_from should be a bad request');
-      $response = $this->callService("taxon-observations", array('edited_date_from' => '2015-01-01'));
+      $response = $this->callService("taxon-observations", ['edited_date_from' => '2015-01-01'));
       $this->assertEquals(400, $response['httpCode'],
         'Requesting taxon observations without proj_id should be a bad request');
       // only test a single project
@@ -1820,7 +1907,7 @@ public function testTaxon_observations_get() {
     Kohana::log('debug', "Running unit test, Rest_ControllerTest::testTaxon_observations_get");
 
     foreach (self::$config['projects'] as $projDef) {
-      $response = $this->callService("taxon-observations", array(
+      $response = $this->callService("taxon-observations", [
           'proj_id' => $projDef['id'],
           'edited_date_from' => '2015-01-01',
           'edited_date_to' => date("Y-m-d\TH:i:s")
@@ -1834,9 +1921,10 @@ public function testTaxon_observations_get() {
       $data = $response['response']['data'];
       $this->assertIsArray($data, 'Taxon-observations data invalid. ' . var_export($data, true));
       $this->assertNotCount(0, $data, 'Taxon-observations data absent. ' . var_export($data, true));
-      foreach ($data as $occurrence)
+      foreach ($data as $occurrence) {
         $this->checkValidTaxonObservation($occurrence);
-      // only test a single project
+      }
+      // Only test a single project.
       break;
     }
   }
@@ -1852,17 +1940,18 @@ public function testAnnotations_get() {
     foreach (self::$config['projects'] as $projDef) {
       $response = $this->callService(
         "annotations",
-        array('proj_id' => $projDef['id'], 'edited_date_from' => '2015-01-01')
+        ['proj_id' => $projDef['id'], 'edited_date_from' => '2015-01-01')
       );
       $this->assertResponseOk($response, '/annotations');
       $this->assertArrayHasKey('paging', $response['response'], 'Paging missing from response to call to annotations');
       $this->assertArrayHasKey('data', $response['response'], 'Data missing from response to call to annotations');
       $data = $response['response']['data'];
-      $this->assertIsArray($data, 'Annotations data invalid. ' . var_export($data, true));
-      $this->assertNotCount(0, $data, 'Annotations data absent. ' . var_export($data, true));
-      foreach ($data as $annotation)
+      $this->assertIsArray($data, 'Annotations data invalid. ' . var_export($data, TRUE));
+      $this->assertNotCount(0, $data, 'Annotations data absent. ' . var_export($data, TRUE));
+      foreach ($data as $annotation) {
         $this->checkValidAnnotation($annotation);
-      // only test a single project
+      }
+      // Only test a single project.
       break;
     }
   }
@@ -1883,8 +1972,14 @@ public function testTaxaSearch_get() {
       'taxon_list_id' => 1,
     ]);
     $this->assertResponseOk($response, '/taxa/search');
-    $this->assertArrayHasKey('paging', $response['response'], 'Paging missing from response to call to taxa/search');
-    $this->assertArrayHasKey('data', $response['response'], 'Data missing from response to call to taxa/search');
+    $this->assertArrayHasKey(
+      'paging', $response['response'],
+      'Paging missing from response to call to taxa/search'
+    );
+    $this->assertArrayHasKey(
+      'data', $response['response'],
+      'Data missing from response to call to taxa/search'
+    );
     $data = $response['response']['data'];
     $this->assertIsArray($data, 'taxa/search data invalid.');
     $this->assertCount(2, $data, 'Taxa/search data wrong count returned.');
@@ -1893,8 +1988,14 @@ public function testTaxaSearch_get() {
       'taxon_list_id' => 1,
     ]);
     $this->assertResponseOk($response, '/taxa/search');
-    $this->assertArrayHasKey('paging', $response['response'], 'Paging missing from response to call to taxa/search');
-    $this->assertArrayHasKey('data', $response['response'], 'Data missing from response to call to taxa/search');
+    $this->assertArrayHasKey(
+      'paging', $response['response'],
+      'Paging missing from response to call to taxa/search'
+    );
+    $this->assertArrayHasKey(
+      'data', $response['response'],
+      'Data missing from response to call to taxa/search'
+    );
     $data = $response['response']['data'];
     $this->assertIsArray($data, 'taxa/search data invalid.');
     $this->assertCount(1, $data, 'Taxa/search data wrong count returned.');
@@ -1920,7 +2021,7 @@ public function testReportsHierarchy_get() {
     Kohana::log('debug', "Running unit test, Rest_ControllerTest::testReportsHierarchy_get");
 
     $projDef = self::$config['projects']['BRC1'];
-    $response = $this->callService("reports", array('proj_id' => $projDef['id']));
+    $response = $this->callService("reports", ['proj_id' => $projDef['id']]);
     $this->assertResponseOk($response, '/reports');
     // Check a folder that should definitely exist.
     $this->checkReportFolderInReponse($response['response'], 'library');
@@ -1931,19 +2032,19 @@ public function testReportsHierarchy_get() {
     // should be an additional featured folder at the top level with shortcuts
     // to favourite reports.
     $this->authMethod = 'hmacWebsite';
-    $response = $this->callService("reports", array('proj_id' => $projDef['id']));
+    $response = $this->callService("reports", ['proj_id' => $projDef['id']]);
     $this->checkReportFolderInReponse($response['response'], 'featured');
     $this->checkReportInReponse($response['response'], 'demo');
 
     // now check some folder contents
     $this->authMethod = 'hmacClient';
-    $response = $this->callService("reports/featured", array('proj_id' => $projDef['id']));
+    $response = $this->callService("reports/featured", ['proj_id' => $projDef['id']]);
     $this->assertResponseOk($response, '/reports/featured');
     $this->checkReportInReponse($response['response'], 'library/occurrences/filterable_explore_list');
-    $response = $this->callService("reports/library", array('proj_id' => $projDef['id']));
+    $response = $this->callService("reports/library", ['proj_id' => $projDef['id']]);
     $this->assertResponseOk($response, '/reports/library');
     $this->checkReportFolderInReponse($response['response'], 'occurrences');
-    $response = $this->callService("reports/library/occurrences", array('proj_id' => $projDef['id']));
+    $response = $this->callService("reports/library/occurrences", ['proj_id' => $projDef['id']]);
     $this->assertResponseOk($response, '/reports/library/occurrences');
     $this->checkReportInReponse($response['response'], 'filterable_explore_list');
   }
@@ -1952,15 +2053,19 @@ public function testMissingReportFile() {
     $this->authMethod = 'jwtUser';
     self::$jwt = $this->getJwt(self::$privateKey, 'http://www.indicia.org.uk', 1, time() + 120);
     $response = $this->callService('reports/some_random_report_name.xml', []);
-    $this->assertEquals(404, $response['httpCode'], 'Request for a missing report does not return 404.');
+    $this->assertEquals(
+      404, $response['httpCode'],
+      'Request for a missing report does not return 404.'
+    );
   }
 
   public function testReportParams_get() {
     Kohana::log('debug', "Running unit test, Rest_ControllerTest::testReportParams_get");
 
-    // First grab a list of reports so we can use the links to get the correct params URL
+    // First grab a list of reports so we can use the links to get the correct
+    // params URL.
     $projDef = self::$config['projects']['BRC1'];
-    $response = $this->callService("reports/library/occurrences", array('proj_id' => $projDef['id']));
+    $response = $this->callService("reports/library/occurrences", ['proj_id' => $projDef['id']]);
     $this->assertResponseOk($response, '/reports/library/occurrences');
     $reportDef = $response['response']['filterable_explore_list'];
     $this->assertArrayHasKey('params', $reportDef, 'Report response does not define parameters');
@@ -1976,9 +2081,9 @@ public function testReportParams_get() {
   public function testReportColumns_get() {
     Kohana::log('debug', "Running unit test, Rest_ControllerTest::testReportColumns_get");
 
-    // First grab a list of reports so we can use the links to get the correct columns URL
-    $projDef = self::$config['projects']['BRC1'];
-    $response = $this->callService("reports/library/occurrences", array('proj_id' => $projDef['id']));
+    // First grab a list of reports so we can use the links to get the correct
+    // columns URL.
+    $response = $this->callService("reports/library/occurrences", ['proj_id' => $projDef['id']]);
     $this->assertResponseOk($response, '/reports/library/occurrences');
     $reportDef = $response['response']['filterable_explore_list'];
     $this->assertArrayHasKey('columns', $reportDef, 'Report response does not define columns');
@@ -1994,9 +2099,10 @@ public function testReportColumns_get() {
   public function testReportOutput_get() {
     Kohana::log('debug', "Running unit test, Rest_ControllerTest::testReportOutput_get");
 
-    // First grab a list of reports so we can use the links to get the correct columns URL
+    // First grab a list of reports so we can use the links to get the correct
+    // columns URL.
     $projDef = self::$config['projects']['BRC1'];
-    $response = $this->callService("reports/library/occurrences", array('proj_id' => $projDef['id']));
+    $response = $this->callService("reports/library/occurrences", ['proj_id' => $projDef['id']]);
     $this->assertResponseOk($response, '/reports/library/occurrences');
     $reportDef = $response['response']['filterable_explore_list'];
     $this->assertArrayHasKey('href', $reportDef, 'Report response missing href');
@@ -2005,34 +2111,57 @@ public function testReportOutput_get() {
     $this->assertResponseOk($response, '/reports/library/occurrences/filterable_explore_list.xml');
     $this->assertArrayHasKey('data', $response['response']);
     $this->assertCount(1, $response['response']['data'], 'Report call returns incorrect record count');
-    $this->assertEquals(1, $response['response']['data'][0]['occurrence_id'], 'Report call returns incorrect record');
+    $this->assertEquals(
+      1, $response['response']['data'][0]['occurrence_id'],
+      'Report call returns incorrect record'
+    );
   }
 
   public function testAcceptHeader() {
     Kohana::log('debug', "Running unit test, Rest_ControllerTest::testAcceptHeader");
     $projDef = self::$config['projects']['BRC1'];
-    $response = $this->callService("reports/library/occurrences", array('proj_id' => $projDef['id']), NULL, ['Accept: application/json']);
+    $response = $this->callService(
+      "reports/library/occurrences",
+      ['proj_id' => $projDef['id']],
+      NULL,
+      ['Accept: application/json']
+    );
     $decoded = json_decode($response['response'], TRUE);
     $this->assertNotEquals(NULL, $decoded, 'JSON response could not be decoded: ' . $response['response']);
     $this->assertEquals(200, $response['httpCode']);
     $this->assertEquals(0, $response['curlErrno']);
-    $response = $this->callService("reports/library/occurrences", array('proj_id' => $projDef['id']), NULL, ['Accept: text/html']);
+    $response = $this->callService(
+      "reports/library/occurrences",
+      ['proj_id' => $projDef['id']],
+      NULL,
+      ['Accept: text/html']
+    );
     $this->assertMatchesRegularExpression('/^<!DOCTYPE HTML>/', $response['response']);
     $this->assertMatchesRegularExpression('/<html>/', $response['response']);
     $this->assertMatchesRegularExpression('/<\/html>$/', $response['response']);
     $this->assertEquals(200, $response['httpCode']);
     $this->assertEquals(0, $response['curlErrno']);
-    // try requesting an invalid content type as first preference - response should select the second.
-    $response = $this->callService("reports/library/occurrences", array('proj_id' => $projDef['id']), NULL, ['Accept: image/png, application/json']);
+    // Try requesting an invalid content type as first preference - response
+    // should select the second.
+    $response = $this->callService(
+      "reports/library/occurrences",
+      ['proj_id' => $projDef['id']],
+      NULL,
+      ['Accept: image/png, application/json']
+    );
     $decoded = json_decode($response['response'], TRUE);
-    $this->assertNotEquals(NULL, $decoded, 'JSON response could not be decoded: ' . $response['response']);
+    $this->assertNotEquals(
+      NULL, $decoded, 
+      'JSON response could not be decoded: ' . $response['response']
+    );
     $this->assertEquals(200, $response['httpCode']);
     $this->assertEquals(0, $response['curlErrno']);
   }
 
   /**
-   * Tests authentication against a resource, by passing incorrect user or secret, then
-   * finally passing the correct details to check a valid response returns.
+   * Tests authentication against a resource, by passing incorrect user or
+   * secret, then finally passing the correct details to check a valid response
+   * returns.
    *
    * @param $resource
    * @param string $user
@@ -2059,11 +2188,16 @@ private function checkResourceAuthentication($resource, $query = []) {
     self::$userPassword = '---';
 
     $response = $this->callService($resource, $query, TRUE);
-    $this->assertEquals(401, $response['httpCode'],
-      "Incorrect secret or password passed to /$resource but request authorised. Http response $response[httpCode].");
-    $this->assertEquals('Unauthorized', $response['response']['status'],
-        "Incorrect secret or password passed to /$resource but data still returned. ".
-      var_export($response, true));
+    $this->assertEquals(
+      401, $response['httpCode'],
+      "Incorrect secret or password passed to /$resource but request " .
+      "authorised. Http response $response[httpCode]."
+    );
+    $this->assertEquals(
+      'Unauthorized', $response['response']['status'],
+      "Incorrect secret or password passed to /$resource but data still returned. " .
+      var_export($response, TRUE)
+    );
     self::$config['shared_secret'] = $correctClientSecret;
     self::$websitePassword = $correctWebsitePassword;
     self::$userPassword = $correctUserPassword;
@@ -2076,10 +2210,16 @@ private function checkResourceAuthentication($resource, $query = []) {
     self::$websitePassword = $correctWebsitePassword;
     self::$userPassword = $correctUserPassword;
     $response = $this->callService($resource, $query, TRUE);
-    $this->assertEquals(401, $response['httpCode'],
-        "Incorrect userId passed to /$resource but request authorised. Http response $response[httpCode].");
-    $this->assertEquals('Unauthorized', $response['response']['status'],
-        "Incorrect userId passed to /$resource but data still returned. " . var_export($response, true));
+    $this->assertEquals(
+      401, $response['httpCode'],
+      "Incorrect userId passed to /$resource but request authorised. Http " .
+      "response $response[httpCode]."
+    );
+    $this->assertEquals(
+      'Unauthorized', $response['response']['status'],
+      "Incorrect userId passed to /$resource but data still returned. " .
+      var_export($response, TRUE)
+    );
 
     // Now test with everything correct.
     self::$clientUserId = $correctClientUserId;
@@ -2115,14 +2255,14 @@ private function assertResponseOk($response, $apiCall) {
    * @param $data Array to be tested as a taxon occurrence resource
    */
   private function checkValidTaxonObservation($data) {
-    $this->assertIsArray($data, 'Taxon-observation object invalid. ' . var_export($data, true));
-    $mustHave = array('id', 'href', 'datasetName', 'taxonVersionKey', 'taxonName',
+    $this->assertIsArray($data, 'Taxon-observation object invalid. ' . var_export($data, TRUE));
+    $mustHave = ['id', 'href', 'datasetName', 'taxonVersionKey', 'taxonName',
         'startDate', 'endDate', 'dateType', 'projection', 'precision', 'recorder', 'lastEditDate');
     foreach ($mustHave as $key) {
       $this->assertArrayHasKey($key, $data,
-          "Missing $key from taxon-observation resource. " . var_export($data, true));
+          "Missing $key from taxon-observation resource. " . var_export($data, TRUE));
       $this->assertNotEmpty($data[$key],
-          "Empty $key in taxon-observation resource" . var_export($data, true));
+          "Empty $key in taxon-observation resource" . var_export($data, TRUE));
     }
     // @todo Format tests
   }
@@ -2132,19 +2272,27 @@ private function checkValidTaxonObservation($data) {
    * @param $data Array to be tested as an annotation resource
    */
   private function checkValidAnnotation($data) {
-    $this->assertIsArray($data, 'Annotation object invalid. ' . var_export($data, true));
-    $mustHave = array('id', 'href', 'taxonObservation', 'taxonVersionKey', 'comment',
+    $this->assertIsArray($data, 'Annotation object invalid. ' . var_export($data, TRUE));
+    $mustHave = ['id', 'href', 'taxonObservation', 'taxonVersionKey', 'comment',
         'question', 'authorName', 'dateTime');
     foreach ($mustHave as $key) {
       $this->assertArrayHasKey($key, $data,
-        "Missing $key from annotation resource. " . var_export($data, true));
+        "Missing $key from annotation resource. " . var_export($data, TRUE));
       $this->assertNotEmpty($data[$key],
-        "Empty $key in annotation resource" . var_export($data, true));
+        "Empty $key in annotation resource" . var_export($data, TRUE));
+    }
+    if (!empty($data['statusCode1'])) {
+      $this->assertMatchesRegularExpression(
+        '/[AUN]/', $data['statusCode1'],
+        'Invalid statusCode1 value for annotation'
+      );
+    }
+    if (!empty($data['statusCode2'])) {
+      $this->assertMatchesRegularExpression(
+        '/[1-6]/', $data['statusCode2'],
+        'Invalid statusCode2 value for annotation'
+      );
     }
-    if (!empty($data['statusCode1']))
-      $this->assertMatchesRegularExpression('/[AUN]/', $data['statusCode1'], 'Invalid statusCode1 value for annotation');
-    if (!empty($data['statusCode2']))
-      $this->assertMatchesRegularExpression('/[1-6]/', $data['statusCode2'], 'Invalid statusCode2 value for annotation');
     // We should be able to request the taxon observation associated with the occurrence
     $session = $this->initCurl($data['taxonObservation']['href']);
     $response = $this->getCurlResponse($session);
@@ -2166,7 +2314,7 @@ private function checkReportFolderInReponse($response, $folder) {
   /**
    * Assert that a folder exists in the response from a call to /reports.
    * @param array $response
-   * @param string $folder
+   * @param string $reportFile
    */
   private function checkReportInReponse($response, $reportFile) {
     $this->assertArrayHasKey($reportFile, $response);
@@ -2180,6 +2328,7 @@ private function checkReportInReponse($response, $reportFile) {
    *
    * @param $session
    * @param $url
+   * @param additionalRequestHeader
    */
   private function setRequestHeader($session, $url, $additionalRequestHeader = []) {
     switch ($this->authMethod) {
@@ -2325,6 +2474,9 @@ private function callUrl($url, $postData = NULL, $additionalRequestHeader = [],
    * @param $method
    * @param mixed|FALSE $query
    * @param string $postData
+   * @param $additionalRequestHeader
+   * @param $customMethod
+   * @param $files
    * @return array
    */
   private function callService($method, $query = FALSE, $postData = NULL, $additionalRequestHeader = [], $customMethod = NULL, $files = FALSE) {
diff --git a/modules/taxon_associations/tests/TaxonAssocations_ServicesTest.php b/modules/taxon_associations/tests/TaxonAssocations_ServicesTest.php
index 75b81c2cd8..2e5c2165c8 100644
--- a/modules/taxon_associations/tests/TaxonAssocations_ServicesTest.php
+++ b/modules/taxon_associations/tests/TaxonAssocations_ServicesTest.php
@@ -3,9 +3,9 @@
 require_once 'client_helpers/data_entry_helper.php';
 require_once 'client_helpers/submission_builder.php';
 
-define ('CORE_FIXTURE_TERMLIST_COUNT', 4);
-define ('CORE_FIXTURE_TERM_COUNT', 5);
-define ('CORE_FIXTURE_TERMLISTS_TERM_COUNT', 5);
+define('CORE_FIXTURE_TERMLIST_COUNT', 4);
+define('CORE_FIXTURE_TERM_COUNT', 5);
+define('CORE_FIXTURE_TERMLISTS_TERM_COUNT', 5);
 
 class TaxonAssociations_ServicesTest extends Indicia_DatabaseTestCase {
 
@@ -13,16 +13,16 @@ class TaxonAssociations_ServicesTest extends Indicia_DatabaseTestCase {
 
   public function getDataSet()
   {
-    $ds1 =  new PHPUnit_Extensions_Database_DataSet_YamlDataSet('modules/phpUnit/config/core_fixture.yaml');
+    $ds1 = new PHPUnit_Extensions_Database_DataSet_YamlDataSet('modules/phpUnit/config/core_fixture.yaml');
     $ds2 = new Indicia_ArrayDataSet(
-      array(
-        'meanings' => array(
-          array(
-            'id' => 20000
-           )
-        ),
-        'termlists' => array(
-          array(
+      [
+        'meanings' => [
+          [
+            'id' => 20000,
+          ],
+        ],
+        'termlists' => [
+          [
             'title' => 'Taxon association types',
             'description' => 'Types of associations between taxa',
             'website_id' => 1,
@@ -30,21 +30,21 @@ public function getDataSet()
             'created_by_id' => 1,
             'updated_on' => '2016-07-22:16:00:00',
             'updated_by_id' => 1,
-            'external_key' => NULL
-          ),
-        ),
-        'terms' => array(
-          array(
+            'external_key' => NULL,
+          ],
+        ],
+        'terms' => [
+          [
             'term' => 'is associated with',
             'language_id' => 1,
             'created_on' => '2016-07-22:16:00:00',
             'created_by_id' => 1,
             'updated_on' => '2016-07-22:16:00:00',
-            'updated_by_id' => 1
-          ),
-        ),
-        'termlists_terms' => array(
-          array(
+            'updated_by_id' => 1,
+          ],
+        ],
+        'termlists_terms' => [
+          [
             'termlist_id' => CORE_FIXTURE_TERMLIST_COUNT + 1,
             'term_id' => CORE_FIXTURE_TERM_COUNT + 1,
             'created_on' => '2016-07-22:16:00:00',
@@ -52,11 +52,11 @@ public function getDataSet()
             'updated_on' => '2016-07-22:16:00:00',
             'updated_by_id' => 1,
             'meaning_id' => 20000,
-            'preferred' => true,
-            'sort_order' => 1
-          ),
-        ),
-      )
+            'preferred' => TRUE,
+            'sort_order' => 1,
+          ],
+        ],
+      ]
     );
 
     $compositeDs = new PHPUnit_Extensions_Database_DataSet_CompositeDataSet();
@@ -67,20 +67,30 @@ public function getDataSet()
 
   public function setup() {
     $this->auth = data_entry_helper::get_read_write_auth(1, 'password');
-    // make the tokens re-usable
-    $this->auth['write_tokens']['persist_auth']=true;
+    // Make the tokens re-usable.
+    $this->auth['write_tokens']['persist_auth'] = true;
     parent::setup();
   }
 
   function testPost() {
-    $array = array(
+    $array = [
       'taxon_association:from_taxon_meaning_id' => 10000,
       'taxon_association:to_taxon_meaning_id' => 10001,
       'taxon_association:association_type_id' => CORE_FIXTURE_TERMLISTS_TERM_COUNT + 1,
+    ];
+    $s = submission_builder::build_submission(
+      $array, ['model' => 'taxon_association']
+    );
+    $r = data_entry_helper::forward_post_to(
+      'taxon_association', $s, $this->auth['write_tokens']
+    );
+    Kohana::log(
+      'debug',
+      "Submission response to taxon_association save " . print_r($r, TRUE)
+    );
+    $this->assertTrue(
+      isset($r['success']),
+      'Submitting a taxon_association did not return success response'
     );
-    $s = submission_builder::build_submission($array, array('model' => 'taxon_association'));
-    $r = data_entry_helper::forward_post_to('taxon_association', $s, $this->auth['write_tokens']);
-    Kohana::log('debug', "Submission response to taxon_association save " . print_r($r, TRUE));
-    $this->assertTrue(isset($r['success']), 'Submitting a taxon_association did not return success response');
   }
 }
\ No newline at end of file

From f7ce64bbc40b79bdbac8f183ece2be8a43536ae4 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Thu, 5 Aug 2021 11:21:08 +0100
Subject: [PATCH 11/93] Linked ID duplicates record_id so hide.

---
 ...notifications_list_for_notifications_centre.xml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/reports/library/notifications/notifications_list_for_notifications_centre.xml b/reports/library/notifications/notifications_list_for_notifications_centre.xml
index 23b734f50b..1967b65a00 100644
--- a/reports/library/notifications/notifications_list_for_notifications_centre.xml
+++ b/reports/library/notifications/notifications_list_for_notifications_centre.xml
@@ -1,6 +1,6 @@
 <report
     title="Notifications list for notifications centre"
-    description="A list of notifications for use with the notifications centre extension, 
+    description="A list of notifications for use with the notifications centre extension,
                  has additional functionality for handling edit and view icons on the grid when compared to the notifications_list.xml report.">
   <query>
   SELECT #columns#
@@ -9,11 +9,11 @@
       LEFT JOIN cache_occurrences_functional coFilter ON n.linked_id=coFilter.id
       #joins#
     WHERE n.acknowledged=false
-      and n.user_id=#user_id#  
+      and n.user_id=#user_id#
       and (o.training=#training# or o.id is null)
       and n.source_type&lt;&gt;'T' -- skip trigger notifications which don't display correctly in the grid
     #order_by#
-  </query>  
+  </query>
   <order_bys>
     <order_by>n.id desc</order_by>
   </order_bys>
@@ -34,7 +34,7 @@
       </wheres>
     </param>
     <!-- parameter for user configurable source types on edit tab. -->
-    <param name="source_types" display="Source Types" 
+    <param name="source_types" display="Source Types"
         description="Comma separated list of source types with each item single quoted, or blank to allow all." datatype="text[]" default=''>
       <wheres>
         <where>n.source_type in (#source_types#)</where>
@@ -46,7 +46,7 @@
     <param name="groups_admin_page_path" display="Groups admin page path" datatype="text" default=""
            description="The path to the page which allows group admins to manage membership" />
     <param name="website_id" display="Website Id for the occurrence record." datatype="integer"/>
-    <param name="group_ids" display="Group Ids" 
+    <param name="group_ids" display="Group Ids"
         description="Comma separated list of groups ids, or blank to allow all. Only notifications associated with these group ids will be displayed." datatype="text" default=''>
       <wheres>
         <where>o.group_id in (#group_ids#)</where>
@@ -60,10 +60,10 @@
     <column name="data" display="Message" datatype="text" sql="replace(replace(n.data, '\n', '&lt;br/&gt;'), '&quot;username&quot;:&quot;admin&quot;', '&quot;username&quot;:&quot;#system_name#&quot;')" />
     <column name="occurrence_id" display="Record ID" datatype="integer" sql="o.id" />
     <column name="sample_id" display="Sample ID" visible="false" datatype="integer" sql="o.sample_id" />
-    <column name="linked_id" display="Linked ID" datatype="integer" sql="n.linked_id" />
+    <column name="linked_id" display="Linked ID" datatype="integer" sql="n.linked_id" visible="false" />
     <column name="acknowledged" display="Acknowledged" datatype="boolean" sql="n.acknowledged" visible="false" />
     <column name="user_id" display="User ID" visible="false" datatype="integer" sql="n.user_id" />
-    <column name="triggered_on" display="Notification date" datatype="date" sql="n.triggered_on" visible="false" />    
+    <column name="triggered_on" display="Notification date" datatype="date" sql="n.triggered_on" visible="false" />
     <column name="triggered_date" display="Notification date" datatype="date" sql="to_char(n.triggered_on, 'DD/MM/YYYY')" orderby="triggered_on" />
     <column name="editing_form" visible="false" sql="case when o.input_form is null AND o.website_id=#website_id# AND o.created_by_id=#user_id# then'#default_edit_page_path#'else o.input_form end" />
     <column name="viewing_form" visible="false" sql="case when o.website_id=#website_id# then '#view_record_page_path#' else null end" />

From 59e9a89d91503e0194f5cfcf9233a3a8c8738201 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Fri, 6 Aug 2021 13:54:22 +0100
Subject: [PATCH 12/93] Prevents an error when using delayed updates

As Kohana detects an insert query, which causes LASTVAL to be called.
Insert_id not available for this type of query.
---
 modules/cache_builder/helpers/cache_builder.php | 1 +
 1 file changed, 1 insertion(+)

diff --git a/modules/cache_builder/helpers/cache_builder.php b/modules/cache_builder/helpers/cache_builder.php
index 69f7b2f376..f540aeae19 100644
--- a/modules/cache_builder/helpers/cache_builder.php
+++ b/modules/cache_builder/helpers/cache_builder.php
@@ -240,6 +240,7 @@ public static function delete($db, $table, array $ids) {
   private static function delayChangesViaWorkQueue($db, $table, $idCsv) {
     $entity = inflector::singular($table);
     $sql = <<<SQL
+-- Comment necessary to prevent Kohana calling LASTVAL().
 INSERT INTO work_queue(task, entity, record_id, params, cost_estimate, priority, created_on)
 SELECT 'task_cache_builder_update', '$entity', t.id, null, 100, 2, now()
 FROM $table t

From 91811335536a796cfd3aa78dca9733a62a385144 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Fri, 6 Aug 2021 13:56:50 +0100
Subject: [PATCH 13/93] New fields supported by api_persist

For the BTO sync code.
---
 modules/rest_api_sync/helpers/api_persist.php | 193 +++++++++++++-----
 1 file changed, 142 insertions(+), 51 deletions(-)

diff --git a/modules/rest_api_sync/helpers/api_persist.php b/modules/rest_api_sync/helpers/api_persist.php
index f4cffd8a22..3e5dd9e6ec 100644
--- a/modules/rest_api_sync/helpers/api_persist.php
+++ b/modules/rest_api_sync/helpers/api_persist.php
@@ -40,6 +40,48 @@ class api_persist {
 
   private static $mediaTypes = [];
 
+  /**
+   * Capture information about Darwin Core linked attributes in the survey.
+   *
+   * @var array
+   */
+  private static $dwcAttributes = [];
+
+  /**
+   * Finds attributes in the survey which have a DwC term name.
+   *
+   * Allows the code to post data into these attributes without continually
+   * looking them up.
+   */
+  public static function initDwcAttributes($db, $surveyId) {
+    self::$dwcAttributes = [
+      'occAttrs' => self::fetchDwcAttrs($db, 'occurrence', $surveyId),
+      'smpAttrs' => self::fetchDwcAttrs($db, 'sample', $surveyId),
+    ];
+    echo '<pre>';
+    var_export(self::$dwcAttributes);
+    echo '</pre>';
+  }
+
+  private static function fetchDwcAttrs($db, $type, $surveyId) {
+    // List of DwC terms that we might process values for.
+    $attrs = $db->select('a.id, a.term_name')
+      ->from("{$type}_attributes as a")
+      ->join("{$type}_attributes_websites as aw", "aw.{$type}_attribute_id", 'a.id')
+      ->where([
+        'aw.restrict_to_survey_id' => $surveyId,
+        'a.deleted' => 'f',
+        'aw.deleted' => 'f',
+      ])
+      ->where('a.term_name IS NOT NULL')
+      ->get();
+    $r = [];
+    foreach ($attrs as $attr) {
+      $r[$attr->term_name] = ($type === 'sample' ? 'smp' : 'occ') . "Attr:$attr->id";
+    }
+    return $r;
+  }
+
   /**
    * Persists a taxon-observation resource.
    *
@@ -64,7 +106,10 @@ class api_persist {
    * @throws \exception
    */
   public static function taxonObservation($db, array $observation, $website_id, $survey_id, $taxon_list_id, $allowUpdateWhenVerified) {
-    if (!empty($observation['taxonVersionKey'])) {
+    if (!empty($observation['organismKey'])) {
+      $lookup = ['organism_key' => $observation['organismKey']];
+    }
+    elseif (!empty($observation['taxonVersionKey'])) {
       $lookup = ['search_code' => $observation['taxonVersionKey']];
     }
     elseif (!empty($observation['taxonName'])) {
@@ -94,7 +139,6 @@ public static function taxonObservation($db, array $observation, $website_id, $s
     // Set the spatial reference depending on the projection information
     // supplied.
     self::setSrefData($values, $observation, 'sample:entered_sref');
-    self::setCoordinateUncertainty($db, $survey_id, $values, $observation);
 
     // Site handling. If a known site with a SiteKey, we can create a record in
     // locations, otherwise use the free text location_name field.
@@ -113,6 +157,27 @@ public static function taxonObservation($db, array $observation, $website_id, $s
     return count($existing) === 0;
   }
 
+  /**
+   * Copies a "standard" Dwc attribute from the observation to the values.
+   *
+   * The DwC attribute will be linked to a custom attribute where the term_name
+   * identifies the DwC term.
+   *
+   * @param array $observation
+   *   Provided observation values.
+   * @param array $values
+   *   Submission values which will be updated with the value.
+   * @param string $type
+   *   Attribute type, smp or occ.
+   * @param string $term
+   *   The DwC term.
+   */
+  private static function copyDwcAttribute(array $observation, array &$values, $type, $term) {
+    if (!empty($observation[$term]) && !empty(self::$dwcAttributes["{$type}Attrs"][$term])) {
+      $values[self::$dwcAttributes["{$type}Attrs"][$term]] = $observation[$term];
+    }
+  }
+
   /**
    * Persists an annotation resource.
    *
@@ -175,20 +240,23 @@ public static function annotation($db, array $annotation, $survey_id) {
    */
   private static function getTaxonObservationValues($db, $website_id, array $observation, $ttl_id) {
     $sensitive = isset($observation['sensitive']) && strtolower($observation['sensitive']) === 't';
-    $values = array(
+    $values = [
       'website_id' => $website_id,
       'sample:date_start'     => $observation['startDate'],
       'sample:date_end'       => $observation['endDate'],
       'sample:date_type'      => $observation['dateType'],
-      'sample:recorder_names' => isset($observation['recorder']) ? $observation['recorder'] : 'Unknown',
+      'sample:recorder_names' => isset($observation['recordedBy']) ? $observation['recordedBy'] : 'Unknown',
       'occurrence:taxa_taxon_list_id' => $ttl_id,
       'occurrence:external_key' => $observation['id'],
       'occurrence:zero_abundance' => isset($observation['zeroAbundance']) ? strtolower($observation['zeroAbundance']) : 'f',
       'occurrence:sensitivity_precision' => $sensitive ? 10000 : NULL,
-    );
+    ];
     if (!empty($observation['licenceCode'])) {
       $values['sample:licence_id'] = self::getLicenceIdFromCode($db, $observation['licenceCode']);
     }
+    if (!empty($observation['occurrenceRemarks'])) {
+      $values['occurrence:comment'] = $observation['occurrenceRemarks'];
+    }
     if (!empty($observation['media'])) {
       foreach ($observation['media'] as $idx => $medium) {
         $values["occurrence_medium:path:$idx"] = $medium['path'];
@@ -205,9 +273,49 @@ private static function getTaxonObservationValues($db, $website_id, array $obser
         $values["occAttr:$id"] = $value;
       }
     }
+    if (!empty($observation['eventId'])) {
+      $values['sample:external_key'] = $observation['eventId'];
+    }
+    if (!empty($observation['eventRemarks'])) {
+      $values['sample:comment'] = $observation['eventRemarks'];
+    }
+    if (!empty($observation['identificationVerificationStatus'])) {
+      self::applyIdentificationVerificationStatus($observation['identificationVerificationStatus'], $values);
+    }
+    self::copyDwcAttribute($observation, $values, 'smp', 'coordinateUncertaintyInMeters');
+    self::copyDwcAttribute($observation, $values, 'smp', 'collectionCode');
+    self::copyDwcAttribute($observation, $values, 'occ', 'individualCount');
+    self::copyDwcAttribute($observation, $values, 'occ', 'lifeStage');
+    self::copyDwcAttribute($observation, $values, 'occ', 'reproductiveCondition');
+    self::copyDwcAttribute($observation, $values, 'occ', 'sex');
+    self::copyDwcAttribute($observation, $values, 'occ', 'identifiedBy');
+    self::copyDwcAttribute($observation, $values, 'occ', 'identificationRemarks');
     return $values;
   }
 
+  private static function applyIdentificationVerificationStatus($identificationVerificationStatus, array &$values) {
+    $mappings = [
+      'accepted' => ['V', NULL],
+      'accepted - correct' => ['V', 1],
+      'accepted - considered correct' => ['V', 2],
+      'unconfirmed' => ['C', NULL],
+      'unconfirmed - plausible' => ['C', 3],
+      'unconfirmed - not reviewed' => ['C', NULL],
+      'not accepted' => ['R', NULL],
+      'not accepted - unable to verify' => ['R', 4],
+      'not accepted - incorrect' => ['R', 5],
+    ];
+    if (isset($mappings[strtolower($identificationVerificationStatus)])) {
+      $statuses = $mappings[strtolower($identificationVerificationStatus)];
+      kohana::log('debug', strtolower($identificationVerificationStatus) . ': ' . var_export($statuses, TRUE));
+      $values['occurrence:record_status'] = $statuses[0];
+      $values['occurrence:record_substatus'] = $statuses[1];
+    }
+    else {
+      throw new exception("Invalid identificationVerificationStatus value: $identificationVerificationStatus");
+    }
+  }
+
   private static function mapOccAttrValueToTermId($db, $occAttrId, &$value) {
     $cacheId = "occAttrIsLookup-$occAttrId";
     $cache = Cache::instance();
@@ -323,10 +431,10 @@ private static function getLicenceIdFromCode($db, $licenceCode) {
       $licenceData = $db->query($qry)->result_array(FALSE);
       self::$licences = [];
       foreach ($licenceData as $licence) {
-        self::$licences[strtolower($licence['code'])] = $licence['id'];
+        self::$licences[strtolower(str_replace($licence['code'], ' ', '-'))] = $licence['id'];
       }
     }
-    return self::$licences[$licenceCode];
+    return self::$licences[strtolower(str_replace($licenceCode, ' ', '-'))];
   }
 
   /**
@@ -360,14 +468,7 @@ private static function findTaxon($db, $taxon_list_id, array $lookup) {
         $exactMatches[] = "'" . pg_escape_string("$words[0] $words[1] subsp. $words[2]") . "'";
       }
       $filter .= 'AND original in (' . implode(',', $exactMatches) . ")\n";
-    }
-    else {
-      // Add in the exact match filter for other search methods.
-      foreach ($lookup as $key => $value) {
-        $filter .= "AND $key='$value'\n";
-      }
-    }
-    $qry = <<<SQL
+      $qry = <<<SQL
 SELECT taxon_meaning_id, taxa_taxon_list_id
 FROM cache_taxon_searchterms
 WHERE taxon_list_id=$taxon_list_id
@@ -375,6 +476,23 @@ private static function findTaxon($db, $taxon_list_id, array $lookup) {
 $filter
 ORDER BY preferred DESC
 SQL;
+    }
+    else {
+      // Add in the exact match filter for other search methods.
+      foreach ($lookup as $key => $value) {
+        $filter .= "AND t.$key='$value'\n";
+      }
+      $qry = <<<SQL
+SELECT ttl.taxon_meaning_id, ttl.id as taxa_taxon_list_id
+FROM taxa_taxon_lists ttl
+JOIN taxa t ON t.id=ttl.taxon_id AND t.deleted=false
+WHERE ttl.taxon_list_id=$taxon_list_id
+AND ttl.deleted=false
+$filter
+ORDER BY ttl.preferred DESC
+SQL;
+    }
+
     $taxa = $db->query($qry)->result_array(FALSE);
     // Need to know if the search found a single unique taxon concept so count
     // the taxon meanings.
@@ -407,7 +525,7 @@ private static function findTaxon($db, $taxon_list_id, array $lookup) {
    * @throws \exception
    */
   private static function checkMandatoryFields(array $array, $resourceName) {
-    $required = array();
+    $required = [];
     // Deletions have no other mandatory fields except the id to delete.
     if (!empty($resource['delete']) && $resource['delete'] === 'T') {
       $array[] = 'id';
@@ -415,16 +533,15 @@ private static function checkMandatoryFields(array $array, $resourceName) {
     else {
       switch ($resourceName) {
         case 'taxon-observation':
-          $required = array(
+          $required = [
             'id',
-            'href', /*'taxonName', */
             'startDate',
             'endDate',
             'dateType',
             'projection',
-            'precision',
-            'recorder',
-          );
+            'coordinateUncertaintyInMeters',
+            'recordedBy',
+          ];
           // Conditionally required fields.
           if (empty($array['gridReference'])) {
             $required[] = 'east';
@@ -433,7 +550,10 @@ private static function checkMandatoryFields(array $array, $resourceName) {
               $required[] = 'gridReference';
             }
           }
-          $required[] = empty($array['taxonVersionKey']) ? 'taxonName' : 'taxonVersionKey';
+          // One of taxonVersionKey, organismKey or taxonName required.
+          if (empty($array['taxonVersionKey']) && empty($array['organismKey'])) {
+            $required[] = 'taxonName';
+          }
           break;
 
         case 'annotation':
@@ -595,35 +715,6 @@ private static function setSrefData(array &$values, array $observation, $fieldna
     }
   }
 
-  /**
-   * Sets the coordinate uncertainty attribute for an imported observation.
-   *
-   * @param object $db
-   *   Database connection.
-   * @param int $survey_id
-   *   ID of the survey (the sref_precision attribute should be linked to this).
-   * @param array $values
-   *   The values array to add the precision information to.
-   * @param array $observation
-   *   The observation data array.
-   */
-  private static function setCoordinateUncertainty($db, $survey_id, array &$values, array $observation) {
-    if (!empty($observation['precision'])) {
-      $attr = $db->select('a.id')
-        ->from('sample_attributes as a')
-        ->join('sample_attributes_websites as aw', 'aw.sample_attribute_id', 'a.id')
-        ->where([
-          'a.system_function' => 'sref_precision',
-          'aw.restrict_to_survey_id' => $survey_id,
-          'a.deleted' => 'f',
-          'aw.deleted' => 'f',
-        ])->get()->current();
-      if ($attr) {
-        $values["smpAttr:$attr->id"] = $observation['precision'];
-      }
-    }
-  }
-
   /**
    * Returns a formatted decimal latitude and longitude string.
    *

From a4da104650effbbde05cb6a7ff99d8e4a7672c96 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Fri, 6 Aug 2021 13:59:03 +0100
Subject: [PATCH 14/93] Changes required to match updated api_persist class.

---
 .../helpers/rest_api_sync_inaturalist.php              | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/modules/rest_api_sync/helpers/rest_api_sync_inaturalist.php b/modules/rest_api_sync/helpers/rest_api_sync_inaturalist.php
index 3723a2a211..ee0c9653ca 100644
--- a/modules/rest_api_sync/helpers/rest_api_sync_inaturalist.php
+++ b/modules/rest_api_sync/helpers/rest_api_sync_inaturalist.php
@@ -121,6 +121,7 @@ public static function loadControlledTerms($serverId, array $server) {
    */
   public static function syncPage($serverId, array $server) {
     $db = Database::instance();
+    api_persist::initDwcAttributes($db, $server['survey_id']);
     $fromDateTime = variable::get("rest_api_sync_{$serverId}_last_run", '1600-01-01T00:00:00+00:00', FALSE);
     $fromId = variable::get("rest_api_sync_{$serverId}_last_id", 0, FALSE);
     $lastId = $fromId;
@@ -156,16 +157,15 @@ public static function syncPage($serverId, array $server) {
           'startDate' => $iNatRecord['observed_on'],
           'endDate' => $iNatRecord['observed_on'],
           'dateType' => 'D',
-          'recorder' => empty($iNatRecord['user']['name']) ? $iNatRecord['user']['login'] : $iNatRecord['user']['name'],
+          'recordedBy' => empty($iNatRecord['user']['name']) ? $iNatRecord['user']['login'] : $iNatRecord['user']['name'],
           'east' => $east,
           'north' => $north,
           'projection' => 'WGS84',
-          'precision' => $iNatRecord['public_positional_accuracy'],
+          'coordinateUncertaintyInMeters' => $iNatRecord['public_positional_accuracy'],
           'siteName' => $iNatRecord['place_guess'],
           'href' => $iNatRecord['uri'],
           // American English in iNat field name - sic.
-          // Also correct extra hyphen in iNat CC licence codes.
-          'licenceCode' => preg_replace('/^cc-/', 'cc ', $iNatRecord['license_code']),
+          'licenceCode' => $iNatRecord['license_code'],
         ];
         if (!empty($iNatRecord['photos'])) {
           $observation['media'] = [];
@@ -176,7 +176,7 @@ public static function syncPage($serverId, array $server) {
                 'path' => $iNatPhoto['url'],
                 'caption' => $iNatPhoto['attribution'],
                 'mediaType' => 'Image:iNaturalist',
-                'licenceCode' => preg_replace('/^cc-/', 'cc ', $iNatPhoto['license_code']),
+                'licenceCode' => $iNatPhoto['license_code'],
               ];
             }
           }

From 87252d5f0ab931baf165142a8569572cd729214c Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Fri, 6 Aug 2021 14:00:57 +0100
Subject: [PATCH 15/93] New class for BTO sync obs reads.

---
 .../helpers/rest_api_sync_indicia.php         |   3 +
 .../rest_api_sync_json_occurrences.php        | 264 ++++++++++++++++++
 2 files changed, 267 insertions(+)
 create mode 100644 modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php

diff --git a/modules/rest_api_sync/helpers/rest_api_sync_indicia.php b/modules/rest_api_sync/helpers/rest_api_sync_indicia.php
index fce524755b..d5b9ad9cbb 100644
--- a/modules/rest_api_sync/helpers/rest_api_sync_indicia.php
+++ b/modules/rest_api_sync/helpers/rest_api_sync_indicia.php
@@ -101,6 +101,9 @@ public static function syncServer($serverId, array $server) {
     }
   }
 
+  public static function loadControlledTerms() {
+  }
+
   /**
    * Currently just a stub function to treat the whole Indicia sync as a page.
    *
diff --git a/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php b/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php
new file mode 100644
index 0000000000..7480e4349b
--- /dev/null
+++ b/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php
@@ -0,0 +1,264 @@
+<?php
+
+/**
+ * @file
+ * Helper class for synchronising records from an iNaturalist server.
+ *
+ * Indicia, the OPAL Online Recording Toolkit.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * any later version.
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/gpl.html.
+ *
+ * @author Indicia Team
+ * @license http://www.gnu.org/licenses/gpl.html GPL
+ * @link https://github.com/indicia-team/warehouse/
+ */
+
+defined('SYSPATH') or die('No direct script access.');
+
+define('MAX_PAGES', 1);
+
+/**
+ * Helper class for syncing to the JSON occurrences API of another server.
+ *
+ * Could be an Indicia warehouse, or another server implementing the same
+ * standard.
+ */
+class rest_api_sync_json_occurrences {
+
+  /**
+   * Synchronise a set of data loaded from the iNat server.
+   *
+   * @param string $serverId
+   *   ID of the server as defined in the configuration.
+   * @param array $server
+   *   Server configuration.
+   */
+  public static function syncServer($serverId, array $server) {
+    kohana::log('debug', 'In syncServer');
+    kohana::log_save();
+    // Count of pages done in this run.
+    $pageCount = 0;
+    // If last run still going, not on first page.
+    $firstPage = !variable::get("rest_api_sync_{$serverId}_next_run");
+    if ($firstPage) {
+      // Track when we started this run, so the next run can pick up all
+      // changes.
+      $timestampAtStart = date('c');
+      variable::set("rest_api_sync_{$serverId}_next_run", $timestampAtStart);
+    }
+    do {
+      $syncStatus = self::syncPage($serverId, $server);
+      $pageCount++;
+      ob_flush();
+    } while ($syncStatus['moreToDo'] && $pageCount < MAX_PAGES);
+    if (!$syncStatus['moreToDo']) {
+      variable::set("rest_api_sync_{$serverId}_last_run", variable::get("rest_api_sync_{$serverId}_next_run"));
+      variable::delete("rest_api_sync_{$serverId}_next_run");
+      variable::delete("rest_api_sync_{$serverId}_last_id");
+    }
+  }
+
+  public static function loadControlledTerms() {
+  }
+
+  /**
+   * Synchronise a single page of data loaded from the server.
+   *
+   * @param string $serverId
+   *   ID of the server as defined in the configuration.
+   * @param array $server
+   *   Server configuration.
+   *
+   * @return array
+   *   Status info.
+   */
+  public static function syncPage($serverId, array $server) {
+    $db = Database::instance();
+    api_persist::initDwcAttributes($db, $server['survey_id']);
+    $nextPage = variable::get("rest_api_sync_{$serverId}_next_page", [], FALSE);
+    $data = rest_api_sync::getDataFromRestUrl(
+      "$server[url]?" . http_build_query($nextPage),
+      $serverId
+    );
+    $taxon_list_id = Kohana::config('rest_api_sync.taxon_list_id');
+    $tracker = ['inserts' => 0, 'updates' => 0, 'errors' => 0];
+    foreach ($data['data'] as $record) {
+      echo '<pre>' . var_export($record, TRUE) . '</pre>';
+      // @todo Handle grid ref if provided
+      // @todo taxonID is ORGANISM_KEY
+      // @todo Ensure non-exact dates are handled.
+      // @todo Make sure all fields in specification are handled
+      // @todo Handle moreToDo code.
+      // @todo occurrence.associated_media
+      // @todo occurrence.occurrence_status
+      // @todo occurrence.organism_quantity
+      // @todo occurrence.organism_quantity_type
+      // @todo occurrence.otherCatalogNumbers
+      // @todo event.samplingProtocol
+      // @todo location.geodeticDatum
+      $parsedDate = self::parseDates($record['event']['eventDate']);
+      try {
+        $observation = [
+          'licenceCode' => empty($record['record-level']['license']) ? NULL : $record['record-level']['license'],
+          'collectionCode' => empty($record['record-level']['collectionCode']) ? NULL : $record['record-level']['collectionCode'],
+          'occurrenceMetadata' => empty($record['record-level']['dynamicProperties']) ? NULL : $record['record-level']['dynamicProperties'],
+          'id' => "$serverId:" . $record['occurrence']['occurrenceID'],
+          'individualCount' => empty($record['occurrence']['individualCount']) ? NULL : $record['occurrence']['individualCount'],
+          'lifeStage' => empty($record['occurrence']['lifeStage']) ? NULL : $record['occurrence']['lifeStage'],
+          'recordedBy' => $record['occurrence']['recordedBy'],
+          'occurrenceRemarks' => empty($record['occurrence']['occurrenceRemarks']) ? NULL : $record['occurrence']['occurrenceRemarks'],
+          'reproductiveCondition' => empty($record['occurrence']['reproductiveCondition']) ? NULL : $record['occurrence']['reproductiveCondition'],
+          'sex' => empty($record['occurrence']['sex']) ? NULL : $record['occurrence']['sex'],
+          'sensitivityPrecision' => empty($record['occurrence']['sensitivityBlur']) ? NULL : $record['occurrence']['sensitivityBlur'],
+          'organismKey' => $record['taxon']['taxonID'],
+          'taxonVersionKey' => empty($record['taxon']['taxonNameID']) ? NULL : $record['taxon']['taxonNameID'],
+          'eventId' => empty($record['event']['eventId']) ? NULL : $record['event']['eventId'],
+          'startDate' => $parsedDate['start'],
+          'endDate' => $parsedDate['end'],
+          'dateType' => $parsedDate['type'],
+          'samplingProtocol' => empty($record['event']['samplingProtocol']) ? NULL : $record['event']['samplingProtocol'],
+          'coordinateUncertaintyInMeters' => $record['location']['coordinateUncertaintyInMeters'],
+          'siteName' => empty($record['location']['locality']) ? NULL : $record['location']['locality'],
+          'identifiedBy' => empty($record['identification']['identifiedBy']) ? NULL : $record['identification']['identifiedBy'],
+          'identificationVerificationStatus' => empty($record['identification']['identificationVerificationStatus']) ? NULL : $record['identification']['identificationVerificationStatus'],
+        ];
+        if (!empty($record['location']['gridReference'])) {
+          $observation['gridReference'] = strtoupper(str_replace(' ', '', $record['location']['gridReference']));
+          if (preg_match('/[A-Z][A-Z]\d*/', $observation['gridReference'])) {
+            $observation['projection'] = 'OSGB';
+          }
+          elseif (preg_match('/[A-Z]\d*/', $observation['gridReference'])) {
+            $observation['projection'] = 'OSIE';
+          }
+          else {
+            throw new exception('Invalid grid reference format: ' . $record['location']['gridReference']);
+          }
+        }
+        else {
+          $observation['east'] = $record['location']['decimalLongitude'];
+          $observation['north'] = $record['location']['decimalLatitude'];
+          $observation['projection'] = 'WGS84';
+        }
+        if (!empty($server['otherFields'])) {
+          foreach ($server['otherFields'] as $src => $dest) {
+            $path = explode('.', $src);
+            if (!empty($record[$path[0]]) && !empty($record[$path[1]])) {
+              // @todo Check multi-value/array handling.
+              $attrTokens = explode(':', $dest);
+              $observation[$attrTokens[0] . 's'][$attrTokens[1]] = $record[$path[0]][$path[1]];
+            }
+          }
+        }
+        echo '<pre>' . var_export($observation, TRUE) . '</pre><br/>';
+
+        $is_new = api_persist::taxonObservation(
+          $db,
+          $observation,
+          $server['website_id'],
+          $server['survey_id'],
+          $taxon_list_id,
+          $server['allowUpdateWhenVerified']
+        );
+        if ($is_new !== NULL) {
+          $tracker[$is_new ? 'inserts' : 'updates']++;
+        }
+        $db->query("UPDATE rest_api_sync_skipped_records SET current=false " .
+          "WHERE server_id='$serverId' AND source_id='{$record['occurrence']['occurrenceID']}' AND dest_table='occurrences'");
+      }
+      catch (exception $e) {
+        rest_api_sync::log(
+          'error',
+          "Error occurred submitting an occurrence with ID {$record['occurrence']['occurrenceID']}\n" . $e->getMessage(),
+          $tracker
+        );
+        $msg = pg_escape_string($e->getMessage());
+        $createdById = isset($_SESSION['auth_user']) ? $_SESSION['auth_user']->id : 1;
+        $sql = <<<QRY
+INSERT INTO rest_api_sync_skipped_records (
+  server_id,
+  source_id,
+  dest_table,
+  error_message,
+  current,
+  created_on,
+  created_by_id
+)
+VALUES (
+  '$serverId',
+  '{$record['occurrence']['occurrenceID']}',
+  'occurrences',
+  '$msg',
+  true,
+  now(),
+  $createdById
+)
+QRY;
+        $db->query($sql);
+      }
+      throw new exception('Stop');
+    }
+    variable::set("rest_api_sync_{$serverId}_next_page", $data['paging']['next']);
+    rest_api_sync::log(
+      'info',
+      "<strong>Observations</strong><br/>Inserts: $tracker[inserts]. Updates: $tracker[updates]. Errors: $tracker[errors]"
+    );
+    $r = [
+      'moreToDo' => count($data['data']) === PAGE_SIZE,
+      'pagesToGo' => ceil($data['total_results'] / PAGE_SIZE),
+      'recordsToGo' => $data['total_results'],
+    ];
+    return $r;
+  }
+
+  /**
+   * Parses a date string into the start, end and type.
+   *
+   * @param string $dateString
+   *   Single date (yyyy-mm-dd), range of dates (yyyy-mm-dd/yyyy-mm-dd), month
+   *   (yyyy-mm) or year (yyyy).
+   *
+   * @return array
+   *   Array containing vague date parts, start, end and type.
+   */
+  private static function parseDates($dateString) {
+    if (preg_match('/^\d{4}-\d{2}-\d{2}$/', $dateString)) {
+      return [
+        'start' => $dateString,
+        'end' => $dateString,
+        'type' => 'D',
+      ];
+    }
+    elseif (preg_match('/^(?P<start>\d{4}-\d{2}-\d{2})\|(?P<end>\d{4}-\d{2}-\d{2})$/', $dateString, $matches)) {
+      return [
+        'start' => $matches['start'],
+        'end' => $matches['end'],
+        'type' => 'DD',
+      ];
+    }
+    elseif (preg_match('/^(?P<year>\d{4})-(?P<month>\d{2})$/', $dateString)) {
+      return [
+        'start' => "$dateString-01",
+        'end' => "$dateString-" . cal_days_in_month(CAL_GREGORIAN, $matches['month'], $matches['year']),
+        'type' => 'O',
+      ];
+    }
+    elseif (preg_match('/^(?P<year>\d{4})$/', $dateString)) {
+      return [
+        'start' => "$dateString-01-01",
+        'end' => "$dateString-12-31",
+        'type' => 'Y',
+      ];
+    }
+
+  }
+
+}

From ec0a2bdbeab48b2341021c8f4e0abecf0612a88e Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Fri, 6 Aug 2021 14:01:17 +0100
Subject: [PATCH 16/93] Correct where recordedBy comes from in structure.

---
 modules/rest_api_sync/helpers/rest_api_sync_rest.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/rest_api_sync/helpers/rest_api_sync_rest.php b/modules/rest_api_sync/helpers/rest_api_sync_rest.php
index 690c4aa163..ed5d3d3f69 100644
--- a/modules/rest_api_sync/helpers/rest_api_sync_rest.php
+++ b/modules/rest_api_sync/helpers/rest_api_sync_rest.php
@@ -85,7 +85,7 @@ public static function syncTaxonObservationsGet($foo, $clientConfig, $projectId)
         $obj['event']['eventRemarks'] = $doc->event->event_remarks;
       }
       if (!empty($doc->event->recorded_by)) {
-        $obj['event']['recordedBy'] = $doc->event->recorded_by;
+        $obj['occurrence']['recordedBy'] = $doc->event->recorded_by;
       }
       if (!empty($doc->event->sampling_protocol)) {
         $obj['event']['samplingProtocol'] = $doc->event->sampling_protocol;

From 47f58b4e24faad0b68f6a89d035e1109966e4fc3 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Fri, 6 Aug 2021 14:48:50 +0100
Subject: [PATCH 17/93] Handle lack of paging info

Also fixes that coordinateUncertainty is optional.
---
 .../helpers/rest_api_sync_json_occurrences.php         | 10 +++++-----
 .../views/rest_api_sync_skipped_record/index.js        |  8 +++++---
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php b/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php
index 7480e4349b..6827440bf3 100644
--- a/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php
+++ b/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php
@@ -126,7 +126,7 @@ public static function syncPage($serverId, array $server) {
           'endDate' => $parsedDate['end'],
           'dateType' => $parsedDate['type'],
           'samplingProtocol' => empty($record['event']['samplingProtocol']) ? NULL : $record['event']['samplingProtocol'],
-          'coordinateUncertaintyInMeters' => $record['location']['coordinateUncertaintyInMeters'],
+          'coordinateUncertaintyInMeters' => empty($record['location']['coordinateUncertaintyInMeters']) ? NULL : $record['location']['coordinateUncertaintyInMeters'],
           'siteName' => empty($record['location']['locality']) ? NULL : $record['location']['locality'],
           'identifiedBy' => empty($record['identification']['identifiedBy']) ? NULL : $record['identification']['identifiedBy'],
           'identificationVerificationStatus' => empty($record['identification']['identificationVerificationStatus']) ? NULL : $record['identification']['identificationVerificationStatus'],
@@ -204,7 +204,6 @@ public static function syncPage($serverId, array $server) {
 QRY;
         $db->query($sql);
       }
-      throw new exception('Stop');
     }
     variable::set("rest_api_sync_{$serverId}_next_page", $data['paging']['next']);
     rest_api_sync::log(
@@ -212,9 +211,10 @@ public static function syncPage($serverId, array $server) {
       "<strong>Observations</strong><br/>Inserts: $tracker[inserts]. Updates: $tracker[updates]. Errors: $tracker[errors]"
     );
     $r = [
-      'moreToDo' => count($data['data']) === PAGE_SIZE,
-      'pagesToGo' => ceil($data['total_results'] / PAGE_SIZE),
-      'recordsToGo' => $data['total_results'],
+      'moreToDo' => count($data['data']) > 0,
+      // No way of determining the following.
+      'pagesToGo' => NULL,
+      'recordsToGo' => NULL,
     ];
     return $r;
   }
diff --git a/modules/rest_api_sync/views/rest_api_sync_skipped_record/index.js b/modules/rest_api_sync/views/rest_api_sync_skipped_record/index.js
index 95327501a3..0a6772a14d 100644
--- a/modules/rest_api_sync/views/rest_api_sync_skipped_record/index.js
+++ b/modules/rest_api_sync/views/rest_api_sync_skipped_record/index.js
@@ -24,9 +24,11 @@ $(document).ready(function docReady() {
             data: startInfo
           });
         } else {
-          chunkPercentage = 100 / startInfo.servers.length;
-          progress = ((response.serverIdx - 1) + ((response.page - 1) / pageCount)) * chunkPercentage;
-          $('#progress').progressbar('value', progress);
+          if (response.pagesToGo) {
+            chunkPercentage = 100 / startInfo.servers.length;
+            progress = ((response.serverIdx - 1) + ((response.page - 1) / pageCount)) * chunkPercentage;
+            $('#progress').progressbar('value', progress);
+          }
           doRequest({
             serverIdx: response.serverIdx,
             page: response.page

From 9c22d2277418bc31e938ba0c464c5ec9a556b9a7 Mon Sep 17 00:00:00 2001
From: Jim Bacon <jame2@ceh.ac.uk>
Date: Fri, 6 Aug 2021 17:50:38 +0100
Subject: [PATCH 18/93] Linting

---
 .../tests/services/data_cleanerTest.php       | 93 +++++++++++--------
 1 file changed, 54 insertions(+), 39 deletions(-)

diff --git a/modules/data_cleaner/tests/services/data_cleanerTest.php b/modules/data_cleaner/tests/services/data_cleanerTest.php
index 3d684c6484..8a5db2244b 100644
--- a/modules/data_cleaner/tests/services/data_cleanerTest.php
+++ b/modules/data_cleaner/tests/services/data_cleanerTest.php
@@ -43,30 +43,29 @@ class Controllers_Services_Data_Cleaner_Test extends Indicia_DatabaseTestCase {
   /**
    * @return PHPUnit_Extensions_Database_DataSet_IDataSet
    */
-  public function getDataSet()
-  {
-    $ds1 =  new DbUDataSetYamlDataSet('modules/phpUnit/config/core_fixture.yaml');
+  public function getDataSet() {
+    $ds1 = new DbUDataSetYamlDataSet('modules/phpUnit/config/core_fixture.yaml');
 
-    // Create a rule to test against
+    // Create a rule to test against.
     $ds2 = new Indicia_ArrayDataSet(
-      array(
-        'verification_rules' => array(
-          array(
+      [
+        'verification_rules' => [
+          [
             'title' => 'Test PeriodWithinYear rule',
             'description' => 'Test rule for unit testing',
             'test_type' => 'PeriodWithinYear',
             'error_message' => 'PeriodWithinYear test failed',
-            'source_url' => null,
-            'source_filename' => null,
+            'source_url' => NULL,
+            'source_filename' => NULL,
             'created_on' => '2016-07-22:16:00:00',
             'created_by_id' => 1,
             'updated_on' => '2016-07-22:16:00:00',
             'updated_by_id' => 1,
             'reverse_rule' => 'F',
-          ),
-        ),
-        'verification_rule_metadata' => array(
-          array(
+          ],
+        ],
+        'verification_rule_metadata' => [
+          [
             'verification_rule_id' => '1',
             'key' => 'Tvk',
             'value' => 'TESTKEY',
@@ -74,8 +73,8 @@ public function getDataSet()
             'created_by_id' => 1,
             'updated_on' => '2016-07-22:16:00:00',
             'updated_by_id' => 1,
-          ),
-          array(
+          ],
+          [
             'verification_rule_id' => '1',
             'key' => 'StartDate',
             'value' => '0801',
@@ -83,8 +82,8 @@ public function getDataSet()
             'created_by_id' => 1,
             'updated_on' => '2016-07-22:16:00:00',
             'updated_by_id' => 1,
-          ),
-          array(
+          ],
+          [
             'verification_rule_id' => '1',
             'key' => 'EndDate',
             'value' => '0831',
@@ -92,10 +91,10 @@ public function getDataSet()
             'created_by_id' => 1,
             'updated_on' => '2016-07-22:16:00:00',
             'updated_by_id' => 1,
-          ),
-        ),
-        'cache_verification_rules_period_within_year' => array(
-          array(
+          ],
+        ],
+        'cache_verification_rules_period_within_year' => [
+          [
             'verification_rule_id' => '1',
             'reverse_rule' => 'f',
             'taxa_taxon_list_external_key' => 'TESTKEY',
@@ -104,9 +103,9 @@ public function getDataSet()
             'survey_id' => NULL,
             'stages' => NULL,
             'error_message' => 'PeriodWithinYear test failed',
-          ),
-        ),
-      )
+          ],
+        ],
+      ]
     );
 
     $compositeDs = new DbUDataSetCompositeDataSet();
@@ -123,7 +122,7 @@ public function setUp(): void {
     $token = $auth['auth_token'];
     $nonce = $auth['nonce'];
     $this->request = data_entry_helper::$base_url .
-            "index.php/services/data_cleaner/verify?auth_token=$token&nonce=$nonce";
+      "index.php/services/data_cleaner/verify?auth_token=$token&nonce=$nonce";
 
     $cache = Cache::instance();
     $cache->delete('data-cleaner-rules');
@@ -134,7 +133,7 @@ public function setUp(): void {
    * incomplete or wrong works.
    */
   public function testIncorrectParams() {
-    $response = data_entry_helper::http_post($this->request, null);
+    $response = data_entry_helper::http_post($this->request, NULL);
     $this->assertEquals($response['output'], 'Invalid parameters');
   }
 
@@ -144,29 +143,45 @@ public function testIncorrectParams() {
    * data_cleaner_period_within_year module must be enabled.
    */
   public function testPeriodWithinYearFail() {
-    $response = data_entry_helper::http_post($this->request, array(
-      'sample' => json_encode(array(
+    $response = data_entry_helper::http_post($this->request, [
+      'sample' => json_encode([
         'sample:survey_id' => 1,
         'sample:date' => '12/09/2012',
         'sample:entered_sref' => 'SU1234',
         'sample:entered_sref_system' => 'osgb',
-      )),
-      'occurrences' => json_encode(array(
-        array(
+      ]),
+      'occurrences' => json_encode([
+        [
           'occurrence:taxa_taxon_list_id' => 1,
-        ),
-      )),
-      'rule_types' => json_encode(array('PeriodWithinYear')),
-    ));
+        ],
+      ]),
+      'rule_types' => json_encode(['PeriodWithinYear']),
+    ]);
     $errors = json_decode($response['output'], TRUE);
 
     $this->assertTrue($response['result'], 'Invalid response');
     $this->assertIsArray($errors, 'Errors list not returned');
-    $this->assertEquals(1, count($errors), 'Errors list empty. Is the data_cleaner_period_within_year module installed?');
-    $this->assertArrayHasKey('taxa_taxon_list_id', $errors[0], 'Errors list missing taxa_taxon_list_id');
-    $this->assertEquals('1', $errors[0]['taxa_taxon_list_id'], 'Incorrect taxa_taxon_list_id returned');
+    $this->assertEquals(
+      1,
+      count($errors),
+      'Errors list empty. Is the data_cleaner_period_within_year module installed?'
+    );
+    $this->assertArrayHasKey(
+      'taxa_taxon_list_id',
+      $errors[0],
+      'Errors list missing taxa_taxon_list_id'
+    );
+    $this->assertEquals(
+      '1',
+      $errors[0]['taxa_taxon_list_id'],
+      'Incorrect taxa_taxon_list_id returned'
+    );
     $this->assertArrayHasKey('message', $errors[0], 'Errors list missing message');
-    $this->assertEquals('PeriodWithinYear test failed', $errors[0]['message'], 'Incorrect message returned');
+    $this->assertEquals(
+      'PeriodWithinYear test failed',
+      $errors[0]['message'],
+      'Incorrect message returned'
+    );
   }
 
   /**

From 44c01d96c8e13ac3a364b82641b966c4b45b4db0 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Tue, 10 Aug 2021 12:22:07 +0100
Subject: [PATCH 19/93] Linting

---
 application/controllers/service_base.php | 12 ++++---
 modules/sref_osie/helpers/osie.php       | 41 ++++++++++++------------
 2 files changed, 27 insertions(+), 26 deletions(-)

diff --git a/application/controllers/service_base.php b/application/controllers/service_base.php
index fbec7cab5d..c80b7feb2a 100644
--- a/application/controllers/service_base.php
+++ b/application/controllers/service_base.php
@@ -125,10 +125,10 @@ protected function authenticate($mode = 'write') {
     $authentic = FALSE; // default
     if (array_key_exists('nonce', $array) && array_key_exists('auth_token', $array)) {
       $nonce = $array['nonce'];
-      $this->cache = new Cache;
+      $this->cache = new Cache();
       // Get all cache entries that match this nonce
       $paths = $this->cache->exists($nonce);
-      foreach($paths as $path) {
+      foreach ($paths as $path) {
         // Find the parts of each file name, which is the cache entry ID, then the mode.
         $tokens = explode('~', basename($path));
         // check this cached nonce is for the correct read or write operation.
@@ -139,14 +139,16 @@ protected function authenticate($mode = 'write') {
             $website = ORM::factory('website', $id);
             if ($website->id)
               $password = $website->password;
-          } else
+          }
+          else {
             $password = kohana::config('indicia.private_key');
+          }
           // calculate the auth token from the nonce and the password. Does it match the request's auth token?
           if (isset($password) && sha1("$nonce:$password")==$array['auth_token']) {
             Kohana::log('info', "Authentication successful.");
             // cache website_password for subsequent use by controllers
             $this->website_password = $password;
-            $authentic=true;
+            $authentic = TRUE;
           }
           if ($authentic) {
             if ($id > 0) {
@@ -166,7 +168,7 @@ protected function authenticate($mode = 'write') {
               $user = ORM::Factory('user', $this->user_id);
               $this->user_is_core_admin = ($user->core_role_id === 1);
               if (!$this->user_is_core_admin) {
-                $this->user_websites = array();
+                $this->user_websites = [];
                 $userWebsites = ORM::Factory('users_website')->where(array(
                   'user_id' => $this->user_id,
                   'site_role_id is not' => NULL,
diff --git a/modules/sref_osie/helpers/osie.php b/modules/sref_osie/helpers/osie.php
index 106af71adf..98d399733c 100644
--- a/modules/sref_osie/helpers/osie.php
+++ b/modules/sref_osie/helpers/osie.php
@@ -13,39 +13,36 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see http://www.gnu.org/licenses/gpl.html.
  *
- * @package Modules
- * @subpackage OSGB Grid References
- * @author  Indicia Team
+ * @author Indicia Team
  * @license http://www.gnu.org/licenses/gpl.html GPL 3.0
- * @link  https://github.com/indicia-team/warehouse/
+ * @link https://github.com/indicia-team/warehouse/
  */
 
 /**
  * Conversion class for OS Ireland grid references (TM75).
- * @package Modules
- * @subpackage OSGB Grid References
- * @author  Indicia Team
  */
 class osie {
 
   /**
    * Returns true if the spatial reference is a recognised Irish Grid square.
    *
-   * @param $sref string Spatial reference to validate
+   * @param $sref string
+   *   Spatial reference to validate
    */
-  public static function is_valid($sref)
-  {
-    // ignore any spaces in the grid ref
-    $sref = str_replace(' ','',$sref);
+  public static function is_valid($sref) {
+    // Ignore any spaces in the grid ref.
+    $sref = str_replace(' ', '', $sref);
     $sq100 = strtoupper(substr($sref, 0, 1));
-    if (!preg_match('([A-HJ-Z])', $sq100))
+    if (!preg_match('([A-HJ-Z])', $sq100)) {
       return FALSE;
-    $eastnorth=substr($sref, 1);
+    }
+    $eastnorth = substr($sref, 1);
     // 2 cases - either remaining chars must be all numeric and an equal number, up to 10 digits
     // OR for DINTY Tetrads, 2 numbers followed by a letter (Excluding O, including I)
-    if ((!preg_match('/^[0-9]*$/', $eastnorth) || strlen($eastnorth) % 2 != 0 || strlen($eastnorth)>10) AND
-                    (!preg_match('/^[0-9][0-9][A-NP-Z]$/', $eastnorth)))
+    if ((!preg_match('/^[0-9]*$/', $eastnorth) || strlen($eastnorth) % 2 != 0 || strlen($eastnorth) > 10) &&
+                    (!preg_match('/^[0-9][0-9][A-NP-Z]$/', $eastnorth))) {
       return FALSE;
+    }
     return TRUE;
   }
 
@@ -53,13 +50,15 @@ public static function is_valid($sref)
    * Converts a grid reference in OSI notation into the WKT text for the polygon, in
    * easting and northings from the zero reference.
    *
-   * @param string $sref The grid reference
-   * @return string String containing the well known text.
+   * @param string $sref
+   *   The grid reference.
+   *
+   * @return string
+   *   String containing the well known text.
    */
-  public static function sref_to_wkt($sref)
-  {
+  public static function sref_to_wkt($sref) {
     // ignore any spaces in the grid ref
-    $sref = str_replace(' ','',$sref);
+    $sref = str_replace(' ', '', $sref);
     if (!self::is_valid($sref))
       throw new InvalidArgumentException('Spatial reference is not a recognisable grid square.', 4001);
     $sq_100 = self::get_100k_square($sref);

From 693541d2de777a3541ace945c33bb8fad8549a80 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Tue, 10 Aug 2021 12:22:46 +0100
Subject: [PATCH 20/93] Finalisation of observations JSON sync format

---
 .../helpers/rest_api_sync_json_occurrences.php           | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php b/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php
index 6827440bf3..ed5e8d27c3 100644
--- a/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php
+++ b/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php
@@ -133,11 +133,12 @@ public static function syncPage($serverId, array $server) {
         ];
         if (!empty($record['location']['gridReference'])) {
           $observation['gridReference'] = strtoupper(str_replace(' ', '', $record['location']['gridReference']));
-          if (preg_match('/[A-Z][A-Z]\d*/', $observation['gridReference'])) {
-            $observation['projection'] = 'OSGB';
-          }
-          elseif (preg_match('/[A-Z]\d*/', $observation['gridReference'])) {
+          if (preg_match('/I?[A-Z]\d*/', $observation['gridReference'])) {
             $observation['projection'] = 'OSIE';
+            $observation['gridReference'] = preg_replace('/^I/', '', $observation['gridReference']);
+          }
+          elseif (preg_match('/[A-Z][A-Z]\d*/', $observation['gridReference'])) {
+            $observation['projection'] = 'OSGB';
           }
           else {
             throw new exception('Invalid grid reference format: ' . $record['location']['gridReference']);

From 1a3b3686994c1c5704de9f24d194917b0efd9b71 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Tue, 10 Aug 2021 12:23:03 +0100
Subject: [PATCH 21/93] Stubs for annotations sync format

---
 .../rest_api_sync/helpers/rest_api_sync_rest.php |  3 +++
 modules/rest_api_sync/plugins/rest_api_sync.php  | 16 +++++++++++++++-
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/modules/rest_api_sync/helpers/rest_api_sync_rest.php b/modules/rest_api_sync/helpers/rest_api_sync_rest.php
index ed5d3d3f69..7dd3e0c416 100644
--- a/modules/rest_api_sync/helpers/rest_api_sync_rest.php
+++ b/modules/rest_api_sync/helpers/rest_api_sync_rest.php
@@ -182,6 +182,9 @@ public static function syncTaxonObservationsGet($foo, $clientConfig, $projectId)
     echo "\n],\"paging\":{\"next\":{\"tracking_from\":$nextFrom}}}";
   }
 
+  public function syncAnnotationsGet($foo, $clientConfig, $projectId) {
+  }
+
   /**
    * Returns male or female for sex term.
    *
diff --git a/modules/rest_api_sync/plugins/rest_api_sync.php b/modules/rest_api_sync/plugins/rest_api_sync.php
index ea3085188f..ddda5631b8 100644
--- a/modules/rest_api_sync/plugins/rest_api_sync.php
+++ b/modules/rest_api_sync/plugins/rest_api_sync.php
@@ -47,7 +47,21 @@ function rest_api_sync_extend_rest_api() {
             'proj_id' => [
               'datatype' => 'text',
             ],
-            'tracking' => [
+            'tracking_from' => [
+              'datatype' => 'integer',
+            ],
+          ],
+        ],
+      ],
+    ],
+    'sync-annotations' => [
+      'GET' => [
+        'sync-annotations' => [
+          'params' => [
+            'proj_id' => [
+              'datatype' => 'text',
+            ],
+            /*************/'tracking_from' => [
               'datatype' => 'integer',
             ],
           ],

From ad5ef6f127332982080981e85943d15da36fa4d7 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Thu, 12 Aug 2021 10:37:12 +0100
Subject: [PATCH 22/93] Remove debug code

---
 modules/rest_api_sync/helpers/api_persist.php | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/modules/rest_api_sync/helpers/api_persist.php b/modules/rest_api_sync/helpers/api_persist.php
index 3e5dd9e6ec..e1a1094066 100644
--- a/modules/rest_api_sync/helpers/api_persist.php
+++ b/modules/rest_api_sync/helpers/api_persist.php
@@ -58,9 +58,6 @@ public static function initDwcAttributes($db, $surveyId) {
       'occAttrs' => self::fetchDwcAttrs($db, 'occurrence', $surveyId),
       'smpAttrs' => self::fetchDwcAttrs($db, 'sample', $surveyId),
     ];
-    echo '<pre>';
-    var_export(self::$dwcAttributes);
-    echo '</pre>';
   }
 
   private static function fetchDwcAttrs($db, $type, $surveyId) {

From b7e2e100fb0a44fb6c38d1b580e61d7d846360c1 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Thu, 12 Aug 2021 10:38:54 +0100
Subject: [PATCH 23/93] Allow links to servers that don't give a total
 records/pages count

I.e. degrade the progress info gracefully.
---
 .../controllers/rest_api_sync.php             |  3 ++-
 .../rest_api_sync_skipped_record/index.js     | 22 +++++++++++++------
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/modules/rest_api_sync/controllers/rest_api_sync.php b/modules/rest_api_sync/controllers/rest_api_sync.php
index 9fe2486235..9d5da9d75e 100644
--- a/modules/rest_api_sync/controllers/rest_api_sync.php
+++ b/modules/rest_api_sync/controllers/rest_api_sync.php
@@ -81,7 +81,7 @@ public function process_batch() {
     $page = empty($_GET['page']) ? 1 : $_GET['page'];
     $serverId = array_keys($servers)[$serverIdx - 1];
     $server = array_merge([
-      'serverType' => 'Indicia', 
+      'serverType' => 'Indicia',
       'allowUpdateWhenVerified' => TRUE,
     ], $servers[$serverId]);
     $helperClass = 'rest_api_sync_' . strtolower($server['serverType']);
@@ -112,6 +112,7 @@ public function process_batch() {
         'log' => rest_api_sync::$log,
         'pagesToGo' => $progressInfo['pagesToGo'],
         'recordsToGo' => $progressInfo['recordsToGo'],
+        'moreToDo' => $progressInfo['moreToDo'],
       ];
       echo json_encode($r);
     }
diff --git a/modules/rest_api_sync/views/rest_api_sync_skipped_record/index.js b/modules/rest_api_sync/views/rest_api_sync_skipped_record/index.js
index 0a6772a14d..92924293e0 100644
--- a/modules/rest_api_sync/views/rest_api_sync_skipped_record/index.js
+++ b/modules/rest_api_sync/views/rest_api_sync_skipped_record/index.js
@@ -1,9 +1,9 @@
 $(document).ready(function docReady() {
   var startInfo;
   var pageCount;
+  var pagesDone = 0;
 
   $('#sync-progress').hide();
-  $('#progress').progressbar();
   function doRequest(data) {
     $.ajax({
       url: 'rest_api_sync/process_batch',
@@ -13,21 +13,29 @@ $(document).ready(function docReady() {
         var chunkPercentage;
         var progress;
         // Get pageCount from pagesToGo on first iteration.
-        pageCount = typeof pageCount === 'undefined' ? response.pagesToGo : pageCount;
+        pageCount = pageCount ?? response.pagesToGo ?? null;
+
         $('#output .panel-body').append('<div>' + response.log.join('<br/>') + '</div>');
         if (response.state === 'done') {
           $('#output .panel-body').append('<div class="alert alert-success">Synchronisation complete</div>');
-          $('#progress').hide();
+          $('#progress, #progress-info').hide();
           $.ajax({
             url: 'rest_api_sync/end',
             dataType: 'json',
             data: startInfo
           });
         } else {
-          if (response.pagesToGo) {
-            chunkPercentage = 100 / startInfo.servers.length;
-            progress = ((response.serverIdx - 1) + ((response.page - 1) / pageCount)) * chunkPercentage;
-            $('#progress').progressbar('value', progress);
+          pagesDone++;
+          if (response.moreToDo) {
+            if (pageCount) {
+              chunkPercentage = 100 / startInfo.servers.length;
+              progress = ((response.serverIdx - 1) + ((response.page - 1) / pageCount)) * chunkPercentage;
+              $('#progress').val(progress);
+            }
+            else {
+              // Alternative if true progress info missing.
+              $('#progress-info').removeClass('hide').text('Batches done so far: ' + pagesDone);
+            }
           }
           doRequest({
             serverIdx: response.serverIdx,

From 3a24e56792469dbc72b00200be5bb89bfb0c978a Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Thu, 12 Aug 2021 10:40:35 +0100
Subject: [PATCH 24/93] Grid ref format detection fixes

---
 .../helpers/rest_api_sync_json_occurrences.php    | 15 ++++-----------
 1 file changed, 4 insertions(+), 11 deletions(-)

diff --git a/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php b/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php
index ed5e8d27c3..b76c5baf30 100644
--- a/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php
+++ b/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php
@@ -43,8 +43,6 @@ class rest_api_sync_json_occurrences {
    *   Server configuration.
    */
   public static function syncServer($serverId, array $server) {
-    kohana::log('debug', 'In syncServer');
-    kohana::log_save();
     // Count of pages done in this run.
     $pageCount = 0;
     // If last run still going, not on first page.
@@ -92,12 +90,8 @@ public static function syncPage($serverId, array $server) {
     $taxon_list_id = Kohana::config('rest_api_sync.taxon_list_id');
     $tracker = ['inserts' => 0, 'updates' => 0, 'errors' => 0];
     foreach ($data['data'] as $record) {
-      echo '<pre>' . var_export($record, TRUE) . '</pre>';
-      // @todo Handle grid ref if provided
-      // @todo taxonID is ORGANISM_KEY
-      // @todo Ensure non-exact dates are handled.
       // @todo Make sure all fields in specification are handled
-      // @todo Handle moreToDo code.
+      // @todo dynamicProperties field.
       // @todo occurrence.associated_media
       // @todo occurrence.occurrence_status
       // @todo occurrence.organism_quantity
@@ -133,11 +127,11 @@ public static function syncPage($serverId, array $server) {
         ];
         if (!empty($record['location']['gridReference'])) {
           $observation['gridReference'] = strtoupper(str_replace(' ', '', $record['location']['gridReference']));
-          if (preg_match('/I?[A-Z]\d*/', $observation['gridReference'])) {
-            $observation['projection'] = 'OSIE';
+          if (preg_match('/^I?[A-Z]\d*[A-NP-Z]?$/', $observation['gridReference'])) {
+            $observation['projection'] = 'OSI';
             $observation['gridReference'] = preg_replace('/^I/', '', $observation['gridReference']);
           }
-          elseif (preg_match('/[A-Z][A-Z]\d*/', $observation['gridReference'])) {
+          elseif (preg_match('/^[A-Z][A-Z]\d*[A-NP-Z]?$/', $observation['gridReference'])) {
             $observation['projection'] = 'OSGB';
           }
           else {
@@ -159,7 +153,6 @@ public static function syncPage($serverId, array $server) {
             }
           }
         }
-        echo '<pre>' . var_export($observation, TRUE) . '</pre><br/>';
 
         $is_new = api_persist::taxonObservation(
           $db,

From 78fdbb5d09f089bfb989a95864d808a74a315420 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Thu, 12 Aug 2021 10:41:11 +0100
Subject: [PATCH 25/93] Updates to config file examples

---
 modules/rest_api/config/rest.example.php               | 8 ++++++++
 modules/rest_api_sync/config/rest_api_sync.example.php | 2 +-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/modules/rest_api/config/rest.example.php b/modules/rest_api/config/rest.example.php
index d6030c86bc..8e7dbe10be 100644
--- a/modules/rest_api/config/rest.example.php
+++ b/modules/rest_api/config/rest.example.php
@@ -166,12 +166,20 @@
         'id_prefix' => 'iBRC',
         'dataset_id_attr_id' => 22,
         'blur' => 'F',
+        // Define an Elasticsearch query for the observations available to this
+        // project.
         'es_bool_query' => [
           'must' => [
             ['term' => ['taxon.class.keyword' => 'Aves']],
             ['term' => ['metadata.website.id' => 2]],
           ],
         ],
+        // Define a filter for the annotations data. This should match the
+        // location that the other server's observations are synced to using
+        // the rest_api_sync module.
+        'annotations_filter' => [
+          'survey_id' => 10,
+        ],
       ],
     ],
     'elasticsearch' => ['es'],
diff --git a/modules/rest_api_sync/config/rest_api_sync.example.php b/modules/rest_api_sync/config/rest_api_sync.example.php
index c8bd3ea45b..59d39f49ba 100644
--- a/modules/rest_api_sync/config/rest_api_sync.example.php
+++ b/modules/rest_api_sync/config/rest_api_sync.example.php
@@ -47,7 +47,7 @@
     'website_id' => 5,
     // Remote API URL.
     'url' => 'http://localhost/indicia/index.php/services/rest',
-    // Remote platform name, iNaturalist or Indicia.
+    // Remote platform name, iNaturalist, Indicia (=REST API), json_occurrences (=JSON sync format).
     'serverType' => 'Indicia',
     // Secret shared with the remote API.
     'shared_secret' => '123password',

From dac5462f727fb1fc9d43c006b03060a0e0cedee1 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Thu, 12 Aug 2021 10:41:38 +0100
Subject: [PATCH 26/93] Change progress bar to HTML5

---
 .../views/rest_api_sync_skipped_record/index.css              | 3 +++
 .../views/rest_api_sync_skipped_record/index.php              | 4 ++--
 2 files changed, 5 insertions(+), 2 deletions(-)
 create mode 100644 modules/rest_api_sync/views/rest_api_sync_skipped_record/index.css

diff --git a/modules/rest_api_sync/views/rest_api_sync_skipped_record/index.css b/modules/rest_api_sync/views/rest_api_sync_skipped_record/index.css
new file mode 100644
index 0000000000..51e8d28f3a
--- /dev/null
+++ b/modules/rest_api_sync/views/rest_api_sync_skipped_record/index.css
@@ -0,0 +1,3 @@
+progress {
+  width: 100%;
+}
\ No newline at end of file
diff --git a/modules/rest_api_sync/views/rest_api_sync_skipped_record/index.php b/modules/rest_api_sync/views/rest_api_sync_skipped_record/index.php
index 1fa8d6611a..6d65c42e40 100644
--- a/modules/rest_api_sync/views/rest_api_sync_skipped_record/index.php
+++ b/modules/rest_api_sync/views/rest_api_sync_skipped_record/index.php
@@ -34,8 +34,8 @@
   <button type="button" class="btn btn-primary" id="start-sync">Start sync</button>
   <div id="sync-progress">
     <h2>Synchronisation progress</h2>
-    <div id="progress"></div><br/>
-    <?php echo $grid; ?>
+    <progress id="progress" class="progress" max="100" title="Synchronisation progress"></progress>
+    <div id="progress-info" class="alert alert-info hide"></div>
     <div id="output" class="panel panel-default">
       <div class="panel-heading">Synchronisation messages</div>
       <div class="panel-body"></div>

From 9f20bede231a8276fa11861fe0e4ad3f09c071a2 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Thu, 12 Aug 2021 11:08:53 +0100
Subject: [PATCH 27/93] Annotations endpoint

---
 .../helpers/rest_api_sync_rest.php            | 72 +++++++++++++++++--
 .../rest_api_sync/filterable_annotations.xml  | 60 ++++++++++++++++
 2 files changed, 126 insertions(+), 6 deletions(-)
 create mode 100644 modules/rest_api_sync/reports/rest_api_sync/filterable_annotations.xml

diff --git a/modules/rest_api_sync/helpers/rest_api_sync_rest.php b/modules/rest_api_sync/helpers/rest_api_sync_rest.php
index 7dd3e0c416..3c7a4b7998 100644
--- a/modules/rest_api_sync/helpers/rest_api_sync_rest.php
+++ b/modules/rest_api_sync/helpers/rest_api_sync_rest.php
@@ -22,7 +22,7 @@
  * @link https://github.com/indicia-team/warehouse/
  */
 
- defined('SYSPATH') or die('No direct script access.');
+defined('SYSPATH') or die('No direct script access.');
 
 /**
  * Helper class for extending the REST API with sync endpoints.
@@ -67,13 +67,26 @@ class rest_api_sync_rest {
     'R5' => 'Not accepted - incorrect',
   ];
 
-  public static function syncTaxonObservationsGet($foo, $clientConfig, $projectId) {
+  /**
+   * Endpoint for sync-taxon-observations.
+   *
+   * Echoes a list of observations in JSON occurrences format.
+   *
+   * @param mixed $foo
+   *   Unused as this endpoint doesn't support retrieving a record by ID.
+   * @param array $clientConfig
+   *   Configuration for this client from the REST API.
+   * @param string $projectId
+   *   Code for the project being retrieved from.
+   */
+  public static function syncTaxonObservationsGet($foo, array $clientConfig, $projectId) {
     if (empty($clientConfig['elasticsearch']) || count($clientConfig['elasticsearch']) !== 1) {
       RestObjects::$apiResponse->fail('Internal Server Error', 500, 'Incorrect elasticsearch configuration for client.');
     }
     $project = ($clientConfig && $projectId) ? $clientConfig['projects'][$projectId] : [];
     $response = self::getEsTaxonObservationsResponse($clientConfig, $project);
     $total = count($response->hits->hits);
+    $nextFrom = NULL;
     echo "{\"data\":[\n";
     foreach ($response->hits->hits as $idx => $hit) {
       $doc = $hit->_source;
@@ -93,7 +106,8 @@ public static function syncTaxonObservationsGet($foo, $clientConfig, $projectId)
       if (!empty($doc->location->coordinate_uncertainty_in_meters)) {
         $obj['location']['coordinateUncertaintyInMeters'] = $doc->location->coordinate_uncertainty_in_meters;
       }
-      if (isset($doc->location->input_sref_system) && in_array($doc->location->input_sref_system, ['OSGB', 'OSI'])) {
+      if (isset($doc->location->input_sref_system)
+          && in_array($doc->location->input_sref_system, ['OSGB', 'OSI'])) {
         $obj['location']['gridReference'] = $doc->location->input_sref;
       }
       else {
@@ -179,10 +193,56 @@ public static function syncTaxonObservationsGet($foo, $clientConfig, $projectId)
         $nextFrom = $doc->metadata->tracking;
       }
     }
-    echo "\n],\"paging\":{\"next\":{\"tracking_from\":$nextFrom}}}";
+    echo "\n]";
+    if ($nextFrom) {
+      echo ",\"paging\":{\"next\":{\"tracking_from\":$nextFrom}}";
+    }
+    echo "}";
   }
 
-  public function syncAnnotationsGet($foo, $clientConfig, $projectId) {
+  /**
+   * Endpoint for sync-annotations.
+   *
+   * Echoes a list of annnotations in JSON occurrences format.
+   *
+   * @param mixed $foo
+   *   Unused as this endpoint doesn't support retrieving a record by ID.
+   * @param array $clientConfig
+   *   Configuration for this client from the REST API.
+   * @param string $projectId
+   *   Code for the project being retrieved from.
+   */
+  public static function syncAnnotationsGet($foo, array $clientConfig, $projectId) {
+    $projectConfig = $clientConfig['projects'][$projectId];
+    $reportEngine = new ReportEngine([$projectConfig['website_id']]);
+    $params = [
+      'limit' => REST_API_DEFAULT_PAGE_SIZE,
+      'system_user_id' => Kohana::config('rest.user_id'),
+    ];
+    foreach ($projectConfig['annotations_filter'] as $key => $value) {
+      $params["context_$key"] = $value;
+    }
+    if (isset($_GET['dateTime_from'])) {
+      $params['dateTime_from'] = $_GET['dateTime_from'];
+    }
+    $output = $reportEngine->requestReport("rest_api_sync/filterable_annotations.xml", 'local', 'xml', $params, FALSE);
+    $dateTimeFrom = NULL;
+    echo "{\"data\":[\n";
+    $total = count($output['content']['records']);
+    foreach ($output['content']['records'] as $idx => $record) {
+      echo json_encode($record);
+      if ($idx < $total - 1) {
+        echo ',';
+      }
+      else {
+        $dateTimeFrom = $record->dateTime;
+      }
+    }
+    echo "\n]";
+    if ($dateTimeFrom) {
+      echo ",\"paging\":{\"next\":{\"dateTime_from\":\"$dateTimeFrom\"}}";
+    }
+    echo "}";
   }
 
   /**
@@ -206,7 +266,7 @@ private static function sexTerm($term) {
   private static function getEsTaxonObservationsResponse($clientConfig, $project) {
     $es = new RestApiElasticsearch($clientConfig['elasticsearch'][0]);
     $format = 'json';
-    if (isset($_GET['tracking_from']) ) {
+    if (isset($_GET['tracking_from'])) {
       if (!preg_match('/^\d+$/', $_GET['tracking_from'])) {
         RestObjects::$apiResponse->fail('Bad Request', 400, 'Invalid tracking from parameter');
       }
diff --git a/modules/rest_api_sync/reports/rest_api_sync/filterable_annotations.xml b/modules/rest_api_sync/reports/rest_api_sync/filterable_annotations.xml
new file mode 100644
index 0000000000..4f3ad45086
--- /dev/null
+++ b/modules/rest_api_sync/reports/rest_api_sync/filterable_annotations.xml
@@ -0,0 +1,60 @@
+<report
+    title="Annotations for REST API"
+    description="Retrieves record annotations to support the REST api module. Compatible with the standard report filter parameters."
+    >
+  <query website_filter_field="o.website_id" samples_id_field="o.sample_id" standard_params="occurrences" count_field="oc.*">
+    select #columns#
+    from occurrence_comments oc
+    join cache_occurrences_functional o on o.id=oc.occurrence_id
+    join occurrences occ on occ.id=o.id
+    join users u on u.id=oc.created_by_id and u.deleted=false
+    join people p on p.id=u.person_id and p.deleted=false
+    #agreements_join#
+    #joins#
+    where #sharing_filter#
+    and oc.deleted=false
+    and o.taxa_taxon_list_external_key is not null
+    #idlist#
+  </query>
+  <order_bys>
+    <order_by>oc.updated_on ASC</order_by>
+  </order_bys>
+  <params>
+    <param name='system_user_id' display='System ID' description='3 character code identifying the system' datatype='text'/>
+    <param name="dateTime_from" display="From dateTime" datatype="date">
+      <where>oc.updated_on&gt;'#dateTime_from#'</where>
+    </param>
+  </params>
+  <columns>
+    <column name="annotationID" datatype="text"
+            sql="COALESCE(oc.external_key, '#system_user_id#' || oc.id::varchar)" />
+    <column name="occurrenceID" datatype="integer"
+            sql="COALESCE(occ.external_key, '#system_user_id#' || o.id::varchar)" />
+    <column name="comment" datatype="text"
+            sql="oc.comment"/>
+    <column name="identificationVerificationStatus" datatype="text"
+            sql="case oc.record_status || coalesce(oc.record_substatus::text, '')
+            when 'V' then 'Accepted'
+            when 'V1' then 'Accepted - correct'
+            when 'V2' then 'Accepted - considered correct'
+            when 'C' then 'Unconfirmed - not reviewed'
+            when 'C3' then 'Unconfirmed - plausible'
+            when 'R' then 'Not accepted'
+            when 'R4' then 'Not accepted - unable to verify'
+            when 'R5' then 'Not accepted - incorrect'
+            else null
+        end" />
+    <column name="question" datatype="boolean"
+            sql="oc.query" />
+    <column name="answeredAnnotationID" datatype="text"
+            sql="case oc.query
+            when false then
+              (select id from occurrence_comments oc2 where oc2.occurrence_id=occ.id and oc2.deleted=false and oc2.query=true and oc2.id&lt;oc.id order by id desc limit 1)
+            else null
+        end" />
+    <column name="authorName" datatype="text"
+            sql="coalesce(oc.person_name, case u.id when 1 then 'anonymous' else p.first_name || ' ' || p.surname end)"/>
+    <column name="dateTime"
+            sql="to_char(oc.updated_on, 'YYYY-MM-DD&quot;T&quot;HH24:MI:SS')" />
+  </columns>
+</report>
\ No newline at end of file

From 6081c779a6489bffb12f9a09fd39a08e2501f2b5 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Thu, 12 Aug 2021 12:30:34 +0100
Subject: [PATCH 28/93] Date from param should be optional for first run

---
 .../reports/rest_api_sync/filterable_annotations.xml            | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/rest_api_sync/reports/rest_api_sync/filterable_annotations.xml b/modules/rest_api_sync/reports/rest_api_sync/filterable_annotations.xml
index 4f3ad45086..e539331421 100644
--- a/modules/rest_api_sync/reports/rest_api_sync/filterable_annotations.xml
+++ b/modules/rest_api_sync/reports/rest_api_sync/filterable_annotations.xml
@@ -21,7 +21,7 @@
   </order_bys>
   <params>
     <param name='system_user_id' display='System ID' description='3 character code identifying the system' datatype='text'/>
-    <param name="dateTime_from" display="From dateTime" datatype="date">
+    <param name="dateTime_from" display="From dateTime" datatype="date" default="">
       <where>oc.updated_on&gt;'#dateTime_from#'</where>
     </param>
   </params>

From 1997affbd5a47349bd4e38361889f4327a20dbee Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Thu, 12 Aug 2021 12:38:20 +0100
Subject: [PATCH 29/93] Corrected context parameter name.

---
 modules/rest_api_sync/helpers/rest_api_sync_rest.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/rest_api_sync/helpers/rest_api_sync_rest.php b/modules/rest_api_sync/helpers/rest_api_sync_rest.php
index 3c7a4b7998..ee59f5cf10 100644
--- a/modules/rest_api_sync/helpers/rest_api_sync_rest.php
+++ b/modules/rest_api_sync/helpers/rest_api_sync_rest.php
@@ -220,7 +220,7 @@ public static function syncAnnotationsGet($foo, array $clientConfig, $projectId)
       'system_user_id' => Kohana::config('rest.user_id'),
     ];
     foreach ($projectConfig['annotations_filter'] as $key => $value) {
-      $params["context_$key"] = $value;
+      $params["{$key}_context"] = $value;
     }
     if (isset($_GET['dateTime_from'])) {
       $params['dateTime_from'] = $_GET['dateTime_from'];

From d1aabee3f961dd2fd2618a73a530c2319462299b Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Thu, 12 Aug 2021 13:35:13 +0100
Subject: [PATCH 30/93] Prefix already included in source data.

---
 .../rest_api_sync/helpers/rest_api_sync_json_occurrences.php    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php b/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php
index b76c5baf30..f56cfeb950 100644
--- a/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php
+++ b/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php
@@ -105,7 +105,7 @@ public static function syncPage($serverId, array $server) {
           'licenceCode' => empty($record['record-level']['license']) ? NULL : $record['record-level']['license'],
           'collectionCode' => empty($record['record-level']['collectionCode']) ? NULL : $record['record-level']['collectionCode'],
           'occurrenceMetadata' => empty($record['record-level']['dynamicProperties']) ? NULL : $record['record-level']['dynamicProperties'],
-          'id' => "$serverId:" . $record['occurrence']['occurrenceID'],
+          'id' => $record['occurrence']['occurrenceID'],
           'individualCount' => empty($record['occurrence']['individualCount']) ? NULL : $record['occurrence']['individualCount'],
           'lifeStage' => empty($record['occurrence']['lifeStage']) ? NULL : $record['occurrence']['lifeStage'],
           'recordedBy' => $record['occurrence']['recordedBy'],

From cbaa2ed698e21e04fceecf18f3ab33fb515979ce Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Thu, 12 Aug 2021 14:24:29 +0100
Subject: [PATCH 31/93] Prefer decimalLatLong as higher precision than
 gridReference

---
 .../helpers/rest_api_sync_json_occurrences.php       | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php b/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php
index f56cfeb950..20f5aa4ac7 100644
--- a/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php
+++ b/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php
@@ -125,7 +125,12 @@ public static function syncPage($serverId, array $server) {
           'identifiedBy' => empty($record['identification']['identifiedBy']) ? NULL : $record['identification']['identifiedBy'],
           'identificationVerificationStatus' => empty($record['identification']['identificationVerificationStatus']) ? NULL : $record['identification']['identificationVerificationStatus'],
         ];
-        if (!empty($record['location']['gridReference'])) {
+        if (!empty($record['location']['decimalLongitude']) && !empty($record['location']['decimalLatitude'])) {
+          $observation['east'] = $record['location']['decimalLongitude'];
+          $observation['north'] = $record['location']['decimalLatitude'];
+          $observation['projection'] = 'WGS84';
+        }
+        elseif (!empty($record['location']['gridReference'])) {
           $observation['gridReference'] = strtoupper(str_replace(' ', '', $record['location']['gridReference']));
           if (preg_match('/^I?[A-Z]\d*[A-NP-Z]?$/', $observation['gridReference'])) {
             $observation['projection'] = 'OSI';
@@ -138,11 +143,6 @@ public static function syncPage($serverId, array $server) {
             throw new exception('Invalid grid reference format: ' . $record['location']['gridReference']);
           }
         }
-        else {
-          $observation['east'] = $record['location']['decimalLongitude'];
-          $observation['north'] = $record['location']['decimalLatitude'];
-          $observation['projection'] = 'WGS84';
-        }
         if (!empty($server['otherFields'])) {
           foreach ($server['otherFields'] as $src => $dest) {
             $path = explode('.', $src);

From a43dcdd674b93ca8aa3e529ee2df1cf8b5d8ad69 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Mon, 16 Aug 2021 09:32:59 +0100
Subject: [PATCH 32/93] Linting

---
 modules/workflow/helpers/workflow.php | 93 ++++++++++++++++-----------
 modules/workflow/plugins/workflow.php | 33 +++++-----
 2 files changed, 71 insertions(+), 55 deletions(-)

diff --git a/modules/workflow/helpers/workflow.php b/modules/workflow/helpers/workflow.php
index 29949a4790..8172753c97 100644
--- a/modules/workflow/helpers/workflow.php
+++ b/modules/workflow/helpers/workflow.php
@@ -45,8 +45,9 @@ public static function getEntityConfig($entity) {
   /**
    * Applies undo data to rewind records to their originally posted state.
    *
-   * This occurs when a record has been modified by the workflow system because of a particular key value linking it to
-   * a workflow event record, then the key value is changed so the workflow event is no longer relevant.
+   * This occurs when a record has been modified by the workflow system because
+   * of a particular key value linking it to a workflow event record, then the
+   * key value is changed so the workflow event is no longer relevant.
    *
    * @param object $db
    *   Database connection.
@@ -71,16 +72,17 @@ public static function getRewoundRecord($db, $entity, $oldRecord, &$newRecord) {
     }
     $eventTypes = [];
     foreach ($entityConfig['keys'] as $keyDef) {
-      $keyChanged = false;
+      $keyChanged = FALSE;
       // We need to know if the key has changed to decide whether to wind back.
       // If the key is in the main entity, we can directly compare the old and new keys.
       if ($keyDef['table'] === $entity) {
         $keyCol = $keyDef['column'];
-        $keyChanged = (string) $oldRecord->$column !== (string) $newRecord->column;
+        $keyChanged = (string) $oldRecord->column !== (string) $newRecord->column;
       }
       else {
-        // Find the definintion of the extra data table that contains the column we need to look for changes in. We can
-        // then look to see if the foreign key pointing to that table has changed.
+        // Find the definintion of the extra data table that contains the
+        // column we need to look for changes in. We can then look to see if
+        // the foreign key pointing to that table has changed.
         foreach ($entityConfig['extraData'] as $extraDataDef) {
           if ($extraDataDef['table'] === $keyDef['table']) {
             $column = $extraDataDef['originating_table_column'];
@@ -94,7 +96,8 @@ public static function getRewoundRecord($db, $entity, $oldRecord, &$newRecord) {
     }
     if ($entity === 'occurrence'
         && $oldRecord->record_status !== $newRecord->record_status) {
-      // Remove previuos verification and rejection workflow changes as the record status is changing.
+      // Remove previuos verification and rejection workflow changes as the
+      // record status is changing.
       $eventTypes[] = 'V';
       $eventTypes[] = 'R';
     }
@@ -129,17 +132,18 @@ public static function getRewoundRecord($db, $entity, $oldRecord, &$newRecord) {
    *   List of event types to rewind ('S', 'V', 'R').
    *
    * @return array
-   *   Associatie array keyed by entity.entity_id, containing an array of the fields with undo values to apply.
+   *   Associate array keyed by entity.entity_id, containing an array of the
+   *   fields with undo values to apply.
    */
   public static function getRewindChangesForRecords($db, $entity, array $entityIdList, array $eventTypes) {
     $r = [];
     $undoRecords = $db
       ->select('DISTINCT workflow_undo.id, workflow_undo.entity_id, workflow_undo.original_values')
       ->from('workflow_undo')
-      ->where(array(
+      ->where([
         'workflow_undo.entity' => $entity,
         'workflow_undo.active' => 't',
-      ))
+      ])
       ->in('event_type', $eventTypes)
       ->in('entity_id', $entityIdList)
       ->orderby('workflow_undo.id', 'DESC')
@@ -154,7 +158,7 @@ public static function getRewindChangesForRecords($db, $entity, array $entityIdL
         $r["$entity.$undoRecord->entity_id"] = array_merge($r["$entity.$undoRecord->entity_id"], $unsetColumns);
       }
       // As this is a hard rewind, disable the undo data.
-      $db->update('workflow_undo', array('active' => 'f'), array('id' => $undoRecord->id));
+      $db->update('workflow_undo', ['active' => 'f'], ['id' => $undoRecord->id]);
     }
     return $r;
   }
@@ -164,7 +168,7 @@ public static function getRewindChangesForRecords($db, $entity, array $entityIdL
    *
    * @param object $db
    *   Database connection.
-   *  @param int $websiteId
+   * @param int $websiteId
    *   ID of the website the update is associated with.
    * @param string $entity
    *   Name of the database entity being saved, e.g. occurrence.
@@ -174,8 +178,8 @@ public static function getRewindChangesForRecords($db, $entity, array $entityIdL
    *   List of event types to include in the results.
    *
    * @return array
-   *   List of records with events attached (keyed by entity.id), with each entry containing an array of the events
-   *   associated with that record.
+   *   List of records with events attached (keyed by entity.id), with each
+   *   entry containing an array of the events associated with that record.
    */
   public static function getEventsForRecords($db, $websiteId, $entity, array $entityIdList, array $eventTypes) {
     $r = [];
@@ -195,10 +199,10 @@ public static function getEventsForRecords($db, $websiteId, $entity, array $enti
         ->select('workflow_events.key_value, workflow_events.event_type, workflow_events.mimic_rewind_first, ' .
           "workflow_events.values, $table.id as {$entity}_id")
         ->from('workflow_events')
-        ->where(array(
+        ->where([
           'workflow_events.deleted' => 'f',
           'key' => $keyDef['db_store_value'],
-        ))
+        ])
         ->in('group_code', $groupCodes)
         ->in('workflow_events.event_type', $eventTypes);
       if ($keyDef['table'] === $entity) {
@@ -207,7 +211,8 @@ public static function getEventsForRecords($db, $websiteId, $entity, array $enti
       }
       else {
         $qry->join($keyDef['table'], "$keyDef[table].$keyDef[column]", 'workflow_events.key_value');
-        // Cross reference to the extraData for the same table to find the field name which matches $newRecord->column.
+        // Cross reference to the extraData for the same table to find the
+        // field name which matches $newRecord->column.
         foreach ($entityConfig['extraData'] as $extraDataDef) {
           if ($extraDataDef['table'] === $keyDef['table']) {
             $qry->join(
@@ -274,8 +279,8 @@ public static function applyWorkflow($db, $websiteId, $entity, $oldRecord, $rewo
   /**
    * Construct a query to retrieve workflow events.
    *
-   * Constructs a query object which will find all the events applicable to the current record for a given key in the
-   * entity's configuration.
+   * Constructs a query object which will find all the events applicable to the
+   * current record for a given key in the entity's configuration.
    *
    * @param object $db
    *   Database connection.
@@ -293,16 +298,16 @@ public static function applyWorkflow($db, $websiteId, $entity, $oldRecord, $rewo
    * @return object
    *   Query object.
    */
-  private static function buildEventQueryForKey($db, $groupCodes, $entity, $oldRecord, $newRecord, array $keyDef) {
+  private static function buildEventQueryForKey($db, array $groupCodes, $entity, $oldRecord, $newRecord, array $keyDef) {
     $entityConfig = self::getEntityConfig($entity);
     $eventTypes = [];
     $qry = $db
       ->select('workflow_events.event_type, workflow_events.mimic_rewind_first, workflow_events.values')
       ->from('workflow_events')
-      ->where(array(
+      ->where([
         'workflow_events.deleted' => 'f',
         'key' => $keyDef['db_store_value'],
-      ))
+      ])
       ->in('group_code', $groupCodes);
     if ($keyDef['table'] === $entity) {
       $column = $keyDef['column'];
@@ -361,8 +366,10 @@ private static function buildEventQueryForKey($db, $groupCodes, $entity, $oldRec
    * Finds configured groups which the current operation's website uses the workflow for.
    *
    * @param int $websiteId
+   *   Website ID.
    *
    * @return array
+   *   List of group names.
    */
   private static function getGroupCodesForThisWebsite($websiteId) {
     $config = kohana::config('workflow_groups', FALSE, FALSE);
@@ -380,8 +387,9 @@ private static function getGroupCodesForThisWebsite($websiteId) {
   /**
    * Applies the events query results to a record.
    *
-   * Applies the field value changes determined by a query against the workflow_events table to the contents of a record
-   * that is about to be saved.
+   * Applies the field value changes determined by a query against the
+   * workflow_events table to the contents of a record that is about to be
+   * saved.
    *
    * @param object $qry
    *   Query object set up to retrieve the events to apply.
@@ -393,12 +401,12 @@ private static function getGroupCodesForThisWebsite($websiteId) {
    * @param object $newRecord
    *   ORM Validation object containing the new record details.
    * @param array $state
-   *   State data to pass through to the post-process hook, containing undo data.
+   *   State data to pass through to the post-process hook, containing undo
+   *   data.
    */
   private static function applyEventsQueryToRecord($qry, $entity, array $oldValues, &$newRecord, array &$state) {
     $events = $qry->get();
     foreach ($events as $event) {
-      $newUndoRecord = array();
       kohana::log('debug', 'Processing event: ' . var_export($event, TRUE));
       $valuesToApply = self::processEvent(
         $event,
@@ -416,8 +424,9 @@ private static function applyEventsQueryToRecord($qry, $entity, array $oldValues
   /**
    * Processes a single workflow event.
    *
-   * Retrieves a list of the values that need to be applied to a database record given an event. The values may include
-   * the results of a mimiced rewind as well as the value changes required for the event.
+   * Retrieves a list of the values that need to be applied to a database
+   * record given an event. The values may include the results of a mimiced
+   * rewind as well as the value changes required for the event.
    *
    * @param object $event
    *   Event object loaded from the database query.
@@ -427,12 +436,15 @@ private static function applyEventsQueryToRecord($qry, $entity, array $oldValues
    *   Array of the record values before the save operation. Used to retrieve
    *   values for undo state data.
    * @param array $newValues
-   *   Array of the record values that were submitted to be saved, causing the event to fire.
+   *   Array of the record values that were submitted to be saved, causing the
+   *   event to fire.
    * @param array $state
-   *   Array of undo state data which will be updated by this method to allow any proposed changes to be undone.
+   *   Array of undo state data which will be updated by this method to allow
+   *   any proposed changes to be undone.
    *
    * @return array
-   *   Associative array of the database fields and values which need to be applied.
+   *   Associative array of the database fields and values which need to be
+   *   applied.
    */
   public static function processEvent($event, $entity, array $oldValues, array $newValues, array &$state) {
     $entityConfig = self::getEntityConfig($entity);
@@ -463,12 +475,15 @@ public static function processEvent($event, $entity, array $oldValues, array $ne
         $valuesToApply[$deltaColumn] = $deltaValue;
       }
     }
-    $state[] = array('event_type' => $event->event_type, 'old_data' => $newUndoRecord);
+    $state[] = [
+      'event_type' => $event->event_type,
+      'old_data' => $newUndoRecord,
+    ];
     return $valuesToApply;
   }
 
   /**
-   * Returns true if the current user is allowed to view the workflow configuration pages.
+   * Returns true if the user is allowed to view the workflow config pages.
    *
    * @param object $auth
    *   Kohana authorisation object.
@@ -496,8 +511,8 @@ public static function allowWorkflowConfigAccess($auth) {
   /**
    * Rewind a record.
    *
-   * If an event wants to mimic a rewind to reset data to its original state, then undoes all changes to the record
-   * caused by workflow.
+   * If an event wants to mimic a rewind to reset data to its original state,
+   * then undoes all changes to the record caused by workflow.
    *
    * @param string $entity
    *   Name of the database entity being saved, e.g. occurrence.
@@ -506,8 +521,8 @@ public static function allowWorkflowConfigAccess($auth) {
    * @param array $columnDeltaList
    *   Array containing the field values that will be changed by the rewind.
    * @param array $state
-   *   Undo state change data from events applied to the record on this transaction which may need to be rewound.
-   * @return void
+   *   Undo state change data from events applied to the record on this
+   *   transaction which may need to be rewound.
    */
   private static function mimicRewind($entity, $entityId, array &$columnDeltaList, array $state) {
     for ($i = count($state) - 1; $i >= 0; $i--) {
@@ -516,11 +531,11 @@ private static function mimicRewind($entity, $entityId, array &$columnDeltaList,
       }
     }
     $undoRecords = ORM::factory('workflow_undo')
-      ->where(array(
+      ->where([
         'entity' => $entity,
         'entity_id' => $entityId,
         'active' => 't',
-      ))
+      ])
       ->orderby('id', 'DESC')->find_all();
     foreach ($undoRecords as $undoRecord) {
       kohana::log('debug', 'mimic rewind record: ' . var_export($undoRecord->as_array(), TRUE));
diff --git a/modules/workflow/plugins/workflow.php b/modules/workflow/plugins/workflow.php
index d3da739359..4487ee5c2f 100644
--- a/modules/workflow/plugins/workflow.php
+++ b/modules/workflow/plugins/workflow.php
@@ -17,8 +17,6 @@
  * You should have received a copy of the GNU General Public License
  * along with this program.  If not, see http://www.gnu.org/licenses/gpl.html.
  *
- * @package Modules
- * @subpackage Workflow
  * @author Indicia Team
  * @license http://www.gnu.org/licenses/gpl.html GPL
  * @link https://github.com/Indicia-Team/
@@ -46,25 +44,27 @@ function workflow_alter_menu($menu, $auth) {
 /**
  * Implements the extend_data_services hook.
  *
- * Determines the data entities which should be added to those available via data services.
+ * Determines the data entities which should be added to those available via
+ * data services.
  *
  * @return array
  *   List of database entities exposed by this plugin with configuration.
  */
 function workflow_extend_data_services() {
-  return array(
-    'workflow_events' => array(),
-    'workflow_metadata' => array('allow_full_access' => TRUE)
-  );
+  return [
+    'workflow_events' => [],
+    'workflow_metadata' => ['allow_full_access' => TRUE],
+  ];
 }
 
 /**
  * Pre-record save processing hook.
  *
- * Potential problem when a record matches multiple events, and they change the same columns,
- * so we are making the assumption that each record will only fire one alert key/key_value combination
- * undo record would require more details on firing event (key and key_value) if this is changed in future
- * In following code, entity means the orm entity, e.g. 'occurrence'
+ * Potential problem when a record matches multiple events, and they change the
+ * same columns, so we are making the assumption that each record will only
+ * fire one alert key/key_value combination undo record would require more
+ * details on firing event (key and key_value) if this is changed in future.
+ * In following code, entity means the orm entity, e.g. 'occurrence'.
  *
  * @param object $db
  *   Database connection.
@@ -81,12 +81,13 @@ function workflow_extend_data_services() {
  *   State data to pass to the post save processing hook.
  */
 function workflow_orm_pre_save_processing($db, $websiteId, $entity, $oldRecord, &$newRecord) {
-  $state = array();
+  $state = [];
   // Abort if no workflow configuration for this entity.
   if (empty(workflow::getEntityConfig($entity))) {
     return $state;
   }
-  // Rewind the record if previous workflow rule changes no longer apply (e.g. after redetermination).
+  // Rewind the record if previous workflow rule changes no longer apply (e.g.
+  // safter redetermination).
   $rewoundRecord = workflow::getRewoundRecord($db, $entity, $oldRecord, $newRecord);
   // Apply any changes in the workflow_events table relevant to the record.
   $state = workflow::applyWorkflow($db, $websiteId, $entity, $oldRecord, $rewoundRecord, $newRecord);
@@ -121,14 +122,14 @@ function workflow_orm_post_save_processing($db, $entity, $record, array $state,
   $userId = security::getUserId();
   // Insert any state undo records.
   foreach ($state as $undoDetails) {
-    $db->insert('workflow_undo', array(
+    $db->insert('workflow_undo', [
       'entity' => $entity,
       'entity_id' => $id,
       'event_type' => $undoDetails['event_type'],
       'created_on' => date("Ymd H:i:s"),
       'created_by_id' => $userId,
-      'original_values' => json_encode($undoDetails['old_data'])
-    ));
+      'original_values' => json_encode($undoDetails['old_data']),
+    ]);
   }
   return TRUE;
 }

From 85883f916d0955b108585e16bdccf7733282e6ad Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Mon, 16 Aug 2021 09:46:23 +0100
Subject: [PATCH 33/93] Adds attrs_filter field.

---
 .../202108160940_workflow_attrs.sql           |  5 +++++
 modules/workflow/models/workflow_event.php    | 21 ++++++++++++-------
 2 files changed, 18 insertions(+), 8 deletions(-)
 create mode 100644 modules/workflow/db/version_6_3_0/202108160940_workflow_attrs.sql

diff --git a/modules/workflow/db/version_6_3_0/202108160940_workflow_attrs.sql b/modules/workflow/db/version_6_3_0/202108160940_workflow_attrs.sql
new file mode 100644
index 0000000000..c0ea068e17
--- /dev/null
+++ b/modules/workflow/db/version_6_3_0/202108160940_workflow_attrs.sql
@@ -0,0 +1,5 @@
+ALTER TABLE workflow_events
+  ADD COLUMN attrs_filter json;
+
+COMMENT ON COLUMN workflow_events.attrs_filter IS
+  'List of occurrence attributes (identified by term, e.g. ReproductiveCondition, Stage or Sex) with the matching attribute values required to trigger this workflow event.'
\ No newline at end of file
diff --git a/modules/workflow/models/workflow_event.php b/modules/workflow/models/workflow_event.php
index 6779a22c19..bf4d45d126 100644
--- a/modules/workflow/models/workflow_event.php
+++ b/modules/workflow/models/workflow_event.php
@@ -27,16 +27,20 @@
  * Model class for the workflow_event table.
  */
 class Workflow_event_Model extends ORM {
-  public $search_field='id';
+  public $search_field = 'id';
 
-  protected $belongs_to = array(
+  protected $belongs_to = [
     'created_by' => 'user',
-    'updated_by' => 'user'
-  );
-  protected $has_and_belongs_to_many = array();
+    'updated_by' => 'user',
+  ];
+  protected $has_and_belongs_to_many = [];
 
+  /**
+   * Define model validation behaviour.
+   */
   public function validate(Validation $array, $save = FALSE) {
-    // uses PHP trim() to remove whitespace from beginning and end of all fields before validation
+    // Uses PHP trim() to remove whitespace from beginning and end of all
+    // fields before validation.
     $array->pre_filter('trim');
     $array->add_rules('entity', 'required');
     $array->add_rules('group_code', 'required');
@@ -46,10 +50,11 @@ public function validate(Validation $array, $save = FALSE) {
     $array->add_rules('values', 'required');
 
     // Explicitly add those fields for which we don't do validation.
-    $this->unvalidatedFields = array(
+    $this->unvalidatedFields = [
       'deleted',
       'mimic_rewind_first',
-    );
+      'attrs_filter',
+    ];
 
     return parent::validate($array, $save);
   }

From 85c3c149b8a7d78aef647a74c1537b64ea6019d6 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Mon, 16 Aug 2021 10:10:06 +0100
Subject: [PATCH 34/93] Linting

---
 .../cache_builder/helpers/cache_builder.php   | 735 +++++++++---------
 1 file changed, 380 insertions(+), 355 deletions(-)

diff --git a/modules/cache_builder/helpers/cache_builder.php b/modules/cache_builder/helpers/cache_builder.php
index f540aeae19..a283aeea21 100644
--- a/modules/cache_builder/helpers/cache_builder.php
+++ b/modules/cache_builder/helpers/cache_builder.php
@@ -1,356 +1,381 @@
-<?php
-
-/**
- * Indicia, the OPAL Online Recording Toolkit.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * any later version.
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see http://www.gnu.org/licenses/gpl.html.
- *
-
- * @author Indicia Team
- * @license http://www.gnu.org/licenses/gpl.html GPL
- * @link http://code.google.com/p/indicia/
- */
-
-/**
- * Helper class for cache_builder functionality.
- */
-class cache_builder {
-
-  public static $delayCacheUpdates = FALSE;
-
-  /**
-   * Performs the actual task of table population.
-   */
-  public static function populate_cache_table($db, $table, $last_run_date) {
-    $queries = kohana::config("cache_builder.$table");
-    try {
-      echo "<h3>$table</h3>";
-      $count = cache_builder::getChangeList($db, $table, $queries, $last_run_date);
-      if ($count > 0) {
-        echo <<<HTML
-<table>
-  <thead>
-    <tr><th></th><th># records affected</th></tr>
-  </thead>
-  <tbody>
-    <tr><th scope=\"row\">Total</th><td>$count</td>
-    </tr>
-
-HTML;
-        cache_builder::makeChanges($db, $table);
-        echo <<<HTML
-  </tbody>
-</table>
-HTML;
-      }
-      else {
-        echo "<p>No changes</p>";
-      }
-      $db->query("drop table needs_update_$table");
-    }
-    catch (Exception $e) {
-      $db->query("drop table needs_update_$table");
-      throw $e;
-    }
-  }
-
-  /**
-   * Apply required database changes to the cache tables.
-   *
-   * When the needs_update_* table already populated, apply the actual cache
-   * update changes to the cached entity.
-   *
-   * @param object $db
-   *   Database connection.
-   * @param string $table
-   *   Entity name to update (e.g. sample, occurrence, taxa_taxon_list).
-   */
-  public static function makeChanges($db, $table) {
-    $queries = kohana::config("cache_builder.$table");
-    cache_builder::do_delete($db, $table, $queries);
-    // preprocess some of the tags in the queries
-    if (is_array($queries['update']))
-      foreach($queries['update'] as $key=>&$sql)
-        $sql = str_replace('#join_needs_update#', $queries['join_needs_update'], $sql);
-    else
-      $queries['update'] = str_replace('#join_needs_update#', $queries['join_needs_update'], $queries['update']);
-    cache_builder::run_statement($db, $table, $queries['update'], 'update');
-    // preprocess some of the tags in the queries
-    if (is_array($queries['insert']))
-      foreach($queries['insert'] as $key=>&$sql)
-        $sql = str_replace('#join_needs_update#', $queries['join_needs_update'] . ' and (nu.deleted=false or nu.deleted is null)', $sql);
-    else
-      $queries['insert'] = str_replace('#join_needs_update#', $queries['join_needs_update'] . ' and (nu.deleted=false or nu.deleted is null)', $queries['insert']);
-    cache_builder::run_statement($db, $table, $queries['insert'], 'insert');
-    if (isset($queries['extra_multi_record_updates']))
-      cache_builder::run_statement($db, $table, $queries['extra_multi_record_updates'], 'final update');
-    if (!variable::get("populated-$table")) {
-      $cacheQuery = $db->query("select count(*) from cache_$table")->result_array(false);
-      if (isset($queries['count']))
-        $totalQuery = $db->query($queries['count'])->result_array(false);
-      else
-        $totalQuery = $db->query("select count(*) from $table where deleted='f'")->result_array(false);
-      $percent = round($cacheQuery[0]['count']*100/$totalQuery[0]['count']);
-      echo "<p>Initial population of $table progress $percent%.</p>";
-    }
-  }
-
-  /**
-   * Inserts a single record into the cache, e.g. could be used as soon as a record is submitted.
-   *
-   * @param object $db
-   *   Database object.
-   * @param string $table
-   *   Plural form of the table name.
-   * @param array $ids
-   *   Record IDs to insert in the cache
-   */
-  public static function insert($db, $table, array $ids) {
-    if (count($ids) > 0) {
-      $idlist = implode(',', $ids);
-      if (self::$delayCacheUpdates && in_array($table, ['occurrences', 'samples'])) {
-        kohana::log('debug', "Delayed inserts for $table ($idlist)");
-        self::delayChangesViaWorkQueue($db, $table, $idlist);
-      }
-      else {
-        $master_list_id = warehouse::getMasterTaxonListId();
-        $queries = kohana::config("cache_builder.$table");
-        if (!isset($queries['key_field']))
-          throw new exception('Cannot do a specific record insert into cache as the key_field configuration not defined in cache_builder configuration');
-        if (!is_array($queries['insert']))
-          $queries['insert'] = array($queries['insert']);
-        foreach ($queries['insert'] as $query) {
-          $insertSql = str_replace(
-            ['#join_needs_update#', '#master_list_id#'],
-            ['', $master_list_id],
-            $query
-          );
-          $insertSql .= ' and ' . $queries['key_field'] . " in ($idlist)";
-          $db->query($insertSql);
-        }
-      }
-      self::final_queries($db, $table, $ids);
-    }
-  }
-
-  /**
-   * Updates a single record in the cache.
-   *
-   * E.g. could be used as soon as a record is edited.
-   *
-   * @param object $db
-   *   Database object.
-   * @param string $table
-   *   Plural form of the table name.
-   * @param array $ids
-   *   Record IDs to insert in the cache.
-   */
-  public static function update($db, $table, array $ids) {
-    if (count($ids) > 0) {
-      $idlist = implode(',', $ids);
-      if (self::$delayCacheUpdates && in_array($table, ['occurrences', 'samples'])) {
-        kohana::log('debug', "Delayed updates for $table ($idlist)");
-        self::delayChangesViaWorkQueue($db, $table, $idlist);
-      }
-      else {
-        $master_list_id = warehouse::getMasterTaxonListId();
-        $queries = kohana::config("cache_builder.$table");
-        if (!isset($queries['key_field']))
-          throw new exception('Cannot do a specific record update into cache as the key_field configuration not defined in cache_builder configuration');
-        if (!is_array($queries['update']))
-          $queries['update'] = array($queries['update']);
-        foreach ($queries['update'] as $query) {
-          $updateSql = str_replace(
-            ['#join_needs_update#', '#master_list_id#'],
-            ['', $master_list_id],
-            $query
-          );
-          $updateSql .= ' and ' . $queries['key_field'] . " in ($idlist)";
-          $db->query($updateSql);
-        }
-        self::final_queries($db, $table, $ids);
-      }
-    }
-  }
-
-  /**
-   * Deletes a single record from the cache.
-   *
-   * E.g. could be used as soon as a record is deleted.
-   *
-   * @param object $db
-   *   Database object.
-   * @param string $table
-   *   Plural form of the table name.
-   * @param array $ids
-   *   Record IDs to delete from the cache.
-   */
-  public static function delete($db, $table, array $ids) {
-    if (self::$delayCacheUpdates && in_array($table, ['occurrences', 'samples'])) {
-      self::delayChangesViaWorkQueue($db, $table, implode(',', $ids));
-    }
-    else {
-      foreach ($ids as $id) {
-        if ($table === 'occurrences' || $table === 'samples') {
-          $db->delete("cache_{$table}_functional", array('id' => $id));
-          $db->delete("cache_{$table}_nonfunctional", array('id' => $id));
-          if ($table === 'samples') {
-            // Slightly more complex delete query to ensure indexes used.
-            $sql = <<<SQL
-DELETE FROM cache_occurrences_functional o
-USING samples s
-JOIN surveys su on su.id=s.survey_id
-WHERE s.id=$id
-AND o.sample_id=s.id
-AND o.survey_id=su.id
-AND o.website_id=su.website_id
-SQL;
-            $db->query($sql);
-            $db->query("delete from cache_occurrences_nonfunctional where id in (select id from occurrences where sample_id=$id)");
-          }
-        }
-        else {
-          $db->delete("cache_$table", array('id' => $id));
-        }
-      }
-    }
-  }
-
-  /**
-   * During an import, add tasks to work queue rather than do immediate update.
-   *
-   * Allows performance improvement during import.
-   *
-   * @param object $db
-   *   Database object.
-   * @param string $table
-   *   Plural form of the table name.
-   * @param string $idCsv
-   *   Record IDs to delete from the cache (comma separated string).
-   */
-  private static function delayChangesViaWorkQueue($db, $table, $idCsv) {
-    $entity = inflector::singular($table);
-    $sql = <<<SQL
--- Comment necessary to prevent Kohana calling LASTVAL().
-INSERT INTO work_queue(task, entity, record_id, params, cost_estimate, priority, created_on)
-SELECT 'task_cache_builder_update', '$entity', t.id, null, 100, 2, now()
-FROM $table t
-LEFT JOIN work_queue w ON w.task='task_cache_builder_update' AND w.entity='$entity' AND w.record_id=t.id
-WHERE t.id IN ($idCsv)
-AND w.id IS NULL;
-SQL;
-    $db->query($sql);
-  }
-
-  public static function final_queries($db, $table, $ids) {
-    $queries = kohana::config("cache_builder.$table");
-    $doneCount = 0;
-    if (isset($queries['extra_single_record_updates'])) {
-      $idlist=implode(',', $ids);
-      if (is_array($queries['extra_single_record_updates']))
-        foreach($queries['extra_single_record_updates'] as $key=>&$sql) {
-          $result=$db->query(str_replace('#ids#', $idlist, $sql));
-          $doneCount += $result->count();
-          if ($doneCount>=count($ids))
-            break; // we've updated all. So can drop out.
-        }
-      else {
-        $db->query(str_replace('#ids#', $idlist, $queries['extra_single_record_updates']));
-      }
-    }
-  }
-
-  /**
-   * Build a temporary table with the list of IDs of records we need to update.
-   * The table has a deleted flag to indicate newly deleted records.
-   * @param objcet $db Database connection.
-   * @param string $table Name of the table being cached, e.g. occurrences.
-   * @param string $query A query which selects a list of IDs for all new, updated or
-   * deleted records (including looking for updates or deletions caused by related
-   * records).
-   * @param string $last_run_date Date/time of the last time the cache builder was
-   * run, used to filter records to only the recent changes. Supplied as a string
-   * suitable for injection into an SQL query.
-   */
-  private static function getChangeList($db, $table, $queries, $last_run_date) {
-    $query = str_replace('#date#', $last_run_date, $queries['get_changed_items_query']);
-    $db->query("create temporary table needs_update_$table as $query");
-    if (!variable::get("populated-$table")) {
-      // as well as the changed records, pick up max 5000 previous records, which is important for initial population.
-      // 5000 is an arbitrary number to compromise between performance and cache population.
-      // of the cache
-      $query = $queries['get_missing_items_query'] . ' limit 5000';
-      $result = $db->query("insert into needs_update_$table $query");
-      if ($result->count() === 0) {
-        // Flag that we don't need to do any more previously existing records as they are all done.
-        // Future cache updates can just pick up changes from now on.
-        variable::set("populated-$table", TRUE);
-        echo "<p>Initial population of $table completed</p>";
-      }
-    }
-    $db->query("ALTER TABLE needs_update_$table ADD CONSTRAINT ix_nu_$table PRIMARY KEY (id)");
-    $r = $db->query("select count(*) as count from needs_update_$table")->result_array(FALSE);
-    $row = $r[0];
-    return $row['count'];
-  }
-
-  /**
-   * Deletes all records from the cache table which are in the table of
-   * records to update and where the deleted flag is true.
-   *
-   * @param object $db
-   *   Database connection.
-   * @param string $table
-   *   Name of the table being cached.
-   * @param array $queries
-   *   List of configured queries for this table, which might include non-default delete queries.
-   */
-  private static function do_delete($db, $table, $queries) {
-    // set up a default delete query if none are specified
-    if (!isset($queries['delete_query'])) {
-      $queries['delete_query'] = array("delete from cache_$table where id in (select id from needs_update_$table where deleted=true)");
-    }
-    $count = 0;
-    foreach ($queries['delete_query'] as $query) {
-      $count += $db->query($query)->count();
-    }
-    if (variable::get("populated-$table")) {
-      echo "    <tr><th scope=\"row\">Delete(s)</th><td>$count</td></tr>\n";
-    }
-  }
-
-  /**
-   * Runs an insert or update statemnet to update one of
-   * the cache tables.
-   * @param object $db Database connection.
-   * @param string $query Query used to perform the update or insert. Can be a string, or an
-   *   associative array of SQL strings if multiple required to do the task.
-   * @param string $action Term describing the action, used for feedback only.
-   */
-  private static function run_statement($db, $table, $query, $action) {
-    $master_list_id = warehouse::getMasterTaxonListId();
-    if (is_array($query)) {
-      foreach ($query as $title => $sql) {
-        $sql = str_replace('#master_list_id#', $master_list_id, $sql);
-        $count = $db->query($sql)->count();
-        if (variable::get("populated-$table"))
-          echo "    <tr><th scope=\"row\">$action(s) for $title</th><td>$count</td></tr>\n";
-      }
-    } else {
-      $sql = str_replace('#master_list_id#', $master_list_id, $query);
-      $count = $db->query($query)->count();
-      if (variable::get("populated-$table")) {
-        echo "    <tr><th scope=\"row\">$action(s)</th><td>$count</td></tr>\n";
-      }
-    }
-  }
+<?php
+
+/**
+ * Indicia, the OPAL Online Recording Toolkit.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * any later version.
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/gpl.html.
+ *
+
+ * @author Indicia Team
+ * @license http://www.gnu.org/licenses/gpl.html GPL
+ * @link http://code.google.com/p/indicia/
+ */
+
+/**
+ * Helper class for cache_builder functionality.
+ */
+class cache_builder {
+
+  /**
+   * Allow cache table updates to be delegated to work_queue.
+   *
+   * Useful when processing large numbers of records where immediate visibility
+   * of the results is not required.
+   */
+  public static $delayCacheUpdates = FALSE;
+
+  /**
+   * Performs the actual task of table population.
+   */
+  public static function populate_cache_table($db, $table, $last_run_date) {
+    $queries = kohana::config("cache_builder.$table");
+    try {
+      echo "<h3>$table</h3>";
+      $count = cache_builder::getChangeList($db, $table, $queries, $last_run_date);
+      if ($count > 0) {
+        echo <<<HTML
+<table>
+  <thead>
+    <tr><th></th><th># records affected</th></tr>
+  </thead>
+  <tbody>
+    <tr><th scope="row">Total</th><td>$count</td>
+    </tr>
+
+HTML;
+        cache_builder::makeChanges($db, $table);
+        echo <<<HTML
+  </tbody>
+</table>
+HTML;
+      }
+      else {
+        echo "<p>No changes</p>";
+      }
+      $db->query("drop table needs_update_$table");
+    }
+    catch (Exception $e) {
+      $db->query("drop table needs_update_$table");
+      throw $e;
+    }
+  }
+
+  /**
+   * Apply required database changes to the cache tables.
+   *
+   * When the needs_update_* table already populated, apply the actual cache
+   * update changes to the cached entity.
+   *
+   * @param object $db
+   *   Database connection.
+   * @param string $table
+   *   Entity name to update (e.g. sample, occurrence, taxa_taxon_list).
+   */
+  public static function makeChanges($db, $table) {
+    $queries = kohana::config("cache_builder.$table");
+    cache_builder::do_delete($db, $table, $queries);
+    // Preprocess some of the tags in the queries.
+    if (is_array($queries['update'])) {
+      foreach ($queries['update'] as &$sql) {
+        $sql = str_replace('#join_needs_update#', $queries['join_needs_update'], $sql);
+      }
+    }
+    else {
+      $queries['update'] = str_replace('#join_needs_update#', $queries['join_needs_update'], $queries['update']);
+    }
+    cache_builder::run_statement($db, $table, $queries['update'], 'update');
+    // Preprocess some of the tags in the queries.
+    if (is_array($queries['insert'])) {
+      foreach ($queries['insert'] as &$sql) {
+        $sql = str_replace('#join_needs_update#', $queries['join_needs_update'] . ' and (nu.deleted=false or nu.deleted is null)', $sql);
+      }
+    }
+    else {
+      $queries['insert'] = str_replace('#join_needs_update#', $queries['join_needs_update'] . ' and (nu.deleted=false or nu.deleted is null)', $queries['insert']);
+    }
+    cache_builder::run_statement($db, $table, $queries['insert'], 'insert');
+    if (isset($queries['extra_multi_record_updates'])) {
+      cache_builder::run_statement($db, $table, $queries['extra_multi_record_updates'], 'final update');
+    }
+    if (!variable::get("populated-$table")) {
+      $cacheQuery = $db->query("select count(*) from cache_$table")->result_array(FALSE);
+      if (isset($queries['count'])) {
+        $totalQuery = $db->query($queries['count'])->result_array(FALSE);
+      }
+      else {
+        $totalQuery = $db->query("select count(*) from $table where deleted='f'")->result_array(FALSE);
+      }
+      $percent = round($cacheQuery[0]['count'] * 100 / $totalQuery[0]['count']);
+      echo "<p>Initial population of $table progress $percent%.</p>";
+    }
+  }
+
+  /**
+   * Inserts a single record into the cache, e.g. could be used as soon as a record is submitted.
+   *
+   * @param object $db
+   *   Database object.
+   * @param string $table
+   *   Plural form of the table name.
+   * @param array $ids
+   *   Record IDs to insert in the cache.
+   */
+  public static function insert($db, $table, array $ids) {
+    if (count($ids) > 0) {
+      $idlist = implode(',', $ids);
+      if (self::$delayCacheUpdates && in_array($table, ['occurrences', 'samples'])) {
+        kohana::log('debug', "Delayed inserts for $table ($idlist)");
+        self::delayChangesViaWorkQueue($db, $table, $idlist);
+      }
+      else {
+        $master_list_id = warehouse::getMasterTaxonListId();
+        $queries = kohana::config("cache_builder.$table");
+        if (!isset($queries['key_field']))
+          throw new exception('Cannot do a specific record insert into cache as the key_field configuration not defined in cache_builder configuration');
+        if (!is_array($queries['insert']))
+          $queries['insert'] = array($queries['insert']);
+        foreach ($queries['insert'] as $query) {
+          $insertSql = str_replace(
+            ['#join_needs_update#', '#master_list_id#'],
+            ['', $master_list_id],
+            $query
+          );
+          $insertSql .= ' and ' . $queries['key_field'] . " in ($idlist)";
+          $db->query($insertSql);
+        }
+      }
+      self::final_queries($db, $table, $ids);
+    }
+  }
+
+  /**
+   * Updates a single record in the cache.
+   *
+   * E.g. could be used as soon as a record is edited.
+   *
+   * @param object $db
+   *   Database object.
+   * @param string $table
+   *   Plural form of the table name.
+   * @param array $ids
+   *   Record IDs to insert in the cache.
+   */
+  public static function update($db, $table, array $ids) {
+    if (count($ids) > 0) {
+      $idlist = implode(',', $ids);
+      if (self::$delayCacheUpdates && in_array($table, ['occurrences', 'samples'])) {
+        kohana::log('debug', "Delayed updates for $table ($idlist)");
+        self::delayChangesViaWorkQueue($db, $table, $idlist);
+      }
+      else {
+        $master_list_id = warehouse::getMasterTaxonListId();
+        $queries = kohana::config("cache_builder.$table");
+        if (!isset($queries['key_field']))
+          throw new exception('Cannot do a specific record update into cache as the key_field configuration not defined in cache_builder configuration');
+        if (!is_array($queries['update']))
+          $queries['update'] = array($queries['update']);
+        foreach ($queries['update'] as $query) {
+          $updateSql = str_replace(
+            ['#join_needs_update#', '#master_list_id#'],
+            ['', $master_list_id],
+            $query
+          );
+          $updateSql .= ' and ' . $queries['key_field'] . " in ($idlist)";
+          $db->query($updateSql);
+        }
+        self::final_queries($db, $table, $ids);
+      }
+    }
+  }
+
+  /**
+   * Deletes a single record from the cache.
+   *
+   * E.g. could be used as soon as a record is deleted.
+   *
+   * @param object $db
+   *   Database object.
+   * @param string $table
+   *   Plural form of the table name.
+   * @param array $ids
+   *   Record IDs to delete from the cache.
+   */
+  public static function delete($db, $table, array $ids) {
+    if (self::$delayCacheUpdates && in_array($table, ['occurrences', 'samples'])) {
+      self::delayChangesViaWorkQueue($db, $table, implode(',', $ids));
+    }
+    else {
+      foreach ($ids as $id) {
+        if ($table === 'occurrences' || $table === 'samples') {
+          $db->delete("cache_{$table}_functional", array('id' => $id));
+          $db->delete("cache_{$table}_nonfunctional", array('id' => $id));
+          if ($table === 'samples') {
+            // Slightly more complex delete query to ensure indexes used.
+            $sql = <<<SQL
+DELETE FROM cache_occurrences_functional o
+USING samples s
+JOIN surveys su on su.id=s.survey_id
+WHERE s.id=$id
+AND o.sample_id=s.id
+AND o.survey_id=su.id
+AND o.website_id=su.website_id
+SQL;
+            $db->query($sql);
+            $db->query("delete from cache_occurrences_nonfunctional where id in (select id from occurrences where sample_id=$id)");
+          }
+        }
+        else {
+          $db->delete("cache_$table", array('id' => $id));
+        }
+      }
+    }
+  }
+
+  /**
+   * During an import, add tasks to work queue rather than do immediate update.
+   *
+   * Allows performance improvement during import.
+   *
+   * @param object $db
+   *   Database object.
+   * @param string $table
+   *   Plural form of the table name.
+   * @param string $idCsv
+   *   Record IDs to delete from the cache (comma separated string).
+   */
+  private static function delayChangesViaWorkQueue($db, $table, $idCsv) {
+    $entity = inflector::singular($table);
+    $sql = <<<SQL
+-- Comment necessary to prevent Kohana calling LASTVAL().
+INSERT INTO work_queue(task, entity, record_id, params, cost_estimate, priority, created_on)
+SELECT 'task_cache_builder_update', '$entity', t.id, null, 100, 2, now()
+FROM $table t
+LEFT JOIN work_queue w ON w.task='task_cache_builder_update' AND w.entity='$entity' AND w.record_id=t.id
+WHERE t.id IN ($idCsv)
+AND w.id IS NULL;
+SQL;
+    $db->query($sql);
+  }
+
+  public static function final_queries($db, $table, $ids) {
+    $queries = kohana::config("cache_builder.$table");
+    $doneCount = 0;
+    if (isset($queries['extra_single_record_updates'])) {
+      $idlist=implode(',', $ids);
+      if (is_array($queries['extra_single_record_updates']))
+        foreach($queries['extra_single_record_updates'] as $key=>&$sql) {
+          $result=$db->query(str_replace('#ids#', $idlist, $sql));
+          $doneCount += $result->count();
+          if ($doneCount>=count($ids))
+            break; // we've updated all. So can drop out.
+        }
+      else {
+        $db->query(str_replace('#ids#', $idlist, $queries['extra_single_record_updates']));
+      }
+    }
+  }
+
+  /**
+   * Build a temporary table with the list of IDs of records we need to update.
+   *
+   * The table has a deleted flag to indicate newly deleted records.
+   *
+   * @param object $db
+   *   Database connection.
+   * @param string $table
+   *   Name of the table being cached, e.g. occurrences.
+   * @param string $queries
+   *   List of configured queries for this table.
+   * @param string $last_run_date
+   *   Date/time of the last time the cache builder was run, used to filter
+   *   records to only the recent changes. Supplied as a string suitable for
+   *   injection into an SQL query.
+   */
+  private static function getChangeList($db, $table, $queries, $last_run_date) {
+    $query = str_replace('#date#', $last_run_date, $queries['get_changed_items_query']);
+    $db->query("create temporary table needs_update_$table as $query");
+    if (!variable::get("populated-$table")) {
+      // As well as the changed records, pick up max 5000 previous records,
+      // which is important for initial population. 5000 is an arbitrary number
+      // to compromise between performance and cache population.
+      $query = $queries['get_missing_items_query'] . ' limit 5000';
+      $result = $db->query("insert into needs_update_$table $query");
+      if ($result->count() === 0) {
+        // Flag that we don't need to do any more previously existing records
+        // as they are all done.
+        // Future cache updates can just pick up changes from now on.
+        variable::set("populated-$table", TRUE);
+        echo "<p>Initial population of $table completed</p>";
+      }
+    }
+    $db->query("ALTER TABLE needs_update_$table ADD CONSTRAINT ix_nu_$table PRIMARY KEY (id)");
+    $r = $db->query("select count(*) as count from needs_update_$table")->result_array(FALSE);
+    $row = $r[0];
+    return $row['count'];
+  }
+
+  /**
+   * Deletes all records from the cache table which are in the table of
+   * records to update and where the deleted flag is true.
+   *
+   * @param object $db
+   *   Database connection.
+   * @param string $table
+   *   Name of the table being cached.
+   * @param array $queries
+   *   List of configured queries for this table, which might include non-default delete queries.
+   */
+  private static function do_delete($db, $table, $queries) {
+    // Set up a default delete query if none are specified.
+    if (!isset($queries['delete_query'])) {
+      $queries['delete_query'] = ["delete from cache_$table where id in (select id from needs_update_$table where deleted=true)"];
+    }
+    $count = 0;
+    foreach ($queries['delete_query'] as $query) {
+      $count += $db->query($query)->count();
+    }
+    if (variable::get("populated-$table")) {
+      echo "    <tr><th scope=\"row\">Delete(s)</th><td>$count</td></tr>\n";
+    }
+  }
+
+  /**
+   * Runs an insert or update statemnet to update one of the cache tables.
+   *
+   * @param object $db
+   *   Database connection.
+   * @param string $query
+   *   Query used to perform the update or insert. Can be a string, or an
+   *   associative array of SQL strings if multiple required to do the task.
+   * @param string $action
+   *   Term describing the action, used for feedback only.
+   */
+  private static function run_statement($db, $table, $query, $action) {
+    $master_list_id = warehouse::getMasterTaxonListId();
+    if (is_array($query)) {
+      foreach ($query as $title => $sql) {
+        $sql = str_replace('#master_list_id#', $master_list_id, $sql);
+        $count = $db->query($sql)->count();
+        if (variable::get("populated-$table")) {
+          echo "    <tr><th scope=\"row\">$action(s) for $title</th><td>$count</td></tr>\n";
+        }
+      }
+    }
+    else {
+      $sql = str_replace('#master_list_id#', $master_list_id, $query);
+      $count = $db->query($query)->count();
+      if (variable::get("populated-$table")) {
+        echo "    <tr><th scope=\"row\">$action(s)</th><td>$count</td></tr>\n";
+      }
+    }
+  }
 }
\ No newline at end of file

From 7c38815a9bfc5df52752e1ea2b0987e9751ff626 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Mon, 16 Aug 2021 11:12:53 +0100
Subject: [PATCH 35/93] Clearer data model

Unlikely to ever need to filter on multiple filter attributes.
---
 .../202108160940_workflow_attrs.sql           | 13 ++++++++++---
 modules/workflow/models/workflow_event.php    | 19 ++++++++++++++++++-
 2 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/modules/workflow/db/version_6_3_0/202108160940_workflow_attrs.sql b/modules/workflow/db/version_6_3_0/202108160940_workflow_attrs.sql
index c0ea068e17..7a43c8826f 100644
--- a/modules/workflow/db/version_6_3_0/202108160940_workflow_attrs.sql
+++ b/modules/workflow/db/version_6_3_0/202108160940_workflow_attrs.sql
@@ -1,5 +1,12 @@
 ALTER TABLE workflow_events
-  ADD COLUMN attrs_filter json;
+  ADD COLUMN attrs_filter_term text;
 
-COMMENT ON COLUMN workflow_events.attrs_filter IS
-  'List of occurrence attributes (identified by term, e.g. ReproductiveCondition, Stage or Sex) with the matching attribute values required to trigger this workflow event.'
\ No newline at end of file
+ALTER TABLE workflow_events
+  ADD COLUMN attrs_filter_values text[];
+
+
+COMMENT ON COLUMN workflow_events.attrs_filter_term IS
+  'When this event should only trigger if a certain attribute value is present, specify the DwC term here which identifies the attribute to use (e.g. ReproductiveCondition or Stage). Typically used to limit bird events to breeding ReproductiveCondition terms.';
+
+COMMENT ON COLUMN workflow_events.attrs_filter_values IS
+  'When this event should only trigger if a certain attribute value is present, specify the list of triggering values here. A record matching any value in the list will trigger the event.';
\ No newline at end of file
diff --git a/modules/workflow/models/workflow_event.php b/modules/workflow/models/workflow_event.php
index bf4d45d126..435a9cc9c1 100644
--- a/modules/workflow/models/workflow_event.php
+++ b/modules/workflow/models/workflow_event.php
@@ -53,10 +53,27 @@ public function validate(Validation $array, $save = FALSE) {
     $this->unvalidatedFields = [
       'deleted',
       'mimic_rewind_first',
-      'attrs_filter',
+      'attrs_filter_term',
+      'attrs_filter_values',
     ];
 
     return parent::validate($array, $save);
   }
 
+  /**
+   * Converts attr_filter_values from form submission string to array.
+   */
+  public function preSubmit() {
+    if (!empty($this->submission['fields']['attrs_filter_values']['value'])
+        && is_string($this->submission['fields']['attrs_filter_values']['value'])) {
+      $keyList = str_replace("\r\n", "\n", $this->submission['fields']['attrs_filter_values']['value']);
+      $keyList = str_replace("\r", "\n", $keyList);
+      $keyList = explode("\n", trim($keyList));
+      $this->submission['fields']['attrs_filter_values'] = ['value' => $keyList];
+    }
+    elseif (isset($this->submission['fields']['attrs_filter_values'])) {
+      $this->submission['fields']['attrs_filter_values'] = ['value' => NULL];
+    }
+  }
+
 }

From 536cc9f53bf69b539ca6c27bb1f2d7bf3d871e83 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Mon, 16 Aug 2021 11:13:27 +0100
Subject: [PATCH 36/93] Add attr filter controls to form

Also changes to make form output bootstrapped.
---
 .../workflow_event/workflow_event_edit.php    | 35 +++++++++++++------
 1 file changed, 25 insertions(+), 10 deletions(-)

diff --git a/modules/workflow/views/workflow_event/workflow_event_edit.php b/modules/workflow/views/workflow_event/workflow_event_edit.php
index 53ac130b28..d82d1f0c51 100644
--- a/modules/workflow/views/workflow_event/workflow_event_edit.php
+++ b/modules/workflow/views/workflow_event/workflow_event_edit.php
@@ -24,13 +24,14 @@
  * @link https://github.com/Indicia-Team/
  */
 
-require_once DOCROOT . 'client_helpers/data_entry_helper.php';
+warehouse::loadHelpers(['data_entry_helper']);
+$id = html::initial_value($values, 'workflow_event:id');
 if (isset($_POST)) {
   data_entry_helper::dump_errors(['errors' => $this->model->getAllErrors()]);
 }
 $readAuth = data_entry_helper::get_read_auth(0 - $_SESSION['auth_user']->id, kohana::config('indicia.private_key'));
 ?>
-<form action="<?php echo url::site(); ?>workflow_event/save" method="post" id="entry-form">
+<form action="<?php echo url::site(); ?>workflow_event/save" method="post" id="entry_form">
   <fieldset>
     <legend>Workflow Event definition details</legend>
     <?php
@@ -38,7 +39,7 @@
     data_entry_helper::enable_validation('entry-form');
     echo data_entry_helper::hidden_text([
       'fieldname' => 'workflow_event:id',
-      'default' => html::initial_value($values, 'workflow_event:id'),
+      'default' => $id,
     ]);
     $messages = [];
     // Where controls have no choice available, show a message instead.
@@ -68,7 +69,7 @@
         'lookupValues' => $other_data['groupSelectItems'],
         'default' => html::initial_value($values, 'workflow_event:group_code'),
         'validation' => ['required'],
-        'helpText' => 'The workflow groups available must be configured by the warehouse administrator and define which website\'s records will be affected by this event definition.'
+        'helpText' => 'The workflow groups available must be configured by the warehouse administrator and define which website\'s records will be affected by this event definition.',
       ]);
     }
     if (count($other_data['entitySelectItems']) !== 1) {
@@ -131,6 +132,24 @@
       'lookupValues' => [],
       'validation' => ['required'],
     ]);
+    echo data_entry_helper::text_input([
+      'label' => 'Attribute value filter term',
+      'fieldname' => 'workflow_event:attrs_filter_term',
+      'helpText' => lang::get('When this event should only trigger if a certain attribute value is present, specify ' .
+        'the DwC term here which identifies the attribute to use (e.g. ReproductiveCondition or Stage). Typically ' .
+        'used to limit bird events to breeding ReproductiveCondition terms.'),
+      'default' => html::initial_value($values, 'workflow_event:attrs_filter_term'),
+    ]);
+    $filterValues = html::initial_value($values, 'workflow_event:attrs_filter_values');
+    if (!empty($filterValues)) {
+      $filterValues = trim($filterValues, '{}');
+      $filterValues = str_replace(',', "\n", $filterValues);
+    }
+    echo data_entry_helper::textarea([
+      'label' => 'Attribute value filter values',
+      'fieldname' => 'workflow_event:attrs_filter_values',
+      'default' => $filterValues,
+    ]);
     echo data_entry_helper::checkbox([
       'label' => 'Rewind record state first',
       'fieldname' => 'workflow_event:mimic_rewind_first',
@@ -142,15 +161,11 @@
       'schema' => $other_data['jsonSchema'],
       'default' => html::initial_value($values, 'workflow_event:values'),
     ]);
-
     echo $metadata;
-    echo html::form_buttons(html::initial_value($values, 'workflow_event:id') != NULL, FALSE, FALSE);
-
     data_entry_helper::$indiciaData['entities'] = $other_data['entities'];
 
-    data_entry_helper::$dumped_resources[] = 'jquery';
-    data_entry_helper::$dumped_resources[] = 'jquery_ui';
-    data_entry_helper::$dumped_resources[] = 'fancybox';
+    echo html::form_buttons(!empty($id), FALSE, FALSE);
+    data_entry_helper::enable_validation('entry_form');
     echo data_entry_helper::dump_javascript();
     ?>
   </fieldset>

From 1c73dac9253c151327e5a4a00d7dbd4bfd2122f4 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Mon, 16 Aug 2021 11:31:28 +0100
Subject: [PATCH 37/93] Remove duplicate stuff from form.

---
 modules/workflow/views/workflow_event/workflow_event_edit.php | 2 --
 1 file changed, 2 deletions(-)

diff --git a/modules/workflow/views/workflow_event/workflow_event_edit.php b/modules/workflow/views/workflow_event/workflow_event_edit.php
index d82d1f0c51..03c62a8789 100644
--- a/modules/workflow/views/workflow_event/workflow_event_edit.php
+++ b/modules/workflow/views/workflow_event/workflow_event_edit.php
@@ -35,8 +35,6 @@
   <fieldset>
     <legend>Workflow Event definition details</legend>
     <?php
-    data_entry_helper::link_default_stylesheet();
-    data_entry_helper::enable_validation('entry-form');
     echo data_entry_helper::hidden_text([
       'fieldname' => 'workflow_event:id',
       'default' => $id,

From 85d70183755f45d16ccb5da463aca2d30f9601ba Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Tue, 17 Aug 2021 09:38:23 +0100
Subject: [PATCH 38/93] Avoid converting lat longs to scientific notation

---
 modules/rest_api_sync/helpers/api_persist.php               | 6 ++++--
 .../helpers/rest_api_sync_json_occurrences.php              | 6 ++++--
 modules/rest_api_sync/helpers/rest_api_sync_rest.php        | 6 ++++--
 3 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/modules/rest_api_sync/helpers/api_persist.php b/modules/rest_api_sync/helpers/api_persist.php
index e1a1094066..ca835e1397 100644
--- a/modules/rest_api_sync/helpers/api_persist.php
+++ b/modules/rest_api_sync/helpers/api_persist.php
@@ -726,8 +726,10 @@ private static function setSrefData(array &$values, array $observation, $fieldna
   private static function formatLatLong($lat, $long) {
     $ns = $lat >= 0 ? 'N' : 'S';
     $ew = $long >= 0 ? 'E' : 'W';
-    $lat = abs($lat);
-    $long = abs($long);
+    // Variant of abs() function using preg_replace avoids changing float to
+    // scientific notation.
+    $lat = preg_replace('/^-/', '', $lat);
+    $long = preg_replace('/^-/', '', $long);
     return "$lat$ns $long$ew";
   }
 
diff --git a/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php b/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php
index 20f5aa4ac7..e7cfb77d23 100644
--- a/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php
+++ b/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php
@@ -126,8 +126,10 @@ public static function syncPage($serverId, array $server) {
           'identificationVerificationStatus' => empty($record['identification']['identificationVerificationStatus']) ? NULL : $record['identification']['identificationVerificationStatus'],
         ];
         if (!empty($record['location']['decimalLongitude']) && !empty($record['location']['decimalLatitude'])) {
-          $observation['east'] = $record['location']['decimalLongitude'];
-          $observation['north'] = $record['location']['decimalLatitude'];
+          // Json_decode() converts some floats to scientific notation, so
+          // reverse that.
+          $observation['east'] = rtrim(number_format($record['location']['decimalLongitude'], 12), 0);
+          $observation['north'] = rtrim(number_format($record['location']['decimalLatitude'], 12), 0);
           $observation['projection'] = 'WGS84';
         }
         elseif (!empty($record['location']['gridReference'])) {
diff --git a/modules/rest_api_sync/helpers/rest_api_sync_rest.php b/modules/rest_api_sync/helpers/rest_api_sync_rest.php
index ee59f5cf10..ae2213fe61 100644
--- a/modules/rest_api_sync/helpers/rest_api_sync_rest.php
+++ b/modules/rest_api_sync/helpers/rest_api_sync_rest.php
@@ -112,8 +112,10 @@ public static function syncTaxonObservationsGet($foo, array $clientConfig, $proj
       }
       else {
         $point = explode(',', $doc->location->point);
-        $obj['location']['decimalLatitude'] = $point[0];
-        $obj['location']['decimalLongitude'] = $point[1];
+        // Rtrim() and number_format() ensure we don't convert float to
+        // scientific notation.
+        $obj['location']['decimalLatitude'] = rtrim(number_format($point[0], 12), 0);
+        $obj['location']['decimalLongitude'] = rtrim(number_format($point[1], 12), 0);
         $obj['location']['geodeticDatum'] = 'WGS84';
       }
       if (!empty($doc->location->verbatim_locality)) {

From 487a20e6c06e3cf4b370b517d1c4e091c61cfef2 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Wed, 18 Aug 2021 11:58:55 +0100
Subject: [PATCH 39/93] Adds a location ID filter field for workflow events

---
 ...orkflow_attrs.sql => 202108160940_workflow_filters.sql} | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)
 rename modules/workflow/db/version_6_3_0/{202108160940_workflow_attrs.sql => 202108160940_workflow_filters.sql} (63%)

diff --git a/modules/workflow/db/version_6_3_0/202108160940_workflow_attrs.sql b/modules/workflow/db/version_6_3_0/202108160940_workflow_filters.sql
similarity index 63%
rename from modules/workflow/db/version_6_3_0/202108160940_workflow_attrs.sql
rename to modules/workflow/db/version_6_3_0/202108160940_workflow_filters.sql
index 7a43c8826f..0892ce934d 100644
--- a/modules/workflow/db/version_6_3_0/202108160940_workflow_attrs.sql
+++ b/modules/workflow/db/version_6_3_0/202108160940_workflow_filters.sql
@@ -4,9 +4,14 @@ ALTER TABLE workflow_events
 ALTER TABLE workflow_events
   ADD COLUMN attrs_filter_values text[];
 
+ALTER TABLE workflow_events
+ADD COLUMN location_ids_filter integer[];
 
 COMMENT ON COLUMN workflow_events.attrs_filter_term IS
   'When this event should only trigger if a certain attribute value is present, specify the DwC term here which identifies the attribute to use (e.g. ReproductiveCondition or Stage). Typically used to limit bird events to breeding ReproductiveCondition terms.';
 
 COMMENT ON COLUMN workflow_events.attrs_filter_values IS
-  'When this event should only trigger if a certain attribute value is present, specify the list of triggering values here. A record matching any value in the list will trigger the event.';
\ No newline at end of file
+  'When this event should only trigger if a certain attribute value is present, specify the list of triggering values here. A record matching any value in the list will trigger the event.';
+
+COMMENT ON COLUMN workflow_events.location_ids_filter IS
+  'When this event should only trigger if the record overlaps an indexed location boundary, specify the location ID or list of IDs here. Used to limit alerts to geographic areas.';
\ No newline at end of file

From 51af4e447ba41ff3ff8cd91433ae1891aace6c65 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Wed, 18 Aug 2021 12:04:58 +0100
Subject: [PATCH 40/93] Add location IDs filter to edit form.

---
 .../workflow/controllers/workflow_event.php   | 21 +++++++++++++++++++
 modules/workflow/models/workflow_event.php    | 21 ++++++++++++-------
 .../workflow_event/workflow_event_edit.php    | 12 +++++++++++
 3 files changed, 47 insertions(+), 7 deletions(-)

diff --git a/modules/workflow/controllers/workflow_event.php b/modules/workflow/controllers/workflow_event.php
index 656b4070cc..d6b6ad02c1 100644
--- a/modules/workflow/controllers/workflow_event.php
+++ b/modules/workflow/controllers/workflow_event.php
@@ -53,6 +53,27 @@ protected function get_action_columns() {
     );
   }
 
+  /**
+   * Convert location_ids_filter to suitable default for the sub_list control.
+   */
+  protected function getModelValues() {
+    $r = parent::getModelValues();
+    $r['location_ids_filter_array'] = [];
+    if (preg_match('/^{(?<list>\d+(,\d+)*)}$/', $r['workflow_event:location_ids_filter'], $matches)) {
+      $ids = explode(',', $matches['list']);
+      foreach ($ids as $id) {
+        $location = ORM::factory('location', $id);
+        $r['location_ids_filter_array'][] = [
+          'caption' => $location->name,
+          'fieldname' => 'workflow_event:location_ids_filter[]',
+          'default' => $id,
+        ];
+      }
+
+    }
+    return $r;
+  }
+
   /**
    * Prepares any additional data required by the edit view.
    *
diff --git a/modules/workflow/models/workflow_event.php b/modules/workflow/models/workflow_event.php
index 435a9cc9c1..0fad0ef59a 100644
--- a/modules/workflow/models/workflow_event.php
+++ b/modules/workflow/models/workflow_event.php
@@ -55,24 +55,31 @@ public function validate(Validation $array, $save = FALSE) {
       'mimic_rewind_first',
       'attrs_filter_term',
       'attrs_filter_values',
+      'location_ids_filter',
     ];
 
     return parent::validate($array, $save);
   }
 
   /**
-   * Converts attr_filter_values from form submission string to array.
+   * Tidy form data to prepare for submission.
+   *
+   * Converts attr_filter_values from form submission string to array. Also
+   * ensures location_ids_filter array is cleaned.
    */
   public function preSubmit() {
     if (!empty($this->submission['fields']['attrs_filter_values']['value'])
         && is_string($this->submission['fields']['attrs_filter_values']['value'])) {
-      $keyList = str_replace("\r\n", "\n", $this->submission['fields']['attrs_filter_values']['value']);
-      $keyList = str_replace("\r", "\n", $keyList);
-      $keyList = explode("\n", trim($keyList));
-      $this->submission['fields']['attrs_filter_values'] = ['value' => $keyList];
+      $valueList = str_replace("\r\n", "\n", $this->submission['fields']['attrs_filter_values']['value']);
+      $valueList = str_replace("\r", "\n", $valueList);
+      $valueList = explode("\n", trim($valueList));
+      $this->submission['fields']['attrs_filter_values'] = ['value' => $valueList];
     }
-    elseif (isset($this->submission['fields']['attrs_filter_values'])) {
-      $this->submission['fields']['attrs_filter_values'] = ['value' => NULL];
+    // Due to the way the sub_list control works, we can have hidden empty
+    // values which need to be cleaned.
+    if (!empty($this->submission['fields']['location_ids_filter']['value'])
+        && is_array($this->submission['fields']['location_ids_filter']['value'])) {
+      $this->submission['fields']['location_ids_filter']['value'] = array_values(array_filter($this->submission['fields']['location_ids_filter']['value']));
     }
   }
 
diff --git a/modules/workflow/views/workflow_event/workflow_event_edit.php b/modules/workflow/views/workflow_event/workflow_event_edit.php
index 03c62a8789..dcd71e5bf7 100644
--- a/modules/workflow/views/workflow_event/workflow_event_edit.php
+++ b/modules/workflow/views/workflow_event/workflow_event_edit.php
@@ -130,6 +130,18 @@
       'lookupValues' => [],
       'validation' => ['required'],
     ]);
+    echo data_entry_helper::sub_list([
+      'label' => 'Location boundary filter',
+      'fieldname' => 'workflow_event:location_ids_filter',
+      'helpText' => lang::get('When this event should only trigger if a record falls inside a geographic region, ' .
+        'specify the list of locations covering that region (or regions). The locations must be indexed.'),
+      'table' => 'location',
+      'captionField' => 'name',
+      'valueField' => 'id',
+      'extraParams' => $readAuth /* + location type IDs filter + */,
+      'addToTable' => FALSE,
+      'default' => $values['location_ids_filter_array'],
+    ]);
     echo data_entry_helper::text_input([
       'label' => 'Attribute value filter term',
       'fieldname' => 'workflow_event:attrs_filter_term',

From 348854ee63bc0c80f42c2133a74fa99d30e67213 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Wed, 18 Aug 2021 15:49:15 +0100
Subject: [PATCH 41/93] Fixes comparison on params::json

---
 application/libraries/WorkQueue.php | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/application/libraries/WorkQueue.php b/application/libraries/WorkQueue.php
index 965704242b..5b96a8547b 100644
--- a/application/libraries/WorkQueue.php
+++ b/application/libraries/WorkQueue.php
@@ -72,7 +72,8 @@ public function enqueue($db, array $fields) {
       'task=' . pg_escape_literal($fields['task']) .
       'AND entity' . (empty($fields['entity']) ? ' IS NULL' : '=' . pg_escape_literal($fields['entity'])) .
       'AND record_id' . (empty($fields['record_id']) ? ' IS NULL' : '=' . pg_escape_literal($fields['record_id'])) .
-      'AND params' . (empty($fields['params']) ? ' IS NULL' : '=' . pg_escape_literal($fields['params']));
+      // Use JSONB to compare as valid in pgSQL.
+      'AND params' . (empty($fields['params']) ? ' IS NULL' : ('::jsonb=' . pg_escape_literal($fields['params']) . '::jsonb'));
     foreach ($fields as $value) {
       $setValues[] = pg_escape_literal($value);
     }

From f877ee345cb4fee8a4f9dab881b40eb622276dac Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Thu, 19 Aug 2021 11:41:28 +0100
Subject: [PATCH 42/93] Work queue tasks for filtering workflow events

Task only created if event has a filter.
---
 .../task_workflow_event_check_filters.php     | 80 +++++++++++++++++++
 modules/workflow/helpers/workflow.php         | 12 ++-
 modules/workflow/plugins/workflow.php         | 13 +++
 3 files changed, 103 insertions(+), 2 deletions(-)
 create mode 100644 modules/workflow/helpers/task_workflow_event_check_filters.php

diff --git a/modules/workflow/helpers/task_workflow_event_check_filters.php b/modules/workflow/helpers/task_workflow_event_check_filters.php
new file mode 100644
index 0000000000..fdc1fe2f96
--- /dev/null
+++ b/modules/workflow/helpers/task_workflow_event_check_filters.php
@@ -0,0 +1,80 @@
+<?php
+
+/**
+ * @file
+ * Queue worker to update cache_occurrences_nonfunctional.attrs_json.
+ *
+ * Indicia, the OPAL Online Recording Toolkit.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * any later version.
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/gpl.html.
+ *
+ * @author Indicia Team
+ * @license http://www.gnu.org/licenses/gpl.html GPL
+ * @link https://github.com/indicia-team/warehouse
+ */
+
+ defined('SYSPATH') or die('No direct script access.');
+
+/**
+ * Queue worker to apply filters which limit the scope of a workflow event.
+ *
+ * Workflow events can have limited scope, e.g. to a particular custom
+ * attribute value or geographic region. The event's record changes must be
+ * applied at the time the record is saved, but this is too early for the
+ * filter criteria to be checked. So a work queue entry is created to undo the
+ * event's outcome if the filter is not met.
+ *
+ * @todo ensure this runs after spatial indexing.
+ */
+class task_workflow_event_check_filters {
+  const BATCH_SIZE = 100;
+
+  public static function process($db, $taskType, $procId) {
+    // Retrieve work queue tasks for this procId, where the task links to an
+    // event with a filter, but the record does not comply with the filter.
+    $qry = <<<SQL
+SELECT q.record_id
+FROM work_queue q
+JOIN occurrences o ON o.id=q.record_id
+JOIN workflow_events e on e.id::text=q.params->>'workflow_events.id'
+LEFT JOIN samples s ON s.id=o.sample_id AND e.location_ids_filter is not null
+LEFT JOIN locations l ON l.id = ANY(e.location_ids_filter) AND st_intersects(l.boundary_geom, s.geom)
+LEFT JOIN (occurrence_attribute_values v
+  JOIN cache_termlists_terms t on t.id=v.int_value
+  JOIN occurrence_attributes a ON a.id=v.occurrence_attribute_id
+) ON v.occurrence_id=o.id AND e.attrs_filter_term IS NOT NULL
+  -- case insensitive array check.
+  AND lower(t.term)=ANY(lower(e.attrs_filter_values::text)::text[])
+  AND lower(a.term_name)=lower(e.attrs_filter_term)
+WHERE q.entity='occurrence' AND q.task='task_workflow_event_check_filters' AND claimed_by='$procId'
+-- Need to either fail on the locations filter, or attribute values filter.
+AND ((e.location_ids_filter IS NOT NULL AND l.id IS NULL)
+OR (e.attrs_filter_term IS NOT NULL AND v.id IS NULL));
+SQL;
+    $tasks = $db->query($qry);
+    $occurrenceIds = [];
+    foreach ($tasks as $task) {
+      $occurrenceIds[] = $task->record_id;
+    }
+    // For the records outside the workflow_event's filter we can rewind them.
+    $rewinds = workflow::getRewindChangesForRecords($db, 'occurrence', $occurrenceIds, ['S', 'V', 'R']);
+    foreach ($rewinds as $key => $rewind) {
+      list($entity, $id) = explode('.', $key);
+      $obj = ORM::factory($entity, $id);
+      foreach ($rewind as $field => $value) {
+        $obj->$field = $value;
+      }
+      $obj->save();
+    }
+  }
+
+}
diff --git a/modules/workflow/helpers/workflow.php b/modules/workflow/helpers/workflow.php
index 8172753c97..0a6eeba920 100644
--- a/modules/workflow/helpers/workflow.php
+++ b/modules/workflow/helpers/workflow.php
@@ -302,7 +302,8 @@ private static function buildEventQueryForKey($db, array $groupCodes, $entity, $
     $entityConfig = self::getEntityConfig($entity);
     $eventTypes = [];
     $qry = $db
-      ->select('workflow_events.event_type, workflow_events.mimic_rewind_first, workflow_events.values')
+      ->select('workflow_events.id, workflow_events.event_type, workflow_events.mimic_rewind_first, workflow_events.values, ' .
+          'workflow_events.attrs_filter_term, workflow_events.location_ids_filter')
       ->from('workflow_events')
       ->where([
         'workflow_events.deleted' => 'f',
@@ -408,8 +409,10 @@ private static function applyEventsQueryToRecord($qry, $entity, array $oldValues
     $events = $qry->get();
     foreach ($events as $event) {
       kohana::log('debug', 'Processing event: ' . var_export($event, TRUE));
+      $needsFilterCheck = !empty($event->attrs_filter_term) || !empty($event->location_ids_filter);
       $valuesToApply = self::processEvent(
         $event,
+        $needsFilterCheck,
         $entity,
         $oldValues,
         $newRecord->as_array(),
@@ -430,6 +433,9 @@ private static function applyEventsQueryToRecord($qry, $entity, array $oldValues
    *
    * @param object $event
    *   Event object loaded from the database query.
+   * @param bool $needsFilterCheck
+   *   If true, then the workflow event has an attribute or location filter
+   *   that needs double checking via a work queue task.
    * @param string $entity
    *   Name of the database entity being saved, e.g. occurrence.
    * @param array $oldValues
@@ -446,7 +452,7 @@ private static function applyEventsQueryToRecord($qry, $entity, array $oldValues
    *   Associative array of the database fields and values which need to be
    *   applied.
    */
-  public static function processEvent($event, $entity, array $oldValues, array $newValues, array &$state) {
+  public static function processEvent($event, $needsFilterCheck, $entity, array $oldValues, array $newValues, array &$state) {
     $entityConfig = self::getEntityConfig($entity);
     $columnDeltaList = [];
     $valuesToApply = [];
@@ -476,6 +482,8 @@ public static function processEvent($event, $entity, array $oldValues, array $ne
       }
     }
     $state[] = [
+      'event_id' => $event->id,
+      'needs_filter_check' => $needsFilterCheck,
       'event_type' => $event->event_type,
       'old_data' => $newUndoRecord,
     ];
diff --git a/modules/workflow/plugins/workflow.php b/modules/workflow/plugins/workflow.php
index 4487ee5c2f..38f07eb79d 100644
--- a/modules/workflow/plugins/workflow.php
+++ b/modules/workflow/plugins/workflow.php
@@ -130,6 +130,19 @@ function workflow_orm_post_save_processing($db, $entity, $record, array $state,
       'created_by_id' => $userId,
       'original_values' => json_encode($undoDetails['old_data']),
     ]);
+    if ($undoDetails['needs_filter_check']) {
+      $q = new WorkQueue();
+      $q->enqueue($db, [
+        'task' => 'task_workflow_event_check_filters',
+        'entity' => $entity,
+        'record_id' => $id,
+        'cost_estimate' => 50,
+        'priority' => 2,
+        'params' => json_encode([
+          'workflow_events.id' => $undoDetails['event_id'],
+        ]),
+      ]);
+    }
   }
   return TRUE;
 }

From 25bff63e1528ee23339165f3b29b713bbd4a2669 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Thu, 19 Aug 2021 12:22:57 +0100
Subject: [PATCH 43/93] UI allows search within location type

---
 modules/workflow/views/workflow_event/edit.js | 14 +++++++++---
 .../workflow_event/workflow_event_edit.php    | 22 +++++++++++++++----
 2 files changed, 29 insertions(+), 7 deletions(-)

diff --git a/modules/workflow/views/workflow_event/edit.js b/modules/workflow/views/workflow_event/edit.js
index df57119755..70e0a20b71 100644
--- a/modules/workflow/views/workflow_event/edit.js
+++ b/modules/workflow/views/workflow_event/edit.js
@@ -1,8 +1,16 @@
 jQuery(document).ready(function($) {
   $('#taxon_list_id').change(function() {
-    var options = $('input#workflow_event\\:key_value\\:taxon').indiciaAutocomplete('option');
-    options.extraParams.taxon_list_id = $('#taxon_list_id').val();
-    $('input#workflow_event\\:key_value\\:taxon').indiciaAutocomplete('option', options);
+    $('input#workflow_event\\:key_value\\:taxon').setExtraParams({
+      taxon_list_id: $('#taxon_list_id').val()
+    });
+  });
+
+  $('#location_type').change(function() {
+    // Remove any hanging autocomplete select list.
+    $('.ac_results').hide();
+    $('#workflow_event\\:location_ids_filter\\:search\\:name').setExtraParams({
+      location_type_id: $('#location_type').val()
+    });
   });
 
   $('#workflow_event\\:entity').change(function entityChange() {
diff --git a/modules/workflow/views/workflow_event/workflow_event_edit.php b/modules/workflow/views/workflow_event/workflow_event_edit.php
index dcd71e5bf7..d1617c08bf 100644
--- a/modules/workflow/views/workflow_event/workflow_event_edit.php
+++ b/modules/workflow/views/workflow_event/workflow_event_edit.php
@@ -125,22 +125,33 @@
       'default' => html::initial_value($values, 'workflow_event:event_type'),
     ]);
     echo data_entry_helper::select([
-      'label' => 'Event Type',
+      'label' => 'Event type',
       'fieldname' => 'workflow_event:event_type',
       'lookupValues' => [],
       'validation' => ['required'],
     ]);
+    echo '<fieldset><legend>Event filters</legend>';
+    echo data_entry_helper::select([
+      'label' => 'Location type',
+      'fieldname' => 'location_type',
+      'table' => 'termlists_term',
+      'valueField' => 'id',
+      'captionField' => 'term',
+      'extraParams' => $readAuth + ['termlist_external_key' => 'indicia:location_types', 'order_by' => 'term'],
+      'blankText' => '<Please select>',
+      'helpText' => 'Choose the location type to search within when adding a location boundary filter below.',
+    ]);
     echo data_entry_helper::sub_list([
       'label' => 'Location boundary filter',
       'fieldname' => 'workflow_event:location_ids_filter',
       'helpText' => lang::get('When this event should only trigger if a record falls inside a geographic region, ' .
-        'specify the list of locations covering that region (or regions). The locations must be indexed.'),
+        'specify the list of locations covering that region (or regions).'),
       'table' => 'location',
       'captionField' => 'name',
       'valueField' => 'id',
-      'extraParams' => $readAuth /* + location type IDs filter + */,
+      'extraParams' => $readAuth /* @todo location type IDs filter + */,
       'addToTable' => FALSE,
-      'default' => $values['location_ids_filter_array'],
+      'default' => html::initial_value($values, 'location_ids_filter_array'),
     ]);
     echo data_entry_helper::text_input([
       'label' => 'Attribute value filter term',
@@ -160,6 +171,8 @@
       'fieldname' => 'workflow_event:attrs_filter_values',
       'default' => $filterValues,
     ]);
+    echo '</fieldset>';
+    echo '<fieldset><legend>Data updates</legend>';
     echo data_entry_helper::checkbox([
       'label' => 'Rewind record state first',
       'fieldname' => 'workflow_event:mimic_rewind_first',
@@ -171,6 +184,7 @@
       'schema' => $other_data['jsonSchema'],
       'default' => html::initial_value($values, 'workflow_event:values'),
     ]);
+    echo '</fieldset>';
     echo $metadata;
     data_entry_helper::$indiciaData['entities'] = $other_data['entities'];
 

From fef93855bcb1680ff372547e9e0b189eed31cb63 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Thu, 19 Aug 2021 13:25:49 +0100
Subject: [PATCH 44/93] Bootstrapped button

---
 modules/workflow/views/workflow_event/index.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/workflow/views/workflow_event/index.php b/modules/workflow/views/workflow_event/index.php
index 69ab0f16d1..4e60a49f41 100644
--- a/modules/workflow/views/workflow_event/index.php
+++ b/modules/workflow/views/workflow_event/index.php
@@ -31,7 +31,7 @@
 echo $grid;
 ?>
 <form action="<?php echo url::site() . 'workflow_event/create'; ?>" method="post">
-  <input type="submit" value="New Workflow event" class="ui-corner-all ui-state-default button" />
+  <input type="submit" value="New Workflow event" class="btn btn-primary" />
 </form>
 <?php
 $systemTableEntries = $this->db->select('*')->from('system')->where('name', 'workflow')->get()->as_array(TRUE);

From c15e7fd9f2c0daee047af3003488891ce22948db Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Thu, 19 Aug 2021 13:27:00 +0100
Subject: [PATCH 45/93] Changes required to workflow after verification events

In order to link with filtered workflow events.
---
 .../indicia_svc_data/helpers/data_utils.php   | 20 ++++++++++++++++++-
 modules/workflow/helpers/workflow.php         |  4 ++--
 .../workflow_event/workflow_event_edit.php    |  4 ++--
 3 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/modules/indicia_svc_data/helpers/data_utils.php b/modules/indicia_svc_data/helpers/data_utils.php
index 21341f421b..f0026bcce8 100644
--- a/modules/indicia_svc_data/helpers/data_utils.php
+++ b/modules/indicia_svc_data/helpers/data_utils.php
@@ -79,7 +79,10 @@ public static function applyWorkflowToOccurrenceUpdates($db, $websiteId, $userId
         && !empty($updates['occurrences']['record_status'])) {
       // As we are verifying or rejecting, we need to rewind any opposing
       // rejections or verifications.
-      $rewinds = workflow::getRewindChangesForRecords($db, 'occurrence', $idList, ['V', 'R']);
+      $rewinds = workflow::getRewindChangesForRecords($db, 'occurrence', $idList, [
+        'V',
+        'R',
+      ]);
       // Fetch any new events to apply when this record is verified.
       $workflowEvents = workflow::getEventsForRecords(
         $db,
@@ -105,8 +108,10 @@ public static function applyWorkflowToOccurrenceUpdates($db, $websiteId, $userId
               $oldRecord = ORM::factory('occurrence', $id);
               $oldRecordVals = $oldRecord->as_array();
               $newRecordVals = array_merge($oldRecordVals, $thisUpdates['occurrences']);
+              $needsFilterCheck = !empty($thisEvent->attrs_filter_term) || !empty($thisEvent->location_ids_filter);
               $valuesToApply = workflow::processEvent(
                 $thisEvent,
+                $needsFilterCheck,
                 'occurrence',
                 $oldRecordVals,
                 $newRecordVals,
@@ -126,6 +131,19 @@ public static function applyWorkflowToOccurrenceUpdates($db, $websiteId, $userId
                 'created_by_id' => $userId,
                 'original_values' => json_encode($undoDetails['old_data']),
               ]);
+              if ($undoDetails['needs_filter_check']) {
+                $q = new WorkQueue();
+                $q->enqueue($db, [
+                  'task' => 'task_workflow_event_check_filters',
+                  'entity' => 'occurrence',
+                  'record_id' => $id,
+                  'cost_estimate' => 50,
+                  'priority' => 2,
+                  'params' => json_encode([
+                    'workflow_events.id' => $undoDetails['event_id'],
+                  ]),
+                ]);
+              }
             }
           }
           // This record is done, so exclude from the bulk operation.
diff --git a/modules/workflow/helpers/workflow.php b/modules/workflow/helpers/workflow.php
index 0a6eeba920..0fb203a632 100644
--- a/modules/workflow/helpers/workflow.php
+++ b/modules/workflow/helpers/workflow.php
@@ -196,8 +196,8 @@ public static function getEventsForRecords($db, $websiteId, $entity, array $enti
     $table = inflector::plural($entity);
     foreach ($entityConfig['keys'] as $keyDef) {
       $qry = $db
-        ->select('workflow_events.key_value, workflow_events.event_type, workflow_events.mimic_rewind_first, ' .
-          "workflow_events.values, $table.id as {$entity}_id")
+        ->select('workflow_events.id, workflow_events.key_value, workflow_events.event_type, workflow_events.mimic_rewind_first, ' .
+          "workflow_events.values, $table.id as {$entity}_id, workflow_events.attrs_filter_term, workflow_events.location_ids_filter")
         ->from('workflow_events')
         ->where([
           'workflow_events.deleted' => 'f',
diff --git a/modules/workflow/views/workflow_event/workflow_event_edit.php b/modules/workflow/views/workflow_event/workflow_event_edit.php
index d1617c08bf..e7ff83e52c 100644
--- a/modules/workflow/views/workflow_event/workflow_event_edit.php
+++ b/modules/workflow/views/workflow_event/workflow_event_edit.php
@@ -149,9 +149,9 @@
       'table' => 'location',
       'captionField' => 'name',
       'valueField' => 'id',
-      'extraParams' => $readAuth /* @todo location type IDs filter + */,
+      'extraParams' => $readAuth,
       'addToTable' => FALSE,
-      'default' => html::initial_value($values, 'location_ids_filter_array'),
+      'default' => empty($values['location_ids_filter_array']) ? NULL : $values['location_ids_filter_array'],
     ]);
     echo data_entry_helper::text_input([
       'label' => 'Attribute value filter term',

From c194ba82f8994b7516f06b70d0ca4edbd181f5e1 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Thu, 19 Aug 2021 13:27:23 +0100
Subject: [PATCH 46/93] Version bump

---
 application/config/version.php | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/application/config/version.php b/application/config/version.php
index 5ef247fdff..3b1da16159 100644
--- a/application/config/version.php
+++ b/application/config/version.php
@@ -29,14 +29,14 @@
  *
  * @var string
  */
-$config['version'] = '6.2.3';
+$config['version'] = '6.3.0';
 
 /**
  * Version release date.
  *
  * @var string
  */
-$config['release_date'] = '2021-08-13';
+$config['release_date'] = '2021-08-19';
 
 /**
  * Link to the code repository downloads page.

From 4ee92082393b9067eaf2e292225980da68d5e00e Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Fri, 20 Aug 2021 13:52:31 +0100
Subject: [PATCH 47/93] Update auth method for BTO servers

---
 modules/rest_api_sync/helpers/rest_api_sync.php | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/modules/rest_api_sync/helpers/rest_api_sync.php b/modules/rest_api_sync/helpers/rest_api_sync.php
index e2a6c595f9..d1a09c9457 100644
--- a/modules/rest_api_sync/helpers/rest_api_sync.php
+++ b/modules/rest_api_sync/helpers/rest_api_sync.php
@@ -52,8 +52,17 @@ public static function getDataFromRestUrl($url, $serverId) {
     if (empty($servers[$serverId]['serverType']) || $servers[$serverId]['serverType'] === 'Indicia') {
       $shared_secret = $servers[$serverId]['shared_secret'];
       $userId = self::$clientUserId;
-      $hmac = hash_hmac("sha1", $url, $shared_secret, $raw_output = FALSE);
-      curl_setopt($session, CURLOPT_HTTPHEADER, array("Authorization: USER:$userId:HMAC:$hmac"));
+      $hmac = hash_hmac("sha1", $url, $shared_secret, FALSE);
+      curl_setopt($session, CURLOPT_HTTPHEADER, ["Authorization: USER:$userId:HMAC:$hmac"]);
+    }
+    elseif (!empty($servers[$serverId]['serverType']) && $servers[$serverId]['serverType'] === 'json_occurrences') {
+      $shared_secret = $servers[$serverId]['shared_secret'];
+      $userId = self::$clientUserId;
+      $time = round(microtime(TRUE) * 1000);
+      $authData = "$userId$time";
+      // Create the authentication HMAC.
+      $hmac = hash_hmac("sha1", $authData, $shared_secret, FALSE);
+      curl_setopt($session, CURLOPT_HTTPHEADER, ["Authorization: USER:$userId:TIME:$time:HMAC:$hmac"]);
     }
     // Do the request.
     $response = curl_exec($session);

From f81c13667761165239e60f30df9fc3e85761a493 Mon Sep 17 00:00:00 2001
From: Jim Bacon <jame2@ceh.ac.uk>
Date: Fri, 20 Aug 2021 17:01:51 +0100
Subject: [PATCH 48/93] Prototype for removing DbUnit dependency

---
 .../tests/services/data_cleanerTest.php       | 124 ++-
 modules/phpUnit/config/core_fixture.php       | 931 ++++++++++++++++++
 .../libraries/SimpleDatabaseTestCase.php      |  87 ++
 3 files changed, 1076 insertions(+), 66 deletions(-)
 create mode 100644 modules/phpUnit/config/core_fixture.php
 create mode 100644 modules/phpUnit/libraries/SimpleDatabaseTestCase.php

diff --git a/modules/data_cleaner/tests/services/data_cleanerTest.php b/modules/data_cleaner/tests/services/data_cleanerTest.php
index 8a5db2244b..b8e8c166a6 100644
--- a/modules/data_cleaner/tests/services/data_cleanerTest.php
+++ b/modules/data_cleaner/tests/services/data_cleanerTest.php
@@ -26,9 +26,6 @@
  * @subpackage Data Cleaner
  */
 
-use PHPUnit\DbUnit\DataSet\YamlDataSet as DbUDataSetYamlDataSet;
-use PHPUnit\DbUnit\DataSet\CompositeDataSet as DbUDataSetCompositeDataSet;
-
 require_once 'client_helpers/data_entry_helper.php';
 
 /**
@@ -36,7 +33,7 @@
  * @backupGlobals disabled
  * @backupStaticAttributes disabled
  */
-class Controllers_Services_Data_Cleaner_Test extends Indicia_DatabaseTestCase {
+class Controllers_Services_Data_Cleaner_Test extends SimpleDatabaseTestCase {
 
   protected $request;
 
@@ -44,74 +41,69 @@ class Controllers_Services_Data_Cleaner_Test extends Indicia_DatabaseTestCase {
    * @return PHPUnit_Extensions_Database_DataSet_IDataSet
    */
   public function getDataSet() {
-    $ds1 = new DbUDataSetYamlDataSet('modules/phpUnit/config/core_fixture.yaml');
+    require 'modules/phpUnit/config/core_fixture.php';
 
     // Create a rule to test against.
-    $ds2 = new Indicia_ArrayDataSet(
-      [
-        'verification_rules' => [
-          [
-            'title' => 'Test PeriodWithinYear rule',
-            'description' => 'Test rule for unit testing',
-            'test_type' => 'PeriodWithinYear',
-            'error_message' => 'PeriodWithinYear test failed',
-            'source_url' => NULL,
-            'source_filename' => NULL,
-            'created_on' => '2016-07-22:16:00:00',
-            'created_by_id' => 1,
-            'updated_on' => '2016-07-22:16:00:00',
-            'updated_by_id' => 1,
-            'reverse_rule' => 'F',
-          ],
+    $local_fixture = [
+      'verification_rules' => [
+        [
+          'title' => 'Test PeriodWithinYear rule',
+          'description' => 'Test rule for unit testing',
+          'test_type' => 'PeriodWithinYear',
+          'error_message' => 'PeriodWithinYear test failed',
+          'source_url' => NULL,
+          'source_filename' => NULL,
+          'created_on' => '2016-07-22 16:00:00',
+          'created_by_id' => 1,
+          'updated_on' => '2016-07-22 16:00:00',
+          'updated_by_id' => 1,
+          'reverse_rule' => 'F',
         ],
-        'verification_rule_metadata' => [
-          [
-            'verification_rule_id' => '1',
-            'key' => 'Tvk',
-            'value' => 'TESTKEY',
-            'created_on' => '2016-07-22:16:00:00',
-            'created_by_id' => 1,
-            'updated_on' => '2016-07-22:16:00:00',
-            'updated_by_id' => 1,
-          ],
-          [
-            'verification_rule_id' => '1',
-            'key' => 'StartDate',
-            'value' => '0801',
-            'created_on' => '2016-07-22:16:00:00',
-            'created_by_id' => 1,
-            'updated_on' => '2016-07-22:16:00:00',
-            'updated_by_id' => 1,
-          ],
-          [
-            'verification_rule_id' => '1',
-            'key' => 'EndDate',
-            'value' => '0831',
-            'created_on' => '2016-07-22:16:00:00',
-            'created_by_id' => 1,
-            'updated_on' => '2016-07-22:16:00:00',
-            'updated_by_id' => 1,
-          ],
+      ],
+      'verification_rule_metadata' => [
+        [
+          'verification_rule_id' => '1',
+          'key' => 'Tvk',
+          'value' => 'TESTKEY',
+          'created_on' => '2016-07-22 16:00:00',
+          'created_by_id' => 1,
+          'updated_on' => '2016-07-22 16:00:00',
+          'updated_by_id' => 1,
         ],
-        'cache_verification_rules_period_within_year' => [
-          [
-            'verification_rule_id' => '1',
-            'reverse_rule' => 'f',
-            'taxa_taxon_list_external_key' => 'TESTKEY',
-            'start_date' => '214',
-            'end_date' => '244',
-            'survey_id' => NULL,
-            'stages' => NULL,
-            'error_message' => 'PeriodWithinYear test failed',
-          ],
+        [
+          'verification_rule_id' => '1',
+          'key' => 'StartDate',
+          'value' => '0801',
+          'created_on' => '2016-07-22 16:00:00',
+          'created_by_id' => 1,
+          'updated_on' => '2016-07-22 16:00:00',
+          'updated_by_id' => 1,
         ],
-      ]
-    );
+        [
+          'verification_rule_id' => '1',
+          'key' => 'EndDate',
+          'value' => '0831',
+          'created_on' => '2016-07-22 16:00:00',
+          'created_by_id' => 1,
+          'updated_on' => '2016-07-22 16:00:00',
+          'updated_by_id' => 1,
+        ],
+      ],
+      'cache_verification_rules_period_within_year' => [
+        [
+          'verification_rule_id' => '1',
+          'reverse_rule' => 'f',
+          'taxa_taxon_list_external_key' => 'TESTKEY',
+          'start_date' => '214',
+          'end_date' => '244',
+          'survey_id' => NULL,
+          'stages' => NULL,
+          'error_message' => 'PeriodWithinYear test failed',
+        ],
+      ],
+    ];
 
-    $compositeDs = new DbUDataSetCompositeDataSet();
-    $compositeDs->addDataSet($ds1);
-    $compositeDs->addDataSet($ds2);
-    return $compositeDs;
+    return array_merge($core_fixture, $local_fixture);
   }
 
   public function setUp(): void {
diff --git a/modules/phpUnit/config/core_fixture.php b/modules/phpUnit/config/core_fixture.php
new file mode 100644
index 0000000000..c5c7fd2622
--- /dev/null
+++ b/modules/phpUnit/config/core_fixture.php
@@ -0,0 +1,931 @@
+<?php
+
+/**
+ * @file
+ * A standard database fixture definition.
+ *
+ * The core fixture sets up the Indicia database with a consistent set of
+ * test data in the core tables.
+ * Id values in tables having sequences are never supplied so that, if a test
+ * adds a record to a table, the sequence will supply it the next valid id.
+ */
+
+$core_fixture = [
+  "websites" => [
+    [
+      "title" => "Test website",
+      "description" => "Website for unit testing",
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+      "url" => "http:,//www.indicia.org.uk",
+      "password" => "password",
+      "verification_checks_enabled" => 'true',
+    ],
+  ],
+  "users_websites" => [
+    [
+      "user_id" => 1,
+      "website_id" => 1,
+      "site_role_id" => 1,
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+    ],
+  ],
+  "surveys" => [
+    [
+      "title" => "Test survey",
+      "description" => "Survey for unit testing",
+      "website_id" => 1,
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+    ],
+    [
+      "title" => "Test survey 2",
+      "description" => "Additional survey for unit testing",
+      "website_id" => 1,
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+    ],
+  ],
+  "taxon_meanings" => [
+    [
+    // No support for INSERT INTO table DEFAULT VALUES.
+    // Use high id values to avoid conflict with any values created by sequence
+    // during testing.
+      "id" => 10000,
+    ],
+    [
+      "id" => 10001,
+    ],
+  ],
+  "taxon_groups" => [
+    [
+      "title" => "Test taxon group",
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+    ],
+  ],
+  "taxon_ranks" => [
+    [
+      "rank" => "Genus",
+      "short_name" => "Genus",
+      "italicise_taxon" => "false",
+      "sort_order" => 290,
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+    ],
+    [
+      "rank" => "Species",
+      "short_name" => "Species",
+      "italicise_taxon" => "true",
+      "sort_order" => 300,
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+    ],
+  ],
+  "taxon_lists" => [
+    [
+      "title" => "Test taxon list",
+      "description" => "Taxon list for unit testing",
+      "website_id" => 1,
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+    ],
+  ],
+  "taxa" => [
+    [
+      "taxon" => "Test taxon",
+      "taxon_group_id" => 1,
+      "language_id" => 2,
+      "external_key" => "TESTKEY",
+      "taxon_rank_id" => 1,
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+    ],
+    [
+      "taxon" => "Test taxon 2",
+      "taxon_group_id" => 1,
+      "language_id" => 2,
+      "external_key" => "TESTKEY2",
+      "taxon_rank_id" => 2,
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+    ],
+  ],
+  "taxa_taxon_lists" => [
+    [
+      "taxon_list_id" => 1,
+      "taxon_id" => 1,
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "taxon_meaning_id" => 10000,
+      "taxonomic_sort_order" => 1,
+      "preferred" => "true",
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+    ],
+    [
+      "taxon_list_id" => 1,
+      "taxon_id" => 2,
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "taxon_meaning_id" => 10001,
+      "taxonomic_sort_order" => 1,
+      "preferred" => "true",
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+    ],
+  ],
+  "cache_taxa_taxon_lists" => [
+    [
+      "id" => 1,
+      "preferred" => true,
+      "taxon_list_id" => 1,
+      "taxon_list_title" => "Test taxa taxon list",
+      "website_id" => 1,
+      "preferred_taxa_taxon_list_id" => 1,
+      "taxonomic_sort_order" => 1,
+      "taxon" => "Test taxon",
+      "language_iso" => "lat",
+      "language" => "Latin",
+      "preferred_taxon" => "Test taxon",
+      "preferred_language_iso" => "lat",
+      "preferred_language" => "Latin",
+      "external_key" => "TESTKEY",
+      "taxon_rank" => "Genus",
+      "taxon_rank_sort_order" => "290",
+      "taxon_meaning_id" => 10000,
+      "taxon_group_id" => 1,
+      "taxon_group" => "Test taxon group",
+      "cache_created_on" => "2016-07-22 16:00:00",
+      "cache_updated_on" => "2016-07-22 16:00:00",
+    ],
+    [
+      "id" => 2,
+      "preferred" => true,
+      "taxon_list_id" => 1,
+      "taxon_list_title" => "Test taxa taxon list 2",
+      "website_id" => 1,
+      "preferred_taxa_taxon_list_id" => 2,
+      "taxonomic_sort_order" => 2,
+      "taxon" => "Test taxon 2",
+      "language_iso" => "lat",
+      "language" => "Latin",
+      "preferred_taxon" => "Test taxon",
+      "preferred_language_iso" => "lat",
+      "preferred_language" => "Latin",
+      "external_key" => "TESTKEY2",
+      "taxon_rank" => "Species",
+      "taxon_rank_sort_order" => "300",
+      "taxon_meaning_id" => 10001,
+      "taxon_group_id" => 1,
+      "taxon_group" => "Test taxon group",
+      "cache_created_on" => "2016-07-22 16:00:00",
+      "cache_updated_on" => "2016-07-22 16:00:00",
+    ],
+  ],
+  "cache_taxon_searchterms" => [
+    [
+      "taxa_taxon_list_id" => 1,
+      "taxon_list_id" => 1,
+      "searchterm" => "testtaxon",
+      "original" => "Test taxon",
+      "taxon_group" => "Test taxon group",
+      "taxon_meaning_id" => 10000,
+      "preferred_taxon" => "Test taxon",
+      "default_common_name" => "Test taxon",
+      "language_iso" => "lat",
+      "name_type" => "L",
+      "simplified" => "t",
+      "taxon_group_id" => 1,
+      "preferred" => "t",
+      "searchterm_length" => 9,
+      "preferred_taxa_taxon_list_id" => 1,
+      "external_key" => "TESTKEY",
+      "taxon_rank_sort_order" => "290",
+    ],
+    [
+      "id" => 2,
+      "taxa_taxon_list_id" => 1,
+      "taxon_list_id" => 1,
+      "searchterm" => "Test taxon",
+      "original" => "Test taxon",
+      "taxon_group" => "Test taxon group",
+      "taxon_meaning_id" => 10000,
+      "preferred_taxon" => "Test taxon",
+      "default_common_name" => "Test taxon",
+      "language_iso" => "lat",
+      "name_type" => "L",
+      "simplified" => "f",
+      "taxon_group_id" => 1,
+      "preferred" => "t",
+      "searchterm_length" => 10,
+      "preferred_taxa_taxon_list_id" => 1,
+      "external_key" => "TESTKEY",
+      "taxon_rank_sort_order" => "290",
+    ],
+    [
+      "id" => 3,
+      "taxa_taxon_list_id" => 2,
+      "taxon_list_id" => 1,
+      "searchterm" => "testtaxon2",
+      "original" => "Test taxon 2",
+      "taxon_group" => "Test taxon group",
+      "taxon_meaning_id" => 10001,
+      "preferred_taxon" => "Test taxon 2",
+      "default_common_name" => "Test taxon 2",
+      "language_iso" => "lat",
+      "name_type" => "L",
+      "simplified" => "t",
+      "taxon_group_id" => 1,
+      "preferred" => "t",
+      "searchterm_length" => 10,
+      "preferred_taxa_taxon_list_id" => 1,
+      "external_key" => "TESTKEY2",
+      "taxon_rank_sort_order" => "300",
+    ],
+    [
+      "id" => 4,
+      "taxa_taxon_list_id" => 2,
+      "taxon_list_id" => 1,
+      "searchterm" => "Test taxon 2",
+      "original" => "Test taxon 2",
+      "taxon_group" => "Test taxon group",
+      "taxon_meaning_id" => 10001,
+      "preferred_taxon" => "Test taxon 2",
+      "default_common_name" => "Test taxon 2",
+      "language_iso" => "lat",
+      "name_type" => "L",
+      "simplified" => "f",
+      "taxon_group_id" => 1,
+      "preferred" => "t",
+      "searchterm_length" => 12,
+      "preferred_taxa_taxon_list_id" => 1,
+      "external_key" => "TESTKEY2",
+      "taxon_rank_sort_order" => "300",
+    ],
+  ],
+  "meanings" => [
+    // No support for INSERT INTO table DEFAULT VALUES.
+    // Use high id values to avoid conflict with any values created by sequence
+    // during testing.
+    [
+      "id" => 10000,
+    ],
+    [
+      "id" => 10001,
+    ],
+    [
+      "id" => 10002,
+    ],
+    [
+      "id" => 10003,
+    ],
+    [
+      "id" => 10004,
+    ],
+    [
+      "id" => 10005,
+    ],
+    [
+      "id" => 10006,
+    ],
+  ],
+  "termlists" => [
+    [
+      "title" => "Test term list",
+      "description" => "Term list list for unit testing",
+      "website_id" => 1,
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+      "external_key" => "TESTKEY",
+    ],
+    [
+      "title" => "Location types",
+      "description" => "Term list for location types",
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+      "external_key" => "indicia:location_types",
+    ],
+    [
+      "title" => "Sample methods",
+      "description" => "Term list for sample methods",
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+      "external_key" => "indicia:sample_methods",
+    ],
+    [
+      "title" => "User identifier types",
+      "description" => "Term list for user identifier types",
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+      "external_key" => "indicia:user_identifier_types",
+    ],
+    [
+      "title" => "Group types",
+      "description" => "Term list for group types",
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+      "external_key" => "indicia:group_types",
+    ],
+    [
+      "title" => "Media types",
+      "description" => "Term list for media types",
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+      "external_key" => "indicia:media_types",
+    ],
+  ],
+
+  "terms" => [
+    [
+      "term" => "Test term",
+      "language_id" => 1,
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+    ],
+    [
+      "term" => "Test location type",
+      "language_id" => 1,
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+    ],
+    [
+      "term" => "Test sample method",
+      "language_id" => 1,
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+    ],
+    [
+      "term" => "email",
+      "language_id" => 1,
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+    ],
+    [
+      "term" => "twitter",
+      "language_id" => 1,
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+    ],
+    [
+      "term" => "Test group type",
+      "language_id" => 1,
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+    ],
+    [
+      "term" => "Image:Local",
+      "language_id" => 1,
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+    ],
+  ],
+  "termlists_terms" => [
+    [
+      // Test term list.
+      "termlist_id" => 1,
+      // Test term.
+      "term_id" => 1,
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+      "meaning_id" => 10000,
+      "preferred" => "true",
+      "sort_order" => 1,
+    ],
+    [
+      // Location types.
+      "termlist_id" => 2,
+      // Test location type.
+      "term_id" => 2,
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+      "meaning_id" => 10001,
+      "preferred" => "true",
+      "sort_order" => 1,
+    ],
+    [
+      // Sample methods.
+      "termlist_id" => 3,
+      // Test sample method.
+      "term_id" => 3,
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+      "meaning_id" => 10002,
+      "preferred" => "true",
+      "sort_order" => 1,
+    ],
+    [
+      // User identifier types.
+      "termlist_id" => 4,
+      // Email.
+      "term_id" => 4,
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+      "meaning_id" => 10003,
+      "preferred" => "true",
+      "sort_order" => 1,
+    ],
+    [
+      // User identifier types.
+      "termlist_id" => 4,
+      // Twitter.
+      "term_id" => 5,
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+      "meaning_id" => 10004,
+      "preferred" => "true",
+      "sort_order" => 2,
+    ],
+    [
+      // Group types.
+      "termlist_id" => 5,
+      // Test group type.
+      "term_id" => 6,
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+      "meaning_id" => 10005,
+      "preferred" => "true",
+      "sort_order" => 2,
+    ],
+    [
+      // Media types.
+      "termlist_id" => 6,
+      // Image:Local.
+      "term_id" => 7,
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+      "meaning_id" => 10005,
+      "preferred" => "true",
+      "sort_order" => 1,
+    ],
+  ],
+  "cache_termlists_terms" => [
+    [
+      "id" => 1,
+      "preferred" => "true",
+      "termlist_id" => 1,
+      "termlist_title" => "Test term list",
+      "website_id" => 1,
+      "preferred_termlists_term_id" => 1,
+      "sort_order" => 1,
+      "term" => "Test term",
+      "language_iso" => "eng",
+      "language" => "English",
+      "preferred_term" => "Test term",
+      "preferred_language_iso" => "eng",
+      "preferred_language" => "English",
+      "meaning_id" => 10000,
+      "cache_created_on" => "2016-07-22 16:00:00",
+      "cache_updated_on" => "2016-07-22 16:00:00",
+    ],
+    [
+      "id" => 2,
+      "preferred" => "true",
+      "termlist_id" => 2,
+      "termlist_title" => "Location types",
+      "website_id" => 1,
+      "preferred_termlists_term_id" => 2,
+      "sort_order" => 1,
+      "term" => "Test location type",
+      "language_iso" => "eng",
+      "language" => "English",
+      "preferred_term" => "Test location type",
+      "preferred_language_iso" => "eng",
+      "preferred_language" => "English",
+      "meaning_id" => 10001,
+      "cache_created_on" => "2016-07-22 16:00:00",
+      "cache_updated_on" => "2016-07-22 16:00:00",
+    ],
+    [
+      "id" => 3,
+      "preferred" => "true",
+      "termlist_id" => 3,
+      "termlist_title" => "Sample methods",
+      "website_id" => 1,
+      "preferred_termlists_term_id" => 3,
+      "sort_order" => 1,
+      "term" => "Test sample method",
+      "language_iso" => "eng",
+      "language" => "English",
+      "preferred_term" => "Test term",
+      "preferred_language_iso" => "eng",
+      "preferred_language" => "English",
+      "meaning_id" => 10002,
+      "cache_created_on" => "2016-07-22 16:00:00",
+      "cache_updated_on" => "2016-07-22 16:00:00",
+    ],
+    [
+      "id" => 4,
+      "preferred" => "true",
+      "termlist_id" => 4,
+      "termlist_title" => "User identifier types",
+      "website_id" => 1,
+      "preferred_termlists_term_id" => 4,
+      "sort_order" => 1,
+      "term" => "email",
+      "language_iso" => "eng",
+      "language" => "English",
+      "preferred_term" => "email",
+      "preferred_language_iso" => "eng",
+      "preferred_language" => "English",
+      "meaning_id" => 10003,
+      "cache_created_on" => "2016-07-22 16:00:00",
+      "cache_updated_on" => "2016-07-22 16:00:00",
+    ],
+    [
+      "id" => 5,
+      "preferred" => "true",
+      "termlist_id" => 4,
+      "termlist_title" => "User identifier types",
+      "website_id" => 1,
+      "preferred_termlists_term_id" => 5,
+      "sort_order" => 2,
+      "term" => "twitter",
+      "language_iso" => "eng",
+      "language" => "English",
+      "preferred_term" => "twitter",
+      "preferred_language_iso" => "eng",
+      "preferred_language" => "English",
+      "meaning_id" => 10004,
+      "cache_created_on" => "2016-07-22 16:00:00",
+      "cache_updated_on" => "2016-07-22 16:00:00",
+    ],
+    [
+      "id" => 6,
+      "preferred" => "true",
+      "termlist_id" => 5,
+      "termlist_title" => "Group types",
+      "website_id" => 1,
+      "preferred_termlists_term_id" => 6,
+      "sort_order" => 1,
+      "term" => "Test group type",
+      "language_iso" => "eng",
+      "language" => "English",
+      "preferred_term" => "Test group type",
+      "preferred_language_iso" => "eng",
+      "preferred_language" => "English",
+      "meaning_id" => 10005,
+      "cache_created_on" => "2016-07-22 16:00:00",
+      "cache_updated_on" => "2016-07-22 16:00:00",
+    ],
+    [
+      "id" => 7,
+      "preferred" => "true",
+      "termlist_id" => 6,
+      "termlist_title" => "Media types",
+      "website_id" => 1,
+      "preferred_termlists_term_id" => 7,
+      "sort_order" => 1,
+      "term" => "Image:Local",
+      "language_iso" => "eng",
+      "language" => "English",
+      "preferred_term" => "Image:Local",
+      "preferred_language_iso" => "eng",
+      "preferred_language" => "English",
+      "meaning_id" => 10006,
+      "cache_created_on" => "2016-07-22 16:00:00",
+      "cache_updated_on" => "2016-07-22 16:00:00",
+    ],
+  ],
+  "samples" => [
+    [
+      "survey_id" => 1,
+      "date_start" => "2016-07-22",
+      "date_end" => "2016-07-22",
+      "date_type" => "D",
+      "entered_sref" => "SU01",
+      "entered_sref_system" => "OSGB",
+      "comment" => "Sample for unit testing with a \" double quote",
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+      "recorder_names" => "PHPUnit",
+      "record_status" => "C",
+    ],
+    [
+      "survey_id" => 1,
+      "date_start" => "2016-07-22",
+      "date_end" => "2016-07-22",
+      "date_type" => "D",
+      "entered_sref" => "SU01",
+      "entered_sref_system" => "OSGB",
+      "comment" => "Sample for unit testing with a \nline break",
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+      "recorder_names" => "PHPUnit",
+      "record_status" => "C",
+    ],
+  ],
+  "map_squares" => [
+    [
+      "geom" => "010300002031BF0D0001000000050000004E9C282E3C320BC18FC3A8120A2F59412CCEAACFA94309C1E75E7B57062F5941A7DE4D3BB84209C11FD1A351893E5941BBB729FE3E320BC1D8E4B8118D3E59414E9C282E3C320BC18FC3A8120A2F5941",
+      "x" => -214871,
+      "y" => 6609705,
+      "size" => 10000,
+    ],
+  ],
+  "occurrences" => [
+    [
+      "sample_id" => 1,
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+      "website_id" => 1,
+      "comment" => "Occurrence for unit testing",
+      "taxa_taxon_list_id" => 1,
+      "record_status" => "C",
+      "release_status" => "R",
+      "confidential" => "f",
+    ],
+    [
+      "sample_id" => 1,
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+      "website_id" => 1,
+      "comment" => "Confidential occurrence for unit testing",
+      "taxa_taxon_list_id" => 1,
+      "record_status" => "C",
+      "release_status" => "R",
+      "confidential" => "t",
+    ],
+  ],
+  "cache_occurrences_functional" => [
+    [
+      "id" => 1,
+      "sample_id" => 1,
+      "website_id" => 1,
+      "survey_id" => 1,
+      "date_start" => "2016-07-22",
+      "date_end" => "2016-07-22",
+      "date_type" => "D",
+      "created_on" => "2016-07-22 16:00:00",
+      "updated_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "taxa_taxon_list_id" => 1,
+      "preferred_taxa_taxon_list_id" => 1,
+      "taxon_meaning_id" => 10000,
+      "taxa_taxon_list_external_key" => "TESTKEY",
+      "taxon_group_id" => 1,
+      "record_status" => "C",
+      "release_status" => "R",
+      "zero_abundance" => "f",
+      "confidential" => "f",
+      "map_sq_1km_id" => 1,
+      "map_sq_2km_id" => 1,
+      "map_sq_10km_id" => 1,
+      "verification_checks_enabled" => "true",
+    ],
+    [
+      "id" => 2,
+      "sample_id" => 1,
+      "website_id" => 1,
+      "survey_id" => 1,
+      "date_start" => "2016-07-22",
+      "date_end" => "2016-07-22",
+      "date_type" => "D",
+      "created_on" => "2016-07-22 16:00:00",
+      "updated_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "taxa_taxon_list_id" => 1,
+      "preferred_taxa_taxon_list_id" => 1,
+      "taxon_meaning_id" => 10000,
+      "taxa_taxon_list_external_key" => "TESTKEY",
+      "taxon_group_id" => 1,
+      "record_status" => "C",
+      "release_status" => "R",
+      "zero_abundance" => "f",
+      "confidential" => "t",
+      "map_sq_1km_id" => 1,
+      "map_sq_2km_id" => 1,
+      "map_sq_10km_id" => 1,
+      "verification_checks_enabled" => "true",
+    ],
+  ],
+  "cache_occurrences_nonfunctional" => [
+    [
+      "id" => 1,
+    ],
+    [
+      "id" => 2,
+    ],
+  ],
+  "cache_samples_functional" => [
+    [
+      "id" => 1,
+      "website_id" => 1,
+      "survey_id" => 1,
+      "date_start" => "2016-07-22",
+      "date_end" => "2016-07-22",
+      "date_type" => "D",
+      "created_on" => "2016-07-22 16:00:00",
+      "updated_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "record_status" => "C",
+      "map_sq_1km_id" => 1,
+      "map_sq_2km_id" => 1,
+      "map_sq_10km_id" => 1,
+    ],
+  ],
+  "cache_samples_nonfunctional" => [
+    [
+      "id" => 1,
+      "website_title" => "Test website",
+      "survey_title" => "Test survey",
+      "public_entered_sref" => "SU01",
+      "entered_sref_system" => "OSGB",
+      "recorders" => "PHPUnit",
+    ],
+  ],
+  "sample_attributes" => [
+    [
+      "caption" => "Altitude",
+      "data_type" => "I",
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+      "public" => "false",
+    ],
+  ],
+  "sample_attributes_websites" => [
+    [
+      "website_id" => 1,
+      "sample_attribute_id" => 1,
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "restrict_to_survey_id" => 1,
+    ],
+  ],
+  "occurrence_attributes" => [
+    [
+      "caption" => "Identified_by",
+      "data_type" => "T",
+      "public" => "false",
+      "system_function" => "det_full_name",
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+    ],
+  ],
+  "occurrence_attributes_websites" => [
+    [
+      "website_id" => 1,
+      "occurrence_attribute_id" => 1,
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "restrict_to_survey_id" => 1,
+    ],
+  ],
+  "locations" => [
+    [
+      "name" => "Test location",
+      "centroid_sref" => "SU01",
+      "centroid_sref_system" => "OSGB",
+      "location_type_id" => "2",
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+      "public" => "true",
+    ],
+  ],
+  "location_attributes" => [
+    [
+      "caption" => "Test text",
+      "data_type" => "T",
+      "public" => "false",
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+    ],
+    [
+      "caption" => "Test lookup",
+      "data_type" => "L",
+      "termlist_id" => 1,
+      "public" => "false",
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+    ],
+    [
+      "caption" => "Test integer",
+      "data_type" => "I",
+      "termlist_id" => 1,
+      "public" => "false",
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+    ],
+  ],
+  "location_attributes_websites" => [
+    [
+      // Test website.
+      "website_id" => 1,
+      // Test text.
+      "location_attribute_id" => 1,
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+    ],
+    [
+      // Test website.
+      "website_id" => 1,
+      // Test lookup.
+      "location_attribute_id" => 2,
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+    ],
+    [
+      // Test website.
+      "website_id" => 1,
+      // Test integer.
+      "location_attribute_id" => 3,
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+    ],
+  ],
+  "location_attribute_values" => [
+    [
+      // Test location.
+      "location_id" => 1,
+      // Test lookup.
+      "location_attribute_id" => 2,
+      // Test term.
+      "int_value" => 1,
+      "created_on" => "2016-07-22 16:00:00",
+      "created_by_id" => 1,
+      "updated_on" => "2016-07-22 16:00:00",
+      "updated_by_id" => 1,
+    ],
+  ],
+];
diff --git a/modules/phpUnit/libraries/SimpleDatabaseTestCase.php b/modules/phpUnit/libraries/SimpleDatabaseTestCase.php
new file mode 100644
index 0000000000..0e3dfa5692
--- /dev/null
+++ b/modules/phpUnit/libraries/SimpleDatabaseTestCase.php
@@ -0,0 +1,87 @@
+<?php
+
+use PHPUnit\Framework\TestCase;
+
+/**
+ * Base class for tests that interact with the database.
+ *
+ * Created as a replacement for Indicia_DatabaseTestCase which depends upon
+ * the now obsolete DbUnit. Allows the set up of fixtures from arrays
+ * containing data for records in tables.
+ */
+class SimpleDatabaseTestCase extends TestCase {
+  /**
+   * One database connection shared by all tests.
+   *
+   * @var conn
+   */
+  public static $conn = NULL;
+
+  /**
+   * Set up function called once before first test.
+   */
+  public static function setUpBeforeClass(): void {
+    $host = 'host = 127.0.0.1';
+    $dbname = 'dbname = indicia';
+    $user = 'user = indicia_user';
+    $pass = 'password = indicia_user_pass';
+    self::$conn = pg_connect("$host $dbname $user $pass");
+  }
+
+  /**
+   * Set up function called before every test.
+   */
+  protected function setUp(): void {
+    // Load base database fixture which does not change.
+    $fixture = $this->getDataSet();
+    self::setupFixture($fixture);
+  }
+
+  /**
+   * Set up database fixture defined in file.
+   */
+  private static function setupFixture($fixture) {
+
+    // Remove any pre-existing data in fixture tables.
+    self::deleteFixture($fixture);
+
+    // Set up fixture.
+    foreach ($fixture as $table => $records) {
+      foreach ($records as $record) {
+        if (!pg_insert(self::$conn, 'indicia.' . $table, $record)) {
+          throw new Exception("Failed inserting $record into $table");
+        }
+      }
+    }
+
+    // Fixture files may also contain the expected form of the survey export
+    // for the configuration established by the fixture.
+    if (isset($export)) {
+      return $export;
+    }
+  }
+
+  /**
+   * Delete database fixture.
+   */
+  private static function deleteFixture($fixture) {
+    // Drop content in fixture tables.
+    $tables = array_keys($fixture);
+    $table_list = implode(',', $tables);
+    pg_query(self::$conn, "TRUNCATE TABLE $table_list CASCADE");
+    // And reset primary key sequences.
+    foreach ($tables as $table) {
+      if (substr($table, 0, 6) !== 'cache_') {
+        // Cache tables don't have their own sequences.
+        $seq = $table . '_id_seq';
+        pg_query(self::$conn, "SELECT setval('$seq', 1, false)");
+      }
+    }
+
+    // Clear the cache to ensure tests use new database contents.
+    $cache = Cache::instance();
+    $cache->delete_all();
+
+  }
+
+}

From 2d093a59260881fbe897d99835f0ce8bd24913d9 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Tue, 7 Sep 2021 11:38:48 +0100
Subject: [PATCH 49/93] Initial drop for syncing annotations from another
 server

---
 modules/rest_api_sync/helpers/api_persist.php |  13 +-
 .../rest_api_sync_json_annotations.php        | 159 ++++++++++++++++++
 .../rest_api_sync_json_occurrences.php        |   2 +-
 3 files changed, 170 insertions(+), 4 deletions(-)
 create mode 100644 modules/rest_api_sync/helpers/rest_api_sync_json_annotations.php

diff --git a/modules/rest_api_sync/helpers/api_persist.php b/modules/rest_api_sync/helpers/api_persist.php
index ca835e1397..aa68f22bea 100644
--- a/modules/rest_api_sync/helpers/api_persist.php
+++ b/modules/rest_api_sync/helpers/api_persist.php
@@ -193,7 +193,7 @@ public static function annotation($db, array $annotation, $survey_id) {
     // Set up a values array for the annotation post.
     $values = self::getAnnotationValues($db, $annotation);
     // Link to the existing observation.
-    $existingObs = self::findExistingObservation($db, $annotation['taxonObservation']['id'], $survey_id);
+    $existingObs = self::findExistingObservation($db, $annotation['occurrenceID'], $survey_id);
     if (!count($existingObs)) {
       // @todo Proper error handling as annotation can't be imported. Perhaps should obtain
       // and import the observation via the API?
@@ -397,7 +397,7 @@ private static function getMediaTypeId($db, $mediaType) {
    *   Values array to use for submission building.
    */
   private static function getAnnotationValues($db, array $annotation) {
-    return array(
+    $values = [
       'occurrence_comment:comment' => $annotation['comment'],
       'occurrence_comment:email_address' => self::valueOrNull($annotation, 'emailAddress'),
       'occurrence_comment:record_status' => self::valueOrNull($annotation, 'record_status'),
@@ -405,7 +405,14 @@ private static function getAnnotationValues($db, array $annotation) {
       'occurrence_comment:query' => $annotation['question'],
       'occurrence_comment:person_name' => $annotation['authorName'],
       'occurrence_comment:external_key' => $annotation['id'],
-    );
+    ];
+    if (!empty($annotation['dateTime'])) {
+      $r['updated_on'] = $annotation['dateTime'];
+    }
+    if (!empty($annotation['identificationVerificationStatus'])) {
+      self::applyIdentificationVerificationStatus($annotation['identificationVerificationStatus'], $values);
+    }
+    return $values;
   }
 
   /**
diff --git a/modules/rest_api_sync/helpers/rest_api_sync_json_annotations.php b/modules/rest_api_sync/helpers/rest_api_sync_json_annotations.php
new file mode 100644
index 0000000000..3271157123
--- /dev/null
+++ b/modules/rest_api_sync/helpers/rest_api_sync_json_annotations.php
@@ -0,0 +1,159 @@
+<?php
+
+/**
+ * @file
+ * Helper class for synchronising records from an iNaturalist server.
+ *
+ * Indicia, the OPAL Online Recording Toolkit.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * any later version.
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see http://www.gnu.org/licenses/gpl.html.
+ *
+ * @author Indicia Team
+ * @license http://www.gnu.org/licenses/gpl.html GPL
+ * @link https://github.com/indicia-team/warehouse/
+ */
+
+defined('SYSPATH') or die('No direct script access.');
+
+define('MAX_PAGES', 1);
+
+/**
+ * Helper class for syncing to the JSON annotations API of another server.
+ *
+ * Could be an Indicia warehouse, or another server implementing the same
+ * standard.
+ */
+class rest_api_sync_json_annotations {
+
+  /**
+   * Synchronise a set of data loaded from the other server.
+   *
+   * @param string $serverId
+   *   ID of the server as defined in the configuration.
+   * @param array $server
+   *   Server configuration.
+   */
+  public static function syncServer($serverId, array $server) {
+    // Count of pages done in this run.
+    $pageCount = 0;
+    // If last run still going, not on first page.
+    $firstPage = !variable::get("rest_api_sync_{$serverId}_next_run");
+    if ($firstPage) {
+      // Track when we started this run, so the next run can pick up all
+      // changes.
+      $timestampAtStart = date('c');
+      variable::set("rest_api_sync_{$serverId}_next_run", $timestampAtStart);
+    }
+    do {
+      $syncStatus = self::syncPage($serverId, $server);
+      $pageCount++;
+      ob_flush();
+    } while ($syncStatus['moreToDo'] && $pageCount < MAX_PAGES);
+    if (!$syncStatus['moreToDo']) {
+      variable::set("rest_api_sync_{$serverId}_last_run", variable::get("rest_api_sync_{$serverId}_next_run"));
+      variable::delete("rest_api_sync_{$serverId}_next_run");
+      variable::delete("rest_api_sync_{$serverId}_last_id");
+    }
+  }
+
+  public static function loadControlledTerms() {
+  }
+
+  /**
+   * Synchronise a single page of data loaded from the server.
+   *
+   * @param string $serverId
+   *   ID of the server as defined in the configuration.
+   * @param array $server
+   *   Server configuration.
+   *
+   * @return array
+   *   Status info.
+   */
+  public static function syncPage($serverId, array $server) {
+    $db = Database::instance();
+    $nextPage = variable::get("rest_api_sync_{$serverId}_next_page", [], FALSE);
+    $data = rest_api_sync::getDataFromRestUrl(
+      "$server[url]?" . http_build_query($nextPage),
+      $serverId
+    );
+    $tracker = ['inserts' => 0, 'updates' => 0, 'errors' => 0];
+    foreach ($data['data'] as $annotation) {
+      // @todo Make sure all fields in specification are handled.
+      try {
+        $annotation = [
+          'id' => $annotation['annotationID'],
+          'occurrenceID' => $annotation['occurrenceID'],
+          'comment' => empty($annotation['record-level']['comment']) ? NULL : $annotation['record-level']['comment'],
+          'identificationVerificationStatus' => empty($annotation['identificationVerificationStatus']) ? NULL : $annotation['identificationVerificationStatus'],
+          'question' => empty($annotation['question']) ? NULL : $annotation['question'],
+          'authorName' => empty($annotation['authorName']) ? 'Unknown' : $annotation['authorName'],
+          'dateTime' => $annotation['dateTime'],
+        ];
+
+        $is_new = api_persist::annotation(
+          $db,
+          $annotation,
+          $server['survey_id']
+        );
+        if ($is_new !== NULL) {
+          $tracker[$is_new ? 'inserts' : 'updates']++;
+        }
+        $db->query("UPDATE rest_api_sync_skipped_records SET current=false " .
+          "WHERE server_id='$serverId' AND source_id='{$annotation['annotation']['annotationID']}' AND dest_table='occurrence_comments'");
+      }
+      catch (exception $e) {
+        rest_api_sync::log(
+          'error',
+          "Error occurred submitting an annotation with ID {$annotation['annotation']['annotationID']}\n" . $e->getMessage(),
+          $tracker
+        );
+        $msg = pg_escape_string($e->getMessage());
+        $createdById = isset($_SESSION['auth_user']) ? $_SESSION['auth_user']->id : 1;
+        $sql = <<<QRY
+INSERT INTO rest_api_sync_skipped_records (
+  server_id,
+  source_id,
+  dest_table,
+  error_message,
+  current,
+  created_on,
+  created_by_id
+)
+VALUES (
+  '$serverId',
+  '{$annotation['annotation']['annotationID']}',
+  'occurrence_comments',
+  '$msg',
+  true,
+  now(),
+  $createdById
+)
+QRY;
+        $db->query($sql);
+      }
+    }
+    variable::set("rest_api_sync_{$serverId}_next_page", $data['paging']['next']);
+    rest_api_sync::log(
+      'info',
+      "<strong>Annotations</strong><br/>Inserts: $tracker[inserts]. Updates: $tracker[updates]. Errors: $tracker[errors]"
+    );
+    $r = [
+      'moreToDo' => count($data['data']) > 0,
+      // No way of determining the following.
+      'pagesToGo' => NULL,
+      'recordsToGo' => NULL,
+    ];
+    return $r;
+  }
+
+}
diff --git a/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php b/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php
index e7cfb77d23..c6d796047d 100644
--- a/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php
+++ b/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php
@@ -35,7 +35,7 @@
 class rest_api_sync_json_occurrences {
 
   /**
-   * Synchronise a set of data loaded from the iNat server.
+   * Synchronise a set of data loaded from the other server.
    *
    * @param string $serverId
    *   ID of the server as defined in the configuration.

From 6c1eaf29587b97e45b7ca56870688bf5d609b39e Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Tue, 7 Sep 2021 12:09:47 +0100
Subject: [PATCH 50/93] Annotations sync bugfixes

---
 .../rest_api_sync/helpers/rest_api_sync.php   |  5 +++-
 .../rest_api_sync_json_annotations.php        | 24 +++++++++----------
 .../rest_api_sync_json_occurrences.php        |  2 --
 3 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/modules/rest_api_sync/helpers/rest_api_sync.php b/modules/rest_api_sync/helpers/rest_api_sync.php
index d1a09c9457..8a8fc4cc84 100644
--- a/modules/rest_api_sync/helpers/rest_api_sync.php
+++ b/modules/rest_api_sync/helpers/rest_api_sync.php
@@ -21,6 +21,8 @@
 
  defined('SYSPATH') or die('No direct script access.');
 
+ define('MAX_PAGES', 1);
+
 /**
  * Helper class for syncing to RESTful APIs.
  */
@@ -55,7 +57,8 @@ public static function getDataFromRestUrl($url, $serverId) {
       $hmac = hash_hmac("sha1", $url, $shared_secret, FALSE);
       curl_setopt($session, CURLOPT_HTTPHEADER, ["Authorization: USER:$userId:HMAC:$hmac"]);
     }
-    elseif (!empty($servers[$serverId]['serverType']) && $servers[$serverId]['serverType'] === 'json_occurrences') {
+    elseif (!empty($servers[$serverId]['serverType']) && substr($servers[$serverId]['serverType'], 0, 5) === 'json_') {
+      // All JSON servers use same HMAC authentication.
       $shared_secret = $servers[$serverId]['shared_secret'];
       $userId = self::$clientUserId;
       $time = round(microtime(TRUE) * 1000);
diff --git a/modules/rest_api_sync/helpers/rest_api_sync_json_annotations.php b/modules/rest_api_sync/helpers/rest_api_sync_json_annotations.php
index 3271157123..8da9a383c5 100644
--- a/modules/rest_api_sync/helpers/rest_api_sync_json_annotations.php
+++ b/modules/rest_api_sync/helpers/rest_api_sync_json_annotations.php
@@ -24,8 +24,6 @@
 
 defined('SYSPATH') or die('No direct script access.');
 
-define('MAX_PAGES', 1);
-
 /**
  * Helper class for syncing to the JSON annotations API of another server.
  *
@@ -87,17 +85,17 @@ public static function syncPage($serverId, array $server) {
       $serverId
     );
     $tracker = ['inserts' => 0, 'updates' => 0, 'errors' => 0];
-    foreach ($data['data'] as $annotation) {
+    foreach ($data['data'] as $record) {
       // @todo Make sure all fields in specification are handled.
       try {
         $annotation = [
-          'id' => $annotation['annotationID'],
-          'occurrenceID' => $annotation['occurrenceID'],
-          'comment' => empty($annotation['record-level']['comment']) ? NULL : $annotation['record-level']['comment'],
-          'identificationVerificationStatus' => empty($annotation['identificationVerificationStatus']) ? NULL : $annotation['identificationVerificationStatus'],
-          'question' => empty($annotation['question']) ? NULL : $annotation['question'],
-          'authorName' => empty($annotation['authorName']) ? 'Unknown' : $annotation['authorName'],
-          'dateTime' => $annotation['dateTime'],
+          'id' => $record['annotationID'],
+          'occurrenceID' => $record['occurrenceID'],
+          'comment' => empty($record['record-level']['comment']) ? NULL : $record['record-level']['comment'],
+          'identificationVerificationStatus' => empty($record['identificationVerificationStatus']) ? NULL : $record['identificationVerificationStatus'],
+          'question' => empty($record['question']) ? NULL : $record['question'],
+          'authorName' => empty($record['authorName']) ? 'Unknown' : $record['authorName'],
+          'dateTime' => $record['dateTime'],
         ];
 
         $is_new = api_persist::annotation(
@@ -109,12 +107,12 @@ public static function syncPage($serverId, array $server) {
           $tracker[$is_new ? 'inserts' : 'updates']++;
         }
         $db->query("UPDATE rest_api_sync_skipped_records SET current=false " .
-          "WHERE server_id='$serverId' AND source_id='{$annotation['annotation']['annotationID']}' AND dest_table='occurrence_comments'");
+          "WHERE server_id='$serverId' AND source_id='$annotation[id]' AND dest_table='occurrence_comments'");
       }
       catch (exception $e) {
         rest_api_sync::log(
           'error',
-          "Error occurred submitting an annotation with ID {$annotation['annotation']['annotationID']}\n" . $e->getMessage(),
+          "Error occurred submitting an annotation with ID $annotation[id]\n" . $e->getMessage(),
           $tracker
         );
         $msg = pg_escape_string($e->getMessage());
@@ -131,7 +129,7 @@ public static function syncPage($serverId, array $server) {
 )
 VALUES (
   '$serverId',
-  '{$annotation['annotation']['annotationID']}',
+  '$annotation[id]',
   'occurrence_comments',
   '$msg',
   true,
diff --git a/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php b/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php
index c6d796047d..4af0a7a331 100644
--- a/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php
+++ b/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php
@@ -24,8 +24,6 @@
 
 defined('SYSPATH') or die('No direct script access.');
 
-define('MAX_PAGES', 1);
-
 /**
  * Helper class for syncing to the JSON occurrences API of another server.
  *

From 93041be6ce31e81e6730057bc9807fe6b5d14499 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Tue, 7 Sep 2021 12:17:16 +0100
Subject: [PATCH 51/93] Field name was incorrect

---
 .../rest_api_sync/helpers/rest_api_sync_json_annotations.php    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/rest_api_sync/helpers/rest_api_sync_json_annotations.php b/modules/rest_api_sync/helpers/rest_api_sync_json_annotations.php
index 8da9a383c5..27b58c0abd 100644
--- a/modules/rest_api_sync/helpers/rest_api_sync_json_annotations.php
+++ b/modules/rest_api_sync/helpers/rest_api_sync_json_annotations.php
@@ -89,7 +89,7 @@ public static function syncPage($serverId, array $server) {
       // @todo Make sure all fields in specification are handled.
       try {
         $annotation = [
-          'id' => $record['annotationID'],
+          'id' => $record['id'],
           'occurrenceID' => $record['occurrenceID'],
           'comment' => empty($record['record-level']['comment']) ? NULL : $record['record-level']['comment'],
           'identificationVerificationStatus' => empty($record['identificationVerificationStatus']) ? NULL : $record['identificationVerificationStatus'],

From cf48fdbe00f06d838eab0829c8ad3c6c9702c4ad Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Tue, 7 Sep 2021 12:54:13 +0100
Subject: [PATCH 52/93] Tidied code for updating occurrence

After an annotation added.
---
 modules/rest_api_sync/helpers/api_persist.php | 121 +++++++-----------
 .../rest_api_sync_json_annotations.php        |   2 +-
 2 files changed, 44 insertions(+), 79 deletions(-)

diff --git a/modules/rest_api_sync/helpers/api_persist.php b/modules/rest_api_sync/helpers/api_persist.php
index aa68f22bea..1b7e6c99d0 100644
--- a/modules/rest_api_sync/helpers/api_persist.php
+++ b/modules/rest_api_sync/helpers/api_persist.php
@@ -208,7 +208,7 @@ public static function annotation($db, array $annotation, $survey_id) {
     $annotationObj = ORM::factory('occurrence_comment');
     $annotationObj->set_submission_data($values);
     $annotationObj->submit();
-    self::updateObservationWithAnnotationDetails($db, $existingObs[0]['id'], $annotation);
+    self::updateObservationWithAnnotationDetails($db, $existingObs[0]['id'], $annotation, $values);
     if (count($annotationObj->getAllErrors()) !== 0) {
       throw new exception("Error occurred submitting an annotation\n" . kohana::debug($annotationObj->getAllErrors()));
     }
@@ -859,88 +859,53 @@ private static function mapRecordStatus(array &$annotation) {
    *   ID of the associated occurrence record in the database.
    * @param array $annotation
    *   Annotation object loaded from the REST API.
+   * @param array $values
+   *   Database values found for the annotation.
    *
    * @throws exception
    */
-  private static function updateObservationWithAnnotationDetails($db, $occurrence_id, array $annotation) {
-    // Find the original record to compare against.
-    $oldRecords = $db
-      ->select('record_status, record_substatus, taxa_taxon_list_id')
-      ->from('cache_occurrences')
-      ->where('id', $occurrence_id)
-      ->get()->result_array(FALSE);
-    if (!count($oldRecords)) {
-      throw new exception('Could not find cache_occurrences record associated with a comment.');
-    }
-
-    // Find the taxon information supplied with the comment's TVK.
-    $newTaxa = $db
-      ->select('id, taxonomic_sort_order, taxon, authority, preferred_taxon, default_common_name, search_name, ' .
-        'external_key, taxon_meaning_id, taxon_group_id, taxon_group')
-      ->from('cache_taxa_taxon_lists')
-      ->where([
-        'preferred' => 't',
-        'external_key' => $annotation['taxonVersionKey'],
-        'taxon_list_id' => kohana::config('rest_api_sync.taxon_list_id'),
-      ])
-      ->limit(1)
-      ->get()->result_array(FALSE);
-    if (!count($newTaxa)) {
-      throw new exception('Could not find cache_taxa_taxon_lists record associated with an update from a comment.');
-    }
-
-    $oldRecord = $oldRecords[0];
-    $newTaxon = $newTaxa[0];
-
-    $new_status = $annotation['record_status'] === $oldRecord['record_status']
-      ? FALSE : $annotation['record_status'];
-    $new_substatus = $annotation['record_substatus'] === $oldRecord['record_substatus']
-      ? FALSE : $annotation['record_substatus'];
-    $new_ttlid = $newTaxon['id'] === $oldRecord['taxa_taxon_list_id']
-      ? FALSE : $newTaxon['id'];
-
-    // Does the comment imply an allowable change to the occurrence's attributes?
-    if ($new_status || $new_substatus || $new_ttlid) {
-      $oupdate = array('updated_on' => date("Ymd H:i:s"));
-      $coupdate = array('cache_updated_on' => date("Ymd H:i:s"));
-      if ($new_status || $new_substatus) {
-        $oupdate['verified_on'] = date("Ymd H:i:s");
-        // @todo Verified_by_id needs to be mapped to a proper user account.
-        $oupdate['verified_by_id'] = 1;
-        $coupdate['verified_on'] = date("Ymd H:i:s");
-        $coupdate['verifier'] = $annotation['authorName'];
-      }
-      if ($new_status) {
-        $oupdate['record_status'] = $new_status;
-        $coupdate['record_status'] = $new_status;
-      }
-      if ($new_substatus) {
-        $oupdate['record_substatus'] = $new_substatus;
-        $coupdate['record_substatus'] = $new_substatus;
+  private static function updateObservationWithAnnotationDetails($db, $occurrence_id, array $annotation, array $values) {
+    $update = [];
+    if (!empty($values['occurrence_comment:record_status'])) {
+      $update['record_status'] = $values['occurrence_comment:record_status'];
+      $update['record_substatus'] = empty($values['occurrence_comment:record_substatus']) ? NULL : $values['occurrence_comment:record_substatus'];
+    }
+    if (!empty($annotation['taxonVersionKey'])) {
+      // Find the taxon information supplied with the comment's TVK.
+      $newTaxa = $db
+        ->select('id')
+        ->from('cache_taxa_taxon_lists')
+        ->where([
+          'preferred' => 't',
+          'external_key' => $annotation['taxonVersionKey'],
+          'taxon_list_id' => kohana::config('rest_api_sync.taxon_list_id'),
+        ])
+        ->limit(1)
+        ->get()->result_array(FALSE);
+      if (!count($newTaxa)) {
+        throw new exception('Could not find cache_taxa_taxon_lists record associated with an update from a comment.');
       }
-      if ($new_ttlid) {
-        $oupdate['taxa_taxon_list_id'] = $new_ttlid;
-        $coupdate['taxa_taxon_list_id'] = $new_ttlid;
-        $coupdate['taxonomic_sort_order'] = $newTaxon['taxonomic_sort_order'];
-        $coupdate['taxon'] = $newTaxon['taxon'];
-        $coupdate['preferred_taxon'] = $newTaxon['preferred_taxon'];
-        $coupdate['authority'] = $newTaxon['authority'];
-        $coupdate['default_common_name'] = $newTaxon['default_common_name'];
-        $coupdate['search_name'] = $newTaxon['search_name'];
-        $coupdate['taxa_taxon_list_external_key'] = $newTaxon['external_key'];
-        $coupdate['taxon_meaning_id'] = $newTaxon['taxon_meaning_id'];
-        $coupdate['taxon_group_id'] = $newTaxon['taxon_group_id'];
-        $coupdate['taxon_group'] = $newTaxon['taxon_group'];
+      $values['taxa_taxon_list_id'] = $newTaxa[0]['id'];
+    }
+    if (count($values)) {
+      $occ = ORM::factory('occurrence', $occurrence_id);
+      foreach ($values as $field => $value) {
+        $occ->$field = $value;
       }
-      $db->update('occurrences',
-        $oupdate,
-        array('id' => $occurrence_id)
-      );
-      $db->update('cache_occurrences',
-        $coupdate,
-        array('id' => $occurrence_id)
-      );
-      // @todo create a determination if this is not automatic
+      $occ->set_metadata();
+      // Save occurrence changes (will update cache).
+      $occ->save();
+    }
+    elseif (!empty($annotation['question']) && $annotation['question'] === 't') {
+      // Need to separately update cache if question asked.
+      $q = new WorkQueue();
+      $q->enqueue($db, [
+        'task' => 'task_cache_builder_update',
+        'entity' => 'occurrence',
+        'record_id' => $occurrence_id,
+        'cost_estimate' => 50,
+        'priority' => 2,
+      ]);
     }
   }
 
diff --git a/modules/rest_api_sync/helpers/rest_api_sync_json_annotations.php b/modules/rest_api_sync/helpers/rest_api_sync_json_annotations.php
index 27b58c0abd..8da9a383c5 100644
--- a/modules/rest_api_sync/helpers/rest_api_sync_json_annotations.php
+++ b/modules/rest_api_sync/helpers/rest_api_sync_json_annotations.php
@@ -89,7 +89,7 @@ public static function syncPage($serverId, array $server) {
       // @todo Make sure all fields in specification are handled.
       try {
         $annotation = [
-          'id' => $record['id'],
+          'id' => $record['annotationID'],
           'occurrenceID' => $record['occurrenceID'],
           'comment' => empty($record['record-level']['comment']) ? NULL : $record['record-level']['comment'],
           'identificationVerificationStatus' => empty($record['identificationVerificationStatus']) ? NULL : $record['identificationVerificationStatus'],

From fe19f6436401585e05ff3a0c93b60b299a75356e Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Tue, 7 Sep 2021 13:42:14 +0100
Subject: [PATCH 53/93] No hierarchy in annotations format

---
 .../rest_api_sync/helpers/rest_api_sync_json_annotations.php    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/rest_api_sync/helpers/rest_api_sync_json_annotations.php b/modules/rest_api_sync/helpers/rest_api_sync_json_annotations.php
index 8da9a383c5..237bf04f19 100644
--- a/modules/rest_api_sync/helpers/rest_api_sync_json_annotations.php
+++ b/modules/rest_api_sync/helpers/rest_api_sync_json_annotations.php
@@ -91,7 +91,7 @@ public static function syncPage($serverId, array $server) {
         $annotation = [
           'id' => $record['annotationID'],
           'occurrenceID' => $record['occurrenceID'],
-          'comment' => empty($record['record-level']['comment']) ? NULL : $record['record-level']['comment'],
+          'comment' => empty($record['comment']) ? 'No comment provided' : $record['comment'],
           'identificationVerificationStatus' => empty($record['identificationVerificationStatus']) ? NULL : $record['identificationVerificationStatus'],
           'question' => empty($record['question']) ? NULL : $record['question'],
           'authorName' => empty($record['authorName']) ? 'Unknown' : $record['authorName'],

From 02d477fdb2335f387424e30b1deb4e7449fbf1ac Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Tue, 7 Sep 2021 13:48:46 +0100
Subject: [PATCH 54/93] Fixes annotation submission bug

---
 modules/rest_api_sync/helpers/api_persist.php | 26 +++++++++----------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/modules/rest_api_sync/helpers/api_persist.php b/modules/rest_api_sync/helpers/api_persist.php
index 1b7e6c99d0..4982c3706f 100644
--- a/modules/rest_api_sync/helpers/api_persist.php
+++ b/modules/rest_api_sync/helpers/api_persist.php
@@ -199,11 +199,11 @@ public static function annotation($db, array $annotation, $survey_id) {
       // and import the observation via the API?
       throw new exception("Attempt to import annotation $annotation[id] but taxon observation not found.");
     }
-    $values['occurrence_comment:occurrence_id'] = $existingObs[0]['id'];
+    $values['occurrence_id'] = $existingObs[0]['id'];
     // Link to existing annotation if appropriate.
     $existing = self::findExistingAnnotation($db, $annotation['id'], $existingObs[0]['id']);
     if ($existing) {
-      $values['occurrence_comment:id'] = $existing[0]['id'];
+      $values['id'] = $existing[0]['id'];
     }
     $annotationObj = ORM::factory('occurrence_comment');
     $annotationObj->set_submission_data($values);
@@ -398,16 +398,16 @@ private static function getMediaTypeId($db, $mediaType) {
    */
   private static function getAnnotationValues($db, array $annotation) {
     $values = [
-      'occurrence_comment:comment' => $annotation['comment'],
-      'occurrence_comment:email_address' => self::valueOrNull($annotation, 'emailAddress'),
-      'occurrence_comment:record_status' => self::valueOrNull($annotation, 'record_status'),
-      'occurrence_comment:record_substatus' => self::valueOrNull($annotation, 'record_substatus'),
-      'occurrence_comment:query' => $annotation['question'],
-      'occurrence_comment:person_name' => $annotation['authorName'],
-      'occurrence_comment:external_key' => $annotation['id'],
+      'comment' => $annotation['comment'],
+      'email_address' => self::valueOrNull($annotation, 'emailAddress'),
+      'record_status' => self::valueOrNull($annotation, 'record_status'),
+      'record_substatus' => self::valueOrNull($annotation, 'record_substatus'),
+      'query' => $annotation['question'],
+      'person_name' => $annotation['authorName'],
+      'external_key' => $annotation['id'],
     ];
     if (!empty($annotation['dateTime'])) {
-      $r['updated_on'] = $annotation['dateTime'];
+      $values['updated_on'] = $annotation['dateTime'];
     }
     if (!empty($annotation['identificationVerificationStatus'])) {
       self::applyIdentificationVerificationStatus($annotation['identificationVerificationStatus'], $values);
@@ -866,9 +866,9 @@ private static function mapRecordStatus(array &$annotation) {
    */
   private static function updateObservationWithAnnotationDetails($db, $occurrence_id, array $annotation, array $values) {
     $update = [];
-    if (!empty($values['occurrence_comment:record_status'])) {
-      $update['record_status'] = $values['occurrence_comment:record_status'];
-      $update['record_substatus'] = empty($values['occurrence_comment:record_substatus']) ? NULL : $values['occurrence_comment:record_substatus'];
+    if (!empty($values['record_status'])) {
+      $update['record_status'] = $values['record_status'];
+      $update['record_substatus'] = empty($values['record_substatus']) ? NULL : $values['record_substatus'];
     }
     if (!empty($annotation['taxonVersionKey'])) {
       // Find the taxon information supplied with the comment's TVK.

From faa47eb4024a5275ed628d2eabcf21e3c1a3e0e3 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Tue, 7 Sep 2021 13:50:35 +0100
Subject: [PATCH 55/93] Annotation submission bugfix

---
 modules/rest_api_sync/helpers/api_persist.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/rest_api_sync/helpers/api_persist.php b/modules/rest_api_sync/helpers/api_persist.php
index 4982c3706f..204224a715 100644
--- a/modules/rest_api_sync/helpers/api_persist.php
+++ b/modules/rest_api_sync/helpers/api_persist.php
@@ -889,7 +889,7 @@ private static function updateObservationWithAnnotationDetails($db, $occurrence_
     }
     if (count($values)) {
       $occ = ORM::factory('occurrence', $occurrence_id);
-      foreach ($values as $field => $value) {
+      foreach ($update as $field => $value) {
         $occ->$field = $value;
       }
       $occ->set_metadata();

From c8db4864666e5bc846267c24d2a85cc41e526e44 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Tue, 7 Sep 2021 16:28:03 +0100
Subject: [PATCH 56/93] Comments exclude system supplied status phrases.

---
 .../rest_api_sync/filterable_annotations.xml       | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/modules/rest_api_sync/reports/rest_api_sync/filterable_annotations.xml b/modules/rest_api_sync/reports/rest_api_sync/filterable_annotations.xml
index e539331421..44548649fe 100644
--- a/modules/rest_api_sync/reports/rest_api_sync/filterable_annotations.xml
+++ b/modules/rest_api_sync/reports/rest_api_sync/filterable_annotations.xml
@@ -30,8 +30,18 @@
             sql="COALESCE(oc.external_key, '#system_user_id#' || oc.id::varchar)" />
     <column name="occurrenceID" datatype="integer"
             sql="COALESCE(occ.external_key, '#system_user_id#' || o.id::varchar)" />
-    <column name="comment" datatype="text"
-            sql="oc.comment"/>
+    <column name="comment" datatype="text" sql="123::text || case oc.comment when case oc.record_status || coalesce(oc.record_substatus::text, '')
+                when 'V' then 'Accepted'
+                when 'V1' then 'Accepted as correct'
+                when 'V2' then 'Accepted as considered correct'
+                when 'C3' then 'Plausible'
+                when 'R' then 'Not accepted'
+                when 'R4' then 'Not accepted as unable to verify'
+                when 'R5' then 'Not accepted as incorrect'
+                else null
+            end then null
+            else oc.comment
+        end"/>
     <column name="identificationVerificationStatus" datatype="text"
             sql="case oc.record_status || coalesce(oc.record_substatus::text, '')
             when 'V' then 'Accepted'

From 54e83b914b5cd546168c0f227d4ede5d4bd436f5 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Tue, 7 Sep 2021 16:30:10 +0100
Subject: [PATCH 57/93] Removed code for debugging

---
 .../reports/rest_api_sync/filterable_annotations.xml            | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/rest_api_sync/reports/rest_api_sync/filterable_annotations.xml b/modules/rest_api_sync/reports/rest_api_sync/filterable_annotations.xml
index 44548649fe..1fb2514521 100644
--- a/modules/rest_api_sync/reports/rest_api_sync/filterable_annotations.xml
+++ b/modules/rest_api_sync/reports/rest_api_sync/filterable_annotations.xml
@@ -30,7 +30,7 @@
             sql="COALESCE(oc.external_key, '#system_user_id#' || oc.id::varchar)" />
     <column name="occurrenceID" datatype="integer"
             sql="COALESCE(occ.external_key, '#system_user_id#' || o.id::varchar)" />
-    <column name="comment" datatype="text" sql="123::text || case oc.comment when case oc.record_status || coalesce(oc.record_substatus::text, '')
+    <column name="comment" datatype="text" sql="case oc.comment when case oc.record_status || coalesce(oc.record_substatus::text, '')
                 when 'V' then 'Accepted'
                 when 'V1' then 'Accepted as correct'
                 when 'V2' then 'Accepted as considered correct'

From 2234400c0c43030a170b178d3fd4e84c1105a38d Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Tue, 7 Sep 2021 16:41:56 +0100
Subject: [PATCH 58/93] Exclude annotations originating elsewhere.

---
 .../reports/rest_api_sync/filterable_annotations.xml            | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/modules/rest_api_sync/reports/rest_api_sync/filterable_annotations.xml b/modules/rest_api_sync/reports/rest_api_sync/filterable_annotations.xml
index 1fb2514521..c903d42c71 100644
--- a/modules/rest_api_sync/reports/rest_api_sync/filterable_annotations.xml
+++ b/modules/rest_api_sync/reports/rest_api_sync/filterable_annotations.xml
@@ -13,6 +13,8 @@
     #joins#
     where #sharing_filter#
     and oc.deleted=false
+    -- Only annotations originating on this warehouse.
+    and oc.external_key is null
     and o.taxa_taxon_list_external_key is not null
     #idlist#
   </query>

From c11cf63fbfa1245777c4de07482e81a26a5d0358 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Wed, 8 Sep 2021 10:47:33 +0100
Subject: [PATCH 59/93] Bug fix in setting synced record's licence ID

---
 modules/rest_api_sync/helpers/api_persist.php | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/rest_api_sync/helpers/api_persist.php b/modules/rest_api_sync/helpers/api_persist.php
index 204224a715..6af36dc89a 100644
--- a/modules/rest_api_sync/helpers/api_persist.php
+++ b/modules/rest_api_sync/helpers/api_persist.php
@@ -435,10 +435,10 @@ private static function getLicenceIdFromCode($db, $licenceCode) {
       $licenceData = $db->query($qry)->result_array(FALSE);
       self::$licences = [];
       foreach ($licenceData as $licence) {
-        self::$licences[strtolower(str_replace($licence['code'], ' ', '-'))] = $licence['id'];
+        self::$licences[strtolower(str_replace(' ', '-', $licence['code']))] = $licence['id'];
       }
     }
-    return self::$licences[strtolower(str_replace($licenceCode, ' ', '-'))];
+    return self::$licences[strtolower(str_replace(' ', '-', $licenceCode))];
   }
 
   /**

From 906c6c5209f49344dc0839056243edd4db9303db Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Wed, 8 Sep 2021 11:40:10 +0100
Subject: [PATCH 60/93] Class renaming for clarity

---
 modules/rest_api/controllers/services/rest.php |  6 +++---
 .../controllers/rest_api_sync.php              |  8 ++++----
 .../controllers/services/rest_api_sync.php     |  6 +++---
 ...hp => rest_api_sync_remote_inaturalist.php} | 10 +++++-----
 ...ia.php => rest_api_sync_remote_indicia.php} | 18 +++++++++---------
 ... rest_api_sync_remote_json_annotations.php} |  8 ++++----
 ... rest_api_sync_remote_json_occurrences.php} |  8 ++++----
 ...st.php => rest_api_sync_rest_endpoints.php} |  2 +-
 ...st_api_sync.php => rest_api_sync_utils.php} | 14 ++++++++++++--
 9 files changed, 45 insertions(+), 35 deletions(-)
 rename modules/rest_api_sync/helpers/{rest_api_sync_inaturalist.php => rest_api_sync_remote_inaturalist.php} (97%)
 rename modules/rest_api_sync/helpers/{rest_api_sync_indicia.php => rest_api_sync_remote_indicia.php} (94%)
 rename modules/rest_api_sync/helpers/{rest_api_sync_json_annotations.php => rest_api_sync_remote_json_annotations.php} (96%)
 rename modules/rest_api_sync/helpers/{rest_api_sync_json_occurrences.php => rest_api_sync_remote_json_occurrences.php} (98%)
 rename modules/rest_api_sync/helpers/{rest_api_sync_rest.php => rest_api_sync_rest_endpoints.php} (99%)
 rename modules/rest_api_sync/helpers/{rest_api_sync.php => rest_api_sync_utils.php} (93%)

diff --git a/modules/rest_api/controllers/services/rest.php b/modules/rest_api/controllers/services/rest.php
index 5e04089aca..c76f981034 100644
--- a/modules/rest_api/controllers/services/rest.php
+++ b/modules/rest_api/controllers/services/rest.php
@@ -857,10 +857,10 @@ public function __call($name, $arguments) {
           if (RestObjects::$handlerModule === 'rest_api' && method_exists($this, $methodName)) {
             $class = $this;
           }
-          elseif (RestObjects::$handlerModule !== 'rest_api' && method_exists(RestObjects::$handlerModule . '_rest', $methodName)) {
+          elseif (RestObjects::$handlerModule !== 'rest_api' && method_exists(RestObjects::$handlerModule . '_rest_endpoints', $methodName)) {
             // Expect any modules extending the API to implement a helper class
-            // <module name>_rest.
-            $class = RestObjects::$handlerModule . '_rest';
+            // <module name>_rest_endpoints.
+            $class = RestObjects::$handlerModule . '_rest_endpoints';
           }
           else {
             RestObjects::$apiResponse->fail('Not Found', 404, "Resource $name not known for method $this->method ($methodName)");
diff --git a/modules/rest_api_sync/controllers/rest_api_sync.php b/modules/rest_api_sync/controllers/rest_api_sync.php
index 9d5da9d75e..8761463e1b 100644
--- a/modules/rest_api_sync/controllers/rest_api_sync.php
+++ b/modules/rest_api_sync/controllers/rest_api_sync.php
@@ -75,7 +75,7 @@ public function end() {
    */
   public function process_batch() {
     $this->auto_render = FALSE;
-    rest_api_sync::$clientUserId = Kohana::config('rest_api_sync.user_id');
+    rest_api_sync_utils::$clientUserId = Kohana::config('rest_api_sync.user_id');
     $servers = Kohana::config('rest_api_sync.servers');
     $serverIdx = empty($_GET['serverIdx']) ? 1 : $_GET['serverIdx'];
     $page = empty($_GET['page']) ? 1 : $_GET['page'];
@@ -84,7 +84,7 @@ public function process_batch() {
       'serverType' => 'Indicia',
       'allowUpdateWhenVerified' => TRUE,
     ], $servers[$serverId]);
-    $helperClass = 'rest_api_sync_' . strtolower($server['serverType']);
+    $helperClass = 'rest_api_sync_remote_' . strtolower($server['serverType']);
     $helperClass::loadControlledTerms($serverId, $server);
     // For performance, just notify work_queue to update cache entries.
     if (class_exists('cache_builder')) {
@@ -101,7 +101,7 @@ public function process_batch() {
     if ($serverIdx > count($servers)) {
       echo json_encode([
         'state' => 'done',
-        'log' => rest_api_sync::$log,
+        'log' => rest_api_sync_utils::$log,
       ]);
     }
     else {
@@ -109,7 +109,7 @@ public function process_batch() {
         'state' => 'in progress',
         'serverIdx' => $serverIdx,
         'page' => $page,
-        'log' => rest_api_sync::$log,
+        'log' => rest_api_sync_utils::$log,
         'pagesToGo' => $progressInfo['pagesToGo'],
         'recordsToGo' => $progressInfo['recordsToGo'],
         'moreToDo' => $progressInfo['moreToDo'],
diff --git a/modules/rest_api_sync/controllers/services/rest_api_sync.php b/modules/rest_api_sync/controllers/services/rest_api_sync.php
index 709ba98595..e5900a917e 100644
--- a/modules/rest_api_sync/controllers/services/rest_api_sync.php
+++ b/modules/rest_api_sync/controllers/services/rest_api_sync.php
@@ -38,7 +38,7 @@ public function index() {
     kohana::log('debug', 'Initiating REST API Sync');
     echo "<h1>REST API Sync</h1>";
     $servers = Kohana::config('rest_api_sync.servers');
-    rest_api_sync::$clientUserId = Kohana::config('rest_api_sync.user_id');
+    rest_api_sync_utils::$clientUserId = Kohana::config('rest_api_sync.user_id');
     // For performance, just notify work_queue to update cache entries.
     if (class_exists('cache_builder')) {
       cache_builder::$delayCacheUpdates = TRUE;
@@ -46,10 +46,10 @@ public function index() {
     foreach ($servers as $serverId => $server) {
       echo "<h2>$serverId</h2>";
       $server = array_merge([
-        'serverType' => 'Indicia', 
+        'serverType' => 'Indicia',
         'allowUpdateWhenVerified' => TRUE,
       ], $server);
-      $helperClass = 'rest_api_sync_' . strtolower($server['serverType']);
+      $helperClass = 'rest_api_sync_remote_' . strtolower($server['serverType']);
       $helperClass::loadControlledTerms($serverId, $server);
       $helperClass::syncServer($serverId, $server);
     }
diff --git a/modules/rest_api_sync/helpers/rest_api_sync_inaturalist.php b/modules/rest_api_sync/helpers/rest_api_sync_remote_inaturalist.php
similarity index 97%
rename from modules/rest_api_sync/helpers/rest_api_sync_inaturalist.php
rename to modules/rest_api_sync/helpers/rest_api_sync_remote_inaturalist.php
index ee0c9653ca..78200be141 100644
--- a/modules/rest_api_sync/helpers/rest_api_sync_inaturalist.php
+++ b/modules/rest_api_sync/helpers/rest_api_sync_remote_inaturalist.php
@@ -30,7 +30,7 @@
 /**
  * Helper class for syncing to the RESTful API on iNaturalist.
  */
-class rest_api_sync_inaturalist {
+class rest_api_sync_remote_inaturalist {
 
   /**
    * Terms loaded from iNat.
@@ -86,7 +86,7 @@ public static function loadControlledTerms($serverId, array $server) {
     $cache = Cache::instance();
     self::$controlledTerms = $cache->get('inaturalist-controlled-terms');
     if (!self::$controlledTerms) {
-      $data = rest_api_sync::getDataFromRestUrl(
+      $data = rest_api_sync_utils::getDataFromRestUrl(
         "$server[url]/controlled_terms",
         $serverId
       );
@@ -125,7 +125,7 @@ public static function syncPage($serverId, array $server) {
     $fromDateTime = variable::get("rest_api_sync_{$serverId}_last_run", '1600-01-01T00:00:00+00:00', FALSE);
     $fromId = variable::get("rest_api_sync_{$serverId}_last_id", 0, FALSE);
     $lastId = $fromId;
-    $data = rest_api_sync::getDataFromRestUrl(
+    $data = rest_api_sync_utils::getDataFromRestUrl(
       "$server[url]/observations?" . http_build_query(array_merge(
         $server['parameters'],
         [
@@ -219,7 +219,7 @@ public static function syncPage($serverId, array $server) {
           "WHERE server_id='$serverId' AND source_id='$iNatRecord[id]' AND dest_table='occurrences'");
       }
       catch (exception $e) {
-        rest_api_sync::log(
+        rest_api_sync_utils::log(
           'error',
           "Error occurred submitting an occurrence with iNaturalist ID $iNatRecord[id]\n" . $e->getMessage(),
           $tracker
@@ -251,7 +251,7 @@ public static function syncPage($serverId, array $server) {
       $lastId = $iNatRecord['id'];
     }
     variable::set("rest_api_sync_{$serverId}_last_id", $lastId);
-    rest_api_sync::log(
+    rest_api_sync_utils::log(
       'info',
       "<strong>Observations</strong><br/>Inserts: $tracker[inserts]. Updates: $tracker[updates]. Errors: $tracker[errors]"
     );
diff --git a/modules/rest_api_sync/helpers/rest_api_sync_indicia.php b/modules/rest_api_sync/helpers/rest_api_sync_remote_indicia.php
similarity index 94%
rename from modules/rest_api_sync/helpers/rest_api_sync_indicia.php
rename to modules/rest_api_sync/helpers/rest_api_sync_remote_indicia.php
index d5b9ad9cbb..d85060e23d 100644
--- a/modules/rest_api_sync/helpers/rest_api_sync_indicia.php
+++ b/modules/rest_api_sync/helpers/rest_api_sync_remote_indicia.php
@@ -27,7 +27,7 @@
 /**
  * Helper class for syncing to the RESTful API on an Indicia warehouses.
  */
-class rest_api_sync_indicia {
+class rest_api_sync_remote_indicia {
 
   /**
    * ISO datetime that the sync was last run.
@@ -79,7 +79,7 @@ public static function syncServer($serverId, array $server) {
     while ($next_page_of_projects_url) {
       $response = self::getServerProjects($next_page_of_projects_url, $serverId);
       if (!isset($response['data'])) {
-        rest_api_sync::log('error', "Invalid response\nURL: $next_page_of_projects_url\nResponse did not include data element.");
+        rest_api_sync_utils::log('error', "Invalid response\nURL: $next_page_of_projects_url\nResponse did not include data element.");
         var_export($response);
         continue;
       }
@@ -237,7 +237,7 @@ private static function syncTaxonObservations(array $server, $serverId, array $p
     while ($next_page_of_taxon_observations_url && ($load_all || $processedCount < MAX_RECORDS_TO_PROCESS)) {
       $data = self::getServerTaxonObservations($next_page_of_taxon_observations_url, $serverId);
       $observations = $data['data'];
-      rest_api_sync::log('debug', count($observations) . ' records found');
+      rest_api_sync_utils::log('debug', count($observations) . ' records found');
       foreach ($observations as $observation) {
         // If the record was originated from a different system, the specified
         // dataset name needs to be stored.
@@ -259,7 +259,7 @@ private static function syncTaxonObservations(array $server, $serverId, array $p
           }
         }
         catch (exception $e) {
-          rest_api_sync::log('error', "Error occurred submitting an occurrence\n" . $e->getMessage() . "\n" .
+          rest_api_sync_utils::log('error', "Error occurred submitting an occurrence\n" . $e->getMessage() . "\n" .
               json_encode($observation), $tracker);
         }
         if ($last_record_date && $last_record_date <> $observation['lastEditDate']) {
@@ -313,14 +313,14 @@ private static function syncAnnotations(array $server, $serverId, array $project
     while ($nextPageOfAnnotationsUrl && ($load_all || $processedCount < MAX_RECORDS_TO_PROCESS)) {
       $data = self::getServerAnnotations($nextPageOfAnnotationsUrl, $serverId);
       $annotations = $data['data'];
-      rest_api_sync::log('debug', count($annotations) . ' records found');
+      rest_api_sync_utils::log('debug', count($annotations) . ' records found');
       foreach ($annotations as $annotation) {
         try {
           $is_new = api_persist::annotation(self::$db, $annotation, $survey_id);
           $tracker[$is_new ? 'inserts' : 'updates']++;
         }
         catch (exception $e) {
-          rest_api_sync::log('error', "Error occurred submitting an annotation\n" . $e->getMessage() . "\n" .
+          rest_api_sync_utils::log('error', "Error occurred submitting an annotation\n" . $e->getMessage() . "\n" .
             json_encode($annotation), $tracker);
         }
         if ($last_record_date && $last_record_date <> $annotation['lastEditDate']) {
@@ -367,15 +367,15 @@ private static function getServerProjectsUrl($server_url) {
   }
 
   public static function getServerProjects($url, $serverId) {
-    return rest_api_sync::getDataFromRestUrl($url, $serverId);
+    return rest_api_sync_utils::getDataFromRestUrl($url, $serverId);
   }
 
   public static function getServerTaxonObservations($url, $serverId) {
-    return rest_api_sync::getDataFromRestUrl($url, $serverId);
+    return rest_api_sync_utils::getDataFromRestUrl($url, $serverId);
   }
 
   public static function getServerAnnotations($url, $serverId) {
-    return rest_api_sync::getDataFromRestUrl($url, $serverId);
+    return rest_api_sync_utils::getDataFromRestUrl($url, $serverId);
   }
 
 }
diff --git a/modules/rest_api_sync/helpers/rest_api_sync_json_annotations.php b/modules/rest_api_sync/helpers/rest_api_sync_remote_json_annotations.php
similarity index 96%
rename from modules/rest_api_sync/helpers/rest_api_sync_json_annotations.php
rename to modules/rest_api_sync/helpers/rest_api_sync_remote_json_annotations.php
index 237bf04f19..a8130c7440 100644
--- a/modules/rest_api_sync/helpers/rest_api_sync_json_annotations.php
+++ b/modules/rest_api_sync/helpers/rest_api_sync_remote_json_annotations.php
@@ -30,7 +30,7 @@
  * Could be an Indicia warehouse, or another server implementing the same
  * standard.
  */
-class rest_api_sync_json_annotations {
+class rest_api_sync_remote_json_annotations {
 
   /**
    * Synchronise a set of data loaded from the other server.
@@ -80,7 +80,7 @@ public static function loadControlledTerms() {
   public static function syncPage($serverId, array $server) {
     $db = Database::instance();
     $nextPage = variable::get("rest_api_sync_{$serverId}_next_page", [], FALSE);
-    $data = rest_api_sync::getDataFromRestUrl(
+    $data = rest_api_sync_utils::getDataFromRestUrl(
       "$server[url]?" . http_build_query($nextPage),
       $serverId
     );
@@ -110,7 +110,7 @@ public static function syncPage($serverId, array $server) {
           "WHERE server_id='$serverId' AND source_id='$annotation[id]' AND dest_table='occurrence_comments'");
       }
       catch (exception $e) {
-        rest_api_sync::log(
+        rest_api_sync_utils::log(
           'error',
           "Error occurred submitting an annotation with ID $annotation[id]\n" . $e->getMessage(),
           $tracker
@@ -141,7 +141,7 @@ public static function syncPage($serverId, array $server) {
       }
     }
     variable::set("rest_api_sync_{$serverId}_next_page", $data['paging']['next']);
-    rest_api_sync::log(
+    rest_api_sync_utils::log(
       'info',
       "<strong>Annotations</strong><br/>Inserts: $tracker[inserts]. Updates: $tracker[updates]. Errors: $tracker[errors]"
     );
diff --git a/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php b/modules/rest_api_sync/helpers/rest_api_sync_remote_json_occurrences.php
similarity index 98%
rename from modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php
rename to modules/rest_api_sync/helpers/rest_api_sync_remote_json_occurrences.php
index 4af0a7a331..4915f2e217 100644
--- a/modules/rest_api_sync/helpers/rest_api_sync_json_occurrences.php
+++ b/modules/rest_api_sync/helpers/rest_api_sync_remote_json_occurrences.php
@@ -30,7 +30,7 @@
  * Could be an Indicia warehouse, or another server implementing the same
  * standard.
  */
-class rest_api_sync_json_occurrences {
+class rest_api_sync_remote_json_occurrences {
 
   /**
    * Synchronise a set of data loaded from the other server.
@@ -81,7 +81,7 @@ public static function syncPage($serverId, array $server) {
     $db = Database::instance();
     api_persist::initDwcAttributes($db, $server['survey_id']);
     $nextPage = variable::get("rest_api_sync_{$serverId}_next_page", [], FALSE);
-    $data = rest_api_sync::getDataFromRestUrl(
+    $data = rest_api_sync_utils::getDataFromRestUrl(
       "$server[url]?" . http_build_query($nextPage),
       $serverId
     );
@@ -169,7 +169,7 @@ public static function syncPage($serverId, array $server) {
           "WHERE server_id='$serverId' AND source_id='{$record['occurrence']['occurrenceID']}' AND dest_table='occurrences'");
       }
       catch (exception $e) {
-        rest_api_sync::log(
+        rest_api_sync_utils::log(
           'error',
           "Error occurred submitting an occurrence with ID {$record['occurrence']['occurrenceID']}\n" . $e->getMessage(),
           $tracker
@@ -200,7 +200,7 @@ public static function syncPage($serverId, array $server) {
       }
     }
     variable::set("rest_api_sync_{$serverId}_next_page", $data['paging']['next']);
-    rest_api_sync::log(
+    rest_api_sync_utils::log(
       'info',
       "<strong>Observations</strong><br/>Inserts: $tracker[inserts]. Updates: $tracker[updates]. Errors: $tracker[errors]"
     );
diff --git a/modules/rest_api_sync/helpers/rest_api_sync_rest.php b/modules/rest_api_sync/helpers/rest_api_sync_rest_endpoints.php
similarity index 99%
rename from modules/rest_api_sync/helpers/rest_api_sync_rest.php
rename to modules/rest_api_sync/helpers/rest_api_sync_rest_endpoints.php
index ae2213fe61..135efdacd3 100644
--- a/modules/rest_api_sync/helpers/rest_api_sync_rest.php
+++ b/modules/rest_api_sync/helpers/rest_api_sync_rest_endpoints.php
@@ -27,7 +27,7 @@
 /**
  * Helper class for extending the REST API with sync endpoints.
  */
-class rest_api_sync_rest {
+class rest_api_sync_rest_endpoints {
 
   /**
    * Attribute types to exclude, either for privacy or duplication reasons.
diff --git a/modules/rest_api_sync/helpers/rest_api_sync.php b/modules/rest_api_sync/helpers/rest_api_sync_utils.php
similarity index 93%
rename from modules/rest_api_sync/helpers/rest_api_sync.php
rename to modules/rest_api_sync/helpers/rest_api_sync_utils.php
index 8a8fc4cc84..8045b6a93d 100644
--- a/modules/rest_api_sync/helpers/rest_api_sync.php
+++ b/modules/rest_api_sync/helpers/rest_api_sync_utils.php
@@ -26,14 +26,24 @@
 /**
  * Helper class for syncing to RESTful APIs.
  */
-class rest_api_sync {
+class rest_api_sync_utils {
 
+  /**
+   * Client user ID for authentication.
+   *
+   * @var string
+   */
   public static $clientUserId;
 
+  /**
+   * Keep track of logged messages so they can be reported back to a UI.
+   *
+   * @var array
+   */
   public static $log = [];
 
   /**
-   * Gets a page of data from another server's REST API
+   * Gets a page of data from another server's REST API.
    *
    * @param string $url
    *   URL of the service to access.

From adaac3b8c49c8f3ef2b260a833fb9626839e9049 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Wed, 8 Sep 2021 11:42:09 +0100
Subject: [PATCH 61/93] Correct param definitions

---
 modules/rest_api_sync/plugins/rest_api_sync.php | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/rest_api_sync/plugins/rest_api_sync.php b/modules/rest_api_sync/plugins/rest_api_sync.php
index ddda5631b8..2f585aa09c 100644
--- a/modules/rest_api_sync/plugins/rest_api_sync.php
+++ b/modules/rest_api_sync/plugins/rest_api_sync.php
@@ -61,8 +61,8 @@ function rest_api_sync_extend_rest_api() {
             'proj_id' => [
               'datatype' => 'text',
             ],
-            /*************/'tracking_from' => [
-              'datatype' => 'integer',
+            'dateTime_from' => [
+              'datatype' => 'text',
             ],
           ],
         ],

From ad9df9c6421baaf8d4252ebb7632ada47fb3edec Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Wed, 8 Sep 2021 12:29:18 +0100
Subject: [PATCH 62/93] New field for verifier only occurrence data

---
 application/models/occurrence.php             | 31 ++++++++++++-------
 ...202109081213_occurrences_verifier_only.sql |  5 +++
 2 files changed, 24 insertions(+), 12 deletions(-)
 create mode 100644 modules/indicia_setup/db/version_6_3_0/202109081213_occurrences_verifier_only.sql

diff --git a/application/models/occurrence.php b/application/models/occurrence.php
index 0c428f6629..1ff0bce896 100644
--- a/application/models/occurrence.php
+++ b/application/models/occurrence.php
@@ -25,24 +25,27 @@
 class Occurrence_Model extends ORM {
   protected $requeuedForVerification = FALSE;
 
-  protected $has_many = array(
+  protected $has_many = [
     'occurrence_attribute_values',
     'determinations',
-    'occurrence_media'
-  );
-  protected $belongs_to = array(
+    'occurrence_media',
+  ];
+
+  protected $belongs_to = [
     'determiner' => 'person',
     'sample',
     'taxa_taxon_list',
     'created_by' => 'user',
     'updated_by' => 'user',
-    'verified_by' => 'user'
-  );
-  // Declare that this model has child attributes, and the name of the node in the submission which contains them.
+    'verified_by' => 'user',
+  ];
+
+  // Declare that this model has child attributes, and the name of the node in
+  // the submission which contains them.
   protected $has_attributes = TRUE;
   protected $attrs_submission_name = 'occAttributes';
   public $attrs_field_prefix = 'occAttr';
-  protected $additional_csv_fields = array(
+  protected $additional_csv_fields = [
     // Extra lookup options.
     'occurrence:fk_taxa_taxon_list:genus' => 'Genus (builds binomial name)',
     'occurrence:fk_taxa_taxon_list:specific' => 'Specific name/epithet (builds binomial name)',
@@ -58,8 +61,8 @@ class Occurrence_Model extends ORM {
     'occurrence_medium:path:3' => 'Media Path 3',
     'occurrence_medium:caption:3' => 'Media Caption 3',
     'occurrence_medium:path:4' => 'Media Path 4',
-    'occurrence_medium:caption:4' => 'Media Caption 4'
-  );
+    'occurrence_medium:caption:4' => 'Media Caption 4',
+  ];
 
   // During an import it is possible to merge different columns in a CSV row to make a database field
   public $specialImportFieldProcessingDefn = [
@@ -108,6 +111,9 @@ class Occurrence_Model extends ORM {
 
   /**
    * Returns a caption to identify this model instance.
+   *
+   * @return string
+   *   Caption for instance.
    */
   public function caption() {
     return 'Record of ' . $this->taxa_taxon_list->taxon->taxon;
@@ -165,7 +171,7 @@ public function validate(Validation $array, $save = FALSE) {
       $array->add_rules('taxa_taxon_list_id', 'required');
     }
     // Explicitly add those fields for which we don't do validation.
-    $this->unvalidatedFields = array(
+    $this->unvalidatedFields = [
       'comment',
       'determiner_id',
       'deleted',
@@ -184,7 +190,8 @@ public function validate(Validation $array, $save = FALSE) {
       'sensitivity_precision',
       'import_guid',
       'metadata',
-    );
+      'verifier_only',
+    ];
     if (array_key_exists('id', $fieldlist)) {
       // Existing data must not be set to download_flag=F (final download) otherwise it
       // is read only.
diff --git a/modules/indicia_setup/db/version_6_3_0/202109081213_occurrences_verifier_only.sql b/modules/indicia_setup/db/version_6_3_0/202109081213_occurrences_verifier_only.sql
new file mode 100644
index 0000000000..ff27f8f619
--- /dev/null
+++ b/modules/indicia_setup/db/version_6_3_0/202109081213_occurrences_verifier_only.sql
@@ -0,0 +1,5 @@
+ALTER TABLE occurrences
+    ADD COLUMN verifier_only json;
+
+COMMENT ON COLUMN occurrences.verifier_only
+    IS 'Data provided by the recorder about a record where they have only given permission for the data to be used for verification purposes, not for public reporting or onward data tranmission.';
\ No newline at end of file

From fef77c7f7f8d9618ed4cd2e917f2d93d61c7ebf7 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Wed, 8 Sep 2021 12:31:40 +0100
Subject: [PATCH 63/93] Field name updated for clarity

---
 application/models/occurrence.php                             | 2 +-
 .../version_6_3_0/202109081213_occurrences_verifier_only.sql  | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/application/models/occurrence.php b/application/models/occurrence.php
index 1ff0bce896..6fe4b6c985 100644
--- a/application/models/occurrence.php
+++ b/application/models/occurrence.php
@@ -190,7 +190,7 @@ public function validate(Validation $array, $save = FALSE) {
       'sensitivity_precision',
       'import_guid',
       'metadata',
-      'verifier_only',
+      'verifier_only_data',
     ];
     if (array_key_exists('id', $fieldlist)) {
       // Existing data must not be set to download_flag=F (final download) otherwise it
diff --git a/modules/indicia_setup/db/version_6_3_0/202109081213_occurrences_verifier_only.sql b/modules/indicia_setup/db/version_6_3_0/202109081213_occurrences_verifier_only.sql
index ff27f8f619..17c0569147 100644
--- a/modules/indicia_setup/db/version_6_3_0/202109081213_occurrences_verifier_only.sql
+++ b/modules/indicia_setup/db/version_6_3_0/202109081213_occurrences_verifier_only.sql
@@ -1,5 +1,5 @@
 ALTER TABLE occurrences
-    ADD COLUMN verifier_only json;
+    ADD COLUMN verifier_only_data json;
 
-COMMENT ON COLUMN occurrences.verifier_only
+COMMENT ON COLUMN occurrences.verifier_only_data
     IS 'Data provided by the recorder about a record where they have only given permission for the data to be used for verification purposes, not for public reporting or onward data tranmission.';
\ No newline at end of file

From a60cf0cdc0e5fcbd3b765dbb24146d659298f851 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Wed, 8 Sep 2021 16:10:50 +0100
Subject: [PATCH 64/93] Dynamic properties and verifier only data support

---
 modules/rest_api_sync/helpers/api_persist.php |  1 +
 .../rest_api_sync_remote_json_occurrences.php | 51 ++++++++++++++-----
 2 files changed, 40 insertions(+), 12 deletions(-)

diff --git a/modules/rest_api_sync/helpers/api_persist.php b/modules/rest_api_sync/helpers/api_persist.php
index 6af36dc89a..e651e23980 100644
--- a/modules/rest_api_sync/helpers/api_persist.php
+++ b/modules/rest_api_sync/helpers/api_persist.php
@@ -247,6 +247,7 @@ private static function getTaxonObservationValues($db, $website_id, array $obser
       'occurrence:external_key' => $observation['id'],
       'occurrence:zero_abundance' => isset($observation['zeroAbundance']) ? strtolower($observation['zeroAbundance']) : 'f',
       'occurrence:sensitivity_precision' => $sensitive ? 10000 : NULL,
+      'occurrence:verifier_only_data' => isset($observation['verifierOnlyData']) ? $observation['verifierOnlyData'] : NULL,
     ];
     if (!empty($observation['licenceCode'])) {
       $values['sample:licence_id'] = self::getLicenceIdFromCode($db, $observation['licenceCode']);
diff --git a/modules/rest_api_sync/helpers/rest_api_sync_remote_json_occurrences.php b/modules/rest_api_sync/helpers/rest_api_sync_remote_json_occurrences.php
index 4915f2e217..c97658e59d 100644
--- a/modules/rest_api_sync/helpers/rest_api_sync_remote_json_occurrences.php
+++ b/modules/rest_api_sync/helpers/rest_api_sync_remote_json_occurrences.php
@@ -89,7 +89,6 @@ public static function syncPage($serverId, array $server) {
     $tracker = ['inserts' => 0, 'updates' => 0, 'errors' => 0];
     foreach ($data['data'] as $record) {
       // @todo Make sure all fields in specification are handled
-      // @todo dynamicProperties field.
       // @todo occurrence.associated_media
       // @todo occurrence.occurrence_status
       // @todo occurrence.organism_quantity
@@ -102,7 +101,6 @@ public static function syncPage($serverId, array $server) {
         $observation = [
           'licenceCode' => empty($record['record-level']['license']) ? NULL : $record['record-level']['license'],
           'collectionCode' => empty($record['record-level']['collectionCode']) ? NULL : $record['record-level']['collectionCode'],
-          'occurrenceMetadata' => empty($record['record-level']['dynamicProperties']) ? NULL : $record['record-level']['dynamicProperties'],
           'id' => $record['occurrence']['occurrenceID'],
           'individualCount' => empty($record['occurrence']['individualCount']) ? NULL : $record['occurrence']['individualCount'],
           'lifeStage' => empty($record['occurrence']['lifeStage']) ? NULL : $record['occurrence']['lifeStage'],
@@ -143,17 +141,11 @@ public static function syncPage($serverId, array $server) {
             throw new exception('Invalid grid reference format: ' . $record['location']['gridReference']);
           }
         }
-        if (!empty($server['otherFields'])) {
-          foreach ($server['otherFields'] as $src => $dest) {
-            $path = explode('.', $src);
-            if (!empty($record[$path[0]]) && !empty($record[$path[1]])) {
-              // @todo Check multi-value/array handling.
-              $attrTokens = explode(':', $dest);
-              $observation[$attrTokens[0] . 's'][$attrTokens[1]] = $record[$path[0]][$path[1]];
-            }
-          }
+        if (!empty($record['record-level']['dynamicProperties']) && !empty($record['record-level']['dynamicProperties']['verifierOnlyData'])) {
+          $observation['verifierOnlyData'] = json_encode($record['record-level']['dynamicProperties']['verifierOnlyData']);
+          unset($record['record-level']['dynamicProperties']['verifierOnlyData']);
         }
-
+        self::processOtherFields($server, $record, $observation);
         $is_new = api_persist::taxonObservation(
           $db,
           $observation,
@@ -213,6 +205,41 @@ public static function syncPage($serverId, array $server) {
     return $r;
   }
 
+  /**
+   * Process any other field mappings defined by the server config.
+   *
+   * @param array $server
+   *   Server configuration.
+   * @param array $record
+   *   Record structure supplied by the remote server.
+   * @param array $observation
+   *   Observation values to store in Indicia. Will be updated as appropriate.
+   */
+  private static function processOtherFields(array $server, array $record, array &$observation) {
+    if (!empty($server['otherFields'])) {
+      foreach ($server['otherFields'] as $src => $dest) {
+        $path = explode('.', $src);
+        $posInDoc = $record;
+        $found = TRUE;
+        foreach ($path as $node) {
+          if (!isset($posInDoc[$node])) {
+            $found = FALSE;
+            break;
+          }
+          $posInDoc = $posInDoc[$node];
+        }
+        if ($found) {
+          // @todo Check multi-value/array handling.
+          $attrTokens = explode(':', $dest);
+          if (is_object($posInDoc) || is_array($posInDoc)) {
+            $posInDoc = json_encode($posInDoc);
+          }
+          $observation[$attrTokens[0] . 's'][$attrTokens[1]] = $posInDoc;
+        }
+      }
+    }
+  }
+
   /**
    * Parses a date string into the start, end and type.
    *

From d0220d6901706d3cf948f93ef31206cbc3270a87 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Thu, 9 Sep 2021 11:30:06 +0100
Subject: [PATCH 65/93] Fix identificationRemarks support

---
 .../helpers/rest_api_sync_remote_json_occurrences.php            | 1 +
 1 file changed, 1 insertion(+)

diff --git a/modules/rest_api_sync/helpers/rest_api_sync_remote_json_occurrences.php b/modules/rest_api_sync/helpers/rest_api_sync_remote_json_occurrences.php
index c97658e59d..f7d971da91 100644
--- a/modules/rest_api_sync/helpers/rest_api_sync_remote_json_occurrences.php
+++ b/modules/rest_api_sync/helpers/rest_api_sync_remote_json_occurrences.php
@@ -120,6 +120,7 @@ public static function syncPage($serverId, array $server) {
           'siteName' => empty($record['location']['locality']) ? NULL : $record['location']['locality'],
           'identifiedBy' => empty($record['identification']['identifiedBy']) ? NULL : $record['identification']['identifiedBy'],
           'identificationVerificationStatus' => empty($record['identification']['identificationVerificationStatus']) ? NULL : $record['identification']['identificationVerificationStatus'],
+          'identificationRemarks' => empty($record['identification']['identificationRemarks']) ? NULL : $record['identification']['identificationRemarks'],
         ];
         if (!empty($record['location']['decimalLongitude']) && !empty($record['location']['decimalLatitude'])) {
           // Json_decode() converts some floats to scientific notation, so

From 86309b40de80b11f471311ce93d72e641d3d4a63 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Thu, 9 Sep 2021 11:48:45 +0100
Subject: [PATCH 66/93] Exclude auto-generatead annotations

---
 .../reports/rest_api_sync/filterable_annotations.xml            | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/modules/rest_api_sync/reports/rest_api_sync/filterable_annotations.xml b/modules/rest_api_sync/reports/rest_api_sync/filterable_annotations.xml
index c903d42c71..ae61e85eef 100644
--- a/modules/rest_api_sync/reports/rest_api_sync/filterable_annotations.xml
+++ b/modules/rest_api_sync/reports/rest_api_sync/filterable_annotations.xml
@@ -15,6 +15,8 @@
     and oc.deleted=false
     -- Only annotations originating on this warehouse.
     and oc.external_key is null
+    -- No system generated notifications
+    and oc.auto_generated=false
     and o.taxa_taxon_list_external_key is not null
     #idlist#
   </query>

From bfbf4b1896e55c7907518e1bafb39708c50f01ec Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Thu, 9 Sep 2021 15:03:19 +0100
Subject: [PATCH 67/93] External_key in cache_samples_functional for
 consistency

---
 .../cache_builder/config/cache_builder.php    | 34 ++++++++++---------
 .../202109091456_sample_ext_key.sql           |  2 ++
 2 files changed, 20 insertions(+), 16 deletions(-)
 create mode 100644 modules/cache_builder/db/version_6_3_0/202109091456_sample_ext_key.sql

diff --git a/modules/cache_builder/config/cache_builder.php b/modules/cache_builder/config/cache_builder.php
index de820d3d1a..9f48222db7 100644
--- a/modules/cache_builder/config/cache_builder.php
+++ b/modules/cache_builder/config/cache_builder.php
@@ -109,7 +109,7 @@
 $config['termlists_terms']['join_needs_update'] = 'join needs_update_termlists_terms nu on nu.id=tlt.id and nu.deleted=false';
 $config['termlists_terms']['key_field'] = 'tlt.id';
 
-//--------------------------------------------------------------------------------------------------------------------------
+//-----------------------------------------------------------------------------
 
 $config['taxa_taxon_lists']['get_missing_items_query'] = "
     select distinct on (ttl.id) ttl.id, tl.deleted or ttl.deleted or ttlpref.deleted or t.deleted
@@ -313,7 +313,7 @@
 $config['taxa_taxon_lists']['join_needs_update'] = 'join needs_update_taxa_taxon_lists nu on nu.id=ttl.id and nu.deleted=false';
 $config['taxa_taxon_lists']['key_field'] = 'ttl.id';
 
-$config['taxa_taxon_lists']['extra_multi_record_updates'] = array(
+$config['taxa_taxon_lists']['extra_multi_record_updates'] = [
   'setup' => "
     -- Find children of updated taxa to ensure they are also changed.
     WITH RECURSIVE q AS (
@@ -428,7 +428,7 @@
     DROP TABLE descendants;
     DROP TABLE ttl_path;
     DROP TABLE master_list_paths;",
-);
+];
 
 // --------------------------------------------------------------------------------------------------------------------------
 
@@ -880,10 +880,10 @@
     group by id
 ";
 
-$config['samples']['delete_query'] = array(
+$config['samples']['delete_query'] = [
   "delete from cache_samples_functional where id in (select id from needs_update_samples where deleted=true);
 delete from cache_samples_nonfunctional where id in (select id from needs_update_samples where deleted=true);",
-);
+];
 
 $config['samples']['update']['functional'] = "
 UPDATE cache_samples_functional s_update
@@ -909,7 +909,8 @@
     else 'A'
   end,
   parent_sample_id=s.parent_id,
-  media_count=(SELECT COUNT(sm.*) FROM sample_media sm WHERE sm.sample_id=s.id AND sm.deleted=false)
+  media_count=(SELECT COUNT(sm.*) FROM sample_media sm WHERE sm.sample_id=s.id AND sm.deleted=false),
+  external_key=s.external_key
 FROM samples s
 #join_needs_update#
 LEFT JOIN samples sp ON sp.id=s.parent_id AND  sp.deleted=false
@@ -1075,7 +1076,7 @@
 INSERT INTO cache_samples_functional(
             id, website_id, survey_id, input_form, location_id, location_name,
             public_geom, date_start, date_end, date_type, created_on, updated_on, verified_on, created_by_id,
-            group_id, record_status, training, query, parent_sample_id, media_count)
+            group_id, record_status, training, query, parent_sample_id, media_count, external_key)
 SELECT distinct on (s.id) s.id, su.website_id, s.survey_id, COALESCE(sp.input_form, s.input_form), s.location_id,
   CASE WHEN s.privacy_precision IS NOT NULL THEN NULL ELSE COALESCE(l.name, s.location_name, lp.name, sp.location_name) END,
   reduce_precision(coalesce(s.geom, l.centroid_geom), false, s.privacy_precision),
@@ -1087,7 +1088,8 @@
     else 'A'
   end,
   s.parent_id,
-  (SELECT COUNT(sm.*) FROM sample_media sm WHERE sm.sample_id=s.id AND sm.deleted=false)
+  (SELECT COUNT(sm.*) FROM sample_media sm WHERE sm.sample_id=s.id AND sm.deleted=false),
+  s.external_key
 FROM samples s
 #join_needs_update#
 LEFT JOIN cache_samples_functional cs on cs.id=s.id
@@ -1263,7 +1265,7 @@
 // Additional update statements to pick up the recorder name from various possible custom attribute places. Faster than
 // loads of left joins. These should be in priority order - i.e. ones where we have recorded the inputter rather than
 // specifically the recorder should come after ones where we have recorded the recorder specifically.
-$config['samples']['extra_multi_record_updates'] = array(
+$config['samples']['extra_multi_record_updates'] = [
   // s.recorder_names is filled in as a starting point. The rest only proceed if this is null.
   // full recorder name
   // or surname, firstname.
@@ -1336,11 +1338,11 @@
     from needs_update_samples nu, users u
     join cache_samples_functional csf on csf.created_by_id=u.id
     where cs.recorders is null and nu.id=cs.id
-    and cs.id=csf.id and u.id<>1;'
-);
+    and cs.id=csf.id and u.id<>1;',
+];
 
 // Final statements to pick up after an insert of a single record.
-$config['samples']['extra_single_record_updates'] = array(
+$config['samples']['extra_single_record_updates'] = [
   // Sample recorder names
   // Or, full recorder name
   // Or, surname, firstname.
@@ -1415,8 +1417,8 @@
     from users u
     join cache_samples_functional csf on csf.created_by_id=u.id
     where cs.recorders is null and cs.id in (#ids#)
-    and cs.id=csf.id and u.id<>1;'
-);
+    and cs.id=csf.id and u.id<>1;',
+];
 
 // ---------------------------------------------------------------------------------------------------------------------
 
@@ -1465,10 +1467,10 @@
     ) as sub
     group by id";
 
-$config['occurrences']['delete_query'] = array(
+$config['occurrences']['delete_query'] = [
   "delete from cache_occurrences_functional where id in (select id from needs_update_occurrences where deleted=true);
 delete from cache_occurrences_nonfunctional where id in (select id from needs_update_occurrences where deleted=true);"
-);
+];
 
 $config['occurrences']['update']['functional'] = "
 UPDATE cache_occurrences_functional
diff --git a/modules/cache_builder/db/version_6_3_0/202109091456_sample_ext_key.sql b/modules/cache_builder/db/version_6_3_0/202109091456_sample_ext_key.sql
new file mode 100644
index 0000000000..8e6ef95429
--- /dev/null
+++ b/modules/cache_builder/db/version_6_3_0/202109091456_sample_ext_key.sql
@@ -0,0 +1,2 @@
+ALTER TABLE cache_samples_functional
+  ADD COLUMN external_key character varying;
\ No newline at end of file

From 281b637b9bc2c4dcce26b45d3f985307f1c9b21e Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Thu, 9 Sep 2021 15:05:09 +0100
Subject: [PATCH 68/93] Update script for external_key cache field.

---
 .../db/version_6_3_0/202109091503_update_ext_key.sql        | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 modules/cache_builder/db/version_6_3_0/202109091503_update_ext_key.sql

diff --git a/modules/cache_builder/db/version_6_3_0/202109091503_update_ext_key.sql b/modules/cache_builder/db/version_6_3_0/202109091503_update_ext_key.sql
new file mode 100644
index 0000000000..83261cf677
--- /dev/null
+++ b/modules/cache_builder/db/version_6_3_0/202109091503_update_ext_key.sql
@@ -0,0 +1,6 @@
+-- #slow script#
+
+UPDATE cache_samples_functional s
+SET external_key=smp.external_key
+FROM samples smp
+WHERE smp.id=s.id;
\ No newline at end of file

From 3db5489b436b70bd54e6ba0a70a1153aa1f08673 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Fri, 10 Sep 2021 12:08:29 +0100
Subject: [PATCH 69/93] New report to tidy code for location select control

---
 .../locations_for_cms_user.xml                | 29 +++++++++++++++++++
 1 file changed, 29 insertions(+)
 create mode 100644 reports/reports_for_prebuilt_forms/report_calendar_grid/locations_for_cms_user.xml

diff --git a/reports/reports_for_prebuilt_forms/report_calendar_grid/locations_for_cms_user.xml b/reports/reports_for_prebuilt_forms/report_calendar_grid/locations_for_cms_user.xml
new file mode 100644
index 0000000000..34a218ab9e
--- /dev/null
+++ b/reports/reports_for_prebuilt_forms/report_calendar_grid/locations_for_cms_user.xml
@@ -0,0 +1,29 @@
+<report
+    title="Locations for drop down"
+    description="List of user's locations available to show on the calendar."
+>
+  <query website_filter_field="lw.website_id">
+    SELECT #columns#
+    FROM locations l
+    JOIN location_attribute_values v ON v.location_id=l.id AND v.deleted=false AND v.int_value=#cms_user_id#
+    JOIN location_attributes a ON a.id=v.location_attribute_id AND a.deleted=false AND a.caption='CMS User ID'
+    JOIN location_attributes_websites aw ON aw.location_attribute_id=a.id AND aw.deleted=false AND aw.restrict_to_survey_id=#survey_id#
+    JOIN locations_websites lw ON lw.location_id=l.id AND lw.deleted=false
+    WHERE l.deleted=false
+  </query>
+  <order_bys>
+    <order_by>l.name ASC</order_by>
+  </order_bys>
+  <params>
+    <param name="cms_user_id" display="CMS user ID" datatype="integer"/>
+    <param name="survey_id" display="Survey ID" datatype="integer"/>
+    <param name="location_type_id" display="Location type ID" datatype="integer" default="">
+      <where>l.location_type_id=#location_type_id#</where>
+    </param>
+  </params>
+  <columns>
+    <column name="id" sql="l.id" />
+    <column name="name" sql="l.name || 'test'" />
+    <column name="centroid_sref" sql="l.centroid_sref" />
+  </columns>
+</report>
\ No newline at end of file

From 8daf7410546122c8f70e57135474d70010b7edde Mon Sep 17 00:00:00 2001
From: Jim Bacon <jame2@ceh.ac.uk>
Date: Fri, 10 Sep 2021 13:39:16 +0100
Subject: [PATCH 70/93] Fix undefined variable exception

Arose when there were no existingAttrs.
---
 application/views/attribute_by_survey/index.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/application/views/attribute_by_survey/index.php b/application/views/attribute_by_survey/index.php
index ab6131b13e..50a5b208ca 100644
--- a/application/views/attribute_by_survey/index.php
+++ b/application/views/attribute_by_survey/index.php
@@ -223,7 +223,7 @@ function get_controls($block_id, array $controlFilter, $db) {
 </form>
 <?php
 // The JavaScript needs a list of attribute captions.
-$attrsData = [];
+$attrs = [];
 foreach ($existingAttrs as $attr) {
   $attrs["id$attr->id"] = $attr->caption;
 }

From 04372bc93c191d0d861d2c5d9fd6082506c55e07 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Fri, 10 Sep 2021 13:39:31 +0100
Subject: [PATCH 71/93] Corrected some debug code

---
 .../report_calendar_grid/locations_for_cms_user.xml             | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/reports/reports_for_prebuilt_forms/report_calendar_grid/locations_for_cms_user.xml b/reports/reports_for_prebuilt_forms/report_calendar_grid/locations_for_cms_user.xml
index 34a218ab9e..bdd52da219 100644
--- a/reports/reports_for_prebuilt_forms/report_calendar_grid/locations_for_cms_user.xml
+++ b/reports/reports_for_prebuilt_forms/report_calendar_grid/locations_for_cms_user.xml
@@ -23,7 +23,7 @@
   </params>
   <columns>
     <column name="id" sql="l.id" />
-    <column name="name" sql="l.name || 'test'" />
+    <column name="name" sql="l.name" />
     <column name="centroid_sref" sql="l.centroid_sref" />
   </columns>
 </report>
\ No newline at end of file

From 2cf764ae4f3ade3f54480099253b424b45b928f9 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Fri, 10 Sep 2021 14:01:52 +0100
Subject: [PATCH 72/93] Fixed report filtering

---
 .../report_calendar_grid/locations_for_cms_user.xml        | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/reports/reports_for_prebuilt_forms/report_calendar_grid/locations_for_cms_user.xml b/reports/reports_for_prebuilt_forms/report_calendar_grid/locations_for_cms_user.xml
index bdd52da219..0f5803ead2 100644
--- a/reports/reports_for_prebuilt_forms/report_calendar_grid/locations_for_cms_user.xml
+++ b/reports/reports_for_prebuilt_forms/report_calendar_grid/locations_for_cms_user.xml
@@ -2,14 +2,15 @@
     title="Locations for drop down"
     description="List of user's locations available to show on the calendar."
 >
-  <query website_filter_field="lw.website_id">
+  <query website_filter_field="">
     SELECT #columns#
     FROM locations l
     JOIN location_attribute_values v ON v.location_id=l.id AND v.deleted=false AND v.int_value=#cms_user_id#
     JOIN location_attributes a ON a.id=v.location_attribute_id AND a.deleted=false AND a.caption='CMS User ID'
     JOIN location_attributes_websites aw ON aw.location_attribute_id=a.id AND aw.deleted=false AND aw.restrict_to_survey_id=#survey_id#
-    JOIN locations_websites lw ON lw.location_id=l.id AND lw.deleted=false
+    JOIN locations_websites lw ON lw.location_id=l.id AND lw.deleted=false AND lw.website_id in (#website_ids#)
     WHERE l.deleted=false
+	#filters#
   </query>
   <order_bys>
     <order_by>l.name ASC</order_by>
@@ -17,7 +18,7 @@
   <params>
     <param name="cms_user_id" display="CMS user ID" datatype="integer"/>
     <param name="survey_id" display="Survey ID" datatype="integer"/>
-    <param name="location_type_id" display="Location type ID" datatype="integer" default="">
+    <param name="location_type_id" display="Location type ID" datatype="integer" default="" >
       <where>l.location_type_id=#location_type_id#</where>
     </param>
   </params>

From 4d152f6ab28bd4f9664feaa013587aff587464b0 Mon Sep 17 00:00:00 2001
From: Jim Bacon <jame2@ceh.ac.uk>
Date: Wed, 15 Sep 2021 13:33:04 +0100
Subject: [PATCH 73/93] Switch to dbunit from github

DbUnit is being updated and now supports PHP8 but this has not been
released on packagist.
---
 composer.json |  8 ++++-
 composer.lock | 81 ++++++++++++++++++++++++++++-----------------------
 2 files changed, 52 insertions(+), 37 deletions(-)

diff --git a/composer.json b/composer.json
index 25a01df3d6..8faa3968d8 100644
--- a/composer.json
+++ b/composer.json
@@ -1,10 +1,16 @@
 {
+  "repositories": [
+    {
+      "type": "vcs",
+      "url": "https://github.com/misantron/dbunit"
+    }
+  ],
   "require": {
     "firebase/php-jwt": "^5.4",
     "phpoffice/phpspreadsheet": "^1.18"
   },
   "require-dev": {
     "phpunit/phpunit": "^9.5",
-    "misantron/dbunit": "^5.1"
+    "misantron/dbunit": "dev-master"
   }
 }
diff --git a/composer.lock b/composer.lock
index f9d9ce3713..6eff9858be 100644
--- a/composer.lock
+++ b/composer.lock
@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "c505b8dc101b76633c8ae935e66c78b2",
+    "content-hash": "2fccf8723ed1dfa22f7247880fd11bd9",
     "packages": [
         {
             "name": "ezyang/htmlpurifier",
@@ -885,37 +885,45 @@
         },
         {
             "name": "misantron/dbunit",
-            "version": "5.1.0",
+            "version": "dev-master",
             "source": {
                 "type": "git",
                 "url": "https://github.com/misantron/dbunit.git",
-                "reference": "a3e5d3c74a2ae78827c86e14e3d06d7a8d44ca65"
+                "reference": "73a9c07dca119c68e92002adb4fab9022235a91f"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/misantron/dbunit/zipball/a3e5d3c74a2ae78827c86e14e3d06d7a8d44ca65",
-                "reference": "a3e5d3c74a2ae78827c86e14e3d06d7a8d44ca65",
+                "url": "https://api.github.com/repos/misantron/dbunit/zipball/73a9c07dca119c68e92002adb4fab9022235a91f",
+                "reference": "73a9c07dca119c68e92002adb4fab9022235a91f",
                 "shasum": ""
             },
             "require": {
-                "ext-libxml": "*",
                 "ext-pdo": "*",
-                "ext-simplexml": "*",
-                "php": "^7.2|^7.3|^7.4",
-                "phpunit/phpunit": "^8.5|^9.2",
-                "symfony/yaml": "^4.4|^5.0"
+                "php": "^7.2 || ^8.0",
+                "phpunit/phpunit": "^8.5 || ^9.2",
+                "symfony/yaml": "^4.4 || ^5.0"
             },
             "require-dev": {
-                "friendsofphp/php-cs-fixer": "^2.16",
-                "php-coveralls/php-coveralls": "^2.2"
+                "friendsofphp/php-cs-fixer": "^2.18 || ^3.0",
+                "php-coveralls/php-coveralls": "^2.4",
+                "phpstan/phpstan": "^0.12.98",
+                "squizlabs/php_codesniffer": "^3.6"
             },
+            "default-branch": true,
             "type": "library",
             "autoload": {
                 "psr-4": {
                     "PHPUnit\\DbUnit\\": "src/"
                 }
             },
-            "notification-url": "https://packagist.org/downloads/",
+            "autoload-dev": {
+                "psr-4": {
+                    "PHPUnit\\DbUnit\\Tests\\": "tests/"
+                },
+                "files": [
+                    "tests/_files/DatabaseTestUtility.php"
+                ]
+            },
             "license": [
                 "MIT"
             ],
@@ -923,21 +931,20 @@
                 {
                     "name": "Aleksandr Ivanov",
                     "email": "misantron@gmail.com",
-                    "role": "developer"
+                    "role": "Developer"
                 }
             ],
             "description": "DbUnit fork supporting PHPUnit 8/9",
-            "homepage": "https://github.com/misantron/dbunit/",
             "keywords": [
                 "database",
-                "dbUnit",
+                "dbunit",
                 "testing"
             ],
             "support": {
-                "issues": "https://github.com/misantron/dbunit/issues",
-                "source": "https://github.com/misantron/dbunit/tree/master"
+                "source": "https://github.com/misantron/dbunit/tree/master",
+                "issues": "https://github.com/misantron/dbunit/issues"
             },
-            "time": "2020-07-07T20:48:08+00:00"
+            "time": "2021-09-09T19:30:23+00:00"
         },
         {
             "name": "myclabs/deep-copy",
@@ -1324,33 +1331,33 @@
         },
         {
             "name": "phpspec/prophecy",
-            "version": "1.13.0",
+            "version": "1.14.0",
             "source": {
                 "type": "git",
                 "url": "https://github.com/phpspec/prophecy.git",
-                "reference": "be1996ed8adc35c3fd795488a653f4b518be70ea"
+                "reference": "d86dfc2e2a3cd366cee475e52c6bb3bbc371aa0e"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/phpspec/prophecy/zipball/be1996ed8adc35c3fd795488a653f4b518be70ea",
-                "reference": "be1996ed8adc35c3fd795488a653f4b518be70ea",
+                "url": "https://api.github.com/repos/phpspec/prophecy/zipball/d86dfc2e2a3cd366cee475e52c6bb3bbc371aa0e",
+                "reference": "d86dfc2e2a3cd366cee475e52c6bb3bbc371aa0e",
                 "shasum": ""
             },
             "require": {
                 "doctrine/instantiator": "^1.2",
-                "php": "^7.2 || ~8.0, <8.1",
+                "php": "^7.2 || ~8.0, <8.2",
                 "phpdocumentor/reflection-docblock": "^5.2",
                 "sebastian/comparator": "^3.0 || ^4.0",
                 "sebastian/recursion-context": "^3.0 || ^4.0"
             },
             "require-dev": {
-                "phpspec/phpspec": "^6.0",
+                "phpspec/phpspec": "^6.0 || ^7.0",
                 "phpunit/phpunit": "^8.0 || ^9.0"
             },
             "type": "library",
             "extra": {
                 "branch-alias": {
-                    "dev-master": "1.11.x-dev"
+                    "dev-master": "1.x-dev"
                 }
             },
             "autoload": {
@@ -1385,9 +1392,9 @@
             ],
             "support": {
                 "issues": "https://github.com/phpspec/prophecy/issues",
-                "source": "https://github.com/phpspec/prophecy/tree/1.13.0"
+                "source": "https://github.com/phpspec/prophecy/tree/1.14.0"
             },
-            "time": "2021-03-17T13:42:18+00:00"
+            "time": "2021-09-10T09:02:12+00:00"
         },
         {
             "name": "phpunit/php-code-coverage",
@@ -1709,16 +1716,16 @@
         },
         {
             "name": "phpunit/phpunit",
-            "version": "9.5.7",
+            "version": "9.5.9",
             "source": {
                 "type": "git",
                 "url": "https://github.com/sebastianbergmann/phpunit.git",
-                "reference": "d0dc8b6999c937616df4fb046792004b33fd31c5"
+                "reference": "ea8c2dfb1065eb35a79b3681eee6e6fb0a6f273b"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/d0dc8b6999c937616df4fb046792004b33fd31c5",
-                "reference": "d0dc8b6999c937616df4fb046792004b33fd31c5",
+                "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/ea8c2dfb1065eb35a79b3681eee6e6fb0a6f273b",
+                "reference": "ea8c2dfb1065eb35a79b3681eee6e6fb0a6f273b",
                 "shasum": ""
             },
             "require": {
@@ -1730,7 +1737,7 @@
                 "ext-xml": "*",
                 "ext-xmlwriter": "*",
                 "myclabs/deep-copy": "^1.10.1",
-                "phar-io/manifest": "^2.0.1",
+                "phar-io/manifest": "^2.0.3",
                 "phar-io/version": "^3.0.2",
                 "php": ">=7.3",
                 "phpspec/prophecy": "^1.12.1",
@@ -1796,7 +1803,7 @@
             ],
             "support": {
                 "issues": "https://github.com/sebastianbergmann/phpunit/issues",
-                "source": "https://github.com/sebastianbergmann/phpunit/tree/9.5.7"
+                "source": "https://github.com/sebastianbergmann/phpunit/tree/9.5.9"
             },
             "funding": [
                 {
@@ -1808,7 +1815,7 @@
                     "type": "github"
                 }
             ],
-            "time": "2021-07-19T06:14:47+00:00"
+            "time": "2021-08-31T06:47:40+00:00"
         },
         {
             "name": "sebastian/cli-parser",
@@ -3107,7 +3114,9 @@
     ],
     "aliases": [],
     "minimum-stability": "stable",
-    "stability-flags": [],
+    "stability-flags": {
+        "misantron/dbunit": 20
+    },
     "prefer-stable": false,
     "prefer-lowest": false,
     "platform": [],

From ad041c00f463b5535bebb8f9e68ecf4f5791d069 Mon Sep 17 00:00:00 2001
From: Jim Bacon <jame2@ceh.ac.uk>
Date: Wed, 15 Sep 2021 13:34:21 +0100
Subject: [PATCH 74/93] Switch to PHP8

---
 docker/phpunit.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/phpunit.sh b/docker/phpunit.sh
index 414ef6f983..8a8b2b1fbb 100755
--- a/docker/phpunit.sh
+++ b/docker/phpunit.sh
@@ -25,7 +25,7 @@ docker-compose -f docker-compose-phpunit.yml build \
   --build-arg GID=$(id -g) \
   --build-arg USER=$(id -un) \
   --build-arg GROUP=$(id -gn) \
-  --build-arg PHP_VERSION=7.3 \
+  --build-arg PHP_VERSION=8 \
   --build-arg PG_VERSION=13 \
   --build-arg PORT=$PORT
 # When the container is brought up, the database will start 

From 0d7a5919295724320b1723865e08c107ce3c0f6a Mon Sep 17 00:00:00 2001
From: Jim Bacon <jame2@ceh.ac.uk>
Date: Wed, 15 Sep 2021 13:37:13 +0100
Subject: [PATCH 75/93] Fix test function signature

---
 modules/phpUnit/tests/Helper_Text_Test.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/phpUnit/tests/Helper_Text_Test.php b/modules/phpUnit/tests/Helper_Text_Test.php
index 0082bc2f22..44357e9743 100644
--- a/modules/phpUnit/tests/Helper_Text_Test.php
+++ b/modules/phpUnit/tests/Helper_Text_Test.php
@@ -277,7 +277,7 @@ public function censor_provider()
 	 * @group core.helpers.text.censor
 	 * @test
 	 */
-	public function censor($str, $badwords, $replacement = '#', $replace_partial_words = FALSE, $expected_result)
+	public function censor($str, $badwords, $replacement, $replace_partial_words, $expected_result)
 	{
 		$result = text::censor($str, $badwords, $replacement, $replace_partial_words);
 		$this->assertEquals($expected_result, $result);

From f2c7aab852c1e9d67a993a6f2c3bba2f884c7183 Mon Sep 17 00:00:00 2001
From: Jim Bacon <jame2@ceh.ac.uk>
Date: Wed, 15 Sep 2021 14:42:04 +0100
Subject: [PATCH 76/93] Fix error in core fixture

---
 modules/phpUnit/config/core_fixture.php | 1 +
 1 file changed, 1 insertion(+)

diff --git a/modules/phpUnit/config/core_fixture.php b/modules/phpUnit/config/core_fixture.php
index c5c7fd2622..dd431f2583 100644
--- a/modules/phpUnit/config/core_fixture.php
+++ b/modules/phpUnit/config/core_fixture.php
@@ -206,6 +206,7 @@
   ],
   "cache_taxon_searchterms" => [
     [
+      "id" => 1,
       "taxa_taxon_list_id" => 1,
       "taxon_list_id" => 1,
       "searchterm" => "testtaxon",

From 7d5060a1f670c9108625a6c05d9db74b6c7f7b55 Mon Sep 17 00:00:00 2001
From: Jim Bacon <jame2@ceh.ac.uk>
Date: Wed, 15 Sep 2021 15:11:24 +0100
Subject: [PATCH 77/93] Fix errors introduced when linting

---
 .../rest_api/tests/Rest_ControllerTest.php    | 20 +++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/modules/rest_api/tests/Rest_ControllerTest.php b/modules/rest_api/tests/Rest_ControllerTest.php
index adce690202..2785f0a83d 100644
--- a/modules/rest_api/tests/Rest_ControllerTest.php
+++ b/modules/rest_api/tests/Rest_ControllerTest.php
@@ -2012,8 +2012,8 @@ public function testProjects_get_id() {
   public function testTaxon_observations_authentication() {
     Kohana::log('debug', "Running unit test, Rest_ControllerTest::testProjects_clientAuthentication");
     $proj_id = self::$config['projects'][array_keys(self::$config['projects'])[0]]['id'];
-    $queryWithProj = ['proj_id' => $proj_id, 'edited_date_from' => '2015-01-01');
-    $query = ['edited_date_from' => '2015-01-01');
+    $queryWithProj = ['proj_id' => $proj_id, 'edited_date_from' => '2015-01-01'];
+    $query = ['edited_date_from' => '2015-01-01'];
 
     $this->authMethod = 'hmacClient';
     $this->checkResourceAuthentication('taxon-observations', $queryWithProj);
@@ -2023,7 +2023,7 @@ public function testTaxon_observations_authentication() {
     $this->checkResourceAuthentication('taxon-observations', $query);
     // @todo The following test needs to check filtered response rather than authentication
     $this->authMethod = 'directUser';
-    $this->checkResourceAuthentication('taxon-observations', $query + ['filter_id' => self::$userFilterId));
+    $this->checkResourceAuthentication('taxon-observations', $query + ['filter_id' => self::$userFilterId]);
     $this->authMethod = 'hmacWebsite';
     $this->checkResourceAuthentication('taxon-observations', $query);
     $this->authMethod = 'directWebsite';
@@ -2038,10 +2038,10 @@ public function testTaxon_observations_get_incorrect_params() {
     $this->assertEquals(400, $response['httpCode'],
         'Requesting taxon observations without params should be a bad request');
     foreach (self::$config['projects'] as $projDef) {
-      $response = $this->callService("taxon-observations", ['proj_id' => $projDef['id']));
+      $response = $this->callService("taxon-observations", ['proj_id' => $projDef['id']]);
       $this->assertEquals(400, $response['httpCode'],
           'Requesting taxon observations without edited_date_from should be a bad request');
-      $response = $this->callService("taxon-observations", ['edited_date_from' => '2015-01-01'));
+      $response = $this->callService("taxon-observations", ['edited_date_from' => '2015-01-01']);
       $this->assertEquals(400, $response['httpCode'],
         'Requesting taxon observations without proj_id should be a bad request');
       // only test a single project
@@ -2062,8 +2062,7 @@ public function testTaxon_observations_get() {
           'proj_id' => $projDef['id'],
           'edited_date_from' => '2015-01-01',
           'edited_date_to' => date("Y-m-d\TH:i:s")
-        )
-      );
+      ]);
       $this->assertResponseOk($response, '/taxon-observations');
       $this->assertArrayHasKey('paging', $response['response'],
           'Paging missing from response to call to taxon-observations');
@@ -2091,7 +2090,7 @@ public function testAnnotations_get() {
     foreach (self::$config['projects'] as $projDef) {
       $response = $this->callService(
         "annotations",
-        ['proj_id' => $projDef['id'], 'edited_date_from' => '2015-01-01')
+        ['proj_id' => $projDef['id'], 'edited_date_from' => '2015-01-01']
       );
       $this->assertResponseOk($response, '/annotations');
       $this->assertArrayHasKey('paging', $response['response'], 'Paging missing from response to call to annotations');
@@ -2234,6 +2233,7 @@ public function testReportColumns_get() {
 
     // First grab a list of reports so we can use the links to get the correct
     // columns URL.
+    $projDef = self::$config['projects']['BRC1'];
     $response = $this->callService("reports/library/occurrences", ['proj_id' => $projDef['id']]);
     $this->assertResponseOk($response, '/reports/library/occurrences');
     $reportDef = $response['response']['filterable_explore_list'];
@@ -2408,7 +2408,7 @@ private function assertResponseOk($response, $apiCall) {
   private function checkValidTaxonObservation($data) {
     $this->assertIsArray($data, 'Taxon-observation object invalid. ' . var_export($data, TRUE));
     $mustHave = ['id', 'href', 'datasetName', 'taxonVersionKey', 'taxonName',
-        'startDate', 'endDate', 'dateType', 'projection', 'precision', 'recorder', 'lastEditDate');
+        'startDate', 'endDate', 'dateType', 'projection', 'precision', 'recorder', 'lastEditDate'];
     foreach ($mustHave as $key) {
       $this->assertArrayHasKey($key, $data,
           "Missing $key from taxon-observation resource. " . var_export($data, TRUE));
@@ -2425,7 +2425,7 @@ private function checkValidTaxonObservation($data) {
   private function checkValidAnnotation($data) {
     $this->assertIsArray($data, 'Annotation object invalid. ' . var_export($data, TRUE));
     $mustHave = ['id', 'href', 'taxonObservation', 'taxonVersionKey', 'comment',
-        'question', 'authorName', 'dateTime');
+        'question', 'authorName', 'dateTime'];
     foreach ($mustHave as $key) {
       $this->assertArrayHasKey($key, $data,
         "Missing $key from annotation resource. " . var_export($data, TRUE));

From 41c394bd65f79ad670d0ecb903b853338a7ca7cf Mon Sep 17 00:00:00 2001
From: Jim Bacon <jame2@ceh.ac.uk>
Date: Thu, 16 Sep 2021 10:16:52 +0100
Subject: [PATCH 78/93] Fix for PHP8 compatability

---
 modules/rest_api/helpers/rest_crud.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/rest_api/helpers/rest_crud.php b/modules/rest_api/helpers/rest_crud.php
index 2bc323b5d2..4fe8f53e90 100644
--- a/modules/rest_api/helpers/rest_crud.php
+++ b/modules/rest_api/helpers/rest_crud.php
@@ -625,7 +625,7 @@ private static function getResponseMetadata(array $responseMetadata) {
         if (preg_match('/_media$/', $subTable)) {
           $subTable = 'media';
         }
-        if (array_key_exists($subTable, self::$entityConfig[$entity]->subModels)) {
+        if (property_exists(self::$entityConfig[$entity]->subModels, $subTable)) {
           if (!isset($r[$subTable])) {
             $r[$subTable] = [];
           }

From 436479927d8ce2c10baad4afc74edf6f639db462 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Thu, 16 Sep 2021 16:43:11 +0100
Subject: [PATCH 79/93] First effort at ES samples extraction report

---
 .../library/samples/list_for_elastic_all.xml  | 202 ++++++++++++++++++
 1 file changed, 202 insertions(+)
 create mode 100644 reports/library/samples/list_for_elastic_all.xml

diff --git a/reports/library/samples/list_for_elastic_all.xml b/reports/library/samples/list_for_elastic_all.xml
new file mode 100644
index 0000000000..557b2cd6dc
--- /dev/null
+++ b/reports/library/samples/list_for_elastic_all.xml
@@ -0,0 +1,202 @@
+<report
+    title="Autofeed samples for Elasticsearch (standard filters, no website limit)"
+    description="A list of samples designed for feeding into Elasticsearch or a similar JSON store.
+      No website limit so designed to collect records from all website registrations."
+    restricted="true"
+>
+  <query website_filter_field="s.website_id" samples_id_field="s.id" training_filter_field=""
+         standard_params="samples" count_field="s.*" blocked_sharing_tasks_field="s.blocked_sharing_tasks">
+    DROP TABLE IF EXISTS filtered_samples;
+    DROP TABLE IF EXISTS occurrence_stats;
+    DROP TABLE IF EXISTS output_rows;
+    DROP TABLE IF EXISTS sample_occurrence_list;
+
+    SET LOCAL enable_incremental_sort = off;
+
+    SELECT DISTINCT s.*, greatest(s.tracking, (SELECT max(o.tracking) FROM cache_occurrences_functional o WHERE o.sample_id=s.id OR o.parent_sample_id=s.id)) AS tracking_inc_occurrences
+    INTO TEMPORARY filtered_samples
+    FROM cache_samples_functional s
+    #joins#
+    WHERE 1=1
+    #filters#
+    #order_by#
+    LIMIT #limit#;
+
+    SELECT DISTINCT s.id as sample_id, o.id as occurrence_id, o.taxa_taxon_list_id, o.confidential, o.release_status
+    INTO TEMPORARY sample_occurrences_list
+    FROM filtered_samples s
+    JOIN cache_occurrences_functional o ON o.sample_id=s.id OR o.parent_sample_id=s.id;
+
+    SELECT s.id as sample_id, COUNT(DISTINCT ol.occurrence_id) AS count_occurrences, COUNT(distinct cttl.taxon_meaning_id) AS count_taxa, COUNT(distinct cttl.taxon_group_id) AS count_taxon_groups,
+      SUM(case when onf.attr_sex_stage_count similar to '[0-9]{1,9}' then onf.attr_sex_stage_count::integer else null end) AS sum_individual_count,
+      MAX(onf.sensitivity_precision) AS max_sensitivity_precision, BOOL_OR(ol.confidential) AS any_confidential, string_agg(DISTINCT ol.release_status, '') as all_release_status
+    INTO TEMPORARY occurrence_stats
+    FROM filtered_samples s
+    LEFT JOIN sample_occurrences_list ol ON ol.sample_id=s.id
+    LEFT JOIN cache_occurrences_nonfunctional onf ON onf.id=ol.occurrence_id
+    LEFT JOIN cache_taxa_taxon_lists cttl ON cttl.id=ol.taxa_taxon_list_id
+    GROUP BY s.id;
+
+    SELECT #columns#
+    INTO TEMPORARY output_rows
+    FROM filtered_samples s
+    JOIN samples smp ON smp.id=s.id
+    JOIN occurrence_stats os ON os.sample_id=s.id
+    JOIN cache_samples_nonfunctional snf ON snf.id=s.id
+    LEFT JOIN occurrences o ON o.sample_id=s.id
+    LEFT JOIN locations l ON l.id=s.location_id AND l.deleted=false;
+
+    UPDATE output_rows o
+    SET recorded_parent_location_id = lp.id,
+      recorded_parent_location_name = lp.name,
+      recorded_parent_location_code = lp.code,
+      parent_sample_attrs_json=snfp.attrs_json
+    FROM samples sp
+    JOIN cache_samples_nonfunctional snfp ON snfp.id=sp.id
+    LEFT JOIN locations lp ON lp.id=sp.location_id AND lp.deleted=false
+    WHERE sp.id=o.parent_sample_id AND sp.deleted=false;
+
+    -- Use parents of locations if no parent sample location.
+    UPDATE output_rows o
+    SET recorded_parent_location_id = lp.id,
+      recorded_parent_location_name = lp.name,
+      recorded_parent_location_code = lp.code
+      FROM samples sc
+    JOIN locations lc ON lc.id=sc.location_id AND lc.deleted=false
+      JOIN locations lp ON lp.id = lc.parent_id AND lp.deleted=false
+      WHERE sc.id=o.sample_id AND sc.deleted=false
+    AND o.recorded_parent_location_id IS NULL;
+
+    SELECT *
+    FROM output_rows s
+    #order_by#
+  </query>
+  <params>
+    <!-- last_id and autofeed_tracking_from parameters required for autofeed to work. -->
+    <param name="last_id" display="Last ID" datatype="integer" default="" order_by="s.id ASC">
+      <where>s.id &gt; #last_id#</where>
+    </param>
+    <param name="autofeed_tracking_from" display="Autofeed tracking from" datatype="integer" default="" order_by="tracking_inc_occurrences ASC">
+      <where>(s.tracking >= #autofeed_tracking_from# OR o.tracking >= #autofeed_tracking_from#)</where> -- WON'T FIRE ON AN OCCURRENCE DELETION
+    </param>
+    <param name="limit" datatype="integer" default="10000" />
+  </params>
+  <columns>
+    <!-- Tracking field called "id" is required for autofeed to work. -->
+    <column name="id" sql="s.id" datatype="integer" />
+    <column name="sample_external_key" sql="s.external_key" datatype="text" />
+    <column name="created_by_id" sql="s.created_by_id" datatype="integer" />
+    <column name="website_id" sql="s.website_id" datatype="integer" />
+    <column name="survey_id" sql="s.survey_id" datatype="integer" />
+    <!-- Deliberate repeat of s.id so it can appear in event.event_id as well as the document ID. -->
+    <column name="sample_id" sql="s.id" datatype="integer" />
+    <column name="parent_sample_id" sql="s.parent_sample_id" datatype="integer" />
+    <column name="group_id" sql="s.group_id" datatype="integer" />
+    <column name="website_title" sql="snf.website_title" datatype="text" />
+    <column name="survey_title" sql="snf.survey_title" datatype="text" />
+    <column name="group_title" sql="snf.group_title" datatype="text" />
+    <column name="recorders" sql="snf.recorders" datatype="text" />
+    <column name="date_start" sql="s.date_start" datatype="date" />
+    <column name="date_end" sql="s.date_end" datatype="date" />
+    <column name="date_type" sql="s.date_type" datatype="text" />
+    <column name="day_of_year" sql="case s.date_type when 'D' then extract(doy from s.date_start) else null end" datatype="integer" />
+    <column name="week" sql="case
+            when coalesce(date_part('year', s.date_start), 0)=coalesce(date_part('year', s.date_end), 0)
+                and coalesce(floor(extract(doy from s.date_start)/7+1), 0)=coalesce(floor(extract(doy from s.date_end)/7+1), 0)
+              then coalesce(floor(extract(doy from s.date_start)/7+1), 0)
+            else null
+          end" datatype="integer" />
+    <column name="ukbms_week" sql="case
+            when coalesce(date_part('year', s.date_start), 0)=coalesce(date_part('year', s.date_end), 0)
+                and
+                  floor((extract('doy' from s.date_start) - extract('doy' from (extract('year' from s.date_start) || '-04-01')::date))/7)
+                  =
+                  floor((extract('doy' from s.date_end) - extract('doy' from (extract('year' from s.date_end) || '-04-01')::date))/7)
+              then floor((extract('doy' from s.date_start) - extract('doy' from (extract('year' from s.date_start) || '-04-01')::date))/7) + 1
+            else null
+          end" datatype="integer" />
+    <column name="month" sql="case
+            when coalesce(date_part('year', s.date_start), 0)=coalesce(date_part('year', s.date_end), 0)
+              and coalesce(date_part('month', s.date_start), 0)=coalesce(date_part('month', s.date_end), 0) then coalesce(date_part('month', s.date_start), 0)
+            else null
+          end" datatype="integer" />
+    <column name="year" sql="case
+            when coalesce(date_part('year', s.date_start), 0)=coalesce(date_part('year', s.date_end), 0) then coalesce(date_part('year', s.date_start), null)
+            else null
+          end" datatype="integer" />
+    <column name="geom" sql="st_astext(st_removerepeatedpoints(st_transform(s.public_geom, 4326)))" datatype="text" />
+    <column name="point" sql="st_y(st_transform(st_centroid(s.public_geom), 4326))::text || ',' || st_x(st_transform(st_centroid(s.public_geom), 4326))::text" datatype="text" />
+    <column name="map_sq_srid" sql="COALESCE(get_output_srid(s.public_geom), 3857)" datatype="integer" />
+    <column name="map_sq_1km" sql="CASE
+      WHEN max_sensitivity_precision > 1000 THEN NULL
+      ELSE TRIM(leading 'POINT(' FROM TRIM(trailing ')' FROM st_astext(st_centroid(st_transform(reduce_precision(s.public_geom, false, 1000), 4326)))))
+    END" datatype="text" />
+    <column name="map_sq_2km" sql="CASE
+      WHEN max_sensitivity_precision > 2000 THEN NULL
+      ELSE TRIM(leading 'POINT(' FROM TRIM(trailing ')' FROM st_astext(st_centroid(st_transform(reduce_precision(s.public_geom, false, 2000), 4326)))))
+    END" datatype="text" />
+    <column name="map_sq_10km" sql="CASE
+      WHEN max_sensitivity_precision > 10000 THEN NULL
+      ELSE TRIM(leading 'POINT(' FROM TRIM(trailing ')' FROM st_astext(st_centroid(st_transform(reduce_precision(s.public_geom, false, 10000), 4326)))))
+    END" datatype="text" />
+    <column name="input_sref" sql="snf.public_entered_sref" datatype="text" />
+    <column name="input_sref_system" sql="snf.entered_sref_system" datatype="text" />
+    <column name="output_sref" sql="snf.output_sref" datatype="text" />
+    <column name="output_sref_system" sql="snf.output_sref_system" datatype="text" />
+    <column name="coordinate_uncertainty_in_meters"
+              sql="CASE
+      WHEN os.max_sensitivity_precision IS NOT NULL OR snf.privacy_precision IS NOT NULL OR snf.entered_sref_system NOT SIMILAR TO '[0-9]+' THEN
+        get_sref_precision(snf.public_entered_sref, snf.entered_sref_system, null)
+      ELSE COALESCE(snf.attr_sref_precision, 50)
+    END"
+            datatype="float"
+            term="http://rs.tdwg.org/dwc/terms/coordinateUncertaintyInMeters" />
+    <column name="given_locality_name" sql="s.location_name" datatype="text" />
+    <column name="recorded_location_id" sql="l.id" datatype="integer" />
+    <column name="recorded_location_name" sql="l.name" datatype="text" />
+    <column name="recorded_location_code" sql="l.code" datatype="text" />
+    <column name="recorded_parent_location_id" sql="null::integer" datatype="integer" />
+    <column name="recorded_parent_location_name" sql="null::text" datatype="text" />
+    <column name="recorded_parent_location_code" sql="null::text" datatype="text" />
+    <column name="indexed_location_ids" sql="array_to_string(s.location_ids, ',')" datatype="text" />
+    <column name="sample_comment" sql="snf.comment" datatype="text" />
+    <column name="attr_biotope" sql="snf.attr_biotope" datatype="text" />
+    <column name="attr_sample_method" sql="snf.attr_sample_method" datatype="text" />
+    <column name="media_data" sql="CASE
+      WHEN s.media_count>0 AND os.max_sensitivity_precision IS NULL AND snf.privacy_precision IS NULL THEN
+        (SELECT string_agg(sm.path || '~~' || COALESCE(sm.caption, '') || '~~' ||  COALESCE(l.code, '') || '~~' ||  COALESCE(t.term, ''), '@@')
+        FROM sample_media sm
+        LEFT JOIN licences l ON l.id=sm.licence_id
+        LEFT JOIN cache_termlists_terms t ON t.id=sm.media_type_id
+        WHERE sm.sample_id=s.id AND sm.deleted=false)
+      ELSE null
+      END" datatype="text" />
+    <column name="created_on" sql="to_char(s.created_on, 'yyyy-mm-dd HH24:MI:SS.MS')" datatype="date" />
+    <column name="updated_on" sql="to_char(s.updated_on, 'yyyy-mm-dd HH24:MI:SS.MS')" datatype="date" />
+    <column name="record_status" sql="s.record_status" datatype="text" />
+    <column name="verified_by_id" sql="smp.verified_by_id" datatype="integer" />
+    <column name="verifier" sql="snf.verifier" datatype="text" />
+    <column name="verified_on" sql="to_char(s.verified_on, 'yyyy-mm-dd HH24:MI:SS.MS')" datatype="date" />
+    <column name="query" sql="s.query" datatype="text" />
+    <column name="trial" sql="s.training::text" datatype="text" />
+    <column name="sensitive" sql="(os.max_sensitivity_precision IS NOT NULL)::text" datatype="text" />
+    <column name="sensitivity_precision" sql="os.max_sensitivity_precision" datatype="integer" />
+    <!-- Flag sensitivity_blur = B for blurred version of a sensitive record. -->
+    <column name="sensitivity_blur" sql="CASE WHEN os.max_sensitivity_precision IS NOT NULL THEN 'B' ELSE null END" datatype="text" />
+    <column name="confidential" sql="os.any_confidential::text" datatype="boolean" />
+    <column name="release_status" description="Worst case release status from the contained occurrences" sql="CASE
+      WHEN POSITION('U' IN os.all_release_status)&lt;&gt;0 THEN 'U'
+	    WHEN POSITION('P' IN os.all_release_status)&lt;&gt;0 THEN 'P'
+		  ELSE 'R'
+	  END" datatype="text" />
+    <column name="sample_attrs_json" sql="snf.attrs_json" datatype="text" />
+    <column name="parent_sample_attrs_json" sql="null::json" datatype="text" />
+    <column name="tracking" sql="s.tracking" datatype="integer" />
+    <column name="tracking_inc_occurrences" sql="s.tracking_inc_occurrences" datatype="integer" />
+    <column name="input_form" sql="s.input_form" datatype="text" />
+    <column name="count_occurrences" sql="os.count_occurrences" datatype="integer" />
+    <column name="count_taxa" sql="os.count_taxa" datatype="integer" />
+    <column name="count_taxon_groups" sql="os.count_taxon_groups" datatype="integer" />
+    <column name="sum_individual_count" sql="os.sum_individual_count" datatype="integer" />
+  </columns>
+</report>
\ No newline at end of file

From daa367e5af2af96a71c86ca5d3c52288912bfd19 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Fri, 17 Sep 2021 14:25:01 +0100
Subject: [PATCH 80/93] Ensure sample tracking updates correctly

E.g. if an occurrence submitted or deleted in isolation to it's sample,
the sample still needs to update the tracking. This ensures feeds that
include occurrence stats are correctly updated.
---
 application/libraries/MY_ORM.php              | 100 +++++++++++-------
 .../cache_builder/helpers/cache_builder.php   |  23 ++++
 2 files changed, 82 insertions(+), 41 deletions(-)

diff --git a/application/libraries/MY_ORM.php b/application/libraries/MY_ORM.php
index 125de9e1e7..697ef492f7 100644
--- a/application/libraries/MY_ORM.php
+++ b/application/libraries/MY_ORM.php
@@ -61,7 +61,7 @@ public function last_query() {
     return $this->db->last_query();
   }
 
-  public $submission = array();
+  public $submission = [];
 
   /**
    * Describes the list of nested models that are present after a submission.
@@ -70,8 +70,8 @@ public function last_query() {
    *
    * @var array
    */
-  private $nestedChildModelIds = array();
-  private $nestedParentModelIds = array();
+  private $nestedChildModelIds = [];
+  private $nestedParentModelIds = [];
 
   /**
    * Default search field name.
@@ -82,7 +82,7 @@ public function last_query() {
    */
   public $search_field = 'title';
 
-  protected $errors = array();
+  protected $errors = [];
 
   /**
    * Flag that gets set if a unique key violation has occurred on save.
@@ -101,12 +101,12 @@ public function last_query() {
    * by a model. If not declared then the model will not transfer them to the saved data when
    * posting a record.
    */
-  protected $unvalidatedFields = array();
+  protected $unvalidatedFields = [];
 
   /**
    * @var array An array which a model can populate to declare additional fields that can be submitted for csv upload.
    */
-  protected $additional_csv_fields = array();
+  protected $additional_csv_fields = [];
 
   /**
    * @var bool Does the model have custom attributes? Defaults to false.
@@ -128,6 +128,7 @@ public function last_query() {
 
   /**
    * Default behaviour on save is to update metadata. If we detect no changes we can skip this.
+   *
    * @var bool
    */
   public $wantToUpdateMetadata = TRUE;
@@ -139,7 +140,7 @@ public function last_query() {
    */
   public $parentChanging = FALSE;
 
-  private $attrValModels = array();
+  private $attrValModels = [];
 
   /**
    * @var array If a submission contains submodels, then the array of submodels can be keyed. This
@@ -147,7 +148,7 @@ public function last_query() {
    * Normally, super/sub-models can handle foreign keys, but this approach is needed for association
    * tables which join across 2 entities created by a submission.
    */
-  private $dynamicRowIdReferences = array();
+  private $dynamicRowIdReferences = [];
 
   /**
    * Indicates database trigger on table which accesses a sequence.
@@ -163,8 +164,11 @@ public function last_query() {
 
   /**
    * Constructor allows plugins to modify the data model.
-   * @var int $id ID of the record to load. If null then creates a new record. If -1 then the ORM
-   * object is not initialised, providing access to the variables only.
+   *
+   * @var int $id
+   *   ID of the record to load. If null then creates a new record. If -1 then
+   *   the ORM object is not initialised, providing access to the variables
+   *   only.
    */
   public function __construct($id = NULL) {
     if (is_object($id) || $id != -1) {
@@ -343,7 +347,7 @@ public function getAllErrors()
    * Retrieve an array containing all page level errors which are marked with the key general.
    */
   public function getPageErrors() {
-    $r = array();
+    $r = [];
     if (array_key_exists('general', $this->errors)) {
       array_push($r, $this->errors['general']);
     }
@@ -567,7 +571,7 @@ protected function canCreateFromCaption() {
    * @return array, an array of record id values for the created records.
    */
   private function createRecordsFromCaptions() {
-    $r = array();
+    $r = [];
 
     // Establish the right model and check it supports create from captions,
     $modelname = $this->submission['fields']['insert_captions_to_create']['value'];
@@ -584,7 +588,7 @@ private function createRecordsFromCaptions() {
       $sub = array(
         'id' => $modelname,
         'fields' => array(
-          'caption' => array()
+          'caption' => []
         )
       );
       // submit each caption to create a record, unless it exists
@@ -621,10 +625,10 @@ private function createRecordsFromCaptions() {
    */
   private function createIdsFromCaptions($ids) {
     $fieldname = $this->submission['fields']['insert_captions_use']['value'];
-    if(empty($ids)){
-	$this->submission['fields'][$fieldname] = array('value'=>array());
+    if (empty($ids)) {
+      $this->submission['fields'][$fieldname] = ['value'=>[]];
     }
-    else{
+    else {
     	$keys = array_fill(0, sizeof($ids), 'value');
     	$a = array_fill_keys($keys, $ids);
     	$this->submission['fields'][$fieldname] = $a;
@@ -688,28 +692,35 @@ protected function preSubmit() {
    */
   protected function populateIdentifiers() {
     if (array_key_exists('website_id', $this->submission['fields'])) {
-      if (is_array($this->submission['fields']['website_id']))
+      if (is_array($this->submission['fields']['website_id'])) {
         $this->identifiers['website_id'] = $this->submission['fields']['website_id']['value'];
-      else
+      }
+      else {
         $this->identifiers['website_id'] = $this->submission['fields']['website_id'];
+      }
     }
     if (array_key_exists('survey_id', $this->submission['fields'])) {
-      if (is_array($this->submission['fields']['survey_id']))
+      if (is_array($this->submission['fields']['survey_id'])) {
         $this->identifiers['survey_id'] = $this->submission['fields']['survey_id']['value'];
-      else
+      }
+      else {
         $this->identifiers['survey_id'] = $this->submission['fields']['survey_id'];
+      }
     }
   }
 
   /**
    * Wraps the process of submission in a transaction.
-   * @return integer If successful, returns the id of the created/found record. If not, returns null - errors are embedded in the model.
+   *
+   * @return int
+   *   If successful, returns the id of the created/found record. If not,
+   *   returns null - errors are embedded in the model.
    */
   public function submit() {
     Kohana::log('debug', 'Commencing new transaction.');
     $this->db->query('BEGIN;');
     try {
-      $this->errors = array();
+      $this->errors = [];
       $this->preProcess();
       $res = $this->inner_submit();
       $this->postProcess();
@@ -720,8 +731,8 @@ public function submit() {
       $res = NULL;
     }
     if ($res) {
-      $allowCommitToDB = (isset($_GET['allow_commit_to_db']) ? $_GET['allow_commit_to_db'] : true);
-      if (!empty($allowCommitToDB)&&$allowCommitToDB==true) {
+      $allowCommitToDB = (isset($_GET['allow_commit_to_db']) ? $_GET['allow_commit_to_db'] : TRUE);
+      if (!empty($allowCommitToDB) && $allowCommitToDB == TRUE) {
         Kohana::log('debug', 'Committing transaction.');
         $this->db->query('COMMIT;');
       }
@@ -747,13 +758,16 @@ private function preProcess() {
   }
 
   /**
-   * Handles any index rebuild requirements as a result of new or updated records, e.g. in
-   * samples or occurrences. Also handles joining of occurrence_associations to the
-   * correct records.
+   * Submission post-processing.
+   *
+   * Handles any index rebuild requirements as a result of new or updated
+   * records, e.g. in samples or occurrences. Also handles joining of
+   * occurrence_associations to the correct records.
    */
   private function postProcess() {
     if (class_exists('cache_builder')) {
       $occurrences = [];
+      $deletedOccurrences = [];
       if (!empty(self::$changedRecords['insert']['occurrence'])) {
         cache_builder::insert($this->db, 'occurrences', self::$changedRecords['insert']['occurrence']);
         $occurrences = self::$changedRecords['insert']['occurrence'];
@@ -764,6 +778,7 @@ private function postProcess() {
       }
       if (!empty(self::$changedRecords['delete']['occurrence'])) {
         cache_builder::delete($this->db, 'occurrences', self::$changedRecords['delete']['occurrence']);
+        $deletedOccurrences = self::$changedRecords['delete']['occurrence'];
       }
       $samples = [];
       if (!empty(self::$changedRecords['insert']['sample'])) {
@@ -785,6 +800,9 @@ private function postProcess() {
         // No need to do occurrence map square update if inserting a sample, as
         // the above code does the occurrences in bulk.
         postgreSQL::insertMapSquaresForOccurrences($occurrences, $this->db);
+        // Need to ensure sample tracking is updated if occurrences change
+        // without a posted sample.
+        cache_builder::updateSampleTrackingForOccurrences($this->db, $occurrences + $deletedOccurrences);
       }
     }
     if (!empty(self::$changedRecords['insert']['occurrence_association']) ||
@@ -801,7 +819,7 @@ private function postProcess() {
         }
       }
       // Reset important if doing an import with multiple submissions.
-      Occurrence_association_Model::$to_occurrence_id_pointers = array();
+      Occurrence_association_Model::$to_occurrence_id_pointers = [];
     }
     $this->createWorkQueueEntries();
   }
@@ -912,7 +930,7 @@ public function inner_submit(){
         else
           $addTo=&self::$changedRecords['update'];
         if (!isset($addTo[$this->object_name]))
-          $addTo[$this->object_name] = array();
+          $addTo[$this->object_name] = [];
         $addTo[$this->object_name][] = $this->id;
       }
       // Call postSubmit
@@ -978,7 +996,7 @@ protected function validateAndSubmit() {
       // The easiest thing here is pretend the current value of any array
       // column doesn't match. These array columns are used so rarely that this
       // less optimised solution is not important.
-      $exactMatches = array();
+      $exactMatches = [];
       foreach ($thisValues as $column => $value) {
         if (array_key_exists($column, $vArray) &&
             !is_array($vArray[$column]) &&
@@ -1326,8 +1344,8 @@ private function checkRequiredAttributes() {
     // Test if this model has an attributes sub-table. Also to have required attributes, we must be posting into a
     // specified survey or website at least.
     if ($this->has_attributes) {
-      $got_values=array();
-      $empties = array();
+      $got_values=[];
+      $empties = [];
       if (isset($this->submission['metaFields'][$this->attrs_submission_name]))
       {
         // Old way of submitting attribute values but still supported - attributes are stored in a metafield. Find the ones we actually have a value for
@@ -1419,7 +1437,7 @@ protected function getRequiredFieldsCacheKey($typeFilter) {
    */
   protected function getAttributes($required = FALSE, $typeFilter = NULL, $hasSurveyRestriction = TRUE) {
     if (empty($this->identifiers['website_id']) && empty($this->identifiers['taxon_list_id'])) {
-      return array();
+      return [];
     }
     $attr_entity = $this->object_name . '_attribute';
     $this->db->select($attr_entity.'s.id', $attr_entity.'s.caption', $attr_entity.'s.data_type');
@@ -1523,7 +1541,7 @@ public function getSubmittableFields($fk = FALSE, array $identifiers = [], $attr
       // currently can only have associations if a single superModel exists.
       if($use_associations && count($struct['superModels']) === 1) {
       	// duplicate all the existing fields, but rename adding a 2 to model end.
-      	$newFields = array();
+      	$newFields = [];
       	foreach($fields as $name=>$caption){
       		$parts=explode(':',$name);
       		if($parts[0]==$struct['model'] || $parts[0] == $struct['model'].'_image' || $parts[0] == $this->attrs_field_prefix) {
@@ -1560,7 +1578,7 @@ public function getRequiredFields($fk = FALSE, array $identifiers = [], $use_ass
     $sub = $this->get_submission_structure();
     $arr = new Validation(array('id'=>1));
     $this->validate($arr, FALSE);
-    $fields = array();
+    $fields = [];
     foreach ($arr->errors() as $column=>$error) {
       if ($error=='required') {
         if ($fk && substr($column, -3) == "_id") {
@@ -1584,7 +1602,7 @@ public function getRequiredFields($fk = FALSE, array $identifiers = [], $use_ass
     	// currently can only have associations if a single superModel exists.
     	if($use_associations && count($sub['superModels'])===1){
     		// duplicate all the existing fields, but rename adding a 2 to model end.
-    		$newFields = array();
+    		$newFields = [];
     		foreach($fields as $id){
     			$parts=explode(':',$id);
     			if($parts[0]==$sub['model'] || $parts[0]==$sub['model'].'_image' || $parts[0]==$this->attrs_field_prefix) {
@@ -1611,7 +1629,7 @@ public function getRequiredFields($fk = FALSE, array $identifiers = [], $use_ass
    * @return array Prefixed key value pairs.
    */
   public function getPrefixedValuesArray($prefix=NULL) {
-    $r = array();
+    $r = [];
     if (!$prefix) {
       $prefix=$this->object_name;
     }
@@ -1627,7 +1645,7 @@ public function getPrefixedValuesArray($prefix=NULL) {
    * @return array Prefixed columns.
    */
   protected function getPrefixedColumnsArray($fk=FALSE, $skipHiddenFields=TRUE) {
-    $r = array();
+    $r = [];
     $prefix=$this->object_name;
     $sub = $this->get_submission_structure();
     foreach ($this->table_columns as $column=>$type) {
@@ -2181,7 +2199,7 @@ public function get_submission_structure() {
    * on creation of a new record.
    */
   public function getDefaults() {
-    return array();
+    return [];
   }
 
   /**
@@ -2200,8 +2218,8 @@ private function sanitise($array) {
    */
   public function clear() {
     parent::clear();
-    $this->errors=array();
-    $this->identifiers = array('website_id'=>NULL,'survey_id'=>NULL);
+    $this->errors = [];
+    $this->identifiers = ['website_id' => NULL, 'survey_id' => NULL];
   }
 
   /**
diff --git a/modules/cache_builder/helpers/cache_builder.php b/modules/cache_builder/helpers/cache_builder.php
index a283aeea21..acf4929fc1 100644
--- a/modules/cache_builder/helpers/cache_builder.php
+++ b/modules/cache_builder/helpers/cache_builder.php
@@ -240,6 +240,29 @@ public static function delete($db, $table, array $ids) {
     }
   }
 
+  /**
+   * If submitting occurrence changes without a sample, update sample tracking.
+   *
+   * This is so that any sample data feeds receive an updated copy of the
+   * sample, as the occurrence statistics will have changed.
+   *
+   * @param object $db
+   *   Database object.
+   * @param array $occurrenceIds
+   *   List of occurrences affected by a submission.
+   */
+  public static function updateSampleTrackingForOccurrences($db, array $ids) {
+    $idList = implode(',', $ids);
+    $sql = <<<SQL
+UPDATE cache_samples_functional s
+SET website_id=s.website_id
+FROM cache_occurrences_functional o
+WHERE o.id IN ($idList)
+AND (s.id=o.sample_id OR s.id=o.parent_sample_id);
+SQL;
+    $db->query($sql);
+  }
+
   /**
    * During an import, add tasks to work queue rather than do immediate update.
    *

From 50da11097dc4bba84ae62f39756ec1aa46424976 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Fri, 17 Sep 2021 14:28:03 +0100
Subject: [PATCH 81/93] Scripts for new fields in samples cache

---
 .../202109091456_sample_ext_key.sql           |  2 -
 .../202109091456_sample_fields.sql            | 16 ++++
 .../202109091503_update_ext_key.sql           |  6 --
 .../202109101509_update_samples.sql           | 73 +++++++++++++++++++
 4 files changed, 89 insertions(+), 8 deletions(-)
 delete mode 100644 modules/cache_builder/db/version_6_3_0/202109091456_sample_ext_key.sql
 create mode 100644 modules/cache_builder/db/version_6_3_0/202109091456_sample_fields.sql
 delete mode 100644 modules/cache_builder/db/version_6_3_0/202109091503_update_ext_key.sql
 create mode 100644 modules/cache_builder/db/version_6_3_0/202109101509_update_samples.sql

diff --git a/modules/cache_builder/db/version_6_3_0/202109091456_sample_ext_key.sql b/modules/cache_builder/db/version_6_3_0/202109091456_sample_ext_key.sql
deleted file mode 100644
index 8e6ef95429..0000000000
--- a/modules/cache_builder/db/version_6_3_0/202109091456_sample_ext_key.sql
+++ /dev/null
@@ -1,2 +0,0 @@
-ALTER TABLE cache_samples_functional
-  ADD COLUMN external_key character varying;
\ No newline at end of file
diff --git a/modules/cache_builder/db/version_6_3_0/202109091456_sample_fields.sql b/modules/cache_builder/db/version_6_3_0/202109091456_sample_fields.sql
new file mode 100644
index 0000000000..83c5fc0536
--- /dev/null
+++ b/modules/cache_builder/db/version_6_3_0/202109091456_sample_fields.sql
@@ -0,0 +1,16 @@
+ALTER TABLE cache_samples_functional
+  ADD COLUMN external_key character varying;
+
+ALTER TABLE cache_samples_nonfunctional
+  ADD COLUMN output_sref character varying,
+  ADD COLUMN output_sref_system character varying,
+  ADD COLUMN verifier character varying;
+
+COMMENT ON COLUMN cache_samples_functional.external_key IS
+    'For samples imported from an external system, provides a field to store the external system''s primary key for the record allowing re-synchronisation.';
+
+COMMENT ON COLUMN cache_samples_nonfunctional.output_sref IS
+    'A display spatial reference created for all samples, using the most appropriate local grid system where possible.';
+
+COMMENT ON COLUMN cache_samples_nonfunctional.output_sref_system IS
+    'Spatial reference system used for the output_sref field.';
\ No newline at end of file
diff --git a/modules/cache_builder/db/version_6_3_0/202109091503_update_ext_key.sql b/modules/cache_builder/db/version_6_3_0/202109091503_update_ext_key.sql
deleted file mode 100644
index 83261cf677..0000000000
--- a/modules/cache_builder/db/version_6_3_0/202109091503_update_ext_key.sql
+++ /dev/null
@@ -1,6 +0,0 @@
--- #slow script#
-
-UPDATE cache_samples_functional s
-SET external_key=smp.external_key
-FROM samples smp
-WHERE smp.id=s.id;
\ No newline at end of file
diff --git a/modules/cache_builder/db/version_6_3_0/202109101509_update_samples.sql b/modules/cache_builder/db/version_6_3_0/202109101509_update_samples.sql
new file mode 100644
index 0000000000..41e12a46e4
--- /dev/null
+++ b/modules/cache_builder/db/version_6_3_0/202109101509_update_samples.sql
@@ -0,0 +1,73 @@
+-- #slow script#
+
+UPDATE cache_samples_functional u
+SET external_key=u.external_key,
+  public_geom=reduce_precision(coalesce(s.geom, l.centroid_geom), false, greatest(s.privacy_precision, (SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id)))
+FROM samples s
+LEFT JOIN locations l ON l.id=s.location_id AND l.deleted=false
+WHERE s.id=u.id;
+
+UPDATE cache_samples_nonfunctional u
+  SET public_entered_sref=case
+    when s.privacy_precision is not null OR (SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id) IS NOT NULL then
+      get_output_sref(
+        greatest(
+          round(sqrt(st_area(st_transform(s.geom, sref_system_to_srid(s.entered_sref_system)))))::integer,
+          (SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id),
+          s.privacy_precision,
+          -- work out best square size to reflect a lat long's true precision
+          case
+          when coalesce(v_sref_precision.int_value, v_sref_precision.float_value)>=501 then 10000
+          when coalesce(v_sref_precision.int_value, v_sref_precision.float_value) between 51 and 500 then 1000
+          when coalesce(v_sref_precision.int_value, v_sref_precision.float_value) between 6 and 50 then 100
+          else 10
+          end,
+          10 -- default minimum square size
+        ), reduce_precision(coalesce(s.geom, l.centroid_geom), (SELECT bool_or(confidential) FROM occurrences WHERE sample_id=s.id), greatest((SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id), s.privacy_precision))
+      )
+   else
+    case
+      when s.entered_sref_system = '4326' and coalesce(s.entered_sref, l.centroid_sref) ~ '^-?[0-9]*\.[0-9]*,[ ]*-?[0-9]*\.[0-9]*' then
+      abs(round(((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[1])::numeric, 3))::varchar
+      || case when ((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[1])::float>0 then 'N' else 'S' end
+      || ', '
+      || abs(round(((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[2])::numeric, 3))::varchar
+      || case when ((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[2])::float>0 then 'E' else 'W' end
+      when s.entered_sref_system = '4326' and coalesce(s.entered_sref, l.centroid_sref) ~ '^-?[0-9]*\.[0-9]*[NS](, |[, ])*-?[0-9]*\.[0-9]*[EW]' then
+      abs(round(((regexp_split_to_array(coalesce(s.entered_sref, l.centroid_sref), '([NS](, |[, ]))|[EW]'))[1])::numeric, 3))::varchar
+      || case when coalesce(s.entered_sref, l.centroid_sref) like '%N%' then 'N' else 'S' end
+      || ', '
+      || abs(round(((regexp_split_to_array(coalesce(s.entered_sref, l.centroid_sref), '([NS](, |[, ]))|[EW]'))[2])::numeric, 3))::varchar
+      || case when coalesce(s.entered_sref, l.centroid_sref) like '%E%' then 'E' else 'W' end
+      else
+      coalesce(s.entered_sref, l.centroid_sref)
+    end
+  end,
+  output_sref=get_output_sref(
+    greatest(
+      round(sqrt(st_area(st_transform(s.geom, sref_system_to_srid(s.entered_sref_system)))))::integer,
+      (SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id),
+      s.privacy_precision,
+      -- work out best square size to reflect a lat long's true precision
+      case
+        when coalesce(v_sref_precision.int_value, v_sref_precision.float_value)>=501 then 10000
+        when coalesce(v_sref_precision.int_value, v_sref_precision.float_value) between 51 and 500 then 1000
+        when coalesce(v_sref_precision.int_value, v_sref_precision.float_value) between 6 and 50 then 100
+        else 10
+      end,
+      10 -- default minimum square size
+    ), reduce_precision(coalesce(s.geom, l.centroid_geom), (SELECT bool_or(confidential) FROM occurrences WHERE sample_id=s.id), greatest((SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id), s.privacy_precision))
+  ),
+  output_sref_system=get_output_system(
+    reduce_precision(coalesce(s.geom, l.centroid_geom), (SELECT bool_or(confidential) FROM occurrences WHERE sample_id=s.id), greatest((SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id), s.privacy_precision))
+  ),
+  verifier=pv.surname || ', ' || pv.first_name
+FROM samples s
+LEFT JOIN locations l ON l.id=s.location_id AND l.deleted=false
+LEFT JOIN (sample_attribute_values v_sref_precision
+  JOIN sample_attributes a_sref_precision on a_sref_precision.id=v_sref_precision.sample_attribute_id and a_sref_precision.deleted=false and a_sref_precision.system_function='sref_precision'
+  LEFT JOIN cache_termlists_terms t_sref_precision on a_sref_precision.data_type='L' and t_sref_precision.id=v_sref_precision.int_value
+) on v_sref_precision.sample_id=s.id and v_sref_precision.deleted=false
+LEFT JOIN users uv on uv.id=s.verified_by_id and uv.deleted=false
+LEFT JOIN people pv on pv.id=uv.person_id and pv.deleted=false
+WHERE s.id=u.id;

From 3577d0d0b7d508d21ad1e00d56d1eb1caf588dc0 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Fri, 17 Sep 2021 14:33:51 +0100
Subject: [PATCH 82/93] Adds missing sample_external_key into occurrences feed

---
 reports/library/occurrences/list_for_elastic.xml          | 8 +++++---
 reports/library/occurrences/list_for_elastic_all.xml      | 8 +++++---
 .../library/occurrences/list_for_elastic_sensitive.xml    | 1 +
 .../occurrences/list_for_elastic_sensitive_all.xml        | 1 +
 4 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/reports/library/occurrences/list_for_elastic.xml b/reports/library/occurrences/list_for_elastic.xml
index fda2ff0165..f80fbf3559 100644
--- a/reports/library/occurrences/list_for_elastic.xml
+++ b/reports/library/occurrences/list_for_elastic.xml
@@ -23,6 +23,7 @@
     FROM filtered_occurrences o
     JOIN cache_occurrences_nonfunctional onf ON onf.id=o.id
     JOIN occurrences occ on occ.id=o.id AND occ.deleted=false
+    JOIN samples s ON s.id=o.sample_id AND s.deleted=false
     JOIN cache_samples_nonfunctional snf ON snf.id=o.sample_id
     JOIN cache_taxa_taxon_lists cttl ON cttl.id=o.taxa_taxon_list_id
     LEFT JOIN locations l ON l.id=o.location_id AND l.deleted=false;
@@ -43,10 +44,10 @@
     SET recorded_parent_location_id = lp.id,
       recorded_parent_location_name = lp.name,
       recorded_parent_location_code = lp.code
-      FROM samples sc
+    FROM samples sc
     JOIN locations lc ON lc.id=sc.location_id AND lc.deleted=false
-      JOIN locations lp ON lp.id = lc.parent_id AND lp.deleted=false
-      WHERE sc.id=o.sample_id AND sc.deleted=false
+    JOIN locations lp ON lp.id = lc.parent_id AND lp.deleted=false
+    WHERE sc.id=o.sample_id AND sc.deleted=false
     AND o.recorded_parent_location_id IS NULL;
 
     SELECT *
@@ -67,6 +68,7 @@
     <!-- Tracking field called "id" is required for autofeed to work. -->
     <column name="id" sql="o.id" datatype="integer" />
     <column name="occurrence_external_key" sql="o.external_key" datatype="text" />
+    <column name="sample_external_key" sql="s.external_key" datatype="text" />
     <column name="created_by_id" sql="o.created_by_id" datatype="integer" />
     <column name="website_id" sql="o.website_id" datatype="integer" />
     <column name="survey_id" sql="o.survey_id" datatype="integer" />
diff --git a/reports/library/occurrences/list_for_elastic_all.xml b/reports/library/occurrences/list_for_elastic_all.xml
index 30912b89b2..6dd18f9402 100644
--- a/reports/library/occurrences/list_for_elastic_all.xml
+++ b/reports/library/occurrences/list_for_elastic_all.xml
@@ -23,6 +23,7 @@
     FROM filtered_occurrences o
     JOIN cache_occurrences_nonfunctional onf ON onf.id=o.id
     JOIN occurrences occ on occ.id=o.id AND occ.deleted=false
+    JOIN samples s ON s.id=o.sample_id AND s.deleted=false
     JOIN cache_samples_nonfunctional snf ON snf.id=o.sample_id
     JOIN cache_taxa_taxon_lists cttl ON cttl.id=o.taxa_taxon_list_id
     LEFT JOIN locations l ON l.id=o.location_id AND l.deleted=false;
@@ -42,10 +43,10 @@
     SET recorded_parent_location_id = lp.id,
       recorded_parent_location_name = lp.name,
       recorded_parent_location_code = lp.code
-      FROM samples sc
+    FROM samples sc
     JOIN locations lc ON lc.id=sc.location_id AND lc.deleted=false
-      JOIN locations lp ON lp.id = lc.parent_id AND lp.deleted=false
-      WHERE sc.id=o.sample_id AND sc.deleted=false
+    JOIN locations lp ON lp.id = lc.parent_id AND lp.deleted=false
+    WHERE sc.id=o.sample_id AND sc.deleted=false
     AND o.recorded_parent_location_id IS NULL;
 
     SELECT *
@@ -66,6 +67,7 @@
     <!-- Tracking field called "id" is required for autofeed to work. -->
     <column name="id" sql="o.id" datatype="integer" />
     <column name="occurrence_external_key" sql="o.external_key" datatype="text" />
+    <column name="sample_external_key" sql="s.external_key" datatype="text" />
     <column name="created_by_id" sql="o.created_by_id" datatype="integer" />
     <column name="website_id" sql="o.website_id" datatype="integer" />
     <column name="survey_id" sql="o.survey_id" datatype="integer" />
diff --git a/reports/library/occurrences/list_for_elastic_sensitive.xml b/reports/library/occurrences/list_for_elastic_sensitive.xml
index 1f90073ecb..e9dd64e68f 100644
--- a/reports/library/occurrences/list_for_elastic_sensitive.xml
+++ b/reports/library/occurrences/list_for_elastic_sensitive.xml
@@ -69,6 +69,7 @@
     <!-- Tracking field called "id" is required for autofeed to work. -->
     <column name="id" sql="o.id" datatype="integer" />
     <column name="occurrence_external_key" sql="o.external_key" datatype="text" />
+    <column name="sample_external_key" sql="s.external_key" datatype="text" />
     <column name="created_by_id" sql="o.created_by_id" datatype="integer" />
     <column name="website_id" sql="o.website_id" datatype="integer" />
     <column name="survey_id" sql="o.survey_id" datatype="integer" />
diff --git a/reports/library/occurrences/list_for_elastic_sensitive_all.xml b/reports/library/occurrences/list_for_elastic_sensitive_all.xml
index a0c35f5478..8dca179f58 100644
--- a/reports/library/occurrences/list_for_elastic_sensitive_all.xml
+++ b/reports/library/occurrences/list_for_elastic_sensitive_all.xml
@@ -68,6 +68,7 @@
     <!-- Tracking field called "id" is required for autofeed to work. -->
     <column name="id" sql="o.id" datatype="integer" />
     <column name="occurrence_external_key" sql="o.external_key" datatype="text" />
+    <column name="sample_external_key" sql="s.external_key" datatype="text" />
     <column name="created_by_id" sql="o.created_by_id" datatype="integer" />
     <column name="website_id" sql="o.website_id" datatype="integer" />
     <column name="survey_id" sql="o.survey_id" datatype="integer" />

From 51da812517ac0ce1697ad889947cae614cd7c386 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Fri, 17 Sep 2021 14:35:18 +0100
Subject: [PATCH 83/93] REST module config allows for Elasticsearch samples
 indexes.

---
 modules/rest_api/config/rest.example.php | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/modules/rest_api/config/rest.example.php b/modules/rest_api/config/rest.example.php
index 8e7dbe10be..2dc871a10c 100644
--- a/modules/rest_api/config/rest.example.php
+++ b/modules/rest_api/config/rest.example.php
@@ -119,6 +119,9 @@
   'es' => [
     // Set open = TRUE if this end-point is available without authentication.
     'open' => FALSE,
+    // Optional type, either occurrence or sample. Default is occurrence if not
+    // specified.
+    'type' => 'occurrence',
     // Name of the elasticsearch index or alias this end-point points to.
     'index' => 'occurrence',
     // URL of the Elasticsearch index.

From c44e831d02ef7930c86d1d20b51b4b35f9acc150 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Fri, 17 Sep 2021 14:38:50 +0100
Subject: [PATCH 84/93] Improvements to cache table population

* Samples public_geom now respects child occurrence sensitivity.
* Samples public_entered_sref now respects child occurrence sensitivity.
* Adds sample output_sref and system (matching occurrence behaviour).
* Adds sample verifier field.
---
 .../cache_builder/config/cache_builder.php    | 191 +++++++++++-------
 1 file changed, 115 insertions(+), 76 deletions(-)

diff --git a/modules/cache_builder/config/cache_builder.php b/modules/cache_builder/config/cache_builder.php
index 9f48222db7..3045e8766d 100644
--- a/modules/cache_builder/config/cache_builder.php
+++ b/modules/cache_builder/config/cache_builder.php
@@ -892,7 +892,7 @@
   input_form=COALESCE(sp.input_form, s.input_form),
   location_id= s.location_id,
   location_name=CASE WHEN s.privacy_precision IS NOT NULL THEN NULL ELSE COALESCE(l.name, s.location_name, lp.name, sp.location_name) END,
-  public_geom=reduce_precision(coalesce(s.geom, l.centroid_geom), false, s.privacy_precision),
+  public_geom=reduce_precision(coalesce(s.geom, l.centroid_geom), false, greatest(s.privacy_precision, (SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id))),
   date_start=s.date_start,
   date_end=s.date_end,
   date_type=s.date_type,
@@ -939,24 +939,59 @@
 SET website_title=w.title,
   survey_title=su.title,
   group_title=g.title,
-  public_entered_sref=case when s.privacy_precision is not null then null else
+  public_entered_sref=case
+    when s.privacy_precision is not null OR (SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id) IS NOT NULL then
+      get_output_sref(
+        greatest(
+          round(sqrt(st_area(st_transform(s.geom, sref_system_to_srid(s.entered_sref_system)))))::integer,
+          (SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id),
+          s.privacy_precision,
+          -- work out best square size to reflect a lat long's true precision
+          case
+          when coalesce(v_sref_precision.int_value, v_sref_precision.float_value)>=501 then 10000
+          when coalesce(v_sref_precision.int_value, v_sref_precision.float_value) between 51 and 500 then 1000
+          when coalesce(v_sref_precision.int_value, v_sref_precision.float_value) between 6 and 50 then 100
+          else 10
+          end,
+          10 -- default minimum square size
+        ), reduce_precision(coalesce(s.geom, l.centroid_geom), (SELECT bool_or(confidential) FROM occurrences WHERE sample_id=s.id), greatest((SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id), s.privacy_precision))
+      )
+   else
     case
       when s.entered_sref_system = '4326' and coalesce(s.entered_sref, l.centroid_sref) ~ '^-?[0-9]*\.[0-9]*,[ ]*-?[0-9]*\.[0-9]*' then
-        abs(round(((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[1])::numeric, 3))::varchar
-        || case when ((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[1])::float>0 then 'N' else 'S' end
-        || ', '
-        || abs(round(((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[2])::numeric, 3))::varchar
-        || case when ((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[2])::float>0 then 'E' else 'W' end
+      abs(round(((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[1])::numeric, 3))::varchar
+      || case when ((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[1])::float>0 then 'N' else 'S' end
+      || ', '
+      || abs(round(((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[2])::numeric, 3))::varchar
+      || case when ((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[2])::float>0 then 'E' else 'W' end
       when s.entered_sref_system = '4326' and coalesce(s.entered_sref, l.centroid_sref) ~ '^-?[0-9]*\.[0-9]*[NS](, |[, ])*-?[0-9]*\.[0-9]*[EW]' then
-        abs(round(((regexp_split_to_array(coalesce(s.entered_sref, l.centroid_sref), '([NS](, |[, ]))|[EW]'))[1])::numeric, 3))::varchar
-        || case when coalesce(s.entered_sref, l.centroid_sref) like '%N%' then 'N' else 'S' end
-        || ', '
-        || abs(round(((regexp_split_to_array(coalesce(s.entered_sref, l.centroid_sref), '([NS](, |[, ]))|[EW]'))[2])::numeric, 3))::varchar
-        || case when coalesce(s.entered_sref, l.centroid_sref) like '%E%' then 'E' else 'W' end
+      abs(round(((regexp_split_to_array(coalesce(s.entered_sref, l.centroid_sref), '([NS](, |[, ]))|[EW]'))[1])::numeric, 3))::varchar
+      || case when coalesce(s.entered_sref, l.centroid_sref) like '%N%' then 'N' else 'S' end
+      || ', '
+      || abs(round(((regexp_split_to_array(coalesce(s.entered_sref, l.centroid_sref), '([NS](, |[, ]))|[EW]'))[2])::numeric, 3))::varchar
+      || case when coalesce(s.entered_sref, l.centroid_sref) like '%E%' then 'E' else 'W' end
       else
-        coalesce(s.entered_sref, l.centroid_sref)
+      coalesce(s.entered_sref, l.centroid_sref)
     end
   end,
+  output_sref=get_output_sref(
+    greatest(
+      round(sqrt(st_area(st_transform(s.geom, sref_system_to_srid(s.entered_sref_system)))))::integer,
+      (SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id),
+      s.privacy_precision,
+      -- work out best square size to reflect a lat long's true precision
+      case
+        when coalesce(v_sref_precision.int_value, v_sref_precision.float_value)>=501 then 10000
+        when coalesce(v_sref_precision.int_value, v_sref_precision.float_value) between 51 and 500 then 1000
+        when coalesce(v_sref_precision.int_value, v_sref_precision.float_value) between 6 and 50 then 100
+        else 10
+      end,
+      10 -- default minimum square size
+    ), reduce_precision(coalesce(s.geom, l.centroid_geom), (SELECT bool_or(confidential) FROM occurrences WHERE sample_id=s.id), greatest((SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id), s.privacy_precision))
+  ),
+  output_sref_system=get_output_system(
+    reduce_precision(coalesce(s.geom, l.centroid_geom), (SELECT bool_or(confidential) FROM occurrences WHERE sample_id=s.id), greatest((SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id), s.privacy_precision))
+  ),
   entered_sref_system=case when s.entered_sref_system is null then l.centroid_sref_system else s.entered_sref_system end,
   recorders = s.recorder_names,
   comment=s.comment,
@@ -1001,7 +1036,8 @@
       WHEN 'L'::bpchar THEN t_sample_method.term
       ELSE NULL::text
   END, t_sample_method_id.term),
-  attr_linked_location_id=v_linked_location_id.int_value
+  attr_linked_location_id=v_linked_location_id.int_value,
+  verifier=pv.surname || ', ' || pv.first_name
 FROM samples s
 #join_needs_update#
 LEFT JOIN samples sp ON sp.id=s.parent_id and sp.deleted=false
@@ -1051,6 +1087,8 @@
   JOIN sample_attributes a_linked_location_id on a_linked_location_id.id=v_linked_location_id.sample_attribute_id
     and a_linked_location_id.deleted=false and a_linked_location_id.system_function='linked_location_id'
 ) ON v_linked_location_id.sample_id=s.id and v_linked_location_id.deleted=false
+LEFT JOIN users uv on uv.id=s.verified_by_id and uv.deleted=false
+LEFT JOIN people pv on pv.id=uv.person_id and pv.deleted=false
 WHERE s.id=cache_samples_nonfunctional.id
 ";
 
@@ -1063,15 +1101,6 @@
 WHERE s.id=cache_samples_nonfunctional.id
 ";
 
-$config['samples']['update']['nonfunctional_sensitive'] = "
-UPDATE cache_samples_nonfunctional
-SET public_entered_sref=null
-FROM samples s
-#join_needs_update#
-JOIN occurrences o ON o.sample_id=s.id AND o.deleted=false AND o.sensitivity_precision IS NOT NULL
-WHERE s.id=cache_samples_nonfunctional.id
-";
-
 $config['samples']['insert']['functional'] = "
 INSERT INTO cache_samples_functional(
             id, website_id, survey_id, input_form, location_id, location_name,
@@ -1079,7 +1108,7 @@
             group_id, record_status, training, query, parent_sample_id, media_count, external_key)
 SELECT distinct on (s.id) s.id, su.website_id, s.survey_id, COALESCE(sp.input_form, s.input_form), s.location_id,
   CASE WHEN s.privacy_precision IS NOT NULL THEN NULL ELSE COALESCE(l.name, s.location_name, lp.name, sp.location_name) END,
-  reduce_precision(coalesce(s.geom, l.centroid_geom), false, s.privacy_precision),
+  reduce_precision(coalesce(s.geom, l.centroid_geom), false, greatest(s.privacy_precision, (SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id))),
   s.date_start, s.date_end, s.date_type, s.created_on, s.updated_on, s.verified_on, s.created_by_id,
   coalesce(s.group_id, sp.group_id), s.record_status, s.training,
   case
@@ -1117,28 +1146,70 @@
 $config['samples']['insert']['nonfunctional'] = "
 INSERT INTO cache_samples_nonfunctional(
             id, website_title, survey_title, group_title, public_entered_sref,
-            entered_sref_system, recorders, comment, privacy_precision, licence_code)
+            entered_sref_system, recorders, comment, privacy_precision, licence_code,
+            attr_sref_precision, output_sref, output_sref_system, verifier)
 SELECT distinct on (s.id) s.id, w.title, su.title, g.title,
-  case when s.privacy_precision is not null then null else
+  case
+    when s.privacy_precision is not null OR (SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id) IS NOT NULL then
+      get_output_sref(
+        greatest(
+          round(sqrt(st_area(st_transform(s.geom, sref_system_to_srid(s.entered_sref_system)))))::integer,
+          (SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id),
+          s.privacy_precision,
+          -- work out best square size to reflect a lat long's true precision
+          case
+          when coalesce(v_sref_precision.int_value, v_sref_precision.float_value)>=501 then 10000
+          when coalesce(v_sref_precision.int_value, v_sref_precision.float_value) between 51 and 500 then 1000
+          when coalesce(v_sref_precision.int_value, v_sref_precision.float_value) between 6 and 50 then 100
+          else 10
+          end,
+          10 -- default minimum square size
+        ), reduce_precision(coalesce(s.geom, l.centroid_geom), (SELECT bool_or(confidential) FROM occurrences WHERE sample_id=s.id), greatest((SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id), s.privacy_precision))
+      )
+   else
     case
       when s.entered_sref_system = '4326' and coalesce(s.entered_sref, l.centroid_sref) ~ '^-?[0-9]*\.[0-9]*,[ ]*-?[0-9]*\.[0-9]*' then
-        abs(round(((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[1])::numeric, 3))::varchar
-        || case when ((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[1])::float>0 then 'N' else 'S' end
-        || ', '
-        || abs(round(((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[2])::numeric, 3))::varchar
-        || case when ((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[2])::float>0 then 'E' else 'W' end
+      abs(round(((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[1])::numeric, 3))::varchar
+      || case when ((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[1])::float>0 then 'N' else 'S' end
+      || ', '
+      || abs(round(((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[2])::numeric, 3))::varchar
+      || case when ((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[2])::float>0 then 'E' else 'W' end
       when s.entered_sref_system = '4326' and coalesce(s.entered_sref, l.centroid_sref) ~ '^-?[0-9]*\.[0-9]*[NS](, |[, ])*-?[0-9]*\.[0-9]*[EW]' then
-        abs(round(((regexp_split_to_array(coalesce(s.entered_sref, l.centroid_sref), '([NS](, |[, ]))|[EW]'))[1])::numeric, 3))::varchar
-        || case when coalesce(s.entered_sref, l.centroid_sref) like '%N%' then 'N' else 'S' end
-        || ', '
-        || abs(round(((regexp_split_to_array(coalesce(s.entered_sref, l.centroid_sref), '([NS](, |[, ]))|[EW]'))[2])::numeric, 3))::varchar
-        || case when coalesce(s.entered_sref, l.centroid_sref) like '%E%' then 'E' else 'W' end
+      abs(round(((regexp_split_to_array(coalesce(s.entered_sref, l.centroid_sref), '([NS](, |[, ]))|[EW]'))[1])::numeric, 3))::varchar
+      || case when coalesce(s.entered_sref, l.centroid_sref) like '%N%' then 'N' else 'S' end
+      || ', '
+      || abs(round(((regexp_split_to_array(coalesce(s.entered_sref, l.centroid_sref), '([NS](, |[, ]))|[EW]'))[2])::numeric, 3))::varchar
+      || case when coalesce(s.entered_sref, l.centroid_sref) like '%E%' then 'E' else 'W' end
       else
-        coalesce(s.entered_sref, l.centroid_sref)
+      coalesce(s.entered_sref, l.centroid_sref)
     end
   end,
   case when s.entered_sref_system is null then l.centroid_sref_system else s.entered_sref_system end,
-  s.recorder_names, s.comment, s.privacy_precision, li.code
+  s.recorder_names, s.comment, s.privacy_precision, li.code,
+  CASE a_sref_precision.data_type
+    WHEN 'I'::bpchar THEN v_sref_precision.int_value::double precision
+    WHEN 'F'::bpchar THEN v_sref_precision.float_value
+    ELSE NULL::double precision
+  END,
+  get_output_sref(
+    greatest(
+      round(sqrt(st_area(st_transform(s.geom, sref_system_to_srid(s.entered_sref_system)))))::integer,
+      (SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id),
+      s.privacy_precision,
+      -- work out best square size to reflect a lat long's true precision
+      case
+        when coalesce(v_sref_precision.int_value, v_sref_precision.float_value)>=501 then 10000
+        when coalesce(v_sref_precision.int_value, v_sref_precision.float_value) between 51 and 500 then 1000
+        when coalesce(v_sref_precision.int_value, v_sref_precision.float_value) between 6 and 50 then 100
+        else 10
+      end,
+      10 -- default minimum square size
+    ), reduce_precision(coalesce(s.geom, l.centroid_geom), (SELECT bool_or(confidential) FROM occurrences WHERE sample_id=s.id), greatest((SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id), s.privacy_precision))
+  ),
+  get_output_system(
+    reduce_precision(coalesce(s.geom, l.centroid_geom),(SELECT bool_or(confidential) FROM occurrences WHERE sample_id=s.id), greatest((SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id), s.privacy_precision))
+  ),
+  pv.surname || ', ' || pv.first_name
 FROM samples s
 #join_needs_update#
 LEFT JOIN samples sp ON sp.id=s.parent_id and sp.deleted=false
@@ -1147,7 +1218,13 @@
 JOIN websites w on w.id=su.website_id and w.deleted=false
 LEFT JOIN groups g on g.id=coalesce(s.group_id, sp.group_id) and g.deleted=false
 LEFT JOIN locations l on l.id=s.location_id and l.deleted=false
+LEFT JOIN (sample_attribute_values v_sref_precision
+  JOIN sample_attributes a_sref_precision on a_sref_precision.id=v_sref_precision.sample_attribute_id and a_sref_precision.deleted=false and a_sref_precision.system_function='sref_precision'
+  LEFT JOIN cache_termlists_terms t_sref_precision on a_sref_precision.data_type='L' and t_sref_precision.id=v_sref_precision.int_value
+) on v_sref_precision.sample_id=s.id and v_sref_precision.deleted=false
 LEFT JOIN licences li on li.id=s.licence_id and li.deleted=false
+LEFT JOIN users uv on uv.id=s.verified_by_id and uv.deleted=false
+LEFT JOIN people pv on pv.id=uv.person_id and pv.deleted=false
 WHERE s.deleted=false
 AND cs.id IS NULL";
 
@@ -1184,11 +1261,6 @@
       WHEN 'L'::bpchar THEN t_biotope.term
       ELSE NULL::text
   END,
-  attr_sref_precision=CASE a_sref_precision.data_type
-      WHEN 'I'::bpchar THEN v_sref_precision.int_value::double precision
-      WHEN 'F'::bpchar THEN v_sref_precision.float_value
-      ELSE NULL::double precision
-  END,
   attr_sample_method=COALESCE(t_sample_method_id.term, CASE a_sample_method.data_type
       WHEN 'T'::bpchar THEN v_sample_method.text_value
       WHEN 'L'::bpchar THEN t_sample_method.term
@@ -1225,10 +1297,6 @@
   JOIN sample_attributes a_biotope on a_biotope.id=v_biotope.sample_attribute_id and a_biotope.deleted=false and a_biotope.system_function='biotope'
   LEFT JOIN cache_termlists_terms t_biotope on a_biotope.data_type='L' and t_biotope.id=v_biotope.int_value
 ) on v_biotope.sample_id=s.id and v_biotope.deleted=false
-LEFT JOIN (sample_attribute_values v_sref_precision
-  JOIN sample_attributes a_sref_precision on a_sref_precision.id=v_sref_precision.sample_attribute_id and a_sref_precision.deleted=false and a_sref_precision.system_function='sref_precision'
-  LEFT JOIN cache_termlists_terms t_sref_precision on a_sref_precision.data_type='L' and t_sref_precision.id=v_sref_precision.int_value
-) on v_sref_precision.sample_id=s.id and v_sref_precision.deleted=false
 LEFT JOIN (sample_attribute_values v_sample_method
   JOIN sample_attributes a_sample_method on a_sample_method.id=v_sample_method.sample_attribute_id and a_sample_method.deleted=false and a_sample_method.system_function='sample_method'
   LEFT JOIN cache_termlists_terms t_sample_method on a_sample_method.data_type='L' and t_sample_method.id=v_sample_method.int_value
@@ -1249,15 +1317,6 @@
 WHERE s.id=cache_samples_nonfunctional.id
 ";
 
-$config['samples']['insert']['nonfunctional_sensitive'] = "
-UPDATE cache_samples_nonfunctional
-SET public_entered_sref=null
-FROM samples s
-#join_needs_update#
-JOIN occurrences o ON o.sample_id=s.id AND o.deleted=false AND o.sensitivity_precision IS NOT NULL
-WHERE s.id=cache_samples_nonfunctional.id
-";
-
 $config['samples']['join_needs_update'] = 'join needs_update_samples nu on nu.id=s.id and nu.deleted=false';
 $config['samples']['key_field'] = 's.id';
 
@@ -1723,16 +1782,6 @@
 AND o.deleted=false
 ";
 
-$config['occurrences']['update']['nonfunctional_sensitive'] = "
-UPDATE cache_samples_nonfunctional cs
-SET public_entered_sref=null
-FROM occurrences o
-#join_needs_update#
-WHERE o.sample_id=cs.id
-AND o.deleted=false
-AND o.sensitivity_precision IS NOT NULL
-";
-
 $config['occurrences']['insert']['functional'] = "INSERT INTO cache_occurrences_functional(
             id, sample_id, website_id, survey_id, input_form, location_id,
             location_name, public_geom,
@@ -1977,15 +2026,5 @@
 AND o.deleted=false
 ";
 
-$config['occurrences']['insert']['nonfunctional_sensitive'] = "
-UPDATE cache_samples_nonfunctional cs
-SET public_entered_sref=null
-FROM occurrences o
-#join_needs_update#
-WHERE o.sample_id=cs.id
-AND o.deleted=false
-AND o.sensitivity_precision IS NOT NULL
-";
-
 $config['occurrences']['join_needs_update'] = 'join needs_update_occurrences nu on nu.id=o.id and nu.deleted=false';
 $config['occurrences']['key_field'] = 'o.id';

From f7582e5f6d76685d1449fd0b399d28cf059dd1f4 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Fri, 17 Sep 2021 14:39:39 +0100
Subject: [PATCH 85/93] Performance fixes

---
 reports/library/samples/list_for_elastic_all.xml | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/reports/library/samples/list_for_elastic_all.xml b/reports/library/samples/list_for_elastic_all.xml
index 557b2cd6dc..2e05cecbeb 100644
--- a/reports/library/samples/list_for_elastic_all.xml
+++ b/reports/library/samples/list_for_elastic_all.xml
@@ -11,9 +11,7 @@
     DROP TABLE IF EXISTS output_rows;
     DROP TABLE IF EXISTS sample_occurrence_list;
 
-    SET LOCAL enable_incremental_sort = off;
-
-    SELECT DISTINCT s.*, greatest(s.tracking, (SELECT max(o.tracking) FROM cache_occurrences_functional o WHERE o.sample_id=s.id OR o.parent_sample_id=s.id)) AS tracking_inc_occurrences
+    SELECT s.*
     INTO TEMPORARY filtered_samples
     FROM cache_samples_functional s
     #joins#
@@ -43,7 +41,6 @@
     JOIN samples smp ON smp.id=s.id
     JOIN occurrence_stats os ON os.sample_id=s.id
     JOIN cache_samples_nonfunctional snf ON snf.id=s.id
-    LEFT JOIN occurrences o ON o.sample_id=s.id
     LEFT JOIN locations l ON l.id=s.location_id AND l.deleted=false;
 
     UPDATE output_rows o
@@ -76,8 +73,8 @@
     <param name="last_id" display="Last ID" datatype="integer" default="" order_by="s.id ASC">
       <where>s.id &gt; #last_id#</where>
     </param>
-    <param name="autofeed_tracking_from" display="Autofeed tracking from" datatype="integer" default="" order_by="tracking_inc_occurrences ASC">
-      <where>(s.tracking >= #autofeed_tracking_from# OR o.tracking >= #autofeed_tracking_from#)</where> -- WON'T FIRE ON AN OCCURRENCE DELETION
+    <param name="autofeed_tracking_from" display="Autofeed tracking from" datatype="integer" default="" order_by="tracking ASC">
+      <where>s.tracking >= #autofeed_tracking_from#</where>
     </param>
     <param name="limit" datatype="integer" default="10000" />
   </params>
@@ -192,7 +189,6 @@
     <column name="sample_attrs_json" sql="snf.attrs_json" datatype="text" />
     <column name="parent_sample_attrs_json" sql="null::json" datatype="text" />
     <column name="tracking" sql="s.tracking" datatype="integer" />
-    <column name="tracking_inc_occurrences" sql="s.tracking_inc_occurrences" datatype="integer" />
     <column name="input_form" sql="s.input_form" datatype="text" />
     <column name="count_occurrences" sql="os.count_occurrences" datatype="integer" />
     <column name="count_taxa" sql="os.count_taxa" datatype="integer" />

From 9ef8ac9774a7ea21709632228761f524f88a4572 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Fri, 17 Sep 2021 15:17:11 +0100
Subject: [PATCH 86/93] Field name corrected.

---
 .../version_6_3_0/202109171515_sensitive_grid_squares.sql | 7 +++++++
 .../library/occurrences/list_for_elastic_sensitive.xml    | 6 +++---
 .../occurrences/list_for_elastic_sensitive_all.xml        | 8 ++++----
 3 files changed, 14 insertions(+), 7 deletions(-)
 create mode 100644 modules/cache_builder/db/version_6_3_0/202109171515_sensitive_grid_squares.sql

diff --git a/modules/cache_builder/db/version_6_3_0/202109171515_sensitive_grid_squares.sql b/modules/cache_builder/db/version_6_3_0/202109171515_sensitive_grid_squares.sql
new file mode 100644
index 0000000000..f543e2e4a9
--- /dev/null
+++ b/modules/cache_builder/db/version_6_3_0/202109171515_sensitive_grid_squares.sql
@@ -0,0 +1,7 @@
+-- #slow script#
+
+-- Trigger re-queuing of sensitive records for Elasticsearch due to previously
+-- incorrect map grid square field names.
+UPDATE cache_occurrences_functional
+SET website_id=website_id
+WHERE sensitive=true;
\ No newline at end of file
diff --git a/reports/library/occurrences/list_for_elastic_sensitive.xml b/reports/library/occurrences/list_for_elastic_sensitive.xml
index e9dd64e68f..726aa87859 100644
--- a/reports/library/occurrences/list_for_elastic_sensitive.xml
+++ b/reports/library/occurrences/list_for_elastic_sensitive.xml
@@ -117,11 +117,11 @@
     <column name="geom" sql="st_astext(st_removerepeatedpoints(st_transform(s.geom, 4326)))" datatype="text" />
     <column name="point" sql="st_y(st_transform(st_centroid(s.geom), 4326))::text || ',' || st_x(st_transform(st_centroid(s.geom), 4326))::text" datatype="text" />
     <column name="map_sq_srid" sql="COALESCE(get_output_srid(o.public_geom), 3857)" datatype="integer" />
-    <column name="map_sq_1km_geom" datatype="text"
+    <column name="map_sq_1km" datatype="text"
       sql="TRIM(leading 'POINT(' FROM TRIM(trailing ')' FROM st_astext(st_centroid(st_transform(reduce_precision(s.geom, false, 1000), 4326)))))" />
-    <column name="map_sq_2km_geom" datatype="text"
+    <column name="map_sq_2km" datatype="text"
       sql="TRIM(leading 'POINT(' FROM TRIM(trailing ')' FROM st_astext(st_centroid(st_transform(reduce_precision(s.geom, false, 2000), 4326)))))" />
-    <column name="map_sq_10km_geom" datatype="text"
+    <column name="map_sq_10km" datatype="text"
       sql="TRIM(leading 'POINT(' FROM TRIM(trailing ')' FROM st_astext(st_centroid(st_transform(reduce_precision(s.geom, false, 10000), 4326)))))" />
     <column name="input_sref" sql="s.entered_sref" datatype="text" />
     <column name="input_sref_system" sql="s.entered_sref_system" datatype="text" />
diff --git a/reports/library/occurrences/list_for_elastic_sensitive_all.xml b/reports/library/occurrences/list_for_elastic_sensitive_all.xml
index 8dca179f58..1b93bf3942 100644
--- a/reports/library/occurrences/list_for_elastic_sensitive_all.xml
+++ b/reports/library/occurrences/list_for_elastic_sensitive_all.xml
@@ -1,7 +1,7 @@
 <report
     title="Autofeed occurrences for Elasticsearch (standard filters, sensitive only, no website limit)"
     description="A list of occurrences designed for feeding into Elasticsearch or a similar JSON store. This report
-      returns full precision copies of the records. No website limit so designed to collect records from all
+      returns full precision copies of sensitive records. No website limit so designed to collect records from all
       website registrations."
     restricted="true"
 >
@@ -116,11 +116,11 @@
     <column name="geom" sql="st_astext(st_removerepeatedpoints(st_transform(s.geom, 4326)))" datatype="text" />
     <column name="point" sql="st_y(st_transform(st_centroid(s.geom), 4326))::text || ',' || st_x(st_transform(st_centroid(s.geom), 4326))::text" datatype="text" />
     <column name="map_sq_srid" sql="COALESCE(get_output_srid(o.public_geom), 3857)" datatype="integer" />
-    <column name="map_sq_1km_geom" datatype="text"
+    <column name="map_sq_1km" datatype="text"
       sql="TRIM(leading 'POINT(' FROM TRIM(trailing ')' FROM st_astext(st_centroid(st_transform(reduce_precision(s.geom, false, 1000), 4326)))))" />
-    <column name="map_sq_2km_geom" datatype="text"
+    <column name="map_sq_2km" datatype="text"
       sql="TRIM(leading 'POINT(' FROM TRIM(trailing ')' FROM st_astext(st_centroid(st_transform(reduce_precision(s.geom, false, 2000), 4326)))))" />
-    <column name="map_sq_10km_geom" datatype="text"
+    <column name="map_sq_10km" datatype="text"
       sql="TRIM(leading 'POINT(' FROM TRIM(trailing ')' FROM st_astext(st_centroid(st_transform(reduce_precision(s.geom, false, 10000), 4326)))))" />
     <column name="input_sref" sql="s.entered_sref" datatype="text" />
     <column name="input_sref_system" sql="s.entered_sref_system" datatype="text" />

From 3c4d609091467701ef2e63a8b72ce17dee0cd7b6 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Mon, 20 Sep 2021 09:47:21 +0100
Subject: [PATCH 87/93] Improved error handling

As proj_ids not only for taxon-observation and annotation end-points.
---
 modules/rest_api/controllers/services/rest.php | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/modules/rest_api/controllers/services/rest.php b/modules/rest_api/controllers/services/rest.php
index c76f981034..5445fcc071 100644
--- a/modules/rest_api/controllers/services/rest.php
+++ b/modules/rest_api/controllers/services/rest.php
@@ -2267,7 +2267,8 @@ private function getAuthHeader() {
       return $headers['authorization'];
     }
     elseif (isset($_SERVER['REDIRECT_HTTP_AUTHORIZATION'])) {
-      // Sometimes on Apache, necessary to redirect the Auth header into the $_SERVER superglobal.
+      // Sometimes on Apache, necessary to redirect the Auth header into the
+      // $_SERVER superglobal.
       // See https://stackoverflow.com/questions/26475885/authorization-header-missing-in-php-post-request.
       return $_SERVER['REDIRECT_HTTP_AUTHORIZATION'];
     }
@@ -2318,7 +2319,7 @@ private function getWebsiteByUrl($url) {
    * @param int $websiteId
    *   Website ID to test against.
    * @param int $userId
-   *   Warehouse user ID to test.   *
+   *   Warehouse user ID to test.
    */
   private function checkWebsiteUser($websiteId, $userId) {
     $cache = Cache::instance();
@@ -2602,10 +2603,13 @@ private function authenticateUsingDirectClient() {
     // Taxon observations and annotations resource end-points will need a
     // proj_id if using client system based authorisation.
     if (($this->resourceName === 'taxon-observations' || $this->resourceName === 'annotations') &&
-        (empty($_REQUEST['proj_id']) || empty($this->projects[$_REQUEST['proj_id']]))) {
-      RestObjects::$apiResponse->fail('Bad request', 400, 'Project ID missing or invalid.');
+        (empty($_REQUEST['proj_id']))) {
+      RestObjects::$apiResponse->fail('Bad request', 400, 'Project ID missing.');
     }
     if (!empty($_REQUEST['proj_id'])) {
+      if (empty($this->projects[$_REQUEST['proj_id']])) {
+        RestObjects::$apiResponse->fail('Bad request', 400, 'Project ID invalid.');
+      }
       $projectConfig = $this->projects[$_REQUEST['proj_id']];
       RestObjects::$clientWebsiteId = $projectConfig['website_id'];
       // The client project config can override the resource options, e.g.

From 5f8180160e56098ea0dcbe47f4c19b44b20b1c52 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Tue, 21 Sep 2021 10:38:56 +0100
Subject: [PATCH 88/93] Addition of sample cache fields

Sensitive, private and verified fields in cache to match occurrences.
---
 modules/cache_builder/config/cache_builder.php    | 11 ++++++++---
 .../version_6_3_0/202109091456_sample_fields.sql  | 15 +++++++++++++--
 .../version_6_3_0/202109101509_update_samples.sql |  4 +++-
 3 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/modules/cache_builder/config/cache_builder.php b/modules/cache_builder/config/cache_builder.php
index 3045e8766d..e6c2376d08 100644
--- a/modules/cache_builder/config/cache_builder.php
+++ b/modules/cache_builder/config/cache_builder.php
@@ -910,7 +910,9 @@
   end,
   parent_sample_id=s.parent_id,
   media_count=(SELECT COUNT(sm.*) FROM sample_media sm WHERE sm.sample_id=s.id AND sm.deleted=false),
-  external_key=s.external_key
+  external_key=s.external_key,
+  sensitive=(SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id) IS NOT NULL,
+  private=s.privacy_precision IS NOT NULL
 FROM samples s
 #join_needs_update#
 LEFT JOIN samples sp ON sp.id=s.parent_id AND  sp.deleted=false
@@ -1105,7 +1107,8 @@
 INSERT INTO cache_samples_functional(
             id, website_id, survey_id, input_form, location_id, location_name,
             public_geom, date_start, date_end, date_type, created_on, updated_on, verified_on, created_by_id,
-            group_id, record_status, training, query, parent_sample_id, media_count, external_key)
+            group_id, record_status, training, query, parent_sample_id, media_count, external_key,
+            sensitive, private)
 SELECT distinct on (s.id) s.id, su.website_id, s.survey_id, COALESCE(sp.input_form, s.input_form), s.location_id,
   CASE WHEN s.privacy_precision IS NOT NULL THEN NULL ELSE COALESCE(l.name, s.location_name, lp.name, sp.location_name) END,
   reduce_precision(coalesce(s.geom, l.centroid_geom), false, greatest(s.privacy_precision, (SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id))),
@@ -1118,7 +1121,9 @@
   end,
   s.parent_id,
   (SELECT COUNT(sm.*) FROM sample_media sm WHERE sm.sample_id=s.id AND sm.deleted=false),
-  s.external_key
+  s.external_key,
+  (SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id) IS NOT NULL,
+  s.privacy_precision IS NOT NULL
 FROM samples s
 #join_needs_update#
 LEFT JOIN cache_samples_functional cs on cs.id=s.id
diff --git a/modules/cache_builder/db/version_6_3_0/202109091456_sample_fields.sql b/modules/cache_builder/db/version_6_3_0/202109091456_sample_fields.sql
index 83c5fc0536..0c888f04e1 100644
--- a/modules/cache_builder/db/version_6_3_0/202109091456_sample_fields.sql
+++ b/modules/cache_builder/db/version_6_3_0/202109091456_sample_fields.sql
@@ -1,5 +1,7 @@
 ALTER TABLE cache_samples_functional
-  ADD COLUMN external_key character varying;
+  ADD COLUMN external_key character varying,
+  ADD COLUMN sensitive boolean,
+  ADD COLUMN private boolean;
 
 ALTER TABLE cache_samples_nonfunctional
   ADD COLUMN output_sref character varying,
@@ -9,8 +11,17 @@ ALTER TABLE cache_samples_nonfunctional
 COMMENT ON COLUMN cache_samples_functional.external_key IS
     'For samples imported from an external system, provides a field to store the external system''s primary key for the record allowing re-synchronisation.';
 
+COMMENT ON COLUMN cache_samples_functional.sensitive IS
+    'Set to true if the sample is blurred because one of the contained occurrences is sensitive.';
+
+COMMENT ON COLUMN cache_samples_functional.private IS
+    'Set to true if the sample has a privacy_precision value set, indicating the data are blurred for site privacy reasons (e.g. private gardens).';
+
 COMMENT ON COLUMN cache_samples_nonfunctional.output_sref IS
     'A display spatial reference created for all samples, using the most appropriate local grid system where possible.';
 
 COMMENT ON COLUMN cache_samples_nonfunctional.output_sref_system IS
-    'Spatial reference system used for the output_sref field.';
\ No newline at end of file
+    'Spatial reference system used for the output_sref field.';
+
+COMMENT ON COLUMN cache_samples_nonfunctional.verifier IS
+    'Name of the person who verified the sample, if any.';
\ No newline at end of file
diff --git a/modules/cache_builder/db/version_6_3_0/202109101509_update_samples.sql b/modules/cache_builder/db/version_6_3_0/202109101509_update_samples.sql
index 41e12a46e4..53b065e435 100644
--- a/modules/cache_builder/db/version_6_3_0/202109101509_update_samples.sql
+++ b/modules/cache_builder/db/version_6_3_0/202109101509_update_samples.sql
@@ -2,7 +2,9 @@
 
 UPDATE cache_samples_functional u
 SET external_key=u.external_key,
-  public_geom=reduce_precision(coalesce(s.geom, l.centroid_geom), false, greatest(s.privacy_precision, (SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id)))
+  public_geom=reduce_precision(coalesce(s.geom, l.centroid_geom), false, greatest(s.privacy_precision, (SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id))),
+  sensitive=(SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id) IS NOT NULL,
+  private=s.privacy_precision IS NOT NULL
 FROM samples s
 LEFT JOIN locations l ON l.id=s.location_id AND l.deleted=false
 WHERE s.id=u.id;

From 75789d20ba0e3d2880c1dd2481862e86036af0b8 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Tue, 21 Sep 2021 10:39:25 +0100
Subject: [PATCH 89/93] Deletions reports for Elasticsearch samples

---
 .../library/samples/list_sample_deletions.xml | 31 +++++++++++++++++++
 .../samples/list_sample_deletions_all.xml     | 29 +++++++++++++++++
 2 files changed, 60 insertions(+)
 create mode 100644 reports/library/samples/list_sample_deletions.xml
 create mode 100644 reports/library/samples/list_sample_deletions_all.xml

diff --git a/reports/library/samples/list_sample_deletions.xml b/reports/library/samples/list_sample_deletions.xml
new file mode 100644
index 0000000000..83be0c0105
--- /dev/null
+++ b/reports/library/samples/list_sample_deletions.xml
@@ -0,0 +1,31 @@
+<report
+    title="Delete samples list"
+    description="A list of samples IDs that have been deleted."
+>
+  <query website_filter_field="srv.website_id" samples_id_field="s.id"
+         count_field="s.*">
+    SELECT #columns#
+    FROM samples s
+    JOIN surveys srv ON srv.id=s.survey_id
+    #agreements_join#
+    #joins#
+    WHERE #sharing_filter#
+    AND s.deleted=true
+  </query>
+  <order_bys>
+    <order_by>s.id</order_by>
+  </order_bys>
+  <params>
+    <param name="last_id" display="Last ID" datatype="integer" default="" order_by="s.id ASC">
+      <where>s.id &gt; #last_id#</where>
+    </param>
+    <param name="autofeed_tracking_date_from" datatype="date" default="" order_by="s.updated_on ASC">
+      <where>s.updated_on >= '#autofeed_tracking_date_from#'</where>
+    </param>
+  </params>
+  <columns>
+    <column name="id" sql="s.id" datatype="integer" />
+    <column name="tracking_date" sql="s.updated_on" datatype="date" />
+    <column name="tracking" sql="s.updated_on" datatype="date" />
+  </columns>
+</report>
\ No newline at end of file
diff --git a/reports/library/samples/list_sample_deletions_all.xml b/reports/library/samples/list_sample_deletions_all.xml
new file mode 100644
index 0000000000..129f19c9d2
--- /dev/null
+++ b/reports/library/samples/list_sample_deletions_all.xml
@@ -0,0 +1,29 @@
+<report
+    title="Delete samples list (no website limit)"
+    description="A list of samples IDs that have been deleted.
+      No website limit so designed to report samples from all website registrations."
+    restricted="true"
+>
+  <query samples_id_field="s.id" count_field="s.*">
+    SELECT #columns#
+    FROM samples s
+    #joins#
+    WHERE s.deleted=true
+  </query>
+  <order_bys>
+    <order_by>s.id</order_by>
+  </order_bys>
+  <params>
+    <param name="last_id" display="Last ID" datatype="integer" default="" order_by="s.id ASC">
+      <where>s.id &gt; #last_id#</where>
+    </param>
+    <param name="autofeed_tracking_date_from" datatype="date" default="" order_by="s.updated_on ASC">
+      <where>s.updated_on >= '#autofeed_tracking_date_from#'</where>
+    </param>
+  </params>
+  <columns>
+    <column name="id" sql="s.id" datatype="integer" />
+    <column name="tracking_date" sql="s.updated_on" datatype="date" />
+    <column name="tracking" sql="s.updated_on" datatype="date" />
+  </columns>
+</report>
\ No newline at end of file

From a6dce35d41362dc7a4f8344247513e08f33b28a8 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Tue, 21 Sep 2021 10:40:07 +0100
Subject: [PATCH 90/93] Completion of sample ES extraction feed reports.

---
 reports/library/samples/list_for_elastic.xml  | 199 +++++++++++++++++
 .../library/samples/list_for_elastic_all.xml  |  11 +-
 .../samples/list_for_elastic_sensitive.xml    | 201 ++++++++++++++++++
 .../list_for_elastic_sensitive_all.xml        | 200 +++++++++++++++++
 4 files changed, 606 insertions(+), 5 deletions(-)
 create mode 100644 reports/library/samples/list_for_elastic.xml
 create mode 100644 reports/library/samples/list_for_elastic_sensitive.xml
 create mode 100644 reports/library/samples/list_for_elastic_sensitive_all.xml

diff --git a/reports/library/samples/list_for_elastic.xml b/reports/library/samples/list_for_elastic.xml
new file mode 100644
index 0000000000..ead11eaf8f
--- /dev/null
+++ b/reports/library/samples/list_for_elastic.xml
@@ -0,0 +1,199 @@
+<report
+    title="Autofeed samples for Elasticsearch (standard filters)"
+    description="A list of samples designed for feeding into Elasticsearch or a similar JSON store."
+    restricted="true"
+>
+  <query website_filter_field="s.website_id" samples_id_field="s.id" training_filter_field=""
+         standard_params="samples" count_field="s.*" blocked_sharing_tasks_field="s.blocked_sharing_tasks">
+    DROP TABLE IF EXISTS filtered_samples;
+    DROP TABLE IF EXISTS occurrence_stats;
+    DROP TABLE IF EXISTS output_rows;
+    DROP TABLE IF EXISTS sample_occurrence_list;
+
+    SELECT s.*
+    INTO TEMPORARY filtered_samples
+    FROM cache_samples_functional s
+    #agreements_join#
+    #joins#
+    WHERE #sharing_filter#
+    #filters#
+    #order_by#
+    LIMIT #limit#;
+
+    SELECT DISTINCT s.id as sample_id, o.id as occurrence_id, o.taxa_taxon_list_id, o.confidential, o.release_status
+    INTO TEMPORARY sample_occurrences_list
+    FROM filtered_samples s
+    JOIN cache_occurrences_functional o ON o.sample_id=s.id OR o.parent_sample_id=s.id;
+
+    SELECT s.id as sample_id, COUNT(DISTINCT ol.occurrence_id) AS count_occurrences, COUNT(distinct cttl.taxon_meaning_id) AS count_taxa, COUNT(distinct cttl.taxon_group_id) AS count_taxon_groups,
+      SUM(case when onf.attr_sex_stage_count similar to '[0-9]{1,9}' then onf.attr_sex_stage_count::integer else null end) AS sum_individual_count,
+      MAX(onf.sensitivity_precision) AS max_sensitivity_precision, BOOL_OR(ol.confidential) AS any_confidential, string_agg(DISTINCT ol.release_status, '') as all_release_status
+    INTO TEMPORARY occurrence_stats
+    FROM filtered_samples s
+    LEFT JOIN sample_occurrences_list ol ON ol.sample_id=s.id
+    LEFT JOIN cache_occurrences_nonfunctional onf ON onf.id=ol.occurrence_id
+    LEFT JOIN cache_taxa_taxon_lists cttl ON cttl.id=ol.taxa_taxon_list_id
+    GROUP BY s.id;
+
+    SELECT #columns#
+    INTO TEMPORARY output_rows
+    FROM filtered_samples s
+    JOIN samples smp ON smp.id=s.id
+    JOIN occurrence_stats os ON os.sample_id=s.id
+    JOIN cache_samples_nonfunctional snf ON snf.id=s.id
+    LEFT JOIN locations l ON l.id=s.location_id AND l.deleted=false;
+
+    UPDATE output_rows o
+    SET recorded_parent_location_id = lp.id,
+      recorded_parent_location_name = lp.name,
+      recorded_parent_location_code = lp.code,
+      parent_sample_attrs_json=snfp.attrs_json
+    FROM samples sp
+    JOIN cache_samples_nonfunctional snfp ON snfp.id=sp.id
+    LEFT JOIN locations lp ON lp.id=sp.location_id AND lp.deleted=false
+    WHERE sp.id=o.parent_sample_id AND sp.deleted=false;
+
+    -- Use parents of locations if no parent sample location.
+    UPDATE output_rows o
+    SET recorded_parent_location_id = lp.id,
+      recorded_parent_location_name = lp.name,
+      recorded_parent_location_code = lp.code
+      FROM samples sc
+    JOIN locations lc ON lc.id=sc.location_id AND lc.deleted=false
+      JOIN locations lp ON lp.id = lc.parent_id AND lp.deleted=false
+      WHERE sc.id=o.sample_id AND sc.deleted=false
+    AND o.recorded_parent_location_id IS NULL;
+
+    SELECT *
+    FROM output_rows s
+    #order_by#
+  </query>
+  <params>
+    <!-- last_id and autofeed_tracking_from parameters required for autofeed to work. -->
+    <param name="last_id" display="Last ID" datatype="integer" default="" order_by="s.id ASC">
+      <where>s.id &gt; #last_id#</where>
+    </param>
+    <param name="autofeed_tracking_from" display="Autofeed tracking from" datatype="integer" default="" order_by="tracking ASC">
+      <where>s.tracking >= #autofeed_tracking_from#</where>
+    </param>
+    <param name="limit" datatype="integer" default="10000" />
+  </params>
+  <columns>
+    <!-- Tracking field called "id" is required for autofeed to work. -->
+    <column name="id" sql="s.id" datatype="integer" />
+    <column name="sample_external_key" sql="s.external_key" datatype="text" />
+    <column name="created_by_id" sql="s.created_by_id" datatype="integer" />
+    <column name="website_id" sql="s.website_id" datatype="integer" />
+    <column name="survey_id" sql="s.survey_id" datatype="integer" />
+    <!-- Deliberate repeat of s.id so it can appear in event.event_id as well as the document ID. -->
+    <column name="sample_id" sql="s.id" datatype="integer" />
+    <column name="parent_sample_id" sql="s.parent_sample_id" datatype="integer" />
+    <column name="group_id" sql="s.group_id" datatype="integer" />
+    <column name="website_title" sql="snf.website_title" datatype="text" />
+    <column name="survey_title" sql="snf.survey_title" datatype="text" />
+    <column name="group_title" sql="snf.group_title" datatype="text" />
+    <column name="recorders" sql="snf.recorders" datatype="text" />
+    <column name="date_start" sql="s.date_start" datatype="date" />
+    <column name="date_end" sql="s.date_end" datatype="date" />
+    <column name="date_type" sql="s.date_type" datatype="text" />
+    <column name="day_of_year" sql="case s.date_type when 'D' then extract(doy from s.date_start) else null end" datatype="integer" />
+    <column name="week" sql="case
+            when coalesce(date_part('year', s.date_start), 0)=coalesce(date_part('year', s.date_end), 0)
+                and coalesce(floor(extract(doy from s.date_start)/7+1), 0)=coalesce(floor(extract(doy from s.date_end)/7+1), 0)
+              then coalesce(floor(extract(doy from s.date_start)/7+1), 0)
+            else null
+          end" datatype="integer" />
+    <column name="ukbms_week" sql="case
+            when coalesce(date_part('year', s.date_start), 0)=coalesce(date_part('year', s.date_end), 0)
+                and
+                  floor((extract('doy' from s.date_start) - extract('doy' from (extract('year' from s.date_start) || '-04-01')::date))/7)
+                  =
+                  floor((extract('doy' from s.date_end) - extract('doy' from (extract('year' from s.date_end) || '-04-01')::date))/7)
+              then floor((extract('doy' from s.date_start) - extract('doy' from (extract('year' from s.date_start) || '-04-01')::date))/7) + 1
+            else null
+          end" datatype="integer" />
+    <column name="month" sql="case
+            when coalesce(date_part('year', s.date_start), 0)=coalesce(date_part('year', s.date_end), 0)
+              and coalesce(date_part('month', s.date_start), 0)=coalesce(date_part('month', s.date_end), 0) then coalesce(date_part('month', s.date_start), 0)
+            else null
+          end" datatype="integer" />
+    <column name="year" sql="case
+            when coalesce(date_part('year', s.date_start), 0)=coalesce(date_part('year', s.date_end), 0) then coalesce(date_part('year', s.date_start), null)
+            else null
+          end" datatype="integer" />
+    <column name="geom" sql="st_astext(st_removerepeatedpoints(st_transform(s.public_geom, 4326)))" datatype="text" />
+    <column name="point" sql="st_y(st_transform(st_centroid(s.public_geom), 4326))::text || ',' || st_x(st_transform(st_centroid(s.public_geom), 4326))::text" datatype="text" />
+    <column name="map_sq_srid" sql="COALESCE(get_output_srid(s.public_geom), 3857)" datatype="integer" />
+    <column name="map_sq_1km" sql="CASE
+      WHEN max_sensitivity_precision > 1000 THEN NULL
+      ELSE TRIM(leading 'POINT(' FROM TRIM(trailing ')' FROM st_astext(st_centroid(st_transform(reduce_precision(s.public_geom, false, 1000), 4326)))))
+    END" datatype="text" />
+    <column name="map_sq_2km" sql="CASE
+      WHEN max_sensitivity_precision > 2000 THEN NULL
+      ELSE TRIM(leading 'POINT(' FROM TRIM(trailing ')' FROM st_astext(st_centroid(st_transform(reduce_precision(s.public_geom, false, 2000), 4326)))))
+    END" datatype="text" />
+    <column name="map_sq_10km" sql="CASE
+      WHEN max_sensitivity_precision > 10000 THEN NULL
+      ELSE TRIM(leading 'POINT(' FROM TRIM(trailing ')' FROM st_astext(st_centroid(st_transform(reduce_precision(s.public_geom, false, 10000), 4326)))))
+    END" datatype="text" />
+    <column name="input_sref" sql="snf.public_entered_sref" datatype="text" />
+    <column name="input_sref_system" sql="snf.entered_sref_system" datatype="text" />
+    <column name="output_sref" sql="snf.output_sref" datatype="text" />
+    <column name="output_sref_system" sql="snf.output_sref_system" datatype="text" />
+    <column name="coordinate_uncertainty_in_meters"
+              sql="CASE
+      WHEN s.sensitive OR s.private OR snf.entered_sref_system NOT SIMILAR TO '[0-9]+' THEN
+        get_sref_precision(snf.public_entered_sref, snf.entered_sref_system, null)
+      ELSE COALESCE(snf.attr_sref_precision, 50)
+    END"
+            datatype="float"
+            term="http://rs.tdwg.org/dwc/terms/coordinateUncertaintyInMeters" />
+    <column name="given_locality_name" sql="s.location_name" datatype="text" />
+    <column name="recorded_location_id" sql="l.id" datatype="integer" />
+    <column name="recorded_location_name" sql="l.name" datatype="text" />
+    <column name="recorded_location_code" sql="l.code" datatype="text" />
+    <column name="recorded_parent_location_id" sql="null::integer" datatype="integer" />
+    <column name="recorded_parent_location_name" sql="null::text" datatype="text" />
+    <column name="recorded_parent_location_code" sql="null::text" datatype="text" />
+    <column name="indexed_location_ids" sql="array_to_string(s.location_ids, ',')" datatype="text" />
+    <column name="sample_comment" sql="snf.comment" datatype="text" />
+    <column name="attr_biotope" sql="snf.attr_biotope" datatype="text" />
+    <column name="attr_sample_method" sql="snf.attr_sample_method" datatype="text" />
+    <column name="media_data" sql="CASE
+      WHEN s.media_count>0 AND sensitive=false AND private=false THEN
+        (SELECT string_agg(sm.path || '~~' || COALESCE(sm.caption, '') || '~~' ||  COALESCE(l.code, '') || '~~' ||  COALESCE(t.term, ''), '@@')
+        FROM sample_media sm
+        LEFT JOIN licences l ON l.id=sm.licence_id
+        LEFT JOIN cache_termlists_terms t ON t.id=sm.media_type_id
+        WHERE sm.sample_id=s.id AND sm.deleted=false)
+      ELSE null
+      END" datatype="text" />
+    <column name="created_on" sql="to_char(s.created_on, 'yyyy-mm-dd HH24:MI:SS.MS')" datatype="date" />
+    <column name="updated_on" sql="to_char(s.updated_on, 'yyyy-mm-dd HH24:MI:SS.MS')" datatype="date" />
+    <column name="record_status" sql="s.record_status" datatype="text" />
+    <column name="verified_by_id" sql="smp.verified_by_id" datatype="integer" />
+    <column name="verifier" sql="snf.verifier" datatype="text" />
+    <column name="verified_on" sql="to_char(s.verified_on, 'yyyy-mm-dd HH24:MI:SS.MS')" datatype="date" />
+    <column name="query" sql="s.query" datatype="text" />
+    <column name="trial" sql="s.training::text" datatype="text" />
+    <column name="sensitive" sql="s.sensitive" datatype="text" />
+    <column name="private" sql="s.private" datatype="text" />
+    <column name="sensitivity_precision" sql="greatest(os.max_sensitivity_precision, s.privacy_precision)" datatype="integer" />
+    <!-- Flag sensitivity_blur = B for blurred version of a sensitive record. -->
+    <column name="sensitivity_blur" sql="CASE WHEN s.sensitive OR s.private THEN 'B' ELSE null END" datatype="text" />
+    <column name="confidential" sql="os.any_confidential::text" datatype="boolean" />
+    <column name="release_status" description="Worst case release status from the contained occurrences" sql="CASE
+      WHEN POSITION('U' IN os.all_release_status)&lt;&gt;0 THEN 'U'
+	    WHEN POSITION('P' IN os.all_release_status)&lt;&gt;0 THEN 'P'
+		  ELSE 'R'
+	  END" datatype="text" />
+    <column name="sample_attrs_json" sql="snf.attrs_json" datatype="text" />
+    <column name="parent_sample_attrs_json" sql="null::json" datatype="text" />
+    <column name="tracking" sql="s.tracking" datatype="integer" />
+    <column name="input_form" sql="s.input_form" datatype="text" />
+    <column name="count_occurrences" sql="os.count_occurrences" datatype="integer" />
+    <column name="count_taxa" sql="os.count_taxa" datatype="integer" />
+    <column name="count_taxon_groups" sql="os.count_taxon_groups" datatype="integer" />
+    <column name="sum_individual_count" sql="os.sum_individual_count" datatype="integer" />
+  </columns>
+</report>
\ No newline at end of file
diff --git a/reports/library/samples/list_for_elastic_all.xml b/reports/library/samples/list_for_elastic_all.xml
index 2e05cecbeb..8203fa34e0 100644
--- a/reports/library/samples/list_for_elastic_all.xml
+++ b/reports/library/samples/list_for_elastic_all.xml
@@ -142,7 +142,7 @@
     <column name="output_sref_system" sql="snf.output_sref_system" datatype="text" />
     <column name="coordinate_uncertainty_in_meters"
               sql="CASE
-      WHEN os.max_sensitivity_precision IS NOT NULL OR snf.privacy_precision IS NOT NULL OR snf.entered_sref_system NOT SIMILAR TO '[0-9]+' THEN
+      WHEN s.sensitive OR s.private OR snf.entered_sref_system NOT SIMILAR TO '[0-9]+' THEN
         get_sref_precision(snf.public_entered_sref, snf.entered_sref_system, null)
       ELSE COALESCE(snf.attr_sref_precision, 50)
     END"
@@ -160,7 +160,7 @@
     <column name="attr_biotope" sql="snf.attr_biotope" datatype="text" />
     <column name="attr_sample_method" sql="snf.attr_sample_method" datatype="text" />
     <column name="media_data" sql="CASE
-      WHEN s.media_count>0 AND os.max_sensitivity_precision IS NULL AND snf.privacy_precision IS NULL THEN
+      WHEN s.media_count>0 AND sensitive=false AND private=false THEN
         (SELECT string_agg(sm.path || '~~' || COALESCE(sm.caption, '') || '~~' ||  COALESCE(l.code, '') || '~~' ||  COALESCE(t.term, ''), '@@')
         FROM sample_media sm
         LEFT JOIN licences l ON l.id=sm.licence_id
@@ -176,10 +176,11 @@
     <column name="verified_on" sql="to_char(s.verified_on, 'yyyy-mm-dd HH24:MI:SS.MS')" datatype="date" />
     <column name="query" sql="s.query" datatype="text" />
     <column name="trial" sql="s.training::text" datatype="text" />
-    <column name="sensitive" sql="(os.max_sensitivity_precision IS NOT NULL)::text" datatype="text" />
-    <column name="sensitivity_precision" sql="os.max_sensitivity_precision" datatype="integer" />
+    <column name="sensitive" sql="s.sensitive" datatype="text" />
+    <column name="private" sql="s.private" datatype="text" />
+    <column name="sensitivity_precision" sql="greatest(os.max_sensitivity_precision, s.privacy_precision)" datatype="integer" />
     <!-- Flag sensitivity_blur = B for blurred version of a sensitive record. -->
-    <column name="sensitivity_blur" sql="CASE WHEN os.max_sensitivity_precision IS NOT NULL THEN 'B' ELSE null END" datatype="text" />
+    <column name="sensitivity_blur" sql="CASE WHEN s.sensitive OR s.private THEN 'B' ELSE null END" datatype="text" />
     <column name="confidential" sql="os.any_confidential::text" datatype="boolean" />
     <column name="release_status" description="Worst case release status from the contained occurrences" sql="CASE
       WHEN POSITION('U' IN os.all_release_status)&lt;&gt;0 THEN 'U'
diff --git a/reports/library/samples/list_for_elastic_sensitive.xml b/reports/library/samples/list_for_elastic_sensitive.xml
new file mode 100644
index 0000000000..56dd315aad
--- /dev/null
+++ b/reports/library/samples/list_for_elastic_sensitive.xml
@@ -0,0 +1,201 @@
+<report
+    title="Autofeed samples for Elasticsearch (standard filters, sensitive only)"
+    description="A list of samples designed for feeding into Elasticsearch or a similar JSON store.
+      This report returns full precision copies of sensitive records. "
+    restricted="true"
+>
+  <query website_filter_field="s.website_id" samples_id_field="s.id" training_filter_field=""
+         standard_params="samples" count_field="s.*" blocked_sharing_tasks_field="s.blocked_sharing_tasks">
+    DROP TABLE IF EXISTS filtered_samples;
+    DROP TABLE IF EXISTS occurrence_stats;
+    DROP TABLE IF EXISTS output_rows;
+    DROP TABLE IF EXISTS sample_occurrence_list;
+
+    SELECT s.*
+    INTO TEMPORARY filtered_samples
+    FROM cache_samples_functional s
+    #agreements_join#
+    #joins#
+    WHERE #sharing_filter#
+    AND s.sensitive=true OR s.private=true
+    #filters#
+    #order_by#
+    LIMIT #limit#;
+
+    SELECT DISTINCT s.id as sample_id, o.id as occurrence_id, o.taxa_taxon_list_id, o.confidential, o.release_status
+    INTO TEMPORARY sample_occurrences_list
+    FROM filtered_samples s
+    JOIN cache_occurrences_functional o ON o.sample_id=s.id OR o.parent_sample_id=s.id;
+
+    SELECT s.id as sample_id, COUNT(DISTINCT ol.occurrence_id) AS count_occurrences, COUNT(distinct cttl.taxon_meaning_id) AS count_taxa, COUNT(distinct cttl.taxon_group_id) AS count_taxon_groups,
+      SUM(case when onf.attr_sex_stage_count similar to '[0-9]{1,9}' then onf.attr_sex_stage_count::integer else null end) AS sum_individual_count,
+      MAX(onf.sensitivity_precision) AS max_sensitivity_precision, BOOL_OR(ol.confidential) AS any_confidential, string_agg(DISTINCT ol.release_status, '') as all_release_status
+    INTO TEMPORARY occurrence_stats
+    FROM filtered_samples s
+    LEFT JOIN sample_occurrences_list ol ON ol.sample_id=s.id
+    LEFT JOIN cache_occurrences_nonfunctional onf ON onf.id=ol.occurrence_id
+    LEFT JOIN cache_taxa_taxon_lists cttl ON cttl.id=ol.taxa_taxon_list_id
+    GROUP BY s.id;
+
+    SELECT #columns#
+    INTO TEMPORARY output_rows
+    FROM filtered_samples s
+    JOIN samples smp ON smp.id=s.id
+    JOIN occurrence_stats os ON os.sample_id=s.id
+    JOIN cache_samples_nonfunctional snf ON snf.id=s.id
+    LEFT JOIN locations l ON l.id=smp.location_id AND l.deleted=false;
+
+    UPDATE output_rows o
+    SET recorded_parent_location_id = lp.id,
+      recorded_parent_location_name = lp.name,
+      recorded_parent_location_code = lp.code,
+      parent_sample_attrs_json=snfp.attrs_json
+    FROM samples sp
+    JOIN cache_samples_nonfunctional snfp ON snfp.id=sp.id
+    LEFT JOIN locations lp ON lp.id=sp.location_id AND lp.deleted=false
+    WHERE sp.id=o.parent_sample_id AND sp.deleted=false;
+
+    -- Use parents of locations if no parent sample location.
+    UPDATE output_rows o
+    SET recorded_parent_location_id = lp.id,
+      recorded_parent_location_name = lp.name,
+      recorded_parent_location_code = lp.code
+      FROM samples sc
+    JOIN locations lc ON lc.id=sc.location_id AND lc.deleted=false
+      JOIN locations lp ON lp.id = lc.parent_id AND lp.deleted=false
+      WHERE sc.id=o.sample_id AND sc.deleted=false
+    AND o.recorded_parent_location_id IS NULL;
+
+    SELECT *
+    FROM output_rows s
+    #order_by#
+  </query>
+  <params>
+    <!-- last_id and autofeed_tracking_from parameters required for autofeed to work. -->
+    <param name="last_id" display="Last ID" datatype="integer" default="" order_by="s.id ASC">
+      <where>s.id &gt; #last_id#</where>
+    </param>
+    <param name="autofeed_tracking_from" display="Autofeed tracking from" datatype="integer" default="" order_by="tracking ASC">
+      <where>s.tracking >= #autofeed_tracking_from#</where>
+    </param>
+    <param name="limit" datatype="integer" default="10000" />
+  </params>
+  <columns>
+    <!-- Tracking field called "id" is required for autofeed to work. -->
+    <column name="id" sql="s.id" datatype="integer" />
+    <column name="sample_external_key" sql="s.external_key" datatype="text" />
+    <column name="created_by_id" sql="s.created_by_id" datatype="integer" />
+    <column name="website_id" sql="s.website_id" datatype="integer" />
+    <column name="survey_id" sql="s.survey_id" datatype="integer" />
+    <!-- Deliberate repeat of s.id so it can appear in event.event_id as well as the document ID. -->
+    <column name="sample_id" sql="s.id" datatype="integer" />
+    <column name="parent_sample_id" sql="s.parent_sample_id" datatype="integer" />
+    <column name="group_id" sql="s.group_id" datatype="integer" />
+    <column name="website_title" sql="snf.website_title" datatype="text" />
+    <column name="survey_title" sql="snf.survey_title" datatype="text" />
+    <column name="group_title" sql="snf.group_title" datatype="text" />
+    <column name="recorders" sql="snf.recorders" datatype="text" />
+    <column name="date_start" sql="s.date_start" datatype="date" />
+    <column name="date_end" sql="s.date_end" datatype="date" />
+    <column name="date_type" sql="s.date_type" datatype="text" />
+    <column name="day_of_year" sql="case s.date_type when 'D' then extract(doy from s.date_start) else null end" datatype="integer" />
+    <column name="week" sql="case
+            when coalesce(date_part('year', s.date_start), 0)=coalesce(date_part('year', s.date_end), 0)
+                and coalesce(floor(extract(doy from s.date_start)/7+1), 0)=coalesce(floor(extract(doy from s.date_end)/7+1), 0)
+              then coalesce(floor(extract(doy from s.date_start)/7+1), 0)
+            else null
+          end" datatype="integer" />
+    <column name="ukbms_week" sql="case
+            when coalesce(date_part('year', s.date_start), 0)=coalesce(date_part('year', s.date_end), 0)
+                and
+                  floor((extract('doy' from s.date_start) - extract('doy' from (extract('year' from s.date_start) || '-04-01')::date))/7)
+                  =
+                  floor((extract('doy' from s.date_end) - extract('doy' from (extract('year' from s.date_end) || '-04-01')::date))/7)
+              then floor((extract('doy' from s.date_start) - extract('doy' from (extract('year' from s.date_start) || '-04-01')::date))/7) + 1
+            else null
+          end" datatype="integer" />
+    <column name="month" sql="case
+            when coalesce(date_part('year', s.date_start), 0)=coalesce(date_part('year', s.date_end), 0)
+              and coalesce(date_part('month', s.date_start), 0)=coalesce(date_part('month', s.date_end), 0) then coalesce(date_part('month', s.date_start), 0)
+            else null
+          end" datatype="integer" />
+    <column name="year" sql="case
+            when coalesce(date_part('year', s.date_start), 0)=coalesce(date_part('year', s.date_end), 0) then coalesce(date_part('year', s.date_start), null)
+            else null
+          end" datatype="integer" />
+    <column name="geom" sql="st_astext(st_removerepeatedpoints(st_transform(smp.geom, 4326)))" datatype="text" />
+    <column name="point" sql="st_y(st_transform(st_centroid(smp.geom), 4326))::text || ',' || st_x(st_transform(st_centroid(smp.geom), 4326))::text" datatype="text" />
+    <column name="map_sq_srid" sql="COALESCE(get_output_srid(smp.geom), 3857)" datatype="integer" />
+    <column name="map_sq_1km" sql="CASE
+      WHEN max_sensitivity_precision > 1000 THEN NULL
+      ELSE TRIM(leading 'POINT(' FROM TRIM(trailing ')' FROM st_astext(st_centroid(st_transform(reduce_precision(smp.geom, false, 1000), 4326)))))
+    END" datatype="text" />
+    <column name="map_sq_2km" sql="CASE
+      WHEN max_sensitivity_precision > 2000 THEN NULL
+      ELSE TRIM(leading 'POINT(' FROM TRIM(trailing ')' FROM st_astext(st_centroid(st_transform(reduce_precision(smp.geom, false, 2000), 4326)))))
+    END" datatype="text" />
+    <column name="map_sq_10km" sql="CASE
+      WHEN max_sensitivity_precision > 10000 THEN NULL
+      ELSE TRIM(leading 'POINT(' FROM TRIM(trailing ')' FROM st_astext(st_centroid(st_transform(reduce_precision(smp.geom, false, 10000), 4326)))))
+    END" datatype="text" />
+    <column name="input_sref" sql="snf.public_entered_sref" datatype="text" />
+    <column name="input_sref_system" sql="snf.entered_sref_system" datatype="text" />
+    <column name="output_sref" sql="snf.output_sref" datatype="text" />
+    <column name="output_sref_system" sql="snf.output_sref_system" datatype="text" />
+    <column name="coordinate_uncertainty_in_meters"
+              sql="CASE
+      WHEN s.sensitive OR s.private OR snf.entered_sref_system NOT SIMILAR TO '[0-9]+' THEN
+        get_sref_precision(snf.public_entered_sref, snf.entered_sref_system, null)
+      ELSE COALESCE(snf.attr_sref_precision, 50)
+    END"
+            datatype="float"
+            term="http://rs.tdwg.org/dwc/terms/coordinateUncertaintyInMeters" />
+    <column name="given_locality_name" sql="s.location_name" datatype="text" />
+    <column name="recorded_location_id" sql="l.id" datatype="integer" />
+    <column name="recorded_location_name" sql="l.name" datatype="text" />
+    <column name="recorded_location_code" sql="l.code" datatype="text" />
+    <column name="recorded_parent_location_id" sql="null::integer" datatype="integer" />
+    <column name="recorded_parent_location_name" sql="null::text" datatype="text" />
+    <column name="recorded_parent_location_code" sql="null::text" datatype="text" />
+    <column name="indexed_location_ids" sql="array_to_string(s.location_ids, ',')" datatype="text" />
+    <column name="sample_comment" sql="snf.comment" datatype="text" />
+    <column name="attr_biotope" sql="snf.attr_biotope" datatype="text" />
+    <column name="attr_sample_method" sql="snf.attr_sample_method" datatype="text" />
+    <column name="media_data" sql="CASE
+      WHEN s.media_count>0 THEN
+        (SELECT string_agg(sm.path || '~~' || COALESCE(sm.caption, '') || '~~' ||  COALESCE(l.code, '') || '~~' ||  COALESCE(t.term, ''), '@@')
+        FROM sample_media sm
+        LEFT JOIN licences l ON l.id=sm.licence_id
+        LEFT JOIN cache_termlists_terms t ON t.id=sm.media_type_id
+        WHERE sm.sample_id=s.id AND sm.deleted=false)
+      ELSE null
+      END" datatype="text" />
+    <column name="created_on" sql="to_char(s.created_on, 'yyyy-mm-dd HH24:MI:SS.MS')" datatype="date" />
+    <column name="updated_on" sql="to_char(s.updated_on, 'yyyy-mm-dd HH24:MI:SS.MS')" datatype="date" />
+    <column name="record_status" sql="s.record_status" datatype="text" />
+    <column name="verified_by_id" sql="smp.verified_by_id" datatype="integer" />
+    <column name="verifier" sql="snf.verifier" datatype="text" />
+    <column name="verified_on" sql="to_char(s.verified_on, 'yyyy-mm-dd HH24:MI:SS.MS')" datatype="date" />
+    <column name="query" sql="s.query" datatype="text" />
+    <column name="trial" sql="s.training::text" datatype="text" />
+    <column name="sensitive" sql="s.sensitive" datatype="text" />
+    <column name="private" sql="s.private" datatype="text" />
+    <column name="sensitivity_precision" sql="greatest(os.max_sensitivity_precision, s.privacy_precision)" datatype="integer" />
+    <!-- Flag sensitivity_blur = F for full precision version of a sensitive record. -->
+    <column name="sensitivity_blur" sql="CASE WHEN s.sensitive OR s.private THEN 'F' ELSE null END" datatype="text" />
+    <column name="confidential" sql="os.any_confidential::text" datatype="boolean" />
+    <column name="release_status" description="Worst case release status from the contained occurrences" sql="CASE
+      WHEN POSITION('U' IN os.all_release_status)&lt;&gt;0 THEN 'U'
+	    WHEN POSITION('P' IN os.all_release_status)&lt;&gt;0 THEN 'P'
+		  ELSE 'R'
+	  END" datatype="text" />
+    <column name="sample_attrs_json" sql="snf.attrs_json" datatype="text" />
+    <column name="parent_sample_attrs_json" sql="null::json" datatype="text" />
+    <column name="tracking" sql="s.tracking" datatype="integer" />
+    <column name="input_form" sql="s.input_form" datatype="text" />
+    <column name="count_occurrences" sql="os.count_occurrences" datatype="integer" />
+    <column name="count_taxa" sql="os.count_taxa" datatype="integer" />
+    <column name="count_taxon_groups" sql="os.count_taxon_groups" datatype="integer" />
+    <column name="sum_individual_count" sql="os.sum_individual_count" datatype="integer" />
+  </columns>
+</report>
\ No newline at end of file
diff --git a/reports/library/samples/list_for_elastic_sensitive_all.xml b/reports/library/samples/list_for_elastic_sensitive_all.xml
new file mode 100644
index 0000000000..c64ad5a143
--- /dev/null
+++ b/reports/library/samples/list_for_elastic_sensitive_all.xml
@@ -0,0 +1,200 @@
+<report
+    title="Autofeed samples for Elasticsearch (standard filters, sensitive only, no website limit)"
+    description="A list of samples designed for feeding into Elasticsearch or a similar JSON store.
+      This report returns full precision copies of sensitive records.
+      No website limit so designed to collect records from all website registrations."
+    restricted="true"
+>
+  <query website_filter_field="s.website_id" samples_id_field="s.id" training_filter_field=""
+         standard_params="samples" count_field="s.*" blocked_sharing_tasks_field="s.blocked_sharing_tasks">
+    DROP TABLE IF EXISTS filtered_samples;
+    DROP TABLE IF EXISTS occurrence_stats;
+    DROP TABLE IF EXISTS output_rows;
+    DROP TABLE IF EXISTS sample_occurrence_list;
+
+    SELECT s.*
+    INTO TEMPORARY filtered_samples
+    FROM cache_samples_functional s
+    #joins#
+    WHERE s.sensitive=true OR s.private=true
+    #filters#
+    #order_by#
+    LIMIT #limit#;
+
+    SELECT DISTINCT s.id as sample_id, o.id as occurrence_id, o.taxa_taxon_list_id, o.confidential, o.release_status
+    INTO TEMPORARY sample_occurrences_list
+    FROM filtered_samples s
+    JOIN cache_occurrences_functional o ON o.sample_id=s.id OR o.parent_sample_id=s.id;
+
+    SELECT s.id as sample_id, COUNT(DISTINCT ol.occurrence_id) AS count_occurrences, COUNT(distinct cttl.taxon_meaning_id) AS count_taxa, COUNT(distinct cttl.taxon_group_id) AS count_taxon_groups,
+      SUM(case when onf.attr_sex_stage_count similar to '[0-9]{1,9}' then onf.attr_sex_stage_count::integer else null end) AS sum_individual_count,
+      MAX(onf.sensitivity_precision) AS max_sensitivity_precision, BOOL_OR(ol.confidential) AS any_confidential, string_agg(DISTINCT ol.release_status, '') as all_release_status
+    INTO TEMPORARY occurrence_stats
+    FROM filtered_samples s
+    LEFT JOIN sample_occurrences_list ol ON ol.sample_id=s.id
+    LEFT JOIN cache_occurrences_nonfunctional onf ON onf.id=ol.occurrence_id
+    LEFT JOIN cache_taxa_taxon_lists cttl ON cttl.id=ol.taxa_taxon_list_id
+    GROUP BY s.id;
+
+    SELECT #columns#
+    INTO TEMPORARY output_rows
+    FROM filtered_samples s
+    JOIN samples smp ON smp.id=s.id
+    JOIN occurrence_stats os ON os.sample_id=s.id
+    JOIN cache_samples_nonfunctional snf ON snf.id=s.id
+    LEFT JOIN locations l ON l.id=smp.location_id AND l.deleted=false;
+
+    UPDATE output_rows o
+    SET recorded_parent_location_id = lp.id,
+      recorded_parent_location_name = lp.name,
+      recorded_parent_location_code = lp.code,
+      parent_sample_attrs_json=snfp.attrs_json
+    FROM samples sp
+    JOIN cache_samples_nonfunctional snfp ON snfp.id=sp.id
+    LEFT JOIN locations lp ON lp.id=sp.location_id AND lp.deleted=false
+    WHERE sp.id=o.parent_sample_id AND sp.deleted=false;
+
+    -- Use parents of locations if no parent sample location.
+    UPDATE output_rows o
+    SET recorded_parent_location_id = lp.id,
+      recorded_parent_location_name = lp.name,
+      recorded_parent_location_code = lp.code
+      FROM samples sc
+    JOIN locations lc ON lc.id=sc.location_id AND lc.deleted=false
+      JOIN locations lp ON lp.id = lc.parent_id AND lp.deleted=false
+      WHERE sc.id=o.sample_id AND sc.deleted=false
+    AND o.recorded_parent_location_id IS NULL;
+
+    SELECT *
+    FROM output_rows s
+    #order_by#
+  </query>
+  <params>
+    <!-- last_id and autofeed_tracking_from parameters required for autofeed to work. -->
+    <param name="last_id" display="Last ID" datatype="integer" default="" order_by="s.id ASC">
+      <where>s.id &gt; #last_id#</where>
+    </param>
+    <param name="autofeed_tracking_from" display="Autofeed tracking from" datatype="integer" default="" order_by="tracking ASC">
+      <where>s.tracking >= #autofeed_tracking_from#</where>
+    </param>
+    <param name="limit" datatype="integer" default="10000" />
+  </params>
+  <columns>
+    <!-- Tracking field called "id" is required for autofeed to work. -->
+    <column name="id" sql="s.id" datatype="integer" />
+    <column name="sample_external_key" sql="s.external_key" datatype="text" />
+    <column name="created_by_id" sql="s.created_by_id" datatype="integer" />
+    <column name="website_id" sql="s.website_id" datatype="integer" />
+    <column name="survey_id" sql="s.survey_id" datatype="integer" />
+    <!-- Deliberate repeat of s.id so it can appear in event.event_id as well as the document ID. -->
+    <column name="sample_id" sql="s.id" datatype="integer" />
+    <column name="parent_sample_id" sql="s.parent_sample_id" datatype="integer" />
+    <column name="group_id" sql="s.group_id" datatype="integer" />
+    <column name="website_title" sql="snf.website_title" datatype="text" />
+    <column name="survey_title" sql="snf.survey_title" datatype="text" />
+    <column name="group_title" sql="snf.group_title" datatype="text" />
+    <column name="recorders" sql="snf.recorders" datatype="text" />
+    <column name="date_start" sql="s.date_start" datatype="date" />
+    <column name="date_end" sql="s.date_end" datatype="date" />
+    <column name="date_type" sql="s.date_type" datatype="text" />
+    <column name="day_of_year" sql="case s.date_type when 'D' then extract(doy from s.date_start) else null end" datatype="integer" />
+    <column name="week" sql="case
+            when coalesce(date_part('year', s.date_start), 0)=coalesce(date_part('year', s.date_end), 0)
+                and coalesce(floor(extract(doy from s.date_start)/7+1), 0)=coalesce(floor(extract(doy from s.date_end)/7+1), 0)
+              then coalesce(floor(extract(doy from s.date_start)/7+1), 0)
+            else null
+          end" datatype="integer" />
+    <column name="ukbms_week" sql="case
+            when coalesce(date_part('year', s.date_start), 0)=coalesce(date_part('year', s.date_end), 0)
+                and
+                  floor((extract('doy' from s.date_start) - extract('doy' from (extract('year' from s.date_start) || '-04-01')::date))/7)
+                  =
+                  floor((extract('doy' from s.date_end) - extract('doy' from (extract('year' from s.date_end) || '-04-01')::date))/7)
+              then floor((extract('doy' from s.date_start) - extract('doy' from (extract('year' from s.date_start) || '-04-01')::date))/7) + 1
+            else null
+          end" datatype="integer" />
+    <column name="month" sql="case
+            when coalesce(date_part('year', s.date_start), 0)=coalesce(date_part('year', s.date_end), 0)
+              and coalesce(date_part('month', s.date_start), 0)=coalesce(date_part('month', s.date_end), 0) then coalesce(date_part('month', s.date_start), 0)
+            else null
+          end" datatype="integer" />
+    <column name="year" sql="case
+            when coalesce(date_part('year', s.date_start), 0)=coalesce(date_part('year', s.date_end), 0) then coalesce(date_part('year', s.date_start), null)
+            else null
+          end" datatype="integer" />
+    <column name="geom" sql="st_astext(st_removerepeatedpoints(st_transform(smp.geom, 4326)))" datatype="text" />
+    <column name="point" sql="st_y(st_transform(st_centroid(smp.geom), 4326))::text || ',' || st_x(st_transform(st_centroid(smp.geom), 4326))::text" datatype="text" />
+    <column name="map_sq_srid" sql="COALESCE(get_output_srid(smp.geom), 3857)" datatype="integer" />
+    <column name="map_sq_1km" sql="CASE
+      WHEN max_sensitivity_precision > 1000 THEN NULL
+      ELSE TRIM(leading 'POINT(' FROM TRIM(trailing ')' FROM st_astext(st_centroid(st_transform(reduce_precision(smp.geom, false, 1000), 4326)))))
+    END" datatype="text" />
+    <column name="map_sq_2km" sql="CASE
+      WHEN max_sensitivity_precision > 2000 THEN NULL
+      ELSE TRIM(leading 'POINT(' FROM TRIM(trailing ')' FROM st_astext(st_centroid(st_transform(reduce_precision(smp.geom, false, 2000), 4326)))))
+    END" datatype="text" />
+    <column name="map_sq_10km" sql="CASE
+      WHEN max_sensitivity_precision > 10000 THEN NULL
+      ELSE TRIM(leading 'POINT(' FROM TRIM(trailing ')' FROM st_astext(st_centroid(st_transform(reduce_precision(smp.geom, false, 10000), 4326)))))
+    END" datatype="text" />
+    <column name="input_sref" sql="snf.public_entered_sref" datatype="text" />
+    <column name="input_sref_system" sql="snf.entered_sref_system" datatype="text" />
+    <column name="output_sref" sql="snf.output_sref" datatype="text" />
+    <column name="output_sref_system" sql="snf.output_sref_system" datatype="text" />
+    <column name="coordinate_uncertainty_in_meters"
+              sql="CASE
+      WHEN s.sensitive OR s.private OR snf.entered_sref_system NOT SIMILAR TO '[0-9]+' THEN
+        get_sref_precision(snf.public_entered_sref, snf.entered_sref_system, null)
+      ELSE COALESCE(snf.attr_sref_precision, 50)
+    END"
+            datatype="float"
+            term="http://rs.tdwg.org/dwc/terms/coordinateUncertaintyInMeters" />
+    <column name="given_locality_name" sql="s.location_name" datatype="text" />
+    <column name="recorded_location_id" sql="l.id" datatype="integer" />
+    <column name="recorded_location_name" sql="l.name" datatype="text" />
+    <column name="recorded_location_code" sql="l.code" datatype="text" />
+    <column name="recorded_parent_location_id" sql="null::integer" datatype="integer" />
+    <column name="recorded_parent_location_name" sql="null::text" datatype="text" />
+    <column name="recorded_parent_location_code" sql="null::text" datatype="text" />
+    <column name="indexed_location_ids" sql="array_to_string(s.location_ids, ',')" datatype="text" />
+    <column name="sample_comment" sql="snf.comment" datatype="text" />
+    <column name="attr_biotope" sql="snf.attr_biotope" datatype="text" />
+    <column name="attr_sample_method" sql="snf.attr_sample_method" datatype="text" />
+    <column name="media_data" sql="CASE
+      WHEN s.media_count>0 THEN
+        (SELECT string_agg(sm.path || '~~' || COALESCE(sm.caption, '') || '~~' ||  COALESCE(l.code, '') || '~~' ||  COALESCE(t.term, ''), '@@')
+        FROM sample_media sm
+        LEFT JOIN licences l ON l.id=sm.licence_id
+        LEFT JOIN cache_termlists_terms t ON t.id=sm.media_type_id
+        WHERE sm.sample_id=s.id AND sm.deleted=false)
+      ELSE null
+      END" datatype="text" />
+    <column name="created_on" sql="to_char(s.created_on, 'yyyy-mm-dd HH24:MI:SS.MS')" datatype="date" />
+    <column name="updated_on" sql="to_char(s.updated_on, 'yyyy-mm-dd HH24:MI:SS.MS')" datatype="date" />
+    <column name="record_status" sql="s.record_status" datatype="text" />
+    <column name="verified_by_id" sql="smp.verified_by_id" datatype="integer" />
+    <column name="verifier" sql="snf.verifier" datatype="text" />
+    <column name="verified_on" sql="to_char(s.verified_on, 'yyyy-mm-dd HH24:MI:SS.MS')" datatype="date" />
+    <column name="query" sql="s.query" datatype="text" />
+    <column name="trial" sql="s.training::text" datatype="text" />
+    <column name="sensitive" sql="s.sensitive" datatype="text" />
+    <column name="private" sql="s.private" datatype="text" />
+    <column name="sensitivity_precision" sql="greatest(os.max_sensitivity_precision, s.privacy_precision)" datatype="integer" />
+    <!-- Flag sensitivity_blur = F for full precision version of a sensitive record. -->
+    <column name="sensitivity_blur" sql="CASE WHEN s.sensitive OR s.private THEN 'F' ELSE null END" datatype="text" />
+    <column name="confidential" sql="os.any_confidential::text" datatype="boolean" />
+    <column name="release_status" description="Worst case release status from the contained occurrences" sql="CASE
+      WHEN POSITION('U' IN os.all_release_status)&lt;&gt;0 THEN 'U'
+	    WHEN POSITION('P' IN os.all_release_status)&lt;&gt;0 THEN 'P'
+		  ELSE 'R'
+	  END" datatype="text" />
+    <column name="sample_attrs_json" sql="snf.attrs_json" datatype="text" />
+    <column name="parent_sample_attrs_json" sql="null::json" datatype="text" />
+    <column name="tracking" sql="s.tracking" datatype="integer" />
+    <column name="input_form" sql="s.input_form" datatype="text" />
+    <column name="count_occurrences" sql="os.count_occurrences" datatype="integer" />
+    <column name="count_taxa" sql="os.count_taxa" datatype="integer" />
+    <column name="count_taxon_groups" sql="os.count_taxon_groups" datatype="integer" />
+    <column name="sum_individual_count" sql="os.sum_individual_count" datatype="integer" />
+  </columns>
+</report>
\ No newline at end of file

From 739c76cad42d5c6d46143b203768755ad1e5f1cf Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Tue, 21 Sep 2021 12:25:44 +0100
Subject: [PATCH 91/93] CHANGELOG for v6.3

---
 CHANGELOG.md | 43 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 589c09db01..3eab4ae432 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,46 @@
+# Version 6.3.0
+*2021-09-21*
+
+* Support for Elasticsearch indexes which contain samples as documents (including empty samples).
+  These can be enabled for access via the REST API.
+* Addition of occurrences.verifier_only_data field to support data synced from other systems where
+  the data is supplied with attribute values that are only permitted to be used for verification.
+* Code updates for PHP 8 compatibility and updated unit test libraries.
+* Improvements to sensitivity handling for sample cache data, including:
+  * Addition of sensitive & private flags.
+  * Blurring the public geometry when any contained occurrences are sensitive,.
+  * The public_entered_sref is now populated with the blurred and localised grid reference when
+    there are sensitive records in a sample. Formerly it was left null.
+  * Fixes a bug where the map square links were not being populated for the full-precision copy of
+    sensitive records.
+* Adds the following fields fields to samples cache for consistency with the occurrences cache
+  tables:
+  * cache_samples_functional.external_key
+  * cache_samples_functional.sensitive
+  * cache_samples_functional.private
+  * cache_samples_nonfunctional.output_sref
+  * cache_samples_nonfunctional.output_sref_system
+  * cache_samples_nonfunctional.private
+* Updating an occurrence in isolation (via web services) now updates the tracking ID associated
+  with the sample that contains the occurrence. This is so that any sample data feeds receive an
+  updated copy of the sample, as the occurrence statistics will have changed.
+* Workflow events now allow filters on location, or stage term. These are applied retrospectively
+  using a Work Queue task, allowing spatial indexing to be applied to the record first. For example
+  this allows a workflow event's effect to be removed from a record if it does not fall inside a
+  boundary or is a juvenile.
+* REST API module provides sync-taxon-observations and sync-annotation end-points designed for
+  synchronising records and verification decisions with remote servers.
+* New json_occurrences server type for the REST API Sync module which sychronises data with any
+  remote (Indicia or otherwise) server that supports the sync-taxon-observations and
+  sync-annotations API format.
+* Bug fixes.
+
+## Deprecation notice
+
+* The previously provided taxon-observations and annotations end-points in the REST API (which were
+  based on the defunct NBN Gateway Exchange Format) are now deprecated and may be removed in a
+  future version.
+
 # Version 6.2.0
 *2021-08-02*
 

From d2b7c3095ba1894761f486c6db99840f9ae3f08f Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Tue, 21 Sep 2021 12:25:59 +0100
Subject: [PATCH 92/93] Version datestamp

---
 application/config/version.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/application/config/version.php b/application/config/version.php
index 1bef0988b3..4dd66d37a5 100644
--- a/application/config/version.php
+++ b/application/config/version.php
@@ -36,7 +36,7 @@
  *
  * @var string
  */
-$config['release_date'] = '2021-09-14';
+$config['release_date'] = '2021-09-21';
 
 /**
  * Link to the code repository downloads page.

From f3f1fab7560b8c2558398abec3ff6b92c818772b Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Tue, 21 Sep 2021 12:26:23 +0100
Subject: [PATCH 93/93] Deprecation annotations

---
 modules/rest_api/controllers/services/rest.php | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/modules/rest_api/controllers/services/rest.php b/modules/rest_api/controllers/services/rest.php
index 5445fcc071..f5b29442c9 100644
--- a/modules/rest_api/controllers/services/rest.php
+++ b/modules/rest_api/controllers/services/rest.php
@@ -1078,6 +1078,11 @@ private function projectsGet() {
    *
    * Outputs a single taxon observations's details.
    *
+   * @deprecated
+   *   Deprecated in version 6.3 and may be removed in future. Use the
+   *   sync-taxon-observations end-point provided by the rest_api_sync module
+   *   instead.
+   *
    * @param string $id
    *   Unique ID for the taxon-observations to output.
    */
@@ -1116,6 +1121,11 @@ private function taxonObservationsGetId($id) {
    *
    * Outputs a list of taxon observation details.
    *
+   * @deprecated
+   *   Deprecated in version 6.3 and may be removed in future. Use the
+   *   sync-taxon-observations end-point provided by the rest_api_sync module
+   *   instead.
+   *
    * @todo Ensure delete information is output.
    */
   private function taxonObservationsGet() {