Merge branch 'develop'

Indicia-Team · Feb 22, 2021 · 7782841 · 7782841
2 parents 6a23a7c + 17661b8
commit 7782841
Show file tree

Hide file tree

Showing 22 changed files with 2,969 additions and 2,458 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,16 @@
+# Version 5.0.0
+
+* Dropped support for oAuth2 password authentication as no longer recommended in the oAuth 2.0
+  Security Best Current Practice. See https://oauth.net/2/grant-types/password/.
+* JWT authentication on the REST API - dropped http://indicia.org.uk/alldata token in the payload.
+  Now, default is to return user's own data only, but this can be expanded by specifying scope
+  in the payload with a space separated list of scopes (sharing modes) - defining which website's
+  records will appear in the response. Options include: userWithinWebsite, user (all records for
+  user across all websites), reporting, verification, data_flow, editing, peer_review, moderation.
+  If a JWT token claims multiple scopes, the URL parameter scope can select the active one for a
+  request.
+* Adds occurrence custom attribute system functions for behaviour and reproductive_condition.
+
 # Version 4.11.0
 *2021-01-19*
 
@@ -7,14 +20,14 @@
 * New `es_key_prefix` option in `application/config/indicia.php` which allows a prefix to be added
   to IDs in Elasticsearch downloads (to uniquely ID the warehouse).
 * Support for Excel (*.xls or *.xlsx) files in the importer (experimental).
-* After upgrades, now more effectively clears appropriate parts of the cache so that UI and ORM 
+* After upgrades, now more effectively clears appropriate parts of the cache so that UI and ORM
   updates are immediate.
-* Performance improvement by indexing notifications table for count of user's outstanding 
+* Performance improvement by indexing notifications table for count of user's outstanding
   notifications.
 * Improvements to Elasticsearch download column templates, e.g. for better MapMate export support.
 * Option to skip overwrite of verified records in the Rest API sync module (e.g. for iNaturalist
   synchronisation), see https://github.com/BiologicalRecordsCentre/iRecord/issues/972.
-# Version 4.10.0 
+# Version 4.10.0
 *2020-12-17*
 
 * New uksi_operations module allows update logs for UKSI to be directly imported, with support for

diff --git a/application/config/version.php b/application/config/version.php
@@ -29,14 +29,14 @@
  *
  * @var string
  */
-$config['version'] = '4.12.1';
+$config['version'] = '5.0.0';
 
 /**
  * Version release date.
  *
  * @var string
  */
-$config['release_date'] = '2021-02-16';
+$config['release_date'] = '2021-02-22';
 
 /**
  * Link to the code repository downloads page.

diff --git a/application/libraries/XMLReportReader.php b/application/libraries/XMLReportReader.php
@@ -375,31 +375,36 @@ public function applyWebsitePermissions(&$query, $websiteIds, $providedParams, $
     if ($websiteIds) {
       if (in_array('', $websiteIds)) {
         foreach ($websiteIds as $key => $value) {
-          if (empty($value))
+          if (empty($value)) {
             unset($websiteIds[$key]);
+          }
         }
       }
       $idList = implode(',', $websiteIds);
-      // query can either pull in the filter or just the list of website ids.
+      // Query can either pull in the filter or just the list of website ids.
       $filter = empty($this->websiteFilterField) ? "1=1" : "({$this->websiteFilterField} in ($idList) or {$this->websiteFilterField} is null)";
-      $query = str_replace(array('#website_filter#', '#website_ids#'), array($filter, $idList), $query);
-    } else
-      // use a dummy filter to return all websites if core admin
-      $query = str_replace(array('#website_filter#', '#website_ids#'), array('1=1', 'SELECT id FROM websites'), $query);
+      $query = str_replace(['#website_filter#', '#website_ids#'], [$filter, $idList], $query);
+    }
+    else {
+      // Use a dummy filter to return all websites if core admin.
+      $query = str_replace(['#website_filter#', '#website_ids#'], ['1=1', 'SELECT id FROM websites'], $query);
+    }
     if (!empty($this->trainingFilterField)) {
       $boolStr = $providedParams['training'] ==='true' ? 'true' : 'false';
       $query = str_replace('#sharing_filter#', "{$this->trainingFilterField}=$boolStr AND #sharing_filter#", $query);
     }
-    // an alternative way to inform a query about training mode....
+    // An alternative way to inform a query about training mode....
     $query = str_replace('#training#', $providedParams['training'], $query);
-    // select the appropriate type of sharing arrangement (i.e. are we reporting, verifying, moderating etc?)
-    if ($sharing==='me' && empty($userId))
-      // revert to website type sharing if we have no known user Id.
-      $sharing='website';
-    $agreementsJoins = array();
-    $sharingFilters = array();
-    if ($sharing==='me') {
-      // my data only so use the UserId if we have it.
+    // Select the appropriate type of sharing arrangement (i.e. are we
+    // reporting, verifying, moderating etc?)
+    if ($sharing === 'me' && empty($userId)) {
+      // Revert to website type sharing if we have no known user Id.
+      $sharing = 'website';
+    }
+    $agreementsJoins = [];
+    $sharingFilters = [];
+    if ($sharing === 'me') {
+      // My data only so use the UserId if we have it.
       $sharingFilters[] = "{$this->createdByField}=$userId";
       // 'me' is a subtype of reporting
       $sharing = 'reporting';
@@ -430,7 +435,7 @@ public function applyWebsitePermissions(&$query, $websiteIds, $providedParams, $
         }
         // If scope not controlled by a survey standard parameter filter, then
         // apply a website_id filter. Avoid doing this unnecessary as it
-        // affects performance
+        // affects performance.
         if (!$this->coveringSurveyFilter($providedParams, $sharedWebsiteIdList)) {
           $sharingFilters[] = "$this->websiteFilterField in ($sharedWebsiteIdList)";
         }
@@ -439,11 +444,12 @@ public function applyWebsitePermissions(&$query, $websiteIds, $providedParams, $
 
     }
     // Add a dummy sharing filter if nothing else set, for the sake of syntax.
-    if (empty($sharingFilters))
+    if (empty($sharingFilters)) {
       $sharingFilters[] = '1=1';
+    }
     $query = str_replace(
-      array('#agreements_join#', '#sharing_filter#', '#sharing#'),
-      array(implode("\n", $agreementsJoins), implode("\n AND ", $sharingFilters), $sharing),
+      ['#agreements_join#', '#sharing_filter#', '#sharing#'],
+      [implode("\n", $agreementsJoins), implode("\n AND ", $sharingFilters), $sharing],
       $query
     );
   }
@@ -488,12 +494,12 @@ private function coveringSurveyFilter(array $providedParams, $sharedWebsiteIdLis
   }
 
   /**
-   * A cached lookup of the websites that are available for a certain sharing mode.
+   * A cached lookup of the websites that are available for a sharing mode.
    *
-   * Only bother to cache the lookup if there is only 1 website (i.e. we are running a
-   * report from a client website or the warehouse user can only see 1 website). Otherwise
-   * there are too many possibilities to be worth it. This is mainly to speed up client
-   * website reporting anyway.
+   * Only bother to cache the lookup if there is only 1 website (i.e. we are
+   * running a report from a client website or the warehouse user can only see
+   * 1 website). Otherwise there are too many possibilities to be worth it.
+   * This is mainly to speed up client website reporting anyway.
    */
   private function getSharedWebsiteList($websiteIds, $sharing) {
     if (count($websiteIds) === 1) {
@@ -509,12 +515,13 @@ private function getSharedWebsiteList($websiteIds, $sharing) {
       ->where("receive_for_$sharing", 't')
       ->in('from_website_id', $websiteIds)
       ->get()->result();
-    $ids = array();
+    $ids = [];
     foreach ($qry as $row) {
       $ids[] = $row->to_website_id;
     }
     $r = implode(',', $ids);
-    // Tag all cache entries for this website so they can be cleared together when changes are saved.
+    // Tag all cache entries for this website so they can be cleared together
+    // when changes are saved.
     $cache->set($cacheId, $r, $tag);
     return $r;
   }
@@ -523,9 +530,9 @@ private function getSharedWebsiteList($websiteIds, $sharing) {
    * Use the sql attributes from the list of columns to auto generate the columns SQL.
    */
   private function autogenColumns() {
-    $sql = array();
-    $distinctSql = array();
-    $countSql = array();
+    $sql = [];
+    $distinctSql = [];
+    $countSql = [];
     foreach ($this->columns as $col => $def) {
       if (!empty($this->colsToInclude) && !in_array($col, $this->colsToInclude))
         continue;

diff --git a/application/models/groups_location.php b/application/models/groups_location.php
@@ -34,4 +34,21 @@ public function validate(Validation $array, $save = FALSE) {
     return parent::validate($array, $save);
   }
 
+  /**
+   * Defines a submission structure for groups_locations.
+   *
+   * Lets locations be submitted at the same time, e.g. during CSV upload.
+   *
+   * @return array
+   *   Submission structure.
+   */
+  public function get_submission_structure() {
+    return [
+      'model' => $this->object_name,
+      'superModels' => [
+        'location' => ['fk' => 'location_id'],
+      ],
+    ];
+  }
+
 }
diff --git a/application/models/occurrence_attribute.php b/application/models/occurrence_attribute.php
@@ -28,19 +28,17 @@
  */
 class Occurrence_Attribute_Model extends ATTR_ORM {
 
-  protected $belongs_to = array(
+  protected $belongs_to = [
     'created_by' => 'user',
     'updated_by' => 'user',
     'termlist_id' => 'termlist',
     'source_id' => 'termlists_term',
     'reporting_category_id' => 'termlists_term',
-  );
+  ];
 
-  protected $has_many = array(
-    'occurrence_attributes_values',
-  );
+  protected $has_many = ['occurrence_attributes_values'];
 
-  protected $has_and_belongs_to_many = array('websites');
+  protected $has_and_belongs_to_many = ['websites'];
 
   /**
    * Retrieve system functions for occurrence attributes.
@@ -54,48 +52,56 @@ class Occurrence_Attribute_Model extends ATTR_ORM {
    *   List of the system known functions that a sample attribute can have.
    */
   public function get_system_functions() {
-    return array(
-      'sex_stage' => array(
-        'title' => 'Sex/stage',
-        'description' => 'A text or lookup attribute where the value corresponds to the sex or life stage of the recorded organism(s).',
-      ),
-      'sex' => array(
-        'title' => 'Sex',
-        'description' => 'A text or lookup attribute where the value corresponds to the sex of the recorded organism(s).',
-      ),
-      'stage' => array(
-        'title' => 'Stage',
-        'description' => 'A text or lookup attribute where the value corresponds to the life stage of the recorded organism(s).',
-      ),
-      'sex_stage_count' => array(
-        'title' => 'Count or abundance of a sex or life stage.',
-        'friendly' => 'Abundance',
-        'description' => 'An attribute corresponding to the abundance of a particular stage, indicated by the caption of the attribute. ' .
-          'The attribute can be an integer count, in which case zero means not present, a checkbox corresponding to presence/absence, ' .
-          'or a lookup where terms "Absent","None", "Not Present" or "0" indicate not present.',
-      ),
-      'certainty' => array(
+    return [
+      'behaviour' => [
+        'title' => 'Behaviour',
+        'description' => 'The behavior shown by the subject at the time the Occurrence was recorded.',
+      ],
+      'certainty' => [
         'title' => 'Certainty of the record accuracy',
         'friendly' => 'Certainty',
         'description' => 'Attribute value describes how certain the recorder is of the record. Please ensure that any terms corresponding ' .
           'to a certain record have a sort order of less than 100, any terms corresponding to a likely record which is not certain have ' .
           'a sort order of 100-199 and any terms corresponding to a record which is not at least considered likely have a sort order of ' .
           '200 or more.',
-      ),
-      'det_first_name' => array(
+      ],
+      'det_first_name' => [
         'title' => 'Determiner first name',
         'description' => 'A text attribute corresponding to the first name of the person determining (identifying) the record.',
-      ),
-      'det_last_name' => array(
-        'title' => 'Determiner last name',
-        'description' => 'A text attribute corresponding to the last name of the person determining (identifying) the record.',
-      ),
-      'det_full_name' => array(
+      ],
+      'det_full_name' => [
         'title' => 'Determiner full name',
         'friendly' => 'Identified by',
         'description' => 'A text attribute corresponding to the full name of the person determining (identifying) the record.',
-      ),
-    );
+      ],
+      'det_last_name' => [
+        'title' => 'Determiner last name',
+        'description' => 'A text attribute corresponding to the last name of the person determining (identifying) the record.',
+      ],
+      'sex_stage' => [
+        'title' => 'Sex/stage',
+        'description' => 'A text or lookup attribute where the value corresponds to the sex or life stage of the recorded organism(s).',
+      ],
+      'reproductive_condition' => [
+        'title' => 'Reproductive condition',
+        'description' => 'The reproductive condition of the biological individual(s) represented in the Occurrence.',
+      ],
+      'sex' => [
+        'title' => 'Sex',
+        'description' => 'A text or lookup attribute where the value corresponds to the sex of the recorded organism(s).',
+      ],
+      'stage' => [
+        'title' => 'Stage',
+        'description' => 'A text or lookup attribute where the value corresponds to the life stage of the recorded organism(s).',
+      ],
+      'sex_stage_count' => [
+        'title' => 'Count or abundance of a sex or life stage.',
+        'friendly' => 'Abundance',
+        'description' => 'An attribute corresponding to the abundance of a particular stage, indicated by the caption of the attribute. ' .
+          'The attribute can be an integer count, in which case zero means not present, a checkbox corresponding to presence/absence, ' .
+          'or a lookup where terms "Absent","None", "Not Present" or "0" indicate not present.',
+      ],
+    ];
   }
 
 }
diff --git a/modules/cache_builder/config/cache_builder.php b/modules/cache_builder/config/cache_builder.php
@@ -1480,6 +1480,16 @@
   ),
   verifier=pv.surname || ', ' || pv.first_name,
   licence_code=li.code,
+  attr_behaviour=CASE a_behaviour.data_type
+    WHEN 'T'::bpchar THEN v_behaviour.text_value
+    WHEN 'L'::bpchar THEN t_behaviour.term
+    ELSE NULL::text
+  END,
+  attr_reproductive_condition=CASE a_reproductive_condition.data_type
+    WHEN 'T'::bpchar THEN v_reproductive_condition.text_value
+    WHEN 'L'::bpchar THEN t_reproductive_condition.term
+    ELSE NULL::text
+  END,
   attr_sex_stage=CASE a_sex_stage.data_type
       WHEN 'T'::bpchar THEN v_sex_stage.text_value
       WHEN 'L'::bpchar THEN t_sex_stage.term
@@ -1536,6 +1546,14 @@
   JOIN sample_attributes spa on spa.id=spv.sample_attribute_id and spa.deleted=false
       and spa.system_function='sref_precision'
 ) on spv.sample_id=s.id and spv.deleted=false
+LEFT JOIN (occurrence_attribute_values v_behaviour
+  JOIN occurrence_attributes a_behaviour on a_behaviour.id=v_behaviour.occurrence_attribute_id and a_behaviour.deleted=false and a_behaviour.system_function='behaviour'
+  LEFT JOIN cache_termlists_terms t_behaviour on a_behaviour.data_type='L' and t_behaviour.id=v_behaviour.int_value
+) on v_behaviour.occurrence_id=o.id and v_behaviour.deleted=false
+LEFT JOIN (occurrence_attribute_values v_reproductive_condition
+  JOIN occurrence_attributes a_reproductive_condition on a_reproductive_condition.id=v_reproductive_condition.occurrence_attribute_id and a_reproductive_condition.deleted=false and a_reproductive_condition.system_function='reproductive_condition'
+  LEFT JOIN cache_termlists_terms t_reproductive_condition on a_reproductive_condition.data_type='L' and t_reproductive_condition.id=v_reproductive_condition.int_value
+) on v_reproductive_condition.occurrence_id=o.id and v_reproductive_condition.deleted=false
 LEFT JOIN (occurrence_attribute_values v_sex_stage
   JOIN occurrence_attributes a_sex_stage on a_sex_stage.id=v_sex_stage.occurrence_attribute_id and a_sex_stage.deleted=false and a_sex_stage.system_function='sex_stage'
   LEFT JOIN cache_termlists_terms t_sex_stage on a_sex_stage.data_type='L' and t_sex_stage.id=v_sex_stage.int_value
@@ -1728,6 +1746,16 @@
 $config['occurrences']['insert']['nonfunctional_attrs'] = "
 UPDATE cache_occurrences_nonfunctional
 SET
+  attr_behaviour=CASE a_behaviour.data_type
+      WHEN 'T'::bpchar THEN v_behaviour.text_value
+      WHEN 'L'::bpchar THEN t_behaviour.term
+      ELSE NULL::text
+  END,
+  attr_reproductive_condition=CASE a_reproductive_condition.data_type
+      WHEN 'T'::bpchar THEN v_reproductive_condition.text_value
+      WHEN 'L'::bpchar THEN t_reproductive_condition.term
+      ELSE NULL::text
+  END,
   attr_sex_stage=CASE a_sex_stage.data_type
       WHEN 'T'::bpchar THEN v_sex_stage.text_value
       WHEN 'L'::bpchar THEN t_sex_stage.term
@@ -1772,6 +1800,14 @@
   END
 FROM occurrences o
 #join_needs_update#
+LEFT JOIN (occurrence_attribute_values v_behaviour
+  JOIN occurrence_attributes a_behaviour on a_behaviour.id=v_behaviour.occurrence_attribute_id and a_behaviour.deleted=false and a_behaviour.system_function='behaviour'
+  LEFT JOIN cache_termlists_terms t_behaviour on a_behaviour.data_type='L' and t_behaviour.id=v_behaviour.int_value
+) on v_behaviour.occurrence_id=o.id and v_behaviour.deleted=false
+LEFT JOIN (occurrence_attribute_values v_reproductive_condition
+  JOIN occurrence_attributes a_reproductive_condition on a_reproductive_condition.id=v_reproductive_condition.occurrence_attribute_id and a_reproductive_condition.deleted=false and a_reproductive_condition.system_function='reproductive_condition'
+  LEFT JOIN cache_termlists_terms t_reproductive_condition on a_reproductive_condition.data_type='L' and t_reproductive_condition.id=v_reproductive_condition.int_value
+) on v_reproductive_condition.occurrence_id=o.id and v_reproductive_condition.deleted=false
 LEFT JOIN (occurrence_attribute_values v_sex_stage
   JOIN occurrence_attributes a_sex_stage on a_sex_stage.id=v_sex_stage.occurrence_attribute_id and a_sex_stage.deleted=false and a_sex_stage.system_function='sex_stage'
   LEFT JOIN cache_termlists_terms t_sex_stage on a_sex_stage.data_type='L' and t_sex_stage.id=v_sex_stage.int_value

diff --git a/modules/cache_builder/db/version_5_0_0/202102081131_sysfuncs.sql b/modules/cache_builder/db/version_5_0_0/202102081131_sysfuncs.sql
@@ -0,0 +1,3 @@
+ALTER TABLE cache_occurrences_nonfunctional
+  ADD COLUMN attr_behaviour character varying,
+  ADD COLUMN attr_reproductive_condition character varying;