Merge branch 'develop' of https://github.com/Indicia-Team/warehouse i…

…nto develop
Indicia-Team · Oct 25, 2019 · 5dcf657 · 5dcf657
2 parents 9b0558b + 592a991
commit 5dcf657
Show file tree

Hide file tree

Showing 12 changed files with 1,233 additions and 354 deletions.
diff --git a/application/config/version.php b/application/config/version.php
@@ -29,14 +29,14 @@
  *
  * @var string
  */
-$config['version'] = '2.35.1';
+$config['version'] = '2.36.0';
 
 /**
  * Version release date.
  *
  * @var string
  */
-$config['release_date'] = '2019-10-03';
+$config['release_date'] = '2019-10-14';
 
 /**
  * Link to the code repository downloads page.

diff --git a/application/controllers/survey.php b/application/controllers/survey.php
@@ -65,9 +65,39 @@ protected function prepareOtherViewData(array $values) {
     $arr = array();
     foreach ($websites->where('deleted','false')->orderby('title','asc')->find_all() as $website)
       $arr[$website->id] = $website->title;
-    return array(
+
+    $otherData = array(
       'websites' => $arr
     );
+
+    $otherData['taxon_restrictions'] = []; 
+    $masterListId = warehouse::getMasterTaxonListId();
+    if ($masterListId) {
+
+      $tmIdVals = $this->db
+          ->select('s.auto_accept_taxa_filters')
+          ->from('surveys AS s')
+          ->where([
+            's.id' => $values['survey:id'],
+          ])
+          ->get()->result();
+
+      $valsCSV=trim($tmIdVals[0]->auto_accept_taxa_filters, "{}");
+
+      $ttlIds = $this->db
+          ->select('id')
+          ->from('cache_taxa_taxon_lists as cttl')
+          ->in('taxon_meaning_id', explode(",", $valsCSV))
+          ->where([
+            'cttl.preferred' => true
+          ])
+          ->get()->result();
+
+      foreach ($ttlIds as $ttlId) {
+        array_push($otherData['taxon_restrictions'], array("taxa_taxon_list_id" => $ttlId->id));
+      }
+    }
+    return $otherData;
   }
 
   /**

diff --git a/application/models/survey.php b/application/models/survey.php
@@ -54,9 +54,32 @@ public function validate(Validation $array, $save = FALSE) {
       'owner_id',
       'auto_accept',
       'auto_accept_max_difficulty',
+      'auto_accept_taxa_filters',
       'core_validation_rules',
     );
     return parent::validate($array, $save);
   }
 
+  protected function preSubmit() {
+    if (!empty($_POST['has-taxon-restriction-data'])) {
+      $ttlIds = [];
+      $tmIds = [];
+      foreach ($_POST as $key => $value) {
+        if (substr($key, -8) === ':present' && $value !== '0') {
+          $ttlIds[] = $value;
+        }  
+      }
+      $tmIdRecs = $this->db
+          ->select('id, taxon_meaning_id')
+          ->from('cache_taxa_taxon_lists')
+          ->in('id', $ttlIds)
+          ->get()->result();
+
+      foreach ($tmIdRecs as $tmIdRec) {
+        $tmIds[] = intVal($tmIdRec->taxon_meaning_id);
+      }
+      $this->submission['fields']['auto_accept_taxa_filters']=array('value' => $tmIds);
+    }
+    return parent::presubmit();
+  }
 }
diff --git a/application/views/survey/survey_edit.php b/application/views/survey/survey_edit.php
@@ -138,6 +138,36 @@
         'helpText' => 'If Auto Accept is set, then this is the minimum identification difficulty that will be auto verified.',
       ));
     }
+    if (array_key_exists('survey:auto_accept_taxa_filters', $values)) {
+      $masterListId = warehouse::getMasterTaxonListId();
+      echo <<<HTML
+<div class="alert alert-info">
+ <p>You can use the taxon selection control below to 
+ select one or more higher level taxa to which recorded taxa must belong in order to
+ quality for auto-verification. Leave the list empty for no filtering. You must also
+ check the Auto Accept box for these filters to take effect.</p>
+</div>
+<label>Taxon restrictions</label>
+<input type="hidden" name="has-taxon-restriction-data" value="1" />
+HTML;
+      require_once 'client_helpers/prebuilt_forms/includes/language_utils.php';
+      $speciesChecklistOptions = [
+        'lookupListId' => $masterListId,
+        'rowInclusionCheck' => 'alwaysRemovable',
+        'extraParams' => $readAuth,
+        'survey_id' => $values['survey:id'],
+        'language' => iform_lang_iso_639_2(kohana::config('indicia.default_lang')),
+      ];
+      if (!empty($other_data['taxon_restrictions'])) {
+        $speciesChecklistOptions['listId'] = $masterListId;
+        $speciesChecklistOptions['preloadTaxa'] = [];
+        foreach ($other_data['taxon_restrictions'] as $restriction) {
+          $speciesChecklistOptions['preloadTaxa'][] = $restriction['taxa_taxon_list_id'];
+        }
+      }
+      echo data_entry_helper::species_checklist($speciesChecklistOptions);
+      echo '<br/>';
+    }
     ?>
   </fieldset>
   <?php if (array_key_exists('attributes', $values) && count($values['attributes']) > 0) : ?>

diff --git a/client_helpers b/client_helpers
diff --git a/modules/auto_verify/config/auto_verify.php.example.php b/modules/auto_verify/config/auto_verify.php.example.php
@@ -20,7 +20,9 @@
  * @link 	http://code.google.com/p/indicia/
  */
 
-$config['auto_accept_occurrences_with_null_id_difficulty']='false';
-//Do we process any data from cache_occurrences, or just look in occDelta for newly changed data.
-//You would probably only need this mode as a one-off run.
-$config['process_old_data']='false';
+$config['auto_accept_occurrences_with_null_id_difficulty']='true';
+// Note that -1 (or less) is unlimited, 0 processes nothing
+$config['max_num_records_to_process_at_once']=0;
+$config['oldest_record_created_date_to_process']='01/01/2000';
+// 1 (or less) is effectively unlimited
+$config['oldest_occurrence_id_to_process']=1;
diff --git a/modules/auto_verify/db/version_2_36_0/201910071330_new_survey_fields.sql b/modules/auto_verify/db/version_2_36_0/201910071330_new_survey_fields.sql
@@ -0,0 +1,24 @@
+CREATE OR REPLACE function f_add_new_survey_fields (OUT success bool)
+    LANGUAGE plpgsql AS
+$func$
+BEGIN 
+
+success := TRUE;
+
+BEGIN
+	ALTER TABLE surveys ADD COLUMN auto_accept_taxa_filters INT[];
+
+EXCEPTION
+    WHEN duplicate_column THEN 
+      RAISE NOTICE 'column exists.';
+      success := FALSE;
+END;
+
+END
+$func$;
+
+SELECT f_add_new_survey_fields();
+
+DROP FUNCTION f_add_new_survey_fields();
+
+COMMENT ON COLUMN surveys.auto_accept_taxa_filters IS 'List of taxon meaning IDs to filter records qualifying for auto-verification';
diff --git a/modules/auto_verify/plugins/auto_verify.php b/modules/auto_verify/plugins/auto_verify.php
@@ -32,35 +32,59 @@
  *   Database object.
  *
  * @todo Config for $autoVerifyNullIdDiff should be a boolean, not a string
- * @todo Config for $processOldData should be a boolean, not a string
  */
 function auto_verify_scheduled_task($last_run_date, $db) {
   $autoVerifyNullIdDiff = kohana::config('auto_verify.auto_accept_occurrences_with_null_id_difficulty', FALSE, FALSE);
-  $processOldData = kohana::config('auto_verify.process_old_data', FALSE, FALSE);
+
+  $oldestRecordCreatedDateToProcess = kohana::config('auto_verify.oldest_record_created_date_to_process', FALSE, FALSE);
+  $oldestOccurrenceIdToProcess = kohana::config('auto_verify.oldest_occurrence_id_to_process', FALSE, FALSE);
+  $maxRecordsNumber = kohana::config('auto_verify.max_num_records_to_process_at_once', FALSE, FALSE);
+
   if (empty($autoVerifyNullIdDiff)) {
     echo "Unable to automatically verify occurrences when the auto_accept_occurrences_with_null_id_difficulty entry is empty.<br>";
     kohana::log('error', 'Unable to automatically verify occurrences when the auto_accept_occurrences_with_null_id_difficulty configuration entry is empty.');
     return FALSE;
   }
-  // Do we need to consider old data (probably as a one-off run) or just newly
-  // changed data.
-  $subQuery = "
-    SELECT delta.id";
-  if (!empty($processOldData)&&$processOldData === 'true') {
-    $subQuery .= "
-      FROM cache_occurrences_functional delta";
+
+  if (empty($oldestRecordCreatedDateToProcess)) {
+    echo "Unable to automatically verify occurrences when the oldest_record_created_date_to_process entry is empty.<br>";
+    kohana::log('error', 'Unable to automatically verify occurrences when the oldest_record_created_date_to_process configuration entry is empty.');
+    return FALSE;
   }
-  else {
-    $subQuery .= "
-      FROM occdelta delta";
+
+  if (empty($oldestOccurrenceIdToProcess)) {
+    echo "Unable to automatically verify occurrences when the oldest_occurrence_id_to_process entry is empty.<br>";
+    kohana::log('error', 'Unable to automatically verify occurrences when the oldest_occurrence_id_to_process configuration entry is empty.');
+    return FALSE;
+  }
+
+  if (empty($maxRecordsNumber)) {
+    echo "Unable to automatically verify occurrences when the max_num_records_to_process_at_once entry is empty.<br>";
+    kohana::log('error', 'Unable to automatically verify occurrences when the max_num_records_to_process_at_once configuration entry is empty.');
+    return FALSE;
   }
-  $subQuery .= "
+
+  $subQuery = "
+    SELECT distinct delta.id
+    FROM cache_occurrences_functional delta
     JOIN surveys s on s.id = delta.survey_id AND s.auto_accept=true AND s.deleted=false
     LEFT JOIN cache_taxon_searchterms cts on cts.taxon_meaning_id = delta.taxon_meaning_id
     WHERE delta.data_cleaner_result=true
     AND delta.record_status='C' AND delta.record_substatus IS NULL
+        AND delta.created_on >= TO_TIMESTAMP('$oldestRecordCreatedDateToProcess', 'DD/MM/YYYY')
         AND (($autoVerifyNullIdDiff=false AND cts.identification_difficulty IS NOT NULL AND cts.identification_difficulty<=s.auto_accept_max_difficulty)
         OR ($autoVerifyNullIdDiff=true AND (cts.identification_difficulty IS NULL OR cts.identification_difficulty<=s.auto_accept_max_difficulty)))";
+
+  if (isset($oldestOccurrenceIdToProcess) && $oldestOccurrenceIdToProcess > -1) {
+    $subQuery .= "
+      AND delta.id >= $oldestOccurrenceIdToProcess";
+  }
+
+  if (isset($maxRecordsNumber) && $maxRecordsNumber > -1) {
+    $subQuery .= "
+      order by delta.id desc limit $maxRecordsNumber";
+  }
+
   $verificationTime = gmdate("Y\/m\/d H:i:s");
   //Need to update cache_occurrences_*, as these tables have already been built at this point.
   $query = "
@@ -77,7 +101,9 @@ function auto_verify_scheduled_task($last_run_date, $db) {
     release_status='R',
     verified_by_id=1,
     verified_on='$verificationTime',
-    record_decision_source='M'
+    record_decision_source='M',
+    updated_on = now(),
+    updated_by_id = 1
     WHERE id in
     ($subQuery);
 
@@ -110,12 +136,3 @@ function auto_verify_scheduled_task($last_run_date, $db) {
   else
     echo 'No occurrence records have been auto-verified.</br>';
 }
-
-/*
- * Tell the system that we need the occdelta table to find out which occurrences have been created/changed recently.
- */
-function auto_verify_metadata() {
-  return array(
-    'requires_occurrences_delta'=>TRUE
-  );
-}
diff --git a/modules/data_cleaner_period_within_year/plugins/data_cleaner_period_within_year.php b/modules/data_cleaner_period_within_year/plugins/data_cleaner_period_within_year.php
@@ -139,12 +139,12 @@ function data_cleaner_period_within_year_data_cleaner_rules() {
 and (vr.stages is null or vr.stages @> string_to_array(co.stage, ''))
 SQL;
   // The groupBy allows us to count the verified records at a similar time of
-  // year and only create messages if less than 2.
+  // year and only create messages if less than 6.
   $groupBy = <<<SQL
 group by co.id, co.date_start, co.taxa_taxon_list_external_key, co.stage,
   co.verification_checks_enabled, co.record_status, vr.error_message, vr.stages
--- at least 2 similar records
-having count(o2.id) < 2
+-- at least 6 similar records
+having count(o2.id) < 6
 SQL;
 
   return array(