diff --git a/.travis.yml b/.travis.yml index b7b0f49ca5..8e945a6454 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ language: php php: # Test oldest and newest maintained versions. - '7.3' - # - '8.0' + - '8.0' env: # Test oldest and newest maintained versions. diff --git a/CHANGELOG.md b/CHANGELOG.md index 589c09db01..3eab4ae432 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,46 @@ +# Version 6.3.0 +*2021-09-21* + +* Support for Elasticsearch indexes which contain samples as documents (including empty samples). + These can be enabled for access via the REST API. +* Addition of occurrences.verifier_only_data field to support data synced from other systems where + the data is supplied with attribute values that are only permitted to be used for verification. +* Code updates for PHP 8 compatibility and updated unit test libraries. +* Improvements to sensitivity handling for sample cache data, including: + * Addition of sensitive & private flags. + * Blurring the public geometry when any contained occurrences are sensitive,. + * The public_entered_sref is now populated with the blurred and localised grid reference when + there are sensitive records in a sample. Formerly it was left null. + * Fixes a bug where the map square links were not being populated for the full-precision copy of + sensitive records. +* Adds the following fields fields to samples cache for consistency with the occurrences cache + tables: + * cache_samples_functional.external_key + * cache_samples_functional.sensitive + * cache_samples_functional.private + * cache_samples_nonfunctional.output_sref + * cache_samples_nonfunctional.output_sref_system + * cache_samples_nonfunctional.private +* Updating an occurrence in isolation (via web services) now updates the tracking ID associated + with the sample that contains the occurrence. This is so that any sample data feeds receive an + updated copy of the sample, as the occurrence statistics will have changed. +* Workflow events now allow filters on location, or stage term. These are applied retrospectively + using a Work Queue task, allowing spatial indexing to be applied to the record first. For example + this allows a workflow event's effect to be removed from a record if it does not fall inside a + boundary or is a juvenile. +* REST API module provides sync-taxon-observations and sync-annotation end-points designed for + synchronising records and verification decisions with remote servers. +* New json_occurrences server type for the REST API Sync module which sychronises data with any + remote (Indicia or otherwise) server that supports the sync-taxon-observations and + sync-annotations API format. +* Bug fixes. + +## Deprecation notice + +* The previously provided taxon-observations and annotations end-points in the REST API (which were + based on the defunct NBN Gateway Exchange Format) are now deprecated and may be removed in a + future version. + # Version 6.2.0 *2021-08-02* diff --git a/application/config/version.php b/application/config/version.php index 534aeb23c3..4dd66d37a5 100644 --- a/application/config/version.php +++ b/application/config/version.php @@ -29,14 +29,14 @@ * * @var string */ -$config['version'] = '6.2.15'; +$config['version'] = '6.3.0'; /** * Version release date. * * @var string */ -$config['release_date'] = '2021-09-14'; +$config['release_date'] = '2021-09-21'; /** * Link to the code repository downloads page. diff --git a/application/controllers/service_base.php b/application/controllers/service_base.php index fbec7cab5d..c80b7feb2a 100644 --- a/application/controllers/service_base.php +++ b/application/controllers/service_base.php @@ -125,10 +125,10 @@ protected function authenticate($mode = 'write') { $authentic = FALSE; // default if (array_key_exists('nonce', $array) && array_key_exists('auth_token', $array)) { $nonce = $array['nonce']; - $this->cache = new Cache; + $this->cache = new Cache(); // Get all cache entries that match this nonce $paths = $this->cache->exists($nonce); - foreach($paths as $path) { + foreach ($paths as $path) { // Find the parts of each file name, which is the cache entry ID, then the mode. $tokens = explode('~', basename($path)); // check this cached nonce is for the correct read or write operation. @@ -139,14 +139,16 @@ protected function authenticate($mode = 'write') { $website = ORM::factory('website', $id); if ($website->id) $password = $website->password; - } else + } + else { $password = kohana::config('indicia.private_key'); + } // calculate the auth token from the nonce and the password. Does it match the request's auth token? if (isset($password) && sha1("$nonce:$password")==$array['auth_token']) { Kohana::log('info', "Authentication successful."); // cache website_password for subsequent use by controllers $this->website_password = $password; - $authentic=true; + $authentic = TRUE; } if ($authentic) { if ($id > 0) { @@ -166,7 +168,7 @@ protected function authenticate($mode = 'write') { $user = ORM::Factory('user', $this->user_id); $this->user_is_core_admin = ($user->core_role_id === 1); if (!$this->user_is_core_admin) { - $this->user_websites = array(); + $this->user_websites = []; $userWebsites = ORM::Factory('users_website')->where(array( 'user_id' => $this->user_id, 'site_role_id is not' => NULL, diff --git a/application/helpers/postgreSQL.php b/application/helpers/postgreSQL.php index 6acd07d0d2..919b79ee51 100644 --- a/application/helpers/postgreSQL.php +++ b/application/helpers/postgreSQL.php @@ -372,7 +372,8 @@ public static function list_fields($entity, $db = NULL) { SELECT column_name, column_default, is_nullable, data_type, udt_name, character_maximum_length, numeric_precision, numeric_precision_radix, numeric_scale FROM information_schema.columns - WHERE table_name = \'' . $entity . '\' + WHERE table_name = \'' . $entity . '\' + AND table_schema != \'information_schema\' ORDER BY ordinal_position '); diff --git a/application/libraries/MY_ORM.php b/application/libraries/MY_ORM.php index 125de9e1e7..697ef492f7 100644 --- a/application/libraries/MY_ORM.php +++ b/application/libraries/MY_ORM.php @@ -61,7 +61,7 @@ public function last_query() { return $this->db->last_query(); } - public $submission = array(); + public $submission = []; /** * Describes the list of nested models that are present after a submission. @@ -70,8 +70,8 @@ public function last_query() { * * @var array */ - private $nestedChildModelIds = array(); - private $nestedParentModelIds = array(); + private $nestedChildModelIds = []; + private $nestedParentModelIds = []; /** * Default search field name. @@ -82,7 +82,7 @@ public function last_query() { */ public $search_field = 'title'; - protected $errors = array(); + protected $errors = []; /** * Flag that gets set if a unique key violation has occurred on save. @@ -101,12 +101,12 @@ public function last_query() { * by a model. If not declared then the model will not transfer them to the saved data when * posting a record. */ - protected $unvalidatedFields = array(); + protected $unvalidatedFields = []; /** * @var array An array which a model can populate to declare additional fields that can be submitted for csv upload. */ - protected $additional_csv_fields = array(); + protected $additional_csv_fields = []; /** * @var bool Does the model have custom attributes? Defaults to false. @@ -128,6 +128,7 @@ public function last_query() { /** * Default behaviour on save is to update metadata. If we detect no changes we can skip this. + * * @var bool */ public $wantToUpdateMetadata = TRUE; @@ -139,7 +140,7 @@ public function last_query() { */ public $parentChanging = FALSE; - private $attrValModels = array(); + private $attrValModels = []; /** * @var array If a submission contains submodels, then the array of submodels can be keyed. This @@ -147,7 +148,7 @@ public function last_query() { * Normally, super/sub-models can handle foreign keys, but this approach is needed for association * tables which join across 2 entities created by a submission. */ - private $dynamicRowIdReferences = array(); + private $dynamicRowIdReferences = []; /** * Indicates database trigger on table which accesses a sequence. @@ -163,8 +164,11 @@ public function last_query() { /** * Constructor allows plugins to modify the data model. - * @var int $id ID of the record to load. If null then creates a new record. If -1 then the ORM - * object is not initialised, providing access to the variables only. + * + * @var int $id + * ID of the record to load. If null then creates a new record. If -1 then + * the ORM object is not initialised, providing access to the variables + * only. */ public function __construct($id = NULL) { if (is_object($id) || $id != -1) { @@ -343,7 +347,7 @@ public function getAllErrors() * Retrieve an array containing all page level errors which are marked with the key general. */ public function getPageErrors() { - $r = array(); + $r = []; if (array_key_exists('general', $this->errors)) { array_push($r, $this->errors['general']); } @@ -567,7 +571,7 @@ protected function canCreateFromCaption() { * @return array, an array of record id values for the created records. */ private function createRecordsFromCaptions() { - $r = array(); + $r = []; // Establish the right model and check it supports create from captions, $modelname = $this->submission['fields']['insert_captions_to_create']['value']; @@ -584,7 +588,7 @@ private function createRecordsFromCaptions() { $sub = array( 'id' => $modelname, 'fields' => array( - 'caption' => array() + 'caption' => [] ) ); // submit each caption to create a record, unless it exists @@ -621,10 +625,10 @@ private function createRecordsFromCaptions() { */ private function createIdsFromCaptions($ids) { $fieldname = $this->submission['fields']['insert_captions_use']['value']; - if(empty($ids)){ - $this->submission['fields'][$fieldname] = array('value'=>array()); + if (empty($ids)) { + $this->submission['fields'][$fieldname] = ['value'=>[]]; } - else{ + else { $keys = array_fill(0, sizeof($ids), 'value'); $a = array_fill_keys($keys, $ids); $this->submission['fields'][$fieldname] = $a; @@ -688,28 +692,35 @@ protected function preSubmit() { */ protected function populateIdentifiers() { if (array_key_exists('website_id', $this->submission['fields'])) { - if (is_array($this->submission['fields']['website_id'])) + if (is_array($this->submission['fields']['website_id'])) { $this->identifiers['website_id'] = $this->submission['fields']['website_id']['value']; - else + } + else { $this->identifiers['website_id'] = $this->submission['fields']['website_id']; + } } if (array_key_exists('survey_id', $this->submission['fields'])) { - if (is_array($this->submission['fields']['survey_id'])) + if (is_array($this->submission['fields']['survey_id'])) { $this->identifiers['survey_id'] = $this->submission['fields']['survey_id']['value']; - else + } + else { $this->identifiers['survey_id'] = $this->submission['fields']['survey_id']; + } } } /** * Wraps the process of submission in a transaction. - * @return integer If successful, returns the id of the created/found record. If not, returns null - errors are embedded in the model. + * + * @return int + * If successful, returns the id of the created/found record. If not, + * returns null - errors are embedded in the model. */ public function submit() { Kohana::log('debug', 'Commencing new transaction.'); $this->db->query('BEGIN;'); try { - $this->errors = array(); + $this->errors = []; $this->preProcess(); $res = $this->inner_submit(); $this->postProcess(); @@ -720,8 +731,8 @@ public function submit() { $res = NULL; } if ($res) { - $allowCommitToDB = (isset($_GET['allow_commit_to_db']) ? $_GET['allow_commit_to_db'] : true); - if (!empty($allowCommitToDB)&&$allowCommitToDB==true) { + $allowCommitToDB = (isset($_GET['allow_commit_to_db']) ? $_GET['allow_commit_to_db'] : TRUE); + if (!empty($allowCommitToDB) && $allowCommitToDB == TRUE) { Kohana::log('debug', 'Committing transaction.'); $this->db->query('COMMIT;'); } @@ -747,13 +758,16 @@ private function preProcess() { } /** - * Handles any index rebuild requirements as a result of new or updated records, e.g. in - * samples or occurrences. Also handles joining of occurrence_associations to the - * correct records. + * Submission post-processing. + * + * Handles any index rebuild requirements as a result of new or updated + * records, e.g. in samples or occurrences. Also handles joining of + * occurrence_associations to the correct records. */ private function postProcess() { if (class_exists('cache_builder')) { $occurrences = []; + $deletedOccurrences = []; if (!empty(self::$changedRecords['insert']['occurrence'])) { cache_builder::insert($this->db, 'occurrences', self::$changedRecords['insert']['occurrence']); $occurrences = self::$changedRecords['insert']['occurrence']; @@ -764,6 +778,7 @@ private function postProcess() { } if (!empty(self::$changedRecords['delete']['occurrence'])) { cache_builder::delete($this->db, 'occurrences', self::$changedRecords['delete']['occurrence']); + $deletedOccurrences = self::$changedRecords['delete']['occurrence']; } $samples = []; if (!empty(self::$changedRecords['insert']['sample'])) { @@ -785,6 +800,9 @@ private function postProcess() { // No need to do occurrence map square update if inserting a sample, as // the above code does the occurrences in bulk. postgreSQL::insertMapSquaresForOccurrences($occurrences, $this->db); + // Need to ensure sample tracking is updated if occurrences change + // without a posted sample. + cache_builder::updateSampleTrackingForOccurrences($this->db, $occurrences + $deletedOccurrences); } } if (!empty(self::$changedRecords['insert']['occurrence_association']) || @@ -801,7 +819,7 @@ private function postProcess() { } } // Reset important if doing an import with multiple submissions. - Occurrence_association_Model::$to_occurrence_id_pointers = array(); + Occurrence_association_Model::$to_occurrence_id_pointers = []; } $this->createWorkQueueEntries(); } @@ -912,7 +930,7 @@ public function inner_submit(){ else $addTo=&self::$changedRecords['update']; if (!isset($addTo[$this->object_name])) - $addTo[$this->object_name] = array(); + $addTo[$this->object_name] = []; $addTo[$this->object_name][] = $this->id; } // Call postSubmit @@ -978,7 +996,7 @@ protected function validateAndSubmit() { // The easiest thing here is pretend the current value of any array // column doesn't match. These array columns are used so rarely that this // less optimised solution is not important. - $exactMatches = array(); + $exactMatches = []; foreach ($thisValues as $column => $value) { if (array_key_exists($column, $vArray) && !is_array($vArray[$column]) && @@ -1326,8 +1344,8 @@ private function checkRequiredAttributes() { // Test if this model has an attributes sub-table. Also to have required attributes, we must be posting into a // specified survey or website at least. if ($this->has_attributes) { - $got_values=array(); - $empties = array(); + $got_values=[]; + $empties = []; if (isset($this->submission['metaFields'][$this->attrs_submission_name])) { // Old way of submitting attribute values but still supported - attributes are stored in a metafield. Find the ones we actually have a value for @@ -1419,7 +1437,7 @@ protected function getRequiredFieldsCacheKey($typeFilter) { */ protected function getAttributes($required = FALSE, $typeFilter = NULL, $hasSurveyRestriction = TRUE) { if (empty($this->identifiers['website_id']) && empty($this->identifiers['taxon_list_id'])) { - return array(); + return []; } $attr_entity = $this->object_name . '_attribute'; $this->db->select($attr_entity.'s.id', $attr_entity.'s.caption', $attr_entity.'s.data_type'); @@ -1523,7 +1541,7 @@ public function getSubmittableFields($fk = FALSE, array $identifiers = [], $attr // currently can only have associations if a single superModel exists. if($use_associations && count($struct['superModels']) === 1) { // duplicate all the existing fields, but rename adding a 2 to model end. - $newFields = array(); + $newFields = []; foreach($fields as $name=>$caption){ $parts=explode(':',$name); if($parts[0]==$struct['model'] || $parts[0] == $struct['model'].'_image' || $parts[0] == $this->attrs_field_prefix) { @@ -1560,7 +1578,7 @@ public function getRequiredFields($fk = FALSE, array $identifiers = [], $use_ass $sub = $this->get_submission_structure(); $arr = new Validation(array('id'=>1)); $this->validate($arr, FALSE); - $fields = array(); + $fields = []; foreach ($arr->errors() as $column=>$error) { if ($error=='required') { if ($fk && substr($column, -3) == "_id") { @@ -1584,7 +1602,7 @@ public function getRequiredFields($fk = FALSE, array $identifiers = [], $use_ass // currently can only have associations if a single superModel exists. if($use_associations && count($sub['superModels'])===1){ // duplicate all the existing fields, but rename adding a 2 to model end. - $newFields = array(); + $newFields = []; foreach($fields as $id){ $parts=explode(':',$id); if($parts[0]==$sub['model'] || $parts[0]==$sub['model'].'_image' || $parts[0]==$this->attrs_field_prefix) { @@ -1611,7 +1629,7 @@ public function getRequiredFields($fk = FALSE, array $identifiers = [], $use_ass * @return array Prefixed key value pairs. */ public function getPrefixedValuesArray($prefix=NULL) { - $r = array(); + $r = []; if (!$prefix) { $prefix=$this->object_name; } @@ -1627,7 +1645,7 @@ public function getPrefixedValuesArray($prefix=NULL) { * @return array Prefixed columns. */ protected function getPrefixedColumnsArray($fk=FALSE, $skipHiddenFields=TRUE) { - $r = array(); + $r = []; $prefix=$this->object_name; $sub = $this->get_submission_structure(); foreach ($this->table_columns as $column=>$type) { @@ -2181,7 +2199,7 @@ public function get_submission_structure() { * on creation of a new record. */ public function getDefaults() { - return array(); + return []; } /** @@ -2200,8 +2218,8 @@ private function sanitise($array) { */ public function clear() { parent::clear(); - $this->errors=array(); - $this->identifiers = array('website_id'=>NULL,'survey_id'=>NULL); + $this->errors = []; + $this->identifiers = ['website_id' => NULL, 'survey_id' => NULL]; } /** diff --git a/application/libraries/WorkQueue.php b/application/libraries/WorkQueue.php index 965704242b..5b96a8547b 100644 --- a/application/libraries/WorkQueue.php +++ b/application/libraries/WorkQueue.php @@ -72,7 +72,8 @@ public function enqueue($db, array $fields) { 'task=' . pg_escape_literal($fields['task']) . 'AND entity' . (empty($fields['entity']) ? ' IS NULL' : '=' . pg_escape_literal($fields['entity'])) . 'AND record_id' . (empty($fields['record_id']) ? ' IS NULL' : '=' . pg_escape_literal($fields['record_id'])) . - 'AND params' . (empty($fields['params']) ? ' IS NULL' : '=' . pg_escape_literal($fields['params'])); + // Use JSONB to compare as valid in pgSQL. + 'AND params' . (empty($fields['params']) ? ' IS NULL' : ('::jsonb=' . pg_escape_literal($fields['params']) . '::jsonb')); foreach ($fields as $value) { $setValues[] = pg_escape_literal($value); } diff --git a/application/models/occurrence.php b/application/models/occurrence.php index 0c428f6629..6fe4b6c985 100644 --- a/application/models/occurrence.php +++ b/application/models/occurrence.php @@ -25,24 +25,27 @@ class Occurrence_Model extends ORM { protected $requeuedForVerification = FALSE; - protected $has_many = array( + protected $has_many = [ 'occurrence_attribute_values', 'determinations', - 'occurrence_media' - ); - protected $belongs_to = array( + 'occurrence_media', + ]; + + protected $belongs_to = [ 'determiner' => 'person', 'sample', 'taxa_taxon_list', 'created_by' => 'user', 'updated_by' => 'user', - 'verified_by' => 'user' - ); - // Declare that this model has child attributes, and the name of the node in the submission which contains them. + 'verified_by' => 'user', + ]; + + // Declare that this model has child attributes, and the name of the node in + // the submission which contains them. protected $has_attributes = TRUE; protected $attrs_submission_name = 'occAttributes'; public $attrs_field_prefix = 'occAttr'; - protected $additional_csv_fields = array( + protected $additional_csv_fields = [ // Extra lookup options. 'occurrence:fk_taxa_taxon_list:genus' => 'Genus (builds binomial name)', 'occurrence:fk_taxa_taxon_list:specific' => 'Specific name/epithet (builds binomial name)', @@ -58,8 +61,8 @@ class Occurrence_Model extends ORM { 'occurrence_medium:path:3' => 'Media Path 3', 'occurrence_medium:caption:3' => 'Media Caption 3', 'occurrence_medium:path:4' => 'Media Path 4', - 'occurrence_medium:caption:4' => 'Media Caption 4' - ); + 'occurrence_medium:caption:4' => 'Media Caption 4', + ]; // During an import it is possible to merge different columns in a CSV row to make a database field public $specialImportFieldProcessingDefn = [ @@ -108,6 +111,9 @@ class Occurrence_Model extends ORM { /** * Returns a caption to identify this model instance. + * + * @return string + * Caption for instance. */ public function caption() { return 'Record of ' . $this->taxa_taxon_list->taxon->taxon; @@ -165,7 +171,7 @@ public function validate(Validation $array, $save = FALSE) { $array->add_rules('taxa_taxon_list_id', 'required'); } // Explicitly add those fields for which we don't do validation. - $this->unvalidatedFields = array( + $this->unvalidatedFields = [ 'comment', 'determiner_id', 'deleted', @@ -184,7 +190,8 @@ public function validate(Validation $array, $save = FALSE) { 'sensitivity_precision', 'import_guid', 'metadata', - ); + 'verifier_only_data', + ]; if (array_key_exists('id', $fieldlist)) { // Existing data must not be set to download_flag=F (final download) otherwise it // is read only. diff --git a/application/views/attribute_by_survey/index.php b/application/views/attribute_by_survey/index.php index ab6131b13e..50a5b208ca 100644 --- a/application/views/attribute_by_survey/index.php +++ b/application/views/attribute_by_survey/index.php @@ -223,7 +223,7 @@ function get_controls($block_id, array $controlFilter, $db) { id"] = $attr->caption; } diff --git a/composer.json b/composer.json index 25a01df3d6..8faa3968d8 100644 --- a/composer.json +++ b/composer.json @@ -1,10 +1,16 @@ { + "repositories": [ + { + "type": "vcs", + "url": "https://github.com/misantron/dbunit" + } + ], "require": { "firebase/php-jwt": "^5.4", "phpoffice/phpspreadsheet": "^1.18" }, "require-dev": { "phpunit/phpunit": "^9.5", - "misantron/dbunit": "^5.1" + "misantron/dbunit": "dev-master" } } diff --git a/composer.lock b/composer.lock index f9d9ce3713..6eff9858be 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "c505b8dc101b76633c8ae935e66c78b2", + "content-hash": "2fccf8723ed1dfa22f7247880fd11bd9", "packages": [ { "name": "ezyang/htmlpurifier", @@ -885,37 +885,45 @@ }, { "name": "misantron/dbunit", - "version": "5.1.0", + "version": "dev-master", "source": { "type": "git", "url": "https://github.com/misantron/dbunit.git", - "reference": "a3e5d3c74a2ae78827c86e14e3d06d7a8d44ca65" + "reference": "73a9c07dca119c68e92002adb4fab9022235a91f" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/misantron/dbunit/zipball/a3e5d3c74a2ae78827c86e14e3d06d7a8d44ca65", - "reference": "a3e5d3c74a2ae78827c86e14e3d06d7a8d44ca65", + "url": "https://api.github.com/repos/misantron/dbunit/zipball/73a9c07dca119c68e92002adb4fab9022235a91f", + "reference": "73a9c07dca119c68e92002adb4fab9022235a91f", "shasum": "" }, "require": { - "ext-libxml": "*", "ext-pdo": "*", - "ext-simplexml": "*", - "php": "^7.2|^7.3|^7.4", - "phpunit/phpunit": "^8.5|^9.2", - "symfony/yaml": "^4.4|^5.0" + "php": "^7.2 || ^8.0", + "phpunit/phpunit": "^8.5 || ^9.2", + "symfony/yaml": "^4.4 || ^5.0" }, "require-dev": { - "friendsofphp/php-cs-fixer": "^2.16", - "php-coveralls/php-coveralls": "^2.2" + "friendsofphp/php-cs-fixer": "^2.18 || ^3.0", + "php-coveralls/php-coveralls": "^2.4", + "phpstan/phpstan": "^0.12.98", + "squizlabs/php_codesniffer": "^3.6" }, + "default-branch": true, "type": "library", "autoload": { "psr-4": { "PHPUnit\\DbUnit\\": "src/" } }, - "notification-url": "https://packagist.org/downloads/", + "autoload-dev": { + "psr-4": { + "PHPUnit\\DbUnit\\Tests\\": "tests/" + }, + "files": [ + "tests/_files/DatabaseTestUtility.php" + ] + }, "license": [ "MIT" ], @@ -923,21 +931,20 @@ { "name": "Aleksandr Ivanov", "email": "misantron@gmail.com", - "role": "developer" + "role": "Developer" } ], "description": "DbUnit fork supporting PHPUnit 8/9", - "homepage": "https://github.com/misantron/dbunit/", "keywords": [ "database", - "dbUnit", + "dbunit", "testing" ], "support": { - "issues": "https://github.com/misantron/dbunit/issues", - "source": "https://github.com/misantron/dbunit/tree/master" + "source": "https://github.com/misantron/dbunit/tree/master", + "issues": "https://github.com/misantron/dbunit/issues" }, - "time": "2020-07-07T20:48:08+00:00" + "time": "2021-09-09T19:30:23+00:00" }, { "name": "myclabs/deep-copy", @@ -1324,33 +1331,33 @@ }, { "name": "phpspec/prophecy", - "version": "1.13.0", + "version": "1.14.0", "source": { "type": "git", "url": "https://github.com/phpspec/prophecy.git", - "reference": "be1996ed8adc35c3fd795488a653f4b518be70ea" + "reference": "d86dfc2e2a3cd366cee475e52c6bb3bbc371aa0e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpspec/prophecy/zipball/be1996ed8adc35c3fd795488a653f4b518be70ea", - "reference": "be1996ed8adc35c3fd795488a653f4b518be70ea", + "url": "https://api.github.com/repos/phpspec/prophecy/zipball/d86dfc2e2a3cd366cee475e52c6bb3bbc371aa0e", + "reference": "d86dfc2e2a3cd366cee475e52c6bb3bbc371aa0e", "shasum": "" }, "require": { "doctrine/instantiator": "^1.2", - "php": "^7.2 || ~8.0, <8.1", + "php": "^7.2 || ~8.0, <8.2", "phpdocumentor/reflection-docblock": "^5.2", "sebastian/comparator": "^3.0 || ^4.0", "sebastian/recursion-context": "^3.0 || ^4.0" }, "require-dev": { - "phpspec/phpspec": "^6.0", + "phpspec/phpspec": "^6.0 || ^7.0", "phpunit/phpunit": "^8.0 || ^9.0" }, "type": "library", "extra": { "branch-alias": { - "dev-master": "1.11.x-dev" + "dev-master": "1.x-dev" } }, "autoload": { @@ -1385,9 +1392,9 @@ ], "support": { "issues": "https://github.com/phpspec/prophecy/issues", - "source": "https://github.com/phpspec/prophecy/tree/1.13.0" + "source": "https://github.com/phpspec/prophecy/tree/1.14.0" }, - "time": "2021-03-17T13:42:18+00:00" + "time": "2021-09-10T09:02:12+00:00" }, { "name": "phpunit/php-code-coverage", @@ -1709,16 +1716,16 @@ }, { "name": "phpunit/phpunit", - "version": "9.5.7", + "version": "9.5.9", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "d0dc8b6999c937616df4fb046792004b33fd31c5" + "reference": "ea8c2dfb1065eb35a79b3681eee6e6fb0a6f273b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/d0dc8b6999c937616df4fb046792004b33fd31c5", - "reference": "d0dc8b6999c937616df4fb046792004b33fd31c5", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/ea8c2dfb1065eb35a79b3681eee6e6fb0a6f273b", + "reference": "ea8c2dfb1065eb35a79b3681eee6e6fb0a6f273b", "shasum": "" }, "require": { @@ -1730,7 +1737,7 @@ "ext-xml": "*", "ext-xmlwriter": "*", "myclabs/deep-copy": "^1.10.1", - "phar-io/manifest": "^2.0.1", + "phar-io/manifest": "^2.0.3", "phar-io/version": "^3.0.2", "php": ">=7.3", "phpspec/prophecy": "^1.12.1", @@ -1796,7 +1803,7 @@ ], "support": { "issues": "https://github.com/sebastianbergmann/phpunit/issues", - "source": "https://github.com/sebastianbergmann/phpunit/tree/9.5.7" + "source": "https://github.com/sebastianbergmann/phpunit/tree/9.5.9" }, "funding": [ { @@ -1808,7 +1815,7 @@ "type": "github" } ], - "time": "2021-07-19T06:14:47+00:00" + "time": "2021-08-31T06:47:40+00:00" }, { "name": "sebastian/cli-parser", @@ -3107,7 +3114,9 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "misantron/dbunit": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": [], diff --git a/docker/phpunit.sh b/docker/phpunit.sh index 414ef6f983..8a8b2b1fbb 100755 --- a/docker/phpunit.sh +++ b/docker/phpunit.sh @@ -25,7 +25,7 @@ docker-compose -f docker-compose-phpunit.yml build \ --build-arg GID=$(id -g) \ --build-arg USER=$(id -un) \ --build-arg GROUP=$(id -gn) \ - --build-arg PHP_VERSION=7.3 \ + --build-arg PHP_VERSION=8 \ --build-arg PG_VERSION=13 \ --build-arg PORT=$PORT # When the container is brought up, the database will start diff --git a/docker/warehouse/Dockerfile b/docker/warehouse/Dockerfile index 54aa6e2a4c..ee7b201644 100644 --- a/docker/warehouse/Dockerfile +++ b/docker/warehouse/Dockerfile @@ -1,8 +1,6 @@ -# This image contains Debian's Apache httpd in conjunction with PHP7.3 -# (as mod_php) and uses mpm_prefork by default. +# This image contains Debian's Apache httpd in conjunction with PHP8.0. # https://hub.docker.com/_/php -# Currently warehouse is not compatible with PHP 7.4 -FROM php:7.3-apache +FROM php:8.0-apache # Use PHP development configuration file RUN mv "$PHP_INI_DIR/php.ini-development" "$PHP_INI_DIR/php.ini" # Increase size of files which can be uploaded. diff --git a/modules/cache_builder/config/cache_builder.php b/modules/cache_builder/config/cache_builder.php index de820d3d1a..e6c2376d08 100644 --- a/modules/cache_builder/config/cache_builder.php +++ b/modules/cache_builder/config/cache_builder.php @@ -109,7 +109,7 @@ $config['termlists_terms']['join_needs_update'] = 'join needs_update_termlists_terms nu on nu.id=tlt.id and nu.deleted=false'; $config['termlists_terms']['key_field'] = 'tlt.id'; -//-------------------------------------------------------------------------------------------------------------------------- +//----------------------------------------------------------------------------- $config['taxa_taxon_lists']['get_missing_items_query'] = " select distinct on (ttl.id) ttl.id, tl.deleted or ttl.deleted or ttlpref.deleted or t.deleted @@ -313,7 +313,7 @@ $config['taxa_taxon_lists']['join_needs_update'] = 'join needs_update_taxa_taxon_lists nu on nu.id=ttl.id and nu.deleted=false'; $config['taxa_taxon_lists']['key_field'] = 'ttl.id'; -$config['taxa_taxon_lists']['extra_multi_record_updates'] = array( +$config['taxa_taxon_lists']['extra_multi_record_updates'] = [ 'setup' => " -- Find children of updated taxa to ensure they are also changed. WITH RECURSIVE q AS ( @@ -428,7 +428,7 @@ DROP TABLE descendants; DROP TABLE ttl_path; DROP TABLE master_list_paths;", -); +]; // -------------------------------------------------------------------------------------------------------------------------- @@ -880,10 +880,10 @@ group by id "; -$config['samples']['delete_query'] = array( +$config['samples']['delete_query'] = [ "delete from cache_samples_functional where id in (select id from needs_update_samples where deleted=true); delete from cache_samples_nonfunctional where id in (select id from needs_update_samples where deleted=true);", -); +]; $config['samples']['update']['functional'] = " UPDATE cache_samples_functional s_update @@ -892,7 +892,7 @@ input_form=COALESCE(sp.input_form, s.input_form), location_id= s.location_id, location_name=CASE WHEN s.privacy_precision IS NOT NULL THEN NULL ELSE COALESCE(l.name, s.location_name, lp.name, sp.location_name) END, - public_geom=reduce_precision(coalesce(s.geom, l.centroid_geom), false, s.privacy_precision), + public_geom=reduce_precision(coalesce(s.geom, l.centroid_geom), false, greatest(s.privacy_precision, (SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id))), date_start=s.date_start, date_end=s.date_end, date_type=s.date_type, @@ -909,7 +909,10 @@ else 'A' end, parent_sample_id=s.parent_id, - media_count=(SELECT COUNT(sm.*) FROM sample_media sm WHERE sm.sample_id=s.id AND sm.deleted=false) + media_count=(SELECT COUNT(sm.*) FROM sample_media sm WHERE sm.sample_id=s.id AND sm.deleted=false), + external_key=s.external_key, + sensitive=(SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id) IS NOT NULL, + private=s.privacy_precision IS NOT NULL FROM samples s #join_needs_update# LEFT JOIN samples sp ON sp.id=s.parent_id AND sp.deleted=false @@ -938,24 +941,59 @@ SET website_title=w.title, survey_title=su.title, group_title=g.title, - public_entered_sref=case when s.privacy_precision is not null then null else + public_entered_sref=case + when s.privacy_precision is not null OR (SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id) IS NOT NULL then + get_output_sref( + greatest( + round(sqrt(st_area(st_transform(s.geom, sref_system_to_srid(s.entered_sref_system)))))::integer, + (SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id), + s.privacy_precision, + -- work out best square size to reflect a lat long's true precision + case + when coalesce(v_sref_precision.int_value, v_sref_precision.float_value)>=501 then 10000 + when coalesce(v_sref_precision.int_value, v_sref_precision.float_value) between 51 and 500 then 1000 + when coalesce(v_sref_precision.int_value, v_sref_precision.float_value) between 6 and 50 then 100 + else 10 + end, + 10 -- default minimum square size + ), reduce_precision(coalesce(s.geom, l.centroid_geom), (SELECT bool_or(confidential) FROM occurrences WHERE sample_id=s.id), greatest((SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id), s.privacy_precision)) + ) + else case when s.entered_sref_system = '4326' and coalesce(s.entered_sref, l.centroid_sref) ~ '^-?[0-9]*\.[0-9]*,[ ]*-?[0-9]*\.[0-9]*' then - abs(round(((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[1])::numeric, 3))::varchar - || case when ((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[1])::float>0 then 'N' else 'S' end - || ', ' - || abs(round(((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[2])::numeric, 3))::varchar - || case when ((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[2])::float>0 then 'E' else 'W' end + abs(round(((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[1])::numeric, 3))::varchar + || case when ((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[1])::float>0 then 'N' else 'S' end + || ', ' + || abs(round(((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[2])::numeric, 3))::varchar + || case when ((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[2])::float>0 then 'E' else 'W' end when s.entered_sref_system = '4326' and coalesce(s.entered_sref, l.centroid_sref) ~ '^-?[0-9]*\.[0-9]*[NS](, |[, ])*-?[0-9]*\.[0-9]*[EW]' then - abs(round(((regexp_split_to_array(coalesce(s.entered_sref, l.centroid_sref), '([NS](, |[, ]))|[EW]'))[1])::numeric, 3))::varchar - || case when coalesce(s.entered_sref, l.centroid_sref) like '%N%' then 'N' else 'S' end - || ', ' - || abs(round(((regexp_split_to_array(coalesce(s.entered_sref, l.centroid_sref), '([NS](, |[, ]))|[EW]'))[2])::numeric, 3))::varchar - || case when coalesce(s.entered_sref, l.centroid_sref) like '%E%' then 'E' else 'W' end + abs(round(((regexp_split_to_array(coalesce(s.entered_sref, l.centroid_sref), '([NS](, |[, ]))|[EW]'))[1])::numeric, 3))::varchar + || case when coalesce(s.entered_sref, l.centroid_sref) like '%N%' then 'N' else 'S' end + || ', ' + || abs(round(((regexp_split_to_array(coalesce(s.entered_sref, l.centroid_sref), '([NS](, |[, ]))|[EW]'))[2])::numeric, 3))::varchar + || case when coalesce(s.entered_sref, l.centroid_sref) like '%E%' then 'E' else 'W' end else - coalesce(s.entered_sref, l.centroid_sref) + coalesce(s.entered_sref, l.centroid_sref) end end, + output_sref=get_output_sref( + greatest( + round(sqrt(st_area(st_transform(s.geom, sref_system_to_srid(s.entered_sref_system)))))::integer, + (SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id), + s.privacy_precision, + -- work out best square size to reflect a lat long's true precision + case + when coalesce(v_sref_precision.int_value, v_sref_precision.float_value)>=501 then 10000 + when coalesce(v_sref_precision.int_value, v_sref_precision.float_value) between 51 and 500 then 1000 + when coalesce(v_sref_precision.int_value, v_sref_precision.float_value) between 6 and 50 then 100 + else 10 + end, + 10 -- default minimum square size + ), reduce_precision(coalesce(s.geom, l.centroid_geom), (SELECT bool_or(confidential) FROM occurrences WHERE sample_id=s.id), greatest((SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id), s.privacy_precision)) + ), + output_sref_system=get_output_system( + reduce_precision(coalesce(s.geom, l.centroid_geom), (SELECT bool_or(confidential) FROM occurrences WHERE sample_id=s.id), greatest((SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id), s.privacy_precision)) + ), entered_sref_system=case when s.entered_sref_system is null then l.centroid_sref_system else s.entered_sref_system end, recorders = s.recorder_names, comment=s.comment, @@ -1000,7 +1038,8 @@ WHEN 'L'::bpchar THEN t_sample_method.term ELSE NULL::text END, t_sample_method_id.term), - attr_linked_location_id=v_linked_location_id.int_value + attr_linked_location_id=v_linked_location_id.int_value, + verifier=pv.surname || ', ' || pv.first_name FROM samples s #join_needs_update# LEFT JOIN samples sp ON sp.id=s.parent_id and sp.deleted=false @@ -1050,6 +1089,8 @@ JOIN sample_attributes a_linked_location_id on a_linked_location_id.id=v_linked_location_id.sample_attribute_id and a_linked_location_id.deleted=false and a_linked_location_id.system_function='linked_location_id' ) ON v_linked_location_id.sample_id=s.id and v_linked_location_id.deleted=false +LEFT JOIN users uv on uv.id=s.verified_by_id and uv.deleted=false +LEFT JOIN people pv on pv.id=uv.person_id and pv.deleted=false WHERE s.id=cache_samples_nonfunctional.id "; @@ -1062,23 +1103,15 @@ WHERE s.id=cache_samples_nonfunctional.id "; -$config['samples']['update']['nonfunctional_sensitive'] = " -UPDATE cache_samples_nonfunctional -SET public_entered_sref=null -FROM samples s -#join_needs_update# -JOIN occurrences o ON o.sample_id=s.id AND o.deleted=false AND o.sensitivity_precision IS NOT NULL -WHERE s.id=cache_samples_nonfunctional.id -"; - $config['samples']['insert']['functional'] = " INSERT INTO cache_samples_functional( id, website_id, survey_id, input_form, location_id, location_name, public_geom, date_start, date_end, date_type, created_on, updated_on, verified_on, created_by_id, - group_id, record_status, training, query, parent_sample_id, media_count) + group_id, record_status, training, query, parent_sample_id, media_count, external_key, + sensitive, private) SELECT distinct on (s.id) s.id, su.website_id, s.survey_id, COALESCE(sp.input_form, s.input_form), s.location_id, CASE WHEN s.privacy_precision IS NOT NULL THEN NULL ELSE COALESCE(l.name, s.location_name, lp.name, sp.location_name) END, - reduce_precision(coalesce(s.geom, l.centroid_geom), false, s.privacy_precision), + reduce_precision(coalesce(s.geom, l.centroid_geom), false, greatest(s.privacy_precision, (SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id))), s.date_start, s.date_end, s.date_type, s.created_on, s.updated_on, s.verified_on, s.created_by_id, coalesce(s.group_id, sp.group_id), s.record_status, s.training, case @@ -1087,7 +1120,10 @@ else 'A' end, s.parent_id, - (SELECT COUNT(sm.*) FROM sample_media sm WHERE sm.sample_id=s.id AND sm.deleted=false) + (SELECT COUNT(sm.*) FROM sample_media sm WHERE sm.sample_id=s.id AND sm.deleted=false), + s.external_key, + (SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id) IS NOT NULL, + s.privacy_precision IS NOT NULL FROM samples s #join_needs_update# LEFT JOIN cache_samples_functional cs on cs.id=s.id @@ -1115,28 +1151,70 @@ $config['samples']['insert']['nonfunctional'] = " INSERT INTO cache_samples_nonfunctional( id, website_title, survey_title, group_title, public_entered_sref, - entered_sref_system, recorders, comment, privacy_precision, licence_code) + entered_sref_system, recorders, comment, privacy_precision, licence_code, + attr_sref_precision, output_sref, output_sref_system, verifier) SELECT distinct on (s.id) s.id, w.title, su.title, g.title, - case when s.privacy_precision is not null then null else + case + when s.privacy_precision is not null OR (SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id) IS NOT NULL then + get_output_sref( + greatest( + round(sqrt(st_area(st_transform(s.geom, sref_system_to_srid(s.entered_sref_system)))))::integer, + (SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id), + s.privacy_precision, + -- work out best square size to reflect a lat long's true precision + case + when coalesce(v_sref_precision.int_value, v_sref_precision.float_value)>=501 then 10000 + when coalesce(v_sref_precision.int_value, v_sref_precision.float_value) between 51 and 500 then 1000 + when coalesce(v_sref_precision.int_value, v_sref_precision.float_value) between 6 and 50 then 100 + else 10 + end, + 10 -- default minimum square size + ), reduce_precision(coalesce(s.geom, l.centroid_geom), (SELECT bool_or(confidential) FROM occurrences WHERE sample_id=s.id), greatest((SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id), s.privacy_precision)) + ) + else case when s.entered_sref_system = '4326' and coalesce(s.entered_sref, l.centroid_sref) ~ '^-?[0-9]*\.[0-9]*,[ ]*-?[0-9]*\.[0-9]*' then - abs(round(((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[1])::numeric, 3))::varchar - || case when ((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[1])::float>0 then 'N' else 'S' end - || ', ' - || abs(round(((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[2])::numeric, 3))::varchar - || case when ((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[2])::float>0 then 'E' else 'W' end + abs(round(((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[1])::numeric, 3))::varchar + || case when ((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[1])::float>0 then 'N' else 'S' end + || ', ' + || abs(round(((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[2])::numeric, 3))::varchar + || case when ((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[2])::float>0 then 'E' else 'W' end when s.entered_sref_system = '4326' and coalesce(s.entered_sref, l.centroid_sref) ~ '^-?[0-9]*\.[0-9]*[NS](, |[, ])*-?[0-9]*\.[0-9]*[EW]' then - abs(round(((regexp_split_to_array(coalesce(s.entered_sref, l.centroid_sref), '([NS](, |[, ]))|[EW]'))[1])::numeric, 3))::varchar - || case when coalesce(s.entered_sref, l.centroid_sref) like '%N%' then 'N' else 'S' end - || ', ' - || abs(round(((regexp_split_to_array(coalesce(s.entered_sref, l.centroid_sref), '([NS](, |[, ]))|[EW]'))[2])::numeric, 3))::varchar - || case when coalesce(s.entered_sref, l.centroid_sref) like '%E%' then 'E' else 'W' end + abs(round(((regexp_split_to_array(coalesce(s.entered_sref, l.centroid_sref), '([NS](, |[, ]))|[EW]'))[1])::numeric, 3))::varchar + || case when coalesce(s.entered_sref, l.centroid_sref) like '%N%' then 'N' else 'S' end + || ', ' + || abs(round(((regexp_split_to_array(coalesce(s.entered_sref, l.centroid_sref), '([NS](, |[, ]))|[EW]'))[2])::numeric, 3))::varchar + || case when coalesce(s.entered_sref, l.centroid_sref) like '%E%' then 'E' else 'W' end else - coalesce(s.entered_sref, l.centroid_sref) + coalesce(s.entered_sref, l.centroid_sref) end end, case when s.entered_sref_system is null then l.centroid_sref_system else s.entered_sref_system end, - s.recorder_names, s.comment, s.privacy_precision, li.code + s.recorder_names, s.comment, s.privacy_precision, li.code, + CASE a_sref_precision.data_type + WHEN 'I'::bpchar THEN v_sref_precision.int_value::double precision + WHEN 'F'::bpchar THEN v_sref_precision.float_value + ELSE NULL::double precision + END, + get_output_sref( + greatest( + round(sqrt(st_area(st_transform(s.geom, sref_system_to_srid(s.entered_sref_system)))))::integer, + (SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id), + s.privacy_precision, + -- work out best square size to reflect a lat long's true precision + case + when coalesce(v_sref_precision.int_value, v_sref_precision.float_value)>=501 then 10000 + when coalesce(v_sref_precision.int_value, v_sref_precision.float_value) between 51 and 500 then 1000 + when coalesce(v_sref_precision.int_value, v_sref_precision.float_value) between 6 and 50 then 100 + else 10 + end, + 10 -- default minimum square size + ), reduce_precision(coalesce(s.geom, l.centroid_geom), (SELECT bool_or(confidential) FROM occurrences WHERE sample_id=s.id), greatest((SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id), s.privacy_precision)) + ), + get_output_system( + reduce_precision(coalesce(s.geom, l.centroid_geom),(SELECT bool_or(confidential) FROM occurrences WHERE sample_id=s.id), greatest((SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id), s.privacy_precision)) + ), + pv.surname || ', ' || pv.first_name FROM samples s #join_needs_update# LEFT JOIN samples sp ON sp.id=s.parent_id and sp.deleted=false @@ -1145,7 +1223,13 @@ JOIN websites w on w.id=su.website_id and w.deleted=false LEFT JOIN groups g on g.id=coalesce(s.group_id, sp.group_id) and g.deleted=false LEFT JOIN locations l on l.id=s.location_id and l.deleted=false +LEFT JOIN (sample_attribute_values v_sref_precision + JOIN sample_attributes a_sref_precision on a_sref_precision.id=v_sref_precision.sample_attribute_id and a_sref_precision.deleted=false and a_sref_precision.system_function='sref_precision' + LEFT JOIN cache_termlists_terms t_sref_precision on a_sref_precision.data_type='L' and t_sref_precision.id=v_sref_precision.int_value +) on v_sref_precision.sample_id=s.id and v_sref_precision.deleted=false LEFT JOIN licences li on li.id=s.licence_id and li.deleted=false +LEFT JOIN users uv on uv.id=s.verified_by_id and uv.deleted=false +LEFT JOIN people pv on pv.id=uv.person_id and pv.deleted=false WHERE s.deleted=false AND cs.id IS NULL"; @@ -1182,11 +1266,6 @@ WHEN 'L'::bpchar THEN t_biotope.term ELSE NULL::text END, - attr_sref_precision=CASE a_sref_precision.data_type - WHEN 'I'::bpchar THEN v_sref_precision.int_value::double precision - WHEN 'F'::bpchar THEN v_sref_precision.float_value - ELSE NULL::double precision - END, attr_sample_method=COALESCE(t_sample_method_id.term, CASE a_sample_method.data_type WHEN 'T'::bpchar THEN v_sample_method.text_value WHEN 'L'::bpchar THEN t_sample_method.term @@ -1223,10 +1302,6 @@ JOIN sample_attributes a_biotope on a_biotope.id=v_biotope.sample_attribute_id and a_biotope.deleted=false and a_biotope.system_function='biotope' LEFT JOIN cache_termlists_terms t_biotope on a_biotope.data_type='L' and t_biotope.id=v_biotope.int_value ) on v_biotope.sample_id=s.id and v_biotope.deleted=false -LEFT JOIN (sample_attribute_values v_sref_precision - JOIN sample_attributes a_sref_precision on a_sref_precision.id=v_sref_precision.sample_attribute_id and a_sref_precision.deleted=false and a_sref_precision.system_function='sref_precision' - LEFT JOIN cache_termlists_terms t_sref_precision on a_sref_precision.data_type='L' and t_sref_precision.id=v_sref_precision.int_value -) on v_sref_precision.sample_id=s.id and v_sref_precision.deleted=false LEFT JOIN (sample_attribute_values v_sample_method JOIN sample_attributes a_sample_method on a_sample_method.id=v_sample_method.sample_attribute_id and a_sample_method.deleted=false and a_sample_method.system_function='sample_method' LEFT JOIN cache_termlists_terms t_sample_method on a_sample_method.data_type='L' and t_sample_method.id=v_sample_method.int_value @@ -1247,15 +1322,6 @@ WHERE s.id=cache_samples_nonfunctional.id "; -$config['samples']['insert']['nonfunctional_sensitive'] = " -UPDATE cache_samples_nonfunctional -SET public_entered_sref=null -FROM samples s -#join_needs_update# -JOIN occurrences o ON o.sample_id=s.id AND o.deleted=false AND o.sensitivity_precision IS NOT NULL -WHERE s.id=cache_samples_nonfunctional.id -"; - $config['samples']['join_needs_update'] = 'join needs_update_samples nu on nu.id=s.id and nu.deleted=false'; $config['samples']['key_field'] = 's.id'; @@ -1263,7 +1329,7 @@ // Additional update statements to pick up the recorder name from various possible custom attribute places. Faster than // loads of left joins. These should be in priority order - i.e. ones where we have recorded the inputter rather than // specifically the recorder should come after ones where we have recorded the recorder specifically. -$config['samples']['extra_multi_record_updates'] = array( +$config['samples']['extra_multi_record_updates'] = [ // s.recorder_names is filled in as a starting point. The rest only proceed if this is null. // full recorder name // or surname, firstname. @@ -1336,11 +1402,11 @@ from needs_update_samples nu, users u join cache_samples_functional csf on csf.created_by_id=u.id where cs.recorders is null and nu.id=cs.id - and cs.id=csf.id and u.id<>1;' -); + and cs.id=csf.id and u.id<>1;', +]; // Final statements to pick up after an insert of a single record. -$config['samples']['extra_single_record_updates'] = array( +$config['samples']['extra_single_record_updates'] = [ // Sample recorder names // Or, full recorder name // Or, surname, firstname. @@ -1415,8 +1481,8 @@ from users u join cache_samples_functional csf on csf.created_by_id=u.id where cs.recorders is null and cs.id in (#ids#) - and cs.id=csf.id and u.id<>1;' -); + and cs.id=csf.id and u.id<>1;', +]; // --------------------------------------------------------------------------------------------------------------------- @@ -1465,10 +1531,10 @@ ) as sub group by id"; -$config['occurrences']['delete_query'] = array( +$config['occurrences']['delete_query'] = [ "delete from cache_occurrences_functional where id in (select id from needs_update_occurrences where deleted=true); delete from cache_occurrences_nonfunctional where id in (select id from needs_update_occurrences where deleted=true);" -); +]; $config['occurrences']['update']['functional'] = " UPDATE cache_occurrences_functional @@ -1721,16 +1787,6 @@ AND o.deleted=false "; -$config['occurrences']['update']['nonfunctional_sensitive'] = " -UPDATE cache_samples_nonfunctional cs -SET public_entered_sref=null -FROM occurrences o -#join_needs_update# -WHERE o.sample_id=cs.id -AND o.deleted=false -AND o.sensitivity_precision IS NOT NULL -"; - $config['occurrences']['insert']['functional'] = "INSERT INTO cache_occurrences_functional( id, sample_id, website_id, survey_id, input_form, location_id, location_name, public_geom, @@ -1975,15 +2031,5 @@ AND o.deleted=false "; -$config['occurrences']['insert']['nonfunctional_sensitive'] = " -UPDATE cache_samples_nonfunctional cs -SET public_entered_sref=null -FROM occurrences o -#join_needs_update# -WHERE o.sample_id=cs.id -AND o.deleted=false -AND o.sensitivity_precision IS NOT NULL -"; - $config['occurrences']['join_needs_update'] = 'join needs_update_occurrences nu on nu.id=o.id and nu.deleted=false'; $config['occurrences']['key_field'] = 'o.id'; diff --git a/modules/cache_builder/db/version_6_3_0/202109091456_sample_fields.sql b/modules/cache_builder/db/version_6_3_0/202109091456_sample_fields.sql new file mode 100644 index 0000000000..0c888f04e1 --- /dev/null +++ b/modules/cache_builder/db/version_6_3_0/202109091456_sample_fields.sql @@ -0,0 +1,27 @@ +ALTER TABLE cache_samples_functional + ADD COLUMN external_key character varying, + ADD COLUMN sensitive boolean, + ADD COLUMN private boolean; + +ALTER TABLE cache_samples_nonfunctional + ADD COLUMN output_sref character varying, + ADD COLUMN output_sref_system character varying, + ADD COLUMN verifier character varying; + +COMMENT ON COLUMN cache_samples_functional.external_key IS + 'For samples imported from an external system, provides a field to store the external system''s primary key for the record allowing re-synchronisation.'; + +COMMENT ON COLUMN cache_samples_functional.sensitive IS + 'Set to true if the sample is blurred because one of the contained occurrences is sensitive.'; + +COMMENT ON COLUMN cache_samples_functional.private IS + 'Set to true if the sample has a privacy_precision value set, indicating the data are blurred for site privacy reasons (e.g. private gardens).'; + +COMMENT ON COLUMN cache_samples_nonfunctional.output_sref IS + 'A display spatial reference created for all samples, using the most appropriate local grid system where possible.'; + +COMMENT ON COLUMN cache_samples_nonfunctional.output_sref_system IS + 'Spatial reference system used for the output_sref field.'; + +COMMENT ON COLUMN cache_samples_nonfunctional.verifier IS + 'Name of the person who verified the sample, if any.'; \ No newline at end of file diff --git a/modules/cache_builder/db/version_6_3_0/202109101509_update_samples.sql b/modules/cache_builder/db/version_6_3_0/202109101509_update_samples.sql new file mode 100644 index 0000000000..53b065e435 --- /dev/null +++ b/modules/cache_builder/db/version_6_3_0/202109101509_update_samples.sql @@ -0,0 +1,75 @@ +-- #slow script# + +UPDATE cache_samples_functional u +SET external_key=u.external_key, + public_geom=reduce_precision(coalesce(s.geom, l.centroid_geom), false, greatest(s.privacy_precision, (SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id))), + sensitive=(SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id) IS NOT NULL, + private=s.privacy_precision IS NOT NULL +FROM samples s +LEFT JOIN locations l ON l.id=s.location_id AND l.deleted=false +WHERE s.id=u.id; + +UPDATE cache_samples_nonfunctional u + SET public_entered_sref=case + when s.privacy_precision is not null OR (SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id) IS NOT NULL then + get_output_sref( + greatest( + round(sqrt(st_area(st_transform(s.geom, sref_system_to_srid(s.entered_sref_system)))))::integer, + (SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id), + s.privacy_precision, + -- work out best square size to reflect a lat long's true precision + case + when coalesce(v_sref_precision.int_value, v_sref_precision.float_value)>=501 then 10000 + when coalesce(v_sref_precision.int_value, v_sref_precision.float_value) between 51 and 500 then 1000 + when coalesce(v_sref_precision.int_value, v_sref_precision.float_value) between 6 and 50 then 100 + else 10 + end, + 10 -- default minimum square size + ), reduce_precision(coalesce(s.geom, l.centroid_geom), (SELECT bool_or(confidential) FROM occurrences WHERE sample_id=s.id), greatest((SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id), s.privacy_precision)) + ) + else + case + when s.entered_sref_system = '4326' and coalesce(s.entered_sref, l.centroid_sref) ~ '^-?[0-9]*\.[0-9]*,[ ]*-?[0-9]*\.[0-9]*' then + abs(round(((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[1])::numeric, 3))::varchar + || case when ((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[1])::float>0 then 'N' else 'S' end + || ', ' + || abs(round(((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[2])::numeric, 3))::varchar + || case when ((string_to_array(coalesce(s.entered_sref, l.centroid_sref), ','))[2])::float>0 then 'E' else 'W' end + when s.entered_sref_system = '4326' and coalesce(s.entered_sref, l.centroid_sref) ~ '^-?[0-9]*\.[0-9]*[NS](, |[, ])*-?[0-9]*\.[0-9]*[EW]' then + abs(round(((regexp_split_to_array(coalesce(s.entered_sref, l.centroid_sref), '([NS](, |[, ]))|[EW]'))[1])::numeric, 3))::varchar + || case when coalesce(s.entered_sref, l.centroid_sref) like '%N%' then 'N' else 'S' end + || ', ' + || abs(round(((regexp_split_to_array(coalesce(s.entered_sref, l.centroid_sref), '([NS](, |[, ]))|[EW]'))[2])::numeric, 3))::varchar + || case when coalesce(s.entered_sref, l.centroid_sref) like '%E%' then 'E' else 'W' end + else + coalesce(s.entered_sref, l.centroid_sref) + end + end, + output_sref=get_output_sref( + greatest( + round(sqrt(st_area(st_transform(s.geom, sref_system_to_srid(s.entered_sref_system)))))::integer, + (SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id), + s.privacy_precision, + -- work out best square size to reflect a lat long's true precision + case + when coalesce(v_sref_precision.int_value, v_sref_precision.float_value)>=501 then 10000 + when coalesce(v_sref_precision.int_value, v_sref_precision.float_value) between 51 and 500 then 1000 + when coalesce(v_sref_precision.int_value, v_sref_precision.float_value) between 6 and 50 then 100 + else 10 + end, + 10 -- default minimum square size + ), reduce_precision(coalesce(s.geom, l.centroid_geom), (SELECT bool_or(confidential) FROM occurrences WHERE sample_id=s.id), greatest((SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id), s.privacy_precision)) + ), + output_sref_system=get_output_system( + reduce_precision(coalesce(s.geom, l.centroid_geom), (SELECT bool_or(confidential) FROM occurrences WHERE sample_id=s.id), greatest((SELECT max(sensitivity_precision) FROM occurrences WHERE sample_id=s.id), s.privacy_precision)) + ), + verifier=pv.surname || ', ' || pv.first_name +FROM samples s +LEFT JOIN locations l ON l.id=s.location_id AND l.deleted=false +LEFT JOIN (sample_attribute_values v_sref_precision + JOIN sample_attributes a_sref_precision on a_sref_precision.id=v_sref_precision.sample_attribute_id and a_sref_precision.deleted=false and a_sref_precision.system_function='sref_precision' + LEFT JOIN cache_termlists_terms t_sref_precision on a_sref_precision.data_type='L' and t_sref_precision.id=v_sref_precision.int_value +) on v_sref_precision.sample_id=s.id and v_sref_precision.deleted=false +LEFT JOIN users uv on uv.id=s.verified_by_id and uv.deleted=false +LEFT JOIN people pv on pv.id=uv.person_id and pv.deleted=false +WHERE s.id=u.id; diff --git a/modules/cache_builder/db/version_6_3_0/202109171515_sensitive_grid_squares.sql b/modules/cache_builder/db/version_6_3_0/202109171515_sensitive_grid_squares.sql new file mode 100644 index 0000000000..f543e2e4a9 --- /dev/null +++ b/modules/cache_builder/db/version_6_3_0/202109171515_sensitive_grid_squares.sql @@ -0,0 +1,7 @@ +-- #slow script# + +-- Trigger re-queuing of sensitive records for Elasticsearch due to previously +-- incorrect map grid square field names. +UPDATE cache_occurrences_functional +SET website_id=website_id +WHERE sensitive=true; \ No newline at end of file diff --git a/modules/cache_builder/helpers/cache_builder.php b/modules/cache_builder/helpers/cache_builder.php index 69f7b2f376..acf4929fc1 100644 --- a/modules/cache_builder/helpers/cache_builder.php +++ b/modules/cache_builder/helpers/cache_builder.php @@ -1,355 +1,404 @@ -$table"; - $count = cache_builder::getChangeList($db, $table, $queries, $last_run_date); - if ($count > 0) { - echo << - - # records affected - - - Total$count - - -HTML; - cache_builder::makeChanges($db, $table); - echo << - -HTML; - } - else { - echo "

No changes

"; - } - $db->query("drop table needs_update_$table"); - } - catch (Exception $e) { - $db->query("drop table needs_update_$table"); - throw $e; - } - } - - /** - * Apply required database changes to the cache tables. - * - * When the needs_update_* table already populated, apply the actual cache - * update changes to the cached entity. - * - * @param object $db - * Database connection. - * @param string $table - * Entity name to update (e.g. sample, occurrence, taxa_taxon_list). - */ - public static function makeChanges($db, $table) { - $queries = kohana::config("cache_builder.$table"); - cache_builder::do_delete($db, $table, $queries); - // preprocess some of the tags in the queries - if (is_array($queries['update'])) - foreach($queries['update'] as $key=>&$sql) - $sql = str_replace('#join_needs_update#', $queries['join_needs_update'], $sql); - else - $queries['update'] = str_replace('#join_needs_update#', $queries['join_needs_update'], $queries['update']); - cache_builder::run_statement($db, $table, $queries['update'], 'update'); - // preprocess some of the tags in the queries - if (is_array($queries['insert'])) - foreach($queries['insert'] as $key=>&$sql) - $sql = str_replace('#join_needs_update#', $queries['join_needs_update'] . ' and (nu.deleted=false or nu.deleted is null)', $sql); - else - $queries['insert'] = str_replace('#join_needs_update#', $queries['join_needs_update'] . ' and (nu.deleted=false or nu.deleted is null)', $queries['insert']); - cache_builder::run_statement($db, $table, $queries['insert'], 'insert'); - if (isset($queries['extra_multi_record_updates'])) - cache_builder::run_statement($db, $table, $queries['extra_multi_record_updates'], 'final update'); - if (!variable::get("populated-$table")) { - $cacheQuery = $db->query("select count(*) from cache_$table")->result_array(false); - if (isset($queries['count'])) - $totalQuery = $db->query($queries['count'])->result_array(false); - else - $totalQuery = $db->query("select count(*) from $table where deleted='f'")->result_array(false); - $percent = round($cacheQuery[0]['count']*100/$totalQuery[0]['count']); - echo "

Initial population of $table progress $percent%.

"; - } - } - - /** - * Inserts a single record into the cache, e.g. could be used as soon as a record is submitted. - * - * @param object $db - * Database object. - * @param string $table - * Plural form of the table name. - * @param array $ids - * Record IDs to insert in the cache - */ - public static function insert($db, $table, array $ids) { - if (count($ids) > 0) { - $idlist = implode(',', $ids); - if (self::$delayCacheUpdates && in_array($table, ['occurrences', 'samples'])) { - kohana::log('debug', "Delayed inserts for $table ($idlist)"); - self::delayChangesViaWorkQueue($db, $table, $idlist); - } - else { - $master_list_id = warehouse::getMasterTaxonListId(); - $queries = kohana::config("cache_builder.$table"); - if (!isset($queries['key_field'])) - throw new exception('Cannot do a specific record insert into cache as the key_field configuration not defined in cache_builder configuration'); - if (!is_array($queries['insert'])) - $queries['insert'] = array($queries['insert']); - foreach ($queries['insert'] as $query) { - $insertSql = str_replace( - ['#join_needs_update#', '#master_list_id#'], - ['', $master_list_id], - $query - ); - $insertSql .= ' and ' . $queries['key_field'] . " in ($idlist)"; - $db->query($insertSql); - } - } - self::final_queries($db, $table, $ids); - } - } - - /** - * Updates a single record in the cache. - * - * E.g. could be used as soon as a record is edited. - * - * @param object $db - * Database object. - * @param string $table - * Plural form of the table name. - * @param array $ids - * Record IDs to insert in the cache. - */ - public static function update($db, $table, array $ids) { - if (count($ids) > 0) { - $idlist = implode(',', $ids); - if (self::$delayCacheUpdates && in_array($table, ['occurrences', 'samples'])) { - kohana::log('debug', "Delayed updates for $table ($idlist)"); - self::delayChangesViaWorkQueue($db, $table, $idlist); - } - else { - $master_list_id = warehouse::getMasterTaxonListId(); - $queries = kohana::config("cache_builder.$table"); - if (!isset($queries['key_field'])) - throw new exception('Cannot do a specific record update into cache as the key_field configuration not defined in cache_builder configuration'); - if (!is_array($queries['update'])) - $queries['update'] = array($queries['update']); - foreach ($queries['update'] as $query) { - $updateSql = str_replace( - ['#join_needs_update#', '#master_list_id#'], - ['', $master_list_id], - $query - ); - $updateSql .= ' and ' . $queries['key_field'] . " in ($idlist)"; - $db->query($updateSql); - } - self::final_queries($db, $table, $ids); - } - } - } - - /** - * Deletes a single record from the cache. - * - * E.g. could be used as soon as a record is deleted. - * - * @param object $db - * Database object. - * @param string $table - * Plural form of the table name. - * @param array $ids - * Record IDs to delete from the cache. - */ - public static function delete($db, $table, array $ids) { - if (self::$delayCacheUpdates && in_array($table, ['occurrences', 'samples'])) { - self::delayChangesViaWorkQueue($db, $table, implode(',', $ids)); - } - else { - foreach ($ids as $id) { - if ($table === 'occurrences' || $table === 'samples') { - $db->delete("cache_{$table}_functional", array('id' => $id)); - $db->delete("cache_{$table}_nonfunctional", array('id' => $id)); - if ($table === 'samples') { - // Slightly more complex delete query to ensure indexes used. - $sql = <<query($sql); - $db->query("delete from cache_occurrences_nonfunctional where id in (select id from occurrences where sample_id=$id)"); - } - } - else { - $db->delete("cache_$table", array('id' => $id)); - } - } - } - } - - /** - * During an import, add tasks to work queue rather than do immediate update. - * - * Allows performance improvement during import. - * - * @param object $db - * Database object. - * @param string $table - * Plural form of the table name. - * @param string $idCsv - * Record IDs to delete from the cache (comma separated string). - */ - private static function delayChangesViaWorkQueue($db, $table, $idCsv) { - $entity = inflector::singular($table); - $sql = <<query($sql); - } - - public static function final_queries($db, $table, $ids) { - $queries = kohana::config("cache_builder.$table"); - $doneCount = 0; - if (isset($queries['extra_single_record_updates'])) { - $idlist=implode(',', $ids); - if (is_array($queries['extra_single_record_updates'])) - foreach($queries['extra_single_record_updates'] as $key=>&$sql) { - $result=$db->query(str_replace('#ids#', $idlist, $sql)); - $doneCount += $result->count(); - if ($doneCount>=count($ids)) - break; // we've updated all. So can drop out. - } - else { - $db->query(str_replace('#ids#', $idlist, $queries['extra_single_record_updates'])); - } - } - } - - /** - * Build a temporary table with the list of IDs of records we need to update. - * The table has a deleted flag to indicate newly deleted records. - * @param objcet $db Database connection. - * @param string $table Name of the table being cached, e.g. occurrences. - * @param string $query A query which selects a list of IDs for all new, updated or - * deleted records (including looking for updates or deletions caused by related - * records). - * @param string $last_run_date Date/time of the last time the cache builder was - * run, used to filter records to only the recent changes. Supplied as a string - * suitable for injection into an SQL query. - */ - private static function getChangeList($db, $table, $queries, $last_run_date) { - $query = str_replace('#date#', $last_run_date, $queries['get_changed_items_query']); - $db->query("create temporary table needs_update_$table as $query"); - if (!variable::get("populated-$table")) { - // as well as the changed records, pick up max 5000 previous records, which is important for initial population. - // 5000 is an arbitrary number to compromise between performance and cache population. - // of the cache - $query = $queries['get_missing_items_query'] . ' limit 5000'; - $result = $db->query("insert into needs_update_$table $query"); - if ($result->count() === 0) { - // Flag that we don't need to do any more previously existing records as they are all done. - // Future cache updates can just pick up changes from now on. - variable::set("populated-$table", TRUE); - echo "

Initial population of $table completed

"; - } - } - $db->query("ALTER TABLE needs_update_$table ADD CONSTRAINT ix_nu_$table PRIMARY KEY (id)"); - $r = $db->query("select count(*) as count from needs_update_$table")->result_array(FALSE); - $row = $r[0]; - return $row['count']; - } - - /** - * Deletes all records from the cache table which are in the table of - * records to update and where the deleted flag is true. - * - * @param object $db - * Database connection. - * @param string $table - * Name of the table being cached. - * @param array $queries - * List of configured queries for this table, which might include non-default delete queries. - */ - private static function do_delete($db, $table, $queries) { - // set up a default delete query if none are specified - if (!isset($queries['delete_query'])) { - $queries['delete_query'] = array("delete from cache_$table where id in (select id from needs_update_$table where deleted=true)"); - } - $count = 0; - foreach ($queries['delete_query'] as $query) { - $count += $db->query($query)->count(); - } - if (variable::get("populated-$table")) { - echo " Delete(s)$count\n"; - } - } - - /** - * Runs an insert or update statemnet to update one of - * the cache tables. - * @param object $db Database connection. - * @param string $query Query used to perform the update or insert. Can be a string, or an - * associative array of SQL strings if multiple required to do the task. - * @param string $action Term describing the action, used for feedback only. - */ - private static function run_statement($db, $table, $query, $action) { - $master_list_id = warehouse::getMasterTaxonListId(); - if (is_array($query)) { - foreach ($query as $title => $sql) { - $sql = str_replace('#master_list_id#', $master_list_id, $sql); - $count = $db->query($sql)->count(); - if (variable::get("populated-$table")) - echo " $action(s) for $title$count\n"; - } - } else { - $sql = str_replace('#master_list_id#', $master_list_id, $query); - $count = $db->query($query)->count(); - if (variable::get("populated-$table")) { - echo " $action(s)$count\n"; - } - } - } +$table"; + $count = cache_builder::getChangeList($db, $table, $queries, $last_run_date); + if ($count > 0) { + echo << + + # records affected + + + Total$count + + +HTML; + cache_builder::makeChanges($db, $table); + echo << + +HTML; + } + else { + echo "

No changes

"; + } + $db->query("drop table needs_update_$table"); + } + catch (Exception $e) { + $db->query("drop table needs_update_$table"); + throw $e; + } + } + + /** + * Apply required database changes to the cache tables. + * + * When the needs_update_* table already populated, apply the actual cache + * update changes to the cached entity. + * + * @param object $db + * Database connection. + * @param string $table + * Entity name to update (e.g. sample, occurrence, taxa_taxon_list). + */ + public static function makeChanges($db, $table) { + $queries = kohana::config("cache_builder.$table"); + cache_builder::do_delete($db, $table, $queries); + // Preprocess some of the tags in the queries. + if (is_array($queries['update'])) { + foreach ($queries['update'] as &$sql) { + $sql = str_replace('#join_needs_update#', $queries['join_needs_update'], $sql); + } + } + else { + $queries['update'] = str_replace('#join_needs_update#', $queries['join_needs_update'], $queries['update']); + } + cache_builder::run_statement($db, $table, $queries['update'], 'update'); + // Preprocess some of the tags in the queries. + if (is_array($queries['insert'])) { + foreach ($queries['insert'] as &$sql) { + $sql = str_replace('#join_needs_update#', $queries['join_needs_update'] . ' and (nu.deleted=false or nu.deleted is null)', $sql); + } + } + else { + $queries['insert'] = str_replace('#join_needs_update#', $queries['join_needs_update'] . ' and (nu.deleted=false or nu.deleted is null)', $queries['insert']); + } + cache_builder::run_statement($db, $table, $queries['insert'], 'insert'); + if (isset($queries['extra_multi_record_updates'])) { + cache_builder::run_statement($db, $table, $queries['extra_multi_record_updates'], 'final update'); + } + if (!variable::get("populated-$table")) { + $cacheQuery = $db->query("select count(*) from cache_$table")->result_array(FALSE); + if (isset($queries['count'])) { + $totalQuery = $db->query($queries['count'])->result_array(FALSE); + } + else { + $totalQuery = $db->query("select count(*) from $table where deleted='f'")->result_array(FALSE); + } + $percent = round($cacheQuery[0]['count'] * 100 / $totalQuery[0]['count']); + echo "

Initial population of $table progress $percent%.

"; + } + } + + /** + * Inserts a single record into the cache, e.g. could be used as soon as a record is submitted. + * + * @param object $db + * Database object. + * @param string $table + * Plural form of the table name. + * @param array $ids + * Record IDs to insert in the cache. + */ + public static function insert($db, $table, array $ids) { + if (count($ids) > 0) { + $idlist = implode(',', $ids); + if (self::$delayCacheUpdates && in_array($table, ['occurrences', 'samples'])) { + kohana::log('debug', "Delayed inserts for $table ($idlist)"); + self::delayChangesViaWorkQueue($db, $table, $idlist); + } + else { + $master_list_id = warehouse::getMasterTaxonListId(); + $queries = kohana::config("cache_builder.$table"); + if (!isset($queries['key_field'])) + throw new exception('Cannot do a specific record insert into cache as the key_field configuration not defined in cache_builder configuration'); + if (!is_array($queries['insert'])) + $queries['insert'] = array($queries['insert']); + foreach ($queries['insert'] as $query) { + $insertSql = str_replace( + ['#join_needs_update#', '#master_list_id#'], + ['', $master_list_id], + $query + ); + $insertSql .= ' and ' . $queries['key_field'] . " in ($idlist)"; + $db->query($insertSql); + } + } + self::final_queries($db, $table, $ids); + } + } + + /** + * Updates a single record in the cache. + * + * E.g. could be used as soon as a record is edited. + * + * @param object $db + * Database object. + * @param string $table + * Plural form of the table name. + * @param array $ids + * Record IDs to insert in the cache. + */ + public static function update($db, $table, array $ids) { + if (count($ids) > 0) { + $idlist = implode(',', $ids); + if (self::$delayCacheUpdates && in_array($table, ['occurrences', 'samples'])) { + kohana::log('debug', "Delayed updates for $table ($idlist)"); + self::delayChangesViaWorkQueue($db, $table, $idlist); + } + else { + $master_list_id = warehouse::getMasterTaxonListId(); + $queries = kohana::config("cache_builder.$table"); + if (!isset($queries['key_field'])) + throw new exception('Cannot do a specific record update into cache as the key_field configuration not defined in cache_builder configuration'); + if (!is_array($queries['update'])) + $queries['update'] = array($queries['update']); + foreach ($queries['update'] as $query) { + $updateSql = str_replace( + ['#join_needs_update#', '#master_list_id#'], + ['', $master_list_id], + $query + ); + $updateSql .= ' and ' . $queries['key_field'] . " in ($idlist)"; + $db->query($updateSql); + } + self::final_queries($db, $table, $ids); + } + } + } + + /** + * Deletes a single record from the cache. + * + * E.g. could be used as soon as a record is deleted. + * + * @param object $db + * Database object. + * @param string $table + * Plural form of the table name. + * @param array $ids + * Record IDs to delete from the cache. + */ + public static function delete($db, $table, array $ids) { + if (self::$delayCacheUpdates && in_array($table, ['occurrences', 'samples'])) { + self::delayChangesViaWorkQueue($db, $table, implode(',', $ids)); + } + else { + foreach ($ids as $id) { + if ($table === 'occurrences' || $table === 'samples') { + $db->delete("cache_{$table}_functional", array('id' => $id)); + $db->delete("cache_{$table}_nonfunctional", array('id' => $id)); + if ($table === 'samples') { + // Slightly more complex delete query to ensure indexes used. + $sql = <<query($sql); + $db->query("delete from cache_occurrences_nonfunctional where id in (select id from occurrences where sample_id=$id)"); + } + } + else { + $db->delete("cache_$table", array('id' => $id)); + } + } + } + } + + /** + * If submitting occurrence changes without a sample, update sample tracking. + * + * This is so that any sample data feeds receive an updated copy of the + * sample, as the occurrence statistics will have changed. + * + * @param object $db + * Database object. + * @param array $occurrenceIds + * List of occurrences affected by a submission. + */ + public static function updateSampleTrackingForOccurrences($db, array $ids) { + $idList = implode(',', $ids); + $sql = <<query($sql); + } + + /** + * During an import, add tasks to work queue rather than do immediate update. + * + * Allows performance improvement during import. + * + * @param object $db + * Database object. + * @param string $table + * Plural form of the table name. + * @param string $idCsv + * Record IDs to delete from the cache (comma separated string). + */ + private static function delayChangesViaWorkQueue($db, $table, $idCsv) { + $entity = inflector::singular($table); + $sql = <<query($sql); + } + + public static function final_queries($db, $table, $ids) { + $queries = kohana::config("cache_builder.$table"); + $doneCount = 0; + if (isset($queries['extra_single_record_updates'])) { + $idlist=implode(',', $ids); + if (is_array($queries['extra_single_record_updates'])) + foreach($queries['extra_single_record_updates'] as $key=>&$sql) { + $result=$db->query(str_replace('#ids#', $idlist, $sql)); + $doneCount += $result->count(); + if ($doneCount>=count($ids)) + break; // we've updated all. So can drop out. + } + else { + $db->query(str_replace('#ids#', $idlist, $queries['extra_single_record_updates'])); + } + } + } + + /** + * Build a temporary table with the list of IDs of records we need to update. + * + * The table has a deleted flag to indicate newly deleted records. + * + * @param object $db + * Database connection. + * @param string $table + * Name of the table being cached, e.g. occurrences. + * @param string $queries + * List of configured queries for this table. + * @param string $last_run_date + * Date/time of the last time the cache builder was run, used to filter + * records to only the recent changes. Supplied as a string suitable for + * injection into an SQL query. + */ + private static function getChangeList($db, $table, $queries, $last_run_date) { + $query = str_replace('#date#', $last_run_date, $queries['get_changed_items_query']); + $db->query("create temporary table needs_update_$table as $query"); + if (!variable::get("populated-$table")) { + // As well as the changed records, pick up max 5000 previous records, + // which is important for initial population. 5000 is an arbitrary number + // to compromise between performance and cache population. + $query = $queries['get_missing_items_query'] . ' limit 5000'; + $result = $db->query("insert into needs_update_$table $query"); + if ($result->count() === 0) { + // Flag that we don't need to do any more previously existing records + // as they are all done. + // Future cache updates can just pick up changes from now on. + variable::set("populated-$table", TRUE); + echo "

Initial population of $table completed

"; + } + } + $db->query("ALTER TABLE needs_update_$table ADD CONSTRAINT ix_nu_$table PRIMARY KEY (id)"); + $r = $db->query("select count(*) as count from needs_update_$table")->result_array(FALSE); + $row = $r[0]; + return $row['count']; + } + + /** + * Deletes all records from the cache table which are in the table of + * records to update and where the deleted flag is true. + * + * @param object $db + * Database connection. + * @param string $table + * Name of the table being cached. + * @param array $queries + * List of configured queries for this table, which might include non-default delete queries. + */ + private static function do_delete($db, $table, $queries) { + // Set up a default delete query if none are specified. + if (!isset($queries['delete_query'])) { + $queries['delete_query'] = ["delete from cache_$table where id in (select id from needs_update_$table where deleted=true)"]; + } + $count = 0; + foreach ($queries['delete_query'] as $query) { + $count += $db->query($query)->count(); + } + if (variable::get("populated-$table")) { + echo " Delete(s)$count\n"; + } + } + + /** + * Runs an insert or update statemnet to update one of the cache tables. + * + * @param object $db + * Database connection. + * @param string $query + * Query used to perform the update or insert. Can be a string, or an + * associative array of SQL strings if multiple required to do the task. + * @param string $action + * Term describing the action, used for feedback only. + */ + private static function run_statement($db, $table, $query, $action) { + $master_list_id = warehouse::getMasterTaxonListId(); + if (is_array($query)) { + foreach ($query as $title => $sql) { + $sql = str_replace('#master_list_id#', $master_list_id, $sql); + $count = $db->query($sql)->count(); + if (variable::get("populated-$table")) { + echo " $action(s) for $title$count\n"; + } + } + } + else { + $sql = str_replace('#master_list_id#', $master_list_id, $query); + $count = $db->query($query)->count(); + if (variable::get("populated-$table")) { + echo " $action(s)$count\n"; + } + } + } } \ No newline at end of file diff --git a/modules/data_cleaner/controllers/verification_rule.php b/modules/data_cleaner/controllers/verification_rule.php index f6ef3bda6a..def0c67d40 100644 --- a/modules/data_cleaner/controllers/verification_rule.php +++ b/modules/data_cleaner/controllers/verification_rule.php @@ -134,7 +134,7 @@ private function get_server_list() { $response = curl_exec($session); if (curl_errno($session)) { $this->session->set_flash('flash_info', 'The list of verification rule servers could not be retrieved from the internet. ' . - 'More information is avaailable in the server logs.'); + 'More information is available in the server logs.'); kohana::log('error', 'Error occurred when retrieving list of verification rule servers. ' . curl_error($session)); return array(); } diff --git a/modules/data_cleaner/tests/services/data_cleanerTest.php b/modules/data_cleaner/tests/services/data_cleanerTest.php index 3d684c6484..b8e8c166a6 100644 --- a/modules/data_cleaner/tests/services/data_cleanerTest.php +++ b/modules/data_cleaner/tests/services/data_cleanerTest.php @@ -26,9 +26,6 @@ * @subpackage Data Cleaner */ -use PHPUnit\DbUnit\DataSet\YamlDataSet as DbUDataSetYamlDataSet; -use PHPUnit\DbUnit\DataSet\CompositeDataSet as DbUDataSetCompositeDataSet; - require_once 'client_helpers/data_entry_helper.php'; /** @@ -36,83 +33,77 @@ * @backupGlobals disabled * @backupStaticAttributes disabled */ -class Controllers_Services_Data_Cleaner_Test extends Indicia_DatabaseTestCase { +class Controllers_Services_Data_Cleaner_Test extends SimpleDatabaseTestCase { protected $request; /** * @return PHPUnit_Extensions_Database_DataSet_IDataSet */ - public function getDataSet() - { - $ds1 = new DbUDataSetYamlDataSet('modules/phpUnit/config/core_fixture.yaml'); - - // Create a rule to test against - $ds2 = new Indicia_ArrayDataSet( - array( - 'verification_rules' => array( - array( - 'title' => 'Test PeriodWithinYear rule', - 'description' => 'Test rule for unit testing', - 'test_type' => 'PeriodWithinYear', - 'error_message' => 'PeriodWithinYear test failed', - 'source_url' => null, - 'source_filename' => null, - 'created_on' => '2016-07-22:16:00:00', - 'created_by_id' => 1, - 'updated_on' => '2016-07-22:16:00:00', - 'updated_by_id' => 1, - 'reverse_rule' => 'F', - ), - ), - 'verification_rule_metadata' => array( - array( - 'verification_rule_id' => '1', - 'key' => 'Tvk', - 'value' => 'TESTKEY', - 'created_on' => '2016-07-22:16:00:00', - 'created_by_id' => 1, - 'updated_on' => '2016-07-22:16:00:00', - 'updated_by_id' => 1, - ), - array( - 'verification_rule_id' => '1', - 'key' => 'StartDate', - 'value' => '0801', - 'created_on' => '2016-07-22:16:00:00', - 'created_by_id' => 1, - 'updated_on' => '2016-07-22:16:00:00', - 'updated_by_id' => 1, - ), - array( - 'verification_rule_id' => '1', - 'key' => 'EndDate', - 'value' => '0831', - 'created_on' => '2016-07-22:16:00:00', - 'created_by_id' => 1, - 'updated_on' => '2016-07-22:16:00:00', - 'updated_by_id' => 1, - ), - ), - 'cache_verification_rules_period_within_year' => array( - array( - 'verification_rule_id' => '1', - 'reverse_rule' => 'f', - 'taxa_taxon_list_external_key' => 'TESTKEY', - 'start_date' => '214', - 'end_date' => '244', - 'survey_id' => NULL, - 'stages' => NULL, - 'error_message' => 'PeriodWithinYear test failed', - ), - ), - ) - ); + public function getDataSet() { + require 'modules/phpUnit/config/core_fixture.php'; + + // Create a rule to test against. + $local_fixture = [ + 'verification_rules' => [ + [ + 'title' => 'Test PeriodWithinYear rule', + 'description' => 'Test rule for unit testing', + 'test_type' => 'PeriodWithinYear', + 'error_message' => 'PeriodWithinYear test failed', + 'source_url' => NULL, + 'source_filename' => NULL, + 'created_on' => '2016-07-22 16:00:00', + 'created_by_id' => 1, + 'updated_on' => '2016-07-22 16:00:00', + 'updated_by_id' => 1, + 'reverse_rule' => 'F', + ], + ], + 'verification_rule_metadata' => [ + [ + 'verification_rule_id' => '1', + 'key' => 'Tvk', + 'value' => 'TESTKEY', + 'created_on' => '2016-07-22 16:00:00', + 'created_by_id' => 1, + 'updated_on' => '2016-07-22 16:00:00', + 'updated_by_id' => 1, + ], + [ + 'verification_rule_id' => '1', + 'key' => 'StartDate', + 'value' => '0801', + 'created_on' => '2016-07-22 16:00:00', + 'created_by_id' => 1, + 'updated_on' => '2016-07-22 16:00:00', + 'updated_by_id' => 1, + ], + [ + 'verification_rule_id' => '1', + 'key' => 'EndDate', + 'value' => '0831', + 'created_on' => '2016-07-22 16:00:00', + 'created_by_id' => 1, + 'updated_on' => '2016-07-22 16:00:00', + 'updated_by_id' => 1, + ], + ], + 'cache_verification_rules_period_within_year' => [ + [ + 'verification_rule_id' => '1', + 'reverse_rule' => 'f', + 'taxa_taxon_list_external_key' => 'TESTKEY', + 'start_date' => '214', + 'end_date' => '244', + 'survey_id' => NULL, + 'stages' => NULL, + 'error_message' => 'PeriodWithinYear test failed', + ], + ], + ]; - $compositeDs = new DbUDataSetCompositeDataSet(); - $compositeDs->addDataSet($ds1); - $compositeDs->addDataSet($ds2); - return $compositeDs; + return array_merge($core_fixture, $local_fixture); } public function setUp(): void { @@ -123,7 +114,7 @@ public function setUp(): void { $token = $auth['auth_token']; $nonce = $auth['nonce']; $this->request = data_entry_helper::$base_url . - "index.php/services/data_cleaner/verify?auth_token=$token&nonce=$nonce"; + "index.php/services/data_cleaner/verify?auth_token=$token&nonce=$nonce"; $cache = Cache::instance(); $cache->delete('data-cleaner-rules'); @@ -134,7 +125,7 @@ public function setUp(): void { * incomplete or wrong works. */ public function testIncorrectParams() { - $response = data_entry_helper::http_post($this->request, null); + $response = data_entry_helper::http_post($this->request, NULL); $this->assertEquals($response['output'], 'Invalid parameters'); } @@ -144,29 +135,45 @@ public function testIncorrectParams() { * data_cleaner_period_within_year module must be enabled. */ public function testPeriodWithinYearFail() { - $response = data_entry_helper::http_post($this->request, array( - 'sample' => json_encode(array( + $response = data_entry_helper::http_post($this->request, [ + 'sample' => json_encode([ 'sample:survey_id' => 1, 'sample:date' => '12/09/2012', 'sample:entered_sref' => 'SU1234', 'sample:entered_sref_system' => 'osgb', - )), - 'occurrences' => json_encode(array( - array( + ]), + 'occurrences' => json_encode([ + [ 'occurrence:taxa_taxon_list_id' => 1, - ), - )), - 'rule_types' => json_encode(array('PeriodWithinYear')), - )); + ], + ]), + 'rule_types' => json_encode(['PeriodWithinYear']), + ]); $errors = json_decode($response['output'], TRUE); $this->assertTrue($response['result'], 'Invalid response'); $this->assertIsArray($errors, 'Errors list not returned'); - $this->assertEquals(1, count($errors), 'Errors list empty. Is the data_cleaner_period_within_year module installed?'); - $this->assertArrayHasKey('taxa_taxon_list_id', $errors[0], 'Errors list missing taxa_taxon_list_id'); - $this->assertEquals('1', $errors[0]['taxa_taxon_list_id'], 'Incorrect taxa_taxon_list_id returned'); + $this->assertEquals( + 1, + count($errors), + 'Errors list empty. Is the data_cleaner_period_within_year module installed?' + ); + $this->assertArrayHasKey( + 'taxa_taxon_list_id', + $errors[0], + 'Errors list missing taxa_taxon_list_id' + ); + $this->assertEquals( + '1', + $errors[0]['taxa_taxon_list_id'], + 'Incorrect taxa_taxon_list_id returned' + ); $this->assertArrayHasKey('message', $errors[0], 'Errors list missing message'); - $this->assertEquals('PeriodWithinYear test failed', $errors[0]['message'], 'Incorrect message returned'); + $this->assertEquals( + 'PeriodWithinYear test failed', + $errors[0]['message'], + 'Incorrect message returned' + ); } /** diff --git a/modules/indicia_setup/db/version_6_3_0/202109081213_occurrences_verifier_only.sql b/modules/indicia_setup/db/version_6_3_0/202109081213_occurrences_verifier_only.sql new file mode 100644 index 0000000000..17c0569147 --- /dev/null +++ b/modules/indicia_setup/db/version_6_3_0/202109081213_occurrences_verifier_only.sql @@ -0,0 +1,5 @@ +ALTER TABLE occurrences + ADD COLUMN verifier_only_data json; + +COMMENT ON COLUMN occurrences.verifier_only_data + IS 'Data provided by the recorder about a record where they have only given permission for the data to be used for verification purposes, not for public reporting or onward data tranmission.'; \ No newline at end of file diff --git a/modules/indicia_svc_data/helpers/data_utils.php b/modules/indicia_svc_data/helpers/data_utils.php index 21341f421b..f0026bcce8 100644 --- a/modules/indicia_svc_data/helpers/data_utils.php +++ b/modules/indicia_svc_data/helpers/data_utils.php @@ -79,7 +79,10 @@ public static function applyWorkflowToOccurrenceUpdates($db, $websiteId, $userId && !empty($updates['occurrences']['record_status'])) { // As we are verifying or rejecting, we need to rewind any opposing // rejections or verifications. - $rewinds = workflow::getRewindChangesForRecords($db, 'occurrence', $idList, ['V', 'R']); + $rewinds = workflow::getRewindChangesForRecords($db, 'occurrence', $idList, [ + 'V', + 'R', + ]); // Fetch any new events to apply when this record is verified. $workflowEvents = workflow::getEventsForRecords( $db, @@ -105,8 +108,10 @@ public static function applyWorkflowToOccurrenceUpdates($db, $websiteId, $userId $oldRecord = ORM::factory('occurrence', $id); $oldRecordVals = $oldRecord->as_array(); $newRecordVals = array_merge($oldRecordVals, $thisUpdates['occurrences']); + $needsFilterCheck = !empty($thisEvent->attrs_filter_term) || !empty($thisEvent->location_ids_filter); $valuesToApply = workflow::processEvent( $thisEvent, + $needsFilterCheck, 'occurrence', $oldRecordVals, $newRecordVals, @@ -126,6 +131,19 @@ public static function applyWorkflowToOccurrenceUpdates($db, $websiteId, $userId 'created_by_id' => $userId, 'original_values' => json_encode($undoDetails['old_data']), ]); + if ($undoDetails['needs_filter_check']) { + $q = new WorkQueue(); + $q->enqueue($db, [ + 'task' => 'task_workflow_event_check_filters', + 'entity' => 'occurrence', + 'record_id' => $id, + 'cost_estimate' => 50, + 'priority' => 2, + 'params' => json_encode([ + 'workflow_events.id' => $undoDetails['event_id'], + ]), + ]); + } } } // This record is done, so exclude from the bulk operation. diff --git a/modules/phpUnit/config/core_fixture.php b/modules/phpUnit/config/core_fixture.php new file mode 100644 index 0000000000..dd431f2583 --- /dev/null +++ b/modules/phpUnit/config/core_fixture.php @@ -0,0 +1,932 @@ + [ + [ + "title" => "Test website", + "description" => "Website for unit testing", + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + "url" => "http:,//www.indicia.org.uk", + "password" => "password", + "verification_checks_enabled" => 'true', + ], + ], + "users_websites" => [ + [ + "user_id" => 1, + "website_id" => 1, + "site_role_id" => 1, + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + ], + ], + "surveys" => [ + [ + "title" => "Test survey", + "description" => "Survey for unit testing", + "website_id" => 1, + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + ], + [ + "title" => "Test survey 2", + "description" => "Additional survey for unit testing", + "website_id" => 1, + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + ], + ], + "taxon_meanings" => [ + [ + // No support for INSERT INTO table DEFAULT VALUES. + // Use high id values to avoid conflict with any values created by sequence + // during testing. + "id" => 10000, + ], + [ + "id" => 10001, + ], + ], + "taxon_groups" => [ + [ + "title" => "Test taxon group", + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + ], + ], + "taxon_ranks" => [ + [ + "rank" => "Genus", + "short_name" => "Genus", + "italicise_taxon" => "false", + "sort_order" => 290, + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + ], + [ + "rank" => "Species", + "short_name" => "Species", + "italicise_taxon" => "true", + "sort_order" => 300, + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + ], + ], + "taxon_lists" => [ + [ + "title" => "Test taxon list", + "description" => "Taxon list for unit testing", + "website_id" => 1, + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + ], + ], + "taxa" => [ + [ + "taxon" => "Test taxon", + "taxon_group_id" => 1, + "language_id" => 2, + "external_key" => "TESTKEY", + "taxon_rank_id" => 1, + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + ], + [ + "taxon" => "Test taxon 2", + "taxon_group_id" => 1, + "language_id" => 2, + "external_key" => "TESTKEY2", + "taxon_rank_id" => 2, + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + ], + ], + "taxa_taxon_lists" => [ + [ + "taxon_list_id" => 1, + "taxon_id" => 1, + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "taxon_meaning_id" => 10000, + "taxonomic_sort_order" => 1, + "preferred" => "true", + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + ], + [ + "taxon_list_id" => 1, + "taxon_id" => 2, + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "taxon_meaning_id" => 10001, + "taxonomic_sort_order" => 1, + "preferred" => "true", + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + ], + ], + "cache_taxa_taxon_lists" => [ + [ + "id" => 1, + "preferred" => true, + "taxon_list_id" => 1, + "taxon_list_title" => "Test taxa taxon list", + "website_id" => 1, + "preferred_taxa_taxon_list_id" => 1, + "taxonomic_sort_order" => 1, + "taxon" => "Test taxon", + "language_iso" => "lat", + "language" => "Latin", + "preferred_taxon" => "Test taxon", + "preferred_language_iso" => "lat", + "preferred_language" => "Latin", + "external_key" => "TESTKEY", + "taxon_rank" => "Genus", + "taxon_rank_sort_order" => "290", + "taxon_meaning_id" => 10000, + "taxon_group_id" => 1, + "taxon_group" => "Test taxon group", + "cache_created_on" => "2016-07-22 16:00:00", + "cache_updated_on" => "2016-07-22 16:00:00", + ], + [ + "id" => 2, + "preferred" => true, + "taxon_list_id" => 1, + "taxon_list_title" => "Test taxa taxon list 2", + "website_id" => 1, + "preferred_taxa_taxon_list_id" => 2, + "taxonomic_sort_order" => 2, + "taxon" => "Test taxon 2", + "language_iso" => "lat", + "language" => "Latin", + "preferred_taxon" => "Test taxon", + "preferred_language_iso" => "lat", + "preferred_language" => "Latin", + "external_key" => "TESTKEY2", + "taxon_rank" => "Species", + "taxon_rank_sort_order" => "300", + "taxon_meaning_id" => 10001, + "taxon_group_id" => 1, + "taxon_group" => "Test taxon group", + "cache_created_on" => "2016-07-22 16:00:00", + "cache_updated_on" => "2016-07-22 16:00:00", + ], + ], + "cache_taxon_searchterms" => [ + [ + "id" => 1, + "taxa_taxon_list_id" => 1, + "taxon_list_id" => 1, + "searchterm" => "testtaxon", + "original" => "Test taxon", + "taxon_group" => "Test taxon group", + "taxon_meaning_id" => 10000, + "preferred_taxon" => "Test taxon", + "default_common_name" => "Test taxon", + "language_iso" => "lat", + "name_type" => "L", + "simplified" => "t", + "taxon_group_id" => 1, + "preferred" => "t", + "searchterm_length" => 9, + "preferred_taxa_taxon_list_id" => 1, + "external_key" => "TESTKEY", + "taxon_rank_sort_order" => "290", + ], + [ + "id" => 2, + "taxa_taxon_list_id" => 1, + "taxon_list_id" => 1, + "searchterm" => "Test taxon", + "original" => "Test taxon", + "taxon_group" => "Test taxon group", + "taxon_meaning_id" => 10000, + "preferred_taxon" => "Test taxon", + "default_common_name" => "Test taxon", + "language_iso" => "lat", + "name_type" => "L", + "simplified" => "f", + "taxon_group_id" => 1, + "preferred" => "t", + "searchterm_length" => 10, + "preferred_taxa_taxon_list_id" => 1, + "external_key" => "TESTKEY", + "taxon_rank_sort_order" => "290", + ], + [ + "id" => 3, + "taxa_taxon_list_id" => 2, + "taxon_list_id" => 1, + "searchterm" => "testtaxon2", + "original" => "Test taxon 2", + "taxon_group" => "Test taxon group", + "taxon_meaning_id" => 10001, + "preferred_taxon" => "Test taxon 2", + "default_common_name" => "Test taxon 2", + "language_iso" => "lat", + "name_type" => "L", + "simplified" => "t", + "taxon_group_id" => 1, + "preferred" => "t", + "searchterm_length" => 10, + "preferred_taxa_taxon_list_id" => 1, + "external_key" => "TESTKEY2", + "taxon_rank_sort_order" => "300", + ], + [ + "id" => 4, + "taxa_taxon_list_id" => 2, + "taxon_list_id" => 1, + "searchterm" => "Test taxon 2", + "original" => "Test taxon 2", + "taxon_group" => "Test taxon group", + "taxon_meaning_id" => 10001, + "preferred_taxon" => "Test taxon 2", + "default_common_name" => "Test taxon 2", + "language_iso" => "lat", + "name_type" => "L", + "simplified" => "f", + "taxon_group_id" => 1, + "preferred" => "t", + "searchterm_length" => 12, + "preferred_taxa_taxon_list_id" => 1, + "external_key" => "TESTKEY2", + "taxon_rank_sort_order" => "300", + ], + ], + "meanings" => [ + // No support for INSERT INTO table DEFAULT VALUES. + // Use high id values to avoid conflict with any values created by sequence + // during testing. + [ + "id" => 10000, + ], + [ + "id" => 10001, + ], + [ + "id" => 10002, + ], + [ + "id" => 10003, + ], + [ + "id" => 10004, + ], + [ + "id" => 10005, + ], + [ + "id" => 10006, + ], + ], + "termlists" => [ + [ + "title" => "Test term list", + "description" => "Term list list for unit testing", + "website_id" => 1, + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + "external_key" => "TESTKEY", + ], + [ + "title" => "Location types", + "description" => "Term list for location types", + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + "external_key" => "indicia:location_types", + ], + [ + "title" => "Sample methods", + "description" => "Term list for sample methods", + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + "external_key" => "indicia:sample_methods", + ], + [ + "title" => "User identifier types", + "description" => "Term list for user identifier types", + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + "external_key" => "indicia:user_identifier_types", + ], + [ + "title" => "Group types", + "description" => "Term list for group types", + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + "external_key" => "indicia:group_types", + ], + [ + "title" => "Media types", + "description" => "Term list for media types", + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + "external_key" => "indicia:media_types", + ], + ], + + "terms" => [ + [ + "term" => "Test term", + "language_id" => 1, + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + ], + [ + "term" => "Test location type", + "language_id" => 1, + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + ], + [ + "term" => "Test sample method", + "language_id" => 1, + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + ], + [ + "term" => "email", + "language_id" => 1, + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + ], + [ + "term" => "twitter", + "language_id" => 1, + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + ], + [ + "term" => "Test group type", + "language_id" => 1, + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + ], + [ + "term" => "Image:Local", + "language_id" => 1, + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + ], + ], + "termlists_terms" => [ + [ + // Test term list. + "termlist_id" => 1, + // Test term. + "term_id" => 1, + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + "meaning_id" => 10000, + "preferred" => "true", + "sort_order" => 1, + ], + [ + // Location types. + "termlist_id" => 2, + // Test location type. + "term_id" => 2, + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + "meaning_id" => 10001, + "preferred" => "true", + "sort_order" => 1, + ], + [ + // Sample methods. + "termlist_id" => 3, + // Test sample method. + "term_id" => 3, + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + "meaning_id" => 10002, + "preferred" => "true", + "sort_order" => 1, + ], + [ + // User identifier types. + "termlist_id" => 4, + // Email. + "term_id" => 4, + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + "meaning_id" => 10003, + "preferred" => "true", + "sort_order" => 1, + ], + [ + // User identifier types. + "termlist_id" => 4, + // Twitter. + "term_id" => 5, + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + "meaning_id" => 10004, + "preferred" => "true", + "sort_order" => 2, + ], + [ + // Group types. + "termlist_id" => 5, + // Test group type. + "term_id" => 6, + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + "meaning_id" => 10005, + "preferred" => "true", + "sort_order" => 2, + ], + [ + // Media types. + "termlist_id" => 6, + // Image:Local. + "term_id" => 7, + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + "meaning_id" => 10005, + "preferred" => "true", + "sort_order" => 1, + ], + ], + "cache_termlists_terms" => [ + [ + "id" => 1, + "preferred" => "true", + "termlist_id" => 1, + "termlist_title" => "Test term list", + "website_id" => 1, + "preferred_termlists_term_id" => 1, + "sort_order" => 1, + "term" => "Test term", + "language_iso" => "eng", + "language" => "English", + "preferred_term" => "Test term", + "preferred_language_iso" => "eng", + "preferred_language" => "English", + "meaning_id" => 10000, + "cache_created_on" => "2016-07-22 16:00:00", + "cache_updated_on" => "2016-07-22 16:00:00", + ], + [ + "id" => 2, + "preferred" => "true", + "termlist_id" => 2, + "termlist_title" => "Location types", + "website_id" => 1, + "preferred_termlists_term_id" => 2, + "sort_order" => 1, + "term" => "Test location type", + "language_iso" => "eng", + "language" => "English", + "preferred_term" => "Test location type", + "preferred_language_iso" => "eng", + "preferred_language" => "English", + "meaning_id" => 10001, + "cache_created_on" => "2016-07-22 16:00:00", + "cache_updated_on" => "2016-07-22 16:00:00", + ], + [ + "id" => 3, + "preferred" => "true", + "termlist_id" => 3, + "termlist_title" => "Sample methods", + "website_id" => 1, + "preferred_termlists_term_id" => 3, + "sort_order" => 1, + "term" => "Test sample method", + "language_iso" => "eng", + "language" => "English", + "preferred_term" => "Test term", + "preferred_language_iso" => "eng", + "preferred_language" => "English", + "meaning_id" => 10002, + "cache_created_on" => "2016-07-22 16:00:00", + "cache_updated_on" => "2016-07-22 16:00:00", + ], + [ + "id" => 4, + "preferred" => "true", + "termlist_id" => 4, + "termlist_title" => "User identifier types", + "website_id" => 1, + "preferred_termlists_term_id" => 4, + "sort_order" => 1, + "term" => "email", + "language_iso" => "eng", + "language" => "English", + "preferred_term" => "email", + "preferred_language_iso" => "eng", + "preferred_language" => "English", + "meaning_id" => 10003, + "cache_created_on" => "2016-07-22 16:00:00", + "cache_updated_on" => "2016-07-22 16:00:00", + ], + [ + "id" => 5, + "preferred" => "true", + "termlist_id" => 4, + "termlist_title" => "User identifier types", + "website_id" => 1, + "preferred_termlists_term_id" => 5, + "sort_order" => 2, + "term" => "twitter", + "language_iso" => "eng", + "language" => "English", + "preferred_term" => "twitter", + "preferred_language_iso" => "eng", + "preferred_language" => "English", + "meaning_id" => 10004, + "cache_created_on" => "2016-07-22 16:00:00", + "cache_updated_on" => "2016-07-22 16:00:00", + ], + [ + "id" => 6, + "preferred" => "true", + "termlist_id" => 5, + "termlist_title" => "Group types", + "website_id" => 1, + "preferred_termlists_term_id" => 6, + "sort_order" => 1, + "term" => "Test group type", + "language_iso" => "eng", + "language" => "English", + "preferred_term" => "Test group type", + "preferred_language_iso" => "eng", + "preferred_language" => "English", + "meaning_id" => 10005, + "cache_created_on" => "2016-07-22 16:00:00", + "cache_updated_on" => "2016-07-22 16:00:00", + ], + [ + "id" => 7, + "preferred" => "true", + "termlist_id" => 6, + "termlist_title" => "Media types", + "website_id" => 1, + "preferred_termlists_term_id" => 7, + "sort_order" => 1, + "term" => "Image:Local", + "language_iso" => "eng", + "language" => "English", + "preferred_term" => "Image:Local", + "preferred_language_iso" => "eng", + "preferred_language" => "English", + "meaning_id" => 10006, + "cache_created_on" => "2016-07-22 16:00:00", + "cache_updated_on" => "2016-07-22 16:00:00", + ], + ], + "samples" => [ + [ + "survey_id" => 1, + "date_start" => "2016-07-22", + "date_end" => "2016-07-22", + "date_type" => "D", + "entered_sref" => "SU01", + "entered_sref_system" => "OSGB", + "comment" => "Sample for unit testing with a \" double quote", + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + "recorder_names" => "PHPUnit", + "record_status" => "C", + ], + [ + "survey_id" => 1, + "date_start" => "2016-07-22", + "date_end" => "2016-07-22", + "date_type" => "D", + "entered_sref" => "SU01", + "entered_sref_system" => "OSGB", + "comment" => "Sample for unit testing with a \nline break", + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + "recorder_names" => "PHPUnit", + "record_status" => "C", + ], + ], + "map_squares" => [ + [ + "geom" => "010300002031BF0D0001000000050000004E9C282E3C320BC18FC3A8120A2F59412CCEAACFA94309C1E75E7B57062F5941A7DE4D3BB84209C11FD1A351893E5941BBB729FE3E320BC1D8E4B8118D3E59414E9C282E3C320BC18FC3A8120A2F5941", + "x" => -214871, + "y" => 6609705, + "size" => 10000, + ], + ], + "occurrences" => [ + [ + "sample_id" => 1, + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + "website_id" => 1, + "comment" => "Occurrence for unit testing", + "taxa_taxon_list_id" => 1, + "record_status" => "C", + "release_status" => "R", + "confidential" => "f", + ], + [ + "sample_id" => 1, + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + "website_id" => 1, + "comment" => "Confidential occurrence for unit testing", + "taxa_taxon_list_id" => 1, + "record_status" => "C", + "release_status" => "R", + "confidential" => "t", + ], + ], + "cache_occurrences_functional" => [ + [ + "id" => 1, + "sample_id" => 1, + "website_id" => 1, + "survey_id" => 1, + "date_start" => "2016-07-22", + "date_end" => "2016-07-22", + "date_type" => "D", + "created_on" => "2016-07-22 16:00:00", + "updated_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "taxa_taxon_list_id" => 1, + "preferred_taxa_taxon_list_id" => 1, + "taxon_meaning_id" => 10000, + "taxa_taxon_list_external_key" => "TESTKEY", + "taxon_group_id" => 1, + "record_status" => "C", + "release_status" => "R", + "zero_abundance" => "f", + "confidential" => "f", + "map_sq_1km_id" => 1, + "map_sq_2km_id" => 1, + "map_sq_10km_id" => 1, + "verification_checks_enabled" => "true", + ], + [ + "id" => 2, + "sample_id" => 1, + "website_id" => 1, + "survey_id" => 1, + "date_start" => "2016-07-22", + "date_end" => "2016-07-22", + "date_type" => "D", + "created_on" => "2016-07-22 16:00:00", + "updated_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "taxa_taxon_list_id" => 1, + "preferred_taxa_taxon_list_id" => 1, + "taxon_meaning_id" => 10000, + "taxa_taxon_list_external_key" => "TESTKEY", + "taxon_group_id" => 1, + "record_status" => "C", + "release_status" => "R", + "zero_abundance" => "f", + "confidential" => "t", + "map_sq_1km_id" => 1, + "map_sq_2km_id" => 1, + "map_sq_10km_id" => 1, + "verification_checks_enabled" => "true", + ], + ], + "cache_occurrences_nonfunctional" => [ + [ + "id" => 1, + ], + [ + "id" => 2, + ], + ], + "cache_samples_functional" => [ + [ + "id" => 1, + "website_id" => 1, + "survey_id" => 1, + "date_start" => "2016-07-22", + "date_end" => "2016-07-22", + "date_type" => "D", + "created_on" => "2016-07-22 16:00:00", + "updated_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "record_status" => "C", + "map_sq_1km_id" => 1, + "map_sq_2km_id" => 1, + "map_sq_10km_id" => 1, + ], + ], + "cache_samples_nonfunctional" => [ + [ + "id" => 1, + "website_title" => "Test website", + "survey_title" => "Test survey", + "public_entered_sref" => "SU01", + "entered_sref_system" => "OSGB", + "recorders" => "PHPUnit", + ], + ], + "sample_attributes" => [ + [ + "caption" => "Altitude", + "data_type" => "I", + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + "public" => "false", + ], + ], + "sample_attributes_websites" => [ + [ + "website_id" => 1, + "sample_attribute_id" => 1, + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "restrict_to_survey_id" => 1, + ], + ], + "occurrence_attributes" => [ + [ + "caption" => "Identified_by", + "data_type" => "T", + "public" => "false", + "system_function" => "det_full_name", + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + ], + ], + "occurrence_attributes_websites" => [ + [ + "website_id" => 1, + "occurrence_attribute_id" => 1, + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "restrict_to_survey_id" => 1, + ], + ], + "locations" => [ + [ + "name" => "Test location", + "centroid_sref" => "SU01", + "centroid_sref_system" => "OSGB", + "location_type_id" => "2", + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + "public" => "true", + ], + ], + "location_attributes" => [ + [ + "caption" => "Test text", + "data_type" => "T", + "public" => "false", + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + ], + [ + "caption" => "Test lookup", + "data_type" => "L", + "termlist_id" => 1, + "public" => "false", + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + ], + [ + "caption" => "Test integer", + "data_type" => "I", + "termlist_id" => 1, + "public" => "false", + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + ], + ], + "location_attributes_websites" => [ + [ + // Test website. + "website_id" => 1, + // Test text. + "location_attribute_id" => 1, + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + ], + [ + // Test website. + "website_id" => 1, + // Test lookup. + "location_attribute_id" => 2, + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + ], + [ + // Test website. + "website_id" => 1, + // Test integer. + "location_attribute_id" => 3, + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + ], + ], + "location_attribute_values" => [ + [ + // Test location. + "location_id" => 1, + // Test lookup. + "location_attribute_id" => 2, + // Test term. + "int_value" => 1, + "created_on" => "2016-07-22 16:00:00", + "created_by_id" => 1, + "updated_on" => "2016-07-22 16:00:00", + "updated_by_id" => 1, + ], + ], +]; diff --git a/modules/phpUnit/libraries/Indicia_ArrayDataSet.php b/modules/phpUnit/libraries/Indicia_ArrayDataSet.php index d3f26b8406..322bb0a25a 100644 --- a/modules/phpUnit/libraries/Indicia_ArrayDataSet.php +++ b/modules/phpUnit/libraries/Indicia_ArrayDataSet.php @@ -9,47 +9,44 @@ /** * Implements a dataset created from a PHP array. - * https://phpunit.de/manual/current/en/database.html#database.available-implementations + * https://phpunit.de/manual/6.5/en/database.html#database.available-implementations */ -class Indicia_ArrayDataSet extends DbUDataSetAbstractDataSet -{ - /** - * @var array - */ - protected $tables = []; - - /** - * @param array $data - */ - public function __construct(array $data) - { - foreach ($data AS $tableName => $rows) { - $columns = []; - if (isset($rows[0])) { - $columns = array_keys($rows[0]); - } - - $metaData = new DbUDataSetDefaultTableMetadata($tableName, $columns); - $table = new DbUDataSetDefaultTable($metaData); - - foreach ($rows AS $row) { - $table->addRow($row); - } - $this->tables[$tableName] = $table; - } +class Indicia_ArrayDataSet extends DbUDataSetAbstractDataSet { + /** + * @var array + */ + protected $tables = []; + + /** + * @param array $data + */ + public function __construct(array $data) { + foreach ($data as $tableName => $rows) { + $columns = []; + if (isset($rows[0])) { + $columns = array_keys($rows[0]); + } + + $metaData = new DbUDataSetDefaultTableMetadata($tableName, $columns); + $table = new DbUDataSetDefaultTable($metaData); + + foreach ($rows as $row) { + $table->addRow($row); + } + $this->tables[$tableName] = $table; } + } - protected function createIterator(bool $reverse = false): DbUDataSetITableIterator - { - return new DbUDataSetDefaultTableIterator($this->tables, $reverse); + protected function createIterator(bool $reverse = false): DbUDataSetITableIterator { + return new DbUDataSetDefaultTableIterator($this->tables, $reverse); + } + + public function getTable(string $tableName): DbUDataSetITable { + if (!isset($this->tables[$tableName])) { + throw new InvalidArgumentException("$tableName is not a table in the current database."); } - public function getTable(string $tableName): DbUDataSetITable - { - if (!isset($this->tables[$tableName])) { - throw new InvalidArgumentException("$tableName is not a table in the current database."); - } + return $this->tables[$tableName]; + } - return $this->tables[$tableName]; - } -} \ No newline at end of file +} diff --git a/modules/phpUnit/libraries/Indicia_DatabaseTestCase.php b/modules/phpUnit/libraries/Indicia_DatabaseTestCase.php index 0a2f191c71..963d4d4706 100644 --- a/modules/phpUnit/libraries/Indicia_DatabaseTestCase.php +++ b/modules/phpUnit/libraries/Indicia_DatabaseTestCase.php @@ -13,47 +13,58 @@ /** * An abstract test case to efficiently make database connections. - * https://phpunit.de/manual/current/en/database.html#database.tip-use-your-own-abstract-database-testcase + * + * https://phpunit.de/manual/6.5/en/database.html#database.tip-use-your-own-abstract-database-testcase */ abstract class Indicia_DatabaseTestCase extends DbUTestCase { // Only instantiate pdo once for test clean-up/fixture load. static private $pdo = NULL; - // only instantiate PHPUnit_Extensions_Database_DB_IDatabaseConnection once per test - private $conn = null; + // Only instantiate PHPUnit_Extensions_Database_DB_IDatabaseConnection once + // per test. + private $conn = NULL; final public function getConnection() { - if ($this->conn === null) { - if (self::$pdo == null) { + if ($this->conn === NULL) { + if (self::$pdo == NULL) { $dsn = 'pgsql:host=127.0.0.1;dbname=indicia'; $user = 'indicia_user'; $pass = 'indicia_user_pass'; self::$pdo = new PDO($dsn, $user, $pass); - } + } $this->conn = $this->createDefaultDBConnection(self::$pdo, 'indicia'); } return $this->conn; } - - // Default implementation which does nothing + + /** + * Default implementation which does nothing. + */ public function getDataSet() { - return new Indicia_ArrayDataSet(array()); + return new Indicia_ArrayDataSet([]); } - // Override the operation used to set up the database. - // The default is CLEAN_INSERT($cascadeTruncates = FALSE) - // We require truncates to be cascaded to prevent foreign key - // violations and sequences to be restarted. + /** + * Override the operation used to set up the database. + * + * The default is CLEAN_INSERT($cascadeTruncates = FALSE) + * We require truncates to be cascaded to prevent foreign key + * violations and sequences to be restarted. + */ protected function getSetUpOperation() { - return My_Operation_Factory::RESTART_INSERT(true); + return My_Operation_Factory::RESTART_INSERT(TRUE); } - - // Override the function to create the database connection so that is uses - // My_DB_DefaultDatabaseConnection. + + /** + * Override the function to create the database connection. + * + * Use My_DB_DefaultDatabaseConnection. + */ protected function createDefaultDBConnection(PDO $connection, $schema = ''): dbUDatabaseDefualtConnection { return new My_DB_DefaultDatabaseConnection($connection, $schema); - } + } + } /** @@ -61,6 +72,7 @@ protected function createDefaultDBConnection(PDO $connection, $schema = ''): dbU * a database specific command to restart sequences. */ class My_DB_MetaData_PgSQL extends DbUDatabaseMetadataPgSQL { + public function getRestartCommand($table) { // Assumes sequence naming convention has been followed. $seq = $table . '_id_seq'; @@ -75,14 +87,16 @@ public function getRestartCommand($table) { } /** - * Extends PHPUnit_Extensions_Database_DB_MetaData in order to replace the + * Extends PHPUnit_Extensions_Database_DB_MetaData in order to replace the * default postgres driver with mine. */ abstract class My_DB_MetaData extends DbUDatabaseMetadataAbstractMetadata { + public static function createMetaData(PDO $pdo, $schema = '') { self::$metaDataClassMap['pgsql'] = 'My_DB_MetaData_PgSQL'; return parent::createMetaData($pdo, $schema); - } + } + } /** @@ -91,6 +105,7 @@ public static function createMetaData(PDO $pdo, $schema = '') { * b. give access to the command that will restart sequences. */ class My_DB_DefaultDatabaseConnection extends dbUDatabaseDefualtConnection { + public function __construct(PDO $connection, $schema = '') { $this->connection = $connection; $this->metaData = My_DB_MetaData::createMetaData($connection, $schema); @@ -100,12 +115,14 @@ public function __construct(PDO $connection, $schema = '') { public function getRestartCommand($table) { return $this->getMetaData()->getRestartCommand($table); } + } /** * New class to add a Restart operation. */ -Class My_Operation_Restart implements DbUOperationOperation { +class My_Operation_Restart implements DbUOperationOperation { + public function execute( DbUDatabaseConnection $connection, DbUDatasetIDataset $dataSet @@ -114,8 +131,9 @@ public function execute( $tableName = $table->getTableMetaData()->getTableName(); $query = $connection->getRestartCommand($tableName); try { - $connection->getConnection()->query($query); - } catch (\Exception $e) { + $connection->getConnection()->query($query); + } + catch (\Exception $e) { if ($e instanceof PDOException) { throw new DbUOperationException('RESTART', $query, [], $table, $e->getMessage()); } @@ -123,23 +141,25 @@ public function execute( } } } + } /** - * Extends PHPUnit_Extensions_Database_Operation_Factory in order to add + * Extends PHPUnit_Extensions_Database_Operation_Factory in order to add * functions that call the Restart operation. */ -Class My_Operation_Factory extends DbUOperationFactory { +class My_Operation_Factory extends DbUOperationFactory { public static function RESTART_INSERT($cascadeTruncates = FALSE) { return new DbUOperationComposite([ self::TRUNCATE($cascadeTruncates), self::RESTART(), - self::INSERT() + self::INSERT(), ]); } public static function RESTART() { return new My_Operation_Restart(); } + } diff --git a/modules/phpUnit/libraries/MY_Session.php b/modules/phpUnit/libraries/MY_Session.php index 2766863265..69c9d685db 100644 --- a/modules/phpUnit/libraries/MY_Session.php +++ b/modules/phpUnit/libraries/MY_Session.php @@ -1,128 +1,127 @@ -destroy(); - - if (Session::$config['driver'] !== 'native') - { - // Set driver name - $driver = 'Session_'.ucfirst(Session::$config['driver']).'_Driver'; - - // Load the driver - if ( ! Kohana::auto_load($driver)) - throw new Kohana_Exception('core.driver_not_found', Session::$config['driver'], get_class($this)); - - // Initialize the driver - Session::$driver = new $driver(); - - // Validate the driver - if ( ! (Session::$driver instanceof Session_Driver)) - throw new Kohana_Exception('core.driver_implements', Session::$config['driver'], get_class($this), 'Session_Driver'); - - // Register non-native driver as the session handler - session_set_save_handler - ( - array(Session::$driver, 'open'), - array(Session::$driver, 'close'), - array(Session::$driver, 'read'), - array(Session::$driver, 'write'), - array(Session::$driver, 'destroy'), - array(Session::$driver, 'gc') - ); - } - - // Validate the session name - if ( ! preg_match('~^(?=.*[a-z])[a-z0-9_]++$~iD', Session::$config['name'])) - throw new Kohana_Exception('session.invalid_session_name', Session::$config['name']); - - // Name the session, this will also be the name of the cookie - session_name(Session::$config['name']); - - // Set the session cookie parameters - session_set_cookie_params - ( - Session::$config['expiration'], - Kohana::config('cookie.path'), - Kohana::config('cookie.domain'), - Kohana::config('cookie.secure'), - Kohana::config('cookie.httponly') - ); - - // DO NOT start the session! phpUnit has done so. - // session_start(); - - // Put session_id in the session variable - $_SESSION['session_id'] = session_id(); - - // Set defaults - if ( ! isset($_SESSION['_kf_flash_'])) - { - $_SESSION['total_hits'] = 0; - $_SESSION['_kf_flash_'] = array(); - - $_SESSION['user_agent'] = Kohana::$user_agent; - $_SESSION['ip_address'] = $this->input->ip_address(); - } - - // Set up flash variables - Session::$flash =& $_SESSION['_kf_flash_']; - - // Increase total hits - $_SESSION['total_hits'] += 1; - - // Validate data only on hits after one - if ($_SESSION['total_hits'] > 1) - { - // Validate the session - foreach (Session::$config['validate'] as $valid) - { - switch ($valid) - { - // Check user agent for consistency - case 'user_agent': - if ($_SESSION[$valid] !== Kohana::$user_agent) - return $this->create(); - break; - - // Check ip address for consistency - case 'ip_address': - if ($_SESSION[$valid] !== $this->input->$valid()) - return $this->create(); - break; - - // Check expiration time to prevent users from manually modifying it - case 'expiration': - if (time() - $_SESSION['last_activity'] > ini_get('session.gc_maxlifetime')) - return $this->create(); - break; - } - } - } - - // Expire flash keys - $this->expire_flash(); - - // Update last activity - $_SESSION['last_activity'] = time(); - - // Set the new data - Session::set($vars); - } + /** + * Create a new session. + * + * @param array variables to set after creation + * + * @return void + */ + public function create($vars = NULL) { + // Destroy any current sessions. + $this->destroy(); + + if (Session::$config['driver'] !== 'native') { + // Set driver name. + $driver = 'Session_' . ucfirst(Session::$config['driver']) . '_Driver'; + + // Load the driver. + if (!Kohana::auto_load($driver)) { + throw new Kohana_Exception('core.driver_not_found', Session::$config['driver'], get_class($this)); + } + + // Initialize the driver. + Session::$driver = new $driver(); + + // Validate the driver. + if (!(Session::$driver instanceof Session_Driver)) { + throw new Kohana_Exception('core.driver_implements', Session::$config['driver'], get_class($this), 'Session_Driver'); + } + // Register non-native driver as the session handler. + session_set_save_handler( + [Session::$driver, 'open'], + [Session::$driver, 'close'], + [Session::$driver, 'read'], + [Session::$driver, 'write'], + [Session::$driver, 'destroy'], + [Session::$driver, 'gc'] + ); + } + + // Validate the session name. + if (!preg_match('~^(?=.*[a-z])[a-z0-9_]++$~iD', Session::$config['name'])) { + throw new Kohana_Exception('session.invalid_session_name', Session::$config['name']); + } + // Name the session, this will also be the name of the cookie. + session_name(Session::$config['name']); + + // Set the session cookie parameters. + session_set_cookie_params( + Session::$config['expiration'], + Kohana::config('cookie.path'), + Kohana::config('cookie.domain'), + Kohana::config('cookie.secure'), + Kohana::config('cookie.httponly') + ); + + // DO NOT start the session! phpUnit has done so. + // session_start(); + + // Put session_id in the session variable. + $_SESSION['session_id'] = session_id(); + + // Set defaults. + if (!isset($_SESSION['_kf_flash_'])) { + $_SESSION['total_hits'] = 0; + $_SESSION['_kf_flash_'] = []; + + $_SESSION['user_agent'] = Kohana::$user_agent; + $_SESSION['ip_address'] = $this->input->ip_address(); + } + + // Set up flash variables. + Session::$flash =& $_SESSION['_kf_flash_']; + + // Increase total hits. + $_SESSION['total_hits'] += 1; + + // Validate data only on hits after one. + if ($_SESSION['total_hits'] > 1) { + // Validate the session. + foreach (Session::$config['validate'] as $valid) { + switch ($valid) { + // Check user agent for consistency. + case 'user_agent': + if ($_SESSION[$valid] !== Kohana::$user_agent) { + return $this->create(); + } + break; + + // Check ip address for consistency. + case 'ip_address': + if ($_SESSION[$valid] !== $this->input->$valid()) { + return $this->create(); + } + break; + + // Check expiration time to prevent users from manually modifying it. + case 'expiration': + if (time() - $_SESSION['last_activity'] > ini_get('session.gc_maxlifetime')) { + return $this->create(); + } + break; + } + } + } + + // Expire flash keys. + $this->expire_flash(); + + // Update last activity. + $_SESSION['last_activity'] = time(); + + // Set the new data. + Session::set($vars); + } } \ No newline at end of file diff --git a/modules/phpUnit/libraries/SimpleDatabaseTestCase.php b/modules/phpUnit/libraries/SimpleDatabaseTestCase.php new file mode 100644 index 0000000000..0e3dfa5692 --- /dev/null +++ b/modules/phpUnit/libraries/SimpleDatabaseTestCase.php @@ -0,0 +1,87 @@ +getDataSet(); + self::setupFixture($fixture); + } + + /** + * Set up database fixture defined in file. + */ + private static function setupFixture($fixture) { + + // Remove any pre-existing data in fixture tables. + self::deleteFixture($fixture); + + // Set up fixture. + foreach ($fixture as $table => $records) { + foreach ($records as $record) { + if (!pg_insert(self::$conn, 'indicia.' . $table, $record)) { + throw new Exception("Failed inserting $record into $table"); + } + } + } + + // Fixture files may also contain the expected form of the survey export + // for the configuration established by the fixture. + if (isset($export)) { + return $export; + } + } + + /** + * Delete database fixture. + */ + private static function deleteFixture($fixture) { + // Drop content in fixture tables. + $tables = array_keys($fixture); + $table_list = implode(',', $tables); + pg_query(self::$conn, "TRUNCATE TABLE $table_list CASCADE"); + // And reset primary key sequences. + foreach ($tables as $table) { + if (substr($table, 0, 6) !== 'cache_') { + // Cache tables don't have their own sequences. + $seq = $table . '_id_seq'; + pg_query(self::$conn, "SELECT setval('$seq', 1, false)"); + } + } + + // Clear the cache to ensure tests use new database contents. + $cache = Cache::instance(); + $cache->delete_all(); + + } + +} diff --git a/modules/phpUnit/tests/Helper_Text_Test.php b/modules/phpUnit/tests/Helper_Text_Test.php index 0082bc2f22..44357e9743 100644 --- a/modules/phpUnit/tests/Helper_Text_Test.php +++ b/modules/phpUnit/tests/Helper_Text_Test.php @@ -277,7 +277,7 @@ public function censor_provider() * @group core.helpers.text.censor * @test */ - public function censor($str, $badwords, $replacement = '#', $replace_partial_words = FALSE, $expected_result) + public function censor($str, $badwords, $replacement, $replace_partial_words, $expected_result) { $result = text::censor($str, $badwords, $replacement, $replace_partial_words); $this->assertEquals($expected_result, $result); diff --git a/modules/rest_api/config/rest.example.php b/modules/rest_api/config/rest.example.php index d6030c86bc..2dc871a10c 100644 --- a/modules/rest_api/config/rest.example.php +++ b/modules/rest_api/config/rest.example.php @@ -119,6 +119,9 @@ 'es' => [ // Set open = TRUE if this end-point is available without authentication. 'open' => FALSE, + // Optional type, either occurrence or sample. Default is occurrence if not + // specified. + 'type' => 'occurrence', // Name of the elasticsearch index or alias this end-point points to. 'index' => 'occurrence', // URL of the Elasticsearch index. @@ -166,12 +169,20 @@ 'id_prefix' => 'iBRC', 'dataset_id_attr_id' => 22, 'blur' => 'F', + // Define an Elasticsearch query for the observations available to this + // project. 'es_bool_query' => [ 'must' => [ ['term' => ['taxon.class.keyword' => 'Aves']], ['term' => ['metadata.website.id' => 2]], ], ], + // Define a filter for the annotations data. This should match the + // location that the other server's observations are synced to using + // the rest_api_sync module. + 'annotations_filter' => [ + 'survey_id' => 10, + ], ], ], 'elasticsearch' => ['es'], diff --git a/modules/rest_api/controllers/services/rest.php b/modules/rest_api/controllers/services/rest.php index 9669ea81aa..f5b29442c9 100644 --- a/modules/rest_api/controllers/services/rest.php +++ b/modules/rest_api/controllers/services/rest.php @@ -65,9 +65,36 @@ class RestApiAbort extends Exception {} * Simple object to keep globally useful stuff in. */ class RestObjects { + + /** + * Database connection. + * + * @var object + */ public static $db; + + /** + * Website ID the client is authenticated as. + * + * @var int + */ public static $clientWebsiteId; + + /** + * User ID the client is authenticated as. + * + * @var int + */ public static $clientUserId; + + /** + * Name of the warehouse module handling the request. + * + * Might not be rest_api if the REST services extended by a modules's plugin + * file. + * + * @var string + */ public static $handlerModule; /** @@ -815,7 +842,8 @@ public function __call($name, $arguments) { $requestForId = $ids[0]; } // When using a client system ID, we also want a project ID in most cases. - if (isset(RestObjects::$clientSystemId) && !in_array($name, ['projects', 'taxa'])) { + if (isset(RestObjects::$clientSystemId) + && !in_array($name, ['projects', 'taxa'])) { if (empty($this->request['proj_id'])) { // Should not have got this far - just in case. RestObjects::$apiResponse->fail('Bad request', 400, 'Missing proj_id parameter'); @@ -829,10 +857,10 @@ public function __call($name, $arguments) { if (RestObjects::$handlerModule === 'rest_api' && method_exists($this, $methodName)) { $class = $this; } - elseif (RestObjects::$handlerModule !== 'rest_api' && method_exists(RestObjects::$handlerModule . '_rest', $methodName)) { + elseif (RestObjects::$handlerModule !== 'rest_api' && method_exists(RestObjects::$handlerModule . '_rest_endpoints', $methodName)) { // Expect any modules extending the API to implement a helper class - // _rest. - $class = RestObjects::$handlerModule . '_rest'; + // _rest_endpoints. + $class = RestObjects::$handlerModule . '_rest_endpoints'; } else { RestObjects::$apiResponse->fail('Not Found', 404, "Resource $name not known for method $this->method ($methodName)"); @@ -937,7 +965,7 @@ private function getRestPluginConfigs() { * @return string * Name of the key in the current HTTP method's config to use. */ - private function getPathConfigPatternMatch($methodConfig, $arguments) { + private function getPathConfigPatternMatch(array $methodConfig, array $arguments) { // Build an array to help locate the correct bit of configuration for // this path inside the resource config's method key. $searchArr = array_merge($arguments); @@ -984,7 +1012,6 @@ private function getMethodName(array $arguments, $usingPath) { return $methodName; } - /** * Check if resource allowed for project. * @@ -1021,10 +1048,10 @@ private function projectsGetId($id) { if (!array_key_exists($id, $this->projects)) { RestObjects::$apiResponse->fail('No Content', 204); } - RestObjects::$apiResponse->succeed($this->projects[$id], array( + RestObjects::$apiResponse->succeed($this->projects[$id], [ 'columnsToUnset' => ['filter_id', 'website_id', 'sharing', 'resources'], 'attachHref' => ['projects', 'id'], - )); + ]); } /** @@ -1051,17 +1078,22 @@ private function projectsGet() { * * Outputs a single taxon observations's details. * + * @deprecated + * Deprecated in version 6.3 and may be removed in future. Use the + * sync-taxon-observations end-point provided by the rest_api_sync module + * instead. + * * @param string $id * Unique ID for the taxon-observations to output. */ private function taxonObservationsGetId($id) { if (substr($id, 0, strlen(kohana::config('rest.user_id'))) === kohana::config('rest.user_id')) { $occurrence_id = substr($id, strlen(kohana::config('rest.user_id'))); - $params = array('occurrence_id' => $occurrence_id); + $params = ['occurrence_id' => $occurrence_id]; } else { // @todo What happens if system not recognised? - $params = array('external_key' => $id); + $params = ['external_key' => $id]; } $params['dataset_name_attr_id'] = kohana::config('rest.dataset_name_attr_id'); @@ -1076,10 +1108,10 @@ private function taxonObservationsGetId($id) { else { RestObjects::$apiResponse->succeed( $report['content']['records'][0], - array( - 'attachHref' => array('taxon-observations', 'id'), + [ + 'attachHref' => ['taxon-observations', 'id'], 'columns' => $report['content']['columns'], - ) + ] ); } } @@ -1089,6 +1121,11 @@ private function taxonObservationsGetId($id) { * * Outputs a list of taxon observation details. * + * @deprecated + * Deprecated in version 6.3 and may be removed in future. Use the + * sync-taxon-observations end-point provided by the rest_api_sync module + * instead. + * * @todo Ensure delete information is output. */ private function taxonObservationsGet() { @@ -1110,7 +1147,7 @@ private function taxonObservationsGet() { RestObjects::$apiResponse->succeed( $this->listResponseStructure($report['content']['records']), [ - 'attachHref' => array('taxon-observations', 'id'), + 'attachHref' => ['taxon-observations', 'id'], 'columns' => $report['content']['columns'], ] ); @@ -1125,7 +1162,7 @@ private function taxonObservationsGet() { * Unique ID for the annotations to output. */ private function annotationsGetId($id) { - $params = array('id' => $id); + $params = ['id' => $id]; $report = $this->loadReport('rest_api/filterable_annotations', $params); if (empty($report['content']['records'])) { RestObjects::$apiResponse->fail('No Content', 204); @@ -1136,11 +1173,11 @@ private function annotationsGetId($id) { } else { $record = $report['content']['records'][0]; - $record['taxonObservation'] = array( + $record['taxonObservation'] = [ 'id' => $record['taxon_observation_id'], // @todo href - ); - RestObjects::$apiResponse->succeed($record, array( + ]; + RestObjects::$apiResponse->succeed($record, [ 'attachHref' => [ 'annotations', 'id', @@ -1151,7 +1188,7 @@ private function annotationsGetId($id) { 'taxon-observations', ], 'columns' => $report['content']['columns'], - )); + ]); } } @@ -1208,10 +1245,10 @@ private function annotationsGet() { * @todo caching option */ private function taxaGetSearch() { - $params = array_merge(array( + $params = array_merge([ 'limit' => REST_API_DEFAULT_PAGE_SIZE, 'include' => ['data', 'count', 'paging', 'columns'], - ), $this->request); + ], $this->request); try { $params['count'] = FALSE; $query = postgreSQL::taxonSearchQuery($params); @@ -1242,30 +1279,30 @@ private function taxaGetSearch() { $result['paging'] = $this->getPagination($count); } } - $columns = array( - 'taxa_taxon_list_id' => array('caption' => 'Taxa taxon list ID'), - 'searchterm' => array('caption' => 'Search term'), - 'highlighted' => array('caption' => 'Highlighted'), - 'taxon' => array('caption' => 'Taxon'), - 'authority' => array('caption' => 'Authority'), - 'language_iso' => array('caption' => 'Language'), - 'preferred_taxon' => array('caption' => 'Preferred name'), - 'preferred_authority' => array('caption' => 'Preferred name authority'), - 'default_common_name' => array('caption' => 'Common name'), - 'taxon_group' => array('caption' => 'Taxon group'), - 'preferred' => array('caption' => 'Preferred'), - 'preferred_taxa_taxon_list_id' => array('caption' => 'Preferred taxa taxon list ID'), - 'taxon_meaning_id' => array('caption' => 'Taxon meaning ID'), - 'external_key' => array('caption' => 'External Key'), - 'taxon_group_id' => array('caption' => 'Taxon group ID'), - 'parent_id' => array('caption' => 'Parent taxa taxon list ID'), - 'identification_difficulty' => array('caption' => 'Ident. difficulty'), - 'id_diff_verification_rule_id' => array('caption' => 'Ident. difficulty verification rule ID'), - ); + $columns = [ + 'taxa_taxon_list_id' => ['caption' => 'Taxa taxon list ID'], + 'searchterm' => ['caption' => 'Search term'], + 'highlighted' => ['caption' => 'Highlighted'], + 'taxon' => ['caption' => 'Taxon'], + 'authority' => ['caption' => 'Authority'], + 'language_iso' => ['caption' => 'Language'], + 'preferred_taxon' => ['caption' => 'Preferred name'], + 'preferred_authority' => ['caption' => 'Preferred name authority'], + 'default_common_name' => ['caption' => 'Common name'], + 'taxon_group' => ['caption' => 'Taxon group'], + 'preferred' => ['caption' => 'Preferred'], + 'preferred_taxa_taxon_list_id' => ['caption' => 'Preferred taxa taxon list ID'], + 'taxon_meaning_id' => ['caption' => 'Taxon meaning ID'], + 'external_key' => ['caption' => 'External Key'], + 'taxon_group_id' => ['caption' => 'Taxon group ID'], + 'parent_id' => ['caption' => 'Parent taxa taxon list ID'], + 'identification_difficulty' => ['caption' => 'Ident. difficulty'], + 'id_diff_verification_rule_id' => ['caption' => 'Ident. difficulty verification rule ID'], + ]; if (in_array('columns', $params['include'])) { $result['columns'] = $columns; } - $resultOptions = array('columns' => $columns); + $resultOptions = ['columns' => $columns]; RestObjects::$apiResponse->succeed( $result, $resultOptions @@ -1329,15 +1366,24 @@ private function reportsGet($featured = FALSE) { } } + /** + * Handler for GET reports/featured. + * + * Returns a list of the reports that have the attribute featured="true". + */ private function reportsGetFeatured() { $this->reportsGet(TRUE); } + /** + * Handler for GET reports/path. + * + * Returns a list of the reports found within a folder path. + */ private function reportsGetPath() { $this->reportsGet(); } - /** * Converts the segments in the URL to a full report path. * @@ -1374,12 +1420,12 @@ private function getPagination($count) { parse_str($parts[1], $params); } else { - $params = array(); + $params = []; } $params['known_count'] = $count; - $pagination = array( + $pagination = [ 'self' => "$urlPrefix$url?" . http_build_query($params), - ); + ]; $limit = empty($params['limit']) ? REST_API_DEFAULT_PAGE_SIZE : $params['limit']; $offset = empty($params['offset']) ? 0 : $params['offset']; if ($offset > 0) { @@ -1460,7 +1506,7 @@ private function getReportOutput(array $segments) { } else { kohana::log('error', 'Rest API getReportOutput method retrieved invalid report response: ' . - var_export($report, true)); + var_export($report, TRUE)); } } finally { @@ -1508,7 +1554,7 @@ private function getReportMetadataItem(array $segments, $item, $description) { } RestObjects::$apiResponse->responseTitle = ucfirst("$item for $reportFile"); RestObjects::$apiResponse->wantIndex = TRUE; - RestObjects::$apiResponse->succeed(array('data' => $list), array('metadata' => array('description' => $description))); + RestObjects::$apiResponse->succeed(['data' => $list], ['metadata' => ['description' => $description]]); } /** @@ -1547,12 +1593,14 @@ private function getReportColumns(array $segments) { * * @param array $segments * URL segments. + * @param bool $featured + * True if returning a list of the featured reports. */ private function getReportHierarchy(array $segments, $featured) { $this->loadReportEngine(); // @todo Cache this $reportHierarchy = $this->reportEngine->reportList(); - $response = array(); + $response = []; $folderReadme = ''; if ($featured) { $folderReadme = kohana::lang("rest_api.reports.featured_folder_description"); @@ -1575,15 +1623,15 @@ private function getReportHierarchy(array $segments, $featured) { // If at the top level of the hierarchy, add a virtual featured folder // unless we are only showing featured reports. if (empty($segments) && empty($this->resourceOptions['featured'])) { - $reportHierarchy = array( - 'featured' => array( + $reportHierarchy = [ + 'featured' => [ 'type' => 'folder', 'description' => kohana::lang("rest_api.reports.featured_folder_description"), - ), - ) + $reportHierarchy; + ], + ] + $reportHierarchy; } if ($featured) { - $response = array(); + $response = []; $this->getFeaturedReports($reportHierarchy, $response); } else { @@ -1604,7 +1652,7 @@ private function getReportHierarchy(array $segments, $featured) { $relativePath = '/reports/' . ($relativePath ? "$relativePath/" : ''); $description = 'A list of reports and report folders stored on the warehouse under ' . "the folder $relativePath. $folderReadme"; - RestObjects::$apiResponse->succeed($response, array('metadata' => array('description' => $description))); + RestObjects::$apiResponse->succeed($response, ['metadata' => ['description' => $description]]); } /** @@ -1664,9 +1712,9 @@ private function addReportLinks(array &$metadata) { 'https://indicia-docs.readthedocs.io/en/latest/developing/reporting/standard-parameters.html'; unset($metadata['standard_params']); } - $metadata['columns'] = array( + $metadata['columns'] = [ 'href' => RestObjects::$apiResponse->getUrlWithCurrentParams("reports/$metadata[path].xml/columns"), - ); + ]; } /** @@ -1741,12 +1789,8 @@ private function checkParamDatatype($paramName, &$value, array $paramDef) { * Validates that the request parameters provided fullful the requirements of * the method being called. * - * @param string $resourceName - * Name of the resource. - * @param string $method - * Method name, e.g. GET or POST. - * @param bool $requestForId - * ID of resource being requested if any. + * @param array $methodConfig + * Configuration for the request. */ private function validateParameters($methodConfig) { // Check through the known list of parameters to ensure data formats are @@ -1812,7 +1856,7 @@ private function loadReportEngine() { // Should also return an object to iterate rather than loading the full // array. if (!isset($this->reportEngine)) { - $this->reportEngine = new ReportEngine(array(RestObjects::$clientWebsiteId)); + $this->reportEngine = new ReportEngine([RestObjects::$clientWebsiteId]); // Resource configuration can provide a list of restricted reports that // are allowed for this client. if (isset($this->resourceOptions['authorise'])) { @@ -2022,7 +2066,9 @@ private function loadFilterForProject($id) { } if (isset($this->projects[$id]['filter_id'])) { $filterId = $this->projects[$id]['filter_id']; - $filters = RestObjects::$db->select('definition')->from('filters')->where(array('id' => $filterId, 'deleted' => 'f')) + $filters = RestObjects::$db->select('definition') + ->from('filters') + ->where(['id' => $filterId, 'deleted' => 'f']) ->get()->result_array(); if (count($filters) !== 1) { RestObjects::$apiResponse->fail('Internal Server Error', 500, 'Failed to find unique project filter record'); @@ -2030,7 +2076,7 @@ private function loadFilterForProject($id) { return json_decode($filters[0]->definition, TRUE); } else { - return array(); + return []; } } @@ -2049,16 +2095,16 @@ private function loadFilterForProject($id) { private function getPermissionsFilterDefinition() { $filters = RestObjects::$db->select('definition') ->from('filters') - ->join('filters_users', array( + ->join('filters_users', [ 'filters_users.filter_id' => 'filters.id', - )) - ->where(array( + ]) + ->where([ 'filters.id' => $_GET['filter_id'], 'filters.deleted' => 'f', 'filters.defines_permissions' => 't', 'filters_users.user_id' => RestObjects::$clientUserId, 'filters_users.deleted' => 'f', - )) + ]) ->get()->result_array(); if (count($filters) !== 1) { RestObjects::$apiResponse->fail('Bad request', 400, 'Filter ID missing or not a permissions filter for the user'); @@ -2174,7 +2220,7 @@ private function authenticate() { $method = ucfirst($method); // Try this authentication method. if (method_exists($this, "authenticateUsing$method")) { - call_user_func(array($this, "authenticateUsing$method")); + call_user_func([$this, "authenticateUsing$method"]); } if ($this->authenticated) { RestObjects::$authMethod = $method; @@ -2231,7 +2277,8 @@ private function getAuthHeader() { return $headers['authorization']; } elseif (isset($_SERVER['REDIRECT_HTTP_AUTHORIZATION'])) { - // Sometimes on Apache, necessary to redirect the Auth header into the $_SERVER superglobal. + // Sometimes on Apache, necessary to redirect the Auth header into the + // $_SERVER superglobal. // See https://stackoverflow.com/questions/26475885/authorization-header-missing-in-php-post-request. return $_SERVER['REDIRECT_HTTP_AUTHORIZATION']; } @@ -2274,6 +2321,16 @@ private function getWebsiteByUrl($url) { return $website; } + /** + * Checks that the current user has website permissions. + * + * Fails if unauthorized. + * + * @param int $websiteId + * Website ID to test against. + * @param int $userId + * Warehouse user ID to test. + */ private function checkWebsiteUser($websiteId, $userId) { $cache = Cache::instance(); $cacheKey = "website-user-$websiteId-$userId"; @@ -2556,10 +2613,13 @@ private function authenticateUsingDirectClient() { // Taxon observations and annotations resource end-points will need a // proj_id if using client system based authorisation. if (($this->resourceName === 'taxon-observations' || $this->resourceName === 'annotations') && - (empty($_REQUEST['proj_id']) || empty($this->projects[$_REQUEST['proj_id']]))) { - RestObjects::$apiResponse->fail('Bad request', 400, 'Project ID missing or invalid.'); + (empty($_REQUEST['proj_id']))) { + RestObjects::$apiResponse->fail('Bad request', 400, 'Project ID missing.'); } if (!empty($_REQUEST['proj_id'])) { + if (empty($this->projects[$_REQUEST['proj_id']])) { + RestObjects::$apiResponse->fail('Bad request', 400, 'Project ID invalid.'); + } $projectConfig = $this->projects[$_REQUEST['proj_id']]; RestObjects::$clientWebsiteId = $projectConfig['website_id']; // The client project config can override the resource options, e.g. @@ -2687,6 +2747,12 @@ private function getFiles() { return $files; } + /** + * Request handler for POST /rest/media-queue. + * + * Allows media to be cached on the server prior to submitting the data the + * media should be attached to. + */ public function mediaQueuePost() { // Upload size. $ups = Kohana::config('indicia.maxUploadSize'); @@ -2719,11 +2785,13 @@ public function mediaQueuePost() { RestObjects::$apiResponse->fail('Bad Request', 400, json_encode($errors)); } foreach ($files as $key => $file) { - kohana::log('debug', var_export($file, TRUE)); $typeParts = explode('/', $file['type']); $fileName = uniqid('', TRUE) . '.' . $typeParts[1]; upload::save($file, $fileName, 'upload-queue'); - $response[$key] = ['name' => $fileName, 'tempPath' => url::base() . "upload-queue/$fileName"]; + $response[$key] = [ + 'name' => $fileName, + 'tempPath' => url::base() . "upload-queue/$fileName", + ]; } RestObjects::$apiResponse->succeed($response); } @@ -2885,7 +2953,7 @@ public function occurrencesDeleteId($id) { /** * API end-point to retrieve a location by ID. * - * @param integer $id + * @param int $id * ID of the location. */ public function locationsGetId($id) { @@ -3313,12 +3381,12 @@ public function sampleAttributesPutId($id) { private function createAttributeTermlist(array &$item) { // Create a new termlist. $termlist = ORM::factory('termlist'); - $termlist->set_submission_data(array( + $termlist->set_submission_data([ 'title' => 'Termlist for ' . $item['values']['caption'], 'description' => 'Termlist created by the REST API for attribute ' . $item['values']['caption'], 'website_id' => RestObjects::$clientWebsiteId, 'deleted' => 'f', - )); + ]); if (!$termlist->submit()) { RestObjects::$apiResponse->fail('Internal Server Error', 500, 'Error occurred creating new termlist: ' . implode("\n", $termlist->getAllErrors())); @@ -3345,7 +3413,11 @@ private function updateAttributeTermlist(array $item) { ->select('tlt.id, t.term, tlt.sort_order') ->from('termlists_terms AS tlt') ->join('terms AS t', 't.id', 'tlt.term_id') - ->where(['tlt.deleted' => 'f', 't.deleted' => 'f', 'tlt.termlist_id' => $item['values']['termlist_id']]) + ->where([ + 'tlt.deleted' => 'f', + 't.deleted' => 'f', + 'tlt.termlist_id' => $item['values']['termlist_id'], + ]) ->orderby(['tlt.sort_order' => 'ASC', 't.term' => 'ASC']) ->get()->result(); foreach ($existingRows as $row) { @@ -3371,13 +3443,13 @@ private function updateAttributeTermlist(array $item) { else { // Create new term. $termlists_term = ORM::factory('termlists_term'); - $termlists_term->set_submission_data(array( + $termlists_term->set_submission_data([ 'term:term' => $term, 'term:fk_language:iso' => kohana::config('indicia.default_lang'), 'sort_order' => $idx + 1, 'termlist_id' => $item['values']['termlist_id'], 'preferred' => 't', - )); + ]); if (!$termlists_term->submit()) { RestObjects::$apiResponse->fail('Internal Server Error', 500, 'Error occurred creating new term: ' . implode("\n", $termlists_term->getAllErrors())); @@ -3447,14 +3519,14 @@ public function sampleAttributesWebsitesPost() { } // Duplicate check. $existing = RestObjects::$db->select('aw.id') - ->from('sample_attributes_websites aw') - ->where([ - 'website_id' => $postArray['values']['website_id'], - 'sample_attribute_id' => $postArray['values']['sample_attribute_id'], - 'restrict_to_survey_id' => $postArray['values']['restrict_to_survey_id'], - 'deleted' => 'f', - ]) - ->get()->current(); + ->from('sample_attributes_websites aw') + ->where([ + 'website_id' => $postArray['values']['website_id'], + 'sample_attribute_id' => $postArray['values']['sample_attribute_id'], + 'restrict_to_survey_id' => $postArray['values']['restrict_to_survey_id'], + 'deleted' => 'f', + ]) + ->get()->current(); if ($existing) { $r = rest_crud::update('sample_attributes_website', $existing->id, $postArray); echo json_encode($r); @@ -3597,14 +3669,14 @@ public function occurrenceAttributesWebsitesPost() { } // Duplicate check. $existing = RestObjects::$db->select('aw.id') - ->from('occurrence_attributes_websites aw') - ->where([ - 'website_id' => $postArray['values']['website_id'], - 'occurrence_attribute_id' => $postArray['values']['occurrence_attribute_id'], - 'restrict_to_survey_id' => $postArray['values']['restrict_to_survey_id'], - 'deleted' => 'f', - ]) - ->get()->current(); + ->from('occurrence_attributes_websites aw') + ->where([ + 'website_id' => $postArray['values']['website_id'], + 'occurrence_attribute_id' => $postArray['values']['occurrence_attribute_id'], + 'restrict_to_survey_id' => $postArray['values']['restrict_to_survey_id'], + 'deleted' => 'f', + ]) + ->get()->current(); if ($existing) { $r = rest_crud::update('occurrence_attributes_website', $existing->id, $postArray); echo json_encode($r); @@ -3618,7 +3690,7 @@ public function occurrenceAttributesWebsitesPost() { } /** - * API end-point to PUT to an existing occurrence attributes website to update. + * End-point to PUT to an existing occurrence attributes website to update. */ public function occurrenceAttributesWebsitesPutId($id) { $this->assertUserHasWebsiteAccess(); diff --git a/modules/rest_api/helpers/rest_crud.php b/modules/rest_api/helpers/rest_crud.php index 2bc323b5d2..4fe8f53e90 100644 --- a/modules/rest_api/helpers/rest_crud.php +++ b/modules/rest_api/helpers/rest_crud.php @@ -625,7 +625,7 @@ private static function getResponseMetadata(array $responseMetadata) { if (preg_match('/_media$/', $subTable)) { $subTable = 'media'; } - if (array_key_exists($subTable, self::$entityConfig[$entity]->subModels)) { + if (property_exists(self::$entityConfig[$entity]->subModels, $subTable)) { if (!isset($r[$subTable])) { $r[$subTable] = []; } diff --git a/modules/rest_api/tests/Rest_ControllerTest.php b/modules/rest_api/tests/Rest_ControllerTest.php index f008acb01a..2785f0a83d 100644 --- a/modules/rest_api/tests/Rest_ControllerTest.php +++ b/modules/rest_api/tests/Rest_ControllerTest.php @@ -90,9 +90,9 @@ public function getDataSet() { * Create an occurrence comment for annotation testing. */ $ds2 = new Indicia_ArrayDataSet( - array( - 'filters' => array( - array( + [ + 'filters' => [ + [ 'title' => 'Test filter', 'description' => 'Filter for unit testing', 'definition' => '{"quality":"!R"}', @@ -101,8 +101,8 @@ public function getDataSet() { 'created_by_id' => 1, 'updated_on' => '2016-07-22:16:00:00', 'updated_by_id' => 1, - ), - array( + ], + [ 'title' => 'Test user permission filter', 'description' => 'Filter for unit testing', 'definition' => '{"quality":"!R","occurrence_id":2}', @@ -111,27 +111,27 @@ public function getDataSet() { 'created_by_id' => 1, 'updated_on' => '2016-07-22:16:00:00', 'updated_by_id' => 1, - ), - ), - 'filters_users' => array( - array( + ], + ], + 'filters_users' => [ + [ 'filter_id' => 2, 'user_id' => 1, 'created_on' => '2016-07-22:16:00:00', 'created_by_id' => 1 - ) - ), - 'occurrence_comments' => array( - array( + ], + ], + 'occurrence_comments' => [ + [ 'comment' => 'Occurrence comment for unit testing', 'created_on' => '2016-07-22:16:00:00', 'created_by_id' => 1, 'updated_on' => '2016-07-22:16:00:00', 'updated_by_id' => 1, 'occurrence_id' => 1, - ), - ), - ) + ], + ], + ] ); $compositeDs = new DbUDataSetCompositeDataSet(); @@ -143,19 +143,19 @@ public function getDataSet() { $db = new Database(); $db->update( 'users', - array('password' => '18d025c6c8809e34371e2ec7d84215bd3eb6031dcd804006f4'), - array('id' => 1) + ['password' => '18d025c6c8809e34371e2ec7d84215bd3eb6031dcd804006f4'], + ['id' => 1] ); return $compositeDs; } public static function setUpBeforeClass(): void { - // grab the clients registered on this system + // Grab the clients registered on this system. $clientUserIds = array_keys(Kohana::config('rest.clients')); $clientConfigs = array_values(Kohana::config('rest.clients')); - // just test the first client + // Just test the first client. self::$clientUserId = $clientUserIds[0]; self::$config = $clientConfigs[0]; } @@ -194,8 +194,8 @@ public function testJwt() { $db = new Database(); $db->update( 'websites', - array('public_key' => NULL), - array('id' => 1) + ['public_key' => NULL], + ['id' => 1] ); $cache->delete($cacheKey); // Make an otherwise valid call - should be unauthorised. @@ -206,8 +206,8 @@ public function testJwt() { $db = new Database(); $db->update( 'websites', - array('public_key' => 'INVALID!!!'), - array('id' => 1) + ['public_key' => 'INVALID!!!'], + ['id' => 1] ); $cache->delete($cacheKey); // Make an otherwise valid call - should be unauthorised. @@ -218,8 +218,8 @@ public function testJwt() { $db = new Database(); $db->update( 'websites', - array('public_key' => self::$publicKey), - array('id' => 1) + ['public_key' => self::$publicKey], + ['id' => 1] ); $cache->delete($cacheKey); // Make a valid call - should be authorised. @@ -255,17 +255,20 @@ public function testJwtHeaderCaseInsensitive() { $response = $this->callService( 'samples', FALSE, - ['values' => [ - 'survey_id' => 1, - 'entered_sref' => 'SU1234', - 'entered_sref_system' => 'OSGB', - 'date' => '01/08/2020', - 'comment' => 'A sample to delete', - ]] + [ + 'values' => [ + 'survey_id' => 1, + 'entered_sref' => 'SU1234', + 'entered_sref_system' => 'OSGB', + 'date' => '01/08/2020', + 'comment' => 'A sample to delete', + ] + ] ); $this->assertEquals(201, $response['httpCode']); $id = $response['response']['values']['id']; - // Now GET to check values stored OK using manually set auth header in lowercase. + // Now GET to check values stored OK using manually set auth header in + // lowercase. $this->authMethod = 'none'; $storedObj = $this->callService("samples/$id", FALSE, NULL, ['authorization: bearer ' . self::$jwt]); $this->assertResponseOk($storedObj, "/samples/$id GET"); @@ -350,8 +353,14 @@ private function putTest($table, array $exampleData, array $updateData) { $storedObj = $this->callService("$table/$id"); $expectedValues = array_merge($exampleData, $updateData); foreach ($expectedValues as $field => $value) { - $this->assertTrue(isset($storedObj['response']['values'][$field]), "Stored info in $table does not include value for $field"); - $this->assertEquals($value, $storedObj['response']['values'][$field], "Stored info in $table does not match value for $field"); + $this->assertTrue( + isset($storedObj['response']['values'][$field]), + "Stored info in $table does not include value for $field" + ); + $this->assertEquals( + $value, $storedObj['response']['values'][$field], + "Stored info in $table does not match value for $field" + ); } } @@ -384,8 +393,14 @@ private function getTest($table, $exampleData) { $this->assertTrue(array_key_exists('ETag', $headers), "$table GET does not return ETag."); $this->assertEquals($insertedRecordETag, $headers['ETag'], 'GET returns ETag which does not match expected'); foreach ($exampleData as $field => $value) { - $this->assertTrue(isset($storedObj['response']['values'][$field]), "Stored info in $table does not include value for $field"); - $this->assertEquals($exampleData[$field], $storedObj['response']['values'][$field], "Stored info in $table does not match value for $field"); + $this->assertTrue( + isset($storedObj['response']['values'][$field]), + "Stored info in $table does not include value for $field" + ); + $this->assertEquals( + $exampleData[$field], $storedObj['response']['values'][$field], + "Stored info in $table does not match value for $field" + ); } } @@ -459,7 +474,10 @@ private function getListTest($table, $exampleData) { break; } } - $this->assertFalse($found, 'POSTed survey found in filtered retrieved list using GET which it should be excluded from.'); + $this->assertFalse( + $found, + 'POSTed survey found in filtered retrieved list using GET which it should be excluded from.' + ); } /** @@ -599,13 +617,15 @@ public function testJwtSamplePostUserAuth() { $response = $this->callService( 'samples', FALSE, - ['values' => [ - 'survey_id' => 1, - 'entered_sref' => 'SU1234', - 'entered_sref_system' => 'OSGB', - 'date' => '01/08/2020', - 'comment' => 'A sample comment test', - ]] + [ + 'values' => [ + 'survey_id' => 1, + 'entered_sref' => 'SU1234', + 'entered_sref_system' => 'OSGB', + 'date' => '01/08/2020', + 'comment' => 'A sample comment test', + ], + ] ); $this->assertEquals(401, $response['httpCode']); // Grant website access. @@ -615,20 +635,25 @@ public function testJwtSamplePostUserAuth() { $response = $this->callService( 'samples', FALSE, - ['values' => [ - 'survey_id' => 1, - 'entered_sref' => 'SU1234', - 'entered_sref_system' => 'OSGB', - 'date' => '01/08/2020', - 'comment' => 'A sample comment test', - ]] + [ + 'values' => [ + 'survey_id' => 1, + 'entered_sref' => 'SU1234', + 'entered_sref_system' => 'OSGB', + 'date' => '01/08/2020', + 'comment' => 'A sample comment test', + ], + ] ); $this->assertEquals(201, $response['httpCode']); $id = $response['response']['values']['id']; // Check created_by_id. $response = $this->callService("samples/$id"); $this->assertEquals(200, $response['httpCode']); - $this->assertEquals($userId, $response['response']['values']['created_by_id'], 'Created_by_id not set correctly for sample'); + $this->assertEquals( + $userId, $response['response']['values']['created_by_id'], + 'Created_by_id not set correctly for sample' + ); // Re-authenticate as user 1. self::$jwt = $this->getJwt(self::$privateKey, 'http://www.indicia.org.uk', 1, time() + 120); // They shouldn't have access. @@ -653,9 +678,7 @@ public function testJwtSamplePostMoreTests() { $response = $this->callService( 'samples', FALSE, - [ - 'values' => $data - ] + ['values' => $data] ); $this->assertEquals(201, $response['httpCode']); $headers = $this->parseHeaders($response['headers']); @@ -697,12 +720,10 @@ public function testJwtSamplePostMoreTests() { $response = $this->callService( 'samples', FALSE, - [ - 'values' => $data - ] + ['values' => $data] ); $this->assertEquals(400, $response['httpCode']); - // GET the posted data; + // GET the posted data. $response = $this->callService("samples/$id"); $this->assertResponseOk($response, "/samples GET"); $this->assertTrue(array_key_exists('comment', $response['response']['values']), @@ -719,7 +740,7 @@ public function testJwtSamplePostMoreTests() { 'GET samples response does not contain processed lat output.'); $this->assertTrue(array_key_exists('lon', $response['response']['values']), 'GET samples response does not contain processed lon output.'); - // PUT a bad update with ID mismatch + // PUT a bad update with ID mismatch. $data = [ 'id' => $id + 1, 'entered_sref' => 'SU121341', @@ -843,7 +864,10 @@ public function testJwtSamplePostList() { FALSE, $data ); - $this->assertEquals(400, $response['httpCode'], 'POSTing a list to normal endpoint should fail'); + $this->assertEquals( + 400, $response['httpCode'], + 'POSTing a list to normal endpoint should fail' + ); $response = $this->callService( 'samples/list', FALSE, @@ -851,11 +875,17 @@ public function testJwtSamplePostList() { ); $this->assertEquals(201, $response['httpCode']); foreach ($response['response'] as $idx => $item) { - $this->assertTrue(is_numeric($idx), 'Response from list post should be a simple list array'); + $this->assertTrue( + is_numeric($idx), + 'Response from list post should be a simple list array' + ); $id = $item['values']['id']; $occCount = $db->query("select count(*) from occurrences where sample_id=$id") ->current()->count; - $this->assertEquals(1, $occCount, 'No occurrence created when submitted with a sample in a list.'); + $this->assertEquals( + 1, $occCount, + 'No occurrence created when submitted with a sample in a list.' + ); } } @@ -883,7 +913,10 @@ public function testJwtSamplePostExtKey() { FALSE, ['values' => $data] ); - $this->assertEquals(409, $response['httpCode'], 'Duplicate external key did not return 409 Conflict response.'); + $this->assertEquals( + 409, $response['httpCode'], + 'Duplicate external key did not return 409 Conflict response.' + ); $this->assertArrayHasKey('duplicate_of', $response['response']); $this->assertArrayHasKey('id', $response['response']['duplicate_of']); $this->assertArrayHasKey('href', $response['response']['duplicate_of']); @@ -895,7 +928,10 @@ public function testJwtSamplePostExtKey() { FALSE, ['values' => $data] ); - $this->assertEquals(201, $response['httpCode'], 'Duplicate external key in different survey not accepted.'); + $this->assertEquals( + 201, $response['httpCode'], + 'Duplicate external key in different survey not accepted.' + ); // PUT with same external key should be OK. $response = $this->callService( "samples/$id", @@ -1135,9 +1171,18 @@ public function testJwtSamplePostWithMedia() { $id = $response['response']['values']['id']; $smpMediaCount = $db->query("select count(*) from sample_media where sample_id=$id and path='$uploadedFileName'") ->current()->count; - $this->assertEquals(1, $smpMediaCount, 'No media created when submitted with a sample.'); - $this->assertFileExists(DOCROOT . 'upload/' . $uploadedFileName, 'Uploaded media file does not exist in destination'); - $this->assertFileExists(DOCROOT . 'upload/thumb-' . $uploadedFileName, 'Uploaded media thumbnail does not exist in destination'); + $this->assertEquals( + 1, $smpMediaCount, + 'No media created when submitted with a sample.' + ); + $this->assertFileExists( + DOCROOT . 'upload/' . $uploadedFileName, + 'Uploaded media file does not exist in destination' + ); + $this->assertFileExists( + DOCROOT . 'upload/thumb-' . $uploadedFileName, + 'Uploaded media thumbnail does not exist in destination' + ); // Post a sample which refers to an incorrect file. $data = [ 'values' => [ @@ -1185,7 +1230,7 @@ public function testJwtSampleOccurrenceMediaPost() { $file, 'image/jpg', basename($file) - ) + ), ], [], NULL, TRUE ); @@ -1222,10 +1267,16 @@ public function testJwtSampleOccurrenceMediaPost() { $this->assertEquals(201, $response['httpCode']); $id = $response['response']['values']['id']; $occurrences = $db->query("select id from occurrences where sample_id=$id"); - $this->assertEquals(1, count($occurrences), 'Posting a sample with occurrence did not create the occurrence'); + $this->assertEquals( + 1, count($occurrences), + 'Posting a sample with occurrence did not create the occurrence' + ); $occurrenceId = $occurrences->current()->id; $occurrences = $db->query("select id from occurrence_media where occurrence_id=$occurrenceId"); - $this->assertEquals(1, count($occurrences), 'Posting a sample with occurrence and media did not create the media'); + $this->assertEquals( + 1, count($occurrences), + 'Posting a sample with occurrence and media did not create the media' + ); // Check occurrence exists. $response = $this->callService("occurrences/$occurrenceId"); $this->assertResponseOk($response, "/occurrences/$occurrenceId GET"); @@ -1310,7 +1361,10 @@ public function testJwtLocationPost() { $db = new Database(); $locationsWebsitesCount = $db->query("select count(*) from locations_websites where location_id=$id") ->current()->count; - $this->assertEquals(1, $locationsWebsitesCount, 'No locations_websites record created for a location POST.'); + $this->assertEquals( + 1, $locationsWebsitesCount, + 'No locations_websites record created for a location POST.' + ); } /** @@ -1320,7 +1374,7 @@ public function testJwtLocationPut() { $this->putTest('locations', [ 'name' => 'Location test', 'centroid_sref' => 'ST1234', - 'centroid_sref_system' => 'OSGB' + 'centroid_sref_system' => 'OSGB', ], [ 'name' => 'Location test updated', ]); @@ -1333,7 +1387,7 @@ public function testJwtLocationGet() { $this->getTest('locations', [ 'name' => 'Location GET test', 'centroid_sref' => 'ST1234', - 'centroid_sref_system' => 'OSGB' + 'centroid_sref_system' => 'OSGB', ]); } @@ -1359,7 +1413,7 @@ public function testJwtLocationOptions() { * Test behaviour around REST support for ETags. */ public function testJwtLocationETags() { - $this->eTagsTest('locations', [ + $this->eTagsTest('locations', [ 'name' => 'Location GET test', 'centroid_sref' => 'ST1234', 'centroid_sref_system' => 'OSGB', @@ -1377,7 +1431,7 @@ public function testJwtSurveyPost() { * Test /surveys PUT behaviour. */ public function testJwtSurveyPut() { - $this->putTest('surveys', [ + $this->putTest('surveys', [ 'title' => 'Test survey', 'description' => 'A test', ], [ @@ -1389,7 +1443,7 @@ public function testJwtSurveyPut() { * A basic test of /surveys GET. */ public function testJwtSurveyGet() { - $this->getTest('surveys', [ + $this->getTest('surveys', [ 'title' => 'Test survey', 'description' => 'A test', ]); @@ -1399,7 +1453,7 @@ public function testJwtSurveyGet() { * A basic test of /surveys GET. */ public function testJwtSurveysGetList() { - $this->getListTest('surveys', [ + $this->getListTest('surveys', [ 'title' => 'Test survey ' . microtime(TRUE), 'description' => 'A test', ]); @@ -1409,7 +1463,7 @@ public function testJwtSurveysGetList() { * Test DELETE for a survey. */ public function testJwtSurveyDelete() { - $this->deleteTest('surveys', [ + $this->deleteTest('surveys', [ 'title' => 'Test survey', 'description' => 'A test', ]); @@ -1418,7 +1472,7 @@ public function testJwtSurveyDelete() { public function testJwtSurveyPostPermissions() { $this->authMethod = 'jwtUser'; self::$jwt = $this->getJwt(self::$privateKey, 'http://www.indicia.org.uk', 1, time() + 120); - $data = [ + $data = [ 'title' => 'Test survey', 'description' => 'A test', ]; @@ -1485,7 +1539,7 @@ public function testJwtSurveyOptions() { * Test behaviour around REST support for ETags. */ public function testJwtSurveyETags() { - $this->eTagsTest('surveys', [ + $this->eTagsTest('surveys', [ 'title' => 'Test survey', 'description' => 'A test', ]); @@ -1502,7 +1556,7 @@ public function testJwtSampleAttributePost() { * Test /sample_attributes PUT behaviour. */ public function testJwtSampleAttributePut() { - $this->putTest('sample_attributes', [ + $this->putTest('sample_attributes', [ 'caption' => 'Test sample attribute', 'data_type' => 'T', ], [ @@ -1514,7 +1568,7 @@ public function testJwtSampleAttributePut() { * A basic test of /sample_attributes GET. */ public function testJwtSampleAttributeGet() { - $this->getTest('sample_attributes', [ + $this->getTest('sample_attributes', [ 'caption' => 'Test sample attribute', 'data_type' => 'T', ]); @@ -1524,7 +1578,7 @@ public function testJwtSampleAttributeGet() { * A basic test of /sample_attributes GET. */ public function testJwtSampleAttributeGetList() { - $this->getListTest('sample_attributes', [ + $this->getListTest('sample_attributes', [ 'caption' => 'Test sample attribute ' . microtime(TRUE), 'data_type' => 'T', ]); @@ -1551,7 +1605,7 @@ public function testJwtSampleAttributeOptions() { * Test behaviour around REST support for ETags. */ public function testJwtSampleAttributeETags() { - $this->eTagsTest('sample_attributes', [ + $this->eTagsTest('sample_attributes', [ 'caption' => 'Test sample attribute', 'data_type' => 'T', ]); @@ -1568,7 +1622,7 @@ public function testJwtOccurrenceAttributePost() { * Test /occurrence_attributes PUT behaviour. */ public function testJwtOccurrenceAttributePut() { - $this->putTest('occurrence_attributes', [ + $this->putTest('occurrence_attributes', [ 'caption' => 'Test occurrence attribute', 'data_type' => 'T', ], [ @@ -1580,7 +1634,7 @@ public function testJwtOccurrenceAttributePut() { * A basic test of /occurrence_attributes GET. */ public function testJwtOccurrenceAttributeGet() { - $this->getTest('occurrence_attributes', [ + $this->getTest('occurrence_attributes', [ 'caption' => 'Test occurrence attribute', 'data_type' => 'T', ]); @@ -1617,7 +1671,7 @@ public function testJwtOccurrenceAttributeOptions() { * Test behaviour around REST support for ETags. */ public function testJwtOccurrenceAttributeETags() { - $this->eTagsTest('occurrence_attributes', [ + $this->eTagsTest('occurrence_attributes', [ 'caption' => 'Test occurrence attribute', 'data_type' => 'T', ]); @@ -1746,7 +1800,7 @@ public function testJwtOccurrencePost() { ], 'taxa_taxon_list_id'); } - /** + /** * Test /occurrences PUT in isolation. */ public function testJwtOccurrencePut() { @@ -1805,7 +1859,7 @@ public function testJwtOccurrenceOptions() { */ public function testJwtOccurrenceETags() { $sampleId = $this->postSampleToAddOccurrencesTo(); - $this->eTagsTest('occurrences', [ + $this->eTagsTest('occurrences', [ 'taxa_taxon_list_id' => 1, 'sample_id' => $sampleId, ]); @@ -1834,7 +1888,10 @@ public function testJwtOccurrencePostExtKey() { FALSE, ['values' => $data] ); - $this->assertEquals(409, $response['httpCode'], 'Duplicate external key did not return 409 Conflict response.'); + $this->assertEquals( + 409, $response['httpCode'], + 'Duplicate external key did not return 409 Conflict response.' + ); } /** @@ -1853,7 +1910,10 @@ public function testJwtOccurrencePostDeletedSample() { FALSE, ['values' => $data] ); - $this->assertEquals(400, $response['httpCode'], 'Adding occurrence to deleted sample did not return 400 Bad request response.'); + $this->assertEquals( + 400, $response['httpCode'], + 'Adding occurrence to deleted sample did not return 400 Bad request response.' + ); } public function testProjects_authentication() { @@ -1866,20 +1926,32 @@ public function testProjects_authentication() { // user and website authentications don't allow access to projects $this->authMethod = 'hmacUser'; $response = $this->callService('projects'); - $this->assertTrue($response['httpCode']===401, 'Invalid authentication method hmacUser for projects ' . - "but response still OK. Http response $response[httpCode]."); + $this->assertTrue( + $response['httpCode'] === 401, + 'Invalid authentication method hmacUser for projects ' . + "but response still OK. Http response $response[httpCode]." + ); $this->authMethod = 'directUser'; $response = $this->callService('projects'); - $this->assertTrue($response['httpCode']===401, 'Invalid authentication method directUser for projects ' . - "but response still OK. Http response $response[httpCode]."); + $this->assertTrue( + $response['httpCode'] === 401, + 'Invalid authentication method directUser for projects ' . + "but response still OK. Http response $response[httpCode]." + ); $this->authMethod = 'hmacWebsite'; $response = $this->callService('projects'); - $this->assertTrue($response['httpCode']===401, 'Invalid authentication method hmacWebsite for projects ' . - "but response still OK. Http response $response[httpCode]."); + $this->assertTrue( + $response['httpCode'] === 401, + 'Invalid authentication method hmacWebsite for projects ' . + "but response still OK. Http response $response[httpCode]." + ); $this->authMethod = 'directWebsite'; $response = $this->callService('projects'); - $this->assertTrue($response['httpCode']===401, 'Invalid authentication method directWebsite for projects ' . - "but response still OK. Http response $response[httpCode]."); + $this->assertTrue( + $response['httpCode'] === 401, + 'Invalid authentication method directWebsite for projects ' . + "but response still OK. Http response $response[httpCode]." + ); $this->authMethod = 'hmacClient'; } @@ -1893,14 +1965,23 @@ public function testProjects_get() { $this->assertEquals(count($viaConfig), count($response['response']['data']), 'Incorrect number of projects returned from /projects.'); foreach ($response['response']['data'] as $projDef) { - $this->assertArrayHasKey($projDef['id'], $viaConfig, "Unexpected project $projDef[id]returned by /projects."); + $this->assertArrayHasKey( + $projDef['id'], $viaConfig, + "Unexpected project $projDef[id]returned by /projects." + ); $this->assertEquals($viaConfig[$projDef['id']]['title'], $projDef['title'], "Unexpected title $projDef[title] returned for project $projDef[id] by /projects."); $this->assertEquals($viaConfig[$projDef['id']]['description'], $projDef['description'], "Unexpected description $projDef[description] returned for project $projDef[id] by /projects."); - // Some project keys are supposed to be removed - $this->assertNotContains('filter_id', $projDef, 'Project definition should not contain filter_id'); - $this->assertNotContains('sharing', $projDef, 'Project definition should not contain sharing'); + // Some project keys are supposed to be removed. + $this->assertNotContains( + 'filter_id', $projDef, + 'Project definition should not contain filter_id' + ); + $this->assertNotContains( + 'sharing', $projDef, + 'Project definition should not contain sharing' + ); } } @@ -1916,17 +1997,23 @@ public function testProjects_get_id() { $this->assertEquals($projDef['description'], $response['response']['description'], "Unexpected description " . $response['response']['description'] . " returned for project $projDef[id] by /projects/$projDef[id]."); - // Some project keys are supposed to be removed - $this->assertNotContains('filter_id', $projDef, 'Project definition should not contain filter_id'); - $this->assertNotContains('sharing', $projDef, 'Project definition should not contain sharing'); + // Some project keys are supposed to be removed. + $this->assertNotContains( + 'filter_id', $projDef, + 'Project definition should not contain filter_id' + ); + $this->assertNotContains( + 'sharing', $projDef, + 'Project definition should not contain sharing' + ); } } public function testTaxon_observations_authentication() { Kohana::log('debug', "Running unit test, Rest_ControllerTest::testProjects_clientAuthentication"); $proj_id = self::$config['projects'][array_keys(self::$config['projects'])[0]]['id']; - $queryWithProj = array('proj_id' => $proj_id, 'edited_date_from' => '2015-01-01'); - $query = array('edited_date_from' => '2015-01-01'); + $queryWithProj = ['proj_id' => $proj_id, 'edited_date_from' => '2015-01-01']; + $query = ['edited_date_from' => '2015-01-01']; $this->authMethod = 'hmacClient'; $this->checkResourceAuthentication('taxon-observations', $queryWithProj); @@ -1936,7 +2023,7 @@ public function testTaxon_observations_authentication() { $this->checkResourceAuthentication('taxon-observations', $query); // @todo The following test needs to check filtered response rather than authentication $this->authMethod = 'directUser'; - $this->checkResourceAuthentication('taxon-observations', $query + array('filter_id' => self::$userFilterId)); + $this->checkResourceAuthentication('taxon-observations', $query + ['filter_id' => self::$userFilterId]); $this->authMethod = 'hmacWebsite'; $this->checkResourceAuthentication('taxon-observations', $query); $this->authMethod = 'directWebsite'; @@ -1951,10 +2038,10 @@ public function testTaxon_observations_get_incorrect_params() { $this->assertEquals(400, $response['httpCode'], 'Requesting taxon observations without params should be a bad request'); foreach (self::$config['projects'] as $projDef) { - $response = $this->callService("taxon-observations", array('proj_id' => $projDef['id'])); + $response = $this->callService("taxon-observations", ['proj_id' => $projDef['id']]); $this->assertEquals(400, $response['httpCode'], 'Requesting taxon observations without edited_date_from should be a bad request'); - $response = $this->callService("taxon-observations", array('edited_date_from' => '2015-01-01')); + $response = $this->callService("taxon-observations", ['edited_date_from' => '2015-01-01']); $this->assertEquals(400, $response['httpCode'], 'Requesting taxon observations without proj_id should be a bad request'); // only test a single project @@ -1971,12 +2058,11 @@ public function testTaxon_observations_get() { Kohana::log('debug', "Running unit test, Rest_ControllerTest::testTaxon_observations_get"); foreach (self::$config['projects'] as $projDef) { - $response = $this->callService("taxon-observations", array( + $response = $this->callService("taxon-observations", [ 'proj_id' => $projDef['id'], 'edited_date_from' => '2015-01-01', 'edited_date_to' => date("Y-m-d\TH:i:s") - ) - ); + ]); $this->assertResponseOk($response, '/taxon-observations'); $this->assertArrayHasKey('paging', $response['response'], 'Paging missing from response to call to taxon-observations'); @@ -1985,9 +2071,10 @@ public function testTaxon_observations_get() { $data = $response['response']['data']; $this->assertIsArray($data, 'Taxon-observations data invalid. ' . var_export($data, true)); $this->assertNotCount(0, $data, 'Taxon-observations data absent. ' . var_export($data, true)); - foreach ($data as $occurrence) + foreach ($data as $occurrence) { $this->checkValidTaxonObservation($occurrence); - // only test a single project + } + // Only test a single project. break; } } @@ -2003,17 +2090,18 @@ public function testAnnotations_get() { foreach (self::$config['projects'] as $projDef) { $response = $this->callService( "annotations", - array('proj_id' => $projDef['id'], 'edited_date_from' => '2015-01-01') + ['proj_id' => $projDef['id'], 'edited_date_from' => '2015-01-01'] ); $this->assertResponseOk($response, '/annotations'); $this->assertArrayHasKey('paging', $response['response'], 'Paging missing from response to call to annotations'); $this->assertArrayHasKey('data', $response['response'], 'Data missing from response to call to annotations'); $data = $response['response']['data']; - $this->assertIsArray($data, 'Annotations data invalid. ' . var_export($data, true)); - $this->assertNotCount(0, $data, 'Annotations data absent. ' . var_export($data, true)); - foreach ($data as $annotation) + $this->assertIsArray($data, 'Annotations data invalid. ' . var_export($data, TRUE)); + $this->assertNotCount(0, $data, 'Annotations data absent. ' . var_export($data, TRUE)); + foreach ($data as $annotation) { $this->checkValidAnnotation($annotation); - // only test a single project + } + // Only test a single project. break; } } @@ -2034,8 +2122,14 @@ public function testTaxaSearch_get() { 'taxon_list_id' => 1, ]); $this->assertResponseOk($response, '/taxa/search'); - $this->assertArrayHasKey('paging', $response['response'], 'Paging missing from response to call to taxa/search'); - $this->assertArrayHasKey('data', $response['response'], 'Data missing from response to call to taxa/search'); + $this->assertArrayHasKey( + 'paging', $response['response'], + 'Paging missing from response to call to taxa/search' + ); + $this->assertArrayHasKey( + 'data', $response['response'], + 'Data missing from response to call to taxa/search' + ); $data = $response['response']['data']; $this->assertIsArray($data, 'taxa/search data invalid.'); $this->assertCount(2, $data, 'Taxa/search data wrong count returned.'); @@ -2044,8 +2138,14 @@ public function testTaxaSearch_get() { 'taxon_list_id' => 1, ]); $this->assertResponseOk($response, '/taxa/search'); - $this->assertArrayHasKey('paging', $response['response'], 'Paging missing from response to call to taxa/search'); - $this->assertArrayHasKey('data', $response['response'], 'Data missing from response to call to taxa/search'); + $this->assertArrayHasKey( + 'paging', $response['response'], + 'Paging missing from response to call to taxa/search' + ); + $this->assertArrayHasKey( + 'data', $response['response'], + 'Data missing from response to call to taxa/search' + ); $data = $response['response']['data']; $this->assertIsArray($data, 'taxa/search data invalid.'); $this->assertCount(1, $data, 'Taxa/search data wrong count returned.'); @@ -2071,7 +2171,7 @@ public function testReportsHierarchy_get() { Kohana::log('debug', "Running unit test, Rest_ControllerTest::testReportsHierarchy_get"); $projDef = self::$config['projects']['BRC1']; - $response = $this->callService("reports", array('proj_id' => $projDef['id'])); + $response = $this->callService("reports", ['proj_id' => $projDef['id']]); $this->assertResponseOk($response, '/reports'); // Check a folder that should definitely exist. $this->checkReportFolderInReponse($response['response'], 'library'); @@ -2082,19 +2182,19 @@ public function testReportsHierarchy_get() { // should be an additional featured folder at the top level with shortcuts // to favourite reports. $this->authMethod = 'hmacWebsite'; - $response = $this->callService("reports", array('proj_id' => $projDef['id'])); + $response = $this->callService("reports", ['proj_id' => $projDef['id']]); $this->checkReportFolderInReponse($response['response'], 'featured'); $this->checkReportInReponse($response['response'], 'demo'); // now check some folder contents $this->authMethod = 'hmacClient'; - $response = $this->callService("reports/featured", array('proj_id' => $projDef['id'])); + $response = $this->callService("reports/featured", ['proj_id' => $projDef['id']]); $this->assertResponseOk($response, '/reports/featured'); $this->checkReportInReponse($response['response'], 'library/occurrences/filterable_explore_list'); - $response = $this->callService("reports/library", array('proj_id' => $projDef['id'])); + $response = $this->callService("reports/library", ['proj_id' => $projDef['id']]); $this->assertResponseOk($response, '/reports/library'); $this->checkReportFolderInReponse($response['response'], 'occurrences'); - $response = $this->callService("reports/library/occurrences", array('proj_id' => $projDef['id'])); + $response = $this->callService("reports/library/occurrences", ['proj_id' => $projDef['id']]); $this->assertResponseOk($response, '/reports/library/occurrences'); $this->checkReportInReponse($response['response'], 'filterable_explore_list'); } @@ -2103,15 +2203,19 @@ public function testMissingReportFile() { $this->authMethod = 'jwtUser'; self::$jwt = $this->getJwt(self::$privateKey, 'http://www.indicia.org.uk', 1, time() + 120); $response = $this->callService('reports/some_random_report_name.xml', []); - $this->assertEquals(404, $response['httpCode'], 'Request for a missing report does not return 404.'); + $this->assertEquals( + 404, $response['httpCode'], + 'Request for a missing report does not return 404.' + ); } public function testReportParams_get() { Kohana::log('debug', "Running unit test, Rest_ControllerTest::testReportParams_get"); - // First grab a list of reports so we can use the links to get the correct params URL + // First grab a list of reports so we can use the links to get the correct + // params URL. $projDef = self::$config['projects']['BRC1']; - $response = $this->callService("reports/library/occurrences", array('proj_id' => $projDef['id'])); + $response = $this->callService("reports/library/occurrences", ['proj_id' => $projDef['id']]); $this->assertResponseOk($response, '/reports/library/occurrences'); $reportDef = $response['response']['filterable_explore_list']; $this->assertArrayHasKey('params', $reportDef, 'Report response does not define parameters'); @@ -2127,9 +2231,10 @@ public function testReportParams_get() { public function testReportColumns_get() { Kohana::log('debug', "Running unit test, Rest_ControllerTest::testReportColumns_get"); - // First grab a list of reports so we can use the links to get the correct columns URL + // First grab a list of reports so we can use the links to get the correct + // columns URL. $projDef = self::$config['projects']['BRC1']; - $response = $this->callService("reports/library/occurrences", array('proj_id' => $projDef['id'])); + $response = $this->callService("reports/library/occurrences", ['proj_id' => $projDef['id']]); $this->assertResponseOk($response, '/reports/library/occurrences'); $reportDef = $response['response']['filterable_explore_list']; $this->assertArrayHasKey('columns', $reportDef, 'Report response does not define columns'); @@ -2145,9 +2250,10 @@ public function testReportColumns_get() { public function testReportOutput_get() { Kohana::log('debug', "Running unit test, Rest_ControllerTest::testReportOutput_get"); - // First grab a list of reports so we can use the links to get the correct columns URL + // First grab a list of reports so we can use the links to get the correct + // columns URL. $projDef = self::$config['projects']['BRC1']; - $response = $this->callService("reports/library/occurrences", array('proj_id' => $projDef['id'])); + $response = $this->callService("reports/library/occurrences", ['proj_id' => $projDef['id']]); $this->assertResponseOk($response, '/reports/library/occurrences'); $reportDef = $response['response']['filterable_explore_list']; $this->assertArrayHasKey('href', $reportDef, 'Report response missing href'); @@ -2156,34 +2262,57 @@ public function testReportOutput_get() { $this->assertResponseOk($response, '/reports/library/occurrences/filterable_explore_list.xml'); $this->assertArrayHasKey('data', $response['response']); $this->assertCount(1, $response['response']['data'], 'Report call returns incorrect record count'); - $this->assertEquals(1, $response['response']['data'][0]['occurrence_id'], 'Report call returns incorrect record'); + $this->assertEquals( + 1, $response['response']['data'][0]['occurrence_id'], + 'Report call returns incorrect record' + ); } public function testAcceptHeader() { Kohana::log('debug', "Running unit test, Rest_ControllerTest::testAcceptHeader"); $projDef = self::$config['projects']['BRC1']; - $response = $this->callService("reports/library/occurrences", array('proj_id' => $projDef['id']), NULL, ['Accept: application/json']); + $response = $this->callService( + "reports/library/occurrences", + ['proj_id' => $projDef['id']], + NULL, + ['Accept: application/json'] + ); $decoded = json_decode($response['response'], TRUE); $this->assertNotEquals(NULL, $decoded, 'JSON response could not be decoded: ' . $response['response']); $this->assertEquals(200, $response['httpCode']); $this->assertEquals(0, $response['curlErrno']); - $response = $this->callService("reports/library/occurrences", array('proj_id' => $projDef['id']), NULL, ['Accept: text/html']); + $response = $this->callService( + "reports/library/occurrences", + ['proj_id' => $projDef['id']], + NULL, + ['Accept: text/html'] + ); $this->assertMatchesRegularExpression('/^/', $response['response']); $this->assertMatchesRegularExpression('//', $response['response']); $this->assertMatchesRegularExpression('/<\/html>$/', $response['response']); $this->assertEquals(200, $response['httpCode']); $this->assertEquals(0, $response['curlErrno']); - // try requesting an invalid content type as first preference - response should select the second. - $response = $this->callService("reports/library/occurrences", array('proj_id' => $projDef['id']), NULL, ['Accept: image/png, application/json']); + // Try requesting an invalid content type as first preference - response + // should select the second. + $response = $this->callService( + "reports/library/occurrences", + ['proj_id' => $projDef['id']], + NULL, + ['Accept: image/png, application/json'] + ); $decoded = json_decode($response['response'], TRUE); - $this->assertNotEquals(NULL, $decoded, 'JSON response could not be decoded: ' . $response['response']); + $this->assertNotEquals( + NULL, $decoded, + 'JSON response could not be decoded: ' . $response['response'] + ); $this->assertEquals(200, $response['httpCode']); $this->assertEquals(0, $response['curlErrno']); } /** - * Tests authentication against a resource, by passing incorrect user or secret, then - * finally passing the correct details to check a valid response returns. + * Tests authentication against a resource, by passing incorrect user or + * secret, then finally passing the correct details to check a valid response + * returns. * * @param $resource * @param string $user @@ -2210,11 +2339,16 @@ private function checkResourceAuthentication($resource, $query = []) { self::$userPassword = '---'; $response = $this->callService($resource, $query, TRUE); - $this->assertEquals(401, $response['httpCode'], - "Incorrect secret or password passed to /$resource but request authorised. Http response $response[httpCode]."); - $this->assertEquals('Unauthorized', $response['response']['status'], - "Incorrect secret or password passed to /$resource but data still returned. ". - var_export($response, true)); + $this->assertEquals( + 401, $response['httpCode'], + "Incorrect secret or password passed to /$resource but request " . + "authorised. Http response $response[httpCode]." + ); + $this->assertEquals( + 'Unauthorized', $response['response']['status'], + "Incorrect secret or password passed to /$resource but data still returned. " . + var_export($response, TRUE) + ); self::$config['shared_secret'] = $correctClientSecret; self::$websitePassword = $correctWebsitePassword; self::$userPassword = $correctUserPassword; @@ -2227,10 +2361,16 @@ private function checkResourceAuthentication($resource, $query = []) { self::$websitePassword = $correctWebsitePassword; self::$userPassword = $correctUserPassword; $response = $this->callService($resource, $query, TRUE); - $this->assertEquals(401, $response['httpCode'], - "Incorrect userId passed to /$resource but request authorised. Http response $response[httpCode]."); - $this->assertEquals('Unauthorized', $response['response']['status'], - "Incorrect userId passed to /$resource but data still returned. " . var_export($response, true)); + $this->assertEquals( + 401, $response['httpCode'], + "Incorrect userId passed to /$resource but request authorised. Http " . + "response $response[httpCode]." + ); + $this->assertEquals( + 'Unauthorized', $response['response']['status'], + "Incorrect userId passed to /$resource but data still returned. " . + var_export($response, TRUE) + ); // Now test with everything correct. self::$clientUserId = $correctClientUserId; @@ -2266,14 +2406,14 @@ private function assertResponseOk($response, $apiCall) { * @param $data Array to be tested as a taxon occurrence resource */ private function checkValidTaxonObservation($data) { - $this->assertIsArray($data, 'Taxon-observation object invalid. ' . var_export($data, true)); - $mustHave = array('id', 'href', 'datasetName', 'taxonVersionKey', 'taxonName', - 'startDate', 'endDate', 'dateType', 'projection', 'precision', 'recorder', 'lastEditDate'); + $this->assertIsArray($data, 'Taxon-observation object invalid. ' . var_export($data, TRUE)); + $mustHave = ['id', 'href', 'datasetName', 'taxonVersionKey', 'taxonName', + 'startDate', 'endDate', 'dateType', 'projection', 'precision', 'recorder', 'lastEditDate']; foreach ($mustHave as $key) { $this->assertArrayHasKey($key, $data, - "Missing $key from taxon-observation resource. " . var_export($data, true)); + "Missing $key from taxon-observation resource. " . var_export($data, TRUE)); $this->assertNotEmpty($data[$key], - "Empty $key in taxon-observation resource" . var_export($data, true)); + "Empty $key in taxon-observation resource" . var_export($data, TRUE)); } // @todo Format tests } @@ -2283,19 +2423,27 @@ private function checkValidTaxonObservation($data) { * @param $data Array to be tested as an annotation resource */ private function checkValidAnnotation($data) { - $this->assertIsArray($data, 'Annotation object invalid. ' . var_export($data, true)); - $mustHave = array('id', 'href', 'taxonObservation', 'taxonVersionKey', 'comment', - 'question', 'authorName', 'dateTime'); + $this->assertIsArray($data, 'Annotation object invalid. ' . var_export($data, TRUE)); + $mustHave = ['id', 'href', 'taxonObservation', 'taxonVersionKey', 'comment', + 'question', 'authorName', 'dateTime']; foreach ($mustHave as $key) { $this->assertArrayHasKey($key, $data, - "Missing $key from annotation resource. " . var_export($data, true)); + "Missing $key from annotation resource. " . var_export($data, TRUE)); $this->assertNotEmpty($data[$key], - "Empty $key in annotation resource" . var_export($data, true)); + "Empty $key in annotation resource" . var_export($data, TRUE)); + } + if (!empty($data['statusCode1'])) { + $this->assertMatchesRegularExpression( + '/[AUN]/', $data['statusCode1'], + 'Invalid statusCode1 value for annotation' + ); + } + if (!empty($data['statusCode2'])) { + $this->assertMatchesRegularExpression( + '/[1-6]/', $data['statusCode2'], + 'Invalid statusCode2 value for annotation' + ); } - if (!empty($data['statusCode1'])) - $this->assertMatchesRegularExpression('/[AUN]/', $data['statusCode1'], 'Invalid statusCode1 value for annotation'); - if (!empty($data['statusCode2'])) - $this->assertMatchesRegularExpression('/[1-6]/', $data['statusCode2'], 'Invalid statusCode2 value for annotation'); // We should be able to request the taxon observation associated with the occurrence $session = $this->initCurl($data['taxonObservation']['href']); $response = $this->getCurlResponse($session); @@ -2317,7 +2465,7 @@ private function checkReportFolderInReponse($response, $folder) { /** * Assert that a folder exists in the response from a call to /reports. * @param array $response - * @param string $folder + * @param string $reportFile */ private function checkReportInReponse($response, $reportFile) { $this->assertArrayHasKey($reportFile, $response); @@ -2331,6 +2479,7 @@ private function checkReportInReponse($response, $reportFile) { * * @param $session * @param $url + * @param additionalRequestHeader */ private function setRequestHeader($session, $url, $additionalRequestHeader = []) { switch ($this->authMethod) { @@ -2476,6 +2625,9 @@ private function callUrl($url, $postData = NULL, $additionalRequestHeader = [], * @param $method * @param mixed|FALSE $query * @param string $postData + * @param $additionalRequestHeader + * @param $customMethod + * @param $files * @return array */ private function callService($method, $query = FALSE, $postData = NULL, $additionalRequestHeader = [], $customMethod = NULL, $files = FALSE) { diff --git a/modules/rest_api_sync/config/rest_api_sync.example.php b/modules/rest_api_sync/config/rest_api_sync.example.php index c8bd3ea45b..59d39f49ba 100644 --- a/modules/rest_api_sync/config/rest_api_sync.example.php +++ b/modules/rest_api_sync/config/rest_api_sync.example.php @@ -47,7 +47,7 @@ 'website_id' => 5, // Remote API URL. 'url' => 'http://localhost/indicia/index.php/services/rest', - // Remote platform name, iNaturalist or Indicia. + // Remote platform name, iNaturalist, Indicia (=REST API), json_occurrences (=JSON sync format). 'serverType' => 'Indicia', // Secret shared with the remote API. 'shared_secret' => '123password', diff --git a/modules/rest_api_sync/controllers/rest_api_sync.php b/modules/rest_api_sync/controllers/rest_api_sync.php index 9fe2486235..8761463e1b 100644 --- a/modules/rest_api_sync/controllers/rest_api_sync.php +++ b/modules/rest_api_sync/controllers/rest_api_sync.php @@ -75,16 +75,16 @@ public function end() { */ public function process_batch() { $this->auto_render = FALSE; - rest_api_sync::$clientUserId = Kohana::config('rest_api_sync.user_id'); + rest_api_sync_utils::$clientUserId = Kohana::config('rest_api_sync.user_id'); $servers = Kohana::config('rest_api_sync.servers'); $serverIdx = empty($_GET['serverIdx']) ? 1 : $_GET['serverIdx']; $page = empty($_GET['page']) ? 1 : $_GET['page']; $serverId = array_keys($servers)[$serverIdx - 1]; $server = array_merge([ - 'serverType' => 'Indicia', + 'serverType' => 'Indicia', 'allowUpdateWhenVerified' => TRUE, ], $servers[$serverId]); - $helperClass = 'rest_api_sync_' . strtolower($server['serverType']); + $helperClass = 'rest_api_sync_remote_' . strtolower($server['serverType']); $helperClass::loadControlledTerms($serverId, $server); // For performance, just notify work_queue to update cache entries. if (class_exists('cache_builder')) { @@ -101,7 +101,7 @@ public function process_batch() { if ($serverIdx > count($servers)) { echo json_encode([ 'state' => 'done', - 'log' => rest_api_sync::$log, + 'log' => rest_api_sync_utils::$log, ]); } else { @@ -109,9 +109,10 @@ public function process_batch() { 'state' => 'in progress', 'serverIdx' => $serverIdx, 'page' => $page, - 'log' => rest_api_sync::$log, + 'log' => rest_api_sync_utils::$log, 'pagesToGo' => $progressInfo['pagesToGo'], 'recordsToGo' => $progressInfo['recordsToGo'], + 'moreToDo' => $progressInfo['moreToDo'], ]; echo json_encode($r); } diff --git a/modules/rest_api_sync/controllers/services/rest_api_sync.php b/modules/rest_api_sync/controllers/services/rest_api_sync.php index 709ba98595..e5900a917e 100644 --- a/modules/rest_api_sync/controllers/services/rest_api_sync.php +++ b/modules/rest_api_sync/controllers/services/rest_api_sync.php @@ -38,7 +38,7 @@ public function index() { kohana::log('debug', 'Initiating REST API Sync'); echo "

REST API Sync

"; $servers = Kohana::config('rest_api_sync.servers'); - rest_api_sync::$clientUserId = Kohana::config('rest_api_sync.user_id'); + rest_api_sync_utils::$clientUserId = Kohana::config('rest_api_sync.user_id'); // For performance, just notify work_queue to update cache entries. if (class_exists('cache_builder')) { cache_builder::$delayCacheUpdates = TRUE; @@ -46,10 +46,10 @@ public function index() { foreach ($servers as $serverId => $server) { echo "

$serverId

"; $server = array_merge([ - 'serverType' => 'Indicia', + 'serverType' => 'Indicia', 'allowUpdateWhenVerified' => TRUE, ], $server); - $helperClass = 'rest_api_sync_' . strtolower($server['serverType']); + $helperClass = 'rest_api_sync_remote_' . strtolower($server['serverType']); $helperClass::loadControlledTerms($serverId, $server); $helperClass::syncServer($serverId, $server); } diff --git a/modules/rest_api_sync/helpers/api_persist.php b/modules/rest_api_sync/helpers/api_persist.php index f4cffd8a22..e651e23980 100644 --- a/modules/rest_api_sync/helpers/api_persist.php +++ b/modules/rest_api_sync/helpers/api_persist.php @@ -40,6 +40,45 @@ class api_persist { private static $mediaTypes = []; + /** + * Capture information about Darwin Core linked attributes in the survey. + * + * @var array + */ + private static $dwcAttributes = []; + + /** + * Finds attributes in the survey which have a DwC term name. + * + * Allows the code to post data into these attributes without continually + * looking them up. + */ + public static function initDwcAttributes($db, $surveyId) { + self::$dwcAttributes = [ + 'occAttrs' => self::fetchDwcAttrs($db, 'occurrence', $surveyId), + 'smpAttrs' => self::fetchDwcAttrs($db, 'sample', $surveyId), + ]; + } + + private static function fetchDwcAttrs($db, $type, $surveyId) { + // List of DwC terms that we might process values for. + $attrs = $db->select('a.id, a.term_name') + ->from("{$type}_attributes as a") + ->join("{$type}_attributes_websites as aw", "aw.{$type}_attribute_id", 'a.id') + ->where([ + 'aw.restrict_to_survey_id' => $surveyId, + 'a.deleted' => 'f', + 'aw.deleted' => 'f', + ]) + ->where('a.term_name IS NOT NULL') + ->get(); + $r = []; + foreach ($attrs as $attr) { + $r[$attr->term_name] = ($type === 'sample' ? 'smp' : 'occ') . "Attr:$attr->id"; + } + return $r; + } + /** * Persists a taxon-observation resource. * @@ -64,7 +103,10 @@ class api_persist { * @throws \exception */ public static function taxonObservation($db, array $observation, $website_id, $survey_id, $taxon_list_id, $allowUpdateWhenVerified) { - if (!empty($observation['taxonVersionKey'])) { + if (!empty($observation['organismKey'])) { + $lookup = ['organism_key' => $observation['organismKey']]; + } + elseif (!empty($observation['taxonVersionKey'])) { $lookup = ['search_code' => $observation['taxonVersionKey']]; } elseif (!empty($observation['taxonName'])) { @@ -94,7 +136,6 @@ public static function taxonObservation($db, array $observation, $website_id, $s // Set the spatial reference depending on the projection information // supplied. self::setSrefData($values, $observation, 'sample:entered_sref'); - self::setCoordinateUncertainty($db, $survey_id, $values, $observation); // Site handling. If a known site with a SiteKey, we can create a record in // locations, otherwise use the free text location_name field. @@ -113,6 +154,27 @@ public static function taxonObservation($db, array $observation, $website_id, $s return count($existing) === 0; } + /** + * Copies a "standard" Dwc attribute from the observation to the values. + * + * The DwC attribute will be linked to a custom attribute where the term_name + * identifies the DwC term. + * + * @param array $observation + * Provided observation values. + * @param array $values + * Submission values which will be updated with the value. + * @param string $type + * Attribute type, smp or occ. + * @param string $term + * The DwC term. + */ + private static function copyDwcAttribute(array $observation, array &$values, $type, $term) { + if (!empty($observation[$term]) && !empty(self::$dwcAttributes["{$type}Attrs"][$term])) { + $values[self::$dwcAttributes["{$type}Attrs"][$term]] = $observation[$term]; + } + } + /** * Persists an annotation resource. * @@ -131,22 +193,22 @@ public static function annotation($db, array $annotation, $survey_id) { // Set up a values array for the annotation post. $values = self::getAnnotationValues($db, $annotation); // Link to the existing observation. - $existingObs = self::findExistingObservation($db, $annotation['taxonObservation']['id'], $survey_id); + $existingObs = self::findExistingObservation($db, $annotation['occurrenceID'], $survey_id); if (!count($existingObs)) { // @todo Proper error handling as annotation can't be imported. Perhaps should obtain // and import the observation via the API? throw new exception("Attempt to import annotation $annotation[id] but taxon observation not found."); } - $values['occurrence_comment:occurrence_id'] = $existingObs[0]['id']; + $values['occurrence_id'] = $existingObs[0]['id']; // Link to existing annotation if appropriate. $existing = self::findExistingAnnotation($db, $annotation['id'], $existingObs[0]['id']); if ($existing) { - $values['occurrence_comment:id'] = $existing[0]['id']; + $values['id'] = $existing[0]['id']; } $annotationObj = ORM::factory('occurrence_comment'); $annotationObj->set_submission_data($values); $annotationObj->submit(); - self::updateObservationWithAnnotationDetails($db, $existingObs[0]['id'], $annotation); + self::updateObservationWithAnnotationDetails($db, $existingObs[0]['id'], $annotation, $values); if (count($annotationObj->getAllErrors()) !== 0) { throw new exception("Error occurred submitting an annotation\n" . kohana::debug($annotationObj->getAllErrors())); } @@ -175,20 +237,24 @@ public static function annotation($db, array $annotation, $survey_id) { */ private static function getTaxonObservationValues($db, $website_id, array $observation, $ttl_id) { $sensitive = isset($observation['sensitive']) && strtolower($observation['sensitive']) === 't'; - $values = array( + $values = [ 'website_id' => $website_id, 'sample:date_start' => $observation['startDate'], 'sample:date_end' => $observation['endDate'], 'sample:date_type' => $observation['dateType'], - 'sample:recorder_names' => isset($observation['recorder']) ? $observation['recorder'] : 'Unknown', + 'sample:recorder_names' => isset($observation['recordedBy']) ? $observation['recordedBy'] : 'Unknown', 'occurrence:taxa_taxon_list_id' => $ttl_id, 'occurrence:external_key' => $observation['id'], 'occurrence:zero_abundance' => isset($observation['zeroAbundance']) ? strtolower($observation['zeroAbundance']) : 'f', 'occurrence:sensitivity_precision' => $sensitive ? 10000 : NULL, - ); + 'occurrence:verifier_only_data' => isset($observation['verifierOnlyData']) ? $observation['verifierOnlyData'] : NULL, + ]; if (!empty($observation['licenceCode'])) { $values['sample:licence_id'] = self::getLicenceIdFromCode($db, $observation['licenceCode']); } + if (!empty($observation['occurrenceRemarks'])) { + $values['occurrence:comment'] = $observation['occurrenceRemarks']; + } if (!empty($observation['media'])) { foreach ($observation['media'] as $idx => $medium) { $values["occurrence_medium:path:$idx"] = $medium['path']; @@ -205,9 +271,49 @@ private static function getTaxonObservationValues($db, $website_id, array $obser $values["occAttr:$id"] = $value; } } + if (!empty($observation['eventId'])) { + $values['sample:external_key'] = $observation['eventId']; + } + if (!empty($observation['eventRemarks'])) { + $values['sample:comment'] = $observation['eventRemarks']; + } + if (!empty($observation['identificationVerificationStatus'])) { + self::applyIdentificationVerificationStatus($observation['identificationVerificationStatus'], $values); + } + self::copyDwcAttribute($observation, $values, 'smp', 'coordinateUncertaintyInMeters'); + self::copyDwcAttribute($observation, $values, 'smp', 'collectionCode'); + self::copyDwcAttribute($observation, $values, 'occ', 'individualCount'); + self::copyDwcAttribute($observation, $values, 'occ', 'lifeStage'); + self::copyDwcAttribute($observation, $values, 'occ', 'reproductiveCondition'); + self::copyDwcAttribute($observation, $values, 'occ', 'sex'); + self::copyDwcAttribute($observation, $values, 'occ', 'identifiedBy'); + self::copyDwcAttribute($observation, $values, 'occ', 'identificationRemarks'); return $values; } + private static function applyIdentificationVerificationStatus($identificationVerificationStatus, array &$values) { + $mappings = [ + 'accepted' => ['V', NULL], + 'accepted - correct' => ['V', 1], + 'accepted - considered correct' => ['V', 2], + 'unconfirmed' => ['C', NULL], + 'unconfirmed - plausible' => ['C', 3], + 'unconfirmed - not reviewed' => ['C', NULL], + 'not accepted' => ['R', NULL], + 'not accepted - unable to verify' => ['R', 4], + 'not accepted - incorrect' => ['R', 5], + ]; + if (isset($mappings[strtolower($identificationVerificationStatus)])) { + $statuses = $mappings[strtolower($identificationVerificationStatus)]; + kohana::log('debug', strtolower($identificationVerificationStatus) . ': ' . var_export($statuses, TRUE)); + $values['occurrence:record_status'] = $statuses[0]; + $values['occurrence:record_substatus'] = $statuses[1]; + } + else { + throw new exception("Invalid identificationVerificationStatus value: $identificationVerificationStatus"); + } + } + private static function mapOccAttrValueToTermId($db, $occAttrId, &$value) { $cacheId = "occAttrIsLookup-$occAttrId"; $cache = Cache::instance(); @@ -292,15 +398,22 @@ private static function getMediaTypeId($db, $mediaType) { * Values array to use for submission building. */ private static function getAnnotationValues($db, array $annotation) { - return array( - 'occurrence_comment:comment' => $annotation['comment'], - 'occurrence_comment:email_address' => self::valueOrNull($annotation, 'emailAddress'), - 'occurrence_comment:record_status' => self::valueOrNull($annotation, 'record_status'), - 'occurrence_comment:record_substatus' => self::valueOrNull($annotation, 'record_substatus'), - 'occurrence_comment:query' => $annotation['question'], - 'occurrence_comment:person_name' => $annotation['authorName'], - 'occurrence_comment:external_key' => $annotation['id'], - ); + $values = [ + 'comment' => $annotation['comment'], + 'email_address' => self::valueOrNull($annotation, 'emailAddress'), + 'record_status' => self::valueOrNull($annotation, 'record_status'), + 'record_substatus' => self::valueOrNull($annotation, 'record_substatus'), + 'query' => $annotation['question'], + 'person_name' => $annotation['authorName'], + 'external_key' => $annotation['id'], + ]; + if (!empty($annotation['dateTime'])) { + $values['updated_on'] = $annotation['dateTime']; + } + if (!empty($annotation['identificationVerificationStatus'])) { + self::applyIdentificationVerificationStatus($annotation['identificationVerificationStatus'], $values); + } + return $values; } /** @@ -323,10 +436,10 @@ private static function getLicenceIdFromCode($db, $licenceCode) { $licenceData = $db->query($qry)->result_array(FALSE); self::$licences = []; foreach ($licenceData as $licence) { - self::$licences[strtolower($licence['code'])] = $licence['id']; + self::$licences[strtolower(str_replace(' ', '-', $licence['code']))] = $licence['id']; } } - return self::$licences[$licenceCode]; + return self::$licences[strtolower(str_replace(' ', '-', $licenceCode))]; } /** @@ -360,14 +473,7 @@ private static function findTaxon($db, $taxon_list_id, array $lookup) { $exactMatches[] = "'" . pg_escape_string("$words[0] $words[1] subsp. $words[2]") . "'"; } $filter .= 'AND original in (' . implode(',', $exactMatches) . ")\n"; - } - else { - // Add in the exact match filter for other search methods. - foreach ($lookup as $key => $value) { - $filter .= "AND $key='$value'\n"; - } - } - $qry = << $value) { + $filter .= "AND t.$key='$value'\n"; + } + $qry = <<query($qry)->result_array(FALSE); // Need to know if the search found a single unique taxon concept so count // the taxon meanings. @@ -407,7 +530,7 @@ private static function findTaxon($db, $taxon_list_id, array $lookup) { * @throws \exception */ private static function checkMandatoryFields(array $array, $resourceName) { - $required = array(); + $required = []; // Deletions have no other mandatory fields except the id to delete. if (!empty($resource['delete']) && $resource['delete'] === 'T') { $array[] = 'id'; @@ -415,16 +538,15 @@ private static function checkMandatoryFields(array $array, $resourceName) { else { switch ($resourceName) { case 'taxon-observation': - $required = array( + $required = [ 'id', - 'href', /*'taxonName', */ 'startDate', 'endDate', 'dateType', 'projection', - 'precision', - 'recorder', - ); + 'coordinateUncertaintyInMeters', + 'recordedBy', + ]; // Conditionally required fields. if (empty($array['gridReference'])) { $required[] = 'east'; @@ -433,7 +555,10 @@ private static function checkMandatoryFields(array $array, $resourceName) { $required[] = 'gridReference'; } } - $required[] = empty($array['taxonVersionKey']) ? 'taxonName' : 'taxonVersionKey'; + // One of taxonVersionKey, organismKey or taxonName required. + if (empty($array['taxonVersionKey']) && empty($array['organismKey'])) { + $required[] = 'taxonName'; + } break; case 'annotation': @@ -595,35 +720,6 @@ private static function setSrefData(array &$values, array $observation, $fieldna } } - /** - * Sets the coordinate uncertainty attribute for an imported observation. - * - * @param object $db - * Database connection. - * @param int $survey_id - * ID of the survey (the sref_precision attribute should be linked to this). - * @param array $values - * The values array to add the precision information to. - * @param array $observation - * The observation data array. - */ - private static function setCoordinateUncertainty($db, $survey_id, array &$values, array $observation) { - if (!empty($observation['precision'])) { - $attr = $db->select('a.id') - ->from('sample_attributes as a') - ->join('sample_attributes_websites as aw', 'aw.sample_attribute_id', 'a.id') - ->where([ - 'a.system_function' => 'sref_precision', - 'aw.restrict_to_survey_id' => $survey_id, - 'a.deleted' => 'f', - 'aw.deleted' => 'f', - ])->get()->current(); - if ($attr) { - $values["smpAttr:$attr->id"] = $observation['precision']; - } - } - } - /** * Returns a formatted decimal latitude and longitude string. * @@ -638,8 +734,10 @@ private static function setCoordinateUncertainty($db, $survey_id, array &$values private static function formatLatLong($lat, $long) { $ns = $lat >= 0 ? 'N' : 'S'; $ew = $long >= 0 ? 'E' : 'W'; - $lat = abs($lat); - $long = abs($long); + // Variant of abs() function using preg_replace avoids changing float to + // scientific notation. + $lat = preg_replace('/^-/', '', $lat); + $long = preg_replace('/^-/', '', $long); return "$lat$ns $long$ew"; } @@ -762,88 +860,53 @@ private static function mapRecordStatus(array &$annotation) { * ID of the associated occurrence record in the database. * @param array $annotation * Annotation object loaded from the REST API. + * @param array $values + * Database values found for the annotation. * * @throws exception */ - private static function updateObservationWithAnnotationDetails($db, $occurrence_id, array $annotation) { - // Find the original record to compare against. - $oldRecords = $db - ->select('record_status, record_substatus, taxa_taxon_list_id') - ->from('cache_occurrences') - ->where('id', $occurrence_id) - ->get()->result_array(FALSE); - if (!count($oldRecords)) { - throw new exception('Could not find cache_occurrences record associated with a comment.'); - } - - // Find the taxon information supplied with the comment's TVK. - $newTaxa = $db - ->select('id, taxonomic_sort_order, taxon, authority, preferred_taxon, default_common_name, search_name, ' . - 'external_key, taxon_meaning_id, taxon_group_id, taxon_group') - ->from('cache_taxa_taxon_lists') - ->where([ - 'preferred' => 't', - 'external_key' => $annotation['taxonVersionKey'], - 'taxon_list_id' => kohana::config('rest_api_sync.taxon_list_id'), - ]) - ->limit(1) - ->get()->result_array(FALSE); - if (!count($newTaxa)) { - throw new exception('Could not find cache_taxa_taxon_lists record associated with an update from a comment.'); - } - - $oldRecord = $oldRecords[0]; - $newTaxon = $newTaxa[0]; - - $new_status = $annotation['record_status'] === $oldRecord['record_status'] - ? FALSE : $annotation['record_status']; - $new_substatus = $annotation['record_substatus'] === $oldRecord['record_substatus'] - ? FALSE : $annotation['record_substatus']; - $new_ttlid = $newTaxon['id'] === $oldRecord['taxa_taxon_list_id'] - ? FALSE : $newTaxon['id']; - - // Does the comment imply an allowable change to the occurrence's attributes? - if ($new_status || $new_substatus || $new_ttlid) { - $oupdate = array('updated_on' => date("Ymd H:i:s")); - $coupdate = array('cache_updated_on' => date("Ymd H:i:s")); - if ($new_status || $new_substatus) { - $oupdate['verified_on'] = date("Ymd H:i:s"); - // @todo Verified_by_id needs to be mapped to a proper user account. - $oupdate['verified_by_id'] = 1; - $coupdate['verified_on'] = date("Ymd H:i:s"); - $coupdate['verifier'] = $annotation['authorName']; - } - if ($new_status) { - $oupdate['record_status'] = $new_status; - $coupdate['record_status'] = $new_status; - } - if ($new_substatus) { - $oupdate['record_substatus'] = $new_substatus; - $coupdate['record_substatus'] = $new_substatus; + private static function updateObservationWithAnnotationDetails($db, $occurrence_id, array $annotation, array $values) { + $update = []; + if (!empty($values['record_status'])) { + $update['record_status'] = $values['record_status']; + $update['record_substatus'] = empty($values['record_substatus']) ? NULL : $values['record_substatus']; + } + if (!empty($annotation['taxonVersionKey'])) { + // Find the taxon information supplied with the comment's TVK. + $newTaxa = $db + ->select('id') + ->from('cache_taxa_taxon_lists') + ->where([ + 'preferred' => 't', + 'external_key' => $annotation['taxonVersionKey'], + 'taxon_list_id' => kohana::config('rest_api_sync.taxon_list_id'), + ]) + ->limit(1) + ->get()->result_array(FALSE); + if (!count($newTaxa)) { + throw new exception('Could not find cache_taxa_taxon_lists record associated with an update from a comment.'); } - if ($new_ttlid) { - $oupdate['taxa_taxon_list_id'] = $new_ttlid; - $coupdate['taxa_taxon_list_id'] = $new_ttlid; - $coupdate['taxonomic_sort_order'] = $newTaxon['taxonomic_sort_order']; - $coupdate['taxon'] = $newTaxon['taxon']; - $coupdate['preferred_taxon'] = $newTaxon['preferred_taxon']; - $coupdate['authority'] = $newTaxon['authority']; - $coupdate['default_common_name'] = $newTaxon['default_common_name']; - $coupdate['search_name'] = $newTaxon['search_name']; - $coupdate['taxa_taxon_list_external_key'] = $newTaxon['external_key']; - $coupdate['taxon_meaning_id'] = $newTaxon['taxon_meaning_id']; - $coupdate['taxon_group_id'] = $newTaxon['taxon_group_id']; - $coupdate['taxon_group'] = $newTaxon['taxon_group']; + $values['taxa_taxon_list_id'] = $newTaxa[0]['id']; + } + if (count($values)) { + $occ = ORM::factory('occurrence', $occurrence_id); + foreach ($update as $field => $value) { + $occ->$field = $value; } - $db->update('occurrences', - $oupdate, - array('id' => $occurrence_id) - ); - $db->update('cache_occurrences', - $coupdate, - array('id' => $occurrence_id) - ); - // @todo create a determination if this is not automatic + $occ->set_metadata(); + // Save occurrence changes (will update cache). + $occ->save(); + } + elseif (!empty($annotation['question']) && $annotation['question'] === 't') { + // Need to separately update cache if question asked. + $q = new WorkQueue(); + $q->enqueue($db, [ + 'task' => 'task_cache_builder_update', + 'entity' => 'occurrence', + 'record_id' => $occurrence_id, + 'cost_estimate' => 50, + 'priority' => 2, + ]); } } diff --git a/modules/rest_api_sync/helpers/rest_api_sync_inaturalist.php b/modules/rest_api_sync/helpers/rest_api_sync_remote_inaturalist.php similarity index 93% rename from modules/rest_api_sync/helpers/rest_api_sync_inaturalist.php rename to modules/rest_api_sync/helpers/rest_api_sync_remote_inaturalist.php index 3723a2a211..78200be141 100644 --- a/modules/rest_api_sync/helpers/rest_api_sync_inaturalist.php +++ b/modules/rest_api_sync/helpers/rest_api_sync_remote_inaturalist.php @@ -30,7 +30,7 @@ /** * Helper class for syncing to the RESTful API on iNaturalist. */ -class rest_api_sync_inaturalist { +class rest_api_sync_remote_inaturalist { /** * Terms loaded from iNat. @@ -86,7 +86,7 @@ public static function loadControlledTerms($serverId, array $server) { $cache = Cache::instance(); self::$controlledTerms = $cache->get('inaturalist-controlled-terms'); if (!self::$controlledTerms) { - $data = rest_api_sync::getDataFromRestUrl( + $data = rest_api_sync_utils::getDataFromRestUrl( "$server[url]/controlled_terms", $serverId ); @@ -121,10 +121,11 @@ public static function loadControlledTerms($serverId, array $server) { */ public static function syncPage($serverId, array $server) { $db = Database::instance(); + api_persist::initDwcAttributes($db, $server['survey_id']); $fromDateTime = variable::get("rest_api_sync_{$serverId}_last_run", '1600-01-01T00:00:00+00:00', FALSE); $fromId = variable::get("rest_api_sync_{$serverId}_last_id", 0, FALSE); $lastId = $fromId; - $data = rest_api_sync::getDataFromRestUrl( + $data = rest_api_sync_utils::getDataFromRestUrl( "$server[url]/observations?" . http_build_query(array_merge( $server['parameters'], [ @@ -156,16 +157,15 @@ public static function syncPage($serverId, array $server) { 'startDate' => $iNatRecord['observed_on'], 'endDate' => $iNatRecord['observed_on'], 'dateType' => 'D', - 'recorder' => empty($iNatRecord['user']['name']) ? $iNatRecord['user']['login'] : $iNatRecord['user']['name'], + 'recordedBy' => empty($iNatRecord['user']['name']) ? $iNatRecord['user']['login'] : $iNatRecord['user']['name'], 'east' => $east, 'north' => $north, 'projection' => 'WGS84', - 'precision' => $iNatRecord['public_positional_accuracy'], + 'coordinateUncertaintyInMeters' => $iNatRecord['public_positional_accuracy'], 'siteName' => $iNatRecord['place_guess'], 'href' => $iNatRecord['uri'], // American English in iNat field name - sic. - // Also correct extra hyphen in iNat CC licence codes. - 'licenceCode' => preg_replace('/^cc-/', 'cc ', $iNatRecord['license_code']), + 'licenceCode' => $iNatRecord['license_code'], ]; if (!empty($iNatRecord['photos'])) { $observation['media'] = []; @@ -176,7 +176,7 @@ public static function syncPage($serverId, array $server) { 'path' => $iNatPhoto['url'], 'caption' => $iNatPhoto['attribution'], 'mediaType' => 'Image:iNaturalist', - 'licenceCode' => preg_replace('/^cc-/', 'cc ', $iNatPhoto['license_code']), + 'licenceCode' => $iNatPhoto['license_code'], ]; } } @@ -219,7 +219,7 @@ public static function syncPage($serverId, array $server) { "WHERE server_id='$serverId' AND source_id='$iNatRecord[id]' AND dest_table='occurrences'"); } catch (exception $e) { - rest_api_sync::log( + rest_api_sync_utils::log( 'error', "Error occurred submitting an occurrence with iNaturalist ID $iNatRecord[id]\n" . $e->getMessage(), $tracker @@ -251,7 +251,7 @@ public static function syncPage($serverId, array $server) { $lastId = $iNatRecord['id']; } variable::set("rest_api_sync_{$serverId}_last_id", $lastId); - rest_api_sync::log( + rest_api_sync_utils::log( 'info', "Observations
Inserts: $tracker[inserts]. Updates: $tracker[updates]. Errors: $tracker[errors]" ); diff --git a/modules/rest_api_sync/helpers/rest_api_sync_indicia.php b/modules/rest_api_sync/helpers/rest_api_sync_remote_indicia.php similarity index 94% rename from modules/rest_api_sync/helpers/rest_api_sync_indicia.php rename to modules/rest_api_sync/helpers/rest_api_sync_remote_indicia.php index fce524755b..d85060e23d 100644 --- a/modules/rest_api_sync/helpers/rest_api_sync_indicia.php +++ b/modules/rest_api_sync/helpers/rest_api_sync_remote_indicia.php @@ -27,7 +27,7 @@ /** * Helper class for syncing to the RESTful API on an Indicia warehouses. */ -class rest_api_sync_indicia { +class rest_api_sync_remote_indicia { /** * ISO datetime that the sync was last run. @@ -79,7 +79,7 @@ public static function syncServer($serverId, array $server) { while ($next_page_of_projects_url) { $response = self::getServerProjects($next_page_of_projects_url, $serverId); if (!isset($response['data'])) { - rest_api_sync::log('error', "Invalid response\nURL: $next_page_of_projects_url\nResponse did not include data element."); + rest_api_sync_utils::log('error', "Invalid response\nURL: $next_page_of_projects_url\nResponse did not include data element."); var_export($response); continue; } @@ -101,6 +101,9 @@ public static function syncServer($serverId, array $server) { } } + public static function loadControlledTerms() { + } + /** * Currently just a stub function to treat the whole Indicia sync as a page. * @@ -234,7 +237,7 @@ private static function syncTaxonObservations(array $server, $serverId, array $p while ($next_page_of_taxon_observations_url && ($load_all || $processedCount < MAX_RECORDS_TO_PROCESS)) { $data = self::getServerTaxonObservations($next_page_of_taxon_observations_url, $serverId); $observations = $data['data']; - rest_api_sync::log('debug', count($observations) . ' records found'); + rest_api_sync_utils::log('debug', count($observations) . ' records found'); foreach ($observations as $observation) { // If the record was originated from a different system, the specified // dataset name needs to be stored. @@ -256,7 +259,7 @@ private static function syncTaxonObservations(array $server, $serverId, array $p } } catch (exception $e) { - rest_api_sync::log('error', "Error occurred submitting an occurrence\n" . $e->getMessage() . "\n" . + rest_api_sync_utils::log('error', "Error occurred submitting an occurrence\n" . $e->getMessage() . "\n" . json_encode($observation), $tracker); } if ($last_record_date && $last_record_date <> $observation['lastEditDate']) { @@ -310,14 +313,14 @@ private static function syncAnnotations(array $server, $serverId, array $project while ($nextPageOfAnnotationsUrl && ($load_all || $processedCount < MAX_RECORDS_TO_PROCESS)) { $data = self::getServerAnnotations($nextPageOfAnnotationsUrl, $serverId); $annotations = $data['data']; - rest_api_sync::log('debug', count($annotations) . ' records found'); + rest_api_sync_utils::log('debug', count($annotations) . ' records found'); foreach ($annotations as $annotation) { try { $is_new = api_persist::annotation(self::$db, $annotation, $survey_id); $tracker[$is_new ? 'inserts' : 'updates']++; } catch (exception $e) { - rest_api_sync::log('error', "Error occurred submitting an annotation\n" . $e->getMessage() . "\n" . + rest_api_sync_utils::log('error', "Error occurred submitting an annotation\n" . $e->getMessage() . "\n" . json_encode($annotation), $tracker); } if ($last_record_date && $last_record_date <> $annotation['lastEditDate']) { @@ -364,15 +367,15 @@ private static function getServerProjectsUrl($server_url) { } public static function getServerProjects($url, $serverId) { - return rest_api_sync::getDataFromRestUrl($url, $serverId); + return rest_api_sync_utils::getDataFromRestUrl($url, $serverId); } public static function getServerTaxonObservations($url, $serverId) { - return rest_api_sync::getDataFromRestUrl($url, $serverId); + return rest_api_sync_utils::getDataFromRestUrl($url, $serverId); } public static function getServerAnnotations($url, $serverId) { - return rest_api_sync::getDataFromRestUrl($url, $serverId); + return rest_api_sync_utils::getDataFromRestUrl($url, $serverId); } } diff --git a/modules/rest_api_sync/helpers/rest_api_sync_remote_json_annotations.php b/modules/rest_api_sync/helpers/rest_api_sync_remote_json_annotations.php new file mode 100644 index 0000000000..a8130c7440 --- /dev/null +++ b/modules/rest_api_sync/helpers/rest_api_sync_remote_json_annotations.php @@ -0,0 +1,157 @@ + 0, 'updates' => 0, 'errors' => 0]; + foreach ($data['data'] as $record) { + // @todo Make sure all fields in specification are handled. + try { + $annotation = [ + 'id' => $record['annotationID'], + 'occurrenceID' => $record['occurrenceID'], + 'comment' => empty($record['comment']) ? 'No comment provided' : $record['comment'], + 'identificationVerificationStatus' => empty($record['identificationVerificationStatus']) ? NULL : $record['identificationVerificationStatus'], + 'question' => empty($record['question']) ? NULL : $record['question'], + 'authorName' => empty($record['authorName']) ? 'Unknown' : $record['authorName'], + 'dateTime' => $record['dateTime'], + ]; + + $is_new = api_persist::annotation( + $db, + $annotation, + $server['survey_id'] + ); + if ($is_new !== NULL) { + $tracker[$is_new ? 'inserts' : 'updates']++; + } + $db->query("UPDATE rest_api_sync_skipped_records SET current=false " . + "WHERE server_id='$serverId' AND source_id='$annotation[id]' AND dest_table='occurrence_comments'"); + } + catch (exception $e) { + rest_api_sync_utils::log( + 'error', + "Error occurred submitting an annotation with ID $annotation[id]\n" . $e->getMessage(), + $tracker + ); + $msg = pg_escape_string($e->getMessage()); + $createdById = isset($_SESSION['auth_user']) ? $_SESSION['auth_user']->id : 1; + $sql = <<query($sql); + } + } + variable::set("rest_api_sync_{$serverId}_next_page", $data['paging']['next']); + rest_api_sync_utils::log( + 'info', + "Annotations
Inserts: $tracker[inserts]. Updates: $tracker[updates]. Errors: $tracker[errors]" + ); + $r = [ + 'moreToDo' => count($data['data']) > 0, + // No way of determining the following. + 'pagesToGo' => NULL, + 'recordsToGo' => NULL, + ]; + return $r; + } + +} diff --git a/modules/rest_api_sync/helpers/rest_api_sync_remote_json_occurrences.php b/modules/rest_api_sync/helpers/rest_api_sync_remote_json_occurrences.php new file mode 100644 index 0000000000..f7d971da91 --- /dev/null +++ b/modules/rest_api_sync/helpers/rest_api_sync_remote_json_occurrences.php @@ -0,0 +1,286 @@ + 0, 'updates' => 0, 'errors' => 0]; + foreach ($data['data'] as $record) { + // @todo Make sure all fields in specification are handled + // @todo occurrence.associated_media + // @todo occurrence.occurrence_status + // @todo occurrence.organism_quantity + // @todo occurrence.organism_quantity_type + // @todo occurrence.otherCatalogNumbers + // @todo event.samplingProtocol + // @todo location.geodeticDatum + $parsedDate = self::parseDates($record['event']['eventDate']); + try { + $observation = [ + 'licenceCode' => empty($record['record-level']['license']) ? NULL : $record['record-level']['license'], + 'collectionCode' => empty($record['record-level']['collectionCode']) ? NULL : $record['record-level']['collectionCode'], + 'id' => $record['occurrence']['occurrenceID'], + 'individualCount' => empty($record['occurrence']['individualCount']) ? NULL : $record['occurrence']['individualCount'], + 'lifeStage' => empty($record['occurrence']['lifeStage']) ? NULL : $record['occurrence']['lifeStage'], + 'recordedBy' => $record['occurrence']['recordedBy'], + 'occurrenceRemarks' => empty($record['occurrence']['occurrenceRemarks']) ? NULL : $record['occurrence']['occurrenceRemarks'], + 'reproductiveCondition' => empty($record['occurrence']['reproductiveCondition']) ? NULL : $record['occurrence']['reproductiveCondition'], + 'sex' => empty($record['occurrence']['sex']) ? NULL : $record['occurrence']['sex'], + 'sensitivityPrecision' => empty($record['occurrence']['sensitivityBlur']) ? NULL : $record['occurrence']['sensitivityBlur'], + 'organismKey' => $record['taxon']['taxonID'], + 'taxonVersionKey' => empty($record['taxon']['taxonNameID']) ? NULL : $record['taxon']['taxonNameID'], + 'eventId' => empty($record['event']['eventId']) ? NULL : $record['event']['eventId'], + 'startDate' => $parsedDate['start'], + 'endDate' => $parsedDate['end'], + 'dateType' => $parsedDate['type'], + 'samplingProtocol' => empty($record['event']['samplingProtocol']) ? NULL : $record['event']['samplingProtocol'], + 'coordinateUncertaintyInMeters' => empty($record['location']['coordinateUncertaintyInMeters']) ? NULL : $record['location']['coordinateUncertaintyInMeters'], + 'siteName' => empty($record['location']['locality']) ? NULL : $record['location']['locality'], + 'identifiedBy' => empty($record['identification']['identifiedBy']) ? NULL : $record['identification']['identifiedBy'], + 'identificationVerificationStatus' => empty($record['identification']['identificationVerificationStatus']) ? NULL : $record['identification']['identificationVerificationStatus'], + 'identificationRemarks' => empty($record['identification']['identificationRemarks']) ? NULL : $record['identification']['identificationRemarks'], + ]; + if (!empty($record['location']['decimalLongitude']) && !empty($record['location']['decimalLatitude'])) { + // Json_decode() converts some floats to scientific notation, so + // reverse that. + $observation['east'] = rtrim(number_format($record['location']['decimalLongitude'], 12), 0); + $observation['north'] = rtrim(number_format($record['location']['decimalLatitude'], 12), 0); + $observation['projection'] = 'WGS84'; + } + elseif (!empty($record['location']['gridReference'])) { + $observation['gridReference'] = strtoupper(str_replace(' ', '', $record['location']['gridReference'])); + if (preg_match('/^I?[A-Z]\d*[A-NP-Z]?$/', $observation['gridReference'])) { + $observation['projection'] = 'OSI'; + $observation['gridReference'] = preg_replace('/^I/', '', $observation['gridReference']); + } + elseif (preg_match('/^[A-Z][A-Z]\d*[A-NP-Z]?$/', $observation['gridReference'])) { + $observation['projection'] = 'OSGB'; + } + else { + throw new exception('Invalid grid reference format: ' . $record['location']['gridReference']); + } + } + if (!empty($record['record-level']['dynamicProperties']) && !empty($record['record-level']['dynamicProperties']['verifierOnlyData'])) { + $observation['verifierOnlyData'] = json_encode($record['record-level']['dynamicProperties']['verifierOnlyData']); + unset($record['record-level']['dynamicProperties']['verifierOnlyData']); + } + self::processOtherFields($server, $record, $observation); + $is_new = api_persist::taxonObservation( + $db, + $observation, + $server['website_id'], + $server['survey_id'], + $taxon_list_id, + $server['allowUpdateWhenVerified'] + ); + if ($is_new !== NULL) { + $tracker[$is_new ? 'inserts' : 'updates']++; + } + $db->query("UPDATE rest_api_sync_skipped_records SET current=false " . + "WHERE server_id='$serverId' AND source_id='{$record['occurrence']['occurrenceID']}' AND dest_table='occurrences'"); + } + catch (exception $e) { + rest_api_sync_utils::log( + 'error', + "Error occurred submitting an occurrence with ID {$record['occurrence']['occurrenceID']}\n" . $e->getMessage(), + $tracker + ); + $msg = pg_escape_string($e->getMessage()); + $createdById = isset($_SESSION['auth_user']) ? $_SESSION['auth_user']->id : 1; + $sql = <<query($sql); + } + } + variable::set("rest_api_sync_{$serverId}_next_page", $data['paging']['next']); + rest_api_sync_utils::log( + 'info', + "Observations
Inserts: $tracker[inserts]. Updates: $tracker[updates]. Errors: $tracker[errors]" + ); + $r = [ + 'moreToDo' => count($data['data']) > 0, + // No way of determining the following. + 'pagesToGo' => NULL, + 'recordsToGo' => NULL, + ]; + return $r; + } + + /** + * Process any other field mappings defined by the server config. + * + * @param array $server + * Server configuration. + * @param array $record + * Record structure supplied by the remote server. + * @param array $observation + * Observation values to store in Indicia. Will be updated as appropriate. + */ + private static function processOtherFields(array $server, array $record, array &$observation) { + if (!empty($server['otherFields'])) { + foreach ($server['otherFields'] as $src => $dest) { + $path = explode('.', $src); + $posInDoc = $record; + $found = TRUE; + foreach ($path as $node) { + if (!isset($posInDoc[$node])) { + $found = FALSE; + break; + } + $posInDoc = $posInDoc[$node]; + } + if ($found) { + // @todo Check multi-value/array handling. + $attrTokens = explode(':', $dest); + if (is_object($posInDoc) || is_array($posInDoc)) { + $posInDoc = json_encode($posInDoc); + } + $observation[$attrTokens[0] . 's'][$attrTokens[1]] = $posInDoc; + } + } + } + } + + /** + * Parses a date string into the start, end and type. + * + * @param string $dateString + * Single date (yyyy-mm-dd), range of dates (yyyy-mm-dd/yyyy-mm-dd), month + * (yyyy-mm) or year (yyyy). + * + * @return array + * Array containing vague date parts, start, end and type. + */ + private static function parseDates($dateString) { + if (preg_match('/^\d{4}-\d{2}-\d{2}$/', $dateString)) { + return [ + 'start' => $dateString, + 'end' => $dateString, + 'type' => 'D', + ]; + } + elseif (preg_match('/^(?P\d{4}-\d{2}-\d{2})\|(?P\d{4}-\d{2}-\d{2})$/', $dateString, $matches)) { + return [ + 'start' => $matches['start'], + 'end' => $matches['end'], + 'type' => 'DD', + ]; + } + elseif (preg_match('/^(?P\d{4})-(?P\d{2})$/', $dateString)) { + return [ + 'start' => "$dateString-01", + 'end' => "$dateString-" . cal_days_in_month(CAL_GREGORIAN, $matches['month'], $matches['year']), + 'type' => 'O', + ]; + } + elseif (preg_match('/^(?P\d{4})$/', $dateString)) { + return [ + 'start' => "$dateString-01-01", + 'end' => "$dateString-12-31", + 'type' => 'Y', + ]; + } + + } + +} diff --git a/modules/rest_api_sync/helpers/rest_api_sync_rest.php b/modules/rest_api_sync/helpers/rest_api_sync_rest_endpoints.php similarity index 79% rename from modules/rest_api_sync/helpers/rest_api_sync_rest.php rename to modules/rest_api_sync/helpers/rest_api_sync_rest_endpoints.php index 690c4aa163..135efdacd3 100644 --- a/modules/rest_api_sync/helpers/rest_api_sync_rest.php +++ b/modules/rest_api_sync/helpers/rest_api_sync_rest_endpoints.php @@ -22,12 +22,12 @@ * @link https://github.com/indicia-team/warehouse/ */ - defined('SYSPATH') or die('No direct script access.'); +defined('SYSPATH') or die('No direct script access.'); /** * Helper class for extending the REST API with sync endpoints. */ -class rest_api_sync_rest { +class rest_api_sync_rest_endpoints { /** * Attribute types to exclude, either for privacy or duplication reasons. @@ -67,13 +67,26 @@ class rest_api_sync_rest { 'R5' => 'Not accepted - incorrect', ]; - public static function syncTaxonObservationsGet($foo, $clientConfig, $projectId) { + /** + * Endpoint for sync-taxon-observations. + * + * Echoes a list of observations in JSON occurrences format. + * + * @param mixed $foo + * Unused as this endpoint doesn't support retrieving a record by ID. + * @param array $clientConfig + * Configuration for this client from the REST API. + * @param string $projectId + * Code for the project being retrieved from. + */ + public static function syncTaxonObservationsGet($foo, array $clientConfig, $projectId) { if (empty($clientConfig['elasticsearch']) || count($clientConfig['elasticsearch']) !== 1) { RestObjects::$apiResponse->fail('Internal Server Error', 500, 'Incorrect elasticsearch configuration for client.'); } $project = ($clientConfig && $projectId) ? $clientConfig['projects'][$projectId] : []; $response = self::getEsTaxonObservationsResponse($clientConfig, $project); $total = count($response->hits->hits); + $nextFrom = NULL; echo "{\"data\":[\n"; foreach ($response->hits->hits as $idx => $hit) { $doc = $hit->_source; @@ -85,7 +98,7 @@ public static function syncTaxonObservationsGet($foo, $clientConfig, $projectId) $obj['event']['eventRemarks'] = $doc->event->event_remarks; } if (!empty($doc->event->recorded_by)) { - $obj['event']['recordedBy'] = $doc->event->recorded_by; + $obj['occurrence']['recordedBy'] = $doc->event->recorded_by; } if (!empty($doc->event->sampling_protocol)) { $obj['event']['samplingProtocol'] = $doc->event->sampling_protocol; @@ -93,13 +106,16 @@ public static function syncTaxonObservationsGet($foo, $clientConfig, $projectId) if (!empty($doc->location->coordinate_uncertainty_in_meters)) { $obj['location']['coordinateUncertaintyInMeters'] = $doc->location->coordinate_uncertainty_in_meters; } - if (isset($doc->location->input_sref_system) && in_array($doc->location->input_sref_system, ['OSGB', 'OSI'])) { + if (isset($doc->location->input_sref_system) + && in_array($doc->location->input_sref_system, ['OSGB', 'OSI'])) { $obj['location']['gridReference'] = $doc->location->input_sref; } else { $point = explode(',', $doc->location->point); - $obj['location']['decimalLatitude'] = $point[0]; - $obj['location']['decimalLongitude'] = $point[1]; + // Rtrim() and number_format() ensure we don't convert float to + // scientific notation. + $obj['location']['decimalLatitude'] = rtrim(number_format($point[0], 12), 0); + $obj['location']['decimalLongitude'] = rtrim(number_format($point[1], 12), 0); $obj['location']['geodeticDatum'] = 'WGS84'; } if (!empty($doc->location->verbatim_locality)) { @@ -179,7 +195,56 @@ public static function syncTaxonObservationsGet($foo, $clientConfig, $projectId) $nextFrom = $doc->metadata->tracking; } } - echo "\n],\"paging\":{\"next\":{\"tracking_from\":$nextFrom}}}"; + echo "\n]"; + if ($nextFrom) { + echo ",\"paging\":{\"next\":{\"tracking_from\":$nextFrom}}"; + } + echo "}"; + } + + /** + * Endpoint for sync-annotations. + * + * Echoes a list of annnotations in JSON occurrences format. + * + * @param mixed $foo + * Unused as this endpoint doesn't support retrieving a record by ID. + * @param array $clientConfig + * Configuration for this client from the REST API. + * @param string $projectId + * Code for the project being retrieved from. + */ + public static function syncAnnotationsGet($foo, array $clientConfig, $projectId) { + $projectConfig = $clientConfig['projects'][$projectId]; + $reportEngine = new ReportEngine([$projectConfig['website_id']]); + $params = [ + 'limit' => REST_API_DEFAULT_PAGE_SIZE, + 'system_user_id' => Kohana::config('rest.user_id'), + ]; + foreach ($projectConfig['annotations_filter'] as $key => $value) { + $params["{$key}_context"] = $value; + } + if (isset($_GET['dateTime_from'])) { + $params['dateTime_from'] = $_GET['dateTime_from']; + } + $output = $reportEngine->requestReport("rest_api_sync/filterable_annotations.xml", 'local', 'xml', $params, FALSE); + $dateTimeFrom = NULL; + echo "{\"data\":[\n"; + $total = count($output['content']['records']); + foreach ($output['content']['records'] as $idx => $record) { + echo json_encode($record); + if ($idx < $total - 1) { + echo ','; + } + else { + $dateTimeFrom = $record->dateTime; + } + } + echo "\n]"; + if ($dateTimeFrom) { + echo ",\"paging\":{\"next\":{\"dateTime_from\":\"$dateTimeFrom\"}}"; + } + echo "}"; } /** @@ -203,7 +268,7 @@ private static function sexTerm($term) { private static function getEsTaxonObservationsResponse($clientConfig, $project) { $es = new RestApiElasticsearch($clientConfig['elasticsearch'][0]); $format = 'json'; - if (isset($_GET['tracking_from']) ) { + if (isset($_GET['tracking_from'])) { if (!preg_match('/^\d+$/', $_GET['tracking_from'])) { RestObjects::$apiResponse->fail('Bad Request', 400, 'Invalid tracking from parameter'); } diff --git a/modules/rest_api_sync/helpers/rest_api_sync.php b/modules/rest_api_sync/helpers/rest_api_sync_utils.php similarity index 76% rename from modules/rest_api_sync/helpers/rest_api_sync.php rename to modules/rest_api_sync/helpers/rest_api_sync_utils.php index e2a6c595f9..8045b6a93d 100644 --- a/modules/rest_api_sync/helpers/rest_api_sync.php +++ b/modules/rest_api_sync/helpers/rest_api_sync_utils.php @@ -21,17 +21,29 @@ defined('SYSPATH') or die('No direct script access.'); + define('MAX_PAGES', 1); + /** * Helper class for syncing to RESTful APIs. */ -class rest_api_sync { +class rest_api_sync_utils { + /** + * Client user ID for authentication. + * + * @var string + */ public static $clientUserId; + /** + * Keep track of logged messages so they can be reported back to a UI. + * + * @var array + */ public static $log = []; /** - * Gets a page of data from another server's REST API + * Gets a page of data from another server's REST API. * * @param string $url * URL of the service to access. @@ -52,8 +64,18 @@ public static function getDataFromRestUrl($url, $serverId) { if (empty($servers[$serverId]['serverType']) || $servers[$serverId]['serverType'] === 'Indicia') { $shared_secret = $servers[$serverId]['shared_secret']; $userId = self::$clientUserId; - $hmac = hash_hmac("sha1", $url, $shared_secret, $raw_output = FALSE); - curl_setopt($session, CURLOPT_HTTPHEADER, array("Authorization: USER:$userId:HMAC:$hmac")); + $hmac = hash_hmac("sha1", $url, $shared_secret, FALSE); + curl_setopt($session, CURLOPT_HTTPHEADER, ["Authorization: USER:$userId:HMAC:$hmac"]); + } + elseif (!empty($servers[$serverId]['serverType']) && substr($servers[$serverId]['serverType'], 0, 5) === 'json_') { + // All JSON servers use same HMAC authentication. + $shared_secret = $servers[$serverId]['shared_secret']; + $userId = self::$clientUserId; + $time = round(microtime(TRUE) * 1000); + $authData = "$userId$time"; + // Create the authentication HMAC. + $hmac = hash_hmac("sha1", $authData, $shared_secret, FALSE); + curl_setopt($session, CURLOPT_HTTPHEADER, ["Authorization: USER:$userId:TIME:$time:HMAC:$hmac"]); } // Do the request. $response = curl_exec($session); diff --git a/modules/rest_api_sync/plugins/rest_api_sync.php b/modules/rest_api_sync/plugins/rest_api_sync.php index ea3085188f..2f585aa09c 100644 --- a/modules/rest_api_sync/plugins/rest_api_sync.php +++ b/modules/rest_api_sync/plugins/rest_api_sync.php @@ -47,12 +47,26 @@ function rest_api_sync_extend_rest_api() { 'proj_id' => [ 'datatype' => 'text', ], - 'tracking' => [ + 'tracking_from' => [ 'datatype' => 'integer', ], ], ], ], ], + 'sync-annotations' => [ + 'GET' => [ + 'sync-annotations' => [ + 'params' => [ + 'proj_id' => [ + 'datatype' => 'text', + ], + 'dateTime_from' => [ + 'datatype' => 'text', + ], + ], + ], + ], + ], ]; } diff --git a/modules/rest_api_sync/reports/rest_api_sync/filterable_annotations.xml b/modules/rest_api_sync/reports/rest_api_sync/filterable_annotations.xml new file mode 100644 index 0000000000..ae61e85eef --- /dev/null +++ b/modules/rest_api_sync/reports/rest_api_sync/filterable_annotations.xml @@ -0,0 +1,74 @@ + + + select #columns# + from occurrence_comments oc + join cache_occurrences_functional o on o.id=oc.occurrence_id + join occurrences occ on occ.id=o.id + join users u on u.id=oc.created_by_id and u.deleted=false + join people p on p.id=u.person_id and p.deleted=false + #agreements_join# + #joins# + where #sharing_filter# + and oc.deleted=false + -- Only annotations originating on this warehouse. + and oc.external_key is null + -- No system generated notifications + and oc.auto_generated=false + and o.taxa_taxon_list_external_key is not null + #idlist# + + + oc.updated_on ASC + + + + + oc.updated_on>'#dateTime_from#' + + + + + + + + + + + + + \ No newline at end of file diff --git a/modules/rest_api_sync/views/rest_api_sync_skipped_record/index.css b/modules/rest_api_sync/views/rest_api_sync_skipped_record/index.css new file mode 100644 index 0000000000..51e8d28f3a --- /dev/null +++ b/modules/rest_api_sync/views/rest_api_sync_skipped_record/index.css @@ -0,0 +1,3 @@ +progress { + width: 100%; +} \ No newline at end of file diff --git a/modules/rest_api_sync/views/rest_api_sync_skipped_record/index.js b/modules/rest_api_sync/views/rest_api_sync_skipped_record/index.js index 95327501a3..92924293e0 100644 --- a/modules/rest_api_sync/views/rest_api_sync_skipped_record/index.js +++ b/modules/rest_api_sync/views/rest_api_sync_skipped_record/index.js @@ -1,9 +1,9 @@ $(document).ready(function docReady() { var startInfo; var pageCount; + var pagesDone = 0; $('#sync-progress').hide(); - $('#progress').progressbar(); function doRequest(data) { $.ajax({ url: 'rest_api_sync/process_batch', @@ -13,20 +13,30 @@ $(document).ready(function docReady() { var chunkPercentage; var progress; // Get pageCount from pagesToGo on first iteration. - pageCount = typeof pageCount === 'undefined' ? response.pagesToGo : pageCount; + pageCount = pageCount ?? response.pagesToGo ?? null; + $('#output .panel-body').append('
' + response.log.join('
') + '
'); if (response.state === 'done') { $('#output .panel-body').append('
Synchronisation complete
'); - $('#progress').hide(); + $('#progress, #progress-info').hide(); $.ajax({ url: 'rest_api_sync/end', dataType: 'json', data: startInfo }); } else { - chunkPercentage = 100 / startInfo.servers.length; - progress = ((response.serverIdx - 1) + ((response.page - 1) / pageCount)) * chunkPercentage; - $('#progress').progressbar('value', progress); + pagesDone++; + if (response.moreToDo) { + if (pageCount) { + chunkPercentage = 100 / startInfo.servers.length; + progress = ((response.serverIdx - 1) + ((response.page - 1) / pageCount)) * chunkPercentage; + $('#progress').val(progress); + } + else { + // Alternative if true progress info missing. + $('#progress-info').removeClass('hide').text('Batches done so far: ' + pagesDone); + } + } doRequest({ serverIdx: response.serverIdx, page: response.page diff --git a/modules/rest_api_sync/views/rest_api_sync_skipped_record/index.php b/modules/rest_api_sync/views/rest_api_sync_skipped_record/index.php index 1fa8d6611a..6d65c42e40 100644 --- a/modules/rest_api_sync/views/rest_api_sync_skipped_record/index.php +++ b/modules/rest_api_sync/views/rest_api_sync_skipped_record/index.php @@ -34,8 +34,8 @@

Synchronisation progress

-

- + +
Synchronisation messages
diff --git a/modules/sref_osie/helpers/osie.php b/modules/sref_osie/helpers/osie.php index 106af71adf..98d399733c 100644 --- a/modules/sref_osie/helpers/osie.php +++ b/modules/sref_osie/helpers/osie.php @@ -13,39 +13,36 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see http://www.gnu.org/licenses/gpl.html. * - * @package Modules - * @subpackage OSGB Grid References - * @author Indicia Team + * @author Indicia Team * @license http://www.gnu.org/licenses/gpl.html GPL 3.0 - * @link https://github.com/indicia-team/warehouse/ + * @link https://github.com/indicia-team/warehouse/ */ /** * Conversion class for OS Ireland grid references (TM75). - * @package Modules - * @subpackage OSGB Grid References - * @author Indicia Team */ class osie { /** * Returns true if the spatial reference is a recognised Irish Grid square. * - * @param $sref string Spatial reference to validate + * @param $sref string + * Spatial reference to validate */ - public static function is_valid($sref) - { - // ignore any spaces in the grid ref - $sref = str_replace(' ','',$sref); + public static function is_valid($sref) { + // Ignore any spaces in the grid ref. + $sref = str_replace(' ', '', $sref); $sq100 = strtoupper(substr($sref, 0, 1)); - if (!preg_match('([A-HJ-Z])', $sq100)) + if (!preg_match('([A-HJ-Z])', $sq100)) { return FALSE; - $eastnorth=substr($sref, 1); + } + $eastnorth = substr($sref, 1); // 2 cases - either remaining chars must be all numeric and an equal number, up to 10 digits // OR for DINTY Tetrads, 2 numbers followed by a letter (Excluding O, including I) - if ((!preg_match('/^[0-9]*$/', $eastnorth) || strlen($eastnorth) % 2 != 0 || strlen($eastnorth)>10) AND - (!preg_match('/^[0-9][0-9][A-NP-Z]$/', $eastnorth))) + if ((!preg_match('/^[0-9]*$/', $eastnorth) || strlen($eastnorth) % 2 != 0 || strlen($eastnorth) > 10) && + (!preg_match('/^[0-9][0-9][A-NP-Z]$/', $eastnorth))) { return FALSE; + } return TRUE; } @@ -53,13 +50,15 @@ public static function is_valid($sref) * Converts a grid reference in OSI notation into the WKT text for the polygon, in * easting and northings from the zero reference. * - * @param string $sref The grid reference - * @return string String containing the well known text. + * @param string $sref + * The grid reference. + * + * @return string + * String containing the well known text. */ - public static function sref_to_wkt($sref) - { + public static function sref_to_wkt($sref) { // ignore any spaces in the grid ref - $sref = str_replace(' ','',$sref); + $sref = str_replace(' ', '', $sref); if (!self::is_valid($sref)) throw new InvalidArgumentException('Spatial reference is not a recognisable grid square.', 4001); $sq_100 = self::get_100k_square($sref); diff --git a/modules/taxon_associations/tests/TaxonAssocations_ServicesTest.php b/modules/taxon_associations/tests/TaxonAssocations_ServicesTest.php index 75b81c2cd8..2e5c2165c8 100644 --- a/modules/taxon_associations/tests/TaxonAssocations_ServicesTest.php +++ b/modules/taxon_associations/tests/TaxonAssocations_ServicesTest.php @@ -3,9 +3,9 @@ require_once 'client_helpers/data_entry_helper.php'; require_once 'client_helpers/submission_builder.php'; -define ('CORE_FIXTURE_TERMLIST_COUNT', 4); -define ('CORE_FIXTURE_TERM_COUNT', 5); -define ('CORE_FIXTURE_TERMLISTS_TERM_COUNT', 5); +define('CORE_FIXTURE_TERMLIST_COUNT', 4); +define('CORE_FIXTURE_TERM_COUNT', 5); +define('CORE_FIXTURE_TERMLISTS_TERM_COUNT', 5); class TaxonAssociations_ServicesTest extends Indicia_DatabaseTestCase { @@ -13,16 +13,16 @@ class TaxonAssociations_ServicesTest extends Indicia_DatabaseTestCase { public function getDataSet() { - $ds1 = new PHPUnit_Extensions_Database_DataSet_YamlDataSet('modules/phpUnit/config/core_fixture.yaml'); + $ds1 = new PHPUnit_Extensions_Database_DataSet_YamlDataSet('modules/phpUnit/config/core_fixture.yaml'); $ds2 = new Indicia_ArrayDataSet( - array( - 'meanings' => array( - array( - 'id' => 20000 - ) - ), - 'termlists' => array( - array( + [ + 'meanings' => [ + [ + 'id' => 20000, + ], + ], + 'termlists' => [ + [ 'title' => 'Taxon association types', 'description' => 'Types of associations between taxa', 'website_id' => 1, @@ -30,21 +30,21 @@ public function getDataSet() 'created_by_id' => 1, 'updated_on' => '2016-07-22:16:00:00', 'updated_by_id' => 1, - 'external_key' => NULL - ), - ), - 'terms' => array( - array( + 'external_key' => NULL, + ], + ], + 'terms' => [ + [ 'term' => 'is associated with', 'language_id' => 1, 'created_on' => '2016-07-22:16:00:00', 'created_by_id' => 1, 'updated_on' => '2016-07-22:16:00:00', - 'updated_by_id' => 1 - ), - ), - 'termlists_terms' => array( - array( + 'updated_by_id' => 1, + ], + ], + 'termlists_terms' => [ + [ 'termlist_id' => CORE_FIXTURE_TERMLIST_COUNT + 1, 'term_id' => CORE_FIXTURE_TERM_COUNT + 1, 'created_on' => '2016-07-22:16:00:00', @@ -52,11 +52,11 @@ public function getDataSet() 'updated_on' => '2016-07-22:16:00:00', 'updated_by_id' => 1, 'meaning_id' => 20000, - 'preferred' => true, - 'sort_order' => 1 - ), - ), - ) + 'preferred' => TRUE, + 'sort_order' => 1, + ], + ], + ] ); $compositeDs = new PHPUnit_Extensions_Database_DataSet_CompositeDataSet(); @@ -67,20 +67,30 @@ public function getDataSet() public function setup() { $this->auth = data_entry_helper::get_read_write_auth(1, 'password'); - // make the tokens re-usable - $this->auth['write_tokens']['persist_auth']=true; + // Make the tokens re-usable. + $this->auth['write_tokens']['persist_auth'] = true; parent::setup(); } function testPost() { - $array = array( + $array = [ 'taxon_association:from_taxon_meaning_id' => 10000, 'taxon_association:to_taxon_meaning_id' => 10001, 'taxon_association:association_type_id' => CORE_FIXTURE_TERMLISTS_TERM_COUNT + 1, + ]; + $s = submission_builder::build_submission( + $array, ['model' => 'taxon_association'] + ); + $r = data_entry_helper::forward_post_to( + 'taxon_association', $s, $this->auth['write_tokens'] + ); + Kohana::log( + 'debug', + "Submission response to taxon_association save " . print_r($r, TRUE) + ); + $this->assertTrue( + isset($r['success']), + 'Submitting a taxon_association did not return success response' ); - $s = submission_builder::build_submission($array, array('model' => 'taxon_association')); - $r = data_entry_helper::forward_post_to('taxon_association', $s, $this->auth['write_tokens']); - Kohana::log('debug', "Submission response to taxon_association save " . print_r($r, TRUE)); - $this->assertTrue(isset($r['success']), 'Submitting a taxon_association did not return success response'); } } \ No newline at end of file diff --git a/modules/workflow/controllers/workflow_event.php b/modules/workflow/controllers/workflow_event.php index 656b4070cc..d6b6ad02c1 100644 --- a/modules/workflow/controllers/workflow_event.php +++ b/modules/workflow/controllers/workflow_event.php @@ -53,6 +53,27 @@ protected function get_action_columns() { ); } + /** + * Convert location_ids_filter to suitable default for the sub_list control. + */ + protected function getModelValues() { + $r = parent::getModelValues(); + $r['location_ids_filter_array'] = []; + if (preg_match('/^{(?\d+(,\d+)*)}$/', $r['workflow_event:location_ids_filter'], $matches)) { + $ids = explode(',', $matches['list']); + foreach ($ids as $id) { + $location = ORM::factory('location', $id); + $r['location_ids_filter_array'][] = [ + 'caption' => $location->name, + 'fieldname' => 'workflow_event:location_ids_filter[]', + 'default' => $id, + ]; + } + + } + return $r; + } + /** * Prepares any additional data required by the edit view. * diff --git a/modules/workflow/db/version_6_3_0/202108160940_workflow_filters.sql b/modules/workflow/db/version_6_3_0/202108160940_workflow_filters.sql new file mode 100644 index 0000000000..0892ce934d --- /dev/null +++ b/modules/workflow/db/version_6_3_0/202108160940_workflow_filters.sql @@ -0,0 +1,17 @@ +ALTER TABLE workflow_events + ADD COLUMN attrs_filter_term text; + +ALTER TABLE workflow_events + ADD COLUMN attrs_filter_values text[]; + +ALTER TABLE workflow_events +ADD COLUMN location_ids_filter integer[]; + +COMMENT ON COLUMN workflow_events.attrs_filter_term IS + 'When this event should only trigger if a certain attribute value is present, specify the DwC term here which identifies the attribute to use (e.g. ReproductiveCondition or Stage). Typically used to limit bird events to breeding ReproductiveCondition terms.'; + +COMMENT ON COLUMN workflow_events.attrs_filter_values IS + 'When this event should only trigger if a certain attribute value is present, specify the list of triggering values here. A record matching any value in the list will trigger the event.'; + +COMMENT ON COLUMN workflow_events.location_ids_filter IS + 'When this event should only trigger if the record overlaps an indexed location boundary, specify the location ID or list of IDs here. Used to limit alerts to geographic areas.'; \ No newline at end of file diff --git a/modules/workflow/helpers/task_workflow_event_check_filters.php b/modules/workflow/helpers/task_workflow_event_check_filters.php new file mode 100644 index 0000000000..fdc1fe2f96 --- /dev/null +++ b/modules/workflow/helpers/task_workflow_event_check_filters.php @@ -0,0 +1,80 @@ +>'workflow_events.id' +LEFT JOIN samples s ON s.id=o.sample_id AND e.location_ids_filter is not null +LEFT JOIN locations l ON l.id = ANY(e.location_ids_filter) AND st_intersects(l.boundary_geom, s.geom) +LEFT JOIN (occurrence_attribute_values v + JOIN cache_termlists_terms t on t.id=v.int_value + JOIN occurrence_attributes a ON a.id=v.occurrence_attribute_id +) ON v.occurrence_id=o.id AND e.attrs_filter_term IS NOT NULL + -- case insensitive array check. + AND lower(t.term)=ANY(lower(e.attrs_filter_values::text)::text[]) + AND lower(a.term_name)=lower(e.attrs_filter_term) +WHERE q.entity='occurrence' AND q.task='task_workflow_event_check_filters' AND claimed_by='$procId' +-- Need to either fail on the locations filter, or attribute values filter. +AND ((e.location_ids_filter IS NOT NULL AND l.id IS NULL) +OR (e.attrs_filter_term IS NOT NULL AND v.id IS NULL)); +SQL; + $tasks = $db->query($qry); + $occurrenceIds = []; + foreach ($tasks as $task) { + $occurrenceIds[] = $task->record_id; + } + // For the records outside the workflow_event's filter we can rewind them. + $rewinds = workflow::getRewindChangesForRecords($db, 'occurrence', $occurrenceIds, ['S', 'V', 'R']); + foreach ($rewinds as $key => $rewind) { + list($entity, $id) = explode('.', $key); + $obj = ORM::factory($entity, $id); + foreach ($rewind as $field => $value) { + $obj->$field = $value; + } + $obj->save(); + } + } + +} diff --git a/modules/workflow/helpers/workflow.php b/modules/workflow/helpers/workflow.php index 29949a4790..0fb203a632 100644 --- a/modules/workflow/helpers/workflow.php +++ b/modules/workflow/helpers/workflow.php @@ -45,8 +45,9 @@ public static function getEntityConfig($entity) { /** * Applies undo data to rewind records to their originally posted state. * - * This occurs when a record has been modified by the workflow system because of a particular key value linking it to - * a workflow event record, then the key value is changed so the workflow event is no longer relevant. + * This occurs when a record has been modified by the workflow system because + * of a particular key value linking it to a workflow event record, then the + * key value is changed so the workflow event is no longer relevant. * * @param object $db * Database connection. @@ -71,16 +72,17 @@ public static function getRewoundRecord($db, $entity, $oldRecord, &$newRecord) { } $eventTypes = []; foreach ($entityConfig['keys'] as $keyDef) { - $keyChanged = false; + $keyChanged = FALSE; // We need to know if the key has changed to decide whether to wind back. // If the key is in the main entity, we can directly compare the old and new keys. if ($keyDef['table'] === $entity) { $keyCol = $keyDef['column']; - $keyChanged = (string) $oldRecord->$column !== (string) $newRecord->column; + $keyChanged = (string) $oldRecord->column !== (string) $newRecord->column; } else { - // Find the definintion of the extra data table that contains the column we need to look for changes in. We can - // then look to see if the foreign key pointing to that table has changed. + // Find the definintion of the extra data table that contains the + // column we need to look for changes in. We can then look to see if + // the foreign key pointing to that table has changed. foreach ($entityConfig['extraData'] as $extraDataDef) { if ($extraDataDef['table'] === $keyDef['table']) { $column = $extraDataDef['originating_table_column']; @@ -94,7 +96,8 @@ public static function getRewoundRecord($db, $entity, $oldRecord, &$newRecord) { } if ($entity === 'occurrence' && $oldRecord->record_status !== $newRecord->record_status) { - // Remove previuos verification and rejection workflow changes as the record status is changing. + // Remove previuos verification and rejection workflow changes as the + // record status is changing. $eventTypes[] = 'V'; $eventTypes[] = 'R'; } @@ -129,17 +132,18 @@ public static function getRewoundRecord($db, $entity, $oldRecord, &$newRecord) { * List of event types to rewind ('S', 'V', 'R'). * * @return array - * Associatie array keyed by entity.entity_id, containing an array of the fields with undo values to apply. + * Associate array keyed by entity.entity_id, containing an array of the + * fields with undo values to apply. */ public static function getRewindChangesForRecords($db, $entity, array $entityIdList, array $eventTypes) { $r = []; $undoRecords = $db ->select('DISTINCT workflow_undo.id, workflow_undo.entity_id, workflow_undo.original_values') ->from('workflow_undo') - ->where(array( + ->where([ 'workflow_undo.entity' => $entity, 'workflow_undo.active' => 't', - )) + ]) ->in('event_type', $eventTypes) ->in('entity_id', $entityIdList) ->orderby('workflow_undo.id', 'DESC') @@ -154,7 +158,7 @@ public static function getRewindChangesForRecords($db, $entity, array $entityIdL $r["$entity.$undoRecord->entity_id"] = array_merge($r["$entity.$undoRecord->entity_id"], $unsetColumns); } // As this is a hard rewind, disable the undo data. - $db->update('workflow_undo', array('active' => 'f'), array('id' => $undoRecord->id)); + $db->update('workflow_undo', ['active' => 'f'], ['id' => $undoRecord->id]); } return $r; } @@ -164,7 +168,7 @@ public static function getRewindChangesForRecords($db, $entity, array $entityIdL * * @param object $db * Database connection. - * @param int $websiteId + * @param int $websiteId * ID of the website the update is associated with. * @param string $entity * Name of the database entity being saved, e.g. occurrence. @@ -174,8 +178,8 @@ public static function getRewindChangesForRecords($db, $entity, array $entityIdL * List of event types to include in the results. * * @return array - * List of records with events attached (keyed by entity.id), with each entry containing an array of the events - * associated with that record. + * List of records with events attached (keyed by entity.id), with each + * entry containing an array of the events associated with that record. */ public static function getEventsForRecords($db, $websiteId, $entity, array $entityIdList, array $eventTypes) { $r = []; @@ -192,13 +196,13 @@ public static function getEventsForRecords($db, $websiteId, $entity, array $enti $table = inflector::plural($entity); foreach ($entityConfig['keys'] as $keyDef) { $qry = $db - ->select('workflow_events.key_value, workflow_events.event_type, workflow_events.mimic_rewind_first, ' . - "workflow_events.values, $table.id as {$entity}_id") + ->select('workflow_events.id, workflow_events.key_value, workflow_events.event_type, workflow_events.mimic_rewind_first, ' . + "workflow_events.values, $table.id as {$entity}_id, workflow_events.attrs_filter_term, workflow_events.location_ids_filter") ->from('workflow_events') - ->where(array( + ->where([ 'workflow_events.deleted' => 'f', 'key' => $keyDef['db_store_value'], - )) + ]) ->in('group_code', $groupCodes) ->in('workflow_events.event_type', $eventTypes); if ($keyDef['table'] === $entity) { @@ -207,7 +211,8 @@ public static function getEventsForRecords($db, $websiteId, $entity, array $enti } else { $qry->join($keyDef['table'], "$keyDef[table].$keyDef[column]", 'workflow_events.key_value'); - // Cross reference to the extraData for the same table to find the field name which matches $newRecord->column. + // Cross reference to the extraData for the same table to find the + // field name which matches $newRecord->column. foreach ($entityConfig['extraData'] as $extraDataDef) { if ($extraDataDef['table'] === $keyDef['table']) { $qry->join( @@ -274,8 +279,8 @@ public static function applyWorkflow($db, $websiteId, $entity, $oldRecord, $rewo /** * Construct a query to retrieve workflow events. * - * Constructs a query object which will find all the events applicable to the current record for a given key in the - * entity's configuration. + * Constructs a query object which will find all the events applicable to the + * current record for a given key in the entity's configuration. * * @param object $db * Database connection. @@ -293,16 +298,17 @@ public static function applyWorkflow($db, $websiteId, $entity, $oldRecord, $rewo * @return object * Query object. */ - private static function buildEventQueryForKey($db, $groupCodes, $entity, $oldRecord, $newRecord, array $keyDef) { + private static function buildEventQueryForKey($db, array $groupCodes, $entity, $oldRecord, $newRecord, array $keyDef) { $entityConfig = self::getEntityConfig($entity); $eventTypes = []; $qry = $db - ->select('workflow_events.event_type, workflow_events.mimic_rewind_first, workflow_events.values') + ->select('workflow_events.id, workflow_events.event_type, workflow_events.mimic_rewind_first, workflow_events.values, ' . + 'workflow_events.attrs_filter_term, workflow_events.location_ids_filter') ->from('workflow_events') - ->where(array( + ->where([ 'workflow_events.deleted' => 'f', 'key' => $keyDef['db_store_value'], - )) + ]) ->in('group_code', $groupCodes); if ($keyDef['table'] === $entity) { $column = $keyDef['column']; @@ -361,8 +367,10 @@ private static function buildEventQueryForKey($db, $groupCodes, $entity, $oldRec * Finds configured groups which the current operation's website uses the workflow for. * * @param int $websiteId + * Website ID. * * @return array + * List of group names. */ private static function getGroupCodesForThisWebsite($websiteId) { $config = kohana::config('workflow_groups', FALSE, FALSE); @@ -380,8 +388,9 @@ private static function getGroupCodesForThisWebsite($websiteId) { /** * Applies the events query results to a record. * - * Applies the field value changes determined by a query against the workflow_events table to the contents of a record - * that is about to be saved. + * Applies the field value changes determined by a query against the + * workflow_events table to the contents of a record that is about to be + * saved. * * @param object $qry * Query object set up to retrieve the events to apply. @@ -393,15 +402,17 @@ private static function getGroupCodesForThisWebsite($websiteId) { * @param object $newRecord * ORM Validation object containing the new record details. * @param array $state - * State data to pass through to the post-process hook, containing undo data. + * State data to pass through to the post-process hook, containing undo + * data. */ private static function applyEventsQueryToRecord($qry, $entity, array $oldValues, &$newRecord, array &$state) { $events = $qry->get(); foreach ($events as $event) { - $newUndoRecord = array(); kohana::log('debug', 'Processing event: ' . var_export($event, TRUE)); + $needsFilterCheck = !empty($event->attrs_filter_term) || !empty($event->location_ids_filter); $valuesToApply = self::processEvent( $event, + $needsFilterCheck, $entity, $oldValues, $newRecord->as_array(), @@ -416,25 +427,32 @@ private static function applyEventsQueryToRecord($qry, $entity, array $oldValues /** * Processes a single workflow event. * - * Retrieves a list of the values that need to be applied to a database record given an event. The values may include - * the results of a mimiced rewind as well as the value changes required for the event. + * Retrieves a list of the values that need to be applied to a database + * record given an event. The values may include the results of a mimiced + * rewind as well as the value changes required for the event. * * @param object $event * Event object loaded from the database query. + * @param bool $needsFilterCheck + * If true, then the workflow event has an attribute or location filter + * that needs double checking via a work queue task. * @param string $entity * Name of the database entity being saved, e.g. occurrence. * @param array $oldValues * Array of the record values before the save operation. Used to retrieve * values for undo state data. * @param array $newValues - * Array of the record values that were submitted to be saved, causing the event to fire. + * Array of the record values that were submitted to be saved, causing the + * event to fire. * @param array $state - * Array of undo state data which will be updated by this method to allow any proposed changes to be undone. + * Array of undo state data which will be updated by this method to allow + * any proposed changes to be undone. * * @return array - * Associative array of the database fields and values which need to be applied. + * Associative array of the database fields and values which need to be + * applied. */ - public static function processEvent($event, $entity, array $oldValues, array $newValues, array &$state) { + public static function processEvent($event, $needsFilterCheck, $entity, array $oldValues, array $newValues, array &$state) { $entityConfig = self::getEntityConfig($entity); $columnDeltaList = []; $valuesToApply = []; @@ -463,12 +481,17 @@ public static function processEvent($event, $entity, array $oldValues, array $ne $valuesToApply[$deltaColumn] = $deltaValue; } } - $state[] = array('event_type' => $event->event_type, 'old_data' => $newUndoRecord); + $state[] = [ + 'event_id' => $event->id, + 'needs_filter_check' => $needsFilterCheck, + 'event_type' => $event->event_type, + 'old_data' => $newUndoRecord, + ]; return $valuesToApply; } /** - * Returns true if the current user is allowed to view the workflow configuration pages. + * Returns true if the user is allowed to view the workflow config pages. * * @param object $auth * Kohana authorisation object. @@ -496,8 +519,8 @@ public static function allowWorkflowConfigAccess($auth) { /** * Rewind a record. * - * If an event wants to mimic a rewind to reset data to its original state, then undoes all changes to the record - * caused by workflow. + * If an event wants to mimic a rewind to reset data to its original state, + * then undoes all changes to the record caused by workflow. * * @param string $entity * Name of the database entity being saved, e.g. occurrence. @@ -506,8 +529,8 @@ public static function allowWorkflowConfigAccess($auth) { * @param array $columnDeltaList * Array containing the field values that will be changed by the rewind. * @param array $state - * Undo state change data from events applied to the record on this transaction which may need to be rewound. - * @return void + * Undo state change data from events applied to the record on this + * transaction which may need to be rewound. */ private static function mimicRewind($entity, $entityId, array &$columnDeltaList, array $state) { for ($i = count($state) - 1; $i >= 0; $i--) { @@ -516,11 +539,11 @@ private static function mimicRewind($entity, $entityId, array &$columnDeltaList, } } $undoRecords = ORM::factory('workflow_undo') - ->where(array( + ->where([ 'entity' => $entity, 'entity_id' => $entityId, 'active' => 't', - )) + ]) ->orderby('id', 'DESC')->find_all(); foreach ($undoRecords as $undoRecord) { kohana::log('debug', 'mimic rewind record: ' . var_export($undoRecord->as_array(), TRUE)); diff --git a/modules/workflow/models/workflow_event.php b/modules/workflow/models/workflow_event.php index 6779a22c19..0fad0ef59a 100644 --- a/modules/workflow/models/workflow_event.php +++ b/modules/workflow/models/workflow_event.php @@ -27,16 +27,20 @@ * Model class for the workflow_event table. */ class Workflow_event_Model extends ORM { - public $search_field='id'; + public $search_field = 'id'; - protected $belongs_to = array( + protected $belongs_to = [ 'created_by' => 'user', - 'updated_by' => 'user' - ); - protected $has_and_belongs_to_many = array(); + 'updated_by' => 'user', + ]; + protected $has_and_belongs_to_many = []; + /** + * Define model validation behaviour. + */ public function validate(Validation $array, $save = FALSE) { - // uses PHP trim() to remove whitespace from beginning and end of all fields before validation + // Uses PHP trim() to remove whitespace from beginning and end of all + // fields before validation. $array->pre_filter('trim'); $array->add_rules('entity', 'required'); $array->add_rules('group_code', 'required'); @@ -46,12 +50,37 @@ public function validate(Validation $array, $save = FALSE) { $array->add_rules('values', 'required'); // Explicitly add those fields for which we don't do validation. - $this->unvalidatedFields = array( + $this->unvalidatedFields = [ 'deleted', 'mimic_rewind_first', - ); + 'attrs_filter_term', + 'attrs_filter_values', + 'location_ids_filter', + ]; return parent::validate($array, $save); } + /** + * Tidy form data to prepare for submission. + * + * Converts attr_filter_values from form submission string to array. Also + * ensures location_ids_filter array is cleaned. + */ + public function preSubmit() { + if (!empty($this->submission['fields']['attrs_filter_values']['value']) + && is_string($this->submission['fields']['attrs_filter_values']['value'])) { + $valueList = str_replace("\r\n", "\n", $this->submission['fields']['attrs_filter_values']['value']); + $valueList = str_replace("\r", "\n", $valueList); + $valueList = explode("\n", trim($valueList)); + $this->submission['fields']['attrs_filter_values'] = ['value' => $valueList]; + } + // Due to the way the sub_list control works, we can have hidden empty + // values which need to be cleaned. + if (!empty($this->submission['fields']['location_ids_filter']['value']) + && is_array($this->submission['fields']['location_ids_filter']['value'])) { + $this->submission['fields']['location_ids_filter']['value'] = array_values(array_filter($this->submission['fields']['location_ids_filter']['value'])); + } + } + } diff --git a/modules/workflow/plugins/workflow.php b/modules/workflow/plugins/workflow.php index d3da739359..38f07eb79d 100644 --- a/modules/workflow/plugins/workflow.php +++ b/modules/workflow/plugins/workflow.php @@ -17,8 +17,6 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see http://www.gnu.org/licenses/gpl.html. * - * @package Modules - * @subpackage Workflow * @author Indicia Team * @license http://www.gnu.org/licenses/gpl.html GPL * @link https://github.com/Indicia-Team/ @@ -46,25 +44,27 @@ function workflow_alter_menu($menu, $auth) { /** * Implements the extend_data_services hook. * - * Determines the data entities which should be added to those available via data services. + * Determines the data entities which should be added to those available via + * data services. * * @return array * List of database entities exposed by this plugin with configuration. */ function workflow_extend_data_services() { - return array( - 'workflow_events' => array(), - 'workflow_metadata' => array('allow_full_access' => TRUE) - ); + return [ + 'workflow_events' => [], + 'workflow_metadata' => ['allow_full_access' => TRUE], + ]; } /** * Pre-record save processing hook. * - * Potential problem when a record matches multiple events, and they change the same columns, - * so we are making the assumption that each record will only fire one alert key/key_value combination - * undo record would require more details on firing event (key and key_value) if this is changed in future - * In following code, entity means the orm entity, e.g. 'occurrence' + * Potential problem when a record matches multiple events, and they change the + * same columns, so we are making the assumption that each record will only + * fire one alert key/key_value combination undo record would require more + * details on firing event (key and key_value) if this is changed in future. + * In following code, entity means the orm entity, e.g. 'occurrence'. * * @param object $db * Database connection. @@ -81,12 +81,13 @@ function workflow_extend_data_services() { * State data to pass to the post save processing hook. */ function workflow_orm_pre_save_processing($db, $websiteId, $entity, $oldRecord, &$newRecord) { - $state = array(); + $state = []; // Abort if no workflow configuration for this entity. if (empty(workflow::getEntityConfig($entity))) { return $state; } - // Rewind the record if previous workflow rule changes no longer apply (e.g. after redetermination). + // Rewind the record if previous workflow rule changes no longer apply (e.g. + // safter redetermination). $rewoundRecord = workflow::getRewoundRecord($db, $entity, $oldRecord, $newRecord); // Apply any changes in the workflow_events table relevant to the record. $state = workflow::applyWorkflow($db, $websiteId, $entity, $oldRecord, $rewoundRecord, $newRecord); @@ -121,14 +122,27 @@ function workflow_orm_post_save_processing($db, $entity, $record, array $state, $userId = security::getUserId(); // Insert any state undo records. foreach ($state as $undoDetails) { - $db->insert('workflow_undo', array( + $db->insert('workflow_undo', [ 'entity' => $entity, 'entity_id' => $id, 'event_type' => $undoDetails['event_type'], 'created_on' => date("Ymd H:i:s"), 'created_by_id' => $userId, - 'original_values' => json_encode($undoDetails['old_data']) - )); + 'original_values' => json_encode($undoDetails['old_data']), + ]); + if ($undoDetails['needs_filter_check']) { + $q = new WorkQueue(); + $q->enqueue($db, [ + 'task' => 'task_workflow_event_check_filters', + 'entity' => $entity, + 'record_id' => $id, + 'cost_estimate' => 50, + 'priority' => 2, + 'params' => json_encode([ + 'workflow_events.id' => $undoDetails['event_id'], + ]), + ]); + } } return TRUE; } diff --git a/modules/workflow/views/workflow_event/edit.js b/modules/workflow/views/workflow_event/edit.js index df57119755..70e0a20b71 100644 --- a/modules/workflow/views/workflow_event/edit.js +++ b/modules/workflow/views/workflow_event/edit.js @@ -1,8 +1,16 @@ jQuery(document).ready(function($) { $('#taxon_list_id').change(function() { - var options = $('input#workflow_event\\:key_value\\:taxon').indiciaAutocomplete('option'); - options.extraParams.taxon_list_id = $('#taxon_list_id').val(); - $('input#workflow_event\\:key_value\\:taxon').indiciaAutocomplete('option', options); + $('input#workflow_event\\:key_value\\:taxon').setExtraParams({ + taxon_list_id: $('#taxon_list_id').val() + }); + }); + + $('#location_type').change(function() { + // Remove any hanging autocomplete select list. + $('.ac_results').hide(); + $('#workflow_event\\:location_ids_filter\\:search\\:name').setExtraParams({ + location_type_id: $('#location_type').val() + }); }); $('#workflow_event\\:entity').change(function entityChange() { diff --git a/modules/workflow/views/workflow_event/index.php b/modules/workflow/views/workflow_event/index.php index 69ab0f16d1..4e60a49f41 100644 --- a/modules/workflow/views/workflow_event/index.php +++ b/modules/workflow/views/workflow_event/index.php @@ -31,7 +31,7 @@ echo $grid; ?>
- +
db->select('*')->from('system')->where('name', 'workflow')->get()->as_array(TRUE); diff --git a/modules/workflow/views/workflow_event/workflow_event_edit.php b/modules/workflow/views/workflow_event/workflow_event_edit.php index 53ac130b28..e7ff83e52c 100644 --- a/modules/workflow/views/workflow_event/workflow_event_edit.php +++ b/modules/workflow/views/workflow_event/workflow_event_edit.php @@ -24,21 +24,20 @@ * @link https://github.com/Indicia-Team/ */ -require_once DOCROOT . 'client_helpers/data_entry_helper.php'; +warehouse::loadHelpers(['data_entry_helper']); +$id = html::initial_value($values, 'workflow_event:id'); if (isset($_POST)) { data_entry_helper::dump_errors(['errors' => $this->model->getAllErrors()]); } $readAuth = data_entry_helper::get_read_auth(0 - $_SESSION['auth_user']->id, kohana::config('indicia.private_key')); ?> -
+
Workflow Event definition details 'workflow_event:id', - 'default' => html::initial_value($values, 'workflow_event:id'), + 'default' => $id, ]); $messages = []; // Where controls have no choice available, show a message instead. @@ -68,7 +67,7 @@ 'lookupValues' => $other_data['groupSelectItems'], 'default' => html::initial_value($values, 'workflow_event:group_code'), 'validation' => ['required'], - 'helpText' => 'The workflow groups available must be configured by the warehouse administrator and define which website\'s records will be affected by this event definition.' + 'helpText' => 'The workflow groups available must be configured by the warehouse administrator and define which website\'s records will be affected by this event definition.', ]); } if (count($other_data['entitySelectItems']) !== 1) { @@ -126,11 +125,54 @@ 'default' => html::initial_value($values, 'workflow_event:event_type'), ]); echo data_entry_helper::select([ - 'label' => 'Event Type', + 'label' => 'Event type', 'fieldname' => 'workflow_event:event_type', 'lookupValues' => [], 'validation' => ['required'], ]); + echo '
Event filters'; + echo data_entry_helper::select([ + 'label' => 'Location type', + 'fieldname' => 'location_type', + 'table' => 'termlists_term', + 'valueField' => 'id', + 'captionField' => 'term', + 'extraParams' => $readAuth + ['termlist_external_key' => 'indicia:location_types', 'order_by' => 'term'], + 'blankText' => '', + 'helpText' => 'Choose the location type to search within when adding a location boundary filter below.', + ]); + echo data_entry_helper::sub_list([ + 'label' => 'Location boundary filter', + 'fieldname' => 'workflow_event:location_ids_filter', + 'helpText' => lang::get('When this event should only trigger if a record falls inside a geographic region, ' . + 'specify the list of locations covering that region (or regions).'), + 'table' => 'location', + 'captionField' => 'name', + 'valueField' => 'id', + 'extraParams' => $readAuth, + 'addToTable' => FALSE, + 'default' => empty($values['location_ids_filter_array']) ? NULL : $values['location_ids_filter_array'], + ]); + echo data_entry_helper::text_input([ + 'label' => 'Attribute value filter term', + 'fieldname' => 'workflow_event:attrs_filter_term', + 'helpText' => lang::get('When this event should only trigger if a certain attribute value is present, specify ' . + 'the DwC term here which identifies the attribute to use (e.g. ReproductiveCondition or Stage). Typically ' . + 'used to limit bird events to breeding ReproductiveCondition terms.'), + 'default' => html::initial_value($values, 'workflow_event:attrs_filter_term'), + ]); + $filterValues = html::initial_value($values, 'workflow_event:attrs_filter_values'); + if (!empty($filterValues)) { + $filterValues = trim($filterValues, '{}'); + $filterValues = str_replace(',', "\n", $filterValues); + } + echo data_entry_helper::textarea([ + 'label' => 'Attribute value filter values', + 'fieldname' => 'workflow_event:attrs_filter_values', + 'default' => $filterValues, + ]); + echo '
'; + echo '
Data updates'; echo data_entry_helper::checkbox([ 'label' => 'Rewind record state first', 'fieldname' => 'workflow_event:mimic_rewind_first', @@ -142,15 +184,12 @@ 'schema' => $other_data['jsonSchema'], 'default' => html::initial_value($values, 'workflow_event:values'), ]); - + echo '
'; echo $metadata; - echo html::form_buttons(html::initial_value($values, 'workflow_event:id') != NULL, FALSE, FALSE); - data_entry_helper::$indiciaData['entities'] = $other_data['entities']; - data_entry_helper::$dumped_resources[] = 'jquery'; - data_entry_helper::$dumped_resources[] = 'jquery_ui'; - data_entry_helper::$dumped_resources[] = 'fancybox'; + echo html::form_buttons(!empty($id), FALSE, FALSE); + data_entry_helper::enable_validation('entry_form'); echo data_entry_helper::dump_javascript(); ?>
diff --git a/reports/library/notifications/notifications_list_for_notifications_centre.xml b/reports/library/notifications/notifications_list_for_notifications_centre.xml index 88604c74f1..fdefc3909d 100644 --- a/reports/library/notifications/notifications_list_for_notifications_centre.xml +++ b/reports/library/notifications/notifications_list_for_notifications_centre.xml @@ -70,7 +70,7 @@ - + diff --git a/reports/library/occurrences/list_for_elastic.xml b/reports/library/occurrences/list_for_elastic.xml index fda2ff0165..f80fbf3559 100644 --- a/reports/library/occurrences/list_for_elastic.xml +++ b/reports/library/occurrences/list_for_elastic.xml @@ -23,6 +23,7 @@ FROM filtered_occurrences o JOIN cache_occurrences_nonfunctional onf ON onf.id=o.id JOIN occurrences occ on occ.id=o.id AND occ.deleted=false + JOIN samples s ON s.id=o.sample_id AND s.deleted=false JOIN cache_samples_nonfunctional snf ON snf.id=o.sample_id JOIN cache_taxa_taxon_lists cttl ON cttl.id=o.taxa_taxon_list_id LEFT JOIN locations l ON l.id=o.location_id AND l.deleted=false; @@ -43,10 +44,10 @@ SET recorded_parent_location_id = lp.id, recorded_parent_location_name = lp.name, recorded_parent_location_code = lp.code - FROM samples sc + FROM samples sc JOIN locations lc ON lc.id=sc.location_id AND lc.deleted=false - JOIN locations lp ON lp.id = lc.parent_id AND lp.deleted=false - WHERE sc.id=o.sample_id AND sc.deleted=false + JOIN locations lp ON lp.id = lc.parent_id AND lp.deleted=false + WHERE sc.id=o.sample_id AND sc.deleted=false AND o.recorded_parent_location_id IS NULL; SELECT * @@ -67,6 +68,7 @@ + diff --git a/reports/library/occurrences/list_for_elastic_all.xml b/reports/library/occurrences/list_for_elastic_all.xml index 30912b89b2..6dd18f9402 100644 --- a/reports/library/occurrences/list_for_elastic_all.xml +++ b/reports/library/occurrences/list_for_elastic_all.xml @@ -23,6 +23,7 @@ FROM filtered_occurrences o JOIN cache_occurrences_nonfunctional onf ON onf.id=o.id JOIN occurrences occ on occ.id=o.id AND occ.deleted=false + JOIN samples s ON s.id=o.sample_id AND s.deleted=false JOIN cache_samples_nonfunctional snf ON snf.id=o.sample_id JOIN cache_taxa_taxon_lists cttl ON cttl.id=o.taxa_taxon_list_id LEFT JOIN locations l ON l.id=o.location_id AND l.deleted=false; @@ -42,10 +43,10 @@ SET recorded_parent_location_id = lp.id, recorded_parent_location_name = lp.name, recorded_parent_location_code = lp.code - FROM samples sc + FROM samples sc JOIN locations lc ON lc.id=sc.location_id AND lc.deleted=false - JOIN locations lp ON lp.id = lc.parent_id AND lp.deleted=false - WHERE sc.id=o.sample_id AND sc.deleted=false + JOIN locations lp ON lp.id = lc.parent_id AND lp.deleted=false + WHERE sc.id=o.sample_id AND sc.deleted=false AND o.recorded_parent_location_id IS NULL; SELECT * @@ -66,6 +67,7 @@ + diff --git a/reports/library/occurrences/list_for_elastic_sensitive.xml b/reports/library/occurrences/list_for_elastic_sensitive.xml index 1f90073ecb..726aa87859 100644 --- a/reports/library/occurrences/list_for_elastic_sensitive.xml +++ b/reports/library/occurrences/list_for_elastic_sensitive.xml @@ -69,6 +69,7 @@ + @@ -116,11 +117,11 @@ - - - diff --git a/reports/library/occurrences/list_for_elastic_sensitive_all.xml b/reports/library/occurrences/list_for_elastic_sensitive_all.xml index a0c35f5478..1b93bf3942 100644 --- a/reports/library/occurrences/list_for_elastic_sensitive_all.xml +++ b/reports/library/occurrences/list_for_elastic_sensitive_all.xml @@ -1,7 +1,7 @@ @@ -68,6 +68,7 @@ + @@ -115,11 +116,11 @@ - - - diff --git a/reports/library/samples/list_for_elastic.xml b/reports/library/samples/list_for_elastic.xml new file mode 100644 index 0000000000..ead11eaf8f --- /dev/null +++ b/reports/library/samples/list_for_elastic.xml @@ -0,0 +1,199 @@ + + + DROP TABLE IF EXISTS filtered_samples; + DROP TABLE IF EXISTS occurrence_stats; + DROP TABLE IF EXISTS output_rows; + DROP TABLE IF EXISTS sample_occurrence_list; + + SELECT s.* + INTO TEMPORARY filtered_samples + FROM cache_samples_functional s + #agreements_join# + #joins# + WHERE #sharing_filter# + #filters# + #order_by# + LIMIT #limit#; + + SELECT DISTINCT s.id as sample_id, o.id as occurrence_id, o.taxa_taxon_list_id, o.confidential, o.release_status + INTO TEMPORARY sample_occurrences_list + FROM filtered_samples s + JOIN cache_occurrences_functional o ON o.sample_id=s.id OR o.parent_sample_id=s.id; + + SELECT s.id as sample_id, COUNT(DISTINCT ol.occurrence_id) AS count_occurrences, COUNT(distinct cttl.taxon_meaning_id) AS count_taxa, COUNT(distinct cttl.taxon_group_id) AS count_taxon_groups, + SUM(case when onf.attr_sex_stage_count similar to '[0-9]{1,9}' then onf.attr_sex_stage_count::integer else null end) AS sum_individual_count, + MAX(onf.sensitivity_precision) AS max_sensitivity_precision, BOOL_OR(ol.confidential) AS any_confidential, string_agg(DISTINCT ol.release_status, '') as all_release_status + INTO TEMPORARY occurrence_stats + FROM filtered_samples s + LEFT JOIN sample_occurrences_list ol ON ol.sample_id=s.id + LEFT JOIN cache_occurrences_nonfunctional onf ON onf.id=ol.occurrence_id + LEFT JOIN cache_taxa_taxon_lists cttl ON cttl.id=ol.taxa_taxon_list_id + GROUP BY s.id; + + SELECT #columns# + INTO TEMPORARY output_rows + FROM filtered_samples s + JOIN samples smp ON smp.id=s.id + JOIN occurrence_stats os ON os.sample_id=s.id + JOIN cache_samples_nonfunctional snf ON snf.id=s.id + LEFT JOIN locations l ON l.id=s.location_id AND l.deleted=false; + + UPDATE output_rows o + SET recorded_parent_location_id = lp.id, + recorded_parent_location_name = lp.name, + recorded_parent_location_code = lp.code, + parent_sample_attrs_json=snfp.attrs_json + FROM samples sp + JOIN cache_samples_nonfunctional snfp ON snfp.id=sp.id + LEFT JOIN locations lp ON lp.id=sp.location_id AND lp.deleted=false + WHERE sp.id=o.parent_sample_id AND sp.deleted=false; + + -- Use parents of locations if no parent sample location. + UPDATE output_rows o + SET recorded_parent_location_id = lp.id, + recorded_parent_location_name = lp.name, + recorded_parent_location_code = lp.code + FROM samples sc + JOIN locations lc ON lc.id=sc.location_id AND lc.deleted=false + JOIN locations lp ON lp.id = lc.parent_id AND lp.deleted=false + WHERE sc.id=o.sample_id AND sc.deleted=false + AND o.recorded_parent_location_id IS NULL; + + SELECT * + FROM output_rows s + #order_by# + + + + + s.id > #last_id# + + + s.tracking >= #autofeed_tracking_from# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/reports/library/samples/list_for_elastic_all.xml b/reports/library/samples/list_for_elastic_all.xml new file mode 100644 index 0000000000..8203fa34e0 --- /dev/null +++ b/reports/library/samples/list_for_elastic_all.xml @@ -0,0 +1,199 @@ + + + DROP TABLE IF EXISTS filtered_samples; + DROP TABLE IF EXISTS occurrence_stats; + DROP TABLE IF EXISTS output_rows; + DROP TABLE IF EXISTS sample_occurrence_list; + + SELECT s.* + INTO TEMPORARY filtered_samples + FROM cache_samples_functional s + #joins# + WHERE 1=1 + #filters# + #order_by# + LIMIT #limit#; + + SELECT DISTINCT s.id as sample_id, o.id as occurrence_id, o.taxa_taxon_list_id, o.confidential, o.release_status + INTO TEMPORARY sample_occurrences_list + FROM filtered_samples s + JOIN cache_occurrences_functional o ON o.sample_id=s.id OR o.parent_sample_id=s.id; + + SELECT s.id as sample_id, COUNT(DISTINCT ol.occurrence_id) AS count_occurrences, COUNT(distinct cttl.taxon_meaning_id) AS count_taxa, COUNT(distinct cttl.taxon_group_id) AS count_taxon_groups, + SUM(case when onf.attr_sex_stage_count similar to '[0-9]{1,9}' then onf.attr_sex_stage_count::integer else null end) AS sum_individual_count, + MAX(onf.sensitivity_precision) AS max_sensitivity_precision, BOOL_OR(ol.confidential) AS any_confidential, string_agg(DISTINCT ol.release_status, '') as all_release_status + INTO TEMPORARY occurrence_stats + FROM filtered_samples s + LEFT JOIN sample_occurrences_list ol ON ol.sample_id=s.id + LEFT JOIN cache_occurrences_nonfunctional onf ON onf.id=ol.occurrence_id + LEFT JOIN cache_taxa_taxon_lists cttl ON cttl.id=ol.taxa_taxon_list_id + GROUP BY s.id; + + SELECT #columns# + INTO TEMPORARY output_rows + FROM filtered_samples s + JOIN samples smp ON smp.id=s.id + JOIN occurrence_stats os ON os.sample_id=s.id + JOIN cache_samples_nonfunctional snf ON snf.id=s.id + LEFT JOIN locations l ON l.id=s.location_id AND l.deleted=false; + + UPDATE output_rows o + SET recorded_parent_location_id = lp.id, + recorded_parent_location_name = lp.name, + recorded_parent_location_code = lp.code, + parent_sample_attrs_json=snfp.attrs_json + FROM samples sp + JOIN cache_samples_nonfunctional snfp ON snfp.id=sp.id + LEFT JOIN locations lp ON lp.id=sp.location_id AND lp.deleted=false + WHERE sp.id=o.parent_sample_id AND sp.deleted=false; + + -- Use parents of locations if no parent sample location. + UPDATE output_rows o + SET recorded_parent_location_id = lp.id, + recorded_parent_location_name = lp.name, + recorded_parent_location_code = lp.code + FROM samples sc + JOIN locations lc ON lc.id=sc.location_id AND lc.deleted=false + JOIN locations lp ON lp.id = lc.parent_id AND lp.deleted=false + WHERE sc.id=o.sample_id AND sc.deleted=false + AND o.recorded_parent_location_id IS NULL; + + SELECT * + FROM output_rows s + #order_by# + + + + + s.id > #last_id# + + + s.tracking >= #autofeed_tracking_from# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/reports/library/samples/list_for_elastic_sensitive.xml b/reports/library/samples/list_for_elastic_sensitive.xml new file mode 100644 index 0000000000..56dd315aad --- /dev/null +++ b/reports/library/samples/list_for_elastic_sensitive.xml @@ -0,0 +1,201 @@ + + + DROP TABLE IF EXISTS filtered_samples; + DROP TABLE IF EXISTS occurrence_stats; + DROP TABLE IF EXISTS output_rows; + DROP TABLE IF EXISTS sample_occurrence_list; + + SELECT s.* + INTO TEMPORARY filtered_samples + FROM cache_samples_functional s + #agreements_join# + #joins# + WHERE #sharing_filter# + AND s.sensitive=true OR s.private=true + #filters# + #order_by# + LIMIT #limit#; + + SELECT DISTINCT s.id as sample_id, o.id as occurrence_id, o.taxa_taxon_list_id, o.confidential, o.release_status + INTO TEMPORARY sample_occurrences_list + FROM filtered_samples s + JOIN cache_occurrences_functional o ON o.sample_id=s.id OR o.parent_sample_id=s.id; + + SELECT s.id as sample_id, COUNT(DISTINCT ol.occurrence_id) AS count_occurrences, COUNT(distinct cttl.taxon_meaning_id) AS count_taxa, COUNT(distinct cttl.taxon_group_id) AS count_taxon_groups, + SUM(case when onf.attr_sex_stage_count similar to '[0-9]{1,9}' then onf.attr_sex_stage_count::integer else null end) AS sum_individual_count, + MAX(onf.sensitivity_precision) AS max_sensitivity_precision, BOOL_OR(ol.confidential) AS any_confidential, string_agg(DISTINCT ol.release_status, '') as all_release_status + INTO TEMPORARY occurrence_stats + FROM filtered_samples s + LEFT JOIN sample_occurrences_list ol ON ol.sample_id=s.id + LEFT JOIN cache_occurrences_nonfunctional onf ON onf.id=ol.occurrence_id + LEFT JOIN cache_taxa_taxon_lists cttl ON cttl.id=ol.taxa_taxon_list_id + GROUP BY s.id; + + SELECT #columns# + INTO TEMPORARY output_rows + FROM filtered_samples s + JOIN samples smp ON smp.id=s.id + JOIN occurrence_stats os ON os.sample_id=s.id + JOIN cache_samples_nonfunctional snf ON snf.id=s.id + LEFT JOIN locations l ON l.id=smp.location_id AND l.deleted=false; + + UPDATE output_rows o + SET recorded_parent_location_id = lp.id, + recorded_parent_location_name = lp.name, + recorded_parent_location_code = lp.code, + parent_sample_attrs_json=snfp.attrs_json + FROM samples sp + JOIN cache_samples_nonfunctional snfp ON snfp.id=sp.id + LEFT JOIN locations lp ON lp.id=sp.location_id AND lp.deleted=false + WHERE sp.id=o.parent_sample_id AND sp.deleted=false; + + -- Use parents of locations if no parent sample location. + UPDATE output_rows o + SET recorded_parent_location_id = lp.id, + recorded_parent_location_name = lp.name, + recorded_parent_location_code = lp.code + FROM samples sc + JOIN locations lc ON lc.id=sc.location_id AND lc.deleted=false + JOIN locations lp ON lp.id = lc.parent_id AND lp.deleted=false + WHERE sc.id=o.sample_id AND sc.deleted=false + AND o.recorded_parent_location_id IS NULL; + + SELECT * + FROM output_rows s + #order_by# + + + + + s.id > #last_id# + + + s.tracking >= #autofeed_tracking_from# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/reports/library/samples/list_for_elastic_sensitive_all.xml b/reports/library/samples/list_for_elastic_sensitive_all.xml new file mode 100644 index 0000000000..c64ad5a143 --- /dev/null +++ b/reports/library/samples/list_for_elastic_sensitive_all.xml @@ -0,0 +1,200 @@ + + + DROP TABLE IF EXISTS filtered_samples; + DROP TABLE IF EXISTS occurrence_stats; + DROP TABLE IF EXISTS output_rows; + DROP TABLE IF EXISTS sample_occurrence_list; + + SELECT s.* + INTO TEMPORARY filtered_samples + FROM cache_samples_functional s + #joins# + WHERE s.sensitive=true OR s.private=true + #filters# + #order_by# + LIMIT #limit#; + + SELECT DISTINCT s.id as sample_id, o.id as occurrence_id, o.taxa_taxon_list_id, o.confidential, o.release_status + INTO TEMPORARY sample_occurrences_list + FROM filtered_samples s + JOIN cache_occurrences_functional o ON o.sample_id=s.id OR o.parent_sample_id=s.id; + + SELECT s.id as sample_id, COUNT(DISTINCT ol.occurrence_id) AS count_occurrences, COUNT(distinct cttl.taxon_meaning_id) AS count_taxa, COUNT(distinct cttl.taxon_group_id) AS count_taxon_groups, + SUM(case when onf.attr_sex_stage_count similar to '[0-9]{1,9}' then onf.attr_sex_stage_count::integer else null end) AS sum_individual_count, + MAX(onf.sensitivity_precision) AS max_sensitivity_precision, BOOL_OR(ol.confidential) AS any_confidential, string_agg(DISTINCT ol.release_status, '') as all_release_status + INTO TEMPORARY occurrence_stats + FROM filtered_samples s + LEFT JOIN sample_occurrences_list ol ON ol.sample_id=s.id + LEFT JOIN cache_occurrences_nonfunctional onf ON onf.id=ol.occurrence_id + LEFT JOIN cache_taxa_taxon_lists cttl ON cttl.id=ol.taxa_taxon_list_id + GROUP BY s.id; + + SELECT #columns# + INTO TEMPORARY output_rows + FROM filtered_samples s + JOIN samples smp ON smp.id=s.id + JOIN occurrence_stats os ON os.sample_id=s.id + JOIN cache_samples_nonfunctional snf ON snf.id=s.id + LEFT JOIN locations l ON l.id=smp.location_id AND l.deleted=false; + + UPDATE output_rows o + SET recorded_parent_location_id = lp.id, + recorded_parent_location_name = lp.name, + recorded_parent_location_code = lp.code, + parent_sample_attrs_json=snfp.attrs_json + FROM samples sp + JOIN cache_samples_nonfunctional snfp ON snfp.id=sp.id + LEFT JOIN locations lp ON lp.id=sp.location_id AND lp.deleted=false + WHERE sp.id=o.parent_sample_id AND sp.deleted=false; + + -- Use parents of locations if no parent sample location. + UPDATE output_rows o + SET recorded_parent_location_id = lp.id, + recorded_parent_location_name = lp.name, + recorded_parent_location_code = lp.code + FROM samples sc + JOIN locations lc ON lc.id=sc.location_id AND lc.deleted=false + JOIN locations lp ON lp.id = lc.parent_id AND lp.deleted=false + WHERE sc.id=o.sample_id AND sc.deleted=false + AND o.recorded_parent_location_id IS NULL; + + SELECT * + FROM output_rows s + #order_by# + + + + + s.id > #last_id# + + + s.tracking >= #autofeed_tracking_from# + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/reports/library/samples/list_sample_deletions.xml b/reports/library/samples/list_sample_deletions.xml new file mode 100644 index 0000000000..83be0c0105 --- /dev/null +++ b/reports/library/samples/list_sample_deletions.xml @@ -0,0 +1,31 @@ + + + SELECT #columns# + FROM samples s + JOIN surveys srv ON srv.id=s.survey_id + #agreements_join# + #joins# + WHERE #sharing_filter# + AND s.deleted=true + + + s.id + + + + s.id > #last_id# + + + s.updated_on >= '#autofeed_tracking_date_from#' + + + + + + + + \ No newline at end of file diff --git a/reports/library/samples/list_sample_deletions_all.xml b/reports/library/samples/list_sample_deletions_all.xml new file mode 100644 index 0000000000..129f19c9d2 --- /dev/null +++ b/reports/library/samples/list_sample_deletions_all.xml @@ -0,0 +1,29 @@ + + + SELECT #columns# + FROM samples s + #joins# + WHERE s.deleted=true + + + s.id + + + + s.id > #last_id# + + + s.updated_on >= '#autofeed_tracking_date_from#' + + + + + + + + \ No newline at end of file diff --git a/reports/reports_for_prebuilt_forms/report_calendar_grid/locations_for_cms_user.xml b/reports/reports_for_prebuilt_forms/report_calendar_grid/locations_for_cms_user.xml new file mode 100644 index 0000000000..0f5803ead2 --- /dev/null +++ b/reports/reports_for_prebuilt_forms/report_calendar_grid/locations_for_cms_user.xml @@ -0,0 +1,30 @@ + + + SELECT #columns# + FROM locations l + JOIN location_attribute_values v ON v.location_id=l.id AND v.deleted=false AND v.int_value=#cms_user_id# + JOIN location_attributes a ON a.id=v.location_attribute_id AND a.deleted=false AND a.caption='CMS User ID' + JOIN location_attributes_websites aw ON aw.location_attribute_id=a.id AND aw.deleted=false AND aw.restrict_to_survey_id=#survey_id# + JOIN locations_websites lw ON lw.location_id=l.id AND lw.deleted=false AND lw.website_id in (#website_ids#) + WHERE l.deleted=false + #filters# + + + l.name ASC + + + + + + l.location_type_id=#location_type_id# + + + + + + + + \ No newline at end of file diff --git a/system/core/utf8.php b/system/core/utf8.php index 9f20f421c7..61f6933aa9 100644 --- a/system/core/utf8.php +++ b/system/core/utf8.php @@ -49,7 +49,8 @@ ); } -if (extension_loaded('mbstring') AND (ini_get('mbstring.func_overload') & MB_OVERLOAD_STRING)) +// Function overloading was deprecated in PHP 7.2 and removed in PHP 8.0. +if (extension_loaded('mbstring') AND defined('MB_OVERLOAD_STRING') AND (ini_get('mbstring.func_overload') & MB_OVERLOAD_STRING)) { trigger_error ( diff --git a/system/libraries/drivers/Database/Pgsql.php b/system/libraries/drivers/Database/Pgsql.php index 0731e6eb1c..3a09101cf0 100644 --- a/system/libraries/drivers/Database/Pgsql.php +++ b/system/libraries/drivers/Database/Pgsql.php @@ -326,7 +326,7 @@ class Pgsql_Result extends Database_Result { * @param boolean return objects or arrays * @param string SQL query that was run */ - public function __construct($result, $link, $object = TRUE, $sql) + public function __construct($result, $link, $object, $sql) { $this->link = $link; $this->result = $result; @@ -453,10 +453,12 @@ public function insert_id() $ER = error_reporting(0); $result = pg_query($this->link, $query); - $insert_id = pg_fetch_array($result, NULL, PGSQL_ASSOC); - - $this->insert_id = $insert_id['insert_id']; - + // $result is false when tables have no serial column. + if ($result !== false) { + $insert_id = pg_fetch_array($result, NULL, PGSQL_ASSOC); + $this->insert_id = $insert_id['insert_id']; + } + // Reset error reporting error_reporting($ER); } diff --git a/system/vendor/swift/Swift/Connection/SMTP.php b/system/vendor/swift/Swift/Connection/SMTP.php index a16f0306c1..393261b8ae 100644 --- a/system/vendor/swift/Swift/Connection/SMTP.php +++ b/system/vendor/swift/Swift/Connection/SMTP.php @@ -253,7 +253,7 @@ public function read() $this->smtpErrors()); } $ret .= trim($tmp) . "\r\n"; - if ($tmp{3} == " ") break; + if ($tmp[3] == " ") break; } return $ret = substr($ret, 0, -2); } @@ -385,7 +385,7 @@ public function runAuthenticators($user, $pass, Swift $swift) foreach ($this->authenticators as $name => $obj) { //Server supports this authentication mechanism - if (in_array($name, $this->getAttributes("AUTH")) || $name{0} == "*") + if (in_array($name, $this->getAttributes("AUTH")) || $name[0] == "*") { $tried++; if ($log->hasLevel(Swift_Log::LOG_EVERYTHING))