diff --git a/.travis.yml b/.travis.yml index ebd177b06f..a68d7e5f9e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -70,8 +70,9 @@ before_script: - .travis/postgres_setup.sh # Enable the phpunit module in config.php (meaning initialise() is not tested) - cp application/config/config.php.travis application/config/config.php - # Provide a config file for the rest_api and request_logging modules + # Provide a config file for the rest_api, spatial_index_builder and request_logging modules - cp modules/rest_api/config/rest.php.travis modules/rest_api/config/rest.php + - cp modules/spatial_index_builder/config/spatial_index_builder.php.travis modules/spatial_index_builder/config/spatial_index_builder.php - cp modules/request_logging/config/request_logging.example.php modules/request_logging/config/request_logging.php # Downgrade to PHPUnit 5.7 - wget https://phar.phpunit.de/phpunit-5.7.phar diff --git a/application/config/sref_notations.php b/application/config/sref_notations.php index 2ea687ce6f..ffd7a12e63 100644 --- a/application/config/sref_notations.php +++ b/application/config/sref_notations.php @@ -35,6 +35,8 @@ '3109' => 'ETRS89 / Jersey Transverse Mercator', '23030' => 'ED50 / UTM zone 30N', '29902' => 'TM65 / Irish Grid', + '3006' => 'SWEREF99 TM / Swedish Transverse Mercator', + '3021' => 'RT90 2.5 gon v / Swedish Grid', ]; // Set the internally stored geoms to use spherical mercator projection. @@ -45,6 +47,8 @@ '4326' => 5, '4277' => 5, '2169' => 0, + '3006' => 0, + '3021' => 0, ]; // provide a list of systems which translate x,y format into a proper Lat/Long format, and the default ouput format diff --git a/application/config/version.php b/application/config/version.php index 949e5bfa5a..682c078ee9 100644 --- a/application/config/version.php +++ b/application/config/version.php @@ -29,14 +29,14 @@ * * @var string */ -$config['version'] = '3.0.1'; +$config['version'] = '3.1.0'; /** * Version release date. * * @var string */ -$config['release_date'] = '2019-12-10'; +$config['release_date'] = '2020-01-15'; /** * Link to the code repository downloads page. diff --git a/application/helpers/spatial_ref.php b/application/helpers/spatial_ref.php index d63c9c52f8..cbb63b1d19 100644 --- a/application/helpers/spatial_ref.php +++ b/application/helpers/spatial_ref.php @@ -27,38 +27,41 @@ class spatial_ref { /** * Retrieves the metadata for all the supported spatial reference systems. * - * @param boolean $refresh Set to true to force a full refresh, otherwise the result is cached. + * @param bool $refresh + * Set to true to force a full refresh, otherwise the result is cached. */ - public static function system_metadata($refresh=false) { + public static function system_metadata($refresh = FALSE) { $cacheId = 'spatial-ref-systems'; if ($refresh) { $cache = Cache::instance(); $cache->delete($cacheId); - self::$system_metadata=false; + self::$system_metadata = FALSE; } - if (self::$system_metadata === false) { + if (self::$system_metadata === FALSE) { $latlong_systems = Kohana::config('sref_notations.lat_long_systems'); $cache = Cache::instance(); if ($cached = $cache->get($cacheId)) { self::$system_metadata = $cached; - } else { + } + else { self::$system_metadata = array(); // fetch any systems that are just declared as srids, with no notation module required foreach (Kohana::config('sref_notations.sref_notations') as $code => $title) { self::$system_metadata["EPSG:$code"] = array( 'title' => $title, 'srid' => $code, - // make a small assumption here, that a non-lat long system is going to be an x y grid of metres. - 'treat_srid_as_x_y_metres' => !isset($latlong_systems[$code]) + // make a small assumption here, that a non-lat long system is + // going to be an x y grid of metres. + 'treat_srid_as_x_y_metres' => !isset($latlong_systems[$code]), ); } - // Now look for any modules which extend the sref systems available + // Now look for any modules which extend the sref systems available. foreach (Kohana::config('config.modules') as $path) { $plugin = basename($path); if (file_exists("$path/plugins/$plugin.php")) { - require_once("$path/plugins/$plugin.php"); - if (function_exists($plugin.'_sref_systems')) { - $metadata = call_user_func($plugin.'_sref_systems'); + require_once "$path/plugins/$plugin.php"; + if (function_exists($plugin . '_sref_systems')) { + $metadata = call_user_func($plugin . '_sref_systems'); self::$system_metadata = array_merge(self::$system_metadata, $metadata); } } diff --git a/application/models/location.php b/application/models/location.php index 2a222bca0b..2e2eacc853 100644 --- a/application/models/location.php +++ b/application/models/location.php @@ -238,19 +238,23 @@ public function calcCentroid($boundary, $system = '4326') { } private static function getConvertedOptionValue($first, $second) { - return str_replace(array(',', ':'), array(',', '8'), $first) . + return str_replace(array(',', ':'), array(',', ':'), $first) . ":" . - str_replace(array(',', ':'), array(',', '8'), $second); + str_replace(array(',', ':'), array(',', ':'), $second); } /** - * Define a form that is used to capture a set of predetermined values that apply to every record during an import. + * Defines inputs required for values that apply to all imported records. + * + * Define a form that is used to capture a set of predetermined values that + * apply to every record during an import. */ - public function fixed_values_form() { - $srefs = array(); - $systems = spatial_ref::system_metadata(); - foreach ($systems as $code => $metadata) - $srefs[] = self::getConvertedOptionValue($code, $metadata['title']); + public function fixedValuesForm() { + $srefs = []; + $systems = spatial_ref::system_list(); + foreach ($systems as $code => $title) { + $srefs[] = self::getConvertedOptionValue($code, $title); + } $location_types = array(":Defined in file"); $parent_location_types = array(":No filter"); @@ -265,28 +269,28 @@ public function fixed_values_form() { 'display' => 'Website', 'description' => 'Select the website to import records into.', 'datatype' => 'lookup', - 'population_call' => 'direct:website:id:title' + 'population_call' => 'direct:website:id:title', ), 'location:centroid_sref_system' => array( 'display' => 'Spatial Ref. System', 'description' => 'Select the spatial reference system used in this import file. Note, if you have a file with a mix of spatial reference systems then you need a ' . 'column in the import file which is mapped to the Location Spatial Reference System field containing the spatial reference system code.', 'datatype' => 'lookup', - 'lookup_values'=>implode(',', $srefs) + 'lookup_values' => implode(',', $srefs), ), 'location:location_type_id' => array( 'display' => 'Location Type', 'description' => 'Select the Location Type for all locations in this import file. Note, if you have a file with a mix of location type then you need a ' . 'column in the import file which is mapped to the Location Type field.', 'datatype' => 'lookup', - 'lookup_values' => implode(',', $location_types) + 'lookup_values' => implode(',', $location_types), ), 'fkFilter:location:location_type_id' => array( 'display' => 'Parent Location Type', 'description' => 'If this import file includes locations which reference parent locations records, you can restrict the type of parent locations looked ' . 'up by setting this location type. It is not currently possible to use a column in the file to do this on a location by location basis.', 'datatype' => 'lookup', - 'lookup_values' => implode(',', $parent_location_types) + 'lookup_values' => implode(',', $parent_location_types), ), ); } diff --git a/application/models/occurrence.php b/application/models/occurrence.php index 323564007c..bc29932c28 100644 --- a/application/models/occurrence.php +++ b/application/models/occurrence.php @@ -570,22 +570,22 @@ private function _check_module_active($module) { * * **occurrence_associations** - Set to 't' to enable occurrence associations options. The * relevant warehouse module must also be enabled. */ - public function fixed_values_form($options = array()) { + public function fixedValuesForm($options = array()) { $srefs = array(); $systems = spatial_ref::system_list(); foreach ($systems as $code => $title) { - $srefs[] = str_replace(array(',', ':'), array(',', '8'), $code) . + $srefs[] = str_replace(array(',', ':'), array(',', ':'), $code) . ":" . - str_replace(array(',', ':'), array(',', '8'), $title); + str_replace(array(',', ':'), array(',', ':'), $title); } $sample_methods = array(":Defined in file"); $parent_sample_methods = array(":No filter"); $terms = $this->db->select('id, term')->from('list_termlists_terms')->where('termlist_external_key', 'indicia:sample_methods')->orderby('term', 'asc')->get()->result(); foreach ($terms as $term) { - $sample_method = str_replace(array(',', ':'), array(',', '8'), $term->id) . + $sample_method = str_replace(array(',', ':'), array(',', ':'), $term->id) . ":" . - str_replace(array(',', ':'), array(',', '8'), $term->term); + str_replace(array(',', ':'), array(',', ':'), $term->term); $sample_methods[] = $sample_method; $parent_sample_methods[] = $sample_method; } @@ -593,9 +593,9 @@ public function fixed_values_form($options = array()) { $locationTypes = array(":No filter"); $terms = $this->db->select('id, term')->from('list_termlists_terms')->where('termlist_external_key', 'indicia:location_types')->orderby('term', 'asc')->get()->result(); foreach ($terms as $term) { - $locationTypes[] = str_replace(array(',', ':'), array(',', '8'), $term->id) . + $locationTypes[] = str_replace(array(',', ':'), array(',', ':'), $term->id) . ":" . - str_replace(array(',', ':'), array(',', '8'), $term->term); + str_replace(array(',', ':'), array(',', ':'), $term->term); } $retVal = array( 'website_id' => array( diff --git a/application/models/sample.php b/application/models/sample.php index 0069e32317..b8da8fa52a 100644 --- a/application/models/sample.php +++ b/application/models/sample.php @@ -346,7 +346,7 @@ public function caption() { /** * Define a form that is used to capture a set of predetermined values that apply to every record during an import. */ - public function fixed_values_form($options = array()) { + public function fixedValuesForm($options = array()) { $srefs = array(); $systems = spatial_ref::system_list(); foreach ($systems as $code => $title) diff --git a/application/models/taxa_taxon_list.php b/application/models/taxa_taxon_list.php index 5c4a01dd38..d1a1153c77 100644 --- a/application/models/taxa_taxon_list.php +++ b/application/models/taxa_taxon_list.php @@ -463,7 +463,7 @@ public function getDefaults() { * Define a form that is used to capture a set of predetermined values that * apply to every record during an import. */ - public function fixed_values_form() { + public function fixedValuesForm() { return array( 'taxa_taxon_list:taxon_list_id' => array( 'display' => 'Species List', diff --git a/application/models/termlists_term.php b/application/models/termlists_term.php index f4fc90ce3c..0f737b5bbf 100644 --- a/application/models/termlists_term.php +++ b/application/models/termlists_term.php @@ -325,7 +325,7 @@ public function getDefaults() { /** * Define a form that is used to capture a set of predetermined values that apply to every record during an import. */ - public function fixed_values_form() { + public function fixedValuesForm() { return array( 'termlists_term:termlist_id' => array( 'display' => 'Termlist', diff --git a/application/tests/helpers/vague_dateTest.php b/application/tests/helpers/vague_dateTest.php index fc5d69eda2..c76a2b41a6 100644 --- a/application/tests/helpers/vague_dateTest.php +++ b/application/tests/helpers/vague_dateTest.php @@ -187,6 +187,7 @@ public function testVagueDateToString($from, $to, $type, $expected) { public function provideStringToVagueDate() { $year = date('Y'); $lastYear = $year - 1; + $lastDayInFeb = date('L') === '1' ? '29' : '28'; return [ 'Date 1997-08-02' => ['1997-08-02', '1997-08-02', '1997-08-02', 'D'], 'Date 02/08/1997' => ['02/08/1997', '1997-08-02', '1997-08-02', 'D'], @@ -269,7 +270,7 @@ public function provideStringToVagueDate() { 'Season Autumn 92' => ['Autumn 92', '1992-09-01', '1992-11-30', 'P'], // Month only and season only years are always for the current year. 'Month only March' => ['March', "$year-03-01", "$year-03-31", 'M'], - 'Season only Winter' => ['Winter', "$lastYear-12-01", "$year-02-28", 'S'], + 'Season only Winter' => ['Winter', "$lastYear-12-01", "$year-02-$lastDayInFeb", 'S'], 'Season only Spring' => ['Spring', "$year-03-01", "$year-05-31", 'S'], 'Season only Summer' => ['Summer', "$year-06-01", "$year-08-31", 'S'], 'Season only Autumn' => ['Autumn', "$year-09-01", "$year-11-30", 'S'], diff --git a/application/views/templates/template.php b/application/views/templates/template.php index 895da24407..846baa2801 100644 --- a/application/views/templates/template.php +++ b/application/views/templates/template.php @@ -56,7 +56,6 @@ ); echo html::script( array( - 'media/js/json2.js', 'media/js/jquery.js?v=3.2.1', 'media/js/jquery.url.js', 'media/js/fancybox/source/jquery.fancybox.pack.js', diff --git a/client_helpers b/client_helpers index cb618a1597..4d64e4b2af 160000 --- a/client_helpers +++ b/client_helpers @@ -1 +1 @@ -Subproject commit cb618a15977df42c21f713e7f92be43a861941f8 +Subproject commit 4d64e4b2af2e5869c1ded5c28402c504619f2cb5 diff --git a/media b/media index 3391d18811..7863d28cc1 160000 --- a/media +++ b/media @@ -1 +1 @@ -Subproject commit 3391d188117dc8e2bb9d251e2c403220c41dee82 +Subproject commit 7863d28cc100906f603746d91f2724464d6a9fc4 diff --git a/modules/attribute_sets/helpers/attribute_sets.php b/modules/attribute_sets/helpers/attribute_sets.php index a269777a8c..12ac7ecc9e 100644 --- a/modules/attribute_sets/helpers/attribute_sets.php +++ b/modules/attribute_sets/helpers/attribute_sets.php @@ -435,13 +435,61 @@ private static function deleteAttributesTaxonRestriction($db, $model) { and aw.{$entity}_attribute_id=attla.{$entity}_attribute_id and aw.deleted=false where astr.deleted=false - and atr.{$entity}_attributes_website_id=atr.id + and atr.{$entity}_attributes_website_id=aw.id and atr.restrict_to_taxon_meaning_id=astr.restrict_to_taxon_meaning_id and coalesce(atr.restrict_to_stage_term_meaning_id, 0)=coalesce(astr.restrict_to_stage_term_meaning_id, 0) ); SQL; $db->query($qry); } + + // taxa_taxon_list_attributes do not use a website table so the above SQL does not work + // This code is a version of the above code for taxa_tax_list_attributes + $ttlQry = <<id + and atr.taxon_lists_taxa_taxon_list_attribute_id=tlttla.id + and atr.restrict_to_taxon_meaning_id=astr.restrict_to_taxon_meaning_id + and coalesce(atr.restrict_to_stage_term_meaning_id, 0)=coalesce(astr.restrict_to_stage_term_meaning_id, 0) + and atr.deleted=false + and atr.id not in ( + -- Exclude deletions for any restriction that is still valid because of + -- another attribute set. + select atr.id + from taxa_taxon_list_attribute_taxon_restrictions, attribute_sets_taxon_restrictions astr + join attribute_sets_surveys ass + on ass.id=astr.attribute_sets_survey_id + and ass.deleted=false + join attribute_sets aset + on aset.id=ass.attribute_set_id + and aset.deleted=false + join attribute_sets_taxa_taxon_list_attributes asttla + on asttla.attribute_set_id=ass.attribute_set_id + and asttla.deleted=false + join taxon_lists_taxa_taxon_list_attributes tlttla + on tlttla.taxon_list_id=aset.taxon_list_id + and tlttla.taxa_taxon_list_attribute_id=asttla.taxa_taxon_list_attribute_id + and tlttla.deleted=false + where astr.deleted=false + --AVB changed this in original code + and atr.taxon_lists_taxa_taxon_list_attribute_id=tlttla.id + and atr.restrict_to_taxon_meaning_id=astr.restrict_to_taxon_meaning_id + and coalesce(atr.restrict_to_stage_term_meaning_id, 0)=coalesce(astr.restrict_to_stage_term_meaning_id, 0) + ); +SQL; + $db->query($ttlQry); } /** diff --git a/modules/auto_verify/config/auto_verify.php.example.php b/modules/auto_verify/config/auto_verify.php.example.php index 0d0265dd0d..8467a9575f 100644 --- a/modules/auto_verify/config/auto_verify.php.example.php +++ b/modules/auto_verify/config/auto_verify.php.example.php @@ -1,4 +1,5 @@ TRUE, + 'always_run' => TRUE, + ); +} + /** * Hook into the task scheduler. When run, the system checks the * cache_occurrences_functional table for records where the data cleaner has * marked the record as data_cleaner_result to true, record_status="C", the system * then sets the record to verified automatically (subject to taxon restriction * tests descrbed below). - * If the survey associated with the record has a value in its - * auto_accept_taxa_filters field, then the taxon_meaning_id associated with + * If the survey associated with the record has a value in its + * auto_accept_taxa_filters field, then the taxon_meaning_id associated with * the record has to be the same as, or a decendent of, one of the * taxa stored in the auto_accept_taxa_filters to qualify for auto verification. * @@ -40,105 +53,107 @@ */ function auto_verify_scheduled_task($last_run_date, $db) { $autoVerifyNullIdDiff = kohana::config('auto_verify.auto_accept_occurrences_with_null_id_difficulty', FALSE, FALSE); - - $oldestRecordCreatedDateToProcess = kohana::config('auto_verify.oldest_record_created_date_to_process', FALSE, FALSE); - $oldestOccurrenceIdToProcess = kohana::config('auto_verify.oldest_occurrence_id_to_process', FALSE, FALSE); - $maxRecordsNumber = kohana::config('auto_verify.max_num_records_to_process_at_once', FALSE, FALSE); - if (empty($autoVerifyNullIdDiff)) { echo "Unable to automatically verify occurrences when the auto_accept_occurrences_with_null_id_difficulty entry is empty.
"; kohana::log('error', 'Unable to automatically verify occurrences when the auto_accept_occurrences_with_null_id_difficulty configuration entry is empty.'); return FALSE; } + $oldestRecordCreatedDateToProcess = kohana::config('auto_verify.oldest_record_created_date_to_process', FALSE, FALSE); if (empty($oldestRecordCreatedDateToProcess)) { echo "Unable to automatically verify occurrences when the oldest_record_created_date_to_process entry is empty.
"; kohana::log('error', 'Unable to automatically verify occurrences when the oldest_record_created_date_to_process configuration entry is empty.'); return FALSE; } - if (empty($oldestOccurrenceIdToProcess)) { - echo "Unable to automatically verify occurrences when the oldest_occurrence_id_to_process entry is empty.
"; - kohana::log('error', 'Unable to automatically verify occurrences when the oldest_occurrence_id_to_process configuration entry is empty.'); - return FALSE; - } + $maxRecordsNumber = kohana::config('auto_verify.max_num_records_to_process_at_once', FALSE, FALSE); + $maxRecordsNumber = $maxRecordsNumber ? $maxRecordsNumber : 1000; - if (empty($maxRecordsNumber)) { - echo "Unable to automatically verify occurrences when the max_num_records_to_process_at_once entry is empty.
"; - kohana::log('error', 'Unable to automatically verify occurrences when the max_num_records_to_process_at_once configuration entry is empty.'); - return FALSE; + $mode = variable::get('auto_verify_mode', 'initial'); + if ($mode === 'initial') { + // Use ID from last scan or config setting if starting from scratch. + $startId = variable::get( + 'auto_verify_start_at_id', + kohana::config('auto_verify.oldest_occurrence_id_to_process', FALSE, FALSE) + ); + // Default lastId to 0 if not configured and first time. + $startId = $startId ? $startId : 1; + $mainTable = 'cache_occurrences_functional'; + $qryEnd = <<= $startId +AND delta.created_on >= TO_TIMESTAMP('$oldestRecordCreatedDateToProcess', 'DD/MM/YYYY') +AND delta.updated_on >= TO_TIMESTAMP('$oldestRecordCreatedDateToProcess', 'DD/MM/YYYY') +ORDER BY delta.id +LIMIT $maxRecordsNumber +SQL; } - - $subQuery = " - SELECT distinct delta.id - FROM cache_occurrences_functional delta - JOIN surveys s on s.id = delta.survey_id AND s.auto_accept=true AND s.deleted=false - LEFT JOIN cache_taxon_searchterms cts on cts.taxon_meaning_id = delta.taxon_meaning_id - WHERE delta.data_cleaner_result=true - AND delta.record_status='C' AND delta.record_substatus IS NULL - AND delta.created_on >= TO_TIMESTAMP('$oldestRecordCreatedDateToProcess', 'DD/MM/YYYY') - AND (($autoVerifyNullIdDiff=false AND cts.identification_difficulty IS NOT NULL AND cts.identification_difficulty<=s.auto_accept_max_difficulty) - OR ($autoVerifyNullIdDiff=true AND (cts.identification_difficulty IS NULL OR cts.identification_difficulty<=s.auto_accept_max_difficulty))) - AND (s.auto_accept_taxa_filters is null OR (s.auto_accept_taxa_filters && delta.taxon_path))"; - - if (isset($oldestOccurrenceIdToProcess) && $oldestOccurrenceIdToProcess > -1) { - $subQuery .= " - AND delta.id >= $oldestOccurrenceIdToProcess"; + else { + $mainTable = 'occdelta'; + // No limit or sort needed, just do contents of occdelta for changes only. + $qryEnd = ''; } + $verificationTime = gmdate("Y\/m\/d H:i:s"); + $query = << -1) { - $subQuery .= " - order by delta.id desc limit $maxRecordsNumber"; - } +SELECT DISTINCT delta.id +INTO TEMPORARY records_to_auto_verify +FROM $mainTable delta +JOIN surveys s ON s.id = delta.survey_id AND s.auto_accept=true AND s.deleted=false +LEFT JOIN cache_taxon_searchterms cts on cts.taxon_meaning_id = delta.taxon_meaning_id +WHERE delta.data_cleaner_result=true +AND delta.record_status='C' AND delta.record_substatus IS NULL +AND (($autoVerifyNullIdDiff=false AND cts.identification_difficulty IS NOT NULL AND cts.identification_difficulty<=s.auto_accept_max_difficulty) +OR ($autoVerifyNullIdDiff=true AND (cts.identification_difficulty IS NULL OR cts.identification_difficulty<=s.auto_accept_max_difficulty))) +AND (s.auto_accept_taxa_filters IS NULL OR (s.auto_accept_taxa_filters && delta.taxon_path)) +$qryEnd; - $verificationTime = gmdate("Y\/m\/d H:i:s"); - //Need to update cache_occurrences_*, as these tables have already been built at this point. - $query = " - INSERT INTO occurrence_comments (comment, generated_by, occurrence_id,record_status,record_substatus,created_by_id,updated_by_id,created_on,updated_on,auto_generated) - SELECT 'Accepted based on automatic checks', 'system', id,'V','2',1,1,'$verificationTime','$verificationTime',true - FROM occurrences - WHERE id in - ($subQuery); +INSERT INTO occurrence_comments (comment, generated_by, occurrence_id, record_status, record_substatus, created_by_id, updated_by_id, created_on, updated_on, auto_generated) +SELECT 'Accepted based on automatic checks', 'system', o.id, 'V', '2', 1, 1, '$verificationTime', '$verificationTime', true +FROM occurrences o +JOIN records_to_auto_verify rav ON rav.id=o.id; + +UPDATE occurrences o +SET + record_status='V', + record_substatus='2', + release_status='R', + verified_by_id=1, + verified_on='$verificationTime', + record_decision_source='M', + updated_on=now(), + updated_by_id=1 +FROM records_to_auto_verify rav +WHERE rav.id=o.id; - UPDATE occurrences - SET - record_status='V', - record_substatus='2', - release_status='R', - verified_by_id=1, - verified_on='$verificationTime', - record_decision_source='M', - updated_on = now(), - updated_by_id = 1 - WHERE id in - ($subQuery); +UPDATE cache_occurrences_functional o +SET + record_status='V', + record_substatus='2', + release_status='R', + verified_on='$verificationTime' +FROM records_to_auto_verify rav +WHERE rav.id=o.id; - UPDATE cache_occurrences_functional - SET - record_status='V', - record_substatus='2', - release_status='R', - verified_on='$verificationTime' - WHERE id in - ($subQuery); +UPDATE cache_occurrences_nonfunctional o +SET verifier='admin, core' +FROM records_to_auto_verify rav +WHERE rav.id=o.id; - UPDATE cache_occurrences_nonfunctional - SET verifier='admin, core' - WHERE id in - ($subQuery);"; - $results=$db->query($query)->result_array(FALSE); - // Query to return count of records, as I was unable to pursuade the above - // query to output the number of updated records correctly. - $query = " - SELECT count(id) - FROM cache_occurrences_functional co - WHERE co.verified_on='$verificationTime';"; - $results = $db->query($query)->result_array(FALSE); - if (!empty($results[0]['count']) && $results[0]['count'] > 1) { - echo $results[0]['count'] . ' occurrence records have been automatically verified.
'; +SQL; + $db->query($query); + // Grab stats about the records processed. + $query = "SELECT count(*), max(id) FROM records_to_auto_verify;"; + $stats = $db->query($query)->current(); + echo "$stats->count occurrence record(s) has been automatically verified.
"; + if ($mode === 'initial') { + if ($stats->count < $maxRecordsNumber) { + // Done, so switch mode. + variable::set('auto_verify_mode', 'updates'); + variable::delete('auto_verify_start_at_id'); + } + else { + variable::set('auto_verify_start_at_id', $stats->max + 1); + } } - elseif (!empty($results[0]['count']) && $results[0]['count'] === "1") - echo '1 occurrence record has been automatically verified.
'; - else - echo 'No occurrence records have been auto-verified.
'; } diff --git a/modules/cache_builder/helpers/task_cache_builder_attr_value_occurrence.php b/modules/cache_builder/helpers/task_cache_builder_attr_value_occurrence.php new file mode 100644 index 0000000000..a8d3d6d4d6 --- /dev/null +++ b/modules/cache_builder/helpers/task_cache_builder_attr_value_occurrence.php @@ -0,0 +1,141 @@ + av.date_start_value THEN ' - '::text || av.date_end_value::text ELSE '' END + ELSE NULL::text + END ORDER BY tlt.sort_order, t.term + ) as v + FROM work_queue q + JOIN occurrence_attribute_values avfilt ON avfilt.id=q.record_id + JOIN occurrence_attribute_values av ON av.occurrence_id=avfilt.occurrence_id AND av.deleted=false + AND COALESCE(av.int_value::text, av.text_value::text, av.float_value::text, av.date_start_value::text) IS NOT NULL + LEFT JOIN occurrence_attributes a ON a.id=av.occurrence_attribute_id AND a.deleted=false + LEFT JOIN termlists_terms tlt ON tlt.id=av.int_value AND a.data_type='L' AND tlt.deleted=false + LEFT JOIN terms t ON t.id=tlt.term_id AND t.deleted=false + WHERE q.entity='occurrence_attribute_value' AND q.task='task_cache_builder_attr_value_occurrence' AND claimed_by='$procId' + GROUP BY avfilt.occurrence_id, av.occurrence_attribute_id, a.multi_value + $langTermSql +) AS subquery +GROUP BY occurrence_id; + +UPDATE cache_occurrences_nonfunctional u +SET attrs_json=a.attrs +FROM attrs a +WHERE a.occurrence_id=u.id; + +DROP TABLE attrs; + +SQL; + $db->query($sql); + kohana::log('debug', $sql); + } + +} diff --git a/modules/cache_builder/helpers/task_cache_builder_attr_value_sample.php b/modules/cache_builder/helpers/task_cache_builder_attr_value_sample.php new file mode 100644 index 0000000000..f801844171 --- /dev/null +++ b/modules/cache_builder/helpers/task_cache_builder_attr_value_sample.php @@ -0,0 +1,140 @@ + av.date_start_value THEN ' - '::text || av.date_end_value::text ELSE '' END + ELSE NULL::text + END ORDER BY tlt.sort_order, t.term + ) as v + FROM work_queue q + JOIN sample_attribute_values avfilt ON avfilt.id=q.record_id + JOIN sample_attribute_values av ON av.sample_id=avfilt.sample_id AND av.deleted=false + AND COALESCE(av.int_value::text, av.text_value::text, av.float_value::text, av.date_start_value::text) IS NOT NULL + LEFT JOIN sample_attributes a ON a.id=av.sample_attribute_id AND a.deleted=false + LEFT JOIN termlists_terms tlt ON tlt.id=av.int_value AND a.data_type='L' AND tlt.deleted=false + LEFT JOIN terms t ON t.id=tlt.term_id AND t.deleted=false + WHERE q.entity='sample' AND q.task='task_cache_builder_attr_value_sample' AND claimed_by='$procId' + GROUP BY sample_id, sample_attribute_id, a.multi_value + $langTermSql +) AS subquery +GROUP BY sample_id; + +UPDATE cache_samples_nonfunctional u +SET attrs_json=a.attrs +FROM attrs a +WHERE a.sample_id=u.id; + +DROP TABLE attrs; + +SQL; + $db->query($sql); + } + +} diff --git a/modules/cache_builder/helpers/task_cache_builder_attr_value_taxa_taxon_list.php b/modules/cache_builder/helpers/task_cache_builder_attr_value_taxa_taxon_list.php new file mode 100644 index 0000000000..3987e95e5f --- /dev/null +++ b/modules/cache_builder/helpers/task_cache_builder_attr_value_taxa_taxon_list.php @@ -0,0 +1,140 @@ + av.date_start_value THEN ' - '::text || av.date_end_value::text ELSE '' END + ELSE NULL::text + END ORDER BY tlt.sort_order, t.term + ) as v + FROM work_queue q + JOIN taxa_taxon_list_attribute_values avfilt ON avfilt.id=q.record_id + JOIN taxa_taxon_list_attribute_values av ON av.taxa_taxon_list_id=avfilt.taxa_taxon_list_id AND av.deleted=false + AND COALESCE(av.int_value::text, av.text_value::text, av.float_value::text, av.date_start_value::text) IS NOT NULL + LEFT JOIN taxa_taxon_list_attributes a ON a.id=av.taxa_taxon_list_attribute_id AND a.deleted=false + LEFT JOIN termlists_terms tlt ON tlt.id=av.int_value AND a.data_type='L' AND tlt.deleted=false + LEFT JOIN terms t ON t.id=tlt.term_id AND t.deleted=false + WHERE q.entity='taxa_taxon_list' AND q.task='task_cache_builder_attrs_taxa_taxon_list' AND claimed_by='$procId' + GROUP BY taxa_taxon_list_id, taxa_taxon_list_attribute_id, a.multi_value + $langTermSql +) AS subquery +GROUP BY taxa_taxon_list_id; + +UPDATE cache_taxa_taxon_lists_nonfunctional u +SET attrs_json=a.attrs +FROM attrs a +WHERE a.taxa_taxon_list_id=u.id; + +DROP TABLE attrs; + +SQL; + $db->query($sql); + } + +} diff --git a/modules/cache_builder/helpers/task_cache_builder_attrs_occurrence.php b/modules/cache_builder/helpers/task_cache_builder_attrs_occurrence.php index 39feca3f93..8936b599f6 100644 --- a/modules/cache_builder/helpers/task_cache_builder_attrs_occurrence.php +++ b/modules/cache_builder/helpers/task_cache_builder_attrs_occurrence.php @@ -90,7 +90,7 @@ public static function process($db, $taskType, $procId) { , ',') || '}')::json AS attrs INTO temporary attrs FROM ( - SELECT occurrence_id, a.multi_value, + SELECT av.occurrence_id, a.multi_value, occurrence_attribute_id::text as f, array_agg( CASE a.data_type diff --git a/modules/cache_builder/plugins/cache_builder.php b/modules/cache_builder/plugins/cache_builder.php index 199a58f5c7..3935200f07 100644 --- a/modules/cache_builder/plugins/cache_builder.php +++ b/modules/cache_builder/plugins/cache_builder.php @@ -114,6 +114,28 @@ function cache_builder_orm_work_queue() { 'cost_estimate' => 30, 'priority' => 2, ], + // To trap direct updates to attribute values tables. + [ + 'entity' => 'occurrence_attribute_value', + 'ops' => ['insert', 'update', 'delete'], + 'task' => 'task_cache_builder_attr_value_occurrence', + 'cost_estimate' => 30, + 'priority' => 2, + ], + [ + 'entity' => 'sample_attribute_value', + 'ops' => ['insert', 'update', 'delete'], + 'task' => 'task_cache_builder_attr_value_sample', + 'cost_estimate' => 30, + 'priority' => 2, + ], + [ + 'entity' => 'taxa_taxon_list_attribute_value', + 'ops' => ['insert', 'update', 'delete'], + 'task' => 'task_cache_builder_attr_value_taxa_taxon_list', + 'cost_estimate' => 30, + 'priority' => 2, + ], [ 'entity' => 'user', 'ops' => ['update'], diff --git a/modules/indicia_setup/db/version_3_1_0/202001061500_attr_views_deleted_restrictions.sql b/modules/indicia_setup/db/version_3_1_0/202001061500_attr_views_deleted_restrictions.sql new file mode 100644 index 0000000000..5fac80352f --- /dev/null +++ b/modules/indicia_setup/db/version_3_1_0/202001061500_attr_views_deleted_restrictions.sql @@ -0,0 +1,155 @@ +CREATE OR REPLACE VIEW list_sample_attributes AS + SELECT * FROM ( + SELECT a.id, + a.caption, + a.caption_i18n, + a.description, + a.description_i18n, + a.image_path, + a.term_name, + a.term_identifier, + fsb2.name AS outer_structure_block, + fsb.name AS inner_structure_block, + a.data_type, + ct.control AS control_type, + a.termlist_id, + a.multi_value, + a.allow_ranges, + aw.website_id, + aw.restrict_to_survey_id, + (((a.id || '|'::text) || a.data_type::text) || '|'::text) || COALESCE(a.termlist_id::text, ''::text) AS signature, + aw.default_text_value, + aw.default_int_value, + aw.default_float_value, + aw.default_upper_value, + aw.default_date_start_value, + aw.default_date_end_value, + aw.default_date_type_value, + COALESCE(aw.validation_rules::text || E'\n', '') || COALESCE(a.validation_rules::text, '') AS validation_rules, + a.deleted, + aw.deleted AS website_deleted, + aw.restrict_to_sample_method_id, + a.system_function, + fsb2.weight as outer_block_weight, + fsb.weight as inner_block_weight, + aw.weight as weight, + ( + SELECT string_agg(restrict_to_taxon_meaning_id::text || '|' || COALESCE(restrict_to_stage_term_meaning_id::text, ''), ';') + FROM sample_attribute_taxon_restrictions tr + WHERE tr.sample_attributes_website_id=aw.id + AND tr.deleted=false + ) as taxon_restrictions, + rc.term as reporting_category, + rc.id as reporting_category_id, + a.unit as unit + FROM sample_attributes a + LEFT JOIN sample_attributes_websites aw ON a.id = aw.sample_attribute_id AND aw.deleted = false + LEFT JOIN control_types ct ON ct.id = aw.control_type_id + LEFT JOIN form_structure_blocks fsb ON fsb.id = aw.form_structure_block_id + LEFT JOIN form_structure_blocks fsb2 ON fsb2.id = fsb.parent_id + LEFT JOIN cache_termlists_terms rc on rc.id=a.reporting_category_id + WHERE a.deleted = false + ) as sub + ORDER BY outer_block_weight, inner_block_weight, weight; + +CREATE OR REPLACE VIEW list_occurrence_attributes AS + SELECT * FROM ( + SELECT + a.id, + a.caption, + a.caption_i18n, + a.description, + a.description_i18n, + a.image_path, + a.term_name, + a.term_identifier, + fsb2.name AS outer_structure_block, + fsb.name AS inner_structure_block, + a.data_type, ct.control AS control_type, + a.termlist_id, + a.multi_value, + a.allow_ranges, + aw.website_id, + aw.restrict_to_survey_id, + (((a.id || '|'::text) || a.data_type::text) || '|'::text) || COALESCE(a.termlist_id::text, ''::text) AS signature, + aw.default_text_value, + aw.default_int_value, + aw.default_float_value, + aw.default_upper_value, + aw.default_date_start_value, + aw.default_date_end_value, + aw.default_date_type_value, + COALESCE(aw.validation_rules::text || E'\n', '') || COALESCE(a.validation_rules::text, '') AS validation_rules, + a.deleted, + aw.deleted AS website_deleted, + a.system_function, + fsb2.weight as outer_block_weight, + fsb.weight as inner_block_weight, + aw.weight as weight, + ( + SELECT string_agg(restrict_to_taxon_meaning_id::text || '|' || COALESCE(restrict_to_stage_term_meaning_id::text, ''), ';') + FROM occurrence_attribute_taxon_restrictions tr + WHERE tr.occurrence_attributes_website_id=aw.id + AND tr.deleted=false + ) as taxon_restrictions, + rc.term as reporting_category, + rc.id as reporting_category_id, + a.unit as unit + FROM occurrence_attributes a + LEFT JOIN occurrence_attributes_websites aw ON a.id = aw.occurrence_attribute_id AND aw.deleted = false + LEFT JOIN control_types ct ON ct.id = aw.control_type_id + LEFT JOIN form_structure_blocks fsb ON fsb.id = aw.form_structure_block_id + LEFT JOIN form_structure_blocks fsb2 ON fsb2.id = fsb.parent_id + LEFT JOIN cache_termlists_terms rc on rc.id=a.reporting_category_id + WHERE a.deleted = false + ) as sub + ORDER BY outer_block_weight, inner_block_weight, weight; + +CREATE OR REPLACE VIEW list_taxa_taxon_list_attributes AS + SELECT a.id, + a.caption, + a.caption_i18n, + a.description, + a.description_i18n, + a.image_path, + a.term_name, + a.term_identifier, + fsb2.name AS outer_structure_block, + fsb.name AS inner_structure_block, + a.data_type, + ct.control AS control_type, + a.termlist_id, + a.multi_value, + a.allow_ranges, + tla.taxon_list_id, + (((a.id || '|'::text) || a.data_type::text) || '|'::text) || COALESCE(a.termlist_id::text, ''::text) AS signature, + tla.default_text_value, + tla.default_int_value, + tla.default_float_value, + tla.default_upper_value, + tla.default_date_start_value, + tla.default_date_end_value, + tla.default_date_type_value, + COALESCE(tla.validation_rules::text || E'\n', '') || COALESCE(a.validation_rules::text, '') AS validation_rules, + a.deleted, + tla.deleted AS taxon_list_deleted, + fsb2.weight as outer_block_weight, + fsb.weight as inner_block_weight, + tla.weight as weight, + ( + SELECT string_agg(restrict_to_taxon_meaning_id::text || '|' || COALESCE(restrict_to_stage_term_meaning_id::text, ''), ';') + FROM taxa_taxon_list_attribute_taxon_restrictions tr + WHERE tr.taxon_lists_taxa_taxon_list_attribute_id=tla.id + AND tr.deleted=false + ) as taxon_restrictions, + rc.term as reporting_category, + rc.id as reporting_category_id, + a.unit as unit + FROM taxa_taxon_list_attributes a + LEFT JOIN taxon_lists_taxa_taxon_list_attributes tla ON tla.taxa_taxon_list_attribute_id=a.id AND tla.deleted=false + LEFT JOIN control_types ct ON ct.id = tla.control_type_id + LEFT JOIN form_structure_blocks fsb ON fsb.id = tla.form_structure_block_id + LEFT JOIN form_structure_blocks fsb2 ON fsb2.id = fsb.parent_id + LEFT JOIN cache_termlists_terms rc on rc.id=a.reporting_category_id + WHERE a.deleted=false + ORDER BY fsb2.weight, fsb.weight, tla.weight; \ No newline at end of file diff --git a/modules/indicia_svc_data/tests/controllers/services/dataTest.php b/modules/indicia_svc_data/tests/controllers/services/dataTest.php index face51e879..775a1b8594 100644 --- a/modules/indicia_svc_data/tests/controllers/services/dataTest.php +++ b/modules/indicia_svc_data/tests/controllers/services/dataTest.php @@ -617,6 +617,10 @@ public function testCreateTermlistTerm() { * values. */ public function testUpdateOccurrenceWithAttr() { + $db = new Database(); + // Start from fresh. + $db->query('delete from work_queue'); + $q = new WorkQueue(); // First, create an attribute using ORM. $attr = ORM::factory('occurrence_attribute'); $array = array( @@ -681,8 +685,35 @@ public function testUpdateOccurrenceWithAttr() { $occ->reload(); $this->assertEquals('This has been partially updated', $occ->comment); $this->assertEquals(1, $occ->taxa_taxon_list_id); + // Run the work queue task and check attrs_json + // Run the task. + $q->process($db); + $attrs = json_decode($db->query( + "select attrs_json from cache_occurrences_nonfunctional where id=$occId" + )->current()->attrs_json, TRUE); + $this->assertEquals('A value', $attrs[$attr->id]); + // Now test if an update fired at the individual attribute value causes a + // work queue task so attrs_json gets updated. + $attrVal = $db->query("select id from occurrence_attribute_values where occurrence_attribute_id=$attr->id limit 1")->current(); + $array = [ + 'website_id' => 1, + 'survey_id' => 1, + 'occurrence_attribute_value:id' => $attrVal->id, + 'occurrence_attribute_value:text_value' => 'Updated', + ]; + $s = submission_builder::build_submission($array, ['model' => 'occurrence_attribute_value']); + $r = data_entry_helper::forward_post_to('occurrence_attribute_value', $s, $this->auth['write_tokens']); + $qCount = $db->query( + "select count(*) from work_queue where task='task_cache_builder_attr_value_occurrence' and record_id=$attrVal->id" + )->current(); + $this->assertEquals(1, $qCount->count, 'Work queue task not generated for attr value update.'); + // Run the task. + $q->process($db); + $attrs = json_decode($db->query( + "select attrs_json from cache_occurrences_nonfunctional where id=$occId" + )->current()->attrs_json, TRUE); + $this->assertEquals('Updated', $attrs[$attr->id]); // Clean up. - $db = new Database(); $db->query("delete from occurrence_attribute_values where occurrence_attribute_id=$attr->id"); $aw->delete(); $attr->delete(); @@ -731,4 +762,5 @@ public function testRequestDataCsvResponseLineFeed() { $this->assertEquals("Sample for unit testing with a \nline break", $sample['comment'], 'Data services CSV format response not encoded correctly for new lines.'); } -} \ No newline at end of file + +} diff --git a/modules/indicia_svc_import/controllers/services/import.php b/modules/indicia_svc_import/controllers/services/import.php index c997c8f490..3e089c2368 100644 --- a/modules/indicia_svc_import/controllers/services/import.php +++ b/modules/indicia_svc_import/controllers/services/import.php @@ -50,13 +50,13 @@ class Import_Controller extends Service_Base_Controller { public function get_import_settings($model) { $this->authenticate('read'); $model = ORM::factory($model); - if (method_exists($model, 'fixed_values_form')) { + if (method_exists($model, 'fixedValuesForm')) { // Pass URL parameters through to the fixed values form in case there are // model specific settings. $options = array_merge($_GET); unset($options['nonce']); unset($options['auth_token']); - echo json_encode($model->fixed_values_form($options)); + echo json_encode($model->fixedValuesForm($options)); } } @@ -358,7 +358,7 @@ private function checkModuleActive($module) { * Requires a $_GET parameter for uploaded_csv - the uploaded file name. */ public function upload() { - $allowCommitToDB = (isset($_GET['allow_commit_to_db']) ? $_GET['allow_commit_to_db'] : true); + $allowCommitToDB = isset($_GET['allow_commit_to_db']) ? $_GET['allow_commit_to_db'] : TRUE; $csvTempFile = DOCROOT . "upload/" . $_GET['uploaded_csv']; $metadata = $this->getMetadata($_GET['uploaded_csv']); if (!empty($metadata['user_id'])) { @@ -832,7 +832,12 @@ public function upload() { } // Get percentage progress. $progress = $filepos * 100 / filesize($csvTempFile); - $r = "{\"uploaded\":$count,\"progress\":$progress,\"filepos\":$filepos}"; + $r = json_encode([ + 'uploaded' => $count, + 'progress' => $progress, + 'filepos' => $filepos, + 'errorCount' => $metadata['errorCount'], + ]); // Allow for a JSONP cross-site request. if (array_key_exists('callback', $_GET)) { $r = $_GET['callback'] . "(" . $r . ")"; @@ -846,7 +851,7 @@ public function upload() { // An AJAX upload request will just receive the number of records // uploaded and progress. $this->auto_render = FALSE; - if (!empty($allowCommitToDB)&&$allowCommitToDB==true) { + if (!empty($allowCommitToDB) && $allowCommitToDB) { $cache->set(basename($csvTempFile) . 'previousSupermodel', $this->previousCsvSupermodel); } if (class_exists('request_logging')) { @@ -910,7 +915,7 @@ private function mergeExistingRecordIds( function ($field) { return $field->fieldName; }, $fields); - $join = self::buildJoin($fieldPrefix,$fields,$table,$saveArray); + $join = self::buildJoin($fieldPrefix,$fields,$table,$saveArray); $wheres = $model->buildWhereFromSaveArray($saveArray, $fields, "(" . $table . ".deleted = 'f')", $in, $assocSuffix); if ($wheres !== FALSE) { $db = Database::instance(); @@ -952,7 +957,7 @@ function ($field) { /* * Need to build a join so the system works correctly when importing taxa with update existing records selected. - * e.g. a problematic scenario would happen if importing new taxa but the external key/search code is still selected + * e.g. a problematic scenario would happen if importing new taxa but the external key/search code is still selected * for existing record update, in this case without building a join, the system would keep overwriting the previous record * as each new one is imported (as it wasn't checking the search code/external key, the final result would be that only one row would import). * Note this function might need improving/generalising for other models, although I did check occurrence/sample import which @@ -965,7 +970,7 @@ public static function buildJoin($fieldPrefix,$fields,$table,$saveArray) { } elseif (!empty($saveArray['taxon:search_code']) && $table=='taxa_taxon_lists') { $r = "join taxa t on t.id = ".$table.".taxon_id AND t.search_code='".$saveArray['taxon:search_code']."' AND t.deleted=false"; - } + } return $r; } diff --git a/modules/indicia_svc_plant_portal_import/controllers/services/plant_portal_import.php b/modules/indicia_svc_plant_portal_import/controllers/services/plant_portal_import.php index 740811596b..1dc9341c47 100644 --- a/modules/indicia_svc_plant_portal_import/controllers/services/plant_portal_import.php +++ b/modules/indicia_svc_plant_portal_import/controllers/services/plant_portal_import.php @@ -45,12 +45,12 @@ class Plant_Portal_Import_Controller extends Service_Base_Controller { public function get_plant_portal_import_settings($model) { $this->authenticate('read'); $model = ORM::factory($model); - if (method_exists($model, 'fixed_values_form')) { + if (method_exists($model, 'fixedValuesForm')) { // Pass URL parameters through to the fixed values form in case there are model specific settings. $options = array_merge($_GET); unset($options['nonce']); unset($options['auth_token']); - echo json_encode($model->fixed_values_form($options)); + echo json_encode($model->fixedValuesForm($options)); } } @@ -869,11 +869,11 @@ public function create_new_plots() { . ",".$userId . "WHERE NOT EXISTS ( - SELECT id - FROM locations - WHERE + SELECT id + FROM locations + WHERE name = '".$plotName."' AND - centroid_sref = '".$explodedPlotSrefs[$plotIdx]."' AND + centroid_sref = '".$explodedPlotSrefs[$plotIdx]."' AND centroid_sref_system = '".$explodedPlotSrefSystems[$plotIdx]."' );" . "insert into locations_websites" @@ -894,16 +894,16 @@ public function create_new_plots() { . ",".$userId . "WHERE NOT EXISTS ( - SELECT id - FROM locations_websites - WHERE + SELECT id + FROM locations_websites + WHERE location_id = (select id from locations where name = '".$plotName."' AND deleted=false order by id desc limit 1) AND website_id = ".$websiteId." );" ); } } - + /* * Create new groups with data passed in from the website */ @@ -923,7 +923,7 @@ public function create_new_groups() { } } } - + /* * After creating the groups, we actually need to assign the group to the user automatically (as they have just imported the group this makes sense to do) */ @@ -933,7 +933,7 @@ private function assign_user_to_new_group($db,$groupName,$userId,$personattribut //duplicate detection should be much earlier, possibly remove entirely if performance becomes an issue $db->query(" insert into person_attribute_values (person_id,person_attribute_id,int_value, created_on, created_by_id, updated_on, updated_by_id) - select ".$personId.", + select ".$personId.", ".$personattributeIdToHoldPlotGroups.", (select tt.id from termlists_terms tt @@ -947,11 +947,11 @@ private function assign_user_to_new_group($db,$groupName,$userId,$personattribut ".$userId." WHERE NOT EXISTS ( - SELECT id - FROM person_attribute_values - WHERE + SELECT id + FROM person_attribute_values + WHERE person_id = ".$personId." AND - person_attribute_id = ".$personattributeIdToHoldPlotGroups." AND + person_attribute_id = ".$personattributeIdToHoldPlotGroups." AND int_value = ( select tt.id from termlists_terms tt @@ -962,7 +962,7 @@ private function assign_user_to_new_group($db,$groupName,$userId,$personattribut ) );")->result(); } - + public function create_new_plot_to_group_attachments() { $db = new Database(); $websiteId = (isset($_GET['websiteId']) ? $_GET['websiteId'] : false); @@ -988,7 +988,7 @@ public function create_new_plot_to_group_attachments() { $db->query($databaseInsertionString)->result_array(false); } } - + private static function get_new_plot_attachments_plot_ids_to_create($db,$explodedPlotPairsForPlotGroupAttachment,$personId) { $plotNamesForAttachmentSet = '('; foreach ($explodedPlotPairsForPlotGroupAttachment as $plotPairsForPlotGroupAttachment) { @@ -996,18 +996,18 @@ private static function get_new_plot_attachments_plot_ids_to_create($db,$explode $plotNamesForAttachmentSet.="'".$explodedPlotNameGroupNamePair[0]."',"; } $plotNamesForAttachmentSet=substr($plotNamesForAttachmentSet, 0, -1); - $plotNamesForAttachmentSet .= ')'; + $plotNamesForAttachmentSet .= ')'; $returnArray=$db-> query( "select l.id as id, l.name as name from locations l - join locations_websites lw on lw.location_id = l.id + join locations_websites lw on lw.location_id = l.id where l.deleted=false AND l.name in ".$plotNamesForAttachmentSet." - order by l.id desc limit ".count($explodedPlotPairsForPlotGroupAttachment) + order by l.id desc limit ".count($explodedPlotPairsForPlotGroupAttachment) )->result_array(false); return $returnArray; } - + private static function get_new_plot_attachments_group_ids_to_create($db,$explodedPlotPairsForPlotGroupAttachment,$personId,$personAttributeIdThatHoldsPlotGroup) { $plotGroupNamesForAttachmentSet = '('; foreach ($explodedPlotPairsForPlotGroupAttachment as $plotPairsForPlotGroupAttachment) { @@ -1022,11 +1022,11 @@ private static function get_new_plot_attachments_group_ids_to_create($db,$explod from terms t join termlists_terms tt on tt.term_id = t.id AND tt.deleted=false join person_attribute_values pav on pav.int_value = tt.id AND pav.person_attribute_id = ".$personAttributeIdThatHoldsPlotGroup." AND pav.deleted=false - where t.deleted=false AND t.term in ".$plotGroupNamesForAttachmentSet + where t.deleted=false AND t.term in ".$plotGroupNamesForAttachmentSet )->result_array(false); return $returnArray; } - + /* * Build a string for inserting the plot location to group attachments */ @@ -1046,7 +1046,7 @@ private static function create_database_insertion_string($explodedPlotPairsForPl } return $insertionString; } - + private static function get_new_plot_attachments_to_create($explodedPlotPairsForPlotGroupAttachments,$plotIdsToCreateAttachmentsFor,$groupIdsToCreateAttachmentsFor) { $explodedPlotPairsForPlotGroupAttachmentAsIds=array(); foreach ($explodedPlotPairsForPlotGroupAttachments as $plotPairForPlotGroupAttachment) { @@ -1065,24 +1065,24 @@ private static function get_new_plot_attachments_to_create($explodedPlotPairsFor } } $explodedPlotPairsForPlotGroupAttachmentAsIds[]=$plotPairForPlotGroupAttachmentAsIds; - + } return $explodedPlotPairsForPlotGroupAttachmentAsIds; } - + private function get_person_from_user_id($db,$userId) { $returnObj=$db->query("select u.person_id AS id from users u where u.id = ".$userId.";")->current(); if (!empty($returnObj->id)) $returnVal=$returnObj->id; - else + else $returnVal=null; return $returnVal; } - - /* - * If spatial reference is missing then automatically generate one using the vice county name or country name + + /* + * If spatial reference is missing then automatically generate one using the vice county name or country name * Note this has an equivalent function with the same name in the Drupal prebuilt form. - * Changes to the logic here should also occur in that function + * Changes to the logic here should also occur in that function */ private static function auto_generate_grid_references($saveArray) { $viceCountyPairs = explode(',',kohana::config('plant_portal_import.vice_counties_list')); @@ -1093,8 +1093,8 @@ private static function auto_generate_grid_references($saveArray) { foreach ($viceCountyPairs as $viceCountyNameGridRefPair) { $viceCountyNameGridRefPairExploded=explode('|',$viceCountyNameGridRefPair); //If we find a match for the vice county then we can set the spatial reference and spatial reference system from the vice county - if (!empty($saveArray['smpAttr:'.kohana::config('plant_portal_import.vice_county_attr_id')])&& - !empty($viceCountyNameGridRefPairExploded[0]) && + if (!empty($saveArray['smpAttr:'.kohana::config('plant_portal_import.vice_county_attr_id')])&& + !empty($viceCountyNameGridRefPairExploded[0]) && $saveArray['smpAttr:'.kohana::config('plant_portal_import.vice_county_attr_id')]==$viceCountyNameGridRefPairExploded[0]) { $saveArray['sample:entered_sref']=$viceCountyNameGridRefPairExploded[1]; $saveArray['sample:entered_sref_system']='4326'; @@ -1106,7 +1106,7 @@ private static function auto_generate_grid_references($saveArray) { foreach ($countryPairs as $countryNameGridRefPair) { $countryNameGridRefPairExploded=explode('|',$countryNameGridRefPair); if (!empty($saveArray['smpAttr:'.kohana::config('plant_portal_import.country_attr_id')])&& - !empty($countryNameGridRefPairExploded[0]) && + !empty($countryNameGridRefPairExploded[0]) && $saveArray['smpAttr:'.kohana::config('plant_portal_import.country_attr_id')]==$countryNameGridRefPairExploded[0]) { $saveArray['sample:entered_sref']=$countryNameGridRefPairExploded[1]; $saveArray['sample:entered_sref_system']='4326'; @@ -1169,7 +1169,7 @@ private function mergeExistingRecordIds( function ($field) { return $field->fieldName; }, $fields); - $join = self::buildJoin($fieldPrefix,$fields,$table,$saveArray); + $join = self::buildJoin($fieldPrefix,$fields,$table,$saveArray); $wheres = $model->buildWhereFromSaveArray($saveArray, $fields, "(" . $table . ".deleted = 'f')", $in, $assocSuffix); if ($wheres !== FALSE) { $db = Database::instance(); @@ -1211,7 +1211,7 @@ function ($field) { /* * Need to build a join so the system works correctly when importing taxa with update existing records selected. - * e.g. a problematic scenario would happen if importing new taxa but the external key/search code is still selected + * e.g. a problematic scenario would happen if importing new taxa but the external key/search code is still selected * for existing record update, in this case without building a join, the system would keep overwriting the previous record * as each new one is imported (as it wasn't checking the search code/external key, the final result would be that only one row would import). * Note this function might need improving/generalising for other models, although I did check occurrence/sample import which @@ -1224,7 +1224,7 @@ public static function buildJoin($fieldPrefix,$fields,$table,$saveArray) { } elseif (!empty($saveArray['taxon:search_code']) && $table=='taxa_taxon_lists') { $r = "join taxa t on t.id = ".$table.".taxon_id AND t.search_code='".$saveArray['taxon:search_code']."' AND t.deleted=false"; - } + } return $r; } diff --git a/modules/spatial_index_builder/config/spatial_index_builder.php.travis b/modules/spatial_index_builder/config/spatial_index_builder.php.travis new file mode 100644 index 0000000000..c584a7be04 --- /dev/null +++ b/modules/spatial_index_builder/config/spatial_index_builder.php.travis @@ -0,0 +1,35 @@ + $entitySelectItems, 'jsonSchema' => json_encode([ 'type' => 'map', - 'title' => 'Colunms to set', + 'title' => 'Columns to set', 'mapping' => $jsonMapping, 'desc' => 'List of columns and the values they are to be set to, when event is triggered.', ]), diff --git a/modules/workflow/i18n/en_GB/form_error_messages.php b/modules/workflow/i18n/en_GB/form_error_messages.php new file mode 100644 index 0000000000..b0086cf975 --- /dev/null +++ b/modules/workflow/i18n/en_GB/form_error_messages.php @@ -0,0 +1,9 @@ + [ + 'required' => 'A species must be selected which has the external key filled in.', + ], +]; diff --git a/modules/workflow/views/workflow_event/edit.js b/modules/workflow/views/workflow_event/edit.js new file mode 100644 index 0000000000..df57119755 --- /dev/null +++ b/modules/workflow/views/workflow_event/edit.js @@ -0,0 +1,55 @@ +jQuery(document).ready(function($) { + $('#taxon_list_id').change(function() { + var options = $('input#workflow_event\\:key_value\\:taxon').indiciaAutocomplete('option'); + options.extraParams.taxon_list_id = $('#taxon_list_id').val(); + $('input#workflow_event\\:key_value\\:taxon').indiciaAutocomplete('option', options); + }); + + $('#workflow_event\\:entity').change(function entityChange() { + var previousValue = $('#workflow_event\\:event_type').val(); + var entityKeys = Object.keys(indiciaData.entities); + var i; + var j; + // First build event types list for select. + if (!previousValue) { + previousValue = $('#old_workflow_event_event_type').val(); + } + $('#workflow_event\\:event_type option').remove(); + for (i = 0; i < entityKeys.length; i++) { + if (entityKeys[i] === $('#workflow_event\\:entity').val()) { + for (j = 0; j < indiciaData.entities[entityKeys[i]].event_types.length; j++) { + $('#workflow_event\\:event_type').append( + '' + ); + } + } + } + $('#workflow_event\\:event_type').val(previousValue); + // now do Keys + previousValue = $('#workflow_event\\:key').val(); + if (!previousValue) { + previousValue = $('#old_workflow_event_key').val(); + } + $('#workflow_event\\:key option').remove(); + for (i = 0; i < entityKeys.length; i++) { + if (entityKeys[i] === $('#workflow_event\\:entity').val()) { + for (j = 0; j < indiciaData.entities[entityKeys[i]].keys.length; j++) { + $('#workflow_event\\:key').append( + '' + ); + } + if (indiciaData.entities[entityKeys[i]].keys.length === 1) { + $('#ctrl-wrap-workflow_event-key').hide(); + } else { + $('#ctrl-wrap-workflow_event-key').show(); + } + } + } + if ($('#workflow_event\\:key option[value="' + previousValue + '"]').length !== 0) { + $('#workflow_event\\:key').val(previousValue); + } + }); + $('#workflow_event\\:entity').change(); +}); diff --git a/modules/workflow/views/workflow_event/workflow_event_edit.php b/modules/workflow/views/workflow_event/workflow_event_edit.php index f401d3ee46..4dade975bc 100644 --- a/modules/workflow/views/workflow_event/workflow_event_edit.php +++ b/modules/workflow/views/workflow_event/workflow_event_edit.php @@ -91,7 +91,8 @@ 'lookupValues' => array(), 'validation' => array('required'), )); - // Code currently assumes only taxa_taxon_list_external_key possible in the key options. + // Code currently assumes only taxa_taxon_list_external_key possible in the + // key options. $params = $readAuth; if ($listId = warehouse::getMasterTaxonListId()) { $params += array('taxon_list_id' => $listId); @@ -106,6 +107,16 @@ 'extraParams' => $params, 'validation' => array('required'), )); + echo data_entry_helper::select(array( + 'label' => 'Alternative species checklist', + 'fieldname' => 'taxon_list_id', + 'table' => 'taxon_list', + 'valueField' => 'id', + 'captionField' => 'title', + 'default' => $listId, + 'extraParams' => $readAuth, + 'helpText' => 'If using taxa not on the master species list, choose the alternative list here before searching.', + )); echo data_entry_helper::hidden_text(array( 'fieldname' => 'old_workflow_event_event_type', 'default' => html::initial_value($values, 'workflow_event:event_type'), @@ -131,54 +142,7 @@ echo $metadata; echo html::form_buttons(html::initial_value($values, 'workflow_event:id') != NULL, FALSE, FALSE); - ?> - - SELECT #columns# FROM filters f - LEFT JOIN locations l ON string_to_array(COALESCE(definition::json->>'location_list', definition::json->>'location_ids'), ',') @> ARRAY[l.id::text ] + LEFT JOIN locations l ON l.id = ANY(string_to_array(COALESCE(definition::json->>'location_list', definition::json->>'location_ids'), ',')::integer[]) WHERE (f.website_id in (#website_ids#) OR f.website_id is null) AND f.deleted=false and f.id=#filter_id# diff --git a/reports/library/taxa/explore_list.xml b/reports/library/taxa/explore_list.xml index 9388d9d52f..80532ca3b8 100644 --- a/reports/library/taxa/explore_list.xml +++ b/reports/library/taxa/explore_list.xml @@ -3,12 +3,12 @@ description="Report designed for species lists used in the explore records facility in iRecord. " > - SELECT distinct #columns# + SELECT #columns# FROM cache_occurrences_functional o JOIN cache_taxa_taxon_lists cttl ON cttl.id=o.taxa_taxon_list_id #agreements_join# #joins# - WHERE #sharing_filter# + WHERE #sharing_filter# AND o.record_status not in ('I','T') AND (#ownData#=1 OR o.record_status not in ('D','R')) AND ('#searchArea#'='' OR st_intersects(o.public_geom, ST_MakeValid(st_geomfromtext('#searchArea#',900913)))) AND (#ownData#=0 OR CAST(o.created_by_id AS character varying)='#currentUser#') @@ -29,10 +29,10 @@ - JOIN locations lfilter ON st_intersects(lfilter.boundary_geom, o.public_geom) AND lfilter.id=#location_id# + JOIN locations lfilter ON st_intersects(lfilter.boundary_geom, o.public_geom) AND lfilter.id=#location_id# - + JOIN taxon_groups tgfilter ON tgfilter.id=o.taxon_group_id AND tgfilter.id IN (#taxon_groups#) diff --git a/reports/library/taxa/explore_list_2.xml b/reports/library/taxa/explore_list_2.xml index 0a2ee4917c..722b73d739 100644 --- a/reports/library/taxa/explore_list_2.xml +++ b/reports/library/taxa/explore_list_2.xml @@ -4,7 +4,7 @@ Second version has additional parameters." > - SELECT distinct #columns# + SELECT #columns# FROM cache_occurrences_functional o JOIN cache_taxa_taxon_lists cttl ON cttl.id=o.taxa_taxon_list_id #agreements_join# diff --git a/reports/library/taxa/explore_list_2_using_spatial_index_builder.xml b/reports/library/taxa/explore_list_2_using_spatial_index_builder.xml index 71001c3d72..6bcdf233f4 100644 --- a/reports/library/taxa/explore_list_2_using_spatial_index_builder.xml +++ b/reports/library/taxa/explore_list_2_using_spatial_index_builder.xml @@ -6,7 +6,7 @@ as their locality, for significantly improved performance." > - SELECT distinct #columns# + SELECT #columns# FROM cache_occurrences_functional o JOIN cache_taxa_taxon_lists cttl ON cttl.id=o.taxa_taxon_list_id #agreements_join# diff --git a/reports/library/taxa/explore_list_3_using_spatial_index_builder.xml b/reports/library/taxa/explore_list_3_using_spatial_index_builder.xml index 5ee62ef939..dfac87d347 100644 --- a/reports/library/taxa/explore_list_3_using_spatial_index_builder.xml +++ b/reports/library/taxa/explore_list_3_using_spatial_index_builder.xml @@ -6,7 +6,7 @@ as their locality, for significantly improved performance." > - SELECT distinct #columns# + SELECT #columns# FROM cache_occurrences_functional o JOIN cache_taxa_taxon_lists cttl ON cttl.id=o.taxa_taxon_list_id #agreements_join# diff --git a/reports/library/taxa/explore_list_using_spatial_index_builder.xml b/reports/library/taxa/explore_list_using_spatial_index_builder.xml index b89becd8c1..68a4d15827 100644 --- a/reports/library/taxa/explore_list_using_spatial_index_builder.xml +++ b/reports/library/taxa/explore_list_using_spatial_index_builder.xml @@ -5,7 +5,7 @@ as their locality, for significantly improved performance." > - SELECT distinct #columns# + SELECT #columns# FROM cache_occurrences_functional o JOIN cache_taxa_taxon_lists cttl ON cttl.id=o.taxa_taxon_list_id #agreements_join# diff --git a/reports/library/users/users_list.xml b/reports/library/users/users_list.xml index e56db415f1..7418dcc920 100644 --- a/reports/library/users/users_list.xml +++ b/reports/library/users/users_list.xml @@ -4,7 +4,7 @@ SELECT #columns# FROM people p LEFT JOIN users u ON u.person_id=p.id AND u.deleted=false - LEFT JOIN users_websites uw on u.id=uw.user_id + LEFT JOIN users_websites uw on u.id=uw.user_id and uw.site_role_id is not null LEFT JOIN websites w on w.id=uw.website_id and w.deleted=false LEFT JOIN core_roles cr ON cr.id=u.core_role_id AND cr.deleted=false #joins# diff --git a/reports/reports_for_prebuilt_forms/dynamic_elasticsearch/record_details.xml b/reports/reports_for_prebuilt_forms/dynamic_elasticsearch/record_details.xml index ef5de161fe..936c9fe4e9 100644 --- a/reports/reports_for_prebuilt_forms/dynamic_elasticsearch/record_details.xml +++ b/reports/reports_for_prebuilt_forms/dynamic_elasticsearch/record_details.xml @@ -3,26 +3,8 @@ description="Obtain record attributes for verification in Elasticsearch configured forms." > - SELECT 0 as group_weight, null::integer as id, 1 as weight, 'Recorder' as attribute_type, 'full_name' as system_function, - 'Text'::bpchar as data_type, 'Recorded by' as caption, - COALESCE(snf.attr_full_name, snf.attr_last_name || COALESCE(', ' || snf.attr_first_name, ''), p.surname || COALESCE(', ' || p.first_name, '')) as value, - COALESCE(snf.attr_full_name, snf.attr_last_name || COALESCE(', ' || snf.attr_first_name, ''), p.surname || COALESCE(', ' || p.first_name, '')) as raw_value - FROM cache_samples_nonfunctional snf - JOIN occurrences o ON o.sample_id=snf.id AND o.id=#occurrence_id# - LEFT JOIN users u ON u.id=o.created_by_id AND u.deleted=false AND u.id<>1 - LEFT JOIN people p ON p.id=u.person_id AND p.deleted=false - UNION - SELECT 0 as group_weight, null::integer as id, 2 as weight, 'Recorder' as attribute_type, 'email' as system_function, - 'Text'::bpchar as data_type, 'Email' as caption, - COALESCE(snf.attr_email, p.email_address, 'Email address not available') as value, - COALESCE(snf.attr_email, p.email_address, 'Email address not available') as raw_value - FROM cache_samples_nonfunctional snf - JOIN occurrences o ON o.sample_id=snf.id AND o.id=#occurrence_id# - LEFT JOIN users u ON u.id=o.created_by_id AND u.deleted=false AND u.id<>1 - LEFT JOIN people p ON p.id=u.person_id AND p.deleted=false - UNION - select - 1 as group_weight, o.id, oaw.weight, 'Record' as attribute_type, a.system_function, + SELECT + 0 as group_weight, o.id, oaw.weight, 'Additional occurrence' as attribute_type, a.system_function, CASE a.data_type WHEN 'T'::bpchar THEN 'Text'::bpchar WHEN 'L'::bpchar THEN 'Lookup List'::bpchar @@ -56,19 +38,19 @@ WHEN 'V'::bpchar THEN (av.date_start_value::character varying::text || ' - '::text) || av.date_end_value::character varying::text ELSE NULL::text END AS raw_value - from occurrences o - join samples s on s.id=o.sample_id and s.deleted=false - join occurrence_attribute_values av on av.occurrence_id=o.id and av.deleted=false - join occurrence_attributes a on a.id=av.occurrence_attribute_id and a.deleted=false - left join cache_termlists_terms lookup on lookup.id=av.int_value - left join occurrence_attributes_websites oaw on oaw.occurrence_attribute_id=a.id and oaw.restrict_to_survey_id=s.survey_id and oaw.deleted=false - where o.id=#occurrence_id# - and o.deleted=false + FROM occurrences o + JOIN samples s on s.id=o.sample_id and s.deleted=false + JOIN occurrence_attribute_values av on av.occurrence_id=o.id and av.deleted=false + JOIN occurrence_attributes a on a.id=av.occurrence_attribute_id and a.deleted=false + LEFT JOIN cache_termlists_terms lookup on lookup.id=av.int_value + LEFT JOIN occurrence_attributes_websites oaw on oaw.occurrence_attribute_id=a.id and oaw.restrict_to_survey_id=s.survey_id and oaw.deleted=false + WHERE o.id=#occurrence_id# + AND o.deleted=false - union + UNION - select - 2 as group_weight, o.id, saw.weight, 'Parent sample' as attribute_type, a.system_function, + SELECT + 1 as group_weight, o.id, saw.weight, 'Parent sample' as attribute_type, a.system_function, CASE a.data_type WHEN 'T'::bpchar THEN 'Text'::bpchar WHEN 'L'::bpchar THEN 'Lookup List'::bpchar @@ -99,20 +81,20 @@ WHEN 'V'::bpchar THEN (av.date_start_value::character varying::text || ' - '::text) || av.date_end_value::character varying::text ELSE NULL::text END AS raw_value - from occurrences o - join samples s on s.id=o.sample_id and s.deleted=false - join sample_attribute_values av on av.sample_id=s.parent_id and av.deleted=false - join sample_attributes a on a.id=av.sample_attribute_id and a.deleted=false - left join cache_termlists_terms lookup on lookup.id=av.int_value - left join sample_attributes_websites saw on saw.sample_attribute_id=a.id and saw.restrict_to_survey_id=s.survey_id and saw.deleted=false - where o.id=#occurrence_id# - and o.deleted=false - and (a.system_function not in ('cms_user_id', 'full_name', 'first_name', 'last_name', 'email') or a.system_function is null) + FROM occurrences o + JOIN samples s on s.id=o.sample_id and s.deleted=false + JOIN sample_attribute_values av on av.sample_id=s.parent_id and av.deleted=false + JOIN sample_attributes a on a.id=av.sample_attribute_id and a.deleted=false + LEFT JOIN cache_termlists_terms lookup on lookup.id=av.int_value + LEFT JOIN sample_attributes_websites saw on saw.sample_attribute_id=a.id and saw.restrict_to_survey_id=s.survey_id and saw.deleted=false + WHERE o.id=#occurrence_id# + AND o.deleted=false + AND (a.system_function not in ('cms_user_id', 'full_name', 'first_name', 'last_name', 'email') or a.system_function is null) - union + UNION - select - 3 as group_weight, o.id, saw.weight, 'Sample' as attribute_type, a.system_function, + SELECT + 2 as group_weight, o.id, saw.weight, 'Additional sample' as attribute_type, a.system_function, CASE a.data_type WHEN 'T'::bpchar THEN 'Text'::bpchar WHEN 'L'::bpchar THEN 'Lookup List'::bpchar @@ -143,16 +125,28 @@ WHEN 'V'::bpchar THEN (av.date_start_value::character varying::text || ' - '::text) || av.date_end_value::character varying::text ELSE NULL::text END AS raw_value - from occurrences o - join samples s on s.id=o.sample_id and s.deleted=false - join sample_attribute_values av on av.sample_id=o.sample_id and av.deleted=false - join sample_attributes a on a.id=av.sample_attribute_id and a.deleted=false - left join cache_termlists_terms lookup on lookup.id=av.int_value - left join sample_attributes_websites saw on saw.sample_attribute_id=a.id and saw.restrict_to_survey_id=s.survey_id and saw.deleted=false - where o.id=#occurrence_id# - and o.deleted=false - and (a.system_function not in ('cms_user_id', 'full_name', 'first_name', 'last_name', 'email') or a.system_function is null) - order by group_weight, attribute_type, weight, caption + FROM occurrences o + JOIN samples s on s.id=o.sample_id and s.deleted=false + JOIN sample_attribute_values av on av.sample_id=o.sample_id and av.deleted=false + JOIN sample_attributes a on a.id=av.sample_attribute_id and a.deleted=false + LEFT JOIN cache_termlists_terms lookup on lookup.id=av.int_value + LEFT JOIN sample_attributes_websites saw on saw.sample_attribute_id=a.id and saw.restrict_to_survey_id=s.survey_id and saw.deleted=false + WHERE o.id=#occurrence_id# + AND o.deleted=false + AND (a.system_function not in ('cms_user_id', 'full_name', 'first_name', 'last_name', 'email') or a.system_function is null) + + UNION + + SELECT 3 as group_weight, null::integer as id, 2 as weight, 'Recorder' as attribute_type, 'email' as system_function, + 'Text'::bpchar as data_type, 'Email' as caption, + COALESCE(snf.attr_email, p.email_address, 'Email address not available') as value, + COALESCE(snf.attr_email, p.email_address, 'Email address not available') as raw_value + FROM cache_samples_nonfunctional snf + JOIN occurrences o ON o.sample_id=snf.id AND o.id=#occurrence_id# + LEFT JOIN users u ON u.id=o.created_by_id AND u.deleted=false AND u.id<>1 + LEFT JOIN people p ON p.id=u.person_id AND p.deleted=false + + ORDER BY group_weight, attribute_type, weight, caption