From 11f2c04473f414bb61eec44241a9d45400f029d1 Mon Sep 17 00:00:00 2001 From: Gary van Breda Date: Wed, 27 Mar 2019 20:54:49 +0000 Subject: [PATCH 1/7] Major refactor of the summary_builder module --- modules/summary_builder/README | 13 + .../config/summary_builder.php | 548 ++++---- .../controllers/summariser_definition.php | 178 +-- .../db/version_2_17_0/201903091200_tables.sql | 58 + .../201903091201_work_queue.sql | 12 + .../helpers/summary_builder.php | 1138 +++++++---------- .../task_summary_builder_location_delete.php | 51 + ...mmary_builder_occurrence_insert_delete.php | 56 + ...task_summary_builder_occurrence_update.php | 58 + .../helpers/task_summary_builder_sample.php | 52 + .../helpers/task_summary_builder_taxon.php | 62 + .../models/summariser_definition.php | 124 +- .../plugins/summary_builder.php | 186 +-- .../views/summariser_definition/index.php | 62 +- .../summariser_definition_edit.php | 343 ++--- 15 files changed, 1535 insertions(+), 1406 deletions(-) create mode 100644 modules/summary_builder/README create mode 100644 modules/summary_builder/db/version_2_17_0/201903091200_tables.sql create mode 100644 modules/summary_builder/db/version_2_17_0/201903091201_work_queue.sql create mode 100644 modules/summary_builder/helpers/task_summary_builder_location_delete.php create mode 100644 modules/summary_builder/helpers/task_summary_builder_occurrence_insert_delete.php create mode 100644 modules/summary_builder/helpers/task_summary_builder_occurrence_update.php create mode 100644 modules/summary_builder/helpers/task_summary_builder_sample.php create mode 100644 modules/summary_builder/helpers/task_summary_builder_taxon.php diff --git a/modules/summary_builder/README b/modules/summary_builder/README new file mode 100644 index 0000000000..a7ea322d29 --- /dev/null +++ b/modules/summary_builder/README @@ -0,0 +1,13 @@ +Notes: +1) Branch/country level summarisation to be done in front end (by summing up the locations) +2) Row and column totals in each grid done by the front end + +Proposed development path +1) Currently only week based summarisation, as that is what UKBMS want. Expand to allow Month based summarisation +2) Allow >1 summarisation per survey +3) Currently deals with super/subsample with locations, as that is what UKBMS want. Expand to handle other cases. +4) Amalgamation of raw data into a daily summary. Would need to store sample list. +5) Make 0-1 rounding configurable +6) Configurable end of year processing: Include data for overlap, include periods that overlap + + \ No newline at end of file diff --git a/modules/summary_builder/config/summary_builder.php b/modules/summary_builder/config/summary_builder.php index 54da82ba5e..1503631294 100644 --- a/modules/summary_builder/config/summary_builder.php +++ b/modules/summary_builder/config/summary_builder.php @@ -1,325 +1,223 @@ -= '#date#' - AND p.deleted = 't' -LIMIT #limit#"; -// second part returns samples that have been created since last run date -$config['get_created_samples_query'] = " - SELECT p.date_start, p.created_by_id as user_id, p.location_id - FROM samples p - WHERE p.location_id IS NOT NULL - AND p.parent_id IS NULL - AND p.deleted = 'f' - AND p.survey_id = #survey_id# - AND p.created_on >= '#date#' -LIMIT #limit#"; - -// first part returns samples that have been deleted since the equivalent summary_created_on was created -// Only need to pick up deleted parent samples, as deleting the subsample automatically deletes the -// occurrences, which are picked up in the items check. Presence of the (potentially empty) parent sample -// is what impacts calculations. -$config['get_missed_deleted_samples_query'] = " - SELECT DISTINCT p.date_start as date_start, p.created_by_id as user_id, p.location_id - FROM samples p - JOIN summary_occurrences so ON so.survey_id = #survey_id# - AND so.location_id = p.location_id - AND so.user_id = p.created_by_id - AND p.date_start <= so.date_end - AND p.date_start >= so.date_start - AND p.updated_on > so.summary_created_on - WHERE p.parent_id IS NULL - AND p.survey_id = #survey_id# - AND p.deleted = 't' -LIMIT #limit#"; -// Other non-delete updates to the samples are not actually important: -// second part returns samples that have been created since the equivalent summary_created_on was created, -// or which have no entries at all. In this case at least one sample for the year must have occurrences -// in order for data to be present, otherwise no point in doing it - this sample would always show up. -// During a rebuild, it is this query that does the vast majority of the work. The order by allows us to tell -// how far through the rebuild it has got. -$config['get_missed_created_samples_query'] = " - SELECT p.date_start as date_start, p.created_by_id as user_id, p.location_id - FROM samples p - LEFT JOIN summary_occurrences so ON so.survey_id = #survey_id# - AND so.location_id = p.location_id - AND p.date_start BETWEEN so.date_start AND so.date_end - AND p.created_on < so.summary_created_on - WHERE p.parent_id IS NULL - AND p.survey_id = #survey_id# - AND p.location_id IS NOT NULL - AND p.deleted = 'f' - AND so.survey_id IS NULL - AND EXISTS (SELECT 1 - FROM samples p2 - JOIN samples s2 ON p2.id = s2.parent_id - JOIN occurrences o2 ON s2.id = o2.sample_id AND o2.deleted = 'f' - WHERE EXTRACT(YEAR FROM p.date_start) = EXTRACT(YEAR FROM p2.date_start) - AND p2.parent_id IS NULL - AND p2.survey_id = #survey_id# - AND p2.location_id = p.location_id - AND p2.deleted = 'f') - ORDER BY date_start -LIMIT #limit#"; - -// return all the taxa ever recorded for this location on this year - includes deleted deliberately -$config['get_taxa_query'] = " - SELECT distinct o.taxa_taxon_list_id - FROM occurrences o - JOIN samples s ON s.id = o.sample_id - JOIN samples p ON s.parent_id = p.id - AND p.survey_id = #survey_id# - AND p.location_id = #location_id# - AND p.date_end>='#year#-01-01' - AND p.date_start<='#year#-12-31' -"; - -// Pick up if an occurrence has been deleted, modified or created -> user_id, location_id, taxon_id. -// Need to rebuild all the entries which have a location_id set to the location (user and non user specific) for that taxa. -// Need to rebuild all the entries which have the user_id, for that taxa. -// Need to rebuild the top level global entries for that taxa. - -// pick up if the sample/parent has been deleted since last run. - -// need to process if any occurrence is flagged as deleted: no deleted flags on this query. -// changes to samples handled elsewhere. -// also pick up if the sample/parent has been deleted, occurrence left alone. -// not checking if dates or locations have changed on samples, or change of taxon on occurrence: -// cant do at moment - but also not possible through UKBMS front end. -$config['get_changed_occurrences_query'] = " - SELECT o.taxa_taxon_list_id, p.date_start, p.created_by_id, p.location_id - FROM occurrences o - JOIN samples s ON s.id = o.sample_id - JOIN samples p ON s.parent_id = p.id AND p.survey_id = #survey_id# AND p.location_id IS NOT NULL - WHERE o.updated_on>='#date#' - LIMIT #limit#"; - -$config['get_missed_changed_occurrences_query'] = " - SELECT distinct o.taxa_taxon_list_id, p.date_start, p.created_by_id, p.location_id - FROM occurrences o - JOIN samples s ON s.id = o.sample_id AND s.deleted = 'f' - JOIN samples p ON s.parent_id = p.id AND p.survey_id = #survey_id# AND p.deleted = 'f' AND p.location_id IS NOT NULL - LEFT JOIN summary_occurrences so ON so.survey_id = #survey_id# AND so.taxa_taxon_list_id = o.taxa_taxon_list_id - AND so.location_id = p.location_id AND so.user_id = p.created_by_id AND p.date_start <= so.date_end AND p.date_start >= so.date_start - AND o.updated_on < so.summary_created_on - WHERE so.survey_id IS NULL AND o.deleted = 'f' - LIMIT #limit#"; - -$config['get_missed_deleted_occurrences_query'] = " - SELECT o.taxa_taxon_list_id, p.date_start, p.created_by_id, p.location_id - FROM occurrences o - JOIN samples s ON s.id = o.sample_id - JOIN samples p ON s.parent_id = p.id AND p.survey_id = #survey_id# AND p.location_id IS NOT NULL - JOIN summary_occurrences so ON so.survey_id = #survey_id# AND so.taxa_taxon_list_id = o.taxa_taxon_list_id - AND so.location_id = p.location_id AND so.user_id = p.created_by_id AND p.date_start <= so.date_end AND p.date_start >= so.date_start - AND o.updated_on>so.summary_created_on - WHERE o.deleted = 't' - LIMIT #limit#"; - -// Rebuild all the data on rotation: this will catch any oddities, like change of taxon -// Order by picks up the oldest entries -// would like to put a distinct on this, but there is a significant performance hit. - -$config['get_rebuild_occurrences_query'] = " - SELECT so.taxa_taxon_list_id, so.location_id, so.user_id, so.year - FROM summary_occurrences so - WHERE so.survey_id = #survey_id# - AND so.user_id IS NOT NULL - AND so.location_id IS NOT NULL - ORDER BY so.summary_created_on - LIMIT #limit#"; - -// Pick up changes to the taxon list -// Need to flag if there are any occurrences attached to deleted taxa. -// If a taxa has been changed, then need to rebuild all the entries for that taxa - if preferred has changed, so will the meaning id. -// But, these this are done en-masse, so there is a real possibility that the limit will be exceed on a single run. -// So pick up if an taxa update date is after an summary occurrence entry -> user_id, location_id, taxon_id. -// This method means no need for a separate missing check. -// would like to do a distinct but there is a significant performance hit. -$config['get_changed_taxa_query'] = " - SELECT so.taxa_taxon_list_id, so.year, so.user_id, so.location_id, so.summary_created_on, cttl.cache_updated_on - FROM summary_occurrences so - JOIN cache_taxa_taxon_lists cttl ON so.taxa_taxon_list_id = cttl.id - AND so.summary_created_on < cttl.cache_updated_on - WHERE so.survey_id = #survey_id# - LIMIT #limit#"; - - - -$config['get_YearTaxonLocationUser_query'] = " - SELECT 1 AS count, p.id AS sample_id, p.date_start, 't' as present - FROM occurrences o - JOIN samples s ON s.id = o.sample_id AND s.deleted = 'f' - JOIN samples p ON s.parent_id = p.id AND p.survey_id = #survey_id# AND p.location_id = #location_id# AND p.created_by_id = #user_id# AND p.deleted = 'f' AND p.date_end>='#year#-01-01' AND p.date_start<='#year#-12-31' - WHERE o.taxa_taxon_list_id = #taxon_id# - AND o.deleted = 'f' AND o.zero_abundance = 'f' - UNION ALL - SELECT 0 AS count, p.id AS sample_id, p.date_start, 'f' as present - FROM samples p - WHERE p.survey_id = #survey_id# - AND p.location_id = #location_id# - AND p.created_by_id = #user_id# - AND p.deleted = 'f' - AND p.date_end>='#year#-01-01' AND p.date_start<='#year#-12-31' - "; - -$config['get_YearTaxonLocationUser_Attr_query'] = " - SELECT oav.int_value AS count, p.id AS sample_id, p.date_start, 't' as present - FROM occurrences o - JOIN occurrence_attribute_values oav ON oav.occurrence_id = o.id AND oav.deleted = 'f' AND oav.occurrence_attribute_id = #attr_id# - JOIN samples s ON s.id = o.sample_id AND s.deleted = 'f' - JOIN samples p ON s.parent_id = p.id AND p.survey_id = #survey_id# AND p.location_id = #location_id# AND p.created_by_id = #user_id# AND p.deleted = 'f' AND p.date_end>='#year#-01-01' AND p.date_start<='#year#-12-31' - WHERE o.taxa_taxon_list_id = #taxon_id# - AND o.deleted = 'f' AND o.zero_abundance = 'f' - UNION ALL - SELECT 0 AS count, p.id AS sample_id, p.date_start, 'f' as present - FROM samples p - WHERE p.survey_id = #survey_id# - AND p.location_id = #location_id# - AND p.created_by_id = #user_id# - AND p.deleted = 'f' - AND p.date_end>='#year#-01-01' AND p.date_start<='#year#-12-31' - "; - -$config['get_YearTaxonLocation_query'] = " - SELECT 1 AS count, p.id AS sample_id, p.date_start, 't' as present - FROM occurrences o - JOIN samples s ON s.id = o.sample_id AND s.deleted = 'f' - JOIN samples p ON s.parent_id = p.id AND p.survey_id = #survey_id# AND p.location_id = #location_id# AND p.deleted = 'f' AND p.date_end>='#year#-01-01' AND p.date_start<='#year#-12-31' - WHERE o.taxa_taxon_list_id = #taxon_id# - AND o.deleted = 'f' AND o.zero_abundance = 'f' - UNION ALL - SELECT 0 AS count, p.id AS sample_id, p.date_start, 'f' as present - FROM samples p - WHERE p.survey_id = #survey_id# - AND p.location_id = #location_id# - AND p.deleted = 'f' - AND p.date_end>='#year#-01-01' AND p.date_start<='#year#-12-31' - "; - -$config['get_YearTaxonLocation_Attr_query'] = " - SELECT oav.int_value AS count, p.id AS sample_id, p.date_start, 't' as present - FROM occurrences o - JOIN occurrence_attribute_values oav ON oav.occurrence_id = o.id AND oav.deleted = 'f' AND oav.occurrence_attribute_id = #attr_id# - JOIN samples s ON s.id = o.sample_id AND s.deleted = 'f' - JOIN samples p ON s.parent_id = p.id AND p.survey_id = #survey_id# AND p.location_id = #location_id# AND p.deleted = 'f' AND p.date_end>='#year#-01-01' AND p.date_start<='#year#-12-31' - WHERE o.taxa_taxon_list_id = #taxon_id# - AND o.deleted = 'f' AND o.zero_abundance = 'f' - UNION ALL - SELECT 0 AS count, p.id AS sample_id, p.date_start, 'f' as present - FROM samples p - WHERE p.survey_id = #survey_id# - AND p.location_id = #location_id# - AND p.deleted = 'f' - AND p.date_end>='#year#-01-01' AND p.date_start<='#year#-12-31' - "; - -$config['get_YearTaxonUser_query'] = " - SELECT count, estimate, date_start - FROM summary_occurrences - WHERE survey_id = #survey_id# - AND user_id = #user_id# - AND date_end>='#year#-01-01' - AND date_start<='#year#-12-31' - AND taxa_taxon_list_id = #taxon_id# - AND location_id IS NOT NULL - "; -$config['get_YearTaxon_query'] = " - SELECT sum(count) as count, sum(estimate) as estimate, date_start - FROM summary_occurrences - WHERE survey_id = #survey_id# - AND user_id IS NULL - AND date_end>='#year#-01-01' - AND date_start<='#year#-12-31' - AND taxa_taxon_list_id = #taxon_id# - AND location_id IS NOT NULL - GROUP BY date_start - "; + parent.updated_on + WHERE parent.id = #sample_id# + AND so.survey_id is NULL ;"; + +$config['sample_existing_taxa'] = +"SELECT so.taxa_taxon_list_id + FROM samples parent + JOIN summary_occurrences so + ON so.website_id = #website_id# + AND so.survey_id = #survey_id# + AND so.user_id = 0 + AND so.location_id = #location_id# + AND so.year = #year# + AND so.summary_created_on <= parent.updated_on + WHERE parent.id = #sample_id# ;"; + +// need to process deleted occurrences: can't use cache +$config['get_occurrences_to_process'] = +"SELECT record_id as occurrence_id, sd.id as definition_id + FROM work_queue wq + JOIN occurrences o ON o.id = wq.record_id + JOIN samples child ON o.sample_id = child.id + JOIN samples parent ON child.parent_id = parent.id AND parent.location_id IS NOT NULL + JOIN summariser_definitions sd ON sd.survey_id = parent.survey_id AND sd.deleted = false + WHERE wq.claimed_by='#procId#' + AND wq.entity='occurrence' + AND wq.task='#task#';"; + +$config['occurrence_detail_lookup'] = +"SELECT EXTRACT(YEAR FROM parent.date_start) as year, parent.created_by_id as user_id, + parent.location_id, occ.taxa_taxon_list_id + FROM samples parent + JOIN samples child ON child.parent_id = parent.id + JOIN occurrences occ ON occ.sample_id = child.id AND occ.id = #occurrence_id# + LEFT JOIN summary_occurrences so + ON so.survey_id = parent.survey_id + AND so.user_id = parent.created_by_id + AND so.location_id = parent.location_id + AND so.taxa_taxon_list_id = occ.taxa_taxon_list_id + AND so.year = EXTRACT(YEAR FROM parent.date_start) + AND so.summary_created_on > occ.updated_on + WHERE parent.location_id IS NOT NULL AND parent.deleted = false + AND so.survey_id is NULL ;"; + +$config['sample_detail_lookup_occurrence'] = +"SELECT parent.id as sample_id, parent.survey_id, EXTRACT(YEAR FROM parent.date_start) as year, + parent.created_by_id as user_id, parent.location_id, s.website_id + FROM samples parent + JOIN samples child ON child.parent_id = parent.id + JOIN occurrences occ ON occ.sample_id = child.id AND occ.id = #occurrence_id# + JOIN surveys s ON parent.survey_id = s.id + WHERE parent.deleted = false ;"; + +$config['get_locations_to_process'] = +"SELECT record_id as location_id + FROM work_queue wq + WHERE wq.claimed_by='#procId#' + AND wq.entity='location' + AND wq.task='task_summary_builder_location_delete';"; + +$config['location_existing_data'] = +"SELECT distinct so.survey_id, so.year, so.user_id, so.taxa_taxon_list_id + FROM summary_occurrences so + WHERE so.user_id > 0 + AND so.location_id = #location_id# ;"; + +$config['get_all_definitions'] = +"SELECT sd.*, s.website_id + FROM summariser_definitions sd + JOIN surveys s ON s.id = sd.survey_id AND s.deleted = 'f' ;"; + +$config['delete_location_data'] = +"DELETE FROM summary_occurrences so + WHERE so.location_id = #location_id# ;"; + + + +$config['get_YearTaxonLocationUser_query'] = " + SELECT 1 AS count, p.id AS sample_id, p.date_start, 't' as present + FROM occurrences o + JOIN samples s ON s.id = o.sample_id AND s.deleted = 'f' + JOIN samples p ON s.parent_id = p.id AND p.survey_id = #survey_id# AND p.location_id = #location_id# AND p.created_by_id = #user_id# AND p.deleted = 'f' AND p.date_end>='#year#-01-01' AND p.date_start<='#year#-12-31' + WHERE o.taxa_taxon_list_id = #taxon_id# + AND o.deleted = 'f' AND o.zero_abundance = 'f' + UNION ALL + SELECT 0 AS count, p.id AS sample_id, p.date_start, 'f' as present + FROM samples p + WHERE p.survey_id = #survey_id# + AND p.location_id = #location_id# + AND p.created_by_id = #user_id# + AND p.deleted = 'f' + AND p.date_end>='#year#-01-01' AND p.date_start<='#year#-12-31' + "; + +$config['get_YearTaxonLocationUser_Attr_query'] = " + SELECT oav.int_value AS count, p.id AS sample_id, p.date_start, 't' as present + FROM occurrences o + JOIN occurrence_attribute_values oav ON oav.occurrence_id = o.id AND oav.deleted = 'f' AND oav.occurrence_attribute_id = #attr_id# + JOIN samples s ON s.id = o.sample_id AND s.deleted = 'f' + JOIN samples p ON s.parent_id = p.id AND p.survey_id = #survey_id# AND p.location_id = #location_id# AND p.created_by_id = #user_id# AND p.deleted = 'f' AND p.date_end>='#year#-01-01' AND p.date_start<='#year#-12-31' + WHERE o.taxa_taxon_list_id = #taxon_id# + AND o.deleted = 'f' AND o.zero_abundance = 'f' + UNION ALL + SELECT 0 AS count, p.id AS sample_id, p.date_start, 'f' as present + FROM samples p + WHERE p.survey_id = #survey_id# + AND p.location_id = #location_id# + AND p.created_by_id = #user_id# + AND p.deleted = 'f' + AND p.date_end>='#year#-01-01' AND p.date_start<='#year#-12-31' + "; + +$config['get_YearTaxonLocation_query'] = " + SELECT 1 AS count, p.id AS sample_id, p.date_start, 't' as present + FROM occurrences o + JOIN samples s ON s.id = o.sample_id AND s.deleted = 'f' + JOIN samples p ON s.parent_id = p.id AND p.survey_id = #survey_id# AND p.location_id = #location_id# AND p.deleted = 'f' AND p.date_end>='#year#-01-01' AND p.date_start<='#year#-12-31' + WHERE o.taxa_taxon_list_id = #taxon_id# + AND o.deleted = 'f' AND o.zero_abundance = 'f' + UNION ALL + SELECT 0 AS count, p.id AS sample_id, p.date_start, 'f' as present + FROM samples p + WHERE p.survey_id = #survey_id# + AND p.location_id = #location_id# + AND p.deleted = 'f' + AND p.date_end>='#year#-01-01' AND p.date_start<='#year#-12-31' + "; + +$config['get_YearTaxonLocation_Attr_query'] = " + SELECT oav.int_value AS count, p.id AS sample_id, p.date_start, 't' as present + FROM occurrences o + JOIN occurrence_attribute_values oav ON oav.occurrence_id = o.id AND oav.deleted = 'f' AND oav.occurrence_attribute_id = #attr_id# + JOIN samples s ON s.id = o.sample_id AND s.deleted = 'f' + JOIN samples p ON s.parent_id = p.id AND p.survey_id = #survey_id# AND p.location_id = #location_id# AND p.deleted = 'f' AND p.date_end>='#year#-01-01' AND p.date_start<='#year#-12-31' + WHERE o.taxa_taxon_list_id = #taxon_id# + AND o.deleted = 'f' AND o.zero_abundance = 'f' + UNION ALL + SELECT 0 AS count, p.id AS sample_id, p.date_start, 'f' as present + FROM samples p + WHERE p.survey_id = #survey_id# + AND p.location_id = #location_id# + AND p.deleted = 'f' + AND p.date_end>='#year#-01-01' AND p.date_start<='#year#-12-31' + "; + +$config['get_YearTaxonUser_query'] = " + SELECT summarised_data + FROM summary_occurrences + WHERE survey_id = #survey_id# + AND user_id = #user_id# + AND year = #year# + AND taxa_taxon_list_id = #taxon_id# + AND location_id > 0 + "; +$config['get_YearTaxon_query'] = " + SELECT summarised_data + FROM summary_occurrences + WHERE survey_id = #survey_id# + AND user_id = 0 + AND year = #year# + AND taxa_taxon_list_id = #taxon_id# + AND location_id > 0 + "; diff --git a/modules/summary_builder/controllers/summariser_definition.php b/modules/summary_builder/controllers/summariser_definition.php index 2e59130e27..7d7e7a37ff 100644 --- a/modules/summary_builder/controllers/summariser_definition.php +++ b/modules/summary_builder/controllers/summariser_definition.php @@ -21,34 +21,8 @@ * @link https://github.com/indicia-team/warehouse/ */ -/* Stage 2 - * TODO: configurable merge taxa with same meaning. - * TODO: sample attribute averaging. store as json in a period level record. - * TODO: new download reports. - * TODO: Store Weeknumber definitions in a period level record - * TODO: comment functions - */ - -/* - * Proposed development path - * 1) Currently only week based summarisation, as that is what UKBMS want. Expand to allow Month based summarisation - * 2) Allow >1 summarisation per survey - * 3) Currently deals with super/subsample with locations, as that is what UKBMS want. Expand to handle other cases. - * 4) Amalgamation of raw data into a daily summary. Would need to store sample list. - * 5) Make 0-1 rounding configurable - * 6) Configurable end of year processing: Include data for overlap, include periods that overlap -*/ - -/* - * Notes: - * 1) user level summarisation for taxon is done in back end - * 2) branch level summarisation to be done in front end (by summing up the locations) - * 3) year level and species level summarisation (row and column totals) done by the front end - * 4) links to be handled by front end. - */ - /** - * Controller providing CRUD access to the surveys list. + * Controller providing CRUD access to the summariser_definition list. * * @package Core * @subpackage Controllers @@ -59,61 +33,81 @@ public function __construct() { parent::__construct('summariser_definition'); $this->columns = array( 'title' => 'Survey title', - 'website' => '' + 'website' => '' ); $this->pagetitle = "Survey based data summariser definition"; } protected function get_action_columns() { - return array( - array( - 'caption'=>'Setup Summariser', - 'url'=>'summariser_definition/edit_from_survey/{survey_id}' - ) - ); + return array( + array( + 'caption' => 'Setup Summariser', + 'url' => 'summariser_definition/edit/{id}', + ) + ); } - public function edit_from_survey($id) { - if (!is_null($id) && !is_null($this->auth_filter) && !in_array($id, $this->auth_filter['values'])) { - $this->access_denied(); - return; - } - $this->model = new Summariser_definition_Model(array('survey_id' => $id, 'deleted'=>'f')); - $values = $this->getModelValues(); - if ( !$this->model->loaded ) { - $values['summariser_definition:survey_id']=$id; - $values['summariser_definition:period_start']='weekday=7'; - $values['summariser_definition:period_one_contains']='Jan-01'; - } + public function edit($id = NULL) { + if ( is_null($id) ) { + $this->access_denied(); + return; + } + $this->model = new Summariser_definition_Model(array('id' => $id, 'deleted'=>'f')); + if ( !$this->model->loaded ) { + $this->access_denied(); + return; + } + $survey = new Survey_Model(array('id' => $this->model->survey_id, 'deleted'=>'f')); + if ( !$this->model->loaded ) { + $this->access_denied(); + return; + } + if ( !is_null($this->auth_filter ) && !in_array($survey->website_id, $this->auth_filter['values']) ) { + $this->access_denied(); + return; + } + + $values = $this->getModelValues(); $other = $this->prepareOtherViewData($values); - $this->setView($this->editViewName(), $this->model->caption(), array( - 'existing'=>$this->model->loaded, - 'values'=>$values - ,'other_data'=>$other + $this->setView($this->editViewName(), $this->model->caption(), array( + 'existing' => TRUE, + 'values' => $values, + 'other_data' => $other )); - $this->defineEditBreadcrumbs(); + $this->defineEditBreadcrumbs(); } - public function edit($id){ - Kohana::show_404(); - } public function create(){ - Kohana::show_404(); + $this->model = new Summariser_definition_Model(); + $values = $this->getModelValues(); + $values['summariser_definition:period_start']='weekday=7'; + $values['summariser_definition:period_one_contains']='Jan-01'; + $other = $this->prepareOtherViewData($values); + if(count($other['surveys'])<1) { + $this->access_denied(); + return; + } + $this->setView($this->editViewName(), $this->model->caption(), array( + 'existing'=>FALSE, + 'values'=>$values, + 'other_data'=>$other + )); + $this->defineEditBreadcrumbs(); } protected function show_submit_fail() { $page_errors=$this->model->getPageErrors(); - if (count($page_errors)!=0) { - $this->session->set_flash('flash_error', implode('
',$page_errors)); - } else { - $this->session->set_flash('flash_error', 'The record could not be saved.'); - } - $values = $this->getDefaults(); - $values = array_merge($values, $_POST); - $other = $this->prepareOtherViewData($values); - $this->setView($this->editViewName(), $this->model->caption(), array( - 'existing'=>$this->model->loaded, + if (count($page_errors)!=0) { + $this->session->set_flash('flash_error', implode('
',$page_errors)); + } else { + $this->session->set_flash('flash_error', 'The record could not be saved.'); + } + $values = $this->getDefaults(); + $values = array_merge($values, $_POST); + $other = $this->prepareOtherViewData($values); + $this->setView($this->editViewName(), $this->model->caption(), array( + 'existing'=>$this->model->loaded, 'values'=>$values ,'other_data'=>$other )); @@ -121,27 +115,46 @@ protected function show_submit_fail() } protected function prepareOtherViewData(array $values) { - $survey = new Survey_Model($values['summariser_definition:survey_id']); - $attrsRet = array(''=>'(Each occurrence has count=1)'); + $survey = new Survey_Model($values['summariser_definition:survey_id']); + $attrsRet = array(''=>'(Each occurrence has count=1)'); $models = ORM::factory('occurrence_attributes_website')-> - where(array('restrict_to_survey_id'=>$values['summariser_definition:survey_id'],'deleted'=>'f'))-> - find_all(); + where(array('deleted'=>'f')); + if(!empty($values['summariser_definition:survey_id'])) { + $models = $models->where(array('restrict_to_survey_id'=>$values['summariser_definition:survey_id'])); + } + $models = $models->find_all(); if(count($models)>0){ - $attrIds = array(); - foreach ($models as $model) - $attrIds[] = $model->occurrence_attribute_id; - $attrs = ORM::factory('occurrence_attribute')-> - where('deleted','f')->in('data_type',array('I','F'))->in('id',$attrIds)-> - orderby('caption')->find_all(); - if(count($attrs)>0) - foreach ($attrs as $attr) - $attrsRet[$attr->id] = $attr->caption.' (ID '.$attr->id.')'; + $attrIds = array(); + foreach ($models as $model) + $attrIds[] = $model->occurrence_attribute_id; + $attrIds = array_unique($attrIds); + $attrs = ORM::factory('occurrence_attribute')-> + where('deleted','f')->in('data_type',array('I','F'))->in('id',$attrIds)-> + orderby('caption')->find_all(); + if(count($attrs)>0) + foreach ($attrs as $attr) + $attrsRet[$attr->id] = $attr->caption.' (ID '.$attr->id.')'; } + $surveys = array(); + $surveyModels = ORM::factory('survey')->where(array('deleted'=>'f'))->find_all(); + foreach ($surveyModels as $model) { + if (is_null($this->auth_filter) || in_array($model->website_id, $this->auth_filter['values'])) { + $surveys["".$model->id] = $model->title; + } + } + $existingModels = ORM::factory('summariser_definition')->where(array('deleted'=>'f'))->find_all(); + foreach ($existingModels as $model) { + if(array_key_exists("".$model->id, $surveys)) { + unset($surveys["".$model->survey_id]); + } + } + asort($surveys); return array( 'survey_title' => $survey->title, - 'occAttrs' => $attrsRet + 'occAttrs' => $attrsRet, + 'surveys' => $surveys, ); } @@ -149,8 +162,9 @@ protected function prepareOtherViewData(array $values) { * Check access to a survey when editing. The survey's website must be in the list * of websites the user is authorised to administer. */ - protected function record_authorised ($id) + protected function record_authorised ($id = NULL) { + $model = new Summariser_definition_Model($id); if (!is_null($id) AND !is_null($this->auth_filter)) { $survey = new Survey_Model($id); @@ -167,7 +181,3 @@ protected function page_authorised() { } } -/* - * This is triggered by changes to occurrence records, not by a report. - * How this is output is determined by the front end. - */ \ No newline at end of file diff --git a/modules/summary_builder/db/version_2_17_0/201903091200_tables.sql b/modules/summary_builder/db/version_2_17_0/201903091200_tables.sql new file mode 100644 index 0000000000..acddbe8e5d --- /dev/null +++ b/modules/summary_builder/db/version_2_17_0/201903091200_tables.sql @@ -0,0 +1,58 @@ + + +DROP VIEW IF EXISTS gv_summariser_definitions; + +ALTER TABLE summariser_definitions + DROP COLUMN check_for_missing, + DROP COLUMN max_records_per_cycle; + +CREATE VIEW gv_summariser_definitions AS + SELECT s.id as survey_id, s.title, w.title AS website, s.website_id, sd.id + FROM summariser_definitions sd + LEFT JOIN surveys s ON sd.survey_id = s.id AND s.deleted = FALSE + LEFT JOIN websites w ON s.website_id = w.id AND w.deleted = FALSE + WHERE sd.survey_id = s.id AND sd.deleted = false; + + +DROP VIEW IF EXISTS list_summary_occurrences; + +DROP TABLE summary_occurrences; + +CREATE TABLE summary_occurrences +( + website_id INTEGER NOT NULL, + survey_id INTEGER NOT NULL, + location_id INTEGER NOT NULL, + user_id INTEGER NOT NULL, + year INTEGER NOT NULL, + type CHARACTER VARYING NOT NULL, + taxon_list_id INTEGER NOT NULL, + taxa_taxon_list_id INTEGER NOT NULL, + preferred_taxa_taxon_list_id INTEGER NOT NULL, + taxonomic_sort_order bigint, + taxon CHARACTER VARYING, + preferred_taxon CHARACTER VARYING, + default_common_name CHARACTER VARYING, + taxon_meaning_id INTEGER NOT NULL, + summarised_data JSON, + created_by_id INTEGER NOT NULL, + summary_created_on timestamp without time zone NOT NULL +) WITH ( + OIDS --- want oids as no pk. +); +CREATE INDEX ix_summary_occurrences_STLU ON summary_occurrences USING btree (survey_id, year, taxa_taxon_list_id, location_id, user_id); +CREATE INDEX ix_summary_occurrences_STU ON summary_occurrences USING btree (survey_id, taxa_taxon_list_id, user_id); + +CREATE OR REPLACE VIEW list_summary_occurrences AS + SELECT website_id, survey_id, + year, location_id, user_id, type, + taxa_taxon_list_id, preferred_taxa_taxon_list_id, taxonomic_sort_order, + taxon, preferred_taxon, default_common_name, taxon_meaning_id, taxon_list_id, + created_by_id, summary_created_on + FROM summary_occurrences; + +COMMENT ON TABLE summary_occurrences IS 'Summary of occurrence data used for reporting.'; +COMMENT ON COLUMN summary_occurrences.location_id IS 'Summary data location id, zero if for all locations.'; +COMMENT ON COLUMN summary_occurrences.user_id IS 'ID of user who created the data summarised in this record, zero if for all users.'; +COMMENT ON COLUMN summary_occurrences.type IS 'summariser_definition period type.'; + diff --git a/modules/summary_builder/db/version_2_17_0/201903091201_work_queue.sql b/modules/summary_builder/db/version_2_17_0/201903091201_work_queue.sql new file mode 100644 index 0000000000..99a655d045 --- /dev/null +++ b/modules/summary_builder/db/version_2_17_0/201903091201_work_queue.sql @@ -0,0 +1,12 @@ +-- #slow script# + +-- trigger a rebuild of the entire data table, using the the work queue. + +DELETE FROM work_queue WHERE task='task_summary_builder_sample'; + +INSERT INTO work_queue (task, entity, record_id, priority, cost_estimate, created_on) +SELECT DISTINCT 'task_summary_builder_sample', 'sample', id, 3, 90, now() +FROM samples +WHERE deleted=false +AND parent_id IS NULL +AND survey_id IN (SELECT survey_id FROM summariser_definitions WHERE deleted = false); \ No newline at end of file diff --git a/modules/summary_builder/helpers/summary_builder.php b/modules/summary_builder/helpers/summary_builder.php index 78c4bdbdc4..1431ba4319 100644 --- a/modules/summary_builder/helpers/summary_builder.php +++ b/modules/summary_builder/helpers/summary_builder.php @@ -1,5 +1,9 @@ '; - $r = $db->query($queries['summary_truncate']); - } - - public static function populate_summary_table(&$db, $last_run_date, $verbose, $rebuild, $clear, $missing_check) { - - self::$verbose = $verbose; - - // this has to be done only once - self::init_lastval($db); - - $queries = kohana::config('summary_builder'); - $r = $db->query($queries['select_definitions'])->result_array(false); - if(count($r)){ - foreach($r as $row){ - if($clear == $row['survey_id']) { - echo date(DATE_ATOM).' Clearing of summary data for survey ID '.$row['survey_id'].' requested.
'; - if(isset($_GET['location_id']) && $_GET['location_id'] != '') - $query = str_replace(array('#survey_id#','#location_id#'), array($row['survey_id'], $_GET['location_id']), $queries['clear_survey_location']); - if(isset($_GET['taxa_taxon_list_id']) && $_GET['taxa_taxon_list_id'] != '') - $query = str_replace(array('#survey_id#','#taxa_taxon_list_id#'), array($row['survey_id'], $_GET['taxa_taxon_list_id']), $queries['clear_survey_taxon']); - else - $query = str_replace(array('#survey_id#'), array($row['survey_id']), $queries['clear_survey']); - $count = $db->query($query)->count(); - if(self::$verbose) { - echo date(DATE_ATOM).' Removed '.$count.' records.
'; - if (!isset($row['check_for_missing']) || $row['check_for_missing'] == 'f') - echo ' Missing checks not currently enabled in the definition.
'; - else - echo ' Rebuild will commence on next invocation of scheduled tasks.
'; - return; - } - } else if($rebuild === true || $rebuild == $row['survey_id']) { - echo date(DATE_ATOM).' Rebuild of summary data for survey ID '.$row['survey_id'].' requested.
'; - $query = str_replace(array('#survey_id#'), array($row['survey_id']), $queries['first_sample_creation_date']); - $sample_date = $db->query($query)->result_array(false); - if(count($sample_date)){ - $query = str_replace(array('#survey_id#','#date#'), - array($row['survey_id'], $sample_date[0]['first_date']), - $queries['rebuild_survey']); - $count = $db->query($query)->count(); - if(self::$verbose){ - echo date(DATE_ATOM).' Date altered on '.$count.' records.
'; - if (!isset($row['check_for_missing']) || $row['check_for_missing'] == 'f') - echo ' Missing checks not currently enabled in the definition.
'; - else - echo ' Rebuild will commence on next invocation of scheduled tasks.
'; - return; - } - } else { - $query = str_replace(array('#survey_id#'), array($row['survey_id']), $queries['clear_survey']); - $count = $db->query($query)->count(); - if(self::$verbose) echo date(DATE_ATOM).' No samples detected for Survey '.$row['survey_id'].'. Precautionary clearance of summary data attempted: '.$count.' summary records removed.
'; - } - } else { - echo date(DATE_ATOM).' Processing summariser_definition ID '.$row['id'].' for survey ID '.$row['survey_id'].'
'; - self::populate_summary_table_for_survey($db, $last_run_date, $row, ($missing_check === true || $missing_check == $row['survey_id'])); + /** + * Performs the actual task of table population. + */ + + + /** + * It is only the presence/absence of a sample that affects the values/estimates, so this is invoked on insert or + * deletion, not update. + */ + /** OK **/ + public static function populate_summary_table_for_sample(&$db, $sampleId, $definitionId) { + // This is the same for an existing sample update, or an insert + // There is a possibility that if the date on a sample changes to a different year, the the old year values + // will not be updated - similar if the location is moved. This is however a very remote possibility, and + // is not possible from the UKBMS front end. + $queries = kohana::config('summary_builder'); + // this has to be done only once + self::init_lastval($db); + + $query = str_replace('#definition_id#', $definitionId, $queries['get_definition']); + $definitionResult = $db->query($query)->result_array(false); + + // This returns all taxa on this sample, merged with all taxa on summary records for this location/year + // Includes a check that the data for the location/year/user/taxon is either not present or was last updated + // before the sample record update time: if not then has been run in another batch and is up to date + list($year, $userId, $locationId, $taxaList) = self::get_changelist_sample($db, $sampleId); + + if (count($taxaList) > 0){ + self::do_summary($db, $definitionResult[0], $year, $userId, $locationId, $taxaList); } - } - } else { - echo date(DATE_ATOM).' No summariser_definitions to be processed.
'; } - } - - /* - * Bit of an oddball situation. Our table does not have a sequence. When inserting, The database driver sets - * the insert_id for the pg_result from the lastval function - with no sequence this returns an error, and the error - * is ignored from a PHP point of view. However from Postgres POV, this causes a transaction abort/rollback, - * so we can't wrap a transaction around it. We need a transaction though, so this is a hack to populate - * the lastval. Use a temporary sequence, as lastval gives an error if the sequence has been dropped. - */ - private static function init_lastval(&$db) { - $db->query("CREATE TEMPORARY SEQUENCE summary_builder_dummy_sequence;"); // dropped automatically at end of process - $db->query("SELECT NEXTVAL('summary_builder_dummy_sequence');"); - } - private static function populate_summary_table_for_survey(&$db, $last_run_date, $definition, $missing_check) { - $YearTaxonLocationUser = array(); - $YearTaxonLocation = array(); - $YearTaxonUser = array(); - $YearTaxon = array(); // list of taxa in a given year. - try { - $count=summary_builder::get_changelist($db, $last_run_date, $definition, $YearTaxonLocationUser, $YearTaxonLocation, $YearTaxonUser, $YearTaxon, $missing_check); - if($count>0) - summary_builder::do_summary($db, $definition, $YearTaxonLocationUser, $YearTaxonLocation, $YearTaxonUser, $YearTaxon); - } catch (Exception $e) { - error_logger::log_error('Building summary', $e); - echo $e->getMessage(); + /** OK **/ + private static function get_changelist_sample(&$db, $sampleId) { + $queries = kohana::config('summary_builder'); + $taxa = []; + + $query = str_replace('#sample_id#', $sampleId, $queries['sample_detail_lookup']); + $sampleResults = $db->query($query)->result_array(false); // returns survey_id, year, created_by_id, location_id + + $query = str_replace(array('#website_id#', '#survey_id#', '#sample_id#', '#year#', '#user_id#', '#location_id#'), + array($sampleResults[0]['website_id'], $sampleResults[0]['survey_id'], $sampleId , + $sampleResults[0]['year'], $sampleResults[0]['user_id'], $sampleResults[0]['location_id']), + $queries['sample_occurrence_lookup']); + $occurrenceResults = $db->query($query)->result_array(false); // returns taxa_taxon_list_id + foreach($occurrenceResults as $row){ + $taxa[] = $row['taxa_taxon_list_id']; + } + + $query = str_replace(array('#website_id#', '#survey_id#', '#sample_id#', '#year#', '#user_id#', '#location_id#'), + array($sampleResults[0]['website_id'], $sampleResults[0]['survey_id'], $sampleId , + $sampleResults[0]['year'], $sampleResults[0]['user_id'], $sampleResults[0]['location_id']), + $queries['sample_existing_taxa']); + $existingResults = $db->query($query)->result_array(false); + foreach($existingResults as $row){ + $taxa[] = $row['taxa_taxon_list_id']; + } + $taxa = array_unique($taxa); + return array($sampleResults[0]['year'], $sampleResults[0]['user_id'], $sampleResults[0]['location_id'], $taxa); } - return; - } - - private static function check_deleted_locations(&$db, $last_run_date, $definition, $YearTaxonLocationUser, $YearTaxonLocation, $YearTaxonUser, $YearTaxon, $missing_check) - { - // Can't pick up setting of locations_websites deleted since last_run as it doesn't have an updated_on field. - // Means can't really do a "location has been deleted since last run": just do a sweep up. - // - // Pick up if a deleted location has any entries in the summary_occurrences (sweeps up all). - // if yes, the link to the sample will have been broken by the cascade trigger, which also doesn't set the updated_on date... - // Retrieve all the users (ulist) which have entries in the cache for this location. - // Need to delete all the entries which have a location_id set to the deleted location (user and non user specific). - // Need to rebuild all the entries which have a user_id in ulist. - // Need to rebuild all the top level global entries. - // Can't put a limit on the numbers returned, as need all from each location processed at the same time. - - // TODO only do this if the first run in the day - - if (!$missing_check && (!isset($definition['check_for_missing']) || $definition['check_for_missing'] == 'f')) return; - - $queries = kohana::config('summary_builder'); - $query = str_replace(array('#survey_id#','#website_id#'), - array($definition['survey_id'], $definition['website_id']), - $queries['get_deleted_locations_query']); - - $r = $db->query($query)->result_array(false); // returns location_id, user_id, taxa_taxon_list_id, year - - $count = count($r); - if($count){ - if(self::$verbose) echo date(DATE_ATOM).' '.$count.' deleted locations records to be processed.
'; - foreach($r as $row){ - $year = $row['year']; - // Flag the rebuild of a top level year/taxon combination. - if(!isset($YearTaxon[$year])) $YearTaxon[$year]=array(); - if(!in_array($row['taxa_taxon_list_id'], $YearTaxon[$year])) $YearTaxon[$year][] = $row['taxa_taxon_list_id']; - // Flag the rebuild of a top level year/taxon/user combination. - if(!isset($YearTaxonUser[$year.':'.$row['taxa_taxon_list_id']])) $YearTaxonUser[$year.':'.$row['taxa_taxon_list_id']] = array(); - if(!in_array($row['user_id'], $YearTaxonUser[$year.':'.$row['taxa_taxon_list_id']])) $YearTaxonUser[$year.':'.$row['taxa_taxon_list_id']][] = $row['user_id']; - // no rebuild for location entries, as the location is deleted, so just remove them immediately. - summary_builder::do_delete($db, $definition, $year, $row['taxa_taxon_list_id'], $row['location_id'], $row['user_id']); - summary_builder::do_delete($db, $definition, $year, $row['taxa_taxon_list_id'], $row['location_id'], false); - } - } else { - if(self::$verbose) echo date(DATE_ATOM).' No deleted locations records processed.
'; + /** OK **/ + public static function populate_summary_table_for_occurrence_insert_delete(&$db, $occurrenceId, $definitionId) { + // This is the same for an occurrence insert or delete: only affects one taxa + $queries = kohana::config('summary_builder'); + // this has to be done only once + self::init_lastval($db); + + $query = str_replace('#definition_id#', $definitionId, + $queries['get_definition']); + $definitionResult = $db->query($query)->result_array(false); + + // This returns just this taxa, following same format as the other methods. + // Includes a check that the data for the location/year/user/taxon is either not present or was last updated + // before the sample record update time: if not then has been run in another batch and is up to date + list($year, $userId, $locationId, $taxa) = self::get_changelist_occurrence_insert_delete($db, $occurrenceId); + + if (count($taxa) > 0){ + self::do_summary($db, $definitionResult[0], $year, $userId, $locationId, $taxa); + } } - } - - private static $locationYearUser; // cache - - private static function check_samples(&$db, $last_run_date, $definition, &$limit, &$YearTaxonLocationUser, &$YearTaxonLocation, &$YearTaxonUser, &$YearTaxon, $missing_check) - { - // 2 modes: since last run and sweep. - - self::$locationYearUser = array(); - - // Pick up if a top level sample or subsample has been deleted -> user_id and location_id. - // May or may not have occurrence records, but still affects calculations. - // Need to rebuild all the entries which have a location_id set to the location (user and non user specific) for all taxa that have been visited at that location that year.. - // Need to rebuild all the entries which have the user_id and no location set for all taxa that have been visited at that location that year.. - // Need to rebuild all the top level global entries (there are no new taxa). - $queries = kohana::config('summary_builder'); - - if ($missing_check || (isset($definition['check_for_missing']) && $definition['check_for_missing'] != 'f')) { - if(self::$verbose) echo date(DATE_ATOM).' Running missed deleted sample check for survey ID '.$definition['survey_id'].'
'; - $query = $queries['get_missed_deleted_samples_query']; - } else { - if(self::$verbose) echo date(DATE_ATOM).' Running deleted sample check for survey ID '.$definition['survey_id'].'
'; - $query = $queries['get_deleted_samples_query']; + /** OK **/ + /** this fills out the data according to a single occurrence insert/delete **/ + private static function get_changelist_occurrence_insert_delete(&$db, $occurrenceId) { + $queries = kohana::config('summary_builder'); + $taxa = []; + + $query = str_replace('#occurrence_id#', $occurrenceId, $queries['occurrence_detail_lookup']); + $occurrenceResults = $db->query($query)->result_array(false); // returns year, created_by_id, location_id, taxa_taxon_list_id + if(count($occurrenceResults) === 0) + return array(0, 0, 0, []); + return array($occurrenceResults[0]['year'], $occurrenceResults[0]['user_id'], + $occurrenceResults[0]['location_id'], [$occurrenceResults[0]['taxa_taxon_list_id']]); } - self::run_samples_query($db, $query, $last_run_date, $definition, $limit, $YearTaxonLocationUser, $YearTaxonLocation, $YearTaxonUser, $YearTaxon); - if($limit <= 0 ) return; - - if ($missing_check || (isset($definition['check_for_missing']) && $definition['check_for_missing'] != 'f')) { - if(self::$verbose) echo date(DATE_ATOM).' Running missed created sample check for survey ID '.$definition['survey_id'].'
'; - $query = $queries['get_missed_created_samples_query']; - } else { - if(self::$verbose) echo date(DATE_ATOM).' Running created sample check for survey ID '.$definition['survey_id'].'
'; - $query = $queries['get_created_samples_query']; + + + /** OK **/ + public static function populate_summary_table_for_occurrence_modify(&$db, $occurrenceId, $definitionId) { + // A modification to an existing taxa may include the changing of the taxa: have to recalculate all taxa + // on that location/user/year. (e.g. on verification) + $queries = kohana::config('summary_builder'); + // this has to be done only once + self::init_lastval($db); + + $query = str_replace('#definition_id#', $definitionId, + $queries['get_definition']); + $definitionResult = $db->query($query)->result_array(false); + + // This returns this taxon, merged with all taxa on summary records for this location/year + // Includes a check that the data for the location/year/user/taxon is either not present or was last updated + // before the sample record update time: if not then has been run in another batch and is up to date + list($year, $userId, $locationId, $taxa) = self::get_changelist_occurrence_modify($db, $occurrenceId); + + if (count($taxa) > 0){ + self::do_summary($db, $definitionResult[0], $year, $userId, $locationId, $taxa); + } } - self::run_samples_query($db, $query, $last_run_date, $definition, $limit, $YearTaxonLocationUser, $YearTaxonLocation, $YearTaxonUser, $YearTaxon); - } - private static function run_samples_query(&$db, $query, $last_run_date, $definition, &$limit, &$YearTaxonLocationUser, &$YearTaxonLocation, &$YearTaxonUser, &$YearTaxon) - { - $query = str_replace(array('#survey_id#', '#date#', '#limit#'), - array($definition['survey_id'], $last_run_date, $limit), - $query); - $r = $db->query($query)->result_array(false); // returns date_start, created_by_id, location_id - - $count = count($r); - $actual = 0; - if($count){ - if(self::$verbose) echo date(DATE_ATOM).' Maximum of '.$count.' sample records to be processed (limit = '.$limit.').
'; - foreach($r as $row) { - $c = summary_builder::flag_all_taxa ($db, $definition['survey_id'], $YearTaxonLocationUser, $YearTaxonLocation, $YearTaxonUser, $YearTaxon, substr($row['date_start'], 0, 4), $row['location_id'], $row['user_id']); - $limit = $limit-$c; - $actual++; - if($limit <= 0 ) { - if(self::$verbose) echo date(DATE_ATOM).' '.$actual.' sample records processed, record count limit reached.
'; - if(self::$verbose) echo date(DATE_ATOM).' Date on sample last processed : '.$row['date_start'].'
'; - $limit = 0; - return; + /** this fills out the data according to a single occurrence modify **/ + private static function get_changelist_occurrence_modify(&$db, $occurrenceId) { + $queries = kohana::config('summary_builder'); + $taxa = []; + + $query = str_replace('#occurrence_id#', $occurrenceId, $queries['sample_detail_lookup_occurrence']); + $sampleResults = $db->query($query)->result_array(false); // returns survey_id, year, created_by_id, location_id + if(count($sampleResults) === 0) + return array(0, 0, 0, []); + + $query = str_replace('#occurrence_id#', $occurrenceId, $queries['occurrence_detail_lookup']); + $occurrenceResults = $db->query($query)->result_array(false); // returns taxa_taxon_list_id + foreach($occurrenceResults as $row){ + $taxa[] = $row['taxa_taxon_list_id']; + } + + $query = str_replace(array('#website_id#', '#survey_id#', '#sample_id#', '#year#', '#user_id#', '#location_id#'), + array($sampleResults[0]['website_id'], $sampleResults[0]['survey_id'], $sampleResults[0]['sample_id'] , + $sampleResults[0]['year'], $sampleResults[0]['user_id'], $sampleResults[0]['location_id']), + $queries['sample_existing_taxa']); + $existingResults = $db->query($query)->result_array(false); + foreach($existingResults as $row){ + $taxa[] = $row['taxa_taxon_list_id']; } - } - if(self::$verbose) echo date(DATE_ATOM).' Record count limit '.$limit.' after sample records processing.
'; - } else { - if(self::$verbose) echo date(DATE_ATOM).' No sample records processed.
'; + $taxa = array_unique($taxa); + return array($sampleResults[0]['year'], $sampleResults[0]['user_id'], $sampleResults[0]['location_id'], $taxa); } - } - - private static function flag_one_taxa (&$YearTaxonLocationUser, &$YearTaxonLocation, &$YearTaxonUser, &$YearTaxon, $year, $taxon, $location_id, $user_id) - { - // The year and taxon are always supplied. - // in a rebuild scenario, the user and the location may not be. - if(!isset($YearTaxon[$year])) $YearTaxon[$year]=array(); - if(!in_array($taxon, $YearTaxon[$year])) $YearTaxon[$year][] = $taxon; - // create list of year/taxon/user - if(!isset($YearTaxonUser[$year.':'.$taxon])) $YearTaxonUser[$year.':'.$taxon] = array(); - if($user_id != null && !in_array($user_id, $YearTaxonUser[$year.':'.$taxon])) $YearTaxonUser[$year.':'.$taxon][] = $user_id; - // create list of year/taxon/location - if(!isset($YearTaxonLocation[$year.':'.$taxon])) $YearTaxonLocation[$year.':'.$taxon] = array(); - if($location_id != null && !in_array($location_id, $YearTaxonLocation[$year.':'.$taxon])) $YearTaxonLocation[$year.':'.$taxon][] = $location_id; - // create list of year/taxon/location/user - if($location_id != null && !isset($YearTaxonLocationUser[$year.':'.$taxon.':'.$location_id])) $YearTaxonLocationUser[$year.':'.$taxon.':'.$location_id] = array(); - if($location_id != null && $user_id != null && !in_array($user_id, $YearTaxonLocationUser[$year.':'.$taxon.':'.$location_id])) $YearTaxonLocationUser[$year.':'.$taxon.':'.$location_id][] = $user_id; - } - - private static function flag_all_taxa (&$db, $survey_id, &$YearTaxonLocationUser, &$YearTaxonLocation, &$YearTaxonUser, &$YearTaxon, $year, $location_id, $user_id) - { - if(isset(self::$locationYearUser[$year.':'.$location_id.':'.$user_id])) - return 0; // previously processed a sample for this year/location/user, so no need to do again - - $queries = kohana::config('summary_builder'); - $query = str_replace(array('#survey_id#', '#location_id#', '#year#'), - array($survey_id, $location_id, $year), - $queries['get_taxa_query']); - $r = $db->query($query)->result_array(false); // returns all taxa_taxon_list_id for this year/location combination - self::$locationYearUser[$year.':'.$location_id.':'.$user_id] = true; - - $count = count($r); - if($count){ - foreach($r as $row){ - // create list of year/taxon - $taxon = $row['taxa_taxon_list_id']; - summary_builder::flag_one_taxa ($YearTaxonLocationUser, $YearTaxonLocation, $YearTaxonUser, $YearTaxon, $year, $taxon, $location_id, $user_id); - } + + + public static function populate_summary_table_for_location_delete(&$db, $locationId) { + $queries = kohana::config('summary_builder'); + // this has to be done only once + self::init_lastval($db); + + $query = str_replace('#location_id#', $locationId, $queries['location_existing_data']); + $existingResults = $db->query($query)->result_array(false); + $definitionResult = $db->query($queries['get_all_definitions'])->result_array(false); + $query = str_replace('#location_id#', $locationId, $queries['delete_location_data']); + $db->query($query); + foreach($existingResults as $row){ + foreach($definitionResult as $definition) { + if($definition['survey_id'] == $row['survey_id']) { + self::do_summary($db, $definitionResult[0], $row[0]['year'], $row[0]['user_id'], $locationId, + [$row['taxa_taxon_list_id']]); + } + } + } } - return $count; - } - private static function get_changelist(&$db, $last_run_date, $definition, &$YearTaxonLocationUser, &$YearTaxonLocation, &$YearTaxonUser, &$YearTaxon, $missing_check) { - $queries = kohana::config('summary_builder'); - $limit = (isset($definition['max_records_per_cycle']) ? $definition['max_records_per_cycle'] : 1000); - if(self::$verbose) echo date(DATE_ATOM).' Record count limit: '.$limit.'
'; + /* + * Bit of an oddball situation. Our table does not have a sequence. When inserting, The database driver sets + * the insert_id for the pg_result from the lastval function - with no sequence this returns an error, and the error + * is ignored from a PHP point of view. However from Postgres POV, this causes a transaction abort/rollback, + * so we can't wrap a transaction around it. We need a transaction though, so this is a hack to populate + * the lastval. Use a temporary sequence, as lastval gives an error if the sequence has been dropped. + */ + /** OK **/ + private static function init_lastval(&$db) { + $db->query("DROP SEQUENCE IF EXISTS summary_builder_dummy_sequence;"); + $db->query("CREATE TEMPORARY SEQUENCE summary_builder_dummy_sequence;"); // dropped automatically at end of process + $db->query("SELECT NEXTVAL('summary_builder_dummy_sequence');"); + } - // the location deletion does not lead to any new calculations, just re-summarising existing data, so no impact on limit. - if(!isset($_GET['only']) || $_GET['only'] == 'locations') - summary_builder::check_deleted_locations($db, $last_run_date, $definition, $YearTaxonLocationUser, $YearTaxonLocation, $YearTaxonUser, $YearTaxon, $missing_check); + /** OK **/ + private static function do_summary(&$db, $definition, $year, $userId, $locationId, $taxa) { + $queries = kohana::config('summary_builder'); + $db->begin(); + $yearStart = new DateTime($year.'-01-01'); + $yearEnd = new DateTime($year.'-01-01'); + // calculate date to period conversions + if($definition['period_type']=='W'){ + // work out week numbers, and period mapping. + // Week 1 = the week with date_from in + $weekstart=explode('=',$definition['period_start']); + if($weekstart[0]=='date'){ + $weekstart_date = date_create($year."-".$weekstart[1]); + if(!$weekstart_date){ + echo date(DATE_ATOM).' ERROR : Weekstart month-day combination unrecognised {'.$weekstart[1].'}
'; + return; + } + $weekstart[1]=$weekstart_date->format('N'); // ISO Day of week - Mon=1, Sun=7 + } + if(intval($weekstart[1])!=$weekstart[1] || $weekstart[1]<1 || $weekstart[1]>7) { + echo date(DATE_ATOM).' ERROR : Weekstart unrecognised or out of range {'.$weekstart[1].'}
'; + return; + } + $consider_date = new DateTime($year.'-01-01'); + $weekNoOffset=0; + while($consider_date->format('N')!=$weekstart[1]) $consider_date->modify('-1 day'); + $weekOne_date = date_create($year.'-'.$definition['period_one_contains']); + if(!$weekOne_date){ + echo date(DATE_ATOM).' ERROR : Week one month-day combination unrecognised {'.$definition['period_one_contains'].'}
'; + return; + } + while($weekOne_date->format('N')!=$weekstart[1]) $weekOne_date->modify('-1 day'); // scan back to start of week + while($weekOne_date > $consider_date){ + $weekOne_date->modify('-7 days'); + $weekNoOffset++; + } - if(!isset($_GET['only']) || $_GET['only'] == 'samples') - summary_builder::check_samples($db, $last_run_date, $definition, $limit, $YearTaxonLocationUser, $YearTaxonLocation, $YearTaxonUser, $YearTaxon, $missing_check); + // the season limits define the start and end of the recording season. The generation of estimates + // is restricted to these weeks. + // if no value is provided for either the start or the end, then the relevant value is assume to be either the start of the year + // or the end of the year. + // If a value is provided, it is inclusive. UKBMS is nominally 1,26: this differs from the old report_calendar_summary + // page, where it was exclusive, and the limits where defined as 0,27. + $season_limits=explode(',',$definition['season_limits']); + $definition['season_limits_array'] = array('start' => ((count($season_limits) && $season_limits[0]!='') ? $season_limits[0] : false), + 'end' => ((count($season_limits)>1 && $season_limits[1]!='') ? $season_limits[1] : false)); + $periods=array(); + $periodMapping=array(); + // Build day number to period mapping. first period = 1, days 1st Jan = 0 + $dayIterator = new DateTime($year.'-01-01'); + $periodNo = 1-$weekNoOffset; + while($dayIterator->format('Y')==$year){ + $periodMapping[$dayIterator->format('z')]=$periodNo; + $dayIterator->modify('+1 day'); + if($dayIterator->format('N') == $weekstart[1]) $periodNo++; + } + // Build period definition. first period = 1, days 1st Jan = 0 + $dayIterator = clone $weekOne_date; + $periodNo = 1-$weekNoOffset; + $periodLength = date_interval_create_from_date_string('6 days'); + while($dayIterator->format('Y')<=$year){ + $endDate = clone $dayIterator; + $endDate->modify('+6 day'); + $periods[$periodNo]=array('date_start'=>$dayIterator->format('Y-m-d'), 'date_end'=>$endDate->format('Y-m-d')); + $dayIterator->modify('+7 days'); + $periodNo++; + } + } else { + echo date(DATE_ATOM).' ERROR : period_type unrecognised {'.$definition['period_type'].'}
'; + return; + } + foreach($taxa as $taxonId) { + $taxon = $db->query("SELECT * FROM cache_taxa_taxon_lists WHERE id = $taxonId")->result_array(false); + if(count($taxon)!=1) { + echo date(DATE_ATOM)." ERROR : Taxon search for id = $taxonId returned wrong number of rows: ".count($taxon)." - expected one row.
"; + continue; + } + summary_builder::do_delete($db, $definition, $year, $taxonId, $locationId, $userId); + summary_builder::do_delete($db, $definition, $year, $taxonId, $locationId, 0); + summary_builder::do_delete($db, $definition, $year, $taxonId, 0, $userId); + summary_builder::do_delete($db, $definition, $year, $taxonId, 0, 0); + + // This updates the data for the user/taxa/location/year combination. + $query = str_replace(array('#year#', '#survey_id#', '#taxon_id#', '#location_id#', '#user_id#', '#attr_id#'), + array($year, $definition['survey_id'], $taxonId, $locationId, $userId, $definition['occurrence_attribute_id']), + $definition['occurrence_attribute_id'] != '' ? $queries['get_YearTaxonLocationUser_Attr_query'] : $queries['get_YearTaxonLocationUser_query']); + $data = array(); + if(summary_builder::load_data($db, $data, $periods, $periodMapping, $query)) { + summary_builder::apply_data_combining($definition, $data); + if($definition['calculate_estimates'] != 'f') { + summary_builder::apply_estimates($db, $definition, $data); + } + summary_builder::do_insert($db, $definition, $year, $taxonId, $locationId, $userId, $data, $periods, $taxon[0]); + } - // Now check occurrences - if($limit > 0 && (!isset($_GET['only']) || $_GET['only'] == 'occurrences')) { - if ($missing_check || (isset($definition['check_for_missing']) && $definition['check_for_missing'] != 'f')) { - if(self::$verbose) echo date(DATE_ATOM).' Start of missing occurrence query for survey ID '.$definition['survey_id'].'
'; - $query = str_replace(array('#date#','#survey_id#','#limit#'), - array($last_run_date,$definition['survey_id'],$limit), - $queries['get_missed_changed_occurrences_query']); - } else { - if(self::$verbose) echo date(DATE_ATOM).' Start of occurrence query for survey ID '.$definition['survey_id'].'
'; - $query = str_replace(array('#date#','#survey_id#','#limit#'), - array($last_run_date,$definition['survey_id'],$limit), - $queries['get_changed_occurrences_query']); - } - $r = $db->query($query)->result_array(false); - $count = count($r); - if($count){ - if(self::$verbose) echo date(DATE_ATOM).' '.$count.' occurrences to be processed.
'; - foreach($r as $row) - summary_builder::flag_one_taxa ($YearTaxonLocationUser, $YearTaxonLocation, $YearTaxonUser, $YearTaxon, substr($row['date_start'], 0, 4), $row['taxa_taxon_list_id'], $row['location_id'], $row['created_by_id']); - $limit = $limit-$count; - } else if(self::$verbose) echo date(DATE_ATOM).' No occurrences to be processed.
'; + // This updates the data for the allusers/taxa/location/year combination. + $query = str_replace(array('#year#', '#survey_id#', '#taxon_id#', '#location_id#', '#attr_id#'), + array($year, $definition['survey_id'], $taxonId, $locationId, $definition['occurrence_attribute_id']), + $definition['occurrence_attribute_id'] != '' ? $queries['get_YearTaxonLocation_Attr_query'] : $queries['get_YearTaxonLocation_query']); + $data = array(); + if(summary_builder::load_data($db, $data, $periods, $periodMapping, $query)) { + summary_builder::apply_data_combining($definition, $data); + if($definition['calculate_estimates'] != 'f') + summary_builder::apply_estimates($db, $definition, $data); + summary_builder::do_insert($db, $definition, $year, $taxonId, $locationId, 0, $data, $periods, $taxon[0]); + } - if($limit > 0 && ($missing_check || (isset($definition['check_for_missing']) && $definition['check_for_missing'] != 'f'))) { - if(self::$verbose) echo date(DATE_ATOM).' Start of missing deleted occurrence query for survey ID '.$definition['survey_id'].'
'; - $query = str_replace(array('#date#','#survey_id#','#limit#'), - array($last_run_date,$definition['survey_id'],$limit), - $queries['get_missed_deleted_occurrences_query']); - $r = $db->query($query)->result_array(false); - $count = count($r); - if($count){ - if(self::$verbose) echo date(DATE_ATOM).' '.$count.' deleted occurrences to be processed.
'; - foreach($r as $row) - summary_builder::flag_one_taxa ($YearTaxonLocationUser, $YearTaxonLocation, $YearTaxonUser, $YearTaxon, substr($row['date_start'], 0, 4), $row['taxa_taxon_list_id'], $row['location_id'], $row['created_by_id']); - $limit = $limit-$count; - } else if(self::$verbose) echo date(DATE_ATOM).' No deleted occurrences to be processed.
'; - } else { - if(self::$verbose) echo date(DATE_ATOM).' Start of entry rebuild query for survey ID '.$definition['survey_id'].'
'; - $query = str_replace(array('#survey_id#','#limit#'), - array($definition['survey_id'],min($limit,50)), - $queries['get_rebuild_occurrences_query']); - $r = $db->query($query)->result_array(false); - $count = count($r); - if($count){ - if(self::$verbose) echo date(DATE_ATOM).' '.$count.' entries to be processed.
'; - foreach($r as $row) - summary_builder::flag_one_taxa ($YearTaxonLocationUser, $YearTaxonLocation, $YearTaxonUser, $YearTaxon, $row['year'], $row['taxa_taxon_list_id'], $row['location_id'], $row['user_id']); - $limit = $limit-$count; + // This run updates the data for the user/taxa/location/year combination. + // Then ads this all together to find the value for the user/taxa/alllocations/year combination + // This run updates the data for the allusers/taxa/location/year combination. + // Then ads this all together to find the value for the allusers/taxa/alllocations/year combination + // In this cycle the year is fixed, the taxa is either one or an array, the location is fixed, the user is fixed. + $query = str_replace(array('#year#', '#survey_id#', '#taxon_id#', '#user_id#'), + array($year, $definition['survey_id'], $taxonId, $userId), $queries['get_YearTaxonUser_query']); + $data = array(); + if(summary_builder::load_summary_data($db, $data, $periods, $periodMapping, $query)) { + summary_builder::do_insert($db, $definition, $year, $taxonId, 0, $userId, $data, $periods, $taxon[0]); + } + // This run updates the data for the user/taxa/location/year combination. + // Then ads this all together to find the value for the user/taxa/alllocations/year combination + // This run updates the data for the allusers/taxa/location/year combination. + // Then ads this all together to find the value for the allusers/taxa/alllocations/year combination + // In this cycle the year is fixed, the taxa is either one or an array, the location is fixed, the user is fixed. + + $query = str_replace(array('#year#', '#survey_id#', '#taxon_id#'), + array($year, $definition['survey_id'], $taxonId), $queries['get_YearTaxon_query']); + $data = array(); + if(summary_builder::load_summary_data($db, $data, $periods, $periodMapping, $query)) { + summary_builder::do_insert($db, $definition, $year, $taxonId, 0, 0, $data, $periods, $taxon[0]); + } } - } + $db->commit(); } - if($limit > 0 && (!isset($_GET['only']) || $_GET['only'] == 'taxa')) { - if(self::$verbose) echo date(DATE_ATOM).' Start of changed taxa query for survey ID '.$definition['survey_id'].'
'; - $query = str_replace(array('#survey_id#', '#limit#'), - array($definition['survey_id'], $limit), - $queries['get_changed_taxa_query']); - $r = $db->query($query)->result_array(false); - $count = count($r); - if($count){ - if(self::$verbose) echo date(DATE_ATOM).' '.$count.' entries to be processed.
'; - foreach($r as $row) { - summary_builder::flag_one_taxa ($YearTaxonLocationUser, $YearTaxonLocation, $YearTaxonUser, $YearTaxon, $row['year'], $row['taxa_taxon_list_id'], $row['location_id'], $row['user_id']); + + /** OK **/ + private static function load_data(&$db, &$data, &$periods, &$periodMapping, $query) { + $present = false; + $r = $db->query($query)->result_array(false); + foreach($periods as $periodNo=>$defn) { + $data[$periodNo] = array('summary'=>0, 'hasData'=>false, 'hasEstimate'=>false,'samples'=>array()); } - } - } - if(count($YearTaxon)>0){ - ksort($YearTaxon); - ksort($YearTaxonUser); - ksort($YearTaxonLocation); - ksort($YearTaxonLocationUser); - } - return count($YearTaxon); - } - - private static function do_summary(&$db, $definition, $YearTaxonLocationUser, $YearTaxonLocation, $YearTaxonUser, $YearTaxon) { - $queries = kohana::config('summary_builder'); - foreach($YearTaxon as $year=>$taxonList) { - $db->begin(); - echo date(DATE_ATOM).' Processing data for '.$year.'
'; - $yearStart = new DateTime($year.'-01-01'); - $yearEnd = new DateTime($year.'-01-01'); - // calculate date to period conversions - if($definition['period_type']=='W'){ - // work out week numbers, and period mapping. - // Week 1 = the week with date_from in - $weekstart=explode('=',$definition['period_start']); - if($weekstart[0]=='date'){ - $weekstart_date = date_create($year."-".$weekstart[1]); - if(!$weekstart_date){ - echo date(DATE_ATOM).' ERROR : Weekstart month-day combination unrecognised {'.$weekstart[1].'}
'; - return; + foreach($r as $row) { + $datetime1 = new DateTime($row['date_start']); + $offset = $datetime1->format('z'); + $period = $periodMapping[$offset]; + $data[$period]['hasData'] = true; + if(isset($data[$period]['samples'][$row['sample_id']])) $data[$period]['samples'][$row['sample_id']] += $row['count']; + else $data[$period]['samples'][$row['sample_id']] = $row['count']; + if(!isset($row['present']) || $row['present']=='t') + $present = true; } - $weekstart[1]=$weekstart_date->format('N'); // ISO Day of week - Mon=1, Sun=7 - } - if(intval($weekstart[1])!=$weekstart[1] || $weekstart[1]<1 || $weekstart[1]>7) { - echo date(DATE_ATOM).' ERROR : Weekstart unrecognised or out of range {'.$weekstart[1].'}
'; - return; - } - $consider_date = new DateTime($year.'-01-01'); - $weekNoOffset=0; - while($consider_date->format('N')!=$weekstart[1]) $consider_date->modify('-1 day'); - $weekOne_date = date_create($year.'-'.$definition['period_one_contains']); - if(!$weekOne_date){ - echo date(DATE_ATOM).' ERROR : Week one month-day combination unrecognised {'.$definition['period_one_contains'].'}
'; - return; - } - while($weekOne_date->format('N')!=$weekstart[1]) $weekOne_date->modify('-1 day'); // scan back to start of week - while($weekOne_date > $consider_date){ - $weekOne_date->modify('-7 days'); - $weekNoOffset++; - } - - // the season limits define the start and end of the recording season. The generation of estimates - // is restricted to these weeks. - // if no value is provided for either the start or the end, then the relevant value is assume to be either the start of the year - // or the end of the year. - // If a value is provided, it is inclusive. UKBMS is nominally 1,26: this differs from the old report_calendar_summary - // page, where it was exclusive, and the limits where defined as 0,27. - $season_limits=explode(',',$definition['season_limits']); - $definition['season_limits_array'] = array('start' => ((count($season_limits) && $season_limits[0]!='') ? $season_limits[0] : false), - 'end' => ((count($season_limits)>1 && $season_limits[1]!='') ? $season_limits[1] : false)); - $periods=array(); - $periodMapping=array(); - // Build day number to period mapping. first period = 1, days 1st Jan = 0 - $dayIterator = new DateTime($year.'-01-01'); - $periodNo = 1-$weekNoOffset; - while($dayIterator->format('Y')==$year){ - $periodMapping[$dayIterator->format('z')]=$periodNo; - $dayIterator->modify('+1 day'); - if($dayIterator->format('N') == $weekstart[1]) $periodNo++; - } - // Build period definition. first period = 1, days 1st Jan = 0 - $dayIterator = clone $weekOne_date; - $periodNo = 1-$weekNoOffset; - $periodLength = date_interval_create_from_date_string('6 days'); - while($dayIterator->format('Y')<=$year){ - $endDate = clone $dayIterator; - $endDate->modify('+6 day'); - $periods[$periodNo]=array('date_start'=>$dayIterator->format('Y-m-d'), 'date_end'=>$endDate->format('Y-m-d')); - $dayIterator->modify('+7 days'); - $periodNo++; - } - } else { - echo date(DATE_ATOM).' ERROR : period_type unrecognised {'.$definition['period_type'].'}
'; - return; + return $present; } - foreach($taxonList as $taxonID) { - $taxon = $db->query("SELECT * FROM cache_taxa_taxon_lists WHERE id = $taxonID")->result_array(false); - if(count($taxon)!=1) { - echo date(DATE_ATOM)." ERROR : Taxon search for id = $taxonID returned wrong number of rows: ".count($taxon)." - expected one row.
"; - continue; + + /** OK **/ + private static function load_summary_data(&$db, &$data, &$periods, &$periodMapping, $query) { + $results = $db->query($query)->result_array(false); + $present = false; + foreach($periods as $periodNo=>$defn) { + $data[$periodNo] = array('summary'=>0, 'hasData'=>false, 'estimate'=>0, 'hasEstimate'=>false,'samples'=>array()); } - - foreach($YearTaxonLocation[$year.':'.$taxonID] as $locationID) { - foreach($YearTaxonLocationUser[$year.':'.$taxonID.':'.$locationID] as $userID){ - $query = str_replace(array('#year#', '#survey_id#', '#taxon_id#', '#location_id#', '#user_id#', '#attr_id#'), - array($year, $definition['survey_id'], $taxonID, $locationID, $userID, $definition['occurrence_attribute_id']), - $definition['occurrence_attribute_id'] != '' ? $queries['get_YearTaxonLocationUser_Attr_query'] : $queries['get_YearTaxonLocationUser_query']); - $data = array(); - if(self::$verbose) echo date(DATE_ATOM).' Processing data for Y'.$year.' T'.$taxonID.' L'.$locationID.' U'.$userID.'
'; - summary_builder::do_delete($db, $definition, $year, $taxonID, $locationID, $userID); - if(!summary_builder::load_data($db, $data, $periods, $periodMapping, $query)) { - if(self::$verbose) echo date(DATE_ATOM).' Data cleared, none inserted
'; - continue; + foreach($results as $row) { + $summary=json_decode($row['summarised_data'], true); + foreach($summary as $period) { + if($period['summary'] !== null && $period['summary'] !== 'NULL') { + $data[$period['period']]['summary'] += $period['summary']; + $data[$period['period']]['hasData'] = true; + $present = true; + } + if($period['estimate'] !== null && $period['estimate'] !== 'NULL') { + $data[$period['period']]['estimate'] += $period['estimate']; + $data[$period['period']]['hasEstimate'] = true; + $present = true; + } + } } - summary_builder::apply_data_combining($definition, $data); - if($definition['calculate_estimates'] != 'f') - summary_builder::apply_estimates($db, $definition, $data); -// summary_builder::dump_data($data); - summary_builder::do_insert($db, $definition, $year, $taxonID, $locationID, $userID, $data, $periods, $taxon[0]); + return $present; + } + + /** OK **/ + private static function apply_data_combining($definition, &$data) { + foreach($data as $period=>$detail){ + switch($definition['data_combination_method']){ + case 'M': + foreach($detail['samples'] as $sampleID=>$value) + $data[$period]['summary'] = max($data[$period]['summary'], $value); + break; + // Not doing an 'S' samples with occurrences + case 'L': + $val=0; + foreach($detail['samples'] as $sampleID=>$value) $val += $value; + $cnt = count($detail['samples']); + $val = $cnt ? ($val.".0")/$cnt : 0; + if($val>0 && $val<1) $val=1; + // data rounding only occurs in this option + $data[$period]['summary'] = summary_builder::apply_data_rounding($definition, $val, true); + break; + default : + case 'A': + foreach($detail['samples'] as $sampleID=>$value) + $data[$period]['summary'] += $value; + break; + } } - if(self::$verbose) echo date(DATE_ATOM).' Processing data for Y'.$year.' T'.$taxonID.' L'.$locationID.'
'; - $query = str_replace(array('#year#', '#survey_id#', '#taxon_id#', '#location_id#', '#attr_id#'), - array($year, $definition['survey_id'], $taxonID, $locationID, $definition['occurrence_attribute_id']), - $definition['occurrence_attribute_id'] != '' ? $queries['get_YearTaxonLocation_Attr_query'] : $queries['get_YearTaxonLocation_query']); - $data = array(); - summary_builder::do_delete($db, $definition, $year, $taxonID, $locationID, false); - if(!summary_builder::load_data($db, $data, $periods, $periodMapping, $query)) { - if(self::$verbose) echo date(DATE_ATOM).' Data cleared, none inserted
'; - continue; + } + + /** OK **/ + private static function apply_estimates(&$db, $definition, &$data) { + $season_start = $definition['season_limits_array']['start']; + $season_end = $definition['season_limits_array']['end']; + $thisLocation=false; + $lastDataPeriod=false; + $minPeriod = min(array_keys($data)); + foreach($data as $period=>$detail) { // we assume this comes out in period order + if($detail['hasData']) { + $data[$period]['estimate'] = $detail['summary']; + $data[$period]['hasEstimate'] = true; + // Half value estimate setup if this is the first count. Previous period (which is where the estimate will be) must be within season limits + if($lastDataPeriod===false && $definition['first_value']=='H') { + $lastDataPeriod = $period-2; + $lastDataPeriodValue = 0; + } + if($lastDataPeriod!==false && ($period-$lastDataPeriod > 1)){ + for($j=1; $j < ($period-$lastDataPeriod); $j++){ // fill in periods between data points + // only consider estimate generation within the season limits. + if(($season_start===false || ($lastDataPeriod+$j)>=$season_start) && ($season_end===false || ($lastDataPeriod+$j)<=$season_end)) { + $estimate = $lastDataPeriodValue+(($j.".0")*($data[$period]['summary']-$lastDataPeriodValue))/($period-$lastDataPeriod); + $data[$lastDataPeriod+$j]['estimate'] = summary_builder::apply_data_rounding($definition, $estimate, false); + $data[$lastDataPeriod+$j]['hasEstimate'] = true; + } + } + } + $lastDataPeriod=$period; + $lastDataPeriodValue=$data[$lastDataPeriod]['summary']; + } } - summary_builder::apply_data_combining($definition, $data); - if($definition['calculate_estimates'] != 'f') - summary_builder::apply_estimates($db, $definition, $data); - //summary_builder::dump_data($data); - summary_builder::do_insert($db, $definition, $year, $taxonID, $locationID, false, $data, $periods, $taxon[0]); - } - foreach($YearTaxonUser[$year.':'.$taxonID] as $userID){ - if(self::$verbose) echo date(DATE_ATOM).' Processing data for Y'.$year.' T'.$taxonID.' U'.$userID.'
'; - $query = str_replace(array('#year#', '#survey_id#', '#taxon_id#', '#user_id#'), - array($year, $definition['survey_id'], $taxonID, $userID), $queries['get_YearTaxonUser_query']); - $data = array(); - summary_builder::do_delete($db, $definition, $year, $taxonID, false, $userID); - if(!summary_builder::load_summary_data($db, $data, $periods, $periodMapping, $query)) { - if(self::$verbose) echo date(DATE_ATOM).' Data cleared, none inserted
'; - continue; + // Have reached end of data, so do half value estimate setup. Next period (which is where the estimate will be) must be within season limits + if($lastDataPeriod && ($season_start===false || $lastDataPeriod+1>=$season_start) && ($season_end===false || $lastDataPeriod+1<=$season_end) && $lastDataPeriod'; - $query = str_replace(array('#year#', '#survey_id#', '#taxon_id#'), - array($year, $definition['survey_id'], $taxonID), $queries['get_YearTaxon_query']); - $data = array(); - summary_builder::do_delete($db, $definition, $year, $taxonID, false, false); - if(summary_builder::load_summary_data($db, $data, $periods, $periodMapping, $query)) { - //summary_builder::dump_data($data); - summary_builder::do_insert($db, $definition, $year, $taxonID, false, false, $data, $periods, $taxon[0]); - } else { - if(self::$verbose) echo date(DATE_ATOM).' Data cleared, none inserted
'; - } - } - $db->commit(); - } - if(self::$verbose) echo date(DATE_ATOM).' End of summarisation for survey ID '.$definition['survey_id'].'

'; - } - - private static function load_data(&$db, &$data, &$periods, &$periodMapping, $query) { - $present = false; - $r = $db->query($query)->result_array(false); - foreach($periods as $periodNo=>$defn) - $data[$periodNo] = array('summary'=>0, 'hasData'=>false, 'hasEstimate'=>false,'samples'=>array()); - foreach($r as $row) { - $datetime1 = new DateTime($row['date_start']); - $offset = $datetime1->format('z'); - $period = $periodMapping[$offset]; - $data[$period]['hasData'] = true; - if(isset($data[$period]['samples'][$row['sample_id']])) $data[$period]['samples'][$row['sample_id']] += $row['count']; - else $data[$period]['samples'][$row['sample_id']] = $row['count']; - if(!isset($row['present']) || $row['present']=='t') - $present = true; - } - return $present; - } - - private static function load_summary_data(&$db, &$data, &$periods, &$periodMapping, $query) { - $r = $db->query($query)->result_array(false); - $present = false; - foreach($periods as $periodNo=>$defn) - $data[$periodNo] = array('summary'=>0, 'hasData'=>false, 'estimate'=>0, 'hasEstimate'=>false,'samples'=>array()); - foreach($r as $row) { - $datetime1 = new DateTime($row['date_start']); - $offset = $datetime1->format('z'); - $period = $periodMapping[$offset]; - if($row['count']!= null) { - $data[$period]['summary'] += $row['count']; - $data[$period]['hasData'] = true; - $present = true; - } - if($row['estimate']!= null) { - $data[$period]['estimate'] += $row['estimate']; - $data[$period]['hasEstimate'] = true; - $present = true; - } } - return $present; - } - private static function dump_data(&$data) { - echo "

"; - foreach($data as $period=>$detail) { - echo ''; - } - echo ''; - foreach($data as $period=>$detail) { - echo ''; - } - echo ''; - foreach($data as $period=>$detail) { - echo ''; + + /** OK **/ + private static function apply_data_rounding($definition, $val, $special) { + if($special && $val>0 && $val<1) return 1; + switch($definition['data_rounding_method']){ + case 'N': + return (int)round($val); + case 'U': + return (int)ceil($val); + case 'D': + return (int)floor($val); + case 'X': + default : + break; + } + return $val; } - echo '
'.$period.'
'.($detail["hasData"] ? $detail["summary"] : '').'
'.($detail["hasEstimate"] ? $detail["estimate"] : '').'

'; - } - - private static function apply_data_combining($definition, &$data) { - foreach($data as $period=>$detail){ - switch($definition['data_combination_method']){ - case 'M': - foreach($detail['samples'] as $sampleID=>$value) - $data[$period]['summary'] = max($data[$period]['summary'], $value); - break; - // Not doing an 'S' samples with occurrences - case 'L': - $val=0; - foreach($detail['samples'] as $sampleID=>$value) $val += $value; - $cnt = count($detail['samples']); - $val = $cnt ? ($val.".0")/$cnt : 0; - if($val>0 && $val<1) $val=1; - // data rounding only occurs in this option - $data[$period]['summary'] = summary_builder::apply_data_rounding($definition, $val, true); - break; - default : - case 'A': - foreach($detail['samples'] as $sampleID=>$value) - $data[$period]['summary'] += $value; - break; - } + + /** OK **/ + private static function do_delete(&$db, $definition, $year, $taxonId, $locationId, $userId) { + // set up a default delete query if none are specified + $query = "delete from summary_occurrences where year = '".$year."' AND ". + "survey_id = ".$definition['survey_id']." AND ". + "taxa_taxon_list_id = ".$taxonId." AND ". + "location_id = ".$locationId." AND ". + "user_id = ".$userId.";"; + $count = $db->query($query)->count(); } - } - - private static function apply_estimates(&$db, $definition, &$data) { - $season_start = $definition['season_limits_array']['start']; - $season_end = $definition['season_limits_array']['end']; - $thisLocation=false; - $lastDataPeriod=false; - $minPeriod = min(array_keys($data)); - foreach($data as $period=>$detail) { // we assume this comes out in period order - if($detail['hasData']) { - $data[$period]['estimate'] = $detail['summary']; - $data[$period]['hasEstimate'] = true; - // Half value estimate setup if this is the first count. Previous period (which is where the estimate will be) must be within season limits - if($lastDataPeriod===false && $definition['first_value']=='H') { - $lastDataPeriod = $period-2; - $lastDataPeriodValue = 0; - } - if($lastDataPeriod!==false && ($period-$lastDataPeriod > 1)){ - for($j=1; $j < ($period-$lastDataPeriod); $j++){ // fill in periods between data points - // only consider estimate generation within the season limits. - if(($season_start===false || ($lastDataPeriod+$j)>=$season_start) && ($season_end===false || ($lastDataPeriod+$j)<=$season_end)) { - $estimate = $lastDataPeriodValue+(($j.".0")*($data[$period]['summary']-$lastDataPeriodValue))/($period-$lastDataPeriod); - $data[$lastDataPeriod+$j]['estimate'] = summary_builder::apply_data_rounding($definition, $estimate, false); - $data[$lastDataPeriod+$j]['hasEstimate'] = true; + + /** OK **/ + private static function do_insert(&$db, $definition, $year, $taxonId, $locationId, $userId, &$data, &$periods, $taxon) { + $summary = array(); + foreach($data as $period=>$details){ + if($details['hasData']||$details['hasEstimate']){ + $summary[] = array( + 'period' => $period, + 'date_start' => $periods[$period]['date_start'], + 'date_end' => $periods[$period]['date_end'], + 'summary' => $details['hasData'] ? $details['summary'] : "NULL", + 'estimate' => $details['hasEstimate'] ? $details['estimate'] : "NULL", + ); } - } } - $lastDataPeriod=$period; - $lastDataPeriodValue=$data[$lastDataPeriod]['summary']; - } - } - // Have reached end of data, so do half value estimate setup. Next period (which is where the estimate will be) must be within season limits - if($lastDataPeriod && ($season_start===false || $lastDataPeriod+1>=$season_start) && ($season_end===false || $lastDataPeriod+1<=$season_end) && $lastDataPeriod0 && $val<1) return 1; - switch($definition['data_rounding_method']){ - case 'N': - return (int)round($val); - case 'U': - return (int)ceil($val); - case 'D': - return (int)floor($val); - case 'X': - default : - break; - } - return $val; - } - - private static function do_delete(&$db, $definition, $year, $taxonID, $locationID, $userID) { - // set up a default delete query if none are specified - $query = "delete from summary_occurrences where year = '".$year."' AND ". - "survey_id = ".$definition['survey_id']." AND ". - "taxa_taxon_list_id = ".$taxonID." AND ". - "location_id ".($locationID ? "= ".$locationID : "IS NULL")." AND ". - "user_id ".($userID ? "= ".$userID : "IS NULL"); - $count = $db->query($query)->count(); - } - - private static function do_insert(&$db, $definition, $year, $taxonID, $locationID, $userID, &$data, &$periods, $taxon) { - // set up a default delete query if none are specified - $rows = array(); - foreach($data as $period=>$details){ - if($details['hasData']||$details['hasEstimate']){ - $rows[] = "(".implode(',',array($definition['website_id'], /* website_id integer */ - $definition['survey_id'], // survey_id integer, - $year, - $locationID ? $locationID : "NULL", //location_id integer, - $userID ? $userID : "NULL", // user_id integer, - $period, - "'".$periods[$period]['date_start']."'", // date_start date, - "'".$periods[$period]['date_end']."'", // date_end date, - "'DD'", // date_type character varying(2), - "'".$definition['period_type']."'", // type character varying, - $taxonID, // taxa_taxon_list_id integer, - $taxon["preferred_taxa_taxon_list_id"], // preferred_taxa_taxon_list_id integer, - $taxon["taxonomic_sort_order"]=="" || $taxon["taxonomic_sort_order"]==null ? "NULL" : $taxon["taxonomic_sort_order"], // taxonomic_sort_order bigint, - "'".str_replace("'","''",$taxon["taxon"])."'", // taxon character varying, - "'".str_replace("'","''",$taxon["preferred_taxon"])."'", // preferred_taxon character varying, - "'".str_replace("'","''",$taxon["default_common_name"])."'", // default_common_name character varying, - $taxon["taxon_meaning_id"], // taxon_meaning_id integer, - $taxon["taxon_list_id"], // taxon_list_id integer, - $details['hasData'] ? $details['summary'] : "NULL", // count double precision, - $details['hasEstimate'] ? $details['estimate'] : "NULL", // estimate double precision, - 1, // created_by_id integer, - "now()" // summary_created_on timestamp without time zone NOT NULL, - )).")"; - } - } - if(!count($rows)) return; - $query = "insert into summary_occurrences (website_id, + if(!count($summary)) return; + + $query = "insert into summary_occurrences ( + website_id, survey_id, year, location_id, user_id, - period_number, - date_start, - date_end, - date_type, type, taxa_taxon_list_id, preferred_taxa_taxon_list_id, taxonomic_sort_order, - taxon, - preferred_taxon, - default_common_name, - taxon_meaning_id, + taxon, + preferred_taxon, + default_common_name, + taxon_meaning_id, taxon_list_id, - count, - estimate, + summarised_data, created_by_id, summary_created_on) - VALUES ".implode(',',$rows); - $db->query($query)->count(); - } + VALUES ( + ".$definition['website_id'].", + ".$definition['survey_id'].", + ".$year.", + ".$locationId.", + ".$userId.", + '".$definition['period_type']."', + ".$taxonId.", + ".$taxon["preferred_taxa_taxon_list_id"].", + ".($taxon["taxonomic_sort_order"] === null ? "NULL" : $taxon["taxonomic_sort_order"]).", + '".str_replace("'","''",$taxon["taxon"])."', + '".str_replace("'","''",$taxon["preferred_taxon"])."', + '".str_replace("'","''",$taxon["default_common_name"])."', + ".$taxon["taxon_meaning_id"].", + ".$taxon["taxon_list_id"].", + '".json_encode($summary)."'::json, + 1, + now() + );"; + $db->query($query); + } + +// throw new exception('Configured survey restriction incorrect in spatial index builder'); + + + + /** + * A utility function used by the work queue task helpers. + * + * Returns the filter SQL to limit indexed locations to the correct types as + * declared in the configuration, with survey limits where appropriate. + * + * @param object $db + * Database connection object. + * + * @return string + * SQL filter clause. + */ -} \ No newline at end of file +} diff --git a/modules/summary_builder/helpers/task_summary_builder_location_delete.php b/modules/summary_builder/helpers/task_summary_builder_location_delete.php new file mode 100644 index 0000000000..20189e3770 --- /dev/null +++ b/modules/summary_builder/helpers/task_summary_builder_location_delete.php @@ -0,0 +1,51 @@ +query($query)->result_array(false); + foreach($result as $row){ + summary_builder::populate_summary_table_for_location_delete($db, $row['location_id']); + } + } +} \ No newline at end of file diff --git a/modules/summary_builder/helpers/task_summary_builder_occurrence_insert_delete.php b/modules/summary_builder/helpers/task_summary_builder_occurrence_insert_delete.php new file mode 100644 index 0000000000..a7ec16a71b --- /dev/null +++ b/modules/summary_builder/helpers/task_summary_builder_occurrence_insert_delete.php @@ -0,0 +1,56 @@ +query($query)->result_array(false); + foreach($result as $row){ + summary_builder::populate_summary_table_for_occurrence_insert_delete($db, $row['occurrence_id'], $row['definition_id']); + } + } + +} diff --git a/modules/summary_builder/helpers/task_summary_builder_occurrence_update.php b/modules/summary_builder/helpers/task_summary_builder_occurrence_update.php new file mode 100644 index 0000000000..20c1bba997 --- /dev/null +++ b/modules/summary_builder/helpers/task_summary_builder_occurrence_update.php @@ -0,0 +1,58 @@ +query($query)->result_array(false); + foreach($result as $row){ + summary_builder::populate_summary_table_for_occurrence_modify($db, $row['occurrence_id'], $row['definition_id']); + } + } + +} diff --git a/modules/summary_builder/helpers/task_summary_builder_sample.php b/modules/summary_builder/helpers/task_summary_builder_sample.php new file mode 100644 index 0000000000..23ae32d2db --- /dev/null +++ b/modules/summary_builder/helpers/task_summary_builder_sample.php @@ -0,0 +1,52 @@ +query($query)->result_array(false); + foreach($result as $row){ + summary_builder::populate_summary_table_for_sample($db, $row['sample_id'], $row['definition_id']); + } + } + +} diff --git a/modules/summary_builder/helpers/task_summary_builder_taxon.php b/modules/summary_builder/helpers/task_summary_builder_taxon.php new file mode 100644 index 0000000000..a1e4c4b10e --- /dev/null +++ b/modules/summary_builder/helpers/task_summary_builder_taxon.php @@ -0,0 +1,62 @@ +'user', 'updated_by'=>'user'); - protected $has_and_belongs_to_many = array(); - - public function validate(Validation $array, $save = FALSE) { - // uses PHP trim() to remove whitespace from beginning and end of all fields before validation - $array->pre_filter('trim'); - $array->add_rules('survey_id', 'required'); - $array->add_rules('period_type', 'required', 'chars[W,M]'); - $array->add_rules('period_start', 'required'); - $array->add_rules('period_one_contains', 'required'); - $array->add_rules('data_combination_method', 'required', 'chars[A,L,M,S]'); - $array->add_rules('data_rounding_method', 'required', 'chars[D,N,U,X]'); - $array->add_rules('interpolation', 'required', 'chars[L]'); - $array->add_rules('first_value', 'required', 'chars[X,H]'); - $array->add_rules('last_value', 'required', 'chars[X,H]'); - $array->add_rules('max_records_per_cycle', 'required', 'integer', 'minimum[1]'); - - // Explicitly add those fields for which we don't do validation - $this->unvalidatedFields = array( - 'occurrence_attribute_id', - 'calculate_estimates', - 'check_for_missing', - 'season_limits', - 'deleted' - ); - - return parent::validate($array, $save); - } - -} +'user', 'updated_by'=>'user'); + protected $has_and_belongs_to_many = array(); + + public function validate(Validation $array, $save = FALSE) { + // uses PHP trim() to remove whitespace from beginning and end of all fields before validation + $array->pre_filter('trim'); + $array->add_rules('survey_id', 'required'); + $array->add_rules('period_type', 'required', 'chars[W,M]'); + $array->add_rules('period_start', 'required'); + $array->add_rules('period_one_contains', 'required'); + $array->add_rules('data_combination_method', 'required', 'chars[A,L,M,S]'); + $array->add_rules('data_rounding_method', 'required', 'chars[D,N,U,X]'); + $array->add_rules('interpolation', 'required', 'chars[L]'); + $array->add_rules('first_value', 'required', 'chars[X,H]'); + $array->add_rules('last_value', 'required', 'chars[X,H]'); + + // Explicitly add those fields for which we don't do validation + $this->unvalidatedFields = array( + 'occurrence_attribute_id', + 'calculate_estimates', + 'season_limits', + 'deleted' + ); + + return parent::validate($array, $save); + } + +} diff --git a/modules/summary_builder/plugins/summary_builder.php b/modules/summary_builder/plugins/summary_builder.php index abee1d5aca..149f0cd9eb 100644 --- a/modules/summary_builder/plugins/summary_builder.php +++ b/modules/summary_builder/plugins/summary_builder.php @@ -1,93 +1,93 @@ -
    '; - echo '
  • help : '.(!isset($_GET['help']) ? 'absent' : 'present').'
    • '; - echo '
  • verbose : present
    • '; - echo '
  • force_summary_truncate : '.(!isset($_GET['force_summary_truncate']) ? 'absent' : 'present').'
    • '; - echo '
  • force_summary_rebuild : '.($rebuild === false ? 'absent' : ($rebuild === true ? 'present (rebuilds all surveys)' : $rebuild.' (Survey ID)')).'
    • '; - echo '
  • force_summary_clear : '.($clear === false ? 'absent or survey not specified' : $clear.' (Survey ID)').'
    • '; - if($clear !== false) echo '
  • location_id : '.($location === false ? 'absent' : $location.' (Location ID)').'
    • '; - if($clear !== false) echo '
  • taxa_taxon_list_id : '.($taxa_taxon_list_id === false ? 'absent' : $taxa_taxon_list_id.' (Taxon ID)').'
    • '; - echo '
  • force_summary_missing_check : '.($missing_check === false ? 'absent (missing checks as defined in summary definition for individual survey)' : ($missing_check === true ? 'present (missing checks forced on all surveys)' : $missing_check.' (Survey ID, missing checks forced on this survey, for all other surveys missing checks as defined in summary definition)')).'
    • '; - echo '
  • only : '.($only === false ? 'absent' : $only).'

'; - } - - if(isset($_GET['help'])) { - echo 'Summary Builder module task help:
Optional URL Parameters
    '; - echo '
  • &help : displays this message detailing the available URL parameters when running the Summary Builder module scheduled task. No other processing takes place.
    '; - echo '
  • &verbose : if present, increases the amount of messages displayed, e.g. include metrics on number of records processed.
    • '; - echo '
  • &force_summary_truncate : if present will truncate the summary entry table. This may have performance advantages over force_summary_clear when removing the entire data set. No other processing will take place on this invocation.
    • '; - echo '
  • &force_summary_rebuild[=<n>] : if present will change the creation date of all the summary entries for either the specified survey ID <n> (if given), or all surveys (if no parameter value given), to the day before the creation date of the first sample on the survey. Using this to rebuild the data leaves the data present whilst the rebuild takes place. No other processing will take place on this invocation for the affected survey(s).
    • '; - echo '
  • &force_summary_clear=<n>[&location_id=<x>|&taxa_taxon_list_id=<y>]] : if present will remove all the summary entries for the specified survey ID <n>, optionally restricting the removal to the data for either a location <x> or a taxon <y>. No other processing will take place on this invocation for the affected survey.
    • '; - echo '
  • &force_summary_missing_check[=<n>] : Carries out extra checks to see if any data has been missed for either the specified survey ID <n> (if given), or all surveys (if no parameter value given). If this option is not provided, or a particular survey is not specified, then whether the missing checks are carried out is determined by the setting in the definition. Normally checks are restricted to those samples created/deleted or occurrences created/updated/deleted since the last run of this scheduled task. With this option, the checks are extended to include any deleted locations with data in the cache, and any samples created/deleted plus any occurrences created/updated/deleted after the relevant summary record was created. These checks have a greater performance hit than the normal checks.
    • '; - echo '
  • &only=[locations|samples|occurrences|taxa] : There are 4 distinct stages to the checks carried out (both normal and missing): this parameter restricts the processing to one of the four stages. This may be especially useful in catch up (missing check) mode, where performance may be marginal.
'; - echo 'The force_summary_truncate, force_summary_rebuild and force_summary_clear parameters all then allow the appropriate data to be rebuilt using the missing summary checks on subsequent calls to this task.'; - return; - } - - if ($last_run_date===null) - // first run, so get all records changed in last day. Missing_check query will automatically gradually pick up the rest. - $last_run_date=date('Y-m-d', time()-60*60*24); - try { - if(isset($_GET['force_summary_truncate'])) - summary_builder::force_summary_truncate($db, $last_run_date, $verbose); - else - // unlike cache builder, summary has a single table. - summary_builder::populate_summary_table($db, $last_run_date, $verbose, $rebuild, $clear, $missing_check); - } catch (Exception $e) { - echo $e->getMessage(); - } -} - -function summary_builder_alter_menu($menu, $auth) { - if ($auth->logged_in('CoreAdmin') || $auth->has_any_website_access('admin')) - $menu['Admin']['Summariser']='summariser_definition'; - return $menu; -} - -function summary_builder_extend_data_services() { - return array( - 'summariser_definitions'=>array(), - 'summary_occurrences'=>array() - ); -} \ No newline at end of file + 'sample', + 'ops' => ['insert', 'delete'], + 'task' => 'task_summary_builder_sample', + 'cost_estimate' => 10, + 'priority' => 2, + ], + [ + /* Insert or delete of an occurrence only affects the records for that occurrence */ + 'entity' => 'occurrence', + 'ops' => ['insert', 'delete'], + 'task' => 'task_summary_builder_occurrence_insert_delete', + 'cost_estimate' => 10, + 'priority' => 2, + ], + [ + /* On the other hand, a modification may include a change of taxon, so must recalculate all existing records */ + 'entity' => 'occurrence', + 'ops' => ['update'], + 'task' => 'task_summary_builder_occurrence_update', + 'cost_estimate' => 10, + 'priority' => 2, + ], + [ + // only location information in the summary_occurrences data is id: only need to worry about delete + 'entity' => 'location', + 'ops' => ['delete'], + 'task' => 'task_summary_builder_location_delete', + 'cost_estimate' => 10, + 'priority' => 2, + ], +/* + [ + // A taxon change needs to change the cached taxon details. + 'entity' => 'taxon', + 'ops' => ['update'], + 'task' => 'task_summary_builder_location_taxon', + 'cost_estimate' => 10, + 'priority' => 2, + ], +*/ + ]; +} + +// summary_builder is no longer a scheduled task. +function summary_builder_alter_menu($menu, $auth) { + if ($auth->logged_in('CoreAdmin') || $auth->has_any_website_access('admin')) + $menu['Admin']['Summariser']='summariser_definition'; + return $menu; +} + +function summary_builder_extend_data_services() { + return array( + 'summariser_definitions'=>array(), + 'summary_occurrences'=>array() + ); +} diff --git a/modules/summary_builder/views/summariser_definition/index.php b/modules/summary_builder/views/summariser_definition/index.php index f57294146d..2f0b4b7ad2 100644 --- a/modules/summary_builder/views/summariser_definition/index.php +++ b/modules/summary_builder/views/summariser_definition/index.php @@ -1,30 +1,34 @@ -db->select('*')->from('system')->where('name','summary_builder')->get()->as_array(true); -foreach($systemTableEntries as $systemTableEntry) { - echo 'Summary Builder module version : '.$systemTableEntry->version.'
Last scheduled tasks ran : '.$systemTableEntry->last_scheduled_task_check.'ID '.$systemTableEntry->id.", last script : ".$systemTableEntry->last_run_script."
"; -} + +
+ +
+db->select('*')->from('system')->where('name','summary_builder')->get()->as_array(true); +foreach($systemTableEntries as $systemTableEntry) { + echo 'Summary Builder module version : '.$systemTableEntry->version.'
Last scheduled tasks ran : '.$systemTableEntry->last_scheduled_task_check.'ID '.$systemTableEntry->id.", last script : ".$systemTableEntry->last_run_script."
"; +} ?> \ No newline at end of file diff --git a/modules/summary_builder/views/summariser_definition/summariser_definition_edit.php b/modules/summary_builder/views/summariser_definition/summariser_definition_edit.php index dce37e86b5..be7317d909 100644 --- a/modules/summary_builder/views/summariser_definition/summariser_definition_edit.php +++ b/modules/summary_builder/views/summariser_definition/summariser_definition_edit.php @@ -1,166 +1,177 @@ -$this->model->getAllErrors())); -?> -
-
-Summariser Definition details -'summariser_definition:id', - 'default'=>html::initial_value($values, 'summariser_definition:id') - )); -else : ?> -

New Record

-'summariser_definition:survey_id', - 'default'=>html::initial_value($values, 'summariser_definition:survey_id') -)); -echo data_entry_helper::text_input(array( - 'label'=>'Survey Title', - 'fieldname'=>'survey:title', - 'default'=>html::initial_value($other_data, 'survey_title'), - 'disabled' => 'disabled' -)); -echo data_entry_helper::checkbox(array( - 'label'=>'Check for missing', - 'fieldname'=>'summariser_definition:check_for_missing', - 'default'=>html::initial_value($values, 'summariser_definition:check_for_missing'), - 'helpText' => 'Enable checks for missed data - this is data entered before the last scheduled task run, but which is not yet represented in the summary table. This check has a performance impact. Needs to be selected during any initial catch up period.', -)); -echo data_entry_helper::text_input(array( - 'label' => 'Max Number of Records', - 'fieldname' => 'summariser_definition:max_records_per_cycle', - 'default' => html::initial_value($values, 'summariser_definition:max_records_per_cycle'), - 'helpText' => 'The maximum number of occurrence records processed for this survey per invocation of the scheduled task.', - 'validation' => array('required','integer','minimum[1]') -)); -?> -

Only one period option (weekly) available at the moment.

-'Summarisation Period', - 'fieldname'=>'summariser_definition:period_type', - 'default'=>'W' // html::initial_value($values, 'summariser_definition:period_type') -)); -echo data_entry_helper::text_input(array( - 'label'=>'Period Start', - 'fieldname'=>'summariser_definition:period_start', - 'default'=>html::initial_value($values, 'summariser_definition:period_start'), - 'helpText' => 'Define the first day of each period. There are 2 options.
'. - "  weekday=<n> where <n> is a number between 1 (for Monday) and 7 (for Sunday).
". - "  date=MMM/DD where MMM/DD is a month/day combination: e.g. choosing Apr-1 will start each week on the day of the week on which the 1st of April occurs.
", - 'validation'=>'required' -)); -echo data_entry_helper::text_input(array( - 'label'=>'Period One Contains', - 'fieldname'=>'summariser_definition:period_one_contains', - 'default'=>html::initial_value($values, 'summariser_definition:period_one_contains'), - 'helpText' => 'Calculate week one as the week containing this date: value should be in the format MMM/DD, which is a month/day combination: e.g. choosing Apr-1 will mean week one contains the date of the 1st of April. Default is the Jan-01', - 'validation'=>'required' -)); -echo data_entry_helper::select(array( - 'label'=>'Attribute to Sum', - 'fieldname'=>'summariser_definition:occurrence_attribute_id', - 'lookupValues' => $other_data['occAttrs'], - 'default'=>html::initial_value($values, 'summariser_definition:occurrence_attribute_id'), - 'helpText' => 'The occurrence attribute which is used as the count associated with the occurrence. If not provided then each occurrence has a count of one.' -)); -echo data_entry_helper::checkbox(array( - 'label'=>'Calculate Estimates', - 'fieldname'=>'summariser_definition:calculate_estimates', - 'default'=>html::initial_value($values, 'summariser_definition:calculate_estimates') -)); -?> -
Data Handling -'Summary Data Combination method', - 'fieldname'=>'summariser_definition:data_combination_method', - 'lookupValues' => array('A'=>'Add all occurrences together', - 'M'=>'Choose the value from the sample with the greatest count', - 'L'=>'Average over all samples for that location during that period'), - 'default'=>html::initial_value($values, 'summariser_definition:data_combination_method'), - 'helpText' => 'When data is aggregated for a location/period combination, this determines how.' -)); -echo data_entry_helper::select(array( - 'label'=>'Data Rounding', - 'fieldname'=>'summariser_definition:data_rounding_method', - 'lookupValues' => array('N'=>'To the nearest integer, .5 rounds up', - 'U'=>'Up: To the integer greater than or equal to the value', - 'D'=>'Down: To the integer less than or equal to the value', - 'X'=>'None (may result in non-integer values)'), - 'default'=>html::initial_value($values, 'summariser_definition:data_rounding_method'), - 'helpText' => 'When data is averaged, this determines what rounding is carried out. Note that for summary data (not estimates) anything between 0 and 1 will be rounded up to 1.' -)); -?> -
Estimate Generation -

Only one interpolation option (linear) available at the moment.

-'Interpolation method', - 'fieldname'=>'summariser_definition:interpolation', - 'default'=>'L' // html::initial_value($values, 'summariser_definition:interpolation') -)); -echo data_entry_helper::text_input(array( - 'label'=>'Season Limits', - 'fieldname'=>'summariser_definition:season_limits', - 'default'=>html::initial_value($values, 'summariser_definition:season_limits'), - 'helpText' => 'This is a comma separated pair of the week numbers for the start and end of the season. When provided, and data is not entered for these weeks, the value is taken as zero, irrespective of the First/Last value processing. First/Last value processing is not carried out outwith these weeks.' -)); - -echo data_entry_helper::select(array( - 'label'=>'First Value Processing', - 'fieldname'=>'summariser_definition:first_value', - 'lookupValues' => array('X'=>'No special processing', 'H'=>'The entry for the previous week is half the entered value'), - 'default'=>html::initial_value($values, 'summariser_definition:first_value'), - 'helpText' => 'When encountering the first entered value, this determines what happens.' -)); -echo data_entry_helper::select(array( - 'label'=>'Last Value Processing', - 'fieldname'=>'summariser_definition:last_value', - 'lookupValues' => array('X'=>'No special processing', 'H'=>'The entry for the next week is half the entered value'), - 'default'=>html::initial_value($values, 'summariser_definition:last_value'), - 'helpText' => 'When encountering the last entered value, this determines what happens.' -)); -?> -
- -
-
\ No newline at end of file +$this->model->getAllErrors())); +} + +?> +
+
+Summariser Definition details +'summariser_definition:id', + 'default'=>html::initial_value($values, 'summariser_definition:id') + )); + + echo data_entry_helper::hidden_text(array( + 'fieldname'=>'summariser_definition:survey_id', + 'default'=>html::initial_value($values, 'summariser_definition:survey_id') + )); + + echo data_entry_helper::text_input(array( + 'label'=>'Survey Title', + 'fieldname'=>'survey:title', + 'default'=>html::initial_value($other_data, 'survey_title'), + 'disabled' => 'disabled' + )); + +} else { + + echo data_entry_helper::select(array( + 'label'=>'Survey', + 'fieldname'=>'summariser_definition:survey_id', + 'lookupValues' => $other_data['surveys'], + 'validation'=>'required' + )); + +} + +echo "
Period"; + +echo "

Only one period option (weekly) available at the moment.

"; +echo data_entry_helper::hidden_text(array( + // 'caption'=>'Summarisation Period', + 'fieldname'=>'summariser_definition:period_type', + 'default'=>'W' // html::initial_value($values, 'summariser_definition:period_type') +)); + +echo data_entry_helper::text_input(array( + 'label'=>'Period Start', + 'fieldname'=>'summariser_definition:period_start', + 'default'=>html::initial_value($values, 'summariser_definition:period_start'), + 'helpText' => 'Define the first day of each period. There are 2 options.
'. + "  weekday=<n> where <n> is a number between 1 (for Monday) and 7 (for Sunday).
". + "  date=MMM/DD where MMM/DD is a month/day combination: e.g. choosing Apr-1 will start each week on the day of the week on which the 1st of April occurs.
", + 'validation'=>'required' +)); + +echo data_entry_helper::text_input(array( + 'label'=>'Period One Contains', + 'fieldname'=>'summariser_definition:period_one_contains', + 'default'=>html::initial_value($values, 'summariser_definition:period_one_contains'), + 'helpText' => 'Calculate week one as the week containing this date: value should be in the format MMM/DD, which is a month/day combination: e.g. choosing Apr-1 will mean week one contains the date of the 1st of April. Default is the Jan-01', + 'validation'=>'required' +)); + +echo "
Data Handling"; + +echo data_entry_helper::select(array( + 'label'=>'Attribute to Sum', + 'fieldname'=>'summariser_definition:occurrence_attribute_id', + 'lookupValues' => $other_data['occAttrs'], + 'default'=>html::initial_value($values, 'summariser_definition:occurrence_attribute_id'), + 'helpText' => 'The occurrence attribute which is used as the count associated with the occurrence. If not provided then each occurrence has a count of one.' +)); + +echo data_entry_helper::select(array( + 'label'=>'Summary Data Combination method', + 'fieldname'=>'summariser_definition:data_combination_method', + 'lookupValues' => array('A'=>'Add all occurrences together', + 'M'=>'Choose the value from the sample with the greatest count', + 'L'=>'Average over all samples for that location during that period'), + 'default'=>html::initial_value($values, 'summariser_definition:data_combination_method'), + 'helpText' => 'When data is aggregated for a location/period combination, this determines how.' +)); + +echo data_entry_helper::select(array( + 'label'=>'Data Rounding', + 'fieldname'=>'summariser_definition:data_rounding_method', + 'lookupValues' => array('N'=>'To the nearest integer, .5 rounds up', + 'U'=>'Up: To the integer greater than or equal to the value', + 'D'=>'Down: To the integer less than or equal to the value', + 'X'=>'None (may result in non-integer values)'), + 'default'=>html::initial_value($values, 'summariser_definition:data_rounding_method'), + 'helpText' => 'When data is averaged, this determines what rounding is carried out. Note that for summary data (not estimates) anything between 0 and 1 will be rounded up to 1.' +)); + +echo "
Estimate Generation"; + +echo data_entry_helper::checkbox(array( + 'label'=>'Calculate Estimates', + 'fieldname'=>'summariser_definition:calculate_estimates', + 'default'=>html::initial_value($values, 'summariser_definition:calculate_estimates') +)); + +// Only one interpolation option at the moment. This may change in future. Keep hidden control until that point. +echo "

Only one interpolation option (linear) available at the moment.

"; +// 'L' = 'Linear interpolation' +echo data_entry_helper::hidden_text(array( + // 'caption'=>'Interpolation method', + 'fieldname'=>'summariser_definition:interpolation', + 'default'=>'L' // html::initial_value($values, 'summariser_definition:interpolation') +)); + +echo data_entry_helper::text_input(array( + 'label'=>'Season Limits', + 'fieldname'=>'summariser_definition:season_limits', + 'default'=>html::initial_value($values, 'summariser_definition:season_limits'), + 'helpText' => 'This is a comma separated pair of the week numbers for the start and end of the season. When provided, and data is not entered for these weeks, the value is taken as zero, irrespective of the First/Last value processing. First/Last value processing is not carried out outwith these weeks.' +)); + +echo data_entry_helper::select(array( + 'label'=>'First Value Processing', + 'fieldname'=>'summariser_definition:first_value', + 'lookupValues' => array('X'=>'No special processing', 'H'=>'The entry for the previous week is half the entered value'), + 'default'=>html::initial_value($values, 'summariser_definition:first_value'), + 'helpText' => 'When encountering the first entered value, this determines what happens.' +)); + +echo data_entry_helper::select(array( + 'label'=>'Last Value Processing', + 'fieldname'=>'summariser_definition:last_value', + 'lookupValues' => array('X'=>'No special processing', 'H'=>'The entry for the next week is half the entered value'), + 'default'=>html::initial_value($values, 'summariser_definition:last_value'), + 'helpText' => 'When encountering the last entered value, this determines what happens.' +)); + +echo "
"; + +echo $metadata; + +echo html::form_buttons($existing, false, false); + +data_entry_helper::$dumped_resources[] = 'jquery'; +data_entry_helper::$dumped_resources[] = 'jquery_ui'; +data_entry_helper::$dumped_resources[] = 'fancybox'; +echo data_entry_helper::dump_javascript(); + +echo "
"; From 7447fc71e7bc44da5b78dd4b7e1009b9ac11e814 Mon Sep 17 00:00:00 2001 From: Gary van Breda Date: Sun, 25 Aug 2019 23:26:42 +0100 Subject: [PATCH 2/7] Changes to new summary builder to support the annual summary page --- .../db/version_2_17_0/201903091200_tables.sql | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/modules/summary_builder/db/version_2_17_0/201903091200_tables.sql b/modules/summary_builder/db/version_2_17_0/201903091200_tables.sql index acddbe8e5d..cd0447a0ec 100644 --- a/modules/summary_builder/db/version_2_17_0/201903091200_tables.sql +++ b/modules/summary_builder/db/version_2_17_0/201903091200_tables.sql @@ -1,6 +1,7 @@  DROP VIEW IF EXISTS gv_summariser_definitions; +DROP VIEW IF EXISTS list_summariser_definitions; ALTER TABLE summariser_definitions DROP COLUMN check_for_missing, @@ -13,6 +14,12 @@ CREATE VIEW gv_summariser_definitions AS LEFT JOIN websites w ON s.website_id = w.id AND w.deleted = FALSE WHERE sd.survey_id = s.id AND sd.deleted = false; +CREATE VIEW list_summariser_definitions AS + SELECT s.website_id, sd.id, sd.survey_id, sd.period_type, sd.period_start, sd.period_one_contains, sd.calculate_estimates + FROM summariser_definitions sd + JOIN surveys s ON sd.survey_id = s.id AND s.deleted = FALSE + JOIN websites w ON s.website_id = w.id AND w.deleted = FALSE + WHERE sd.survey_id = s.id AND sd.deleted = false; DROP VIEW IF EXISTS list_summary_occurrences; @@ -48,7 +55,7 @@ CREATE OR REPLACE VIEW list_summary_occurrences AS year, location_id, user_id, type, taxa_taxon_list_id, preferred_taxa_taxon_list_id, taxonomic_sort_order, taxon, preferred_taxon, default_common_name, taxon_meaning_id, taxon_list_id, - created_by_id, summary_created_on + created_by_id, summary_created_on, summarised_data FROM summary_occurrences; COMMENT ON TABLE summary_occurrences IS 'Summary of occurrence data used for reporting.'; From 5dcbec8d39343e3c59d3278dd69e8ecf54bb8c6f Mon Sep 17 00:00:00 2001 From: Gary van Breda Date: Sun, 25 Aug 2019 23:40:19 +0100 Subject: [PATCH 3/7] Rename summary builder db update directory 2_17 to 2_29 --- .../db/{version_2_17_0 => version_2_29_0}/201903091200_tables.sql | 0 .../201903091201_work_queue.sql | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename modules/summary_builder/db/{version_2_17_0 => version_2_29_0}/201903091200_tables.sql (100%) rename modules/summary_builder/db/{version_2_17_0 => version_2_29_0}/201903091201_work_queue.sql (100%) diff --git a/modules/summary_builder/db/version_2_17_0/201903091200_tables.sql b/modules/summary_builder/db/version_2_29_0/201903091200_tables.sql similarity index 100% rename from modules/summary_builder/db/version_2_17_0/201903091200_tables.sql rename to modules/summary_builder/db/version_2_29_0/201903091200_tables.sql diff --git a/modules/summary_builder/db/version_2_17_0/201903091201_work_queue.sql b/modules/summary_builder/db/version_2_29_0/201903091201_work_queue.sql similarity index 100% rename from modules/summary_builder/db/version_2_17_0/201903091201_work_queue.sql rename to modules/summary_builder/db/version_2_29_0/201903091201_work_queue.sql From d4b78da017365239738e382d9135b213e9fc4617 Mon Sep 17 00:00:00 2001 From: Gary van Breda Date: Mon, 26 Aug 2019 21:54:26 +0100 Subject: [PATCH 4/7] Update to priorities and cost estimates for summary builder jobs --- .../db/version_2_29_0/201903091201_work_queue.sql | 2 +- modules/summary_builder/plugins/summary_builder.php | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/summary_builder/db/version_2_29_0/201903091201_work_queue.sql b/modules/summary_builder/db/version_2_29_0/201903091201_work_queue.sql index 99a655d045..19fa4808a9 100644 --- a/modules/summary_builder/db/version_2_29_0/201903091201_work_queue.sql +++ b/modules/summary_builder/db/version_2_29_0/201903091201_work_queue.sql @@ -5,7 +5,7 @@ DELETE FROM work_queue WHERE task='task_summary_builder_sample'; INSERT INTO work_queue (task, entity, record_id, priority, cost_estimate, created_on) -SELECT DISTINCT 'task_summary_builder_sample', 'sample', id, 3, 90, now() +SELECT DISTINCT 'task_summary_builder_sample', 'sample', id, 2, 50, now() FROM samples WHERE deleted=false AND parent_id IS NULL diff --git a/modules/summary_builder/plugins/summary_builder.php b/modules/summary_builder/plugins/summary_builder.php index 149f0cd9eb..56d5a220f2 100644 --- a/modules/summary_builder/plugins/summary_builder.php +++ b/modules/summary_builder/plugins/summary_builder.php @@ -38,7 +38,7 @@ function summary_builder_orm_work_queue() { 'entity' => 'sample', 'ops' => ['insert', 'delete'], 'task' => 'task_summary_builder_sample', - 'cost_estimate' => 10, + 'cost_estimate' => 50, 'priority' => 2, ], [ @@ -46,7 +46,7 @@ function summary_builder_orm_work_queue() { 'entity' => 'occurrence', 'ops' => ['insert', 'delete'], 'task' => 'task_summary_builder_occurrence_insert_delete', - 'cost_estimate' => 10, + 'cost_estimate' => 40, 'priority' => 2, ], [ @@ -54,7 +54,7 @@ function summary_builder_orm_work_queue() { 'entity' => 'occurrence', 'ops' => ['update'], 'task' => 'task_summary_builder_occurrence_update', - 'cost_estimate' => 10, + 'cost_estimate' => 50, 'priority' => 2, ], [ @@ -62,8 +62,8 @@ function summary_builder_orm_work_queue() { 'entity' => 'location', 'ops' => ['delete'], 'task' => 'task_summary_builder_location_delete', - 'cost_estimate' => 10, - 'priority' => 2, + 'cost_estimate' => 40, + 'priority' => 3, ], /* [ From f891a3a60d9dcece97b0d4cb6cb07423408a665b Mon Sep 17 00:00:00 2001 From: Gary van Breda Date: Mon, 26 Aug 2019 22:01:09 +0100 Subject: [PATCH 5/7] Alter initial population so does most recent samples first --- .../db/version_2_29_0/201903091201_work_queue.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/summary_builder/db/version_2_29_0/201903091201_work_queue.sql b/modules/summary_builder/db/version_2_29_0/201903091201_work_queue.sql index 19fa4808a9..b6d09b020f 100644 --- a/modules/summary_builder/db/version_2_29_0/201903091201_work_queue.sql +++ b/modules/summary_builder/db/version_2_29_0/201903091201_work_queue.sql @@ -9,4 +9,5 @@ SELECT DISTINCT 'task_summary_builder_sample', 'sample', id, 2, 50, now() FROM samples WHERE deleted=false AND parent_id IS NULL -AND survey_id IN (SELECT survey_id FROM summariser_definitions WHERE deleted = false); \ No newline at end of file +AND survey_id IN (SELECT survey_id FROM summariser_definitions WHERE deleted = false) +ORDER BY date desc; \ No newline at end of file From e0ab1993a2948e665386ff38f3d3ee12b26c756c Mon Sep 17 00:00:00 2001 From: John van Breda Date: Fri, 30 Aug 2019 13:50:52 +0100 Subject: [PATCH 6/7] Update change log --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 69db12c906..de54d38937 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +# Version 2.31.0 +*2019-08-29* + +* Refactor of the Summary Builder module to use the work_queue for greater efficiency. + # Version 2.30.0 *2019-08-28* From 5148f0498b4411e658ac3df9a72aead73fa0e75d Mon Sep 17 00:00:00 2001 From: John van Breda Date: Fri, 30 Aug 2019 13:52:02 +0100 Subject: [PATCH 7/7] Version bump --- application/config/version.php | 4 ++-- client_helpers | 2 +- media | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/application/config/version.php b/application/config/version.php index e54af4dca2..883b92f390 100644 --- a/application/config/version.php +++ b/application/config/version.php @@ -29,14 +29,14 @@ * * @var string */ -$config['version'] = '2.30.0'; +$config['version'] = '2.31.0'; /** * Version release date. * * @var string */ -$config['release_date'] = '2019-08-28'; +$config['release_date'] = '2019-08-29'; /** * Link to the code repository downloads page. diff --git a/client_helpers b/client_helpers index 00d3d9a3cb..1fe3f04367 160000 --- a/client_helpers +++ b/client_helpers @@ -1 +1 @@ -Subproject commit 00d3d9a3cb1460c0f52e588304dfc4ed2d83e1bb +Subproject commit 1fe3f04367bb887aadcf902e41e0841c42cb95cf diff --git a/media b/media index 629695e344..50a18734b1 160000 --- a/media +++ b/media @@ -1 +1 @@ -Subproject commit 629695e3444972bef049319a90358fcc8e5147ee +Subproject commit 50a18734b11ae657639cb8a31b257e2a1b33f43b