Skip to content

Commit

Permalink
Merge branch 'develop'
Browse files Browse the repository at this point in the history
  • Loading branch information
@johnvanbreda
johnvanbreda committed Aug 30, 2019
2 parents e99a6e4 + 5148f04 commit 9f65a9c
Show file tree
Hide file tree
Showing 17 changed files with 1,550 additions and 1,408 deletions.
5 changes: 5 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
# Version 2.31.0
*2019-08-29*

* Refactor of the Summary Builder module to use the work_queue for greater efficiency.

# Version 2.30.0
*2019-08-28*

Expand Down
4 changes: 2 additions & 2 deletions application/config/version.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,14 @@
*
* @var string
*/
$config['version'] = '2.30.0';
$config['version'] = '2.31.0';

/**
* Version release date.
*
* @var string
*/
$config['release_date'] = '2019-08-28';
$config['release_date'] = '2019-08-29';

/**
* Link to the code repository downloads page.
Expand Down
13 changes: 13 additions & 0 deletions modules/summary_builder/README
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
Notes:
1) Branch/country level summarisation to be done in front end (by summing up the locations)
2) Row and column totals in each grid done by the front end

Proposed development path
1) Currently only week based summarisation, as that is what UKBMS want. Expand to allow Month based summarisation
2) Allow >1 summarisation per survey
3) Currently deals with super/subsample with locations, as that is what UKBMS want. Expand to handle other cases.
4) Amalgamation of raw data into a daily summary. Would need to store sample list.
5) Make 0-1 rounding configurable
6) Configurable end of year processing: Include data for overlap, include periods that overlap


548 changes: 223 additions & 325 deletions modules/summary_builder/config/summary_builder.php
View file

Large diffs are not rendered by default.

178 changes: 94 additions & 84 deletions modules/summary_builder/controllers/summariser_definition.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,34 +21,8 @@
* @link https://github.com/indicia-team/warehouse/
*/

/* Stage 2
* TODO: configurable merge taxa with same meaning.
* TODO: sample attribute averaging. store as json in a period level record.
* TODO: new download reports.
* TODO: Store Weeknumber definitions in a period level record
* TODO: comment functions
*/

/*
* Proposed development path
* 1) Currently only week based summarisation, as that is what UKBMS want. Expand to allow Month based summarisation
* 2) Allow >1 summarisation per survey
* 3) Currently deals with super/subsample with locations, as that is what UKBMS want. Expand to handle other cases.
* 4) Amalgamation of raw data into a daily summary. Would need to store sample list.
* 5) Make 0-1 rounding configurable
* 6) Configurable end of year processing: Include data for overlap, include periods that overlap
*/

/*
* Notes:
* 1) user level summarisation for taxon is done in back end
* 2) branch level summarisation to be done in front end (by summing up the locations)
* 3) year level and species level summarisation (row and column totals) done by the front end
* 4) links to be handled by front end.
*/

/**
* Controller providing CRUD access to the surveys list.
* Controller providing CRUD access to the summariser_definition list.
*
* @package Core
* @subpackage Controllers
Expand All @@ -59,98 +33,138 @@ public function __construct() {
parent::__construct('summariser_definition');
$this->columns = array(
'title' => 'Survey title',
'website' => ''
'website' => ''
);
$this->pagetitle = "Survey based data summariser definition";
}

protected function get_action_columns() {
return array(
array(
'caption'=>'Setup Summariser',
'url'=>'summariser_definition/edit_from_survey/{survey_id}'
)
);
return array(
array(
'caption' => 'Setup Summariser',
'url' => 'summariser_definition/edit/{id}',
)
);
}

public function edit_from_survey($id) {
if (!is_null($id) && !is_null($this->auth_filter) && !in_array($id, $this->auth_filter['values'])) {
$this->access_denied();
return;
}
$this->model = new Summariser_definition_Model(array('survey_id' => $id, 'deleted'=>'f'));
$values = $this->getModelValues();
if ( !$this->model->loaded ) {
$values['summariser_definition:survey_id']=$id;
$values['summariser_definition:period_start']='weekday=7';
$values['summariser_definition:period_one_contains']='Jan-01';
}
public function edit($id = NULL) {
if ( is_null($id) ) {
$this->access_denied();
return;
}
$this->model = new Summariser_definition_Model(array('id' => $id, 'deleted'=>'f'));
if ( !$this->model->loaded ) {
$this->access_denied();
return;
}
$survey = new Survey_Model(array('id' => $this->model->survey_id, 'deleted'=>'f'));
if ( !$this->model->loaded ) {
$this->access_denied();
return;
}
if ( !is_null($this->auth_filter ) && !in_array($survey->website_id, $this->auth_filter['values']) ) {
$this->access_denied();
return;
}

$values = $this->getModelValues();
$other = $this->prepareOtherViewData($values);
$this->setView($this->editViewName(), $this->model->caption(), array(
'existing'=>$this->model->loaded,
'values'=>$values
,'other_data'=>$other
$this->setView($this->editViewName(), $this->model->caption(), array(
'existing' => TRUE,
'values' => $values,
'other_data' => $other
));
$this->defineEditBreadcrumbs();
$this->defineEditBreadcrumbs();
}

public function edit($id){
Kohana::show_404();
}
public function create(){
Kohana::show_404();
$this->model = new Summariser_definition_Model();
$values = $this->getModelValues();
$values['summariser_definition:period_start']='weekday=7';
$values['summariser_definition:period_one_contains']='Jan-01';
$other = $this->prepareOtherViewData($values);
if(count($other['surveys'])<1) {
$this->access_denied();
return;
}
$this->setView($this->editViewName(), $this->model->caption(), array(
'existing'=>FALSE,
'values'=>$values,
'other_data'=>$other
));
$this->defineEditBreadcrumbs();
}

protected function show_submit_fail()
{
$page_errors=$this->model->getPageErrors();
if (count($page_errors)!=0) {
$this->session->set_flash('flash_error', implode('<br/>',$page_errors));
} else {
$this->session->set_flash('flash_error', 'The record could not be saved.');
}
$values = $this->getDefaults();
$values = array_merge($values, $_POST);
$other = $this->prepareOtherViewData($values);
$this->setView($this->editViewName(), $this->model->caption(), array(
'existing'=>$this->model->loaded,
if (count($page_errors)!=0) {
$this->session->set_flash('flash_error', implode('<br/>',$page_errors));
} else {
$this->session->set_flash('flash_error', 'The record could not be saved.');
}
$values = $this->getDefaults();
$values = array_merge($values, $_POST);
$other = $this->prepareOtherViewData($values);
$this->setView($this->editViewName(), $this->model->caption(), array(
'existing'=>$this->model->loaded,
'values'=>$values
,'other_data'=>$other
));
$this->defineEditBreadcrumbs();
}

protected function prepareOtherViewData(array $values) {
$survey = new Survey_Model($values['summariser_definition:survey_id']);
$attrsRet = array(''=>'(Each occurrence has count=1)');
$survey = new Survey_Model($values['summariser_definition:survey_id']);
$attrsRet = array(''=>'(Each occurrence has count=1)');

$models = ORM::factory('occurrence_attributes_website')->
where(array('restrict_to_survey_id'=>$values['summariser_definition:survey_id'],'deleted'=>'f'))->
find_all();
where(array('deleted'=>'f'));
if(!empty($values['summariser_definition:survey_id'])) {
$models = $models->where(array('restrict_to_survey_id'=>$values['summariser_definition:survey_id']));
}
$models = $models->find_all();
if(count($models)>0){
$attrIds = array();
foreach ($models as $model)
$attrIds[] = $model->occurrence_attribute_id;
$attrs = ORM::factory('occurrence_attribute')->
where('deleted','f')->in('data_type',array('I','F'))->in('id',$attrIds)->
orderby('caption')->find_all();
if(count($attrs)>0)
foreach ($attrs as $attr)
$attrsRet[$attr->id] = $attr->caption.' (ID '.$attr->id.')';
$attrIds = array();
foreach ($models as $model)
$attrIds[] = $model->occurrence_attribute_id;
$attrIds = array_unique($attrIds);
$attrs = ORM::factory('occurrence_attribute')->
where('deleted','f')->in('data_type',array('I','F'))->in('id',$attrIds)->
orderby('caption')->find_all();
if(count($attrs)>0)
foreach ($attrs as $attr)
$attrsRet[$attr->id] = $attr->caption.' (ID '.$attr->id.')';
}

$surveys = array();
$surveyModels = ORM::factory('survey')->where(array('deleted'=>'f'))->find_all();
foreach ($surveyModels as $model) {
if (is_null($this->auth_filter) || in_array($model->website_id, $this->auth_filter['values'])) {
$surveys["".$model->id] = $model->title;
}
}
$existingModels = ORM::factory('summariser_definition')->where(array('deleted'=>'f'))->find_all();
foreach ($existingModels as $model) {
if(array_key_exists("".$model->id, $surveys)) {
unset($surveys["".$model->survey_id]);
}
}
asort($surveys);
return array(
'survey_title' => $survey->title,
'occAttrs' => $attrsRet
'occAttrs' => $attrsRet,
'surveys' => $surveys,
);
}

/**
* Check access to a survey when editing. The survey's website must be in the list
* of websites the user is authorised to administer.
*/
protected function record_authorised ($id)
protected function record_authorised ($id = NULL)
{
$model = new Summariser_definition_Model($id);
if (!is_null($id) AND !is_null($this->auth_filter))
{
$survey = new Survey_Model($id);
Expand All @@ -167,7 +181,3 @@ protected function page_authorised() {
}

}
/*
* This is triggered by changes to occurrence records, not by a report.
* How this is output is determined by the front end.
*/
65 changes: 65 additions & 0 deletions modules/summary_builder/db/version_2_29_0/201903091200_tables.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@


DROP VIEW IF EXISTS gv_summariser_definitions;
DROP VIEW IF EXISTS list_summariser_definitions;

ALTER TABLE summariser_definitions
DROP COLUMN check_for_missing,
DROP COLUMN max_records_per_cycle;

CREATE VIEW gv_summariser_definitions AS
SELECT s.id as survey_id, s.title, w.title AS website, s.website_id, sd.id
FROM summariser_definitions sd
LEFT JOIN surveys s ON sd.survey_id = s.id AND s.deleted = FALSE
LEFT JOIN websites w ON s.website_id = w.id AND w.deleted = FALSE
WHERE sd.survey_id = s.id AND sd.deleted = false;

CREATE VIEW list_summariser_definitions AS
SELECT s.website_id, sd.id, sd.survey_id, sd.period_type, sd.period_start, sd.period_one_contains, sd.calculate_estimates
FROM summariser_definitions sd
JOIN surveys s ON sd.survey_id = s.id AND s.deleted = FALSE
JOIN websites w ON s.website_id = w.id AND w.deleted = FALSE
WHERE sd.survey_id = s.id AND sd.deleted = false;

DROP VIEW IF EXISTS list_summary_occurrences;

DROP TABLE summary_occurrences;

CREATE TABLE summary_occurrences
(
website_id INTEGER NOT NULL,
survey_id INTEGER NOT NULL,
location_id INTEGER NOT NULL,
user_id INTEGER NOT NULL,
year INTEGER NOT NULL,
type CHARACTER VARYING NOT NULL,
taxon_list_id INTEGER NOT NULL,
taxa_taxon_list_id INTEGER NOT NULL,
preferred_taxa_taxon_list_id INTEGER NOT NULL,
taxonomic_sort_order bigint,
taxon CHARACTER VARYING,
preferred_taxon CHARACTER VARYING,
default_common_name CHARACTER VARYING,
taxon_meaning_id INTEGER NOT NULL,
summarised_data JSON,
created_by_id INTEGER NOT NULL,
summary_created_on timestamp without time zone NOT NULL
) WITH (
OIDS --- want oids as no pk.
);
CREATE INDEX ix_summary_occurrences_STLU ON summary_occurrences USING btree (survey_id, year, taxa_taxon_list_id, location_id, user_id);
CREATE INDEX ix_summary_occurrences_STU ON summary_occurrences USING btree (survey_id, taxa_taxon_list_id, user_id);

CREATE OR REPLACE VIEW list_summary_occurrences AS
SELECT website_id, survey_id,
year, location_id, user_id, type,
taxa_taxon_list_id, preferred_taxa_taxon_list_id, taxonomic_sort_order,
taxon, preferred_taxon, default_common_name, taxon_meaning_id, taxon_list_id,
created_by_id, summary_created_on, summarised_data
FROM summary_occurrences;

COMMENT ON TABLE summary_occurrences IS 'Summary of occurrence data used for reporting.';
COMMENT ON COLUMN summary_occurrences.location_id IS 'Summary data location id, zero if for all locations.';
COMMENT ON COLUMN summary_occurrences.user_id IS 'ID of user who created the data summarised in this record, zero if for all users.';
COMMENT ON COLUMN summary_occurrences.type IS 'summariser_definition period type.';

13 changes: 13 additions & 0 deletions modules/summary_builder/db/version_2_29_0/201903091201_work_queue.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
-- #slow script#

-- trigger a rebuild of the entire data table, using the the work queue.

DELETE FROM work_queue WHERE task='task_summary_builder_sample';

INSERT INTO work_queue (task, entity, record_id, priority, cost_estimate, created_on)
SELECT DISTINCT 'task_summary_builder_sample', 'sample', id, 2, 50, now()
FROM samples
WHERE deleted=false
AND parent_id IS NULL
AND survey_id IN (SELECT survey_id FROM summariser_definitions WHERE deleted = false)
ORDER BY date desc;
Loading

0 comments on commit 9f65a9c

Please sign in to comment.