diff --git a/application/config/version.php b/application/config/version.php index 3421d336c5..ae90e8960a 100644 --- a/application/config/version.php +++ b/application/config/version.php @@ -29,14 +29,14 @@ * * @var string */ -$config['version'] = '2.37.2'; +$config['version'] = '3.0.0'; /** * Version release date. * * @var string */ -$config['release_date'] = '2019-11-21'; +$config['release_date'] = '2019-12-03'; /** * Link to the code repository downloads page. diff --git a/application/controllers/attribute_by_survey.php b/application/controllers/attribute_by_survey.php index 0d91569dff..a43f6d18c7 100644 --- a/application/controllers/attribute_by_survey.php +++ b/application/controllers/attribute_by_survey.php @@ -47,7 +47,7 @@ public function index() { $segments = $this->uri->segment_array(); $this->_survey_id = $segments[2]; $this->pagetitle = 'Attributes for ' . $this->getSurvey()->title; - $this->page_breadcrumbs[] = html::anchor('survey', 'Surveys'); + $this->page_breadcrumbs[] = html::anchor('survey', 'Survey datasets'); $this->page_breadcrumbs[] = $this->pagetitle; $this->template->content = new View('attribute_by_survey/index'); $this->template->title = $this->pagetitle; @@ -348,7 +348,7 @@ protected function get_return_page() { * Set the edit page breadcrumbs to cope with the fact this controller handles all *_attributes_website models. */ protected function defineEditBreadcrumbs() { - $this->page_breadcrumbs[] = html::anchor('survey', 'Surveys'); + $this->page_breadcrumbs[] = html::anchor('survey', 'Survey datasets'); $survey = ORM::Factory('survey', $this->model->restrict_to_survey_id); $this->page_breadcrumbs[] = html::anchor('/attribute_by_survey/'.$this->model->restrict_to_survey_id.'?type='.$this->type, 'Attributes for '.$survey->title); $this->page_breadcrumbs[] = $this->model->caption(); diff --git a/application/controllers/occurrence.php b/application/controllers/occurrence.php index 9669d5ef8d..c32807f6c2 100644 --- a/application/controllers/occurrence.php +++ b/application/controllers/occurrence.php @@ -36,7 +36,7 @@ public function __construct() { $this->columns = [ 'id' => 'ID', 'website' => 'Website', - 'survey' => 'Survey', + 'survey' => 'Survey dataset', 'taxon' => 'Taxon', 'entered_sref' => 'Spatial Ref', 'date_start' => 'Date', diff --git a/application/controllers/sample.php b/application/controllers/sample.php index 5dca220325..90653883cb 100644 --- a/application/controllers/sample.php +++ b/application/controllers/sample.php @@ -32,10 +32,10 @@ public function __construct() ( 'id' => 'ID', 'website' => 'Website', - 'survey' => 'Survey', + 'survey' => 'Survey dataset', 'entered_sref' => 'Spatial Ref.', 'location' => 'Location', - 'date_start' => 'Date' + 'date_start' => 'Date', ); $this->set_website_access('editor'); } diff --git a/application/controllers/survey_comment.php b/application/controllers/survey_comment.php index 2f96385c2e..ff781fe0d4 100644 --- a/application/controllers/survey_comment.php +++ b/application/controllers/survey_comment.php @@ -77,7 +77,7 @@ protected function get_return_page() { * Define non-standard behaviuor for the breadcrumbs, since this is accessed via a taxon list */ protected function defineEditBreadcrumbs() { - $this->page_breadcrumbs[] = html::anchor('survey', 'Surveys'); + $this->page_breadcrumbs[] = html::anchor('survey', 'Survey datasets'); if ($this->model->id) { // editing an existing item, so our argument is the survey_comment_id $survey_id = $this->model->survey_id; diff --git a/application/controllers/survey_medium.php b/application/controllers/survey_medium.php index 13e5cec88d..b370fb1c9e 100644 --- a/application/controllers/survey_medium.php +++ b/application/controllers/survey_medium.php @@ -73,8 +73,9 @@ protected function getDefaults() { */ protected function get_return_page() { if (array_key_exists('survey_medium:survey_id', $_POST)) { - return "survey/edit/".$_POST['survey_medium:survey_id']."?tab=Media_Files"; - } else { + return "survey/edit/" . $_POST['survey_medium:survey_id'] . "?tab=Media_Files"; + } + else { return $this->model->object_name; } } @@ -82,28 +83,28 @@ protected function get_return_page() { /** * Get the list of terms ready for the media types list. */ - protected function prepareOtherViewData(array $values) - { - return array( - 'media_type_terms' => $this->get_termlist_terms('indicia:media_types') - ); + protected function prepareOtherViewData(array $values) { + return [ + 'media_type_terms' => $this->get_termlist_terms('indicia:media_types'), + ]; } /** * Define non-standard behaviuor for the breadcrumbs, since this is accessed via a survey */ protected function defineEditBreadcrumbs() { - $this->page_breadcrumbs[] = html::anchor('survey', 'Surveys'); + $this->page_breadcrumbs[] = html::anchor('survey', 'Survey datasets'); if ($this->model->id) { - // editing an existing item + // Editing an existing item. $surveyId = $this->model->survey_id; - } else { - // creating a new one so our argument is the survey id + } + else { + // Creating a new one so our argument is the survey id. $surveyId = $this->uri->argument(1); } $survey = ORM::factory('survey', $surveyId); - $this->page_breadcrumbs[] = html::anchor('survey/edit/'.$surveyId, $survey->caption()); + $this->page_breadcrumbs[] = html::anchor("survey/edit/$surveyId", $survey->caption()); $this->page_breadcrumbs[] = $this->model->caption(); } -} \ No newline at end of file +} diff --git a/application/controllers/user.php b/application/controllers/user.php index cb28507473..523c94dcc7 100644 --- a/application/controllers/user.php +++ b/application/controllers/user.php @@ -66,9 +66,26 @@ protected function get_action_columns() { } protected function password_fields($password = '', $password2 = '') { - return '
  • ' . - html::error_message($this->model->getError('password')) . - '
    • '; + $pw1 = html::specialchars($password); + $pw2 = html::specialchars($password2); + $error = html::error_message($this->model->getError('password')); + if (!empty($error)) { + $error = "$error"; + } + $errorClass = empty($error) ? '' : ' has-error'; + return << + + + $error + + +
    + + +
    + +HTML; } // Due to the way the Users gridview is displayed (ie driven off the person table) diff --git a/application/models/occurrence.php b/application/models/occurrence.php index ae482e9511..323564007c 100644 --- a/application/models/occurrence.php +++ b/application/models/occurrence.php @@ -607,8 +607,8 @@ public function fixed_values_form($options = array()) { 'validation' => ['required'], ), 'survey_id' => array( - 'display' => 'Survey', - 'description' => 'Select the survey to import records into.', + 'display' => 'Survey dataset', + 'description' => 'Select the survey dataset to import records into.', 'datatype' => 'lookup', 'population_call' => 'direct:survey:id:title', 'linked_to' => 'website_id', diff --git a/application/models/sample.php b/application/models/sample.php index 1d0c16007c..0069e32317 100644 --- a/application/models/sample.php +++ b/application/models/sample.php @@ -380,8 +380,8 @@ public function fixed_values_form($options = array()) { 'population_call'=>'direct:website:id:title' ), 'survey_id' => array( - 'display'=>'Survey', - 'description'=>'Select the survey to import records into.', + 'display'=>'Survey dataset', + 'description'=>'Select the survey dataset to import records into.', 'datatype'=>'lookup', 'population_call'=>'direct:survey:id:title', 'linked_to'=>'website_id', diff --git a/application/models/user.php b/application/models/user.php index ac5bf022ad..edead0673c 100644 --- a/application/models/user.php +++ b/application/models/user.php @@ -53,11 +53,7 @@ public function validate(Validation $array, $save = FALSE) { $array->add_rules('password', 'required', 'length[7,30]', 'matches_post[password2]'); } $this->unvalidatedFields = [ - 'interests', - 'location_name', 'core_role_id', - 'email_visible', - 'view_common_names', 'person_id', 'allow_share_for_reporting', 'allow_share_for_peer_review', diff --git a/application/views/occurrence/occurrence_edit.php b/application/views/occurrence/occurrence_edit.php index f6b11a994a..cf93fabdba 100644 --- a/application/views/occurrence/occurrence_edit.php +++ b/application/views/occurrence/occurrence_edit.php @@ -41,7 +41,7 @@ ID id;?>
    'Survey', + 'label' => 'Survey dataset', 'fieldname' => 'sample-survey', 'default' => $sample->survey->title, 'readonly' => TRUE, diff --git a/application/views/user/user_edit.php b/application/views/user/user_edit.php index 2c6870e626..1809923b8a 100644 --- a/application/views/user/user_edit.php +++ b/application/views/user/user_edit.php @@ -27,107 +27,93 @@ ?>

    This page allows you to specify a users details.

    -
    - User's Details - - - 'Username', - 'fieldname' => 'user:username', - 'default' => html::initial_value($values, 'user:username'), - 'validation' => ['required'], - ]); - echo data_entry_helper::textarea([ - 'label' => 'Interests', - 'fieldname' => 'user:interests', - 'default' => html::initial_value($values, 'user:interests'), - ]); - echo data_entry_helper::textarea([ - 'label' => 'Location name', - 'fieldname' => 'user:location_name', - 'default' => html::initial_value($values, 'user:location_name'), - ]); - echo data_entry_helper::checkbox([ - 'label' => 'Email visible', - 'fieldname' => 'user:email_visible', - 'default' => html::initial_value($values, 'user:email_visible'), - ]); - echo data_entry_helper::checkbox([ - 'label' => 'View common names', - 'fieldname' => 'user:view_common_names', - 'default' => html::initial_value($values, 'user:view_common_names'), - ]); - echo data_entry_helper::checkbox([ - 'label' => 'This user allows records to be shared for reporting', - 'fieldname' => 'user:allow_share_for_reporting', - 'default' => html::initial_value($values, 'user:allow_share_for_reporting'), - ]); - echo data_entry_helper::checkbox([ - 'label' => 'This user allows records to be shared for peer review', - 'fieldname' => 'user:allow_share_for_peer_review', - 'default' => html::initial_value($values, 'user:allow_share_for_peer_review'), - ]); - echo data_entry_helper::checkbox([ - 'label' => 'This user allows records to be shared for verification', - 'fieldname' => 'user:allow_share_for_verification', - 'default' => html::initial_value($values, 'user:allow_share_for_verification'), - ]); - echo data_entry_helper::checkbox([ - 'label' => 'This user allows records to be shared for data flow', - 'fieldname' => 'user:allow_share_for_data_flow', - 'default' => html::initial_value($values, 'user:allow_share_for_data_flow'), - ]); - echo data_entry_helper::checkbox([ - 'label' => 'This user allows records to be shared for moderation', - 'fieldname' => 'user:allow_share_for_moderation', - 'default' => html::initial_value($values, 'user:allow_share_for_moderation'), - ]); - echo data_entry_helper::checkbox([ - 'label' => 'This user allows records to be shared for editing', - 'fieldname' => 'user:allow_share_for_editing', - 'default' => html::initial_value($values, 'user:allow_share_for_editing'), - ]); - if ($this->auth->logged_in('CoreAdmin')) { - $roles = ORM::factory('core_role')->orderby('title', 'asc')->find_all(); - $lookupValues = []; - foreach ($roles as $role) { - $lookupValues[$role->id] = $role->title; - } - echo data_entry_helper::select([ - 'label' => 'Role within Warehouse', - 'fieldname' => 'user:core_role_id', - 'default' => html::initial_value($values, 'user:core_role_id'), - 'lookupValues' => $lookupValues, - 'blankText' => '', - ]); + + + 'Username', + 'fieldname' => 'user:username', + 'default' => html::initial_value($values, 'user:username'), + 'validation' => ['required'], + ]); + echo '
    User data sharing options'; + echo data_entry_helper::checkbox([ + 'label' => 'This user allows records to be shared for reporting', + 'fieldname' => 'user:allow_share_for_reporting', + 'default' => html::initial_value($values, 'user:allow_share_for_reporting'), + ]); + echo data_entry_helper::checkbox([ + 'label' => 'This user allows records to be shared for peer review', + 'fieldname' => 'user:allow_share_for_peer_review', + 'default' => html::initial_value($values, 'user:allow_share_for_peer_review'), + ]); + echo data_entry_helper::checkbox([ + 'label' => 'This user allows records to be shared for verification', + 'fieldname' => 'user:allow_share_for_verification', + 'default' => html::initial_value($values, 'user:allow_share_for_verification'), + ]); + echo data_entry_helper::checkbox([ + 'label' => 'This user allows records to be shared for data flow', + 'fieldname' => 'user:allow_share_for_data_flow', + 'default' => html::initial_value($values, 'user:allow_share_for_data_flow'), + ]); + echo data_entry_helper::checkbox([ + 'label' => 'This user allows records to be shared for moderation', + 'fieldname' => 'user:allow_share_for_moderation', + 'default' => html::initial_value($values, 'user:allow_share_for_moderation'), + ]); + echo data_entry_helper::checkbox([ + 'label' => 'This user allows records to be shared for editing', + 'fieldname' => 'user:allow_share_for_editing', + 'default' => html::initial_value($values, 'user:allow_share_for_editing'), + ]); + echo '
    '; + if ($this->auth->logged_in('CoreAdmin')) { + $roles = ORM::factory('core_role')->orderby('title', 'asc')->find_all(); + $lookupValues = []; + foreach ($roles as $role) { + $lookupValues[$role->id] = $role->title; } + echo data_entry_helper::select([ + 'label' => 'Role within Warehouse', + 'fieldname' => 'user:core_role_id', + 'default' => html::initial_value($values, 'user:core_role_id'), + 'lookupValues' => $lookupValues, + 'blankText' => '', + ]); + } - if (isset($password_field) and $password_field != '') { - echo $password_field; - echo html::error_message($model->getError('user:password')); - } ?> -
    + if (isset($password_field) and $password_field != '') { + echo $password_field; + echo html::error_message($model->getError('user:password')); + } ?>
    - Website Roles -
      + Website roles users_websites as $website) { - echo '
    1. '; - echo '
      2. '; + $otherOptions = implode("\n ", $otherOptionList); + echo << +
      + +
      + +
      +
      + + +HTML; } ?> -
    'AFTER' THEN RAISE EXCEPTION 'audit.if_modified_func() may only run as an AFTER trigger'; END IF; - + audit_row = ROW( NEXTVAL('audit.logged_actions_id_seq'), -- id txid_current(), -- transaction ID @@ -71,7 +71,7 @@ BEGIN NEW.updated_by_id, -- Indicia updated_by_id NULL, NULL -- row_data, changed_fields ); - + --- Override search details for child records IF (TG_ARGV[0] = 'sample_id') THEN audit_row.search_table_name = 'samples'; @@ -87,7 +87,7 @@ BEGIN IF TG_ARGV[1] IS NOT NULL THEN excluded_cols = TG_ARGV[1]::text[]; END IF; - + IF (TG_OP = 'UPDATE' AND TG_LEVEL = 'ROW') THEN audit_row.row_data = hstore(OLD.*); audit_row.changed_fields = (hstore(NEW.*) - audit_row.row_data) - excluded_cols; @@ -115,47 +115,47 @@ BEGIN website_row.website_id = websiteID; INSERT INTO audit.logged_actions_websites VALUES (website_row.*); END LOOP; - + RETURN NULL; END; $body$ LANGUAGE plpgsql SECURITY DEFINER; - + COMMENT ON FUNCTION audit.if_modified_func() IS $body$ Track changes TO a TABLE at the statement AND/OR ROW level. - + Optional parameters TO TRIGGER IN CREATE TRIGGER CALL: - + param 0: text, COLUMN used to source primary key; for subtables this will point to the parent records primary key. param 1: text[], COLUMNS TO IGNORE IN updates. DEFAULT []. - + Updates TO ignored cols are omitted FROM changed_fields. - + Updates WITH ONLY ignored cols changed are NOT inserted INTO the audit log. - + Almost ALL the processing WORK IS still done FOR updates that ignored. IF you need TO save the LOAD, you need TO USE WHEN clause ON the TRIGGER instead. - + No warning OR error IS issued IF ignored_cols contains COLUMNS that do NOT exist IN the target TABLE. This lets you specify a standard SET OF ignored COLUMNS. - + There IS no parameter TO disable logging OF VALUES. ADD this TRIGGER AS a 'FOR EACH STATEMENT' rather than 'FOR EACH ROW' TRIGGER IF you do NOT want TO log ROW VALUES. - + Note that the USER name logged IS the login ROLE FOR the SESSION. The audit TRIGGER cannot obtain the active ROLE because it IS reset BY the SECURITY DEFINER invocation OF the audit TRIGGER its SELF. $body$; - - - + + + CREATE OR REPLACE FUNCTION audit.audit_table(target_table regclass, audit_rows BOOLEAN, audit_inserts BOOLEAN, primary_column text, ignored_cols text[]) RETURNS void AS $body$ DECLARE stm_targets text = 'UPDATE OR DELETE OR TRUNCATE'; @@ -170,13 +170,13 @@ BEGIN stm_targets = 'INSERT OR ' || stm_targets ; row_targets = 'INSERT OR ' || row_targets; END IF; - + IF audit_rows THEN IF array_length(ignored_cols,1) > 0 THEN _ignored_cols_snip = ', ' || quote_literal(ignored_cols); END IF; - _q_txt = 'CREATE TRIGGER audit_trigger_row AFTER ' || row_targets || ' ON ' || - quote_ident(target_table::text) || + _q_txt = 'CREATE TRIGGER audit_trigger_row AFTER ' || row_targets || ' ON ' || + quote_ident(target_table::text) || ' FOR EACH ROW EXECUTE PROCEDURE audit.if_modified_func(' || primary_column || _ignored_cols_snip || ');'; RAISE NOTICE '%',_q_txt; @@ -184,21 +184,21 @@ BEGIN stm_targets = 'TRUNCATE'; ELSE END IF; - + _q_txt = 'CREATE TRIGGER audit_trigger_stm AFTER ' || stm_targets || ' ON ' || quote_ident(target_table::text) || ' FOR EACH STATEMENT EXECUTE PROCEDURE audit.if_modified_func(' || primary_column || ');'; RAISE NOTICE '%',_q_txt; EXECUTE _q_txt; - + END; $body$ LANGUAGE 'plpgsql'; - + COMMENT ON FUNCTION audit.audit_table(regclass, BOOLEAN, BOOLEAN, text, text[]) IS $body$ ADD auditing support TO a TABLE. - + Arguments: target_table: TABLE name, schema qualified IF NOT ON search_path audit_rows: Record each ROW CHANGE, OR ONLY audit at a statement level @@ -206,14 +206,14 @@ Arguments: parent_column: In tables with a parent entity, the column name ignored_cols: COLUMNS TO exclude FROM UPDATE diffs, IGNORE updates that CHANGE ONLY ignored cols. $body$; - - + + -- And provide a convenient call wrappers for the simplest cases -- CREATE OR REPLACE FUNCTION audit.audit_table(target_table regclass, audit_inserts BOOLEAN, parent_column text) RETURNS void AS $$ SELECT audit.audit_table($1, true, audit_inserts, parent_column, ARRAY[]::text[]); $$ LANGUAGE 'sql'; - + COMMENT ON FUNCTION audit.audit_table(regclass, BOOLEAN, text) IS $body$ ADD auditing support TO the given TABLE. Subtable - parent column as supplied, Rows Audited, inserts audited, No cols are ignored. $body$; @@ -221,7 +221,7 @@ $body$; CREATE OR REPLACE FUNCTION audit.audit_table(target_table regclass) RETURNS void AS $$ SELECT audit.audit_table($1, true, false, 'id', ARRAY[]::text[]); $$ LANGUAGE 'sql'; - + COMMENT ON FUNCTION audit.audit_table(regclass) IS $body$ ADD auditing support TO the given TABLE. Not subtable so no inserts, Rows Audited, No cols are ignored. $body$; \ No newline at end of file diff --git a/modules/audit/dbuninstall/remove_auditing.sql b/modules/audit/dbuninstall/remove_auditing.sql index 534e782ffa..6e3d996692 100755 --- a/modules/audit/dbuninstall/remove_auditing.sql +++ b/modules/audit/dbuninstall/remove_auditing.sql @@ -1,3 +1,3 @@ DROP SCHEMA audit CASCADE; -DELETE FROM indicia.system WHERE name = 'audit'; +DELETE FROM system WHERE name = 'audit'; diff --git a/modules/cache_builder/config/cache_builder.php b/modules/cache_builder/config/cache_builder.php index c2785c75f6..8c90f2b9a1 100644 --- a/modules/cache_builder/config/cache_builder.php +++ b/modules/cache_builder/config/cache_builder.php @@ -1309,12 +1309,6 @@ join locations l on l.id=s.location_id where l.updated_on>'#date#' union - select o.id, su.deleted - from occurrences o - join samples s on s.id=o.sample_id - join surveys su on su.id=s.survey_id - where su.updated_on>'#date#' - union select o.id, ttl.deleted from occurrences o join taxa_taxon_lists ttl on ttl.id=o.taxa_taxon_list_id diff --git a/modules/indicia_setup/controllers/setup_check.php b/modules/indicia_setup/controllers/setup_check.php index 8977420f9a..1984abd8bf 100644 --- a/modules/indicia_setup/controllers/setup_check.php +++ b/modules/indicia_setup/controllers/setup_check.php @@ -222,9 +222,11 @@ public function config_db() { $description = str_replace( array('*code*', '*code_user*', '*code_perm*'), array( - "CREATE DATABASE indicia TEMPLATE=template_postgis;", + "CREATE DATABASE indicia;", "CREATE USER indicia_user WITH PASSWORD 'indicia';\n" . "GRANT ALL PRIVILEGES ON DATABASE indicia TO indicia_user;", + "CREATE EXTENSION postgis;\n" . + "CREATE EXTENSION postgis_topology;\n" . "CREATE EXTENSION btree_gin;\n" . "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO indicia_user;\n" . "GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO indicia_user;\n" . diff --git a/modules/indicia_setup/db/version_1_28_0/201703170934_confidential_comment.sql b/modules/indicia_setup/db/version_1_28_0/201703170934_confidential_comment.sql index 921b8750a7..cbf0da97d8 100644 --- a/modules/indicia_setup/db/version_1_28_0/201703170934_confidential_comment.sql +++ b/modules/indicia_setup/db/version_1_28_0/201703170934_confidential_comment.sql @@ -1,2 +1,2 @@ -COMMENT ON COLUMN indicia.occurrences.confidential IS +COMMENT ON COLUMN occurrences.confidential IS 'Flag set to true if this record is flagged confidential by the dataset administrator. The confidential flag relates to the need to control communications around a record rather then simply an indicator that a record is sensitive (which should be done via the sensitivity_precision field) so this flag prevents notifications about this record being sent to the recorder.'; diff --git a/modules/indicia_setup/db/version_3_0_0/201911291600_tidy_user_table.sql b/modules/indicia_setup/db/version_3_0_0/201911291600_tidy_user_table.sql new file mode 100644 index 0000000000..cf70716903 --- /dev/null +++ b/modules/indicia_setup/db/version_3_0_0/201911291600_tidy_user_table.sql @@ -0,0 +1,17 @@ +ALTER TABLE users +DROP COLUMN email_visible; + +ALTER TABLE users +DROP COLUMN view_common_names; + +ALTER TABLE users +DROP COLUMN location_name; + +ALTER TABLE users +DROP COLUMN interests; + +ALTER TABLE users +DROP COLUMN home_entered_sref; + +ALTER TABLE users +DROP COLUMN home_entered_sref_system; \ No newline at end of file diff --git a/modules/indicia_setup/helpers/config_test.php b/modules/indicia_setup/helpers/config_test.php index 779aaeb19b..4912b3cbf4 100644 --- a/modules/indicia_setup/helpers/config_test.php +++ b/modules/indicia_setup/helpers/config_test.php @@ -84,7 +84,7 @@ public static function check_db(&$messages, $problems_only) { ); } else { - $problem['description'] .= 'Fix the other issues listed on this page before proceeding to configure and ' . + $problem['description'] .= ' Fix the other issues listed on this page before proceeding to configure and ' . 'install the database.'; } array_push($messages, $problem); diff --git a/modules/indicia_setup/i18n/en_GB/setup.php b/modules/indicia_setup/i18n/en_GB/setup.php index 995ad5313b..742dd4027e 100644 --- a/modules/indicia_setup/i18n/en_GB/setup.php +++ b/modules/indicia_setup/i18n/en_GB/setup.php @@ -13,7 +13,8 @@ *code_user* Finally, connect to your new database and run the following script to prepare the required extensions and grant -permissions. +permissions. Note that if installing the warehouse in a hosted environment, you may not be able to create the extensions +yourself using SQL and will either have to do that via your control panel or by asking your host. 
     *code_perm*
    diff --git a/modules/indicia_setup/models/upgrade.php b/modules/indicia_setup/models/upgrade.php index 1fa3c5fa52..bd1168e8ba 100644 --- a/modules/indicia_setup/models/upgrade.php +++ b/modules/indicia_setup/models/upgrade.php @@ -141,8 +141,9 @@ public function run() { // can run the whole upgrade. If more than 1000, then scripts which perform // a lot of processing on occurrences can be marked as slow and run after // the upgrade. We use an approx count as it is much faster than count(*). + $this->couldBeSlow = $this->db - ->query("SELECT reltuples as approx_count FROM pg_class WHERE oid = 'indicia.occurrences'::regclass;") + ->query("SELECT reltuples as approx_count FROM pg_class WHERE oid = 'occurrences'::regclass;") ->current()->approx_count > 1000; // Run the core upgrade. $last_run_script = $system->getLastRunScript('Indicia'); diff --git a/modules/indicia_svc_data/tests/controllers/services/dataTest.php b/modules/indicia_svc_data/tests/controllers/services/dataTest.php index fc712ae261..face51e879 100644 --- a/modules/indicia_svc_data/tests/controllers/services/dataTest.php +++ b/modules/indicia_svc_data/tests/controllers/services/dataTest.php @@ -361,12 +361,11 @@ public function testCreateUser() { $this->assertTrue(isset($r['success']), 'Submitting a new person did not work'); $personId = $r['success']; - $array=array( + $array = [ 'user:person_id' => $personId, - 'user:email_visible' => 'f', 'user:core_role_id' => 1, - 'user:username' => 'testUser' - ); + 'user:username' => 'testUser', + ]; $s = submission_builder::build_submission($array, array('model' => 'user')); $r = data_entry_helper::forward_post_to('user', $s, $this->auth['write_tokens']); diff --git a/modules/indicia_svc_security/controllers/services/site_user.php b/modules/indicia_svc_security/controllers/services/site_user.php index b9d436f9d9..c12002b2b1 100644 --- a/modules/indicia_svc_security/controllers/services/site_user.php +++ b/modules/indicia_svc_security/controllers/services/site_user.php @@ -77,10 +77,10 @@ public function authenticate_user() { $this->handle_error($e); } } - + /** * Implements the webservice call from remote websites to send a password reset email. - * + * * Does not support requests from users on the warehouse itself (website_id < 0) * Expects HTTP POST with userid, options and usual service credentials. * Catches any Exceptions and passes them to handle_error() @@ -92,9 +92,9 @@ public function request_password_reset() { try { $this->authenticate('read'); - + $username_or_email = $_POST['userid']; - + $this->auth = new Auth; $returned = $this->auth->user_and_person_by_username_or_email($_POST['userid']); if (array_key_exists('error_message', $returned)) { @@ -104,7 +104,7 @@ public function request_password_reset() { } $user = $returned['user']; $person = $returned['person']; - + if (! $this->auth->is_website_user($user->id, $this->website_id) ) { $result = array('result' => false, @@ -112,7 +112,7 @@ public function request_password_reset() { $this->response = json_encode($result); return; } - + $this->auth->send_forgotten_password_mail($user, $person); $result = array('result' => true); @@ -125,47 +125,41 @@ public function request_password_reset() { $this->handle_error($e); } } - + /** * Returns a json string containing user profile data for the supplied use id. * The user must have a role on the requesting warehouse and not be logically deleted. * Data for banned users is returned. - * + * * Does not support requests from users on the warehouse itself (website_id < 0) * Expects HTTP POST with usual service credentials. * User_id is supplied as part of the URI and passed to this function as an argument. * Catches any Exceptions and passes them to handle_error() * - * @return json array with profile data as follows + * @return json + * Array with profile data as follows: + * * title + * * first_name + * * surname + * * initials + * * email_address + * * website_url + * * address + * * username + * * default_digest_mode + * * activated + * * banned + * * site_role + * * registration_datetime + * * last_login_datetime + * * preferred_sref_system */ public function get_user_profile($user_id) { try { $this->authenticate('read'); - + $profile = $this->_get_user_profile($user_id); // set response @@ -177,7 +171,7 @@ public function get_user_profile($user_id) { $this->handle_error($e); } } - + /** * Reusable private implementation of get_user_profile which just returns an array * and doesn't authenticate service. @@ -207,30 +201,24 @@ private function _get_user_profile($user_id) { if (is_numeric($website->site_role_id)) { $site_role = new Site_Role_Model($website->site_role_id); } - + $profile = array( - 'result' => true, - 'title' => is_numeric($person->title_id) ? $title->title : '', - 'first_name' => $person->first_name, - 'surname' => $person->surname, - 'initials' => $person->initials, - 'email_address' => $person->email_address, - 'website_url' => $person->website_url, - 'address' => $person->address, - 'home_entered_sref' => $user->home_entered_sref, - 'home_entered_sref_system' => $user->home_entered_sref_system, - 'interests' => $user->interests, - 'location_name' => $user->location_name, - 'email_visible' => $user->email_visible, - 'view_common_names' => $user->view_common_names, - 'username' => $user->username, - 'default_digest_mode' => $user->default_digest_mode, - 'activated' => $website->activated, - 'banned' => $website->banned, - 'site_role' => is_numeric($website->site_role_id) ? $site_role->title : '', - 'registration_datetime' => $website->registration_datetime, - 'last_login_datetime' => $website->last_login_datetime, - 'preferred_sref_system' => $website->preferred_sref_system, + 'result' => TRUE, + 'title' => is_numeric($person->title_id) ? $title->title : '', + 'first_name' => $person->first_name, + 'surname' => $person->surname, + 'initials' => $person->initials, + 'email_address' => $person->email_address, + 'website_url' => $person->website_url, + 'address' => $person->address, + 'username' => $user->username, + 'default_digest_mode' => $user->default_digest_mode, + 'activated' => $website->activated, + 'banned' => $website->banned, + 'site_role' => is_numeric($website->site_role_id) ? $site_role->title : '', + 'registration_datetime' => $website->registration_datetime, + 'last_login_datetime' => $website->last_login_datetime, + 'preferred_sref_system' => $website->preferred_sref_system, ); return $profile; @@ -240,5 +228,5 @@ private function _get_user_profile($user_id) { $this->handle_error($e); } } - + } \ No newline at end of file diff --git a/modules/indicia_svc_security/helpers/user_identifier.php b/modules/indicia_svc_security/helpers/user_identifier.php index b8d7ff6ffc..c2a34b2e0d 100644 --- a/modules/indicia_svc_security/helpers/user_identifier.php +++ b/modules/indicia_svc_security/helpers/user_identifier.php @@ -321,7 +321,6 @@ private static function createUser($email, $userPersonObj) { $user = ORM::factory('user'); $data = array( 'person_id' => $person->id, - 'email_visible' => 'f', 'username' => $person->newUsername(), // User will not actually have warehouse access, so password fairly irrelevant. 'password' => 'P4ssw0rd', diff --git a/modules/rest_api/controllers/services/rest.php b/modules/rest_api/controllers/services/rest.php index 7a819491bf..80f79ebc38 100644 --- a/modules/rest_api/controllers/services/rest.php +++ b/modules/rest_api/controllers/services/rest.php @@ -252,6 +252,20 @@ class Rest_Controller extends Controller { */ private $request; + /** + * For ES paged downloads, holds the mode (scroll or composite). + * + * @var string + */ + private $pagingMode = 'off'; + + /** + * For ES paged downloads, holds the current request state (initial or nextPage). + * + * @var string + */ + private $pagingModeState; + /** * List of project definitions that are available to the authorised client. * @@ -774,20 +788,12 @@ private function checkAllowedResource($proj_id, $resourceName) { ]; /** - * Calculate the data to post to an Elasticsearch search. - * - * @param string $scrollMode - * Scroll mode for ES, either off, initial, or nextpage. - * @param string $format - * Format identifier. If CSV then we can use this to do source filtering - * to lower memory consumption. + * Works out the list of columns for an ES CSV download. * - * @return string - * Data to post. + * @param obj $postObj + * Request object. */ - private function getEsPostData($scrollMode, $format) { - $postData = file_get_contents('php://input'); - $postObj = empty($postData) ? [] : json_decode($postData); + private function getColumnsTemplate(&$postObj) { // Params for configuring an ES CSV download template get extracted and not // sent to ES. if (isset($postObj->columnsTemplate)) { @@ -802,7 +808,24 @@ private function getEsPostData($scrollMode, $format) { $this->esCsvTemplateRemoveColumns = (array) $postObj->removeColumns; unset($postObj->removeColumns); } - if ($scrollMode === 'nextpage') { + } + + /** + * Calculate the data to post to an Elasticsearch search. + * + * @param obj $postObj + * Request object. + * @param string $format + * Format identifier. If CSV then we can use this to do source filtering + * to lower memory consumption. + * @param array|NULL $file + * Cached info about the file if paging. + * + * @return string + * Data to post. + */ + private function getEsPostData($postObj, $format, $file) { + if ($this->pagingMode === 'scroll' && $this->pagingModeState === 'nextPage') { // A subsequent hit on a scrolled request. $postObj = [ 'scroll_id' => $_GET['scroll_id'], @@ -811,9 +834,14 @@ private function getEsPostData($scrollMode, $format) { return json_encode($postObj); } // Either unscrolled, or the first call to a scroll. So post the query. - if ($scrollMode !== 'off') { + if ($this->pagingMode === 'scroll') { $postObj->size = MAX_ES_SCROLL_SIZE; } + elseif ($this->pagingMode === 'composite' && isset($file['after_key'])) { + foreach ($postObj->aggs as &$agg) { + $agg->composite->after = $file['after_key']; + } + } if ($format === 'csv') { $csvTemplate = $this->getEsCsvTemplate(); $fields = []; @@ -854,14 +882,12 @@ private function getEsPostData($scrollMode, $format) { * * @param string $url * Elasticsearch alias URL. - * @param string $scrollMode - * Current scroll mode, either off, initial or nextpage. * * @return string * Revised URL. */ - private function getEsActualUrl($url, $scrollMode) { - if ($scrollMode === 'nextpage') { + private function getEsActualUrl($url) { + if ($this->pagingMode === 'scroll' && $this->pagingModeState === 'nextPage') { // On subsequent hits to a scrolled request, the URL is different. return preg_replace('/[a-z0-9_-]*\/_search$/', '_search/scroll', $url); } @@ -876,8 +902,11 @@ private function getEsActualUrl($url, $scrollMode) { unset($params['scroll']); unset($params['scroll_id']); unset($params['callback']); + unset($params['aggregation_type']); + unset($params['state']); + unset($params['uniq_id']); unset($params['_']); - if ($scrollMode === 'initial') { + if ($this->pagingMode === 'scroll' && $this->pagingModeState === 'initial') { $params['scroll'] = SCROLL_TIMEOUT; } $url .= '?' . http_build_query($params); @@ -948,27 +977,6 @@ private static function purgeDownloadFiles() { } } - /** - * Builds an empty CSV file ready to received a scrolled ES request. - * - * @param string $format - * Data format, either json or csv. - * - * @return array - * File array containing the name and handle. - */ - private function prepareScrollFile($format) { - $this->purgeDownloadFiles(); - $filename = uniqid() . ".$format"; - // Reopen file for appending. - $handle = fopen(DOCROOT . "download/$filename", "w"); - fwrite($handle, $this->getEsOutputHeader($format)); - return [ - 'filename' => $filename, - 'handle' => $handle, - ]; - } - /** * Builds the header for the top of a scrolled Elasticsearch output. * @@ -1019,22 +1027,66 @@ private function getEsCsvTemplate() { return $csvTemplate; } + /** + * Builds an empty CSV file ready to received a paged ES request. + * + * @param string $format + * Data format, either json or csv. + * + * @return array + * File array containing the name and handle. + */ + private function preparePagingFile($format) { + $this->purgeDownloadFiles(); + $uniqId = uniqid('', TRUE); + $filename = "download-$uniqId.$format"; + // Reopen file for appending. + $handle = fopen(DOCROOT . "download/$filename", "w"); + fwrite($handle, $this->getEsOutputHeader($format)); + return [ + 'uniq_id' => $uniqId, + 'filename' => $filename, + 'handle' => $handle, + 'done' => 0, + ]; + } + /** * Create a temporary file that will be used to build an ES download. * + * @param string $format + * Data format, either json or csv. + * * @return array * File details, array containing filename and handle. */ - private function openScrollFile() { + private function openPagingFile($format) { + $uniqId = isset($_GET['uniq_id']) ? $_GET['uniq_id'] : $_GET['scroll_id']; $cache = Cache::instance(); - $info = $cache->get("es-scroll-$_GET[scroll_id]"); + $info = $cache->get("es-paging-$uniqId"); if ($info === NULL) { - $this->apiResponse->fail('Bad request', 400, 'Invalid scroll ID.'); + $this->apiResponse->fail('Bad request', 400, 'Invalid scroll_id or uniq_id parameter.'); } $info['handle'] = fopen(DOCROOT . "download/$info[filename]", 'a'); return $info; } + /** + * Works out the mode of paging for chunked downloads. + * + * Supports Elasticsearch scroll or composite aggregations for paging. Mode + * is stored in $this->pagingMode. + */ + private function getPagingMode($format) { + $this->pagingModeState = empty($_GET['state']) ? 'initial' : $_GET['state']; + if (isset($_GET['aggregation_type']) && $_GET['aggregation_type'] === 'composite') { + $this->pagingMode = 'composite'; + } + elseif ($format === 'csv') { + $this->pagingMode = 'scroll'; + } + } + /** * Proxies the current request to a provided URL. * @@ -1045,24 +1097,28 @@ private function openScrollFile() { */ private function proxyToEs($url) { $format = isset($_GET['format']) && $_GET['format'] === 'csv' ? 'csv' : 'json'; - $scrollMode = isset($_GET['scroll']) ? 'initial' : (empty($_GET['scroll_id']) ? 'off' : 'nextpage'); - $postData = $this->getEsPostData($scrollMode, $format); - $actualUrl = $this->getEsActualUrl($url, $scrollMode); - $session = curl_init($actualUrl); - if (!empty($postData) && $postData !== '[]') { - curl_setopt($session, CURLOPT_POST, 1); - curl_setopt($session, CURLOPT_POSTFIELDS, $postData); - } - if ($scrollMode === 'initial') { + $postData = file_get_contents('php://input'); + $postObj = empty($postData) ? [] : json_decode($postData); + $this->getPagingMode($format); + $this->getColumnsTemplate($postObj); + $file = NULL; + if ($this->pagingModeState === 'initial') { // First iteration of a scrolled request, so prepare an output file. - $file = $this->prepareScrollFile($format); + $file = $this->preparePagingFile($format); } - elseif ($scrollMode === 'nextpage') { - $file = $this->openScrollFile(); + elseif ($this->pagingModeState === 'nextPage') { + $file = $this->openPagingFile($format); } else { echo $this->getEsOutputHeader($format); } + $postData = $this->getEsPostData($postObj, $format, $file); + $actualUrl = $this->getEsActualUrl($url); + $session = curl_init($actualUrl); + if (!empty($postData) && $postData !== '[]') { + curl_setopt($session, CURLOPT_POST, 1); + curl_setopt($session, CURLOPT_POSTFIELDS, $postData); + } if ($_SERVER['REQUEST_METHOD'] !== 'GET') { curl_setopt($session, CURLOPT_CUSTOMREQUEST, $_SERVER['REQUEST_METHOD']); } @@ -1083,27 +1139,34 @@ private function proxyToEs($url) { $this->apiResponse->fail('Internal server error', 500, json_encode($error)); } curl_close($session); - // First response from a scroll, need to grab the scroll ID. - if ($scrollMode === 'initial') { + // Will need decoded data for processing CSV. + if ($format === 'csv') { $data = json_decode($response, TRUE); if (!empty($data['error'])) { kohana::log('error', 'Bad ES Rest query response: ' . json_encode($data['error'])); kohana::log('error', 'Query: ' . $postData); $this->apiResponse->fail('Bad request', 400, json_encode($data['error'])); } + // Find the list of documents or aggregation output to add to the CSV. + $itemList = $this->pagingMode === 'composite' + ? $data['aggregations']['samples']['buckets'] + : $data['hits']['hits']; + } + // First response from a scroll, need to grab the scroll ID. + if ($this->pagingMode === 'scroll' && $this->pagingModeState === 'initial') { $file['scroll_id'] = $data['_scroll_id']; - $file['total'] = $data['hits']['total']; - $file['done'] = 0; + // ES6/7 tolerance. + $file['total'] = isset($data['hits']['total']['value']) ? $data['hits']['total']['value'] : $data['hits']['total']; } - elseif ($scrollMode === 'nextpage') { + elseif ($this->pagingMode === 'scroll' && $this->pagingModeState === 'nextPage') { $file['scroll_id'] = $_GET['scroll_id']; } - if ($scrollMode === 'off') { + if ($this->pagingMode === 'off') { switch ($format) { case 'csv': header('Content-type: text/csv'); - $this->esToCsv($response); + $this->esToCsv($itemList); break; case 'json': @@ -1121,7 +1184,7 @@ private function proxyToEs($url) { else { switch ($format) { case 'csv': - $this->esToCsv($response, $file); + $this->esToCsv($itemList, $file); break; case 'json': @@ -1134,16 +1197,36 @@ private function proxyToEs($url) { } fclose($file['handle']); unset($file['handle']); - $file['done'] = min($file['total'], $file['done'] + MAX_ES_SCROLL_SIZE); - $cache = Cache::instance(); - if ($file['done'] < $file['total']) { - $cache->set("es-scroll-$file[scroll_id]", $file); + $done = FALSE; + if ($this->pagingMode === 'scroll') { + $file['done'] = min($file['total'], $file['done'] + MAX_ES_SCROLL_SIZE); + $done = $file['done'] >= $file['total']; + } + elseif ($this->pagingMode === 'composite') { + if ($format === 'csv') { + $file['done'] = $file['done'] + count($itemList); + } + // Composite aggregation has to run till we get an empty response. + $data = json_decode($response, TRUE); + $list = $data['aggregations'][array_keys($data['aggregations'])[0]]['buckets']; + $done = count($list) === 0; + if (empty($data['aggregations'][array_keys($data['aggregations'])[0]]['after_key'])) { + unset($file['after_key']); + } + else { + $file['after_key'] = $data['aggregations'][array_keys($data['aggregations'])[0]]['after_key']; + } } - else { - $cache->delete("es-scroll-$file[scroll_id]", $file); + $file['state'] = $done ? 'done' : 'nextPage'; + $cache = Cache::instance(); + if ($done) { + $cache->delete("es-paging-$file[uniq_id]", $file); unset($file['scroll_id']); $this->zip($file); } + else { + $cache->set("es-paging-$file[uniq_id]", $file); + } $file['filename'] = url::base() . 'download/' . $file['filename']; header('Content-type: application/json'); // Allow for a JSONP cross-site request. @@ -1177,22 +1260,21 @@ private function zip(array &$file) { /** * Converts an Elasticsearch response to a chunk of CSV data. * - * @param string $response - * Response from an Elasticsearch search. + * @param string $itemList + * Decoded list of data from an Elasticsearch search. * @param array $file * File data, or NULL if not writing to a file in which case the output * is echoed. */ - private function esToCsv($response, array $file = NULL) { - $data = json_decode($response, TRUE); - if (empty($data['hits']['hits'])) { + private function esToCsv($itemList, array $file = NULL) { + if (empty($itemList)) { return; } $esCsvTemplate = $this->getEsCsvTemplate(); - foreach ($data['hits']['hits'] as $doc) { + foreach ($itemList as $item) { $row = []; foreach ($esCsvTemplate as $source) { - $this->copyIntoCsvRow($doc, $source, $row); + $this->copyIntoCsvRow($item, $source, $row); } $row = array_map(function ($cell) { // Cells containing a quote, a comma or a new line will need to be @@ -1415,7 +1497,7 @@ private function getRawEsFieldValue(array $doc, $source) { private function copyIntoCsvRow(array $doc, $sourceField, array &$row) { // Fields starting '_' are special fields in the root of the doc. Others // are in the _source element. - $docSource = strpos($sourceField, '_') === 0 ? $doc : $doc['_source']; + $docSource = strpos($sourceField, '_') === 0 || !isset($doc['_source']) ? $doc : $doc['_source']; if (preg_match('/^\[(?P[a-z ]*)\](\((?[a-z0-9]*=[^,=\)]*(,[a-z0-9]*=[^,=\)]*)*)\))?$/', $sourceField, $matches)) { $fn = 'esGetSpecialField' . str_replace(' ', '', ucwords(str_replace(['['], '', $matches['sourceType']))); @@ -2580,7 +2662,6 @@ private function authenticate() { $this->authenticated = FALSE; $this->checkElasticsearchRequest(); if ($this->authenticated) { - kohana::log('debug', "Open elasticsearch request"); return; } // Provide a default if not configured.