From 5afa31dfd3bb1e3edf50d67a4c7060c02b7bfaf6 Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Thu, 30 Mar 2023 14:08:57 +0100
Subject: [PATCH 1/3] Version bump/changelog

---
 CHANGELOG.md                   | 7 +++++++
 application/config/version.php | 4 ++--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4af0e0f62f..1f47210c2d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,10 @@
+#Version 8.17.0
+*2023-03-30*
+
+* Adds organism_key to the cache_taxa_taxon_lists and cache_taxon_searchterms tables to provide
+  a more reliable option for filtering species lists when taxonomy is dynamic. See
+  https://github.com/BiologicalRecordsCentre/NPMS/issues/268.
+
 # Version 8.16.0
 *2023-03-17*
 
diff --git a/application/config/version.php b/application/config/version.php
index 4bcfd6f1d3..586b0f05ce 100644
--- a/application/config/version.php
+++ b/application/config/version.php
@@ -29,14 +29,14 @@
  *
  * @var string
  */
-$config['version'] = '8.16.4';
+$config['version'] = '8.17.0';
 
 /**
  * Version release date.
  *
  * @var string
  */
-$config['release_date'] = '2023-03-24';
+$config['release_date'] = '2023-03-30';
 
 /**
  * Link to the code repository downloads page.

From 63884f56c243d7e43bf9d1f29a7d39f296bec98c Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Thu, 30 Mar 2023 14:09:58 +0100
Subject: [PATCH 2/3] Taxon search service support for filtering by organism
 keys

---
 application/helpers/postgreSQL.php                 |  6 +++++-
 .../indicia_svc_data/controllers/services/data.php | 14 ++++++++++++--
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/application/helpers/postgreSQL.php b/application/helpers/postgreSQL.php
index cd6c486118..5c00319b1b 100644
--- a/application/helpers/postgreSQL.php
+++ b/application/helpers/postgreSQL.php
@@ -591,6 +591,7 @@ private static function taxonSearchCheckOptions($db, array &$options) {
     self::integerListOption($options, 'exclude_preferred_taxa_taxon_list_id');
     self::stringListOption($db, $options, 'preferred_taxon');
     self::stringListOption($db, $options, 'external_key');
+    self::stringListOption($db, $options, 'organism_key');
     self::integerListOption($options, 'parent_id');
     self::stringListOption($db, $options, 'language');
     self::checkBooleanOptions($options, [
@@ -660,7 +661,7 @@ private static function taxonSearchGetQueryContextFilter(array $options) {
     $params = [
       'taxon_list_id', 'scratchpad_list_id', 'taxon_group_id', 'taxon_group', 'taxon_meaning_id',
       'taxa_taxon_list_id', 'preferred_taxa_taxon_list_id', 'exclude_taxon_meaning_id', 'exclude_taxa_taxon_list_id',
-      'exclude_preferred_taxa_taxon_list_id', 'preferred_taxon', 'external_key', 'parent_id',
+      'exclude_preferred_taxa_taxon_list_id', 'preferred_taxon', 'external_key', 'organism_key', 'parent_id',
     ];
     foreach ($params as $param) {
       if (!empty($options[$param])) {
@@ -891,6 +892,7 @@ private static function taxonSearchGetColsListSql($isCount, array $searchFilterD
   cts.preferred_taxa_taxon_list_id,
   cts.taxon_meaning_id,
   cts.external_key,
+  cts.organism_key,
   cts.taxon_group_id,
   cts.parent_id,
   cts.identification_difficulty,
@@ -1014,6 +1016,8 @@ private static function taxonSearchGetLimitOffsetSql(array $options) {
    *     matches required.
    *   * external_key - External key or array of external keys to limit the
    *     search to (e.g. limit to a list of TVKs).
+   *   * organism_key - External key or array of organism keys to limit the
+   *     search to (e.g. limit to a list of UKSI keys).
    *   * parent_id - ID of a taxa_taxon_list record limit the search to
    *     children of, e.g. a species when searching the subspecies. May be set
    *     to null to force top level only. Pass null to search top level taxa
diff --git a/modules/indicia_svc_data/controllers/services/data.php b/modules/indicia_svc_data/controllers/services/data.php
index 4afd513e0c..0b8efc877b 100644
--- a/modules/indicia_svc_data/controllers/services/data.php
+++ b/modules/indicia_svc_data/controllers/services/data.php
@@ -964,8 +964,18 @@ protected function getDataTaxaSearch() {
     }
     unset($params['auth_token']);
     unset($params['nonce']);
-    $possibleArrays = ['taxon_list_id', 'language', 'taxon_group_id', 'taxon_group', 'family_taxa_taxon_list_id',
-        'taxon_meaning_id', 'preferred_taxon', 'external_key', 'taxa_taxon_list_id'];
+    $possibleArrays = [
+      'taxon_list_id',
+      'language',
+      'taxon_group_id',
+      'taxon_group',
+      'family_taxa_taxon_list_id',
+      'taxon_meaning_id',
+      'preferred_taxon',
+      'external_key',
+      'organism_key',
+      'taxa_taxon_list_id',
+    ];
     foreach ($possibleArrays as $possibleArrayParam) {
       if (isset($params[$possibleArrayParam])) {
         $params[$possibleArrayParam] = $this->decodeArrayParameter($params[$possibleArrayParam]);

From 1522bcff19e8247dbb9dca11813267bec72c5eee Mon Sep 17 00:00:00 2001
From: John van Breda <john.vanbreda@biodiverseit.co.uk>
Date: Thu, 30 Mar 2023 14:10:39 +0100
Subject: [PATCH 3/3] Taxon cache tables include organism_key

---
 .../cache_builder/config/cache_builder.php    | 27 +++++++++++--------
 .../202303301226_organism_key.sql             | 11 ++++++++
 .../202303301227_organism_key_populate.sql    | 13 +++++++++
 3 files changed, 40 insertions(+), 11 deletions(-)
 create mode 100644 modules/cache_builder/db/version_8_17_0/202303301226_organism_key.sql
 create mode 100644 modules/cache_builder/db/version_8_17_0/202303301227_organism_key_populate.sql

diff --git a/modules/cache_builder/config/cache_builder.php b/modules/cache_builder/config/cache_builder.php
index c384e048f9..4c4eb85f03 100644
--- a/modules/cache_builder/config/cache_builder.php
+++ b/modules/cache_builder/config/cache_builder.php
@@ -234,6 +234,7 @@
       default_common_name=tcommon.taxon,
       search_name=regexp_replace(regexp_replace(regexp_replace(lower(t.taxon), E'\\\\(.+\\\\)', '', 'g'), 'ae', 'e', 'g'), E'[^a-z0-9\\\\?\\\\+]', '', 'g'),
       external_key=tpref.external_key,
+      organism_key=tpref.organism_key,
       taxon_meaning_id=ttlpref.taxon_meaning_id,
       taxon_group_id = tpref.taxon_group_id,
       taxon_group = tg.title,
@@ -270,8 +271,8 @@
       id, preferred, taxon_list_id, taxon_list_title, website_id,
       preferred_taxa_taxon_list_id, parent_id, taxonomic_sort_order,
       taxon, authority, language_iso, language, preferred_taxon, preferred_authority,
-      preferred_language_iso, preferred_language, default_common_name, search_name, external_key,
-      taxon_meaning_id, taxon_group_id, taxon_group,
+      preferred_language_iso, preferred_language, default_common_name, search_name,
+      external_key, organism_key, taxon_meaning_id, taxon_group_id, taxon_group,
       taxon_rank_id, taxon_rank, taxon_rank_sort_order,
       cache_created_on, cache_updated_on, allow_data_entry,
       marine_flag, freshwater_flag, terrestrial_flag, non_native_flag,
@@ -286,7 +287,7 @@
       lpref.iso as preferred_language_iso, lpref.language as preferred_language,
       tcommon.taxon as default_common_name,
       regexp_replace(regexp_replace(regexp_replace(lower(t.taxon), E'\\\\(.+\\\\)', '', 'g'), 'ae', 'e', 'g'), E'[^a-z0-9\\\\?\\\\+]', '', 'g'),
-      tpref.external_key, ttlpref.taxon_meaning_id, tpref.taxon_group_id, tg.title,
+      tpref.external_key, tpref.organism_key, ttlpref.taxon_meaning_id, tpref.taxon_group_id, tg.title,
       tr.id, tr.rank, tr.sort_order,
       now(), now(), ttl.allow_data_entry,
       t.marine_flag, t.freshwater_flag, t.terrestrial_flag, t.non_native_flag,
@@ -570,6 +571,7 @@
       terrestrial_flag=cttl.terrestrial_flag,
       non_native_flag=cttl.non_native_flag,
       external_key=cttl.external_key,
+      organism_key=cttl.organism_key,
       authority=cttl.authority,
       search_code=cttl.search_code,
       taxonomic_sort_order=cttl.taxonomic_sort_order,
@@ -607,6 +609,7 @@
       terrestrial_flag=cttl.terrestrial_flag,
       non_native_flag=cttl.non_native_flag,
       external_key=cttl.external_key,
+      organism_key=cttl.organism_key,
       authority=cttl.authority,
       search_code=cttl.search_code,
       taxonomic_sort_order=cttl.taxonomic_sort_order,
@@ -649,6 +652,7 @@
       terrestrial_flag=cttl.terrestrial_flag,
       non_native_flag=cttl.non_native_flag,
       external_key=cttl.external_key,
+      organism_key=cttl.organism_key,
       authority=cttl.authority,
       search_code=cttl.search_code,
       taxonomic_sort_order=cttl.taxonomic_sort_order,
@@ -682,6 +686,7 @@
       terrestrial_flag=cttl.terrestrial_flag,
       non_native_flag=cttl.non_native_flag,
       external_key=cttl.external_key,
+      organism_key=cttl.organism_key,
       authority=cttl.authority,
       search_code=cttl.search_code,
       taxonomic_sort_order=cttl.taxonomic_sort_order,
@@ -708,7 +713,7 @@
       default_common_name, preferred_authority, language_iso,
       name_type, simplified, code_type_id, preferred, searchterm_length, parent_id, preferred_taxa_taxon_list_id,
       marine_flag, freshwater_flag, terrestrial_flag, non_native_flag,
-      external_key, authority, search_code, taxonomic_sort_order, taxon_rank
+      external_key, organism_key, authority, search_code, taxonomic_sort_order, taxon_rank
     )
     select distinct on (cttl.id) cttl.id, cttl.taxon_list_id, cttl.taxon || coalesce(' ' || cttl.authority, ''),
       cttl.taxon, cttl.taxon_group_id, cttl.taxon_group, cttl.taxon_meaning_id,
@@ -719,7 +724,7 @@
         else 'V'
       end, false, null, cttl.preferred, length(cttl.taxon), cttl.parent_id, cttl.preferred_taxa_taxon_list_id,
       cttl.marine_flag, cttl.freshwater_flag, cttl.terrestrial_flag, cttl.non_native_flag,
-      cttl.external_key, cttl.authority, cttl.search_code, cttl.taxonomic_sort_order, cttl.taxon_rank
+      cttl.external_key, cttl.organism_key, cttl.authority, cttl.search_code, cttl.taxonomic_sort_order, cttl.taxon_rank
     from cache_taxa_taxon_lists cttl
     #join_needs_update#
     left join cache_taxon_searchterms cts on cts.taxa_taxon_list_id=cttl.id and cts.name_type in ('L','S','V') and cts.simplified='f'
@@ -729,13 +734,13 @@
       taxa_taxon_list_id, taxon_list_id, searchterm, original, taxon_group_id, taxon_group, taxon_meaning_id, preferred_taxon,
       default_common_name, preferred_authority, language_iso,
       name_type, simplified, code_type_id, preferred, searchterm_length, parent_id, preferred_taxa_taxon_list_id,
-      marine_flag, freshwater_flag, terrestrial_flag, non_native_flag, external_key, authority, search_code, taxonomic_sort_order, taxon_rank
+      marine_flag, freshwater_flag, terrestrial_flag, non_native_flag, external_key, organism_key, authority, search_code, taxonomic_sort_order, taxon_rank
     )
     select distinct on (cttl.id) cttl.id, cttl.taxon_list_id, taxon_abbreviation(cttl.taxon), cttl.taxon, cttl.taxon_group_id, cttl.taxon_group, cttl.taxon_meaning_id, cttl.preferred_taxon,
       cttl.default_common_name, cttl.authority, cttl.language_iso,
       'A', null, null, cttl.preferred, length(taxon_abbreviation(cttl.taxon)), cttl.parent_id, cttl.preferred_taxa_taxon_list_id,
       cttl.marine_flag, cttl.freshwater_flag, cttl.terrestrial_flag, cttl.non_native_flag,
-      cttl.external_key, cttl.authority, cttl.search_code, cttl.taxonomic_sort_order, cttl.taxon_rank
+      cttl.external_key, cttl.organism_key, cttl.authority, cttl.search_code, cttl.taxonomic_sort_order, cttl.taxon_rank
     from cache_taxa_taxon_lists cttl
     #join_needs_update#
     left join cache_taxon_searchterms cts on cts.taxa_taxon_list_id=cttl.id and cts.name_type='A'
@@ -745,7 +750,7 @@
       taxa_taxon_list_id, taxon_list_id, searchterm, original, taxon_group_id, taxon_group, taxon_meaning_id, preferred_taxon,
       default_common_name, preferred_authority, language_iso,
       name_type, simplified, code_type_id, preferred, searchterm_length, parent_id, preferred_taxa_taxon_list_id,
-      marine_flag, freshwater_flag, terrestrial_flag, non_native_flag, external_key, authority, search_code, taxonomic_sort_order, taxon_rank
+      marine_flag, freshwater_flag, terrestrial_flag, non_native_flag, external_key, organism_key, authority, search_code, taxonomic_sort_order, taxon_rank
     )
     select distinct on (cttl.id) cttl.id, cttl.taxon_list_id,
       regexp_replace(lower(
@@ -762,7 +767,7 @@
           regexp_replace(regexp_replace(cttl.taxon, E'\\\\(.+\\\\)', '', 'g') || coalesce(cttl.authority, ''), 'ae', 'e', 'g')
         ), E'[^a-z0-9\\\\?\\\\+]', '', 'g')),
       cttl.parent_id, cttl.preferred_taxa_taxon_list_id,
-      cttl.marine_flag, cttl.freshwater_flag, cttl.terrestrial_flag, cttl.non_native_flag, cttl.external_key, cttl.authority,
+      cttl.marine_flag, cttl.freshwater_flag, cttl.terrestrial_flag, cttl.non_native_flag, cttl.external_key, cttl.organism_key, cttl.authority,
       cttl.search_code, cttl.taxonomic_sort_order, cttl.taxon_rank
     from cache_taxa_taxon_lists cttl
     #join_needs_update#
@@ -775,13 +780,13 @@
       name_type, simplified, code_type_id, source_id, preferred, searchterm_length,
       parent_id, preferred_taxa_taxon_list_id,
       marine_flag, freshwater_flag, terrestrial_flag, non_native_flag,
-      external_key, authority, search_code, taxonomic_sort_order, taxon_rank
+      external_key, organism_key, authority, search_code, taxonomic_sort_order, taxon_rank
     )
     select distinct on (tc.id) cttl.id, cttl.taxon_list_id, tc.code, tc.code, cttl.taxon_group_id, cttl.taxon_group, cttl.taxon_meaning_id, cttl.preferred_taxon,
       cttl.default_common_name, cttl.authority, null, 'C', null, tc.code_type_id, tc.id, cttl.preferred, length(tc.code),
       cttl.parent_id, cttl.preferred_taxa_taxon_list_id,
       cttl.marine_flag, cttl.freshwater_flag, cttl.terrestrial_flag, cttl.non_native_flag,
-      cttl.external_key, cttl.authority, cttl.search_code, cttl.taxonomic_sort_order, cttl.taxon_rank
+      cttl.external_key, cttl.organism_key, cttl.authority, cttl.search_code, cttl.taxonomic_sort_order, cttl.taxon_rank
     from cache_taxa_taxon_lists cttl
     #join_needs_update#
     join taxon_codes tc on tc.taxon_meaning_id=cttl.taxon_meaning_id and tc.deleted=false
diff --git a/modules/cache_builder/db/version_8_17_0/202303301226_organism_key.sql b/modules/cache_builder/db/version_8_17_0/202303301226_organism_key.sql
new file mode 100644
index 0000000000..7f7e3ac0f7
--- /dev/null
+++ b/modules/cache_builder/db/version_8_17_0/202303301226_organism_key.sql
@@ -0,0 +1,11 @@
+ALTER TABLE cache_taxa_taxon_lists
+  ADD COLUMN organism_key varchar;
+
+COMMENT ON COLUMN cache_taxa_taxon_lists.organism_key
+    IS 'Identifier for the organism concept, e.g. when linking to UKSI.';
+
+ALTER TABLE cache_taxon_searchterms
+  ADD COLUMN organism_key varchar;
+
+COMMENT ON COLUMN cache_taxon_searchterms.organism_key
+    IS 'Identifier for the organism concept, e.g. when linking to UKSI.';
\ No newline at end of file
diff --git a/modules/cache_builder/db/version_8_17_0/202303301227_organism_key_populate.sql b/modules/cache_builder/db/version_8_17_0/202303301227_organism_key_populate.sql
new file mode 100644
index 0000000000..582567605c
--- /dev/null
+++ b/modules/cache_builder/db/version_8_17_0/202303301227_organism_key_populate.sql
@@ -0,0 +1,13 @@
+-- #slow script#
+
+UPDATE cache_taxa_taxon_lists cttl
+SET organism_key=t.organism_key
+FROM taxa t
+WHERE t.id=cttl.taxon_id
+AND t.organism_key IS NOT NULL;
+
+UPDATE cache_taxon_searchterms cts
+SET organism_key=cttl.organism_key
+FROM cache_taxa_taxon_lists cttl
+WHERE cttl.id=cts.taxa_taxon_list_id
+AND cttl.organism_key IS NOT NULL;
\ No newline at end of file