Merged in r2-3050-searchable-identifiers (pull request #6955)

R2-3050 - Add searchable_identifiers table to search records

app/models/concerns/phonetic_searchable.rb

@@ -3,13 +3,19 @@
 # Copyright (c) 2014 - 2023 UNICEF. All rights reserved.
 
 # Concern that allow records to be searchable through phonetics
+# TODO: Once the Searchable concern is deprecated this concern will be renamed to Searchable
 module PhoneticSearchable
   extend ActiveSupport::Concern
 
   included do
+    has_many :searchable_identifiers, as: :record
     store_accessor :phonetic_data, :tokens
 
+    accepts_nested_attributes_for :searchable_identifiers
+
     before_save :recalculate_phonetic_tokens
+    before_create :recalculate_searchable_identifiers
+    before_update :recalculate_searchable_identifiers
   end
 
   # Class methods to indicate the phonetic_field_names of a record
@@ -25,6 +31,36 @@ def recalculate_phonetic_tokens
     self.tokens = generate_tokens
   end
 
+  def recalculate_searchable_identifiers
+    return unless filterable_id_fields_to_save.present?
+
+    identifiers_to_save = identifiers_to_update
+
+    filterable_id_fields_to_save.each do |field_name|
+      next if data[field_name].blank? || identifiers_to_save.any? { |identifier| identifier[:field_name] == field_name }
+
+      identifiers_to_save << { field_name:, value: data[field_name] }
+    end
+
+    self.searchable_identifiers_attributes = identifiers_to_save
+  end
+
+  def identifiers_to_update
+    return [] unless filterable_id_fields_to_save.present?
+
+    searchable_identifiers.map do |searchable_identifier|
+      {
+        field_name: searchable_identifier.field_name,
+        value: data[searchable_identifier.field_name],
+        id: searchable_identifier.id
+      }
+    end
+  end
+
+  def filterable_id_fields_to_save
+    changes_to_save_for_record.keys & self.class.filterable_id_fields
+  end
+
   def generate_tokens
     self.class.phonetic_field_names.reduce([]) do |memo, field_name|
       value = data[field_name]

app/models/searchable_identifier.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2014 - 2023 UNICEF. All rights reserved.
+
+# Class for SearchableIdentifier
+class SearchableIdentifier < ApplicationRecord
+  belongs_to :record, polymorphic: true
+end

app/services/data_removal_service.rb

@@ -6,7 +6,8 @@
 class DataRemovalService
   RECORD_MODELS = [Child, Incident, TracingRequest].freeze
   DATA_MODELS = [
-    Trace, Flag, Alert, Attachment, AuditLog, BulkExport, RecordHistory, SavedSearch, Transition, Violation
+    Trace, Flag, Alert, Attachment, AuditLog, BulkExport, RecordHistory, SavedSearch, Transition, Violation,
+    SearchableIdentifier
   ].freeze
   METADATA_MODELS = [
     Agency, ContactInformation, Field, FormSection, Location, Lookup, PrimeroModule, PrimeroProgram, Report, Role,

app/services/model_deletion_service.rb

@@ -5,7 +5,7 @@
 # Service to handle deletion of data for a specific model and its associations
 # rubocop:disable Metrics/ClassLength
 class ModelDeletionService < ValueObject
-  UUID_REFERENCED_MODELS = [Alert, Attachment, Trace, Violation].freeze
+  UUID_REFERENCED_MODELS = [Alert, Attachment, Trace, Violation, SearchableIdentifier].freeze
   attr_accessor :model_class
 
   def delete_records!(query)

app/services/search/search_query.rb

@@ -47,13 +47,13 @@ def phonetic(value)
   def filter_ids(value)
     return self unless value.present?
 
-    filterable_id_queries = record_class.filterable_id_fields.map do |id_field|
-      ActiveRecord::Base.sanitize_sql_for_conditions(
-        ['data->>:id_field ILIKE :value', { id_field:, value: "#{ActiveRecord::Base.sanitize_sql_like(value)}%" }]
+    @query = @query.where(
+      'id in (:records)',
+      records: SearchableIdentifier.select('record_id').where(record_type: record_class.name).where(
+        'value ilike :value', value: "#{ActiveRecord::Base.sanitize_sql_like(value)}%"
       )
-    end
+    )
 
-    @query = @query.where("(#{filterable_id_queries.join(' OR ')})")
     self
   end
 

db/data_migration/v2.11/README.md

@@ -19,6 +19,11 @@ returns reports which contain location fields that will be updated:
 rails r ./db/data_migration/v2.11/migrate_location_fields.rb
 ```
 
+returns the searchable identifiers to be generated:
+```bash
+rails r ./db/data_migration/v2.11/calculate_searchable_identifiers.rb Child false file/path.txt
+```
+
 ## Executing scripts
 Once you validate that the info is correct you can execute the script to modify the data using:
 
@@ -29,6 +34,11 @@ rails r ./db/data_migration/v2.11/calculate_solr_fields.rb Child true file/path.
 ```bash
 rails r ./db/data_migration/v2.11/migrate_location_fields.rb true
 ```
+
+```bash
+rails r ./db/data_migration/v2.11/calculate_searchable_identifiers.rb  Child true file/path.txt
+```
+
 Remove stale solr jobs. pass until what date you want to delete them.
 ```bash
 rails r ./db/data_migration/v2.11/remove_solr_jobs.rb 2024-09-30

db/data_migration/v2.11/calculate_searchable_identifiers.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2014 - 2023 UNICEF. All rights reserved.
+
+# Example of usage:
+# rails r bin/calculate_solr_fields Child,Incident,TracingRequest true file/path.txt
+
+def print_log(message)
+  message = "[#{DateTime.now.strftime('%m/%d/%Y %H:%M')}]: #{message}"
+  puts message
+end
+
+models = (ARGV[0] || '').split(',')
+save_records = ARGV[1] == 'true'
+file_path = ARGV[2]
+
+return unless models.present?
+
+def records_to_process(model_class, ids_file_path)
+  return model_class unless ids_file_path.present?
+
+  print_log("Loading record ids from #{ids_file_path}...")
+  ids_to_update = File.read(ids_file_path).split
+  model_class.where(id: ids_to_update)
+end
+
+models.map(&:constantize).each do |model|
+  records_to_process(model, file_path).find_in_batches(batch_size: 1000).with_index do |records, batch|
+    print_log("Process #{model.name} batch #{batch}...")
+    records.each do |record|
+      if save_records
+        current_size = record.searchable_identifiers.size
+        record.generate_searchable_identifiers
+        generated_size = record.searchable_identifiers.size
+        total = generated_size - current_size
+        print_log("#{total} Searchable Identifiers generated for record_type: #{model.name}, record_id: #{record.id}")
+      elsif record.searchable_identifiers.present?
+        identifiers_size = record.searchable_identifiers.size
+        print_log("record_type: #{model.name}, record_id: #{record.id} has #{identifiers_size} Searchable Identifiers")
+      else
+        print_log("record_type: #{model.name}, record_id: #{record.id} does not have Searchable Identifiers")
+      end
+    end
+  end
+end

db/migrate/20241009000000_create_searchable_identifiers.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2014 - 2023 UNICEF. All rights reserved.
+class CreateSearchableIdentifiers < ActiveRecord::Migration[6.1]
+  enable_extension 'pg_trgm' unless ENV['PRIMERO_PG_APP_ROLE'] || extension_enabled?('pg_trgm')
+
+  def change
+    create_table :searchable_identifiers do |t|
+      t.references :record, polymorphic: true, type: :uuid
+      t.string 'field_name'
+      t.string 'value'
+    end
+
+    add_index :searchable_identifiers,
+              :value,
+              using: :gin,
+              opclass: :gin_trgm_ops,
+              name: 'searchable_identifiers_value_idx'
+  end
+end

db/schema.rb

@@ -10,10 +10,11 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2024_03_06_154915) do
+ActiveRecord::Schema.define(version: 2024_10_09_000000) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "ltree"
+  enable_extension "pg_trgm"
   enable_extension "pgcrypto"
   enable_extension "plpgsql"
 
@@ -521,6 +522,15 @@
     t.index ["user_id"], name: "index_saved_searches_on_user_id"
   end
 
+  create_table "searchable_identifiers", force: :cascade do |t|
+    t.string "record_type"
+    t.uuid "record_id"
+    t.string "field_name"
+    t.string "value"
+    t.index ["record_type", "record_id"], name: "index_searchable_identifiers_on_record"
+    t.index ["value"], name: "searchable_identifiers_value_idx", opclass: :gin_trgm_ops, using: :gin
+  end
+
   create_table "sources", id: :uuid, default: -> { "gen_random_uuid()" }, force: :cascade do |t|
     t.jsonb "data", default: {}
     t.index ["data"], name: "index_sources_on_data", using: :gin