More refactoring

app/lib/smooch_nlu.rb

@@ -9,10 +9,7 @@ class SmoochBotNotInstalledError < ::ArgumentError
     Bot::Alegre::MEAN_TOKENS_MODEL => 0.6
   }
 
-  ALEGRE_CONTEXT_KEY = {
-    menu: 'smooch_nlu_menu',
-    resource: 'smooch_nlu_resource'
-  }
+  include SmoochNluMenus
 
   def initialize(team_slug)
     @team_slug = team_slug
@@ -32,87 +29,40 @@ def enabled?
     !!@smooch_bot_installation.get_nlu_enabled
   end
 
-  def add_keyword_to_menu_option(language, menu, menu_option_index, keyword)
-    update_menu_option_keywords(language, menu, menu_option_index, keyword, 'add')
-  end
-
-  def remove_keyword_from_menu_option(language, menu, menu_option_index, keyword)
-    update_menu_option_keywords(language, menu, menu_option_index, keyword, 'remove')
-  end
-
-  def list_menu_keywords(languages = nil, menus = nil)
-    if languages.nil?
-      languages = @smooch_bot_installation.get_smooch_workflows.map { |w| w['smooch_workflow_language'] }
-    elsif languages.is_a? String
-      languages = [languages]
-    end
-    if menus.nil?
-      menus = ['main', 'secondary']
-    elsif menus.is_a? String
-      menus = [menus]
-    end
-
-    output = {}
-    languages.each do |language|
-      output[language] = {}
-      workflow = @smooch_bot_installation.get_smooch_workflows.find { |w| w['smooch_workflow_language'] == language }
-      menus.each do |menu|
-        output[language][menu] = []
-        i = 0
-        workflow.fetch("smooch_state_#{menu}",{}).fetch('smooch_menu_options', []).each do |option|
-          output[language][menu] << {
-            'index' => i,
-            'title' => option.dig('smooch_menu_option_label'),
-            'keywords' => option.dig('smooch_menu_option_nlu_keywords').to_a,
-            'id' => option.dig('smooch_menu_option_id'),
-          }
-          i += 1
-        end
-      end
-    end
-    output
-  end
-
-  def self.menu_option_from_message(message, language, options)
+  def self.alegre_matches_from_message(message, language, context, alegre_result_key)
     # FIXME: Raise exception if not in a tipline context (so, if Bot::Smooch.config is nil)
-    option = nil
+    matches = []
     team_slug = Team.find(Bot::Smooch.config['team_id']).slug
     params = nil
     response = nil
-    if Bot::Smooch.config.to_h['nlu_enabled'] && !options.nil?
-      # FIXME: In the future we could consider menus across all languages when options is nil
+    if Bot::Smooch.config.to_h['nlu_enabled']
+      # FIXME: In the future we could consider matches across all languages when options is nil
       # FIXME: No need to call Alegre if it's an exact match to one of the keywords
       # FIXME: No need to call Alegre if message has no word characters
       # FIXME: Handle error responses from Alegre
       params = {
         text: message,
-        models: ALEGRE_MODELS_AND_THRESHOLDS.keys,
-        per_model_threshold: ALEGRE_MODELS_AND_THRESHOLDS,
+        models: SmoochNlu::ALEGRE_MODELS_AND_THRESHOLDS.keys,
+        per_model_threshold: SmoochNlu::ALEGRE_MODELS_AND_THRESHOLDS,
         context: {
-          context: ALEGRE_CONTEXT_KEY[:menu],
           team: team_slug,
           language: language,
-        }
+        }.merge(context)
       }
       response = Bot::Alegre.request_api('get', '/text/similarity/', params)
 
       # One approach would be to take the option that has the most matches
       # Unfortunately this approach is influenced by the number of keywords per option
       # So, we are not using this approach right now
-      # Get the menu_option_id of all results returned
-      # option_counts = response['result'].to_a.map{|o| o.dig('_source', 'context', 'menu_option_id')}
-      # Count how many of each menu_option_id we have and sort (high to low)
+      # Get the `alegre_result_key` of all results returned
+      # option_counts = response['result'].to_a.map{|o| o.dig('_source', 'context', alegre_result_key)}
+      # Count how many of each alegre_result_key we have and sort (high to low)
       # ranked_options = option_counts.group_by(&:itself).transform_values(&:count).sort_by{|_k,v| v}.reverse()
 
       # Second approach is to sort the results from best to worst
-      sorted_options = response['result'].to_a.sort_by{ |result| result['_score'] }.reverse()
-      ranked_options = sorted_options.map{|o| o.dig('_source', 'context', 'menu_option_id')}
-
-      # Select the top menu option that exists in `options`
-      ranked_options.each do | r |
-        option = options.find{ |o| !o['smooch_menu_option_id'].blank? && o['smooch_menu_option_id'] == r }
-        break if !option.nil?
-      end
+      sorted_options = response['result'].to_a.sort_by{ |result| result['_score'] }.reverse
+      ranked_options = sorted_options.map{ |o| o.dig('_source', 'context', alegre_result_key) }
+      matches = ranked_options
 
       # FIXME: Deal with ties (i.e., where two options have an equal _score or count)
     end
@@ -124,10 +74,10 @@ def self.menu_option_from_message(message, language, options)
       user_query: message,
       alegre_query: params,
       alegre_response: response,
-      selected_option: option
+      matches: matches
     }
-    Rails.logger.info("[Smooch NLU] [Menu Option From Message] #{log.to_json}")
-    option
+    Rails.logger.info("[Smooch NLU] [Matches From Message] #{log.to_json}")
+    matches
   end
 
   private
@@ -138,27 +88,15 @@ def toggle!(enabled)
     @smooch_bot_installation.reload
   end
 
-  # "menu" is "main" or "secondary"
-  # "operation" is "add" or "remove"
-  # FIXME: Validate the two things above
-  def update_menu_option_keywords(language, menu, menu_option_index, keyword, operation)
+  def update_keywords(language, keywords, keyword, operation, doc_id, context)
     alegre_operation = nil
     alegre_params = nil
-    workflow = @smooch_bot_installation.get_smooch_workflows.find { |w| w['smooch_workflow_language'] == language }
-    keywords = workflow["smooch_state_#{menu}"]['smooch_menu_options'][menu_option_index]['smooch_menu_option_nlu_keywords'].to_a
-    # Make sure there is a unique identifier for this menu option
-    # FIXME: This whole thing should be a model :(
-    menu_option_id = (workflow["smooch_state_#{menu}"]['smooch_menu_options'][menu_option_index]['smooch_menu_option_id'] ||= SecureRandom.uuid)
-    doc_id = Digest::MD5.hexdigest([ALEGRE_CONTEXT_KEY[:menu], @team_slug, menu, menu_option_id, keyword].join(':'))
     common_alegre_params = {
       doc_id: doc_id,
       context: {
-        context: ALEGRE_CONTEXT_KEY[:menu],
         team: @team_slug,
-        language: language,
-        menu: menu,
-        menu_option_id: menu_option_id
-      }
+        language: language
+      }.merge(context)
     }
     if operation == 'add' && !keywords.include?(keyword)
       keywords << keyword
@@ -169,10 +107,8 @@ def update_menu_option_keywords(language, menu, menu_option_index, keyword, oper
       alegre_operation = 'delete'
       alegre_params = common_alegre_params.merge({ quiet: true })
     end
-    workflow["smooch_state_#{menu}"]['smooch_menu_options'][menu_option_index]['smooch_menu_option_nlu_keywords'] = keywords
-    @smooch_bot_installation.save!
-    @smooch_bot_installation.reload
     # FIXME: Add error handling and better logging
     Bot::Alegre.request_api(alegre_operation, '/text/similarity/', alegre_params) if alegre_operation && alegre_params
+    keywords
   end
 end

app/lib/smooch_nlu_menus.rb

@@ -0,0 +1,85 @@
+module SmoochNluMenus
+  ALEGRE_CONTEXT_KEY_MENU = 'smooch_nlu_menu'
+
+  def self.included(base)
+    base.extend(ClassMethods)
+  end
+
+  def add_keyword_to_menu_option(language, menu, menu_option_index, keyword)
+    update_menu_option_keywords(language, menu, menu_option_index, keyword, 'add')
+  end
+
+  def remove_keyword_from_menu_option(language, menu, menu_option_index, keyword)
+    update_menu_option_keywords(language, menu, menu_option_index, keyword, 'remove')
+  end
+
+  def list_menu_keywords(languages = nil, menus = nil)
+    if languages.nil?
+      languages = @smooch_bot_installation.get_smooch_workflows.map { |w| w['smooch_workflow_language'] }
+    elsif languages.is_a? String
+      languages = [languages]
+    end
+    if menus.nil?
+      menus = ['main', 'secondary']
+    elsif menus.is_a? String
+      menus = [menus]
+    end
+
+    output = {}
+    languages.each do |language|
+      output[language] = {}
+      workflow = @smooch_bot_installation.get_smooch_workflows.find { |w| w['smooch_workflow_language'] == language }
+      menus.each do |menu|
+        output[language][menu] = []
+        i = 0
+        workflow.fetch("smooch_state_#{menu}",{}).fetch('smooch_menu_options', []).each do |option|
+          output[language][menu] << {
+            'index' => i,
+            'title' => option.dig('smooch_menu_option_label'),
+            'keywords' => option.dig('smooch_menu_option_nlu_keywords').to_a,
+            'id' => option.dig('smooch_menu_option_id'),
+          }
+          i += 1
+        end
+      end
+    end
+    output
+  end
+
+  # "menu" is "main" or "secondary"
+  # "operation" is "add" or "remove"
+  # FIXME: Validate the two things above
+  def update_menu_option_keywords(language, menu, menu_option_index, keyword, operation)
+    workflow = @smooch_bot_installation.get_smooch_workflows.find { |w| w['smooch_workflow_language'] == language }
+    keywords = workflow["smooch_state_#{menu}"]['smooch_menu_options'][menu_option_index]['smooch_menu_option_nlu_keywords'].to_a
+    menu_option_id = (workflow["smooch_state_#{menu}"]['smooch_menu_options'][menu_option_index]['smooch_menu_option_id'] ||= SecureRandom.uuid)
+    doc_id = Digest::MD5.hexdigest([ALEGRE_CONTEXT_KEY_MENU, @team_slug, menu, menu_option_id, keyword].join(':'))
+    context = {
+      context: ALEGRE_CONTEXT_KEY_MENU,
+      menu: menu,
+      menu_option_id: menu_option_id
+    }
+    new_keywords = update_keywords(language, keywords, keyword, operation, doc_id, context)
+    workflow["smooch_state_#{menu}"]['smooch_menu_options'][menu_option_index]['smooch_menu_option_nlu_keywords'] = new_keywords
+    @smooch_bot_installation.save!
+    @smooch_bot_installation.reload
+  end
+
+  module ClassMethods
+    def menu_option_from_message(message, language, options)
+      return nil if options.blank?
+      option = nil
+      context = {
+        context: ALEGRE_CONTEXT_KEY_MENU
+      }
+      matches = SmoochNlu.alegre_matches_from_message(message, language, context, 'menu_option_id')
+      # Select the top menu option that exists in `options`
+      matches.each do |r|
+        option = options.find{ |o| !o['smooch_menu_option_id'].blank? && o['smooch_menu_option_id'] == r }
+        break unless option.nil?
+      end
+      Rails.logger.info("[Smooch NLU] [Menu Option From Message] Menu option: #{option} | Message: #{message}")
+      option
+    end
+  end
+end

db/migrate/20230922174044_add_keywords_to_tipline_resources.rb

@@ -0,0 +1,5 @@
+class AddKeywordsToTiplineResources < ActiveRecord::Migration[6.1]
+  def change
+    add_column :tipline_resources, :keywords, :string, array: true, default: []
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2023_09_14_152816) do
+ActiveRecord::Schema.define(version: 2023_09_22_174044) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -261,7 +261,7 @@
     t.jsonb "value_json", default: "{}"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
-    t.index "dynamic_annotation_fields_value(field_name, value)", name: "dynamic_annotation_fields_value", where: "((field_name)::text = ANY ((ARRAY['external_id'::character varying, 'smooch_user_id'::character varying, 'verification_status_status'::character varying])::text[]))"
+    t.index "dynamic_annotation_fields_value(field_name, value)", name: "dynamic_annotation_fields_value", where: "((field_name)::text = ANY (ARRAY[('external_id'::character varying)::text, ('smooch_user_id'::character varying)::text, ('verification_status_status'::character varying)::text]))"
     t.index ["annotation_id", "field_name"], name: "index_dynamic_annotation_fields_on_annotation_id_and_field_name"
     t.index ["annotation_id"], name: "index_dynamic_annotation_fields_on_annotation_id"
     t.index ["annotation_type"], name: "index_dynamic_annotation_fields_on_annotation_type"
@@ -700,6 +700,7 @@
     t.string "header_file"
     t.string "header_overlay_text"
     t.string "header_media_url"
+    t.string "keywords", default: [], array: true
     t.index ["team_id"], name: "index_tipline_resources_on_team_id"
     t.index ["uuid"], name: "index_tipline_resources_on_uuid", unique: true
   end