diff --git a/app/lib/smooch_nlu.rb b/app/lib/smooch_nlu.rb index 8ac5878de2..35774975b9 100644 --- a/app/lib/smooch_nlu.rb +++ b/app/lib/smooch_nlu.rb @@ -9,10 +9,7 @@ class SmoochBotNotInstalledError < ::ArgumentError Bot::Alegre::MEAN_TOKENS_MODEL => 0.6 } - ALEGRE_CONTEXT_KEY = { - menu: 'smooch_nlu_menu', - resource: 'smooch_nlu_resource' - } + include SmoochNluMenus def initialize(team_slug) @team_slug = team_slug @@ -32,87 +29,40 @@ def enabled? !!@smooch_bot_installation.get_nlu_enabled end - def add_keyword_to_menu_option(language, menu, menu_option_index, keyword) - update_menu_option_keywords(language, menu, menu_option_index, keyword, 'add') - end - - def remove_keyword_from_menu_option(language, menu, menu_option_index, keyword) - update_menu_option_keywords(language, menu, menu_option_index, keyword, 'remove') - end - - def list_menu_keywords(languages = nil, menus = nil) - if languages.nil? - languages = @smooch_bot_installation.get_smooch_workflows.map { |w| w['smooch_workflow_language'] } - elsif languages.is_a? String - languages = [languages] - end - if menus.nil? - menus = ['main', 'secondary'] - elsif menus.is_a? String - menus = [menus] - end - - output = {} - languages.each do |language| - output[language] = {} - workflow = @smooch_bot_installation.get_smooch_workflows.find { |w| w['smooch_workflow_language'] == language } - menus.each do |menu| - output[language][menu] = [] - i = 0 - workflow.fetch("smooch_state_#{menu}",{}).fetch('smooch_menu_options', []).each do |option| - output[language][menu] << { - 'index' => i, - 'title' => option.dig('smooch_menu_option_label'), - 'keywords' => option.dig('smooch_menu_option_nlu_keywords').to_a, - 'id' => option.dig('smooch_menu_option_id'), - } - i += 1 - end - end - end - output - end - - def self.menu_option_from_message(message, language, options) + def self.alegre_matches_from_message(message, language, context, alegre_result_key) # FIXME: Raise exception if not in a tipline context (so, if Bot::Smooch.config is nil) - option = nil + matches = [] team_slug = Team.find(Bot::Smooch.config['team_id']).slug params = nil response = nil - if Bot::Smooch.config.to_h['nlu_enabled'] && !options.nil? - # FIXME: In the future we could consider menus across all languages when options is nil + if Bot::Smooch.config.to_h['nlu_enabled'] + # FIXME: In the future we could consider matches across all languages when options is nil # FIXME: No need to call Alegre if it's an exact match to one of the keywords # FIXME: No need to call Alegre if message has no word characters # FIXME: Handle error responses from Alegre params = { text: message, - models: ALEGRE_MODELS_AND_THRESHOLDS.keys, - per_model_threshold: ALEGRE_MODELS_AND_THRESHOLDS, + models: SmoochNlu::ALEGRE_MODELS_AND_THRESHOLDS.keys, + per_model_threshold: SmoochNlu::ALEGRE_MODELS_AND_THRESHOLDS, context: { - context: ALEGRE_CONTEXT_KEY[:menu], team: team_slug, language: language, - } + }.merge(context) } response = Bot::Alegre.request_api('get', '/text/similarity/', params) # One approach would be to take the option that has the most matches # Unfortunately this approach is influenced by the number of keywords per option # So, we are not using this approach right now - # Get the menu_option_id of all results returned - # option_counts = response['result'].to_a.map{|o| o.dig('_source', 'context', 'menu_option_id')} - # Count how many of each menu_option_id we have and sort (high to low) + # Get the `alegre_result_key` of all results returned + # option_counts = response['result'].to_a.map{|o| o.dig('_source', 'context', alegre_result_key)} + # Count how many of each alegre_result_key we have and sort (high to low) # ranked_options = option_counts.group_by(&:itself).transform_values(&:count).sort_by{|_k,v| v}.reverse() # Second approach is to sort the results from best to worst - sorted_options = response['result'].to_a.sort_by{ |result| result['_score'] }.reverse() - ranked_options = sorted_options.map{|o| o.dig('_source', 'context', 'menu_option_id')} - - # Select the top menu option that exists in `options` - ranked_options.each do | r | - option = options.find{ |o| !o['smooch_menu_option_id'].blank? && o['smooch_menu_option_id'] == r } - break if !option.nil? - end + sorted_options = response['result'].to_a.sort_by{ |result| result['_score'] }.reverse + ranked_options = sorted_options.map{ |o| o.dig('_source', 'context', alegre_result_key) } + matches = ranked_options # FIXME: Deal with ties (i.e., where two options have an equal _score or count) end @@ -124,10 +74,10 @@ def self.menu_option_from_message(message, language, options) user_query: message, alegre_query: params, alegre_response: response, - selected_option: option + matches: matches } - Rails.logger.info("[Smooch NLU] [Menu Option From Message] #{log.to_json}") - option + Rails.logger.info("[Smooch NLU] [Matches From Message] #{log.to_json}") + matches end private @@ -138,27 +88,15 @@ def toggle!(enabled) @smooch_bot_installation.reload end - # "menu" is "main" or "secondary" - # "operation" is "add" or "remove" - # FIXME: Validate the two things above - def update_menu_option_keywords(language, menu, menu_option_index, keyword, operation) + def update_keywords(language, keywords, keyword, operation, doc_id, context) alegre_operation = nil alegre_params = nil - workflow = @smooch_bot_installation.get_smooch_workflows.find { |w| w['smooch_workflow_language'] == language } - keywords = workflow["smooch_state_#{menu}"]['smooch_menu_options'][menu_option_index]['smooch_menu_option_nlu_keywords'].to_a - # Make sure there is a unique identifier for this menu option - # FIXME: This whole thing should be a model :( - menu_option_id = (workflow["smooch_state_#{menu}"]['smooch_menu_options'][menu_option_index]['smooch_menu_option_id'] ||= SecureRandom.uuid) - doc_id = Digest::MD5.hexdigest([ALEGRE_CONTEXT_KEY[:menu], @team_slug, menu, menu_option_id, keyword].join(':')) common_alegre_params = { doc_id: doc_id, context: { - context: ALEGRE_CONTEXT_KEY[:menu], team: @team_slug, - language: language, - menu: menu, - menu_option_id: menu_option_id - } + language: language + }.merge(context) } if operation == 'add' && !keywords.include?(keyword) keywords << keyword @@ -169,10 +107,8 @@ def update_menu_option_keywords(language, menu, menu_option_index, keyword, oper alegre_operation = 'delete' alegre_params = common_alegre_params.merge({ quiet: true }) end - workflow["smooch_state_#{menu}"]['smooch_menu_options'][menu_option_index]['smooch_menu_option_nlu_keywords'] = keywords - @smooch_bot_installation.save! - @smooch_bot_installation.reload # FIXME: Add error handling and better logging Bot::Alegre.request_api(alegre_operation, '/text/similarity/', alegre_params) if alegre_operation && alegre_params + keywords end end diff --git a/app/lib/smooch_nlu_menus.rb b/app/lib/smooch_nlu_menus.rb new file mode 100644 index 0000000000..3857f1b6e0 --- /dev/null +++ b/app/lib/smooch_nlu_menus.rb @@ -0,0 +1,85 @@ +module SmoochNluMenus + ALEGRE_CONTEXT_KEY_MENU = 'smooch_nlu_menu' + + def self.included(base) + base.extend(ClassMethods) + end + + def add_keyword_to_menu_option(language, menu, menu_option_index, keyword) + update_menu_option_keywords(language, menu, menu_option_index, keyword, 'add') + end + + def remove_keyword_from_menu_option(language, menu, menu_option_index, keyword) + update_menu_option_keywords(language, menu, menu_option_index, keyword, 'remove') + end + + def list_menu_keywords(languages = nil, menus = nil) + if languages.nil? + languages = @smooch_bot_installation.get_smooch_workflows.map { |w| w['smooch_workflow_language'] } + elsif languages.is_a? String + languages = [languages] + end + if menus.nil? + menus = ['main', 'secondary'] + elsif menus.is_a? String + menus = [menus] + end + + output = {} + languages.each do |language| + output[language] = {} + workflow = @smooch_bot_installation.get_smooch_workflows.find { |w| w['smooch_workflow_language'] == language } + menus.each do |menu| + output[language][menu] = [] + i = 0 + workflow.fetch("smooch_state_#{menu}",{}).fetch('smooch_menu_options', []).each do |option| + output[language][menu] << { + 'index' => i, + 'title' => option.dig('smooch_menu_option_label'), + 'keywords' => option.dig('smooch_menu_option_nlu_keywords').to_a, + 'id' => option.dig('smooch_menu_option_id'), + } + i += 1 + end + end + end + output + end + + # "menu" is "main" or "secondary" + # "operation" is "add" or "remove" + # FIXME: Validate the two things above + def update_menu_option_keywords(language, menu, menu_option_index, keyword, operation) + workflow = @smooch_bot_installation.get_smooch_workflows.find { |w| w['smooch_workflow_language'] == language } + keywords = workflow["smooch_state_#{menu}"]['smooch_menu_options'][menu_option_index]['smooch_menu_option_nlu_keywords'].to_a + menu_option_id = (workflow["smooch_state_#{menu}"]['smooch_menu_options'][menu_option_index]['smooch_menu_option_id'] ||= SecureRandom.uuid) + doc_id = Digest::MD5.hexdigest([ALEGRE_CONTEXT_KEY_MENU, @team_slug, menu, menu_option_id, keyword].join(':')) + context = { + context: ALEGRE_CONTEXT_KEY_MENU, + menu: menu, + menu_option_id: menu_option_id + } + new_keywords = update_keywords(language, keywords, keyword, operation, doc_id, context) + workflow["smooch_state_#{menu}"]['smooch_menu_options'][menu_option_index]['smooch_menu_option_nlu_keywords'] = new_keywords + @smooch_bot_installation.save! + @smooch_bot_installation.reload + end + + module ClassMethods + def menu_option_from_message(message, language, options) + return nil if options.blank? + option = nil + context = { + context: ALEGRE_CONTEXT_KEY_MENU + } + matches = SmoochNlu.alegre_matches_from_message(message, language, context, 'menu_option_id') + # Select the top menu option that exists in `options` + matches.each do |r| + option = options.find{ |o| !o['smooch_menu_option_id'].blank? && o['smooch_menu_option_id'] == r } + break unless option.nil? + end + Rails.logger.info("[Smooch NLU] [Menu Option From Message] Menu option: #{option} | Message: #{message}") + option + end + end +end diff --git a/db/migrate/20230922174044_add_keywords_to_tipline_resources.rb b/db/migrate/20230922174044_add_keywords_to_tipline_resources.rb new file mode 100644 index 0000000000..06b2673ecf --- /dev/null +++ b/db/migrate/20230922174044_add_keywords_to_tipline_resources.rb @@ -0,0 +1,5 @@ +class AddKeywordsToTiplineResources < ActiveRecord::Migration[6.1] + def change + add_column :tipline_resources, :keywords, :string, array: true, default: [] + end +end diff --git a/db/schema.rb b/db/schema.rb index 40e1696ac6..47a8fc682e 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2023_09_14_152816) do +ActiveRecord::Schema.define(version: 2023_09_22_174044) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -261,7 +261,7 @@ t.jsonb "value_json", default: "{}" t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.index "dynamic_annotation_fields_value(field_name, value)", name: "dynamic_annotation_fields_value", where: "((field_name)::text = ANY ((ARRAY['external_id'::character varying, 'smooch_user_id'::character varying, 'verification_status_status'::character varying])::text[]))" + t.index "dynamic_annotation_fields_value(field_name, value)", name: "dynamic_annotation_fields_value", where: "((field_name)::text = ANY (ARRAY[('external_id'::character varying)::text, ('smooch_user_id'::character varying)::text, ('verification_status_status'::character varying)::text]))" t.index ["annotation_id", "field_name"], name: "index_dynamic_annotation_fields_on_annotation_id_and_field_name" t.index ["annotation_id"], name: "index_dynamic_annotation_fields_on_annotation_id" t.index ["annotation_type"], name: "index_dynamic_annotation_fields_on_annotation_type" @@ -700,6 +700,7 @@ t.string "header_file" t.string "header_overlay_text" t.string "header_media_url" + t.string "keywords", default: [], array: true t.index ["team_id"], name: "index_tipline_resources_on_team_id" t.index ["uuid"], name: "index_tipline_resources_on_uuid", unique: true end