Cv2 3435 alegre with presto for audio

app/controllers/api/v1/webhooks_controller.rb

@@ -7,7 +7,8 @@ def index
         bot_name_to_class = {
           smooch: Bot::Smooch,
           keep: Bot::Keep,
-          fetch: Bot::Fetch
+          fetch: Bot::Fetch,
+          alegre: Bot::Alegre,
         }
         unless bot_name_to_class.has_key?(params[:name].to_sym)
           render_error('Bot not found', 'ID_NOT_FOUND', 404) and return

app/models/bot/alegre.rb

@@ -6,6 +6,7 @@ class Error < ::StandardError
   end
 
   include AlegreSimilarity
+  include AlegreWebhooks
 
   # Text similarity models
   MEAN_TOKENS_MODEL = 'xlm-r-bert-base-nli-stsb-mean-tokens'
@@ -126,6 +127,12 @@ def get_language_from_transcription
     end
   end
 
+  def self.valid_request?(request)
+    token = request.params["token"] || request.query_parameters["token"]
+    !token.blank? && token == CheckConfig.get('alegre_token')
+  end
+
+
   def self.default_model
     CheckConfig.get('alegre_default_model') || Bot::Alegre::ELASTICSEARCH_MODEL
   end
@@ -490,7 +497,13 @@ def self.request_api(method, path, params = {}, query_or_body = 'body', retries
       response_body = response.body
       Rails.logger.info("[Alegre Bot] Alegre response: #{response_body.inspect}")
       ActiveRecord::Base.connection.reconnect! if RequestStore.store[:pause_database_connection]
-      JSON.parse(response_body)
+      parsed_response = JSON.parse(response_body)
+      if parsed_response.dig("queue") == 'audio__Model' && parsed_response.dig("body", "callback_url") != nil
+        redis = Redis.new(REDIS_CONFIG)
+        redis_response = redis.blpop("alegre:webhook:#{parsed_response.dig("body", "id")}", 120)
+        return JSON.parse(redis_response[1])
+      end
+      parsed_response
     rescue StandardError => e
       if retries > 0
         sleep 1

app/models/concerns/alegre_similarity.rb

@@ -19,7 +19,7 @@ def get_similar_items(pm)
       type = Bot::Alegre.get_pm_type(pm)
       Rails.logger.info "[Alegre Bot] [ProjectMedia ##{pm.id}] [Similarity 2/5] Type is #{type.blank? ? "blank" : type}"
       unless type.blank?
-        if !self.should_get_similar_items_of_type?('master', pm.team_id) || !self.should_get_similar_items_of_type?(type, pm.team_id)
+        if !Bot::Alegre.should_get_similar_items_of_type?('master', pm.team_id) || !Bot::Alegre.should_get_similar_items_of_type?(type, pm.team_id)
           Rails.logger.info "[Alegre Bot] [ProjectMedia ##{pm.id}] [Similarity 3/5] ProjectMedia cannot be checked for similar items"
           return {}
         else

app/models/concerns/alegre_webhooks.rb

@@ -0,0 +1,18 @@
+class AlegreCallbackError < StandardError
+end
+module AlegreWebhooks
+  extend ActiveSupport::Concern
+  module ClassMethods
+    def webhook(request)
+      begin
+        doc_id = request.params["data"]["requested"]["body"]["id"]
+        redis = Redis.new(REDIS_CONFIG)
+        key = "alegre:webhook:#{doc_id}"
+        redis.lpush(key, request.params.to_json)
+        redis.expire(key, 1.day.to_i)
+      rescue StandardError => e
+        CheckSentry.notify(AlegreCallbackError.new(e.message), { alegre_response: body })
+      end
+    end
+  end
+end

config/config.yml.example

@@ -11,7 +11,7 @@ development: &default
   elasticsearch_index:
   elasticsearch_log: true
   elasticsearch_sync: false
-
+  alegre_token: dev
   # WARNING For production, don't use a wildcard: set the allowed domains explicitly as a regular expression, e.g.
   # '(https?://.*\.?(meedan.com|meedan.org))'
   allowed_origins: '.*'

test/controllers/webhooks_controller_test.rb

@@ -232,4 +232,44 @@ def setup
     assert_equal '200', response.code
     assert_match /ignored/, response.body
   end
+
+  test "should process alegre webhook zzz" do
+    pm1 = create_project_media team: @team, media: create_uploaded_audio
+    pm2 = create_project_media team: @team, media: create_uploaded_audio
+    pm3 = create_project_media team: @team, media: create_uploaded_audio
+    params = {:url => nil, :context => {:has_custom_id => true, :team_id => @team.id}, :match_across_content_types => true, :threshold => 0.9}
+    Bot::Alegre.stubs(:request_api).with('get', '/audio/similarity/', params, 'body').returns({
+      result: [
+        {
+          id: 1,
+          doc_id: random_string,
+          chromaprint_fingerprint: [6581800, 2386744, 2583368, 2488648, 6343163, 14978026, 300191082, 309757210, 304525578, 304386106, 841261098, 841785386],
+          url: 'https://foo.com/bar.wav',
+          context: [
+            { team_id: @team.id.to_s, project_media_id: pm1.id.to_s }
+          ],
+          score: 0.971234,
+        },
+        {
+          id: 2,
+          doc_id: random_string,
+          chromaprint_fingerprint: [2386744, 2583368, 2488648, 6343163, 14978026, 300191082, 309757210, 304525578, 304386106, 841261098, 841785386, 858042410, 825593963, 823509230],
+          url: 'https://bar.com/foo.wav',
+          context: [
+            { team_id: @team.id.to_s, project_media_id: pm2.id.to_s }
+          ],
+          score: 0.983167,
+        }
+      ]
+    }.with_indifferent_access)
+    Bot::Alegre.stubs(:media_file_url).with(pm3).returns(@media_path)
+    redis = Redis.new(REDIS_CONFIG)
+    redis.del("foo")
+    payload = { 'action' => 'audio', 'data' => {'requested' => {'body' => {'id' => 'foo', 'context' => {'project_media_id' => pm3.id} }}}}
+    assert_nil redis.lpop('alegre:webhook:foo')
+    post :index, params: { name: :alegre, token: CheckConfig.get('alegre_token') }.merge(payload), body: payload
+    response = JSON.parse(redis.lpop('alegre:webhook:foo'))
+    assert_equal response["data"]["requested"]["body"]["id"], "foo"
+    Bot::Alegre.unstub(:media_file_url)
+  end
 end

test/models/bot/alegre_2_test.rb

@@ -315,6 +315,16 @@ def teardown
     RequestStore.store[:pause_database_connection] = false
   end
 
+  test "should block calls on redis blpop for audio request" do
+    stubbed_response = Net::HTTPSuccess.new(1.0, '200', 'OK')
+    stubbed_response.stubs(:body).returns({"queue" => "audio__Model", "body" => {"id" => "123", "callback_url" => "http://example.com/callback"}}.to_json)
+    Net::HTTP.any_instance.stubs(:request).returns(stubbed_response)
+    Redis.any_instance.stubs(:blpop).with("alegre:webhook:123", 120).returns(["alegre:webhook:123", {"tested" => true}.to_json])
+    assert_equal Bot::Alegre.request_api('get', '/audio/similarity/', @params, 'body'), {"tested" => true}
+    Net::HTTP.any_instance.unstub(:request)
+    Redis.any_instance.unstub(:blpop)
+  end
+
   test "should get items with similar title" do
     create_verification_status_stuff
     RequestStore.store[:skip_cached_field_update] = false

test/test_helper.rb

@@ -151,7 +151,7 @@ def before_all
     @start = Time.now
 
     create_metadata_stuff
-    @exporter = Check::OpenTelemetryTestConfig.current_exporter
+    # @exporter = Check::OpenTelemetryTestConfig.current_exporter
     # URL mocked by pender-client
     @url = 'https://www.youtube.com/user/MeedanTube'
   end