CV2-5628: fix OR condition and remove projects filter (#2125)

* CV2-5628: fix OR condition and remove projects filter

* CV2-5628: fix tests 1/2

* CV2-5628: fix tests 2/2

* CV2-5628: fix feed query

* CV2-5628: fix tests

* CV2-5628: cleanup

* CV2-5628: fix feed conditions

* CV2-5628: cleanup

lib/check_search.rb

@@ -2,7 +2,7 @@ class CheckSearch
   include SearchHelper
 
   def initialize(options, file = nil, team_id = Team.current&.id)
-    # Options include keywords, projects, tags, status, report status
+    # Options include search filters
     options = begin JSON.parse(options) rescue {} end
     @options = options.to_h.clone.with_indifferent_access
     @options['input'] = options.clone
@@ -22,9 +22,6 @@ def initialize(options, file = nil, team_id = Team.current&.id)
     @options['esoffset'] ||= 0
     adjust_es_window_size
 
-    # Check for non project
-    @options['none_project'] = @options['projects'].include?('-1') unless @options['projects'].blank?
-    adjust_project_filter
     adjust_channel_filter
     adjust_numeric_range_filter
     adjust_archived_filter
@@ -37,8 +34,6 @@ def initialize(options, file = nil, team_id = Team.current&.id)
     # Apply feed filters
     @feed_view = @options['feed_view'] || :fact_check
     @options.merge!(@feed.get_feed_filters(@feed_view)) if feed_query?
-
-    (Project.current ||= Project.where(id: @options['projects'].last).last) if @options['projects'].to_a.size == 1
     @file = file
   end
 
@@ -244,9 +239,7 @@ def get_pg_results_for_media
     core_conditions = {}
     core_conditions['team_id'] = @options['team_id'] if @options['team_id'].is_a?(Array)
     # Add custom conditions for array values
-    {
-      'project_id' => 'projects', 'user_id' => 'users', 'source_id' => 'sources', 'read' => 'read', 'unmatched' => 'unmatched'
-    }.each do |k, v|
+    { 'user_id' => 'users', 'source_id' => 'sources', 'read' => 'read', 'unmatched' => 'unmatched'}.each do |k, v|
       custom_conditions[k] = [@options[v]].flatten if @options.has_key?(v)
     end
     core_conditions.merge!({ archived: @options['archived'] })
@@ -286,6 +279,17 @@ def get_search_field
   end
 
   def medias_query
+    return build_feed_conditions if feed_query?
+    and_conditions, or_conditions, not_conditions = build_es_medias_query
+    # Build ES query using this format: `bool: { must: [{and_conditions}], should: [{or_conditions}, must_not: [{not_conditions}]] }`
+    query = {}
+    { must: and_conditions, should: or_conditions, must_not: not_conditions }.each do |k, v|
+      query[k] = v.flatten unless v.blank?
+    end
+    { bool: query }
+  end
+
+  def build_es_medias_query
     core_conditions = []
     custom_conditions = []
     core_conditions << { terms: { get_search_field => @options['project_media_ids'] } } unless @options['project_media_ids'].blank?
@@ -313,16 +317,16 @@ def medias_query
     custom_conditions.concat request_language_conditions
     custom_conditions.concat report_language_conditions
     custom_conditions.concat team_tasks_conditions
-    feed_conditions = build_feed_conditions
-    conditions = []
+    and_conditions = core_conditions
+    or_conditions = []
+    not_conditions = []
     if @options['operator'].upcase == 'OR'
-      and_conditions = { bool: { must: core_conditions } }
-      or_conditions = { bool: { should: custom_conditions } }
-      conditions = [and_conditions, or_conditions, feed_conditions]
+      or_conditions << custom_conditions
+      not_conditions << { term: { associated_type: { value: "Blank" } } }
     else
-      conditions = [{ bool: { must: (core_conditions + custom_conditions) } }, feed_conditions]
+      and_conditions.concat(custom_conditions)
     end
-    { bool: { must: conditions.reject{ |c| c.blank? } } }
+    return and_conditions, or_conditions, not_conditions
   end
 
   def medias_get_search_result(query)
@@ -418,25 +422,6 @@ def adjust_es_window_size
     @options['eslimit'] = window_size - @options['esoffset'].to_i if current_size > window_size
   end
 
-  def adjust_project_filter
-    team_id = [@options['team_id']].flatten.first
-    project_group_ids = [@options['project_group_id']].flatten.reject{ |pgid| pgid.blank? }.map(&:to_i)
-    unless project_group_ids.empty?
-      project_ids = @options['projects'].to_a.map(&:to_i)
-      project_groups_project_ids = Project.where(project_group_id: project_group_ids, team_id: team_id).map(&:id)
-
-      project_ids = project_ids.blank? ? project_groups_project_ids : (project_ids & project_groups_project_ids)
-
-      # Invalidate the search if empty... otherwise, adjust the projects filter
-      @options['projects'] = project_ids.empty? ? [0] : project_ids
-    end
-    if Team.current && !feed_query? && [@options['team_id']].flatten.size == 1
-      t = Team.find(team_id)
-      @options['projects'] = @options['projects'].blank? ? (Project.where(team_id: t.id).map(&:id) + [nil]) : Project.where(id: @options['projects'], team_id: t.id).map(&:id)
-    end
-    @options['projects'] += [nil] if @options['none_project']
-  end
-
   def adjust_channel_filter
     if @options['channels'].is_a?(Array) && @options['channels'].include?('any_tipline')
       channels = @options['channels'] - ['any_tipline']
@@ -700,7 +685,7 @@ def doc_conditions
       doc_c << { terms: { 'associated_type': types } }
     end
 
-    fields = { 'project_id' => 'projects', 'user_id' => 'users' }
+    fields = { 'user_id' => 'users' }
     status_search_fields.each do |field|
       fields[field] = field
     end
@@ -780,12 +765,21 @@ def hit_es_for_range_filter
   end
 
   def build_feed_conditions
-    return {} unless feed_query?
+    return [] unless feed_query?
     conditions = []
+    feed_options = @options.clone
+    feed_options.delete('feed_id')
+    feed_options.delete('input')
+    and_conditions, or_conditions, not_conditions = CheckSearch.new(feed_options.to_json, nil, @options['team_id']).build_es_medias_query
     @feed.get_team_filters(@options['feed_team_ids']).each do |filters|
       team_id = filters['team_id'].to_i
       conditions << CheckSearch.new(filters.merge({ show_similar: !!@options['show_similar'] }).to_json, nil, team_id).medias_query
     end
-    { bool: { should: conditions } }
+    or_conditions.concat(conditions)
+    query = []
+    { must: and_conditions, should: or_conditions, must_not: not_conditions}.each do |k, v|
+      query << { bool: { "#{k}": v } } unless v.blank?
+    end
+    { bool: { must: query } }
   end
 end

test/controllers/elastic_search_10_test.rb

@@ -140,22 +140,14 @@ def setup
     end
   end
 
-  test "should filter items by non project and read-unread" do
+  test "should filter items by read-unread" do
     t = create_team
-    p = create_project team: t
     u = create_user
     create_team_user team: t, user: u, role: 'admin'
     with_current_user_and_team(u ,t) do
       pm = create_project_media team: t, disable_es_callbacks: false
-      pm2 = create_project_media project: p, disable_es_callbacks: false
+      pm2 = create_project_media team: t, disable_es_callbacks: false
       pm3 = create_project_media team: t, quote: 'claim a', disable_es_callbacks: false
-      results = CheckSearch.new({ projects: ['-1'] }.to_json)
-      # result should return empty as now all items should have a project CHECK-1150
-      assert_empty results.medias.map(&:id)
-      results = CheckSearch.new({ projects: [p.id, '-1'] }.to_json)
-      assert_equal [pm2.id], results.medias.map(&:id)
-      results = CheckSearch.new({ keyword: 'claim', projects: ['-1'] }.to_json)
-      assert_empty results.medias.map(&:id)
       # test read/unread
       pm.read = true
       pm.save!

test/controllers/elastic_search_4_test.rb

@@ -8,10 +8,8 @@ def setup
 
   test "should search with multiple filters" do
     t = create_team
-    p = create_project team: t
-    p2 = create_project team: t
-    pm = create_project_media project: p, quote: 'report_title', disable_es_callbacks: false
-    pm2 = create_project_media project: p2, quote: 'report_title', disable_es_callbacks: false
+    pm = create_project_media team: t, quote: 'report_title', disable_es_callbacks: false
+    pm2 = create_project_media team: t, quote: 'report_title', disable_es_callbacks: false
     create_tag tag: 'sports', annotated: pm, disable_es_callbacks: false
     create_tag tag: 'sports', annotated: pm2, disable_es_callbacks: false
     create_status status: 'verified', annotated: pm, disable_es_callbacks: false
@@ -20,37 +18,16 @@ def setup
     Team.current = t
     result = CheckSearch.new({keyword: 'report_title', tags: ['sports']}.to_json)
     assert_equal [pm2.id, pm.id], result.medias.map(&:id)
-    # keyword & context
-    result = CheckSearch.new({keyword: 'report_title', projects: [p.id]}.to_json)
-    assert_equal [pm.id], result.medias.map(&:id)
     # keyword & status
     result = CheckSearch.new({keyword: 'report_title', verification_status: ['verified']}.to_json)
     assert_equal [pm.id], result.medias.map(&:id)
-    # tags & context
-    result = CheckSearch.new({projects: [p.id], tags: ['sports']}.to_json)
-    assert_equal [pm.id], result.medias.map(&:id)
-    # status & context
-    result = CheckSearch.new({projects: [p.id], verification_status: ['verified']}.to_json)
-    assert_equal [pm.id], result.medias.map(&:id)
-    # keyword & tags & context
-    result = CheckSearch.new({keyword: 'report_title', tags: ['sports'], projects: [p.id]}.to_json)
-    assert_equal [pm.id], result.medias.map(&:id)
-    # keyword & status & context
-    result = CheckSearch.new({keyword: 'report_title', verification_status: ['verified'], projects: [p.id]}.to_json)
-    assert_equal [pm.id], result.medias.map(&:id)
-    # tags & context & status
-    result = CheckSearch.new({tags: ['sports'], verification_status: ['verified'], projects: [p.id]}.to_json)
-    assert_equal [pm.id], result.medias.map(&:id)
     # keyword & tags & status
     result = CheckSearch.new({keyword: 'report_title', tags: ['sports'], verification_status: ['verified']}.to_json)
     assert_equal [pm.id], result.medias.map(&:id)
-    # keyword & tags & context & status
-    result = CheckSearch.new({keyword: 'report_title', tags: ['sports'], verification_status: ['verified'], projects: [p.id]}.to_json)
-    assert_equal [pm.id], result.medias.map(&:id)
     # search keyword in comments
     create_comment text: 'add_comment', annotated: pm, disable_es_callbacks: false
     sleep 1
-    result = CheckSearch.new({keyword: 'add_comment', projects: [p.id]}.to_json)
+    result = CheckSearch.new({keyword: 'add_comment'}.to_json)
     assert_equal [pm.id], result.medias.map(&:id)
   end
 
@@ -200,34 +177,21 @@ def setup
     c2 = create_claim_media
     m = create_valid_media
     t1 = create_team
-    p1a = create_project team: t1
-    p1b = create_project team: t1
-    pm1a = create_project_media project: p1a, media: c, disable_es_callbacks: false
-    sleep 1
-    pm1b = create_project_media project: p1b, media: c2, disable_es_callbacks: false
-
+    pm1a = create_project_media team: t1, media: c, disable_es_callbacks: false
+    pm1b = create_project_media team: t1, media: c2, disable_es_callbacks: false
     t2 = create_team
-    p2a = create_project team: t2
-    p2b = create_project team: t2
-    pm2a = create_project_media project: p2a, media: m, disable_es_callbacks: false
-    sleep 1
-    pm2b = create_project_media project: p2b, disable_es_callbacks: false
-
+    pm2a = create_project_media team: t2, media: m, disable_es_callbacks: false
+    pm2b = create_project_media team: t2, disable_es_callbacks: false
+    sleep 2
     Team.current = t1
     assert_equal [pm1b, pm1a], CheckSearch.new('{}').medias
     assert_equal 2, CheckSearch.new('{}').project_medias.count
-    assert_equal 1, CheckSearch.new({ projects: [p1a.id], show: ['claims']}.to_json).project_medias.count
-    assert_equal [pm1a], CheckSearch.new({ projects: [p1a.id] }.to_json).medias
-    assert_equal 1, CheckSearch.new({ projects: [p1a.id] }.to_json).project_medias.count
+    assert_equal 2, CheckSearch.new({ show: ['claims']}.to_json).project_medias.count
     assert_equal [pm1a, pm1b], CheckSearch.new({ sort_type: 'ASC' }.to_json).medias
     assert_equal 2, CheckSearch.new({ sort_type: 'ASC' }.to_json).project_medias.count
-    Team.current = nil
-
     Team.current = t2
     assert_equal [pm2b, pm2a], CheckSearch.new('{}').medias
     assert_equal 2, CheckSearch.new('{}').project_medias.count
-    assert_equal [pm2a], CheckSearch.new({ projects: [p2a.id] }.to_json).medias
-    assert_equal 1, CheckSearch.new({ projects: [p2a.id] }.to_json).project_medias.count
     assert_equal [pm2a, pm2b], CheckSearch.new({ sort_type: 'ASC' }.to_json).medias
     assert_equal 2, CheckSearch.new({ sort_type: 'ASC' }.to_json).project_medias.count
     Team.current = nil

test/controllers/elastic_search_7_test.rb

@@ -159,16 +159,14 @@ def setup
 
   test "should parse search options" do
     t = create_team
-    p = create_project team: t
-    p2 = create_project team: t
-    pm = create_project_media project: p, disable_es_callbacks: false
-    pm2 = create_project_media project: p2, disable_es_callbacks: false
+    pm = create_project_media team: t, archived: 0, disable_es_callbacks: false
+    pm2 = create_project_media team: t, archived: 2, disable_es_callbacks: false
     sleep 1
     Team.current = t
-    result = CheckSearch.new({projects: [p.id]}.to_json)
+    result = CheckSearch.new({archived: [0]}.to_json)
     assert_equal [pm.id], result.medias.map(&:id)
     # pass wrong format should map to all items
-    result = CheckSearch.new({projects: [p.id]})
+    result = CheckSearch.new({archived: [0]})
     assert_equal [pm.id, pm2.id], result.medias.map(&:id).sort
   end
 

test/controllers/elastic_search_test.rb

@@ -157,40 +157,6 @@ def setup
     assert_equal [pm.id], result.medias.map(&:id)
   end
 
-  test "should search with context" do
-    t = create_team
-    p = create_project team: t
-    pender_url = CheckConfig.get('pender_url_private') + '/api/medias'
-    url = 'http://test.com'
-    response = '{"type":"media","data":{"url":"' + url + '/normalized","type":"item", "title": "search_title", "description":"search_desc"}}'
-    WebMock.stub_request(:get, pender_url).with({ query: { url: url } }).to_return(body: response)
-    url2 = 'http://test2.com'
-    response = '{"type":"media","data":{"url":"' + url2 + '/normalized","type":"item", "title": "search_title", "description":"search_desc"}}'
-    WebMock.stub_request(:get, pender_url).with({ query: { url: url2 } }).to_return(body: response)
-    m = create_media(account: create_valid_account, url: url)
-    m1 = create_media(account: create_valid_account, url: url2)
-    pm = create_project_media project: p, media: m, disable_es_callbacks: false
-    keyword = { projects: [0,0,0] }.to_json
-    sleep 1
-    Team.current = t
-    result = CheckSearch.new(keyword)
-    assert_empty result.medias
-    result = CheckSearch.new({projects: [p.id]}.to_json)
-    assert_equal [pm.id], result.medias.map(&:id)
-    # add a new context to existing media
-    p2 = create_project team: t
-    pm2 = create_project_media project: p2, media: m1, disable_es_callbacks: false
-    sleep 1
-    result = CheckSearch.new({projects: [p.id]}.to_json)
-    assert_equal [pm.id].sort, result.medias.map(&:id).sort
-    # add a new media to same context
-    m2 = create_valid_media
-    pm2 = create_project_media project: p, media: m2, disable_es_callbacks: false
-    sleep 1
-    result = CheckSearch.new({projects: [p.id]}.to_json)
-    assert_equal [pm.id, pm2.id].sort, result.medias.map(&:id).sort
-  end
-
   test "should search with tags or status" do
     t = create_team
     p = create_project team: t

test/controllers/graphql_controller_10_test.rb

@@ -515,35 +515,6 @@ def setup
     assert_equal [], JSON.parse(@response.body)['data']['search']['medias']['edges'].collect{ |pm| pm['node']['dbid'] }
   end
 
-  test "should search by project group" do
-    u = create_user is_admin: true
-    t = create_team
-    create_team_user user: u, team: t, role: 'admin'
-    authenticate_with_user(u)
-
-    pg = create_project_group team: t
-    p1 = create_project team: t
-    p1.project_group = pg
-    p1.save!
-    create_project_media project: p1
-    p2 = create_project team: t
-    p2.project_group = pg
-    p2.save!
-    create_project_media project: p2
-    p3 = create_project team: t
-    create_project_media project: p3
-
-    query = 'query CheckSearch { search(query: "{}") { number_of_results } }'
-    post :create, params: { query: query, team: t.slug }
-    assert_response :success
-    assert_equal 3, JSON.parse(@response.body)['data']['search']['number_of_results']
-
-    query = 'query CheckSearch { search(query: "{\"project_group_id\":' + pg.id.to_s + '}") { number_of_results } }'
-    post :create, params: { query: query, team: t.slug }
-    assert_response :success
-    assert_equal 2, JSON.parse(@response.body)['data']['search']['number_of_results']
-  end
-
   test "should not access GraphQL mutation if not authenticated" do
     post :create, params: { query: 'mutation Test' }
     assert_response 401