diff --git a/.codeclimate.yml b/.codeclimate.yml index ff19a8b13d..3a28cf1b74 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -2,7 +2,7 @@ version: "2" checks: argument-count: config: - threshold: 8 + threshold: 9 complex-logic: config: threshold: 4 diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index d5126ae9ed..685787e3c8 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -5,14 +5,10 @@ on: - cron: '0 5 * * *' #Runs daily at 5 AM UTC push: branches: - - master - - develop - - epic* - - cv2* + - '*' pull_request: branches: - develop - env: CC_TEST_REPORTER_ID: "${{ secrets.CC_TEST_REPORTER_ID }}" diff --git a/.rubocop.yml b/.rubocop.yml index 2ff1fbf74d..78aae1b289 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -243,10 +243,10 @@ Metrics/ModuleLength: Max: 250 Metrics/ParameterLists: - Description: 'Avoid parameter lists longer than three or four parameters.' + Description: 'Avoid parameter lists longer than 9 parameters.' StyleGuide: 'https://github.com/bbatsov/ruby-style-guide#too-many-params' Enabled: true - Max: 8 + Max: 9 Metrics/PerceivedComplexity: Description: >- diff --git a/Gemfile.lock b/Gemfile.lock index 202a6f447d..a89311714b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -726,8 +726,7 @@ GEM mime-types (>= 1.16, < 4.0) netrc (~> 0.8) retriable (3.1.2) - rexml (3.3.6) - strscan + rexml (3.3.9) rotp (6.3.0) rqrcode (2.1.1) chunky_png (~> 1.0) @@ -817,7 +816,6 @@ GEM ssrf_filter (1.0.7) streamio-ffmpeg (3.0.2) multi_json (~> 1.8) - strscan (3.1.0) swagger-docs (0.2.9) activesupport (>= 3) rails (>= 3) diff --git a/app/graph/mutations/graphql_crud_operations.rb b/app/graph/mutations/graphql_crud_operations.rb index 1520987d43..3d47bb6246 100644 --- a/app/graph/mutations/graphql_crud_operations.rb +++ b/app/graph/mutations/graphql_crud_operations.rb @@ -7,7 +7,20 @@ def self.safe_save(obj, attrs, parent_names = []) obj.send(method, value) if obj.respond_to?(method) end obj.disable_es_callbacks = Rails.env.to_s == "test" - obj.save_with_version! + + begin + obj.save_with_version! + rescue RuntimeError => e + if e.message.include?("\"code\":#{LapisConstants::ErrorCodes::const_get('DUPLICATED')}") && + obj.is_a?(ProjectMedia) && + obj.set_fact_check.present? && + obj.set_original_claim.present? + existing_pm = ProjectMedia.find(JSON.parse(e.message)['data']['id']) + obj = ProjectMedia.handle_fact_check_for_existing_claim(existing_pm,obj) + else + raise e + end + end name = obj.class_name.underscore { name.to_sym => obj }.merge( diff --git a/app/graph/types/team_statistics_type.rb b/app/graph/types/team_statistics_type.rb new file mode 100644 index 0000000000..5c98bfddde --- /dev/null +++ b/app/graph/types/team_statistics_type.rb @@ -0,0 +1,40 @@ +class TeamStatisticsType < DefaultObject + description 'Workspace statistics.' + + implements GraphQL::Types::Relay::Node + + # For articles + + field :number_of_articles_created_by_date, JsonStringType, null: true + field :number_of_articles_updated_by_date, JsonStringType, null: true + field :number_of_explainers_created, GraphQL::Types::Int, null: true + field :number_of_fact_checks_created, GraphQL::Types::Int, null: true + field :number_of_published_fact_checks, GraphQL::Types::Int, null: true + field :number_of_fact_checks_by_rating, JsonStringType, null: true + field :top_articles_sent, JsonStringType, null: true + field :top_articles_tags, JsonStringType, null: true + + # For tiplines + + field :number_of_messages, GraphQL::Types::Int, null: true + field :number_of_conversations, GraphQL::Types::Int, null: true + field :number_of_messages_by_date, JsonStringType, null: true + field :number_of_conversations_by_date, JsonStringType, null: true + field :number_of_search_results_by_feedback_type, JsonStringType, null: true + field :average_response_time, GraphQL::Types::Int, null: true + field :number_of_unique_users, GraphQL::Types::Int, null: true + field :number_of_total_users, GraphQL::Types::Int, null: true + field :number_of_returning_users, GraphQL::Types::Int, null: true + field :number_of_subscribers, GraphQL::Types::Int, null: true + field :number_of_new_subscribers, GraphQL::Types::Int, null: true + field :number_of_newsletters_sent, GraphQL::Types::Int, null: true + field :number_of_newsletters_delivered, GraphQL::Types::Int, null: true + field :top_media_tags, JsonStringType, null: true + field :top_requested_media_clusters, JsonStringType, null: true + field :number_of_media_received_by_media_type, JsonStringType, null: true + + # For both articles and tiplines + + field :number_of_articles_sent, GraphQL::Types::Int, null: true + field :number_of_matched_results_by_article_type, JsonStringType, null: true +end diff --git a/app/graph/types/team_type.rb b/app/graph/types/team_type.rb index 97c4f77fbf..80ad198b0e 100644 --- a/app/graph/types/team_type.rb +++ b/app/graph/types/team_type.rb @@ -380,6 +380,7 @@ def api_key(dbid:) end field :api_keys, ApiKeyType.connection_type, null: true + def api_keys ability = context[:ability] || Ability.new api_keys = object.api_keys.order(created_at: :desc) @@ -388,4 +389,15 @@ def api_keys ability.can?(:read, api_key) end end + + field :statistics, TeamStatisticsType, null: true do + argument :period, GraphQL::Types::String, required: true # FIXME: List/validate possible values + argument :language, GraphQL::Types::String, required: false + argument :platform, GraphQL::Types::String, required: false # FIXME: List/validate possible values + end + + def statistics(period:, language: nil, platform: nil) + return nil unless User.current&.is_admin + TeamStatistics.new(object, period, language, platform) + end end diff --git a/app/lib/check_cached_fields.rb b/app/lib/check_cached_fields.rb index 4deb04fe39..fedc37191b 100644 --- a/app/lib/check_cached_fields.rb +++ b/app/lib/check_cached_fields.rb @@ -47,7 +47,7 @@ def cached_field(name, options = {}) klass = self update_on[:events].each do |event, callback| model.send "after_#{event}", ->(obj) do - klass.update_cached_field(name, obj, update_on[:if], update_on[:affected_ids], callback, options) + klass.update_cached_field(name, obj, update_on[:if], update_on[:affected_ids], callback, options, event) end end end @@ -97,12 +97,13 @@ def index_cached_field(options, value, name, obj) update_pg: options[:update_pg], pg_field_name: options[:pg_field_name], } - self.delay_for(1.second).index_cached_field_bg(index_options, value, name, obj) + self.delay_for(1.second).index_cached_field_bg(index_options, value, name, obj.class.name, obj.id) end end - def index_cached_field_bg(index_options, value, name, obj) - self.index_and_pg_cached_field(index_options, value, name, obj) + def index_cached_field_bg(index_options, value, name, klass, id) + obj = klass.constantize.find_by_id id + self.index_and_pg_cached_field(index_options, value, name, obj) unless obj.nil? end def update_pg_cache_field(options, value, name, target) @@ -120,7 +121,7 @@ def create_cached_field(options, name, obj) self.index_cached_field(options, value, name, obj) unless Rails.env == 'test' end - def update_cached_field(name, obj, condition, ids, callback, options) + def update_cached_field(name, obj, condition, ids, callback, options, event) return if self.skip_cached_field_update? condition ||= proc { true } return unless condition.call(obj) @@ -136,17 +137,20 @@ def update_cached_field(name, obj, condition, ids, callback, options) pg_field_name: options[:pg_field_name], recalculate: options[:recalculate], } - self.delay_for(1.second).update_cached_field_bg(name, obj, ids, callback, index_options) + self.delay_for(1.second).update_cached_field_bg(name, ids, callback, index_options, obj.class.name, obj.id, event) end end - def update_cached_field_bg(name, obj, ids, callback, options) - recalculate = options[:recalculate] - self.where(id: ids).each do |target| - value = callback == :recalculate ? target.send(recalculate) : obj.send(callback, target) - Rails.cache.write(self.check_cache_key(self, target.id, name), value, expires_in: self.cached_field_expiration(options)) - # Update ES index and PG, if needed - self.index_and_pg_cached_field(options, value, name, target) + def update_cached_field_bg(name, ids, callback, options, klass, id, event) + obj = event == 'destroy' ? klass.constantize : klass.constantize.find_by_id(id) + unless obj.nil? + recalculate = options[:recalculate] + self.where(id: ids).each do |target| + value = callback == :recalculate ? target.send(recalculate) : obj.send(callback, target) + Rails.cache.write(self.check_cache_key(self, target.id, name), value, expires_in: self.cached_field_expiration(options)) + # Update ES index and PG, if needed + self.index_and_pg_cached_field(options, value, name, target) + end end end end diff --git a/app/lib/check_elastic_search.rb b/app/lib/check_elastic_search.rb index 35268006bb..300c094777 100644 --- a/app/lib/check_elastic_search.rb +++ b/app/lib/check_elastic_search.rb @@ -63,7 +63,7 @@ def update_elasticsearch_doc_bg(options) create_doc_if_not_exists(options) sleep 1 client = $repository.client - client.update index: CheckElasticSearchModel.get_index_alias, id: options[:doc_id], retry_on_conflict: 3, body: { doc: fields } + client.update index: CheckElasticSearchModel.get_index_alias, id: options[:doc_id], body: { doc: fields } end end @@ -98,7 +98,7 @@ def create_update_nested_obj_bg(options) end values = store_elasticsearch_data(options[:keys], options[:data]) client = $repository.client - client.update index: CheckElasticSearchModel.get_index_alias, id: options[:doc_id], retry_on_conflict: 3, + client.update index: CheckElasticSearchModel.get_index_alias, id: options[:doc_id], body: { script: { source: source, params: { value: values, id: values['id'] } } } end @@ -178,7 +178,7 @@ def destroy_elasticsearch_doc_nested(options) begin client = $repository.client source = "for (int i = 0; i < ctx._source.#{nested_type}.size(); i++) { if(ctx._source.#{nested_type}[i].id == params.id){ctx._source.#{nested_type}.remove(i);}}" - client.update index: CheckElasticSearchModel.get_index_alias, id: options[:doc_id], retry_on_conflict: 3, + client.update index: CheckElasticSearchModel.get_index_alias, id: options[:doc_id], body: { script: { source: source, params: { id: options[:model_id] } } } rescue Rails.logger.info "[ES destroy] doc with id #{options[:doc_id]} not exists" diff --git a/app/lib/smooch_nlu.rb b/app/lib/smooch_nlu.rb index 01e15fb32f..9932ed5ec8 100644 --- a/app/lib/smooch_nlu.rb +++ b/app/lib/smooch_nlu.rb @@ -33,8 +33,6 @@ def enabled? end def update_keywords(language, keywords, keyword, operation, doc_id, context) - alegre_operation = nil - alegre_params = nil common_alegre_params = { doc_id: doc_id, context: { @@ -44,15 +42,11 @@ def update_keywords(language, keywords, keyword, operation, doc_id, context) } if operation == 'add' && !keywords.include?(keyword) keywords << keyword - alegre_operation = 'post' - alegre_params = common_alegre_params.merge({ text: keyword, models: ALEGRE_MODELS_AND_THRESHOLDS.keys }) + Bot::Alegre.index_sync_with_params(common_alegre_params.merge({ text: keyword, models: ALEGRE_MODELS_AND_THRESHOLDS.keys }), "text") elsif operation == 'remove' keywords -= [keyword] - alegre_operation = 'delete' - alegre_params = common_alegre_params.merge({ quiet: true }) + Bot::Alegre.request_delete_from_raw(common_alegre_params.merge({ quiet: true }), "text") end - # FIXME: Add error handling and better logging - Bot::Alegre.request(alegre_operation, '/text/similarity/', alegre_params) if alegre_operation && alegre_params keywords end @@ -91,19 +85,19 @@ def self.alegre_matches_from_message(message, language, context, alegre_result_k language: language, }.merge(context) } - response = Bot::Alegre.request('post', '/text/similarity/search/', params) + response = Bot::Alegre.query_sync_with_params(params, "text") # One approach would be to take the option that has the most matches # Unfortunately this approach is influenced by the number of keywords per option # So, we are not using this approach right now # Get the `alegre_result_key` of all results returned - # option_counts = response['result'].to_a.map{|o| o.dig('_source', 'context', alegre_result_key)} + # option_counts = response['result'].to_a.map{|o| o.dig('context', alegre_result_key)} # Count how many of each alegre_result_key we have and sort (high to low) # ranked_options = option_counts.group_by(&:itself).transform_values(&:count).sort_by{|_k,v| v}.reverse() # Second approach is to sort the results from best to worst - sorted_options = response['result'].to_a.sort_by{ |result| result['_score'] }.reverse - ranked_options = sorted_options.map{ |o| { 'key' => o.dig('_source', 'context', alegre_result_key), 'score' => o['_score'] } } + sorted_options = response['result'].to_a.sort_by{ |result| result['score'] }.reverse + ranked_options = sorted_options.map{ |o| { 'key' => o.dig('context', alegre_result_key), 'score' => o['score'] } } matches = ranked_options # In all cases log for analysis diff --git a/app/models/annotations/tag.rb b/app/models/annotations/tag.rb index bc6a02b9b5..65642e86c6 100644 --- a/app/models/annotations/tag.rb +++ b/app/models/annotations/tag.rb @@ -93,14 +93,15 @@ def hit_nested_objects_limit? end def self.create_project_media_tags(project_media_id, tags_json) - project_media = ProjectMedia.find_by_id(project_media_id) - - if !project_media.nil? - tags = JSON.parse(tags_json) - clean_tags(tags).each { |tag| Tag.create annotated: project_media, tag: tag.strip, skip_check_ability: true } - else - error = StandardError.new("[ProjectMedia] Exception creating project media's tags in background. Project media is nil.") - CheckSentry.notify(error, project_media_id: project_media_id) + tags = JSON.parse(tags_json).reject { |t| t.blank? } + unless tags.empty? + project_media = ProjectMedia.find_by_id(project_media_id) + if !project_media.nil? + clean_tags(tags).each { |tag| Tag.create annotated: project_media, tag: tag.strip, skip_check_ability: true } + else + error = StandardError.new("[ProjectMedia] Exception creating project media's tags in background. Project media is nil.") + CheckSentry.notify(error, project_media_id: project_media_id) + end end end diff --git a/app/models/bot/alegre.rb b/app/models/bot/alegre.rb index 877813d1a7..a1d05d8950 100644 --- a/app/models/bot/alegre.rb +++ b/app/models/bot/alegre.rb @@ -41,7 +41,7 @@ def similar_items_ids_and_scores(team_ids, thresholds = {}) ALL_TEXT_SIMILARITY_FIELDS.each do |field| text = self.send(field) next if text.blank? - threads << Thread.new { ids_and_scores.merge!(Bot::Alegre.get_similar_texts(team_ids, text, Bot::Alegre::ALL_TEXT_SIMILARITY_FIELDS, thresholds[:text]).to_h) } + threads << Thread.new { ids_and_scores.merge!(Bot::Alegre.get_items_from_similar_text(team_ids, text, Bot::Alegre::ALL_TEXT_SIMILARITY_FIELDS, thresholds[:text]).to_h) } end threads.map(&:join) end @@ -155,10 +155,8 @@ def self.run(body) if ['audio', 'image', 'video'].include?(self.get_pm_type(pm)) self.relate_project_media_async(pm) else - Bot::Alegre.send_to_media_similarity_index(pm) - Bot::Alegre.send_field_to_similarity_index(pm, 'original_title') - Bot::Alegre.send_field_to_similarity_index(pm, 'original_description') - Bot::Alegre.relate_project_media_to_similar_items(pm) + self.relate_project_media_async(pm, 'original_title') + self.relate_project_media_async(pm, 'original_description') end self.get_extracted_text(pm) self.get_flags(pm) @@ -206,7 +204,7 @@ def self.get_items_from_similar_text(team_id, text, fields = nil, threshold = ni threshold ||= self.get_threshold_for_query('text', nil, true) models ||= [self.matching_model_to_use(team_ids)].flatten Hash[self.get_similar_items_from_api( - '/text/similarity/search/', + 'text', self.similar_texts_from_api_conditions(text, models, fuzzy, team_ids, fields, threshold), threshold ).collect{|k,v| [k, v.merge(model: v[:model]||Bot::Alegre.default_matching_model)]}] @@ -722,8 +720,4 @@ def self.is_text_too_short?(pm, length_threshold) is_short end - class < CheckConfig.get('min_number_of_words_for_tipline_submit_shortcut', 10, :integer) || + ::Bot::Alegre.get_number_of_words(message['text'].to_s) > self.min_number_of_words_for_tipline_long_text || !Twitter::TwitterText::Extractor.extract_urls(message['text'].to_s).blank? # URL in message? ) end @@ -851,7 +851,7 @@ def self.save_text_message(message) extra = { quote: claim } pm = ProjectMedia.joins(:media).where('trim(lower(quote)) = ?', claim.downcase).where('project_medias.team_id' => team.id).last # Don't create a new text media if it's an unconfirmed request with just a few words - if pm.nil? && message['archived'] == CheckArchivedFlags::FlagCodes::UNCONFIRMED && ::Bot::Alegre.get_number_of_words(claim) < CheckConfig.get('min_number_of_words_for_tipline_submit_shortcut', 10, :integer) + if pm.nil? && message['archived'] == CheckArchivedFlags::FlagCodes::UNCONFIRMED && ::Bot::Alegre.get_number_of_words(claim) < self.min_number_of_words_for_tipline_long_text return team end else diff --git a/app/models/claim_description.rb b/app/models/claim_description.rb index d331358166..b2d1ece896 100644 --- a/app/models/claim_description.rb +++ b/app/models/claim_description.rb @@ -16,7 +16,7 @@ class ClaimDescription < ApplicationRecord validates_uniqueness_of :project_media_id, allow_nil: true validate :cant_apply_article_to_item_if_article_is_in_the_trash after_commit :update_fact_check, on: [:update] - after_update :update_report_status + after_update :update_report after_update :reset_item_rating_if_removed after_update :replace_media, unless: proc { |cd| cd.disable_replace_media } after_update :migrate_claim_and_fact_check_logs, if: proc { |cd| cd.saved_change_to_project_media_id? && !cd.project_media_id.nil? } @@ -69,16 +69,17 @@ def update_fact_check end end - # Pause report when claim/fact-check is removed - def update_report_status + # Pause and update report when claim/fact-check is removed + def update_report if self.project_media_id.nil? && !self.project_media_id_before_last_save.nil? - # Update report status + # Update report status and text fields pm = ProjectMedia.find(self.project_media_id_before_last_save) report = Annotation.where(annotation_type: 'report_design', annotated_type: 'ProjectMedia', annotated_id: pm.id).last unless report.nil? report = report.load data = report.data.clone.with_indifferent_access data[:state] = 'paused' + data[:options] = data[:options].to_h.merge({ description: '', headline: '', title: '', text: '' }) report.data = data report.save! end diff --git a/app/models/cluster.rb b/app/models/cluster.rb index a1f124ad60..8c0d3c3e4a 100644 --- a/app/models/cluster.rb +++ b/app/models/cluster.rb @@ -70,7 +70,7 @@ def self.import_other_medias_to_team(cluster_id, parent_id, max) cluster.project_medias.where.not(team_id: team.id).limit(max).select(:id, :media_id).find_each do |pm| next if ProjectMedia.where(team_id: team.id, media_id: pm.media_id).exists? target = cluster.import_media_to_team(team, pm) - Relationship.create(source: parent, target: target, relationship_type: Relationship.confirmed_type) # Didn't use "!" so if fails silently if the similarity bot creates a relationship first + Relationship.create_unless_exists(parent.id, target.id, Relationship.confirmed_type) end end end diff --git a/app/models/concerns/alegre_similarity.rb b/app/models/concerns/alegre_similarity.rb index 65b88acb2d..5bcd596fea 100644 --- a/app/models/concerns/alegre_similarity.rb +++ b/app/models/concerns/alegre_similarity.rb @@ -125,7 +125,8 @@ def send_to_text_similarity_index_package(pm, field, text, doc_id) doc_id: doc_id, text: text, models: models, - context: self.get_context(pm, field) + context: self.get_context(pm, field), + requires_callback: true } params[:language] = language if !language.nil? params @@ -133,10 +134,9 @@ def send_to_text_similarity_index_package(pm, field, text, doc_id) def send_to_text_similarity_index(pm, field, text, doc_id) if !text.blank? && Bot::Alegre::BAD_TITLE_REGEX !~ text - self.request( - 'post', - '/text/similarity/', - self.send_to_text_similarity_index_package(pm, field, text, doc_id) + self.query_sync_with_params( + self.send_to_text_similarity_index_package(pm, field, text, doc_id), + "text" ) end end @@ -207,10 +207,10 @@ def get_merged_similar_items(pm, threshold, fields, value, team_ids = [pm&.team_ es_matches end - def get_similar_items_from_api(path, conditions, _threshold = {}) - Rails.logger.error("[Alegre Bot] Sending request to alegre : #{path} , #{conditions.to_json}") + def get_similar_items_from_api(type, conditions, _threshold = {}) + Rails.logger.error("[Alegre Bot] Sending request to alegre : #{type} , #{conditions.to_json}") response = {} - result = self.request('post', path, conditions)&.dig('result') + result = self.query_sync_with_params(conditions, type)&.dig('result') project_medias = result.collect{ |r| self.extract_project_medias_from_context(r) } if !result.nil? && result.is_a?(Array) project_medias.each do |request_response| request_response.each do |pmid, score_with_context| diff --git a/app/models/concerns/alegre_v2.rb b/app/models/concerns/alegre_v2.rb index 2dde5fc159..9bc2a70ab4 100644 --- a/app/models/concerns/alegre_v2.rb +++ b/app/models/concerns/alegre_v2.rb @@ -1,7 +1,7 @@ require 'active_support/concern' class AlegreTimeoutError < StandardError; end class TemporaryProjectMedia - attr_accessor :team_id, :id, :url, :type + attr_accessor :team_id, :id, :url, :text, :type, :field def media media_type_map = { "claim" => "Claim", @@ -36,6 +36,10 @@ def is_video? def is_audio? self.type == "audio" end + + def is_uploaded_media? + self.is_image? || self.is_audio? || self.is_video? + end end module AlegreV2 @@ -55,11 +59,18 @@ def sync_path_for_type(type) end def async_path(project_media) - "/similarity/async/#{get_type(project_media)}" + self.async_path_for_type(get_type(project_media)) + end + + def async_path_for_type(type) + "/similarity/async/#{type}" end def delete_path(project_media) - type = get_type(project_media) + self.delete_path_for_type(get_type(project_media)) + end + + def delete_path_for_type(type) "/#{type}/similarity/" end @@ -122,6 +133,10 @@ def request(method, path, params, retries=3) end end + def request_delete_from_raw(params, type) + request("delete", delete_path_for_type(type), params) + end + def request_delete(data, project_media) request("delete", delete_path(project_media), data) end @@ -148,28 +163,32 @@ def get_type(project_media) type end + def content_hash_for_value(value) + value.nil? ? nil : Digest::MD5.hexdigest(value) + end + def content_hash(project_media, field) if Bot::Alegre::ALL_TEXT_SIMILARITY_FIELDS.include?(field) - Digest::MD5.hexdigest(project_media.send(field)) + content_hash_for_value(project_media.send(field)) + elsif project_media.is_link? + return content_hash_for_value(project_media.media.url) + elsif project_media.is_a?(TemporaryProjectMedia) + return Rails.cache.read("url_sha:#{project_media.url}") + elsif project_media.is_uploaded_media? + return project_media.media.file.filename.split(".").first else - if project_media.is_link? - return Digest::MD5.hexdigest(project_media.media.url) - elsif project_media.is_a?(TemporaryProjectMedia) - return Rails.cache.read("url_sha:#{project_media.url}") - elsif !project_media.is_text? - return project_media.media.file.filename.split(".").first - else - return Digest::MD5.hexdigest(project_media.send(field).to_s) - end + return content_hash_for_value(project_media.send(field).to_s) end end def generic_package(project_media, field) - { - content_hash: content_hash(project_media, field), + content_hash_value = content_hash(project_media, field) + params = { doc_id: item_doc_id(project_media, field), context: get_context(project_media, field) } + params[:content_hash] = content_hash_value if !content_hash_value.nil? + params end def delete_package(project_media, field, params={}, quiet=false) @@ -267,6 +286,22 @@ def store_package_text(project_media, field, params) generic_package_text(project_media, field, params) end + def index_async_with_params(params, type, suppress_search_response=true) + request("post", async_path_for_type(type), params.merge(suppress_search_response: suppress_search_response)) + end + + def index_sync_with_params(params, type) + query_sync_with_params(params, type) + end + + def query_sync_with_params(params, type) + request("post", sync_path_for_type(type), params) + end + + def query_async_with_params(params, type) + request("post", async_path_for_type(type), params) + end + def get_sync(project_media, field=nil, params={}) request_sync( store_package(project_media, field, params), @@ -286,6 +321,10 @@ def delete(project_media, field=nil, params={}) delete_package(project_media, field, params), project_media ) + rescue StandardError => e + error = Bot::Alegre::Error.new(e) + Rails.logger.error("[Alegre Bot] Exception on Delete for ProjectMedia ##{project_media.id}: #{error.class} - #{error.message}") + CheckSentry.notify(error, bot: "alegre", project_media: project_media, params: params, field: field) end def get_per_model_threshold(project_media, threshold) @@ -298,7 +337,7 @@ def get_per_model_threshold(project_media, threshold) end def isolate_relevant_context(project_media, result) - result["context"].select{|x| ([x["team_id"]].flatten & [project_media.team_id].flatten).count > 0 && !x["temporary_media"]}.first + (result["contexts"]||result["context"]).select{|x| ([x["team_id"]].flatten & [project_media.team_id].flatten).count > 0 && !x["temporary_media"]}.first end def get_target_field(project_media, field) @@ -319,7 +358,7 @@ def parse_similarity_results(project_media, field, results, relationship_type) context: result["context"], model: result["model"], source_field: get_target_field(project_media, field), - target_field: get_target_field(project_media, result["field"]), + target_field: get_target_field(project_media, result["field"] || result["context"]["field"]), relationship_type: relationship_type } ] @@ -485,25 +524,27 @@ def wait_for_results(project_media, args) end def get_items_with_similar_media_v2(args={}) + text = args[:text] + field = args[:field] media_url = args[:media_url] project_media = args[:project_media] threshold = args[:threshold] team_ids = args[:team_ids] type = args[:type] - if ['audio', 'image', 'video'].include?(type) - if project_media.nil? - project_media = TemporaryProjectMedia.new - project_media.url = media_url - project_media.id = Digest::MD5.hexdigest(project_media.url).to_i(16) - project_media.team_id = team_ids - project_media.type = type - end - get_similar_items_v2_async(project_media, nil, threshold) - wait_for_results(project_media, args) - response = get_similar_items_v2_callback(project_media, nil) - delete(project_media, nil) if project_media.is_a?(TemporaryProjectMedia) - return response + if project_media.nil? + project_media = TemporaryProjectMedia.new + project_media.text = text + project_media.field = field + project_media.url = media_url + project_media.id = Digest::MD5.hexdigest(project_media.url).to_i(16) + project_media.team_id = team_ids + project_media.type = type end + get_similar_items_v2_async(project_media, nil, threshold) + wait_for_results(project_media, args) + response = get_similar_items_v2_callback(project_media, nil) + delete(project_media, nil) if project_media.is_a?(TemporaryProjectMedia) + return response end def process_alegre_callback(params) @@ -512,9 +553,11 @@ def process_alegre_callback(params) should_relate = true if project_media.nil? project_media = TemporaryProjectMedia.new + project_media.text = params.dig('data', 'item', 'raw', 'text') project_media.url = params.dig('data', 'item', 'raw', 'url') project_media.id = params.dig('data', 'item', 'raw', 'context', 'project_media_id') project_media.team_id = params.dig('data', 'item', 'raw', 'context', 'team_id') + project_media.field = params.dig('data', 'item', 'raw', 'context', 'field') project_media.type = params['model_type'] should_relate = false end diff --git a/app/models/concerns/alegre_webhooks.rb b/app/models/concerns/alegre_webhooks.rb index 9832e44417..75246680d7 100644 --- a/app/models/concerns/alegre_webhooks.rb +++ b/app/models/concerns/alegre_webhooks.rb @@ -25,6 +25,8 @@ def webhook(request) doc_id = body.dig('data', 'requested', 'id') # search for doc_id on completed full-circuit callbacks doc_id = body.dig('data', 'item', 'id') if doc_id.nil? + # search for doc_id on indexed-but-not-searched callbacks + doc_id = body.dig('data', 'item', 'doc_id') if doc_id.nil? # search for doc_id on completed short-circuit callbacks (i.e. items already known to Alegre but added context TODO make these the same structure) doc_id = body.dig('data', 'item', 'raw', 'doc_id') if doc_id.nil? if doc_id.blank? diff --git a/app/models/concerns/project_media_cached_fields.rb b/app/models/concerns/project_media_cached_fields.rb index d5f1161aef..bfad29395c 100644 --- a/app/models/concerns/project_media_cached_fields.rb +++ b/app/models/concerns/project_media_cached_fields.rb @@ -70,14 +70,23 @@ def title_or_description_update } } - FACT_CHECK_EVENT = { - model: FactCheck, - affected_ids: proc { |fc| [fc.claim_description.project_media_id] }, - events: { - save: :recalculate, - destroy: :recalculate + FACT_CHECK_EVENTS = [ + { + model: FactCheck, + affected_ids: proc { |fc| [fc.claim_description.project_media_id] }, + events: { + save: :recalculate, + destroy: :recalculate + } + }, + { + model: ClaimDescription, + affected_ids: proc { |cd| [cd.project_media_id, cd.project_media_id_before_last_save] }, + events: { + save: :recalculate + } } - } + ] { is_suggested: Relationship.suggested_type, is_confirmed: Relationship.confirmed_type }.each do |field_name, _type| cached_field field_name, @@ -181,28 +190,28 @@ def title_or_description_update cached_field :fact_check_id, start_as: nil, recalculate: :recalculate_fact_check_id, - update_on: [FACT_CHECK_EVENT] + update_on: FACT_CHECK_EVENTS cached_field :fact_check_title, start_as: nil, recalculate: :recalculate_fact_check_title, - update_on: [FACT_CHECK_EVENT] + update_on: FACT_CHECK_EVENTS cached_field :fact_check_summary, start_as: nil, recalculate: :recalculate_fact_check_summary, - update_on: [FACT_CHECK_EVENT] + update_on: FACT_CHECK_EVENTS cached_field :fact_check_url, start_as: nil, recalculate: :recalculate_fact_check_url, - update_on: [FACT_CHECK_EVENT] + update_on: FACT_CHECK_EVENTS cached_field :fact_check_published_on, start_as: 0, update_es: true, recalculate: :recalculate_fact_check_published_on, - update_on: [FACT_CHECK_EVENT] + update_on: FACT_CHECK_EVENTS cached_field :description, recalculate: :recalculate_description, @@ -272,8 +281,8 @@ def title_or_description_update model: Tag, affected_ids: proc { |t| [t.annotated_id.to_i] }, events: { - save: :cached_field_project_media_tags_as_sentence_save, - destroy: :cached_field_project_media_tags_as_sentence_destroy, + save: :recalculate, + destroy: :recalculate, } } ] @@ -720,11 +729,11 @@ def cached_field_project_media_added_as_similar_by_name_create(_target) self.user && self.user == BotUser.alegre_user ? 'Check' : self.user&.name end - def cached_field_project_media_added_as_similar_by_name_destroy(_target) + def self.cached_field_project_media_added_as_similar_by_name_destroy(_target) nil end - def cached_field_project_media_confirmed_as_similar_by_name_destroy(_target) + def self.cached_field_project_media_confirmed_as_similar_by_name_destroy(_target) nil end end @@ -744,14 +753,4 @@ def cached_field_project_media_folder_save(_target) self.title end end - - Tag.class_eval do - def cached_field_project_media_tags_as_sentence_save(target) - target.tags_as_sentence.split(', ').concat([self.tag_text]).uniq.join(', ') - end - - def cached_field_project_media_tags_as_sentence_destroy(target) - target.tags_as_sentence.split(', ').reject{ |tt| tt == self.tag_text }.uniq.join(', ') - end - end end diff --git a/app/models/concerns/project_media_creators.rb b/app/models/concerns/project_media_creators.rb index 032bd86d5c..dca9f1566b 100644 --- a/app/models/concerns/project_media_creators.rb +++ b/app/models/concerns/project_media_creators.rb @@ -11,6 +11,28 @@ def create_metrics_annotation end end + def create_claim_description_and_fact_check + cd = ClaimDescription.create!(description: self.set_claim_description, context: self.set_claim_context, project_media: self, skip_check_ability: true) unless self.set_claim_description.blank? + fc = nil + unless self.set_fact_check.blank? + fact_check = self.set_fact_check.with_indifferent_access + fc = FactCheck.create!({ + title: fact_check['title'], + summary: fact_check['summary'], + language: fact_check['language'], + url: fact_check['url'], + publish_report: !!fact_check['publish_report'], + signature: Digest::MD5.hexdigest([self.set_fact_check.to_json, self.team_id].join(':')), + claim_description: cd, + report_status: (fact_check['publish_report'] ? 'published' : 'unpublished'), + rating: self.set_status, + tags: self.set_tags.to_a.map(&:strip), + skip_check_ability: true + }) + end + fc + end + private def create_team_tasks @@ -236,33 +258,10 @@ def create_relationship(type = Relationship.confirmed_type) end end - def create_claim_description_and_fact_check - cd = ClaimDescription.create!(description: self.set_claim_description, context: self.set_claim_context, project_media: self, skip_check_ability: true) unless self.set_claim_description.blank? - fc = nil - unless self.set_fact_check.blank? - fact_check = self.set_fact_check.with_indifferent_access - fc = FactCheck.create!({ - title: fact_check['title'], - summary: fact_check['summary'], - language: fact_check['language'], - url: fact_check['url'], - publish_report: !!fact_check['publish_report'], - signature: Digest::MD5.hexdigest([self.set_fact_check.to_json, self.team_id].join(':')), - claim_description: cd, - report_status: (fact_check['publish_report'] ? 'published' : 'unpublished'), - rating: self.set_status, - tags: self.set_tags.to_a.map(&:strip), - skip_check_ability: true - }) - end - fc - end - def create_tags_in_background if self.set_tags.is_a?(Array) - project_media_id = self.id - tags_json = self.set_tags.to_json - Tag.run_later_in(1.second, 'create_project_media_tags', project_media_id, tags_json, user_id: self.user_id) + tags = self.set_tags.reject { |t| t.blank? } + Tag.run_later_in(1.second, 'create_project_media_tags', self.id, tags.to_json, user_id: self.user_id) unless tags.empty? end end end diff --git a/app/models/concerns/project_media_getters.rb b/app/models/concerns/project_media_getters.rb index b8662224e7..0ce240c337 100644 --- a/app/models/concerns/project_media_getters.rb +++ b/app/models/concerns/project_media_getters.rb @@ -11,10 +11,6 @@ def is_link? self.media.type == "Link" end - def is_uploaded_image? - self.media.type == "UploadedImage" - end - def is_blank? self.media.type == "Blank" end @@ -28,7 +24,11 @@ def is_audio? end def is_image? - self.is_uploaded_image? + self.media.type == "UploadedImage" + end + + def is_uploaded_media? + self.is_image? || self.is_audio? || self.is_video? end def is_text? diff --git a/app/models/concerns/smooch_messages.rb b/app/models/concerns/smooch_messages.rb index 6c83c938a4..88913b58ce 100644 --- a/app/models/concerns/smooch_messages.rb +++ b/app/models/concerns/smooch_messages.rb @@ -281,8 +281,8 @@ def bundle_list_of_messages(list, last, reject_payload = false) def bundle_list_of_messages_to_items(list, last) # Collect messages from list based on media files, long text and short text # so we have three types of messages - # Long text (text with number of words > min_number_of_words_for_tipline_submit_shortcut) - # Short text (text with number of words <= min_number_of_words_for_tipline_submit_shortcut) + # Long text (text with number of words > min_number_of_words_for_tipline_long_text) + # Short text (text with number of words <= min_number_of_words_for_tipline_long_text) # Media (image, audio, video, etc) messages = [] # Define a text variable to hold short text @@ -291,20 +291,34 @@ def bundle_list_of_messages_to_items(list, last) if message['type'] == 'text' # Get an item for long text (message that match number of words condition) if message['payload'].nil? - contains_link = Twitter::TwitterText::Extractor.extract_urls(message['text']) - messages << message if !contains_link.blank? || ::Bot::Alegre.get_number_of_words(message['text'].to_s) > CheckConfig.get('min_number_of_words_for_tipline_submit_shortcut', 10, :integer) - text << message['text'] + link_from_message = nil + begin + link_from_message = self.extract_url(message['text']) + rescue SecurityError + link_from_message = nil + end + messages << message if !link_from_message.blank? || ::Bot::Alegre.get_number_of_words(message['text'].to_s) > self.min_number_of_words_for_tipline_long_text + # Text should be a link only in case we have two matched items (link and long text) + text << (link_from_message.blank? ? message['text'] : link_from_message.url) end elsif !message['mediaUrl'].blank? # Get an item for each media file - if !message['text'].blank? && ::Bot::Alegre.get_number_of_words(message['text'].to_s) > CheckConfig.get('min_number_of_words_for_tipline_submit_shortcut', 10, :integer) + if !message['text'].blank? && ::Bot::Alegre.get_number_of_words(message['text'].to_s) > self.min_number_of_words_for_tipline_long_text message['caption'] = message['text'] + # Text should be a media url in case we have two matched items (media and caption) + message['text'] = message['mediaUrl'].to_s + else + message['text'] = [message['text'], message['mediaUrl'].to_s].compact.join("\n#{Bot::Smooch::MESSAGE_BOUNDARY}") end - message['text'] = [message['text'], message['mediaUrl'].to_s].compact.join("\n#{Bot::Smooch::MESSAGE_BOUNDARY}") text << message['text'] messages << self.adjust_media_type(message) end end + # Attach text to exising messages and return all messages + self.attach_text_to_messages(text, messages, last) + end + + def attach_text_to_messages(text, messages, last) # collect all text in right order and add a boundary so we can easily split messages if needed all_text = text.reject{ |t| t.blank? }.join("\n#{Bot::Smooch::MESSAGE_BOUNDARY}") if messages.blank? @@ -400,8 +414,7 @@ def save_message(message_json, app_id, author = nil, request_type = 'default_req end unless associated.nil? self.smoooch_post_save_message_actions(message, associated, app_id, author, request_type, associated_obj) - # Check if message contains caption then create an item and force relationship - self.relate_item_and_caption(message, associated, app_id, author, request_type, associated_obj) unless message['caption'].blank? + self.smooch_relate_items_for_same_message(message, associated, app_id, author, request_type, associated_obj) end end end @@ -416,21 +429,39 @@ def smoooch_post_save_message_actions(message, associated, app_id, author, reque self.send_report_to_user(message['authorId'], message, associated, message['language'], 'fact_check_report') if self.should_try_to_send_report?(request_type, associated) end - def relate_item_and_caption(message, associated, app_id, author, request_type, associated_obj) + def smooch_relate_items_for_same_message(message, associated, app_id, author, request_type, associated_obj) + if !message['caption'].blank? + # Check if message contains caption then create an item and force relationship + self.relate_item_and_text(message, associated, app_id, author, request_type, associated_obj, Relationship.confirmed_type) + elsif message['type'] == 'text' && associated.class.name == 'ProjectMedia' && associated.media.type == 'Link' + # Check if message of type text contain a link and long text + # Text words equal the number of words - 1(which is the link size) + text_words = ::Bot::Alegre.get_number_of_words(message['text']) - 1 + if text_words > self.min_number_of_words_for_tipline_long_text + # Remove link from text + link = self.extract_url(message['text']) + message['text'] = message['text'].remove(link.url) + self.relate_item_and_text(message, associated, app_id, author, request_type, associated_obj, Relationship.confirmed_type) + end + end + end + + def relate_item_and_text(message, associated, app_id, author, request_type, associated_obj, relationship_type) message['_id'] = SecureRandom.hex message['type'] = 'text' + message['text'] = message['caption'] unless message['caption'].nil? message['request_body'] = message['text'] - message['text'] = message['caption'] message.delete('caption') message.delete('mediaUrl') target = self.create_project_media_from_message(message) unless target.nil? smoooch_post_save_message_actions(message, target, app_id, author, request_type, associated_obj) - Relationship.create_unless_exists(associated.id, target.id, Relationship.suggested_type) + Relationship.create_unless_exists(associated.id, target.id, relationship_type) end end def smooch_save_tipline_request(message, associated, app_id, author, request_type, associated_obj) + text = message['text'] message['text'] = message['request_body'] unless message['request_body'].blank? message.delete('request_body') fields = { smooch_data: message.merge({ app_id: app_id }) } @@ -450,6 +481,8 @@ def smooch_save_tipline_request(message, associated, app_id, author, request_typ associated.save! end end + # Back message text to original one + message['text'] = text end def create_tipline_requests(associated, author, fields) @@ -525,5 +558,10 @@ def send_custom_message_to_user(team, uid, timestamp, message, language) success = (response && response.code.to_i < 400) success end + + def min_number_of_words_for_tipline_long_text + # Define a min number of words to create a media + CheckConfig.get('min_number_of_words_for_tipline_long_text') || CheckConfig.get('min_number_of_words_for_tipline_submit_shortcut', 10, :integer) + end end end diff --git a/app/models/concerns/smooch_resend.rb b/app/models/concerns/smooch_resend.rb index 455ce33ac8..ea96af55ad 100644 --- a/app/models/concerns/smooch_resend.rb +++ b/app/models/concerns/smooch_resend.rb @@ -109,13 +109,19 @@ def send_message_on_template_button_click(_message, uid, language, info) end def clicked_on_template_button?(message) - ['report', 'message'].include?(self.get_information_from_clicked_template_button(message).first.to_s) + ['report', 'message', 'newsletter'].include?(self.get_information_from_clicked_template_button(message).first.to_s) end def get_information_from_clicked_template_button(message, delete = false) quoted_id = message.dig('quotedMessage', 'content', '_id') unless quoted_id.blank? - info = Rails.cache.read("smooch:original:#{quoted_id}").to_s.split(':') + info = Rails.cache.read("smooch:original:#{quoted_id}").to_s + begin + original = JSON.parse(info) + info = ['newsletter', original['language']] if original['fallback_template'] == 'newsletter' + rescue + info = info.split(':') + end Rails.cache.delete("smooch:original:#{quoted_id}") if delete return info end @@ -130,6 +136,10 @@ def template_button_click_callback(message, uid, language) self.send_report_on_template_button_click(message, uid, language, info) when 'message' self.send_message_on_template_button_click(message, uid, language, info) + when 'newsletter' + team_id = self.config['team_id'].to_i + language = info[1] || language + self.toggle_subscription(uid, language, team_id, self.get_platform_from_message(message), self.get_workflow(language)) if self.user_is_subscribed_to_newsletter?(uid, language, team_id) end end end diff --git a/app/models/concerns/smooch_search.rb b/app/models/concerns/smooch_search.rb index 1454747f85..ca6941093b 100644 --- a/app/models/concerns/smooch_search.rb +++ b/app/models/concerns/smooch_search.rb @@ -246,8 +246,9 @@ def should_restrict_by_language?(team_ids) end def search_by_keywords_for_similar_published_fact_checks(words, after, team_ids, feed_id = nil, language = nil) + types = CheckSearch::MEDIA_TYPES.clone.push('blank') search_fields = %w(title description fact_check_title fact_check_summary extracted_text url claim_description_content) - filters = { keyword: words.join('+'), keyword_fields: { fields: search_fields }, sort: 'recent_activity', eslimit: 3 } + filters = { keyword: words.join('+'), keyword_fields: { fields: search_fields }, sort: 'recent_activity', eslimit: 3, show: types } filters.merge!({ fc_language: [language] }) if should_restrict_by_language?(team_ids) filters.merge!({ sort: 'score' }) if words.size > 1 # We still want to be able to return the latest fact-checks if a meaninful query is not passed feed_id.blank? ? filters.merge!({ report_status: ['published'] }) : filters.merge!({ feed_id: feed_id }) diff --git a/app/models/concerns/team_private.rb b/app/models/concerns/team_private.rb index 568ddea800..61c817076b 100644 --- a/app/models/concerns/team_private.rb +++ b/app/models/concerns/team_private.rb @@ -130,4 +130,11 @@ def remove_is_default_project_flag # as admin not allowed to delete the default project self.default_folder.update_columns(is_default: false) end + + def empty_data_structure + data_structure = MonthlyTeamStatistic.new.formatted_hash + data_structure["Language"] = self.default_language + data_structure["Org"] = self.name + [data_structure] + end end diff --git a/app/models/explainer.rb b/app/models/explainer.rb index 5e2992bd8d..aa25ea9e42 100644 --- a/app/models/explainer.rb +++ b/app/models/explainer.rb @@ -73,24 +73,26 @@ def self.update_paragraphs_in_alegre(id, previous_paragraphs_count, timestamp) # Index title params = { + content_hash: Bot::Alegre.content_hash_for_value(explainer.title), doc_id: Digest::MD5.hexdigest(['explainer', explainer.id, 'title'].join(':')), + context: base_context.merge({ field: 'title' }), text: explainer.title, models: ALEGRE_MODELS_AND_THRESHOLDS.keys, - context: base_context.merge({ field: 'title' }) } - Bot::Alegre.request('post', '/text/similarity/', params) + Bot::Alegre.index_async_with_params(params, "text") # Index paragraphs count = 0 explainer.description.to_s.gsub(/\r\n?/, "\n").split(/\n+/).reject{ |paragraph| paragraph.strip.blank? }.each do |paragraph| count += 1 params = { + content_hash: Bot::Alegre.content_hash_for_value(paragraph.strip), doc_id: Digest::MD5.hexdigest(['explainer', explainer.id, 'paragraph', count].join(':')), + context: base_context.merge({ paragraph: count }), text: paragraph.strip, models: ALEGRE_MODELS_AND_THRESHOLDS.keys, - context: base_context.merge({ paragraph: count }) } - Bot::Alegre.request('post', '/text/similarity/', params) + Bot::Alegre.index_async_with_params(params, "text") end # Remove paragraphs that don't exist anymore (we delete after updating in order to avoid race conditions) @@ -101,7 +103,7 @@ def self.update_paragraphs_in_alegre(id, previous_paragraphs_count, timestamp) quiet: true, context: base_context.merge({ paragraph: count }) } - Bot::Alegre.request('delete', '/text/similarity/', params) + Bot::Alegre.request_delete_from_raw(params, "text") end end @@ -116,9 +118,9 @@ def self.search_by_similarity(text, language, team_id) language: language } } - response = Bot::Alegre.request('post', '/text/similarity/search/', params) + response = Bot::Alegre.query_sync_with_params(params, "text") results = response['result'].to_a.sort_by{ |result| result['_score'] } - explainer_ids = results.collect{ |result| result.dig('_source', 'context', 'explainer_id').to_i }.uniq.first(3) + explainer_ids = results.collect{ |result| result.dig('context', 'explainer_id').to_i }.uniq.first(3) explainer_ids.empty? ? Explainer.none : Explainer.where(team_id: team_id, id: explainer_ids) end diff --git a/app/models/link.rb b/app/models/link.rb index 6a74256e2e..66bc3707c1 100644 --- a/app/models/link.rb +++ b/app/models/link.rb @@ -3,7 +3,7 @@ class Link < Media validates :url, presence: true, on: :create validate :validate_pender_result_and_retry, on: :create - validate :url_is_unique, on: :create + validate :url_is_unique, :url_max_size, on: :create after_create :set_pender_result_as_annotation, :set_account @@ -68,6 +68,11 @@ def url_is_unique end end + def url_max_size + # Use 2k as max size to stay within safe limits for a unique URL index in PostgreSQL as max size is 2712 bytes. + errors.add(:base, "Media URL exceeds the maximum size (2000 bytes)") if !self.url.nil? && self.url.bytesize > CheckConfig.get('url_max_size', 2000, :integer) + end + def validate_pender_result_and_retry self.validate_pender_result(false, true) # raise error for invalid links diff --git a/app/models/project_media.rb b/app/models/project_media.rb index 223403cc93..ab169298d3 100644 --- a/app/models/project_media.rb +++ b/app/models/project_media.rb @@ -430,6 +430,31 @@ def apply_rules_and_actions_on_update self.team.apply_rules_and_actions(self, rule_ids) end + def self.handle_fact_check_for_existing_claim(existing_pm,new_pm) + if existing_pm.fact_check.blank? + existing_pm.append_fact_check_from(new_pm) + return existing_pm + elsif existing_pm.fact_check.present? + if existing_pm.fact_check.language != new_pm.set_fact_check['language'] + new_pm.replace_with_blank_media + return new_pm + end + end + end + + def append_fact_check_from(new_pm) + self.set_claim_description = new_pm.set_claim_description + self.set_fact_check = new_pm.set_fact_check + self.create_claim_description_and_fact_check + end + + def replace_with_blank_media + m = Blank.create! + self.set_original_claim = nil + self.media_id = m.id + self.save! + end + protected def add_extra_elasticsearch_data(ms) diff --git a/app/models/relationship.rb b/app/models/relationship.rb index fbded86531..868b74fe09 100644 --- a/app/models/relationship.rb +++ b/app/models/relationship.rb @@ -12,18 +12,21 @@ class Relationship < ApplicationRecord before_validation :set_user, on: :create before_validation :set_confirmed, if: :is_being_confirmed?, on: :update + before_validation :move_fact_check_and_report_to_main, on: :create validate :relationship_type_is_valid, :items_are_from_the_same_team validate :target_not_published_report, on: :create validate :similar_item_exists, on: :create, if: proc { |r| r.is_suggested? } validate :cant_be_related_to_itself validates :relationship_type, uniqueness: { scope: [:source_id, :target_id], message: :already_exists }, on: :create + before_create :point_targets_to_new_source before_create :destroy_same_suggested_item, if: proc { |r| r.is_confirmed? } after_create :move_to_same_project_as_main, prepend: true - after_create :point_targets_to_new_source, :update_counters, prepend: true + after_create :update_counters, prepend: true after_update :reset_counters, prepend: true after_update :propagate_inversion after_save :turn_off_unmatched_field, if: proc { |r| r.is_confirmed? || r.is_suggested? } + after_save :move_explainers_to_source, if: proc { |r| r.is_confirmed? } before_destroy :archive_detach_to_list after_destroy :update_counters, prepend: true after_destroy :turn_on_unmatched_field, if: proc { |r| r.is_confirmed? || r.is_suggested? } @@ -158,6 +161,21 @@ def self.create_unless_exists(source_id, target_id, relationship_type, options = exception_message = nil exception_class = nil if r.nil? + # Add to existing media cluster if source is already a target: + # If we're trying to create a relationship between C (target_id) and B (source_id), but there is already a relationship between A (source_id) and B (target_id), + # then, instead, create the relationship between A (source_id) and C (target_id) (so, if A's cluster contains B, then C comes in and our algorithm says C is similar + # to B, it is added to A's cluster). Exception: If the relationship between A (source_id) and B (target_id) is a suggestion, we should not create any relationship + # at all when trying to create a relationship between C (target_id) and B (source_id) (regardless if it’s a suggestion or a confirmed match) - but we should log that case. + existing = Relationship.where(target_id: source_id).first + unless existing.nil? + if existing.relationship_type == Relationship.suggested_type + error_msg = StandardError.new('Not creating relationship because requested source_id is already suggested to another item.') + CheckSentry.notify(error_msg, source_id: source_id, target_id: target_id, relationship_type: relationship_type, options: options) + return nil + end + source_id = existing.source_id + end + begin r = Relationship.new r.skip_check_ability = true @@ -187,21 +205,20 @@ def self.create_unless_exists(source_id, target_id, relationship_type, options = def update_elasticsearch_parent(action = 'create_or_update') return if self.is_default? || self.disable_es_callbacks || RequestStore.store[:disable_es_callbacks] - # touch target to update `updated_at` date - target = self.target - unless target.nil? + [self.source, self.target].compact.each do |pm| updated_at = Time.now - target.update_columns(updated_at: updated_at) + # touch item to update `updated_at` date + pm.update_columns(updated_at: updated_at) data = { updated_at: updated_at.utc } data['parent_id'] = { method: "#{action}_parent_id", klass: self.class.name, id: self.id, - default: target_id, + default: pm.id, type: 'int' - } if self.is_confirmed? - target.update_elasticsearch_doc(data.keys, data, target.id, true) - end + } + pm.update_elasticsearch_doc(data.keys, data, pm.id, true) + end if self.is_confirmed? end def set_unmatched_field(value) @@ -252,7 +269,11 @@ def propagate_inversion claim.project_media_id = self.source_id claim.save end - Relationship.where(source_id: self.target_id).update_all({ source_id: self.source_id }) + Relationship.where(source_id: self.target_id).find_each do |r| + r.source_id = self.source_id + r.skip_check_ability = true + r.save! + end self.source&.clear_cached_fields self.target&.clear_cached_fields Relationship.delay_for(1.second).propagate_inversion(ids, self.source_id) @@ -286,7 +307,7 @@ def similar_item_exists def point_targets_to_new_source # Get existing targets for the source target_ids = Relationship.where(source_id: self.source_id).map(&:target_id) - # Delete duplicate relation from target(CHECK-1603) + # Delete duplicate relationships from target (CHECK-1603) Relationship.where(source_id: self.target_id, target_id: target_ids).delete_all Relationship.where(source_id: self.target_id).find_each do |old_relationship| old_relationship.delete @@ -295,6 +316,8 @@ def point_targets_to_new_source weight: old_relationship.weight } Relationship.create_unless_exists(self.source_id, old_relationship.target_id, old_relationship.relationship_type, options) + old_relationship.source.clear_cached_fields + old_relationship.target.clear_cached_fields end end @@ -332,6 +355,27 @@ def move_to_same_project_as_main end end + def move_explainers_to_source + # Three cases to move explainers + # 1) Relationship is new and confirmed + # 2) Item is being confirmed (move from suggested to confirmed) + # 3) Pin item (sawp source_id & target_id) + if self.relationship_type_before_last_save.nil? || self.is_being_confirmed? || (self.source_id_before_last_save && self.source_id_before_last_save == self.target_id) + ExplainerItem.transaction do + # Destroy common Explainer from target item (use destroy to log this event) + explainer_ids = ExplainerItem.where(project_media_id: [self.source_id, self.target_id]) + .group('explainer_id').having("count(explainer_id) = ?", 2).pluck(:explainer_id) + ExplainerItem.where(explainer_id: explainer_ids, project_media_id: self.target_id).destroy_all + # Move the Explainer from target to source by using update_all(as no callbacks) and then update logs + ExplainerItem.where(project_media_id: self.target_id).update_all(project_media_id: self.source_id) + # Update logs (to make item history consistent with Explainers attached to item) + Version.from_partition(self.source.team_id) + .where(event_type: 'create_explaineritem', associated_type: 'ProjectMedia', associated_id: self.target_id) + .update_all(associated_id: self.source_id) + end + end + end + def destroy_same_suggested_item Relationship.transaction do # Check if same item already exists as a suggested item by a bot @@ -344,4 +388,32 @@ def destroy_same_suggested_item def cant_be_related_to_itself errors.add(:base, I18n.t(:item_cant_be_related_to_itself)) if self.source_id == self.target_id end + + def move_fact_check_and_report_to_main + Relationship.transaction do + source = self.source + target = self.target + if source && target && source.team_id == target.team_id # Must verify since this method runs before the validation + target_report = target.get_annotations('report_design').to_a.map(&:load).last + + # If the child item has a claim/fact-check and published report but the parent item doesn't, then move the claim/fact-check/report from the child to the parent + if !source.claim_description && target.claim_description && target_report && target_report.get_field_value('state') == 'published' + # Move report + target_report.annotated_id = source.id + target_report.save! + + # Move claim/fact-check + claim_description = target.claim_description + claim_description.project_media = source + claim_description.save! + + # Clear caches + source.clear_cached_fields + target.clear_cached_fields + source.reload + target.reload + end + end + end + end end diff --git a/app/models/request.rb b/app/models/request.rb index 81c9e58346..8f48f5b809 100644 --- a/app/models/request.rb +++ b/app/models/request.rb @@ -53,7 +53,7 @@ def attach_to_similar_request!(alegre_limit = 20) models_thresholds = self.text_similarity_settings.reject{ |_k, v| v['min_words'] > words } if models_thresholds.count > 0 params = { text: media.quote, models: models_thresholds.keys, per_model_threshold: models_thresholds.transform_values{ |v| v['threshold'] }, limit: alegre_limit, context: context } - similar_request_id = ::Bot::Alegre.request('post', '/text/similarity/search/', params)&.dig('result').to_a.collect{ |result| result&.dig('_source', 'context', 'request_id').to_i }.find{ |id| id != 0 && id < self.id } + similar_request_id = ::Bot::Alegre.query_sync_with_params(params, "text")&.dig('result').to_a.collect{ |result| result&.dig('context', 'request_id').to_i }.find{ |id| id != 0 && id < self.id } end # elsif ['UploadedImage', 'UploadedAudio', 'UploadedVideo'].include?(media.type) # threshold = 0.85 #FIXME: Should be feed setting @@ -194,7 +194,7 @@ def self.send_to_alegre(id) models: request.text_similarity_settings.keys(), context: context } - ::Bot::Alegre.request('post', '/text/similarity/', params) + ::Bot::Alegre.index_async_with_params(params, "text") # elsif ['UploadedImage', 'UploadedAudio', 'UploadedVideo'].include?(media.type) # type = media.type.gsub(/^Uploaded/, '').downcase # url = media.file&.file&.public_url diff --git a/app/models/team.rb b/app/models/team.rb index c1f33d07f3..329b044e62 100644 --- a/app/models/team.rb +++ b/app/models/team.rb @@ -443,11 +443,14 @@ def data_report end else data = Rails.cache.read("data:report:#{self.id}") - return nil if data.blank? - data.map.with_index do |row, i| - row['Month'] = "#{i + 1}. #{row['Month']}" - row.reject { |key, _value| ['Average number of conversations per day', 'Number of messages sent'].include?(key) } + if data.blank? + empty_data_structure + else + data.map.with_index do |row, i| + row['Month'] = "#{i + 1}. #{row['Month']}" + row.reject { |key, _value| ['Average number of conversations per day', 'Number of messages sent'].include?(key) } + end end end end diff --git a/app/models/tipline_request.rb b/app/models/tipline_request.rb index 62e5bf28d8..3b753a78d6 100644 --- a/app/models/tipline_request.rb +++ b/app/models/tipline_request.rb @@ -2,6 +2,7 @@ class TiplineRequest < ApplicationRecord include CheckElasticSearch belongs_to :associated, polymorphic: true + belongs_to :project_media, -> { where(tipline_requests: { associated_type: 'ProjectMedia' }) }, foreign_key: 'associated_id', optional: true belongs_to :user, optional: true before_validation :set_team_and_user, :set_smooch_data_fields, on: :create diff --git a/app/resources/api/v2/report_resource.rb b/app/resources/api/v2/report_resource.rb index 29ee00cc87..3289b0b61a 100644 --- a/app/resources/api/v2/report_resource.rb +++ b/app/resources/api/v2/report_resource.rb @@ -57,7 +57,7 @@ def self.apply_text_similarity_filter(organization_ids, threshold, filters) ids = nil unless text.blank? fields = filters[:similarity_fields].blank? ? nil : filters[:similarity_fields].to_a.flatten - ids_and_scores = Bot::Alegre.get_similar_texts(organization_ids, text[0], fields, [{ value: threshold }], nil, filters.dig(:fuzzy, 0)) + ids_and_scores = Bot::Alegre.get_items_from_similar_text(organization_ids, text[0], fields, [{ value: threshold }], nil, filters.dig(:fuzzy, 0)) RequestStore.store[:scores] = ids_and_scores # Store the scores so we can return them ids = ids_and_scores.keys.uniq ids = [0] if ids.blank? diff --git a/app/workers/reindex_alegre_workspace.rb b/app/workers/reindex_alegre_workspace.rb index 093f8e56af..557a3db088 100644 --- a/app/workers/reindex_alegre_workspace.rb +++ b/app/workers/reindex_alegre_workspace.rb @@ -45,12 +45,15 @@ def get_default_query(team_id, last_id=nil) end def get_request_doc(pm, field, field_value) - Bot::Alegre.send_to_text_similarity_index_package( - pm, - field, - field_value, - Bot::Alegre.item_doc_id(pm, field) - ) + { + doc: Bot::Alegre.send_to_text_similarity_index_package( + pm, + field, + field_value, + Bot::Alegre.item_doc_id(pm, field) + ), + type: Bot::Alegre.get_pm_type(pm) + } end def get_request_docs_for_project_media(pm) @@ -64,11 +67,14 @@ def get_request_docs_for_project_media(pm) end end - def check_for_write(running_bucket, event_id, team_id, write_remains=false, in_processes=3) + def check_for_write(running_bucket, event_id, team_id, write_remains=false) # manage dispatch of documents to bulk similarity api call in parallel if running_bucket.length > 500 || write_remains log(event_id, 'Writing to Alegre...') - Parallel.map(running_bucket.each_slice(30).to_a, in_processes: in_processes) { |bucket_slice| Bot::Alegre.request('post', '/text/bulk_similarity/', { documents: bucket_slice }) } + running_bucket.each do |item| + # FIXME we need to go back to bulk uploads eventually + Bot::Alegre.query_async_with_params(item[:doc], item[:type]) + end log(event_id, 'Wrote to Alegre.') # track state in case job needs to restart write_last_id(event_id, team_id, running_bucket.last[:context][:project_media_id]) if running_bucket.length > 0 && running_bucket.last[:context] diff --git a/config/config.yml.example b/config/config.yml.example index cd9054374b..5c9b545d2e 100644 --- a/config/config.yml.example +++ b/config/config.yml.example @@ -279,6 +279,8 @@ development: &default export_csv_expire: 604800 # Seconds: Default is 7 days header_file_video_max_size_whatsapp: 16 # Megabytes header_file_video_max_size_check: 10 # Megabytes, should be less than WhatsApp limit + url_max_size: 2000 + min_number_of_words_for_tipline_long_text: 10 # Session # diff --git a/config/initializers/plugins.rb b/config/initializers/plugins.rb index b928f936ff..52e80f5d9d 100644 --- a/config/initializers/plugins.rb +++ b/config/initializers/plugins.rb @@ -1,2 +1,2 @@ # Load classes on boot, in production, that otherwise wouldn't be auto-loaded by default -CcDeville && Bot::Keep && Workflow::Workflow.workflows && CheckS3 && Bot::Tagger && Bot::Fetch && Bot::Smooch && Bot::Slack && Bot::Alegre && CheckChannels && RssFeed && UrlRewriter && ClusterTeam && ListExport +CcDeville && Bot::Keep && Workflow::Workflow.workflows && CheckS3 && Bot::Tagger && Bot::Fetch && Bot::Smooch && Bot::Slack && Bot::Alegre && CheckChannels && RssFeed && UrlRewriter && ClusterTeam && ListExport && TeamStatistics && CheckDataPoints diff --git a/db/migrate/20241015223059_add_index_to_created_at_for_articles.rb b/db/migrate/20241015223059_add_index_to_created_at_for_articles.rb new file mode 100644 index 0000000000..cc06680c6a --- /dev/null +++ b/db/migrate/20241015223059_add_index_to_created_at_for_articles.rb @@ -0,0 +1,8 @@ +class AddIndexToCreatedAtForArticles < ActiveRecord::Migration[6.1] + def change + execute "CREATE INDEX fact_check_created_at_day ON fact_checks (date_trunc('day', created_at))" + add_index :fact_checks, :created_at + execute "CREATE INDEX explainer_created_at_day ON explainers (date_trunc('day', created_at))" + add_index :explainers, :created_at + end +end diff --git a/db/migrate/20241123001206_add_unique_index_to_relationships_table.rb b/db/migrate/20241123001206_add_unique_index_to_relationships_table.rb new file mode 100644 index 0000000000..e8e83818c3 --- /dev/null +++ b/db/migrate/20241123001206_add_unique_index_to_relationships_table.rb @@ -0,0 +1,33 @@ +class AddUniqueIndexToRelationshipsTable < ActiveRecord::Migration[6.1] + def change + # The code below is a copy-paste from the rake task lib/tasks/migrate/20230216030351_delete_duplicate_relationships.rake + # and it's responsible for deleting any remaining duplicate relationship, but first, before running this migration, be + # sure to run the rake task above, with: rake check:migrate:delete_duplicate_relationships + duplicates = Relationship.group(:target_id).having('COUNT(id) > 1').count + n = duplicates.size + i = 0 + duplicates.each do |pm_id, count| + i += 1 + puts "[#{Time.now}] #{i}/#{n}" + if count > 1 + relationships = Relationship.where(target_id: pm_id).order('id ASC').to_a + # Keep the confirmed relationship, or the one whose model is image, video or audio... if none, keep the first one + keep = relationships.find{ |r| r.relationship_type == Relationship.confirmed_type } || relationships.find{ |r| ['image', 'video', 'audio'].include?(r.model) } || relationships.first + raise "No relationship to keep for target_id #{pm_id}!" if keep.nil? + relationships.each do |relationship| + if relationship.id == keep.id + puts " Keeping relationship ##{relationship.id}" + else + puts " Deleting relationship ##{relationship.id}" + relationship.delete + end + relationship.source.clear_cached_fields + relationship.target.clear_cached_fields + end + end + end + + remove_index :relationships, name: 'index_relationships_on_target_id' + add_index :relationships, :target_id, unique: true + end +end diff --git a/db/migrate/20241123135242_add_trigger_to_relationships_table.rb b/db/migrate/20241123135242_add_trigger_to_relationships_table.rb new file mode 100644 index 0000000000..c252a55983 --- /dev/null +++ b/db/migrate/20241123135242_add_trigger_to_relationships_table.rb @@ -0,0 +1,37 @@ +class AddTriggerToRelationshipsTable < ActiveRecord::Migration[6.1] + def up + # Create the trigger function + execute <<~SQL + CREATE OR REPLACE FUNCTION validate_relationships() + RETURNS TRIGGER AS $$ + BEGIN + -- Check if source_id exists as a target_id + IF EXISTS (SELECT 1 FROM relationships WHERE target_id = NEW.source_id) THEN + RAISE EXCEPTION 'source_id % already exists as a target_id', NEW.source_id; + END IF; + + -- Check if target_id exists as a source_id + IF EXISTS (SELECT 1 FROM relationships WHERE source_id = NEW.target_id) THEN + RAISE EXCEPTION 'target_id % already exists as a source_id', NEW.target_id; + END IF; + + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + SQL + + # Attach the trigger to the table (only on INSERT - we shouldn't have it for UPDATE since we need to support reverting a relationship + execute <<~SQL + CREATE TRIGGER enforce_relationships + BEFORE INSERT ON relationships + FOR EACH ROW + EXECUTE FUNCTION validate_relationships(); + SQL + end + + def down + # Remove the trigger and function if rolling back + execute "DROP TRIGGER IF EXISTS enforce_relationships ON relationships;" + execute "DROP FUNCTION IF EXISTS validate_relationships();" + end +end diff --git a/db/schema.rb b/db/schema.rb index bccdebba3c..636e74117f 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2024_10_09_192811) do +ActiveRecord::Schema.define(version: 2024_11_23_135242) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -121,6 +121,26 @@ END; $function$ SQL + create_function :validate_relationships, sql_definition: <<-'SQL' + CREATE OR REPLACE FUNCTION public.validate_relationships() + RETURNS trigger + LANGUAGE plpgsql + AS $function$ + BEGIN + -- Check if source_id exists as a target_id + IF EXISTS (SELECT 1 FROM relationships WHERE target_id = NEW.source_id) THEN + RAISE EXCEPTION 'source_id % already exists as a target_id', NEW.source_id; + END IF; + + -- Check if target_id exists as a source_id + IF EXISTS (SELECT 1 FROM relationships WHERE source_id = NEW.target_id) THEN + RAISE EXCEPTION 'target_id % already exists as a source_id', NEW.target_id; + END IF; + + RETURN NEW; + END; + $function$ + SQL create_table "account_sources", id: :serial, force: :cascade do |t| t.integer "account_id" @@ -292,7 +312,7 @@ t.jsonb "value_json", default: "{}" t.datetime "created_at", null: false t.datetime "updated_at", null: false - t.index "dynamic_annotation_fields_value(field_name, value)", name: "dynamic_annotation_fields_value", where: "((field_name)::text = ANY (ARRAY[('external_id'::character varying)::text, ('smooch_user_id'::character varying)::text, ('verification_status_status'::character varying)::text]))" + t.index "dynamic_annotation_fields_value(field_name, value)", name: "dynamic_annotation_fields_value", where: "((field_name)::text = ANY ((ARRAY['external_id'::character varying, 'smooch_user_id'::character varying, 'verification_status_status'::character varying])::text[]))" t.index ["annotation_id", "field_name"], name: "index_dynamic_annotation_fields_on_annotation_id_and_field_name" t.index ["annotation_id"], name: "index_dynamic_annotation_fields_on_annotation_id" t.index ["annotation_type"], name: "index_dynamic_annotation_fields_on_annotation_type" @@ -326,6 +346,8 @@ t.datetime "updated_at", precision: 6, null: false t.string "tags", default: [], array: true t.boolean "trashed", default: false + t.index "date_trunc('day'::text, created_at)", name: "explainer_created_at_day" + t.index ["created_at"], name: "index_explainers_on_created_at" t.index ["tags"], name: "index_explainers_on_tags", using: :gin t.index ["team_id"], name: "index_explainers_on_team_id" t.index ["user_id"], name: "index_explainers_on_user_id" @@ -347,7 +369,9 @@ t.string "rating" t.boolean "imported", default: false t.boolean "trashed", default: false + t.index "date_trunc('day'::text, created_at)", name: "fact_check_created_at_day" t.index ["claim_description_id"], name: "index_fact_checks_on_claim_description_id", unique: true + t.index ["created_at"], name: "index_fact_checks_on_created_at" t.index ["imported"], name: "index_fact_checks_on_imported" t.index ["language"], name: "index_fact_checks_on_language" t.index ["publisher_id"], name: "index_fact_checks_on_publisher_id" @@ -592,7 +616,7 @@ t.index ["source_id", "target_id", "relationship_type"], name: "relationship_index", unique: true t.index ["source_id"], name: "index_relationships_on_source_id" t.index ["target_id", "relationship_type"], name: "index_relationships_on_target_id_and_relationship_type" - t.index ["target_id"], name: "index_relationships_on_target_id" + t.index ["target_id"], name: "index_relationships_on_target_id", unique: true t.check_constraint "source_id <> target_id", name: "source_target_must_be_different" end @@ -958,4 +982,8 @@ add_foreign_key "project_media_requests", "project_medias" add_foreign_key "project_media_requests", "requests" add_foreign_key "requests", "feeds" + + create_trigger :enforce_relationships, sql_definition: <<-SQL + CREATE TRIGGER enforce_relationships BEFORE INSERT ON public.relationships FOR EACH ROW EXECUTE PROCEDURE validate_relationships() + SQL end diff --git a/lib/check_data_points.rb b/lib/check_data_points.rb index 160056f0a4..83c70d7df5 100644 --- a/lib/check_data_points.rb +++ b/lib/check_data_points.rb @@ -4,44 +4,47 @@ class << self GRANULARITY_VALUES = ['year', 'quarter', 'month', 'week', 'day'] # Number of tipline messages - def tipline_messages(team_id, start_date, end_date, granularity = nil) + def tipline_messages(team_id, start_date, end_date, granularity = nil, platform = nil, language = nil) start_date, end_date = parse_start_end_dates(start_date, end_date) query = TiplineMessage.where(team_id: team_id, created_at: start_date..end_date) - query_based_on_granularity(query, granularity) + query_based_on_granularity(query, platform, language, granularity) end # Number of tipline requests - def tipline_requests(team_id, start_date, end_date, granularity = nil) + def tipline_requests(team_id, start_date, end_date, granularity = nil, platform = nil, language = nil) start_date, end_date = parse_start_end_dates(start_date, end_date) query = TiplineRequest.where(team_id: team_id, created_at: start_date..end_date) - query_based_on_granularity(query, granularity) + query_based_on_granularity(query, platform, language, granularity) end # Number of tipline requests grouped by type of search result - def tipline_requests_by_search_type(team_id, start_date, end_date) + def tipline_requests_by_search_type(team_id, start_date, end_date, platform = nil, language = nil) start_date, end_date = parse_start_end_dates(start_date, end_date) - TiplineRequest.where( + query = TiplineRequest.where( team_id: team_id, smooch_request_type: SEARCH_RESULT_TYPES, created_at: start_date..end_date, - ).group('smooch_request_type').count + ) + query = query.where(platform: platform) unless platform.blank? + query = query.where(language: language) unless language.blank? + query.group('smooch_request_type').count end # Number of Subscribers - def tipline_subscriptions(team_id, start_date, end_date, granularity = nil) + def tipline_subscriptions(team_id, start_date, end_date, granularity = nil, platform = nil, language = nil) start_date, end_date = parse_start_end_dates(start_date, end_date) query = TiplineSubscription.where(team_id: team_id, created_at: start_date..end_date) - query_based_on_granularity(query, granularity) + query_based_on_granularity(query, platform, language, granularity) end # Number of Newsletters sent - def newsletters_sent(team_id, start_date, end_date, granularity = nil) + def newsletters_sent(team_id, start_date, end_date, granularity = nil, platform = nil, language = nil) start_date, end_date = parse_start_end_dates(start_date, end_date) query = TiplineNewsletterDelivery .joins(:tipline_newsletter) .where('tipline_newsletters.team_id': team_id) .where(created_at: start_date..end_date) - query_based_on_granularity(query, granularity, 'newsletter') + query_based_on_granularity(query, platform, language, granularity, 'newsletter') end # Number of Media received, by type @@ -53,48 +56,62 @@ def media_received_by_type(team_id, start_date, end_date) end # Top clusters - def top_clusters(team_id, start_date, end_date, limit = 5) - elastic_search_top_items(team_id, start_date, end_date, limit) + def top_clusters(team_id, start_date, end_date, limit = 5, range_field = 'created_at', language = nil, language_field = 'language', platform = nil) + elastic_search_top_items(team_id, start_date, end_date, limit, false, range_field, language, language_field, platform) end # Top media tags - def top_media_tags(team_id, start_date, end_date, limit = 5) - elastic_search_top_items(team_id, start_date, end_date, limit, true) + def top_media_tags(team_id, start_date, end_date, limit = 5, range_field = 'created_at', language = nil, language_field = 'language', platform = nil) + elastic_search_top_items(team_id, start_date, end_date, limit, true, range_field, language, language_field, platform) end # Articles sent - def articles_sent(team_id, start_date, end_date) + def articles_sent(team_id, start_date, end_date, platform = nil, language = nil) start_date, end_date = parse_start_end_dates(start_date, end_date) # Get number of articles sent as search results - search_result_c = TiplineRequest.where(team_id: team_id, smooch_request_type: SEARCH_RESULT_TYPES, created_at: start_date..end_date).count + search_results_query = TiplineRequest.where(team_id: team_id, smooch_request_type: SEARCH_RESULT_TYPES, created_at: start_date..end_date) + search_results_query = search_results_query.where(platform: platform) unless platform.blank? + search_results_query = search_results_query.where(language: language) unless language.blank? + search_results_count = search_results_query.count # Get the number of articles sent as reports - reports_c = TiplineRequest + reports_query = TiplineRequest .where(team_id: team_id, created_at: start_date..end_date) - .where('smooch_report_received_at > 0 OR smooch_report_update_received_at > 0 OR smooch_report_sent_at > 0 OR smooch_report_correction_sent_at > 0').count - search_result_c + reports_c + .where('smooch_report_received_at > 0 OR smooch_report_update_received_at > 0 OR smooch_report_sent_at > 0 OR smooch_report_correction_sent_at > 0') + reports_query = reports_query.where(platform: platform) unless platform.blank? + reports_query = reports_query.where(language: language) unless language.blank? + reports_count = reports_query.count + search_results_count + reports_count end # Average response time - def average_response_time(team_id, start_date, end_date) - TiplineRequest - .where(team_id: team_id, smooch_report_received_at: start_date.to_datetime.to_i..end_date.to_datetime.to_i) - .average("smooch_report_received_at - CAST(DATE_PART('EPOCH', created_at::timestamp) AS INTEGER)").to_f + def average_response_time(team_id, start_date, end_date, platform = nil, language = nil) + query = TiplineRequest.where(team_id: team_id, smooch_report_received_at: start_date.to_datetime.to_i..end_date.to_datetime.to_i) + query = query.where(platform: platform) unless platform.blank? + query = query.where(language: language) unless language.blank? + query.average("smooch_report_received_at - CAST(DATE_PART('EPOCH', created_at::timestamp) AS INTEGER)").to_f end # All users - def all_users(team_id, start_date, end_date) + def all_users(team_id, start_date, end_date, platform = nil, language = nil) start_date, end_date = parse_start_end_dates(start_date, end_date) - TiplineRequest.where(team_id: team_id, created_at: start_date..end_date) - .count('DISTINCT(tipline_user_uid)') + query = TiplineRequest.where(team_id: team_id, created_at: start_date..end_date) + query = query.where(platform: platform) unless platform.blank? + query = query.where(language: language) unless language.blank? + query.count('DISTINCT(tipline_user_uid)') end # Returning users - def returning_users(team_id, start_date, end_date) + def returning_users(team_id, start_date, end_date, platform = nil, language = nil) # Number of returning users (at least one session in the current month, and at least one session in the last previous 2 months) start_date, end_date = parse_start_end_dates(start_date, end_date) - uids = TiplineRequest.where(team_id: team_id, created_at: start_date.ago(2.months)..start_date).map(&:tipline_user_uid).uniq - TiplineRequest.where(team_id: team_id, tipline_user_uid: uids, created_at: start_date..end_date) - .count('DISTINCT(tipline_user_uid)') + uids_query = TiplineRequest.where(team_id: team_id, created_at: start_date.ago(2.months)..start_date) + uids_query = uids_query.where(platform: platform) unless platform.blank? + uids_query = uids_query.where(language: language) unless language.blank? + uids = uids_query.select(:tipline_user_uid).map(&:tipline_user_uid).uniq + query = TiplineRequest.where(team_id: team_id, tipline_user_uid: uids, created_at: start_date..end_date) + query = query.where(platform: platform) unless platform.blank? + query = query.where(language: language) unless language.blank? + query.count('DISTINCT(tipline_user_uid)') end # New users @@ -110,13 +127,15 @@ def new_users(team_id, start_date, end_date) def parse_start_end_dates(start_date, end_date) # date format is `2023-08-23` - start_date = Time.parse(start_date) - end_date = Time.parse(end_date) + start_date = Time.parse(start_date).beginning_of_day + end_date = Time.parse(end_date).end_of_day raise 'End date should be greater than start date' if start_date > end_date return start_date, end_date end - def query_based_on_granularity(query, granularity, type = nil) + def query_based_on_granularity(query, platform, language, granularity, type = nil) + query = query.where(platform: platform) unless platform.blank? + query = query.where(language: language) unless language.blank? # For PG the allowed values for granularity can be one of the following # [millennium, century, decade, year, quarter, month, week, day, hour, # minute, second, milliseconds, microseconds] @@ -132,15 +151,17 @@ def query_based_on_granularity(query, granularity, type = nil) end end - def elastic_search_top_items(team_id, start_date, end_date, limit, with_tags = false) + def elastic_search_top_items(team_id, start_date, end_date, limit, with_tags = false, range_field = 'created_at', language = nil, language_field = 'language', platform = nil) data = {} query = { - range: { 'created_at': { start_time: start_date, end_time: end_date } }, + range: { range_field => { start_time: start_date, end_time: end_date } }, demand: { min: 1 }, sort: 'demand', eslimit: limit } query[:tags_as_sentence] = { min: 1 } if with_tags + query[language_field.to_sym] = [language].flatten if language + query[:channels] = [CheckChannels::ChannelCodes.all_channels['TIPLINE'][platform.upcase]] if platform result = CheckSearch.new(query.to_json, nil, team_id) result.medias.each{ |pm| data[pm.id] = pm.demand } data diff --git a/lib/check_search.rb b/lib/check_search.rb index 208848b6dd..3953f762e7 100644 --- a/lib/check_search.rb +++ b/lib/check_search.rb @@ -29,11 +29,7 @@ def initialize(options, file = nil, team_id = Team.current&.id) adjust_numeric_range_filter adjust_archived_filter adjust_language_filter - - # Set fuzzy matching for keyword search, right now with automatic Levenshtein Edit Distance - # https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-simple-query-string-query.html - # https://github.com/elastic/elasticsearch/issues/23366 - @options['keyword'] = "#{@options['keyword']}~" if !@options['keyword'].blank? && @options['fuzzy'] + adjust_keyword_filter # Set es_id option @options['es_id'] = Base64.encode64("ProjectMedia/#{@options['id']}") if @options['id'] && ['GraphQL::Types::String', 'GraphQL::Types::Int', 'String', 'Integer'].include?(@options['id'].class.name) @@ -60,6 +56,19 @@ def initialize(options, file = nil, team_id = Team.current&.id) 'fact_check_published_on' => 'fact_check_published_on' } + def adjust_keyword_filter + unless @options['keyword'].blank? + # This regex removes all characters except letters, numbers, hashtag, search operators, emojis and whitespace + # in any language - stripping out special characters can improve match results + @options['keyword'].gsub!(/[^[:word:]\s#~+\-|()"\u{1F600}-\u{1F64F}\u{1F300}-\u{1F5FF}\u{1F680}-\u{1F6FF}\u{1F700}-\u{1F77F}\u{1F780}-\u{1F7FF}\u{1F800}-\u{1F8FF}\u{1F900}-\u{1F9FF}\u{1FA00}-\u{1FA6F}\u{1F1E6}-\u{1F1FF}\u{2600}-\u{26FF}\u{2700}-\u{27BF}]/, ' ') + + # Set fuzzy matching for keyword search, right now with automatic Levenshtein Edit Distance + # https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-simple-query-string-query.html + # https://github.com/elastic/elasticsearch/issues/23366 + @options['keyword'] = "#{@options['keyword']}~" if @options['fuzzy'] + end + end + def team_condition(team_id = nil) if feed_query? feed_teams = @options['feed_team_ids'].is_a?(Array) ? (@feed.team_ids & @options['feed_team_ids']) : @feed.team_ids @@ -332,10 +341,11 @@ def self.get_exported_data(query, team_id) # Prepare the export data = [] header = nil + fields = [] if feed_sharing_only_fact_checks header = ['Fact-check title', 'Fact-check summary', 'Fact-check URL', 'Tags', 'Workspace', 'Updated at', 'Rating'] else - header = ['Claim', 'Item page URL', 'Status', 'Created by', 'Submitted at', 'Published at', 'Number of media', 'Tags'] + header = ['Claim', 'Item page URL', 'Status', 'Created by', 'Submitted at', 'Social Media Posted at', 'Report Published at', 'Number of media', 'Tags'] fields = team.team_tasks.sort fields.each { |tt| header << tt.label } end @@ -346,39 +356,15 @@ def self.get_exported_data(query, team_id) while !search_after.empty? result = $repository.search(_source: 'annotated_id', query: search.medias_query, sort: [{ annotated_id: { order: :asc } }], size: 10000, search_after: search_after).results ids = result.collect{ |i| i['annotated_id'] }.uniq.compact.map(&:to_i) + pm_report = {} + Dynamic.where(annotation_type: 'report_design', annotated_type: 'ProjectMedia', annotated_id: ids) + .find_each do |raw| + pm_report[raw.annotated_id] = (raw.data['last_published'] || raw.updated_at).to_i if raw.data['state'] == 'published' + end # Iterate through each result and generate an output row for the CSV ProjectMedia.where(id: ids, team_id: search.team_condition(team_id)).find_each do |pm| - row = nil - if feed_sharing_only_fact_checks - row = [ - pm.fact_check_title, - pm.fact_check_summary, - pm.fact_check_url, - pm.tags_as_sentence, - pm.team_name, - pm.updated_at_timestamp, - pm.status - ] - else - row = [ - pm.claim_description&.description, - pm.full_url, - pm.status_i18n, - pm.author_name.to_s.gsub(/ \[.*\]$/, ''), - pm.created_at.strftime("%Y-%m-%d %H:%M:%S"), - pm.published_at&.strftime("%Y-%m-%d %H:%M:%S"), - pm.linked_items_count, - pm.tags_as_sentence - ] - annotations = pm.get_annotations('task').map(&:load) - fields.each do |field| - annotation = annotations.find { |a| a.team_task_id == field.id } - answer = (annotation ? (begin annotation.first_response_obj.file_data[:file_urls].join("\n") rescue annotation.first_response.to_s end) : '') - answer = begin JSON.parse(answer).collect{ |x| x['url'] }.join(', ') rescue answer end - row << answer - end - end + row = self.get_exported_data_row(feed_sharing_only_fact_checks, pm, pm_report[pm.id], fields) data << row end @@ -388,6 +374,42 @@ def self.get_exported_data(query, team_id) data end + def self.get_exported_data_row(feed_sharing_only_fact_checks, pm, report_published_at, fields) + row = nil + if feed_sharing_only_fact_checks + row = [ + pm.fact_check_title, + pm.fact_check_summary, + pm.fact_check_url, + pm.tags_as_sentence, + pm.team_name, + pm.updated_at_timestamp, + pm.status + ] + else + report_published_at_value = report_published_at ? Time.at(report_published_at).strftime("%Y-%m-%d %H:%M:%S") : nil + row = [ + pm.claim_description&.description, + pm.full_url, + pm.status_i18n, + pm.author_name.to_s.gsub(/ \[.*\]$/, ''), + pm.created_at.strftime("%Y-%m-%d %H:%M:%S"), + pm.published_at&.strftime("%Y-%m-%d %H:%M:%S"), + report_published_at_value, + pm.linked_items_count, + pm.tags_as_sentence + ] + annotations = pm.get_annotations('task').map(&:load) + fields.each do |field| + annotation = annotations.find { |a| a.team_task_id == field.id } + answer = (annotation ? (begin annotation.first_response_obj.file_data[:file_urls].join("\n") rescue annotation.first_response.to_s end) : '') + answer = begin JSON.parse(answer).collect{ |x| x['url'] }.join(', ') rescue answer end + row << answer + end + end + row + end + private def adjust_es_window_size diff --git a/lib/relay.idl b/lib/relay.idl index bee36de83a..b672574b3a 100644 --- a/lib/relay.idl +++ b/lib/relay.idl @@ -13372,6 +13372,7 @@ type Team implements Node { ): SourceConnection sources_count(keyword: String): Int spam_count: Int + statistics(language: String, period: String!, platform: String): TeamStatistics tag_texts( """ Returns the elements in the list that come after the specified cursor. @@ -13688,6 +13689,42 @@ type TeamEdge { node: Team } +""" +Workspace statistics. +""" +type TeamStatistics implements Node { + average_response_time: Int + created_at: String + id: ID! + number_of_articles_created_by_date: JsonStringType + number_of_articles_sent: Int + number_of_articles_updated_by_date: JsonStringType + number_of_conversations: Int + number_of_conversations_by_date: JsonStringType + number_of_explainers_created: Int + number_of_fact_checks_by_rating: JsonStringType + number_of_fact_checks_created: Int + number_of_matched_results_by_article_type: JsonStringType + number_of_media_received_by_media_type: JsonStringType + number_of_messages: Int + number_of_messages_by_date: JsonStringType + number_of_new_subscribers: Int + number_of_newsletters_delivered: Int + number_of_newsletters_sent: Int + number_of_published_fact_checks: Int + number_of_returning_users: Int + number_of_search_results_by_feedback_type: JsonStringType + number_of_subscribers: Int + number_of_total_users: Int + number_of_unique_users: Int + permissions: String + top_articles_sent: JsonStringType + top_articles_tags: JsonStringType + top_media_tags: JsonStringType + top_requested_media_clusters: JsonStringType + updated_at: String +} + """ Team task type """ diff --git a/lib/tasks/migrate/20220224213611_fix_parent_id_for_suggested_items.rake b/lib/tasks/migrate/20220224213611_fix_parent_id_for_suggested_items.rake index afc086d34a..bae56ad777 100644 --- a/lib/tasks/migrate/20220224213611_fix_parent_id_for_suggested_items.rake +++ b/lib/tasks/migrate/20220224213611_fix_parent_id_for_suggested_items.rake @@ -1,5 +1,18 @@ namespace :check do namespace :migrate do + def parse_args(args) + output = {} + return output if args.blank? + args.each do |a| + arg = a.split('&') + arg.each do |pair| + key, value = pair.split(':') + output.merge!({ key => value }) + end + end + output + end + task fix_parent_id_and_sources_count_for_suggested_items: :environment do started = Time.now.to_i index_alias = CheckElasticSearchModel.get_index_alias @@ -56,5 +69,59 @@ namespace :check do minutes = (Time.now.to_i - started) / 60 puts "[#{Time.now}] Done in #{minutes} minutes." end + + # bundle exec rails check:migrate:fix_parent_id_for_suggested_list['slug:team_slug&ids:1-2-3'] + task fix_parent_id_for_suggested_list: :environment do |_t, args| + data_args = parse_args args.extras + started = Time.now.to_i + pm_ids = [] + pm_ids = begin ids.split('-').map{ |s| s.to_i } rescue [] end + # Add Team condition + team_condition = {} + if data_args['slug'].blank? + last_team_id = Rails.cache.read('check:migrate:fix_parent_id_for_suggested_list:team_id') || 0 + else + last_team_id = 0 + team_condition = { slug: data_args['slug'] } + end + index_alias = CheckElasticSearchModel.get_index_alias + Team.where('id > ?', last_team_id).where(team_condition).find_each do |team| + result_ids = CheckSearch.new({"suggestions_count"=>{"min"=>1}}.to_json, nil, team.id).medias.map(&:id) + result_ids.concat(pm_ids) unless pm_ids.blank? + # Confirmed items + Relationship.where(source_id: result_ids, relationship_type: Relationship.confirmed_type).find_in_batches(:batch_size => 1000) do |relations| + es_body = [] + # Update parent_id for sources + source_ids = relations.map(&:source_id).uniq + source_ids.each do |source_id| + print '.' + doc_id = Base64.encode64("ProjectMedia/#{source_id}") + fields = { "parent_id" => source_id } + es_body << { update: { _index: index_alias, _id: doc_id, retry_on_conflict: 3, data: { doc: fields } } } + end + relations.each do |r| + print '.' + doc_id = Base64.encode64("ProjectMedia/#{r.target_id}") + fields = { "parent_id" => r.source_id } + es_body << { update: { _index: index_alias, _id: doc_id, retry_on_conflict: 3, data: { doc: fields } } } + end + $repository.client.bulk body: es_body unless es_body.blank? + end + # Suggested items + Relationship.where(source_id: result_ids, relationship_type: Relationship.suggested_type).find_in_batches(:batch_size => 1000) do |relations| + es_body = [] + relations.each do |r| + print '.' + doc_id = Base64.encode64("ProjectMedia/#{r.target_id}") + fields = { "parent_id" => r.target_id } + es_body << { update: { _index: index_alias, _id: doc_id, retry_on_conflict: 3, data: { doc: fields } } } + end + $repository.client.bulk body: es_body unless es_body.blank? + end + Rails.cache.write('check:migrate:fix_parent_id_for_suggested_list:team_id', team.id) if data_args['slug'].blank? + end + minutes = ((Time.now.to_i - started) / 60).to_i + puts "[#{Time.now}] Done in #{minutes} minutes." + end end end \ No newline at end of file diff --git a/lib/tasks/migrate/20230216030351_delete_duplicate_relationships.rake b/lib/tasks/migrate/20230216030351_delete_duplicate_relationships.rake index 77c7aff648..0f733ab030 100644 --- a/lib/tasks/migrate/20230216030351_delete_duplicate_relationships.rake +++ b/lib/tasks/migrate/20230216030351_delete_duplicate_relationships.rake @@ -8,17 +8,19 @@ namespace :check do i += 1 puts "[#{Time.now}] #{i}/#{n}" if count > 1 - relationships = Relationship.where(target_id: pm_id).to_a - # Keep the relationship whose model is image, video or audio... if none, keep the first one - keep = relationships.find{ |r| ['image', 'video', 'audio'].include?(r.model) } || relationships.first - raise "No relationship to keeo for target_id #{pm_id}!" if keep.nil? + relationships = Relationship.where(target_id: pm_id).order('id ASC').to_a + # Keep the confirmed relationship, or the one whose model is image, video or audio... if none, keep the first one + keep = relationships.find{ |r| r.relationship_type == Relationship.confirmed_type } || relationships.find{ |r| ['image', 'video', 'audio'].include?(r.model) } || relationships.first + raise "No relationship to keep for target_id #{pm_id}!" if keep.nil? relationships.each do |relationship| if relationship.id == keep.id - puts " Keeping relationship ##{r.id}" + puts " Keeping relationship ##{relationship.id}" else - puts " Deleting relationship ##{r.id}" - relationship.destroy! + puts " Deleting relationship ##{relationship.id}" + relationship.delete end + relationship.source.clear_cached_fields + relationship.target.clear_cached_fields end end end diff --git a/lib/tasks/migrate/20230725053637_add_unmatched_to_project_medias.rake b/lib/tasks/migrate/20230725053637_add_unmatched_to_project_medias.rake index b4b05f7aba..282754a357 100644 --- a/lib/tasks/migrate/20230725053637_add_unmatched_to_project_medias.rake +++ b/lib/tasks/migrate/20230725053637_add_unmatched_to_project_medias.rake @@ -43,6 +43,52 @@ namespace :check do $repository.client.update_by_query options end end + Rails.cache.write('check:migrate:add_unmatched_to_project_media:team_id', team.id) + end + minutes = ((Time.now.to_i - started) / 60).to_i + puts "[#{Time.now}] Done in #{minutes} minutes." + end + + # bundle exec rails check:migrate:fix_unmatched_list + task fix_unmatched_list: :environment do |_t, args| + started = Time.now.to_i + slug = args.extras.last + team_condition = {} + if slug.blank? + last_team_id = Rails.cache.read('check:migrate:fix_unmatched_list:team_id') || 0 + else + last_team_id = 0 + team_condition = { slug: slug } + end + Team.where('id > ?', last_team_id).where(team_condition).find_each do |team| + puts "Processing team #{team.slug} .... \n" + unmatched_ids = team.project_medias.where(unmatched: 1).map(&:id) + # Get re-matched items (suggested or confirmed) + relationships = Relationship.where('source_id IN (?) OR target_id IN (?)', unmatched_ids, unmatched_ids) + .where('relationship_type = ? OR relationship_type = ?', Relationship.suggested_type.to_yaml, Relationship.confirmed_type.to_yaml) + s_ids = relationships.map(&:source_id).uniq + t_ids = relationships.map(&:target_id).uniq + matched_ids = s_ids.concat(t_ids).uniq + unless matched_ids.blank? + index_alias = CheckElasticSearchModel.get_index_alias + ProjectMedia.where(id: matched_ids).find_in_batches(:batch_size => 500) do |pms| + print '.' + pm_ids = pms.map(&:id) + # Update PG + ProjectMedia.where(id: pm_ids).update_all(unmatched: 0) + # Update ES + options = { + index: index_alias, + conflicts: 'proceed', + body: { + script: { source: "ctx._source.unmatched = params.unmatched", params: { unmatched: 0 } }, + query: { terms: { annotated_id: pm_ids } } + } + } + $repository.client.update_by_query options + end + end + Rails.cache.write('check:migrate:fix_unmatched_list:team_id', team.id) end minutes = ((Time.now.to_i - started) / 60).to_i puts "[#{Time.now}] Done in #{minutes} minutes." diff --git a/lib/team_statistics.rb b/lib/team_statistics.rb new file mode 100644 index 0000000000..28b32010b5 --- /dev/null +++ b/lib/team_statistics.rb @@ -0,0 +1,309 @@ +class TeamStatistics + PERIODS = ['past_week', 'past_2_weeks', 'past_month', 'past_3_months', 'past_6_months', 'year_to_date'] + + PLATFORMS = Bot::Smooch::SUPPORTED_INTEGRATION_NAMES + + def initialize(team, period, language, platform = nil) + @team = team + unless @team.is_a?(Team) + raise ArgumentError.new('Invalid workspace provided') + end + + @period = period + unless PERIODS.include?(@period) + raise ArgumentError.new("Invalid period provided. Allowed values: #{PERIODS.join(', ')}") + end + + range = time_range.to_a + @start_date, @end_date = range.first.to_datetime.beginning_of_day, range.last.to_datetime.end_of_day + @start_date_str, @end_date_str = @start_date.strftime('%Y-%m-%d'), @end_date.strftime('%Y-%m-%d') + + @platform = platform + if !@platform.blank? && !PLATFORMS.keys.include?(@platform) + # For `Bot::Smooch::SUPPORTED_INTEGRATION_NAMES`, the keys (e.g., 'whatsapp') are used by `TiplineRequest`, + # while the values (e.g., 'WhatsApp') are used by `TiplineMessage` + raise ArgumentError.new("Invalid platform provided. Allowed values: #{PLATFORMS.keys.join(', ')}") + end + @platform_name = PLATFORMS[@platform] unless @platform.blank? + + @language = language + @all_languages = [@team.get_languages.to_a, 'und'].flatten + end + + # For GraphQL + def id + Base64.encode64("TeamStatistics/#{@team.id}") + end + + # For articles + + def number_of_articles_created_by_date + number_of_articles_saved_by_date(:created_at) + end + + def number_of_articles_updated_by_date + number_of_articles_saved_by_date(:updated_at) + end + + def number_of_explainers_created + explainers_base_query.count + end + + def number_of_fact_checks_created + fact_checks_base_query.count + end + + def number_of_published_fact_checks + fact_checks_base_query.where(report_status: 'published').count + end + + def number_of_fact_checks_by_rating + data = {} + fact_checks_base_query.group(:rating).count.each do |status_id, count| + data[status_id.to_s] = count + end + data.sort.to_h + end + + # FIXME: Only fact-checks for now (need to add explainers) and the "demand" is across languages and platforms + def top_articles_sent + data = [] + clusters = CheckDataPoints.top_clusters(@team.id, @start_date, @end_date, 5, 'last_seen', @language || @all_languages, 'fc_language') + clusters.each do |pm_id, demand| + item = ProjectMedia.find(pm_id) + title = item.fact_check_title || item.title + data << { id: item.fact_check_id, label: title, value: demand } + end + data.sort_by{ |object| object[:value] }.reverse + end + + def top_articles_tags + sql = <<-SQL + SELECT tag, COUNT(*) as tag_count + FROM ( + SELECT unnest(fcs.tags) AS tag FROM fact_checks fcs + INNER JOIN claim_descriptions cds ON fcs.claim_description_id = cds.id + WHERE cds.team_id = :team_id AND fcs.created_at BETWEEN :start_date AND :end_date AND fcs.language IN (:language) + UNION ALL + SELECT unnest(explainers.tags) AS tag FROM explainers + WHERE explainers.team_id = :team_id AND explainers.created_at BETWEEN :start_date AND :end_date AND explainers.language IN (:language) + ) AS all_tags + GROUP BY tag + ORDER BY tag_count DESC + LIMIT 5 + SQL + + language = @language ? [@language] : @all_languages + result = ActiveRecord::Base.connection.execute(ApplicationRecord.sanitize_sql_for_assignment([sql, team_id: @team.id, start_date: @start_date, end_date: @end_date, language: language])) + data = [] + result.each do |row| + data << { id: row['tag'], label: row['tag'], value: row['tag_count'].to_i } + end + data.sort_by{ |object| object[:value] }.reverse + end + + # For tiplines + + def number_of_messages + CheckDataPoints.tipline_messages(@team.id, @start_date_str, @end_date_str, nil, @platform_name, @language) + end + + def number_of_conversations + CheckDataPoints.tipline_requests(@team.id, @start_date_str, @end_date_str, nil, @platform, @language) + end + + def number_of_messages_by_date + data = CheckDataPoints.tipline_messages(@team.id, @start_date_str, @end_date_str, 'day', @platform_name, @language) + number_of_tipline_data_points_by_date(data) + end + + def number_of_conversations_by_date + data = CheckDataPoints.tipline_requests(@team.id, @start_date_str, @end_date_str, 'day', @platform, @language) + number_of_tipline_data_points_by_date(data) + end + + def number_of_search_results_by_feedback_type + mapping = { + relevant_search_result_requests: 'Positive', + irrelevant_search_result_requests: 'Negative', + timeout_search_requests: 'No Response' + } + data = { + 'Positive' => 0, + 'Negative' => 0, + 'No Response' => 0 + } + CheckDataPoints.tipline_requests_by_search_type(@team.id, @start_date_str, @end_date_str, @platform, @language).each do |type, count| + data[mapping[type.to_sym]] = count + end + data + end + + def average_response_time + CheckDataPoints.average_response_time(@team.id, @start_date, @end_date, @platform, @language) + end + + def number_of_unique_users + number_of_total_users - number_of_returning_users + end + + def number_of_total_users + CheckDataPoints.all_users(@team.id, @start_date_str, @end_date_str, @platform, @language) + end + + def number_of_returning_users + CheckDataPoints.returning_users(@team.id, @start_date_str, @end_date_str, @platform, @language) + end + + def number_of_subscribers + CheckDataPoints.tipline_subscriptions(@team.id, @team.created_at.strftime('%Y-%m-%d'), @end_date_str, nil, @platform_name, @language) + end + + def number_of_new_subscribers + CheckDataPoints.tipline_subscriptions(@team.id, @start_date_str, @end_date_str, nil, @platform_name, @language) + end + + def number_of_newsletters_sent + number_of_newsletters('sent') + end + + def number_of_newsletters_delivered + number_of_newsletters('delivered') + end + + def number_of_media_received_by_media_type + conditions = { team_id: @team.id, created_at: @start_date..@end_date } + conditions[:language] = @language unless @language.blank? + conditions[:platform] = @platform unless @platform.blank? + data = TiplineRequest + .joins("INNER JOIN project_medias pm ON tipline_requests.associated_type = 'ProjectMedia' AND pm.id = tipline_requests.associated_id") + .joins("INNER JOIN medias m ON m.id = pm.media_id") + .where(conditions) + .group('m.type') + .count + { 'Claim' => 0, 'Link' => 0, 'UploadedAudio' => 0, 'UploadedImage' => 0, 'UploadedVideo' => 0 }.merge(data).reject{ |k, _v| k == 'Blank' } + end + + # FIXME: The "demand" is across languages and platforms + def top_requested_media_clusters + data = [] + clusters = CheckDataPoints.top_clusters(@team.id, @start_date, @end_date, 5, 'last_seen', @language || @all_languages, 'request_language', @platform) + clusters.each do |pm_id, demand| + item = ProjectMedia.find(pm_id) + data << { id: item.id, label: item.title, value: demand } + end + data.sort_by{ |object| object[:value] }.reverse + end + + # FIXME: The "demand" is across languages and platforms + def top_media_tags + tags = {} + clusters = CheckDataPoints.top_clusters(@team.id, @start_date, @end_date, 5, 'last_seen', @language || @all_languages, 'language', @platform) + clusters.each do |pm_id, demand| + item = ProjectMedia.find(pm_id) + item.tags_as_sentence.split(',').map(&:strip).each do |tag| + tags[tag] ||= 0 + tags[tag] += demand + end + end + data = [] + tags.each { |tag, value| data << { id: tag, label: tag, value: value } } + data.sort_by{ |object| object[:value] }.reverse.first(5) + end + + # For both articles and tiplines + + def number_of_articles_sent + CheckDataPoints.articles_sent(@team.id, @start_date_str, @end_date_str, @platform, @language) + end + + def number_of_matched_results_by_article_type + query = TiplineRequest.where(team_id: @team.id, smooch_request_type: ['relevant_search_result_requests', 'irrelevant_search_result_requests', 'timeout_search_requests'], created_at: @start_date..@end_date) + query = query.where(platform: @platform) unless @platform.blank? + query = query.where(language: @language) unless @language.blank? + { 'FactCheck' => query.joins(project_media: { claim_description: :fact_check }).count, 'Explainer' => query.joins(project_media: :explainers).count } + end + + private + + def time_range + ago = { + past_week: 1.week, + past_2_weeks: 2.weeks, + past_month: 1.month, + past_3_months: 3.months, + past_6_months: 6.months + }[@period.to_sym] + from = Time.now.ago(ago) unless ago.nil? + from = Time.now.beginning_of_year if @period.to_s == 'year_to_date' + from.to_datetime.beginning_of_day..Time.now.to_datetime.end_of_day + end + + def fact_checks_base_query(timestamp_field = :created_at, group_by_day = false) + query = FactCheck.joins(:claim_description).where(timestamp_field => time_range, 'claim_descriptions.team_id' => @team.id) + query = query.where('fact_checks.created_at != fact_checks.updated_at') if timestamp_field.to_sym == :updated_at + query = query.where(language: @language) unless @language.blank? + if group_by_day + # Avoid SQL injection warning + group = { + created_at: "date_trunc('day', fact_checks.created_at)", + updated_at: "date_trunc('day', fact_checks.updated_at)" + }[timestamp_field.to_sym] + query = query.group(group) + end + query + end + + def explainers_base_query(timestamp_field = :created_at, group_by_day = false) + query = Explainer.where(timestamp_field => time_range, 'team_id' => @team.id) + query = query.where(language: @language) unless @language.blank? + query = query.where('explainers.created_at != explainers.updated_at') if timestamp_field.to_sym == :updated_at + if group_by_day + # Avoid SQL injection warning + group = { + created_at: "date_trunc('day', explainers.created_at)", + updated_at: "date_trunc('day', explainers.updated_at)" + }[timestamp_field.to_sym] + query = query.group(group) + end + query + end + + def number_of_articles_saved_by_date(timestamp_field) # timestamp_field = :created_at or :updated_at + raise ArgumentError if timestamp_field != :created_at && timestamp_field != :updated_at + number_of_fact_checks = fact_checks_base_query(timestamp_field, true).count + number_of_explainers = explainers_base_query(timestamp_field, true).count + number_of_articles = {} + + # Pre-fill with zeros + time_range.to_a.each do |day| + number_of_articles[day.strftime("%Y-%m-%d")] = 0 + end + + # Replace zeros by the days we have data for + (number_of_fact_checks.keys + number_of_explainers.keys).uniq.sort.each do |day| + number_of_articles[day.strftime("%Y-%m-%d")] = number_of_fact_checks[day].to_i + number_of_explainers[day].to_i + end + + number_of_articles + end + + def number_of_tipline_data_points_by_date(results) + data = {} + # Pre-fill with zeros + time_range.to_a.each do |day| + data[day.strftime("%Y-%m-%d")] = 0 + end + results.each do |day, count| + data[day.strftime("%Y-%m-%d")] = count + end + data + end + + def number_of_newsletters(state) + query = TiplineMessage.where(created_at: @start_date..@end_date, team_id: @team.id, state: state, event: 'newsletter') + query = query.where(language: @language) unless @language.blank? + query = query.where(platform: @platform_name) unless @platform.blank? + query.count + end +end diff --git a/public/relay.json b/public/relay.json index 2d4750a1bb..2fc923085e 100644 --- a/public/relay.json +++ b/public/relay.json @@ -56461,6 +56461,11 @@ "name": "TeamBotInstallation", "ofType": null }, + { + "kind": "OBJECT", + "name": "TeamStatistics", + "ofType": null + }, { "kind": "OBJECT", "name": "TeamTask", @@ -70508,6 +70513,59 @@ "isDeprecated": false, "deprecationReason": null }, + { + "name": "statistics", + "description": null, + "args": [ + { + "name": "period", + "description": null, + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "String", + "ofType": null + } + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "language", + "description": null, + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "platform", + "description": null, + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "defaultValue": null, + "isDeprecated": false, + "deprecationReason": null + } + ], + "type": { + "kind": "OBJECT", + "name": "TeamStatistics", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, { "name": "tag_texts", "description": null, @@ -71877,6 +71935,447 @@ "enumValues": null, "possibleTypes": null }, + { + "kind": "OBJECT", + "name": "TeamStatistics", + "description": "Workspace statistics.", + "fields": [ + { + "name": "average_response_time", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "created_at", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "id", + "description": null, + "args": [ + + ], + "type": { + "kind": "NON_NULL", + "name": null, + "ofType": { + "kind": "SCALAR", + "name": "ID", + "ofType": null + } + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_articles_created_by_date", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "JsonStringType", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_articles_sent", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_articles_updated_by_date", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "JsonStringType", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_conversations", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_conversations_by_date", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "JsonStringType", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_explainers_created", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_fact_checks_by_rating", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "JsonStringType", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_fact_checks_created", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_matched_results_by_article_type", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "JsonStringType", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_media_received_by_media_type", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "JsonStringType", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_messages", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_messages_by_date", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "JsonStringType", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_new_subscribers", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_newsletters_delivered", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_newsletters_sent", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_published_fact_checks", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_returning_users", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_search_results_by_feedback_type", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "JsonStringType", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_subscribers", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_total_users", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "number_of_unique_users", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "Int", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "permissions", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "top_articles_sent", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "JsonStringType", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "top_articles_tags", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "JsonStringType", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "top_media_tags", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "JsonStringType", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "top_requested_media_clusters", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "JsonStringType", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + }, + { + "name": "updated_at", + "description": null, + "args": [ + + ], + "type": { + "kind": "SCALAR", + "name": "String", + "ofType": null + }, + "isDeprecated": false, + "deprecationReason": null + } + ], + "inputFields": null, + "interfaces": [ + { + "kind": "INTERFACE", + "name": "Node", + "ofType": null + } + ], + "enumValues": null, + "possibleTypes": null + }, { "kind": "OBJECT", "name": "TeamTask", diff --git a/test/controllers/elastic_search_9_test.rb b/test/controllers/elastic_search_9_test.rb index 6493472a94..3c22995bde 100644 --- a/test/controllers/elastic_search_9_test.rb +++ b/test/controllers/elastic_search_9_test.rb @@ -60,7 +60,7 @@ def setup WebMock.stub_request(:post, 'http://alegre/text/langid/').to_return(body: { 'result' => { 'language' => 'es' }}.to_json) WebMock.stub_request(:post, 'http://alegre/text/similarity/').to_return(body: 'success') WebMock.stub_request(:delete, 'http://alegre/text/similarity/').to_return(body: {success: true}.to_json) - WebMock.stub_request(:post, 'http://alegre/text/similarity/search/').to_return(body: {success: true}.to_json) + WebMock.stub_request(:post, 'http://alegre/similarity/sync/text').to_return(body: {success: true}.to_json) WebMock.stub_request(:post, 'http://alegre/image/classification/').with({ body: { uri: 'some/path' } }).to_return(body: { "result": valid_flags_data }.to_json) diff --git a/test/controllers/graphql_controller_11_test.rb b/test/controllers/graphql_controller_11_test.rb index ed47c24ba7..5e5c544f65 100644 --- a/test/controllers/graphql_controller_11_test.rb +++ b/test/controllers/graphql_controller_11_test.rb @@ -201,6 +201,73 @@ def teardown end end + test "should get team statistics if super admin" do + user = create_user is_admin: true + team = create_team + create_team_user user: user, team: team, role: 'admin' + + authenticate_with_user(user) + query = <<~GRAPHQL + query { + team(slug: "#{team.slug}") { + statistics(period: "past_week", platform: "whatsapp", language: "en") { + number_of_articles_created_by_date + number_of_articles_updated_by_date + number_of_explainers_created + number_of_fact_checks_created + number_of_published_fact_checks + number_of_fact_checks_by_rating + top_articles_sent + top_articles_tags + number_of_messages + number_of_conversations + number_of_messages_by_date + number_of_conversations_by_date + number_of_search_results_by_feedback_type + average_response_time + number_of_unique_users + number_of_total_users + number_of_returning_users + number_of_subscribers + number_of_new_subscribers + number_of_newsletters_sent + number_of_newsletters_delivered + top_media_tags + top_requested_media_clusters + number_of_media_received_by_media_type + number_of_articles_sent + number_of_matched_results_by_article_type + } + } + } + GRAPHQL + + post :create, params: { query: query } + assert_response :success + assert_not_nil JSON.parse(@response.body).dig('data', 'team', 'statistics') + end + + test "should not get team statistics if not super admin" do + user = create_user is_admin: false + team = create_team + create_team_user user: user, team: team, role: 'admin' + + authenticate_with_user(user) + query = <<~GRAPHQL + query { + team(slug: "#{team.slug}") { + statistics(period: "past_week", platform: "whatsapp", language: "en") { + number_of_articles_created_by_date + } + } + } + GRAPHQL + + post :create, params: { query: query } + assert_response :success + assert_nil JSON.parse(@response.body).dig('data', 'team', 'statistics') + end + test "should not get requests if interval is more than one month" do u = create_user t = create_team @@ -222,6 +289,7 @@ def teardown } } GRAPHQL + post :create, params: { query: query, team: t.slug } assert_response 400 assert_equal 'Maximum interval is one month.', JSON.parse(@response.body)['errors'][0]['message'] @@ -250,6 +318,7 @@ def teardown } } GRAPHQL + post :create, params: { query: query, team: t.slug } assert_response :success assert_equal 2, JSON.parse(@response.body).dig('data', 'team', 'tipline_requests', 'edges').size diff --git a/test/controllers/graphql_controller_12_test.rb b/test/controllers/graphql_controller_12_test.rb index bf666da1ef..4b0b16a3a5 100644 --- a/test/controllers/graphql_controller_12_test.rb +++ b/test/controllers/graphql_controller_12_test.rb @@ -731,4 +731,128 @@ def teardown assert_equal 1, pm_tags.count assert_equal 1, fc_tags.count end + + test "should append FactCheck to ProjectMedia, if ProjectMedia already exists and does not have a FactCheck" do + Sidekiq::Testing.fake! + url = 'http://example.com' + pender_url = CheckConfig.get('pender_url_private') + '/api/medias' + response_body = '{"type":"media","data":{"url":"' + url + '","type":"item"}}' + WebMock.stub_request(:get, pender_url).with({ query: { url: url } }).to_return(body: response_body) + + t = create_team + p = create_project team: t + pm = create_project_media team: t, set_original_claim: url + + assert_not_nil pm + + a = ApiKey.create! + b = create_bot_user api_key_id: a.id + create_team_user team: t, user: b + authenticate_with_token(a) + + query = <<~GRAPHQL + mutation { + createProjectMedia(input: { + project_id: #{p.id}, + media_type: "Blank", + channel: { main: 1 }, + set_tags: ["tag"], + set_status: "verified", + set_claim_description: "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", + set_original_claim: "#{url}", + set_fact_check: { + title: "Title #1", + language: "en", + } + }) { + project_media { + dbid + full_url + claim_description { + fact_check { + dbid + } + } + } + } + } + GRAPHQL + + assert_no_difference 'ProjectMedia.count' do + assert_difference 'FactCheck.count' do + post :create, params: { query: query, team: t.slug } + assert_response :success + end + end + + response_pm = JSON.parse(@response.body)['data']['createProjectMedia']['project_media'] + fact_check = response_pm['claim_description']['fact_check'] + + assert_not_nil fact_check + assert_equal response_pm['dbid'], pm.id + end + + test "should create a FactCheck with a Blank ProjectMedia, if ProjectMedia already exists and has a FactCheck in a different language" do + Sidekiq::Testing.fake! + url = 'http://example.com' + pender_url = CheckConfig.get('pender_url_private') + '/api/medias' + response_body = '{"type":"media","data":{"url":"' + url + '","type":"item"}}' + WebMock.stub_request(:get, pender_url).with({ query: { url: url } }).to_return(body: response_body) + + t = create_team + t.settings[:languages] << 'pt' + t.save! + p = create_project team: t + pm = create_project_media team: t, set_original_claim: url + c = create_claim_description project_media: pm + fc_1 = create_fact_check claim_description: c + + assert_not_nil fc_1 + + a = ApiKey.create! + b = create_bot_user api_key_id: a.id + create_team_user team: t, user: b + authenticate_with_token(a) + + query = <<~GRAPHQL + mutation { + createProjectMedia(input: { + project_id: #{p.id}, + media_type: "Blank", + channel: { main: 1 }, + set_tags: ["tag"], + set_status: "verified", + set_claim_description: "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", + set_original_claim: "#{url}", + set_fact_check: { + title: "Title #1", + language: "pt", + } + }) { + project_media { + dbid + full_url + claim_description { + fact_check { + dbid + } + } + } + } + } + GRAPHQL + + assert_difference 'ProjectMedia.count' do + assert_difference 'FactCheck.count' do + post :create, params: { query: query, team: t.slug } + assert_response :success + end + end + + response_pm = JSON.parse(@response.body)['data']['createProjectMedia']['project_media'] + fc_2 = response_pm['claim_description']['fact_check'] + + assert_not_nil fc_2 + assert_not_equal response_pm['dbid'], pm.id + end end diff --git a/test/controllers/graphql_controller_5_test.rb b/test/controllers/graphql_controller_5_test.rb index 8264680ecb..e85a1918b3 100644 --- a/test/controllers/graphql_controller_5_test.rb +++ b/test/controllers/graphql_controller_5_test.rb @@ -57,14 +57,14 @@ def setup pm = create_project_media project: p, media: m pm2 = create_project_media project: p, media: m2 create_claim_description project_media: pm2 - Bot::Alegre.stubs(:get_similar_texts).returns({ pm2.id => 0.9, pm.id => 0.8 }) + Bot::Alegre.stubs(:get_items_from_similar_text).returns({ pm2.id => 0.9, pm.id => 0.8 }) query = 'query { project_media(ids: "' + [pm.id, p.id, t.id].join(',') + '") { similar_items(first: 10000) { edges { node { dbid, claim_description { id, fact_check { id } } } } } } }' post :create, params: { query: query, team: t.slug } assert_response :success assert_equal pm2.id, JSON.parse(@response.body)['data']['project_media']['similar_items']['edges'][0]['node']['dbid'] - Bot::Alegre.unstub(:get_similar_texts) + Bot::Alegre.unstub(:get_items_from_similar_text) end test "should create and update flags and content warning" do diff --git a/test/controllers/reports_controller_test.rb b/test/controllers/reports_controller_test.rb index 0af43ffdd5..45b1785a4a 100644 --- a/test/controllers/reports_controller_test.rb +++ b/test/controllers/reports_controller_test.rb @@ -13,17 +13,15 @@ def setup def from_alegre(pm) { - '_index' => 'alegre_similarity', - '_type' => '_doc', + 'index' => 'alegre_similarity', '_id' => 'tMXj53UB36CYclMPXp14', - '_score' => 0.9, - '_source' => { - 'content' => 'Test', - 'context' => { - 'team_id' => pm.team_id.to_s, - 'field' => 'original_title', - 'project_media_id' => pm.id.to_s - } + 'id' => 'tMXj53UB36CYclMPXp14', + 'score' => 0.9, + 'content' => 'Test', + 'context' => { + 'team_id' => pm.team_id.to_s, + 'field' => 'original_title', + 'project_media_id' => pm.id.to_s } } end diff --git a/test/lib/check_config_test.rb b/test/lib/check_config_test.rb index 6341d8b6e0..5e290fc35d 100644 --- a/test/lib/check_config_test.rb +++ b/test/lib/check_config_test.rb @@ -57,4 +57,10 @@ class CheckConfigTest < ActiveSupport::TestCase ENV.delete('test_other_env_key') end + test "should handle json value " do + stub_configs({ 'test_json_key' => '{"lang":{"en":"default.html"}}', 'test_invalid_json_key' => 'invalid_json_value' }) do + assert_equal 'default.html', CheckConfig.get('test_json_key', nil, :json) + assert_equal 'invalid_json_value', CheckConfig.get('test_invalid_json_key', nil, :json) + end + end end diff --git a/test/lib/check_search_test.rb b/test/lib/check_search_test.rb new file mode 100644 index 0000000000..9288ba3db3 --- /dev/null +++ b/test/lib/check_search_test.rb @@ -0,0 +1,20 @@ +require_relative '../test_helper' + +class CheckSearchTest < ActiveSupport::TestCase + def setup + @team = create_team + end + + def teardown + end + + test "should strip special characters from keyword parameter" do + query = 'Something has increased 1000% last year' + search = CheckSearch.new({ keyword: query }.to_json, nil, @team.id) + assert_equal 'Something has increased 1000 last year', search.instance_variable_get('@options')['keyword'] + + query = 'Something is going to happen on 04/11, reportedly' + search = CheckSearch.new({ keyword: query }.to_json, nil, @team.id) + assert_equal 'Something is going to happen on 04 11 reportedly', search.instance_variable_get('@options')['keyword'] + end +end diff --git a/test/lib/smooch_nlu_test.rb b/test/lib/smooch_nlu_test.rb index 86e01015f8..ee73b38ba3 100644 --- a/test/lib/smooch_nlu_test.rb +++ b/test/lib/smooch_nlu_test.rb @@ -64,7 +64,7 @@ def create_team_with_smooch_bot_installed team = create_team_with_smooch_bot_installed nlu = SmoochNlu.new(team.slug) nlu.enable! - Bot::Alegre.expects(:request).with{ |x, y, _z| x == 'post' && y == '/text/similarity/' }.once + Bot::Alegre.expects(:request).with{ |x, y, _z| x == 'post' && y == '/similarity/sync/text' }.once nlu.add_keyword_to_menu_option('en', 'main', 0, 'subscribe') expected_output = { 'en' => { @@ -85,7 +85,7 @@ def create_team_with_smooch_bot_installed end test 'should add keyword if it does not exist' do - Bot::Alegre.expects(:request).with{ |x, y, _z| x == 'post' && y == '/text/similarity/' }.once + Bot::Alegre.expects(:request).with{ |x, y, _z| x == 'post' && y == '/similarity/sync/text' }.once team = create_team_with_smooch_bot_installed SmoochNlu.new(team.slug).add_keyword_to_menu_option('en', 'main', 0, 'subscribe to the newsletter') end @@ -93,9 +93,9 @@ def create_team_with_smooch_bot_installed test 'should not add keyword if it exists' do team = create_team_with_smooch_bot_installed nlu = SmoochNlu.new(team.slug) - Bot::Alegre.expects(:request).with{ |x, y, _z| x == 'post' && y == '/text/similarity/' }.once + Bot::Alegre.expects(:request).with{ |x, y, _z| x == 'post' && y == '/similarity/sync/text' }.once nlu.add_keyword_to_menu_option('en', 'main', 0, 'subscribe to the newsletter') - Bot::Alegre.expects(:request).with{ |x, y, _z| x == 'post' && y == '/text/similarity/' }.never + Bot::Alegre.expects(:request).with{ |x, y, _z| x == 'post' && y == '/similarity/sync/text' }.never nlu.add_keyword_to_menu_option('en', 'main', 0, 'subscribe to the newsletter') end @@ -114,8 +114,8 @@ def create_team_with_smooch_bot_installed end test 'should return a menu option if NLU is enabled' do - Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/text/similarity/search/' && z[:text] =~ /newsletter/ }.returns({ 'result' => [ - { '_score' => 0.9, '_source' => { 'context' => { 'menu_option_id' => 'test' } } }, + Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/similarity/sync/text' && z[:text] =~ /newsletter/ }.returns({ 'result' => [ + { 'score' => 0.9, 'context' => { 'menu_option_id' => 'test' } }, ]}) team = create_team_with_smooch_bot_installed SmoochNlu.new(team.slug).enable! diff --git a/test/lib/tasks/statistics_test.rb b/test/lib/tasks/statistics_test.rb index d00519537c..32e18de50a 100644 --- a/test/lib/tasks/statistics_test.rb +++ b/test/lib/tasks/statistics_test.rb @@ -37,9 +37,9 @@ def teardown test "check:data:statistics generates statistics data for teams with no tipline" do TeamBotInstallation.delete_all + Team.current = nil - api_key = create_api_key - bot_user = create_bot_user api_key_id: api_key.id + bot_user = create_bot_user bot_user.approve! non_tipline_team = create_team(slug: 'other-team') diff --git a/test/lib/team_statistics_test.rb b/test/lib/team_statistics_test.rb new file mode 100644 index 0000000000..7cd7a46f61 --- /dev/null +++ b/test/lib/team_statistics_test.rb @@ -0,0 +1,211 @@ +require_relative '../test_helper' + +class TeamStatisticsTest < ActiveSupport::TestCase + def setup + @team = create_team + @team.set_languages = ['en', 'pt'] + @team.save! + end + + def teardown + end + + test "should provide a valid period" do + assert_raises ArgumentError do + TeamStatistics.new(@team, 'past_century', 'en', 'whatsapp') + end + + assert_nothing_raised do + TeamStatistics.new(@team, 'past_month', 'en', 'whatsapp') + end + end + + test "should provide a valid workspace" do + assert_raises ArgumentError do + TeamStatistics.new(Class.new, 'past_month', 'en', 'whatsapp') + end + + assert_nothing_raised do + TeamStatistics.new(@team, 'past_month', 'en', 'whatsapp') + end + end + + test "should provide a valid platform" do + assert_raises ArgumentError do + TeamStatistics.new(@team, 'past_month', 'en', 'icq') + end + + assert_nothing_raised do + TeamStatistics.new(@team, 'past_month', 'en', 'whatsapp') + end + end + + test "should have a GraphQL ID" do + assert_kind_of String, TeamStatistics.new(@team, 'past_month', 'en', 'whatsapp').id + end + + test "should return articles statistics" do + team = create_team + exp = nil + + travel_to Time.parse('2024-01-01') do + create_fact_check(tags: ['foo', 'bar'], language: 'en', rating: 'false', claim_description: create_claim_description(project_media: create_project_media(team: @team))) + create_fact_check(tags: ['foo', 'bar'], claim_description: create_claim_description(project_media: create_project_media(team: team))) + exp = create_explainer team: @team, language: 'en', tags: ['foo'] + create_explainer team: @team, tags: ['foo', 'bar'] + create_explainer language: 'en', team: team, tags: ['foo', 'bar'] + end + + travel_to Time.parse('2024-01-02') do + create_fact_check(tags: ['bar'], report_status: 'published', rating: 'verified', language: 'en', claim_description: create_claim_description(project_media: create_project_media(team: @team))) + create_fact_check(tags: ['foo', 'bar'], claim_description: create_claim_description(project_media: create_project_media(team: team))) + create_explainer team: @team, language: 'en', tags: ['foo'] + create_explainer team: @team, tags: ['foo', 'bar'] + create_explainer language: 'en', team: team, tags: ['foo', 'bar'] + exp.updated_at = Time.now + exp.save! + end + + travel_to Time.parse('2024-01-08') do + object = TeamStatistics.new(@team, 'past_week', 'en') + assert_equal({ '2024-01-01' => 2, '2024-01-02' => 2, '2024-01-03' => 0, '2024-01-04' => 0, '2024-01-05' => 0, '2024-01-06' => 0, '2024-01-07' => 0, '2024-01-08' => 0 }, + object.number_of_articles_created_by_date) + assert_equal({ '2024-01-01' => 0, '2024-01-02' => 1, '2024-01-03' => 0, '2024-01-04' => 0, '2024-01-05' => 0, '2024-01-06' => 0, '2024-01-07' => 0, '2024-01-08' => 0 }, + object.number_of_articles_updated_by_date) + assert_equal 2, object.number_of_explainers_created + assert_equal 2, object.number_of_fact_checks_created + assert_equal 1, object.number_of_published_fact_checks + assert_equal({ 'false' => 1, 'verified' => 1 }, object.number_of_fact_checks_by_rating) + assert_equal([{ id: 'foo', label: 'foo', value: 3 }, { id: 'bar', label: 'bar', value: 2 }], object.top_articles_tags) + end + end + + test "should return number of articles sent" do + setup_elasticsearch + RequestStore.store[:skip_cached_field_update] = false + + pm1 = create_project_media team: @team, disable_es_callbacks: false + fc1 = create_fact_check title: 'Bar', report_status: 'published', rating: 'verified', language: 'en', claim_description: create_claim_description(project_media: pm1), disable_es_callbacks: false + create_tipline_request team_id: @team.id, associated: pm1 + + pm2 = create_project_media team: @team, disable_es_callbacks: false + fc2 = create_fact_check title: 'Foo', report_status: 'published', rating: 'verified', language: 'en', claim_description: create_claim_description(project_media: pm2), disable_es_callbacks: false + create_tipline_request team_id: @team.id, associated: pm2 + create_tipline_request team_id: @team.id, associated: pm2 + + sleep 2 + + object = TeamStatistics.new(@team, 'past_week', 'en') + expected = [{ id: fc2.id, label: 'Foo', value: 2 }, { id: fc1.id, label: 'Bar', value: 1 }] + assert_equal expected, object.top_articles_sent + end + + test "should return tipline statistics" do + pm1 = create_project_media team: @team, quote: 'Test' + create_fact_check claim_description: create_claim_description(project_media: pm1) + exp = create_explainer team: @team + pm1.explainers << exp + team = create_team + pm2 = create_project_media team: team + + travel_to Time.parse('2024-01-01') do + 2.times { create_tipline_message team_id: @team.id, language: 'en', platform: 'WhatsApp' } + create_tipline_message team_id: @team.id, language: 'en', platform: 'Telegram' + create_tipline_message team_id: @team.id, language: 'pt', platform: 'WhatsApp' + create_tipline_message team_id: team.id, language: 'en', platform: 'WhatsApp' + + create_tipline_request team_id: @team.id, associated: pm1, language: 'en', platform: 'whatsapp', smooch_request_type: 'relevant_search_result_requests' + create_tipline_request team_id: team.id, associated: pm2, language: 'en', platform: 'whatsapp', smooch_request_type: 'relevant_search_result_requests' + create_tipline_request team_id: @team.id, associated: pm1, language: 'pt', platform: 'whatsapp', smooch_request_type: 'relevant_search_result_requests' + create_tipline_request team_id: @team.id, associated: pm1, language: 'en', platform: 'telegram', smooch_request_type: 'relevant_search_result_requests' + end + + travel_to Time.parse('2024-01-03') do + 3.times { create_tipline_message team_id: @team.id, language: 'en', platform: 'WhatsApp' } + create_tipline_message team_id: @team.id, language: 'en', platform: 'Telegram' + create_tipline_message team_id: @team.id, language: 'pt', platform: 'WhatsApp' + create_tipline_message team_id: team.id, language: 'en', platform: 'WhatsApp' + + 2.times { create_tipline_request team_id: @team.id, associated: pm1, language: 'en', platform: 'whatsapp', smooch_request_type: 'irrelevant_search_result_requests' } + create_tipline_request team_id: team.id, associated: pm2, language: 'en', platform: 'whatsapp', smooch_request_type: 'relevant_search_result_requests' + create_tipline_request team_id: @team.id, associated: pm1, language: 'pt', platform: 'whatsapp', smooch_request_type: 'relevant_search_result_requests' + create_tipline_request team_id: @team.id, associated: pm1, language: 'en', platform: 'telegram', smooch_request_type: 'relevant_search_result_requests' + end + + travel_to Time.parse('2024-01-08') do + object = TeamStatistics.new(@team, 'past_week', 'en', 'whatsapp') + assert_equal 5, object.number_of_messages + assert_equal({ '2024-01-01' => 2, '2024-01-02' => 0, '2024-01-03' => 3, '2024-01-04' => 0, '2024-01-05' => 0, '2024-01-06' => 0, '2024-01-07' => 0, '2024-01-08' => 0 }, + object.number_of_messages_by_date) + assert_equal 3, object.number_of_conversations + assert_equal({ '2024-01-01' => 1, '2024-01-02' => 0, '2024-01-03' => 2, '2024-01-04' => 0, '2024-01-05' => 0, '2024-01-06' => 0, '2024-01-07' => 0, '2024-01-08' => 0 }, + object.number_of_conversations_by_date) + assert_equal({ 'Positive' => 1, 'Negative' => 2, 'No Response' => 0 }, object.number_of_search_results_by_feedback_type) + assert_equal({ 'Claim' => 3, 'Link' => 0, 'UploadedAudio' => 0, 'UploadedImage' => 0, 'UploadedVideo' => 0 }, object.number_of_media_received_by_media_type) + assert_equal 3, object.number_of_articles_sent + assert_equal({ 'FactCheck' => 3, 'Explainer' => 3 }, object.number_of_matched_results_by_article_type) + end + end + + test "should return top requested media clusters" do + setup_elasticsearch + RequestStore.store[:skip_cached_field_update] = false + channel = CheckChannels::ChannelCodes::WHATSAPP + Sidekiq::Testing.inline! do + pm1 = create_project_media team: @team, quote: 'Bar', channel: { main: channel, others: [channel] }, disable_es_callbacks: false + create_tipline_request team_id: @team.id, associated: pm1, platform: 'whatsapp', language: 'en', disable_es_callbacks: false + + pm2 = create_project_media team: @team, quote: 'Foo', channel: { main: channel, others: [channel] }, disable_es_callbacks: false + create_tipline_request team_id: @team.id, associated: pm2, platform: 'whatsapp', language: 'en', disable_es_callbacks: false + create_tipline_request team_id: @team.id, associated: pm2, platform: 'whatsapp', language: 'en', disable_es_callbacks: false + + pm3 = create_project_media team: @team, quote: 'Test 1', channel: { main: 0, others: [0] }, disable_es_callbacks: false + create_tipline_request team_id: @team.id, associated: pm3, platform: 'whatsapp', language: 'en', disable_es_callbacks: false + + pm4 = create_project_media team: @team, quote: 'Test 2', channel: { main: channel, others: [channel] }, disable_es_callbacks: false + create_tipline_request team_id: @team.id, associated: pm4, platform: 'whatsapp', language: 'pt', disable_es_callbacks: false + + sleep 3 + + object = TeamStatistics.new(@team, 'past_week', 'en', 'whatsapp') + expected = [{ id: pm2.id, label: 'Foo', value: 2 }, { id: pm1.id, label: 'Bar', value: 1 }] + assert_equal expected, object.top_requested_media_clusters + end + end + + test "should return top media tags" do + setup_elasticsearch + RequestStore.store[:skip_cached_field_update] = false + channel = CheckChannels::ChannelCodes::WHATSAPP + TestDynamicAnnotationTables.load! + create_annotation_type_and_fields('Language', { 'Language' => ['Text', true] }) + Sidekiq::Testing.inline! do + pm1 = create_project_media team: @team, channel: { main: channel, others: [channel] }, tags: ['foo', 'bar'], disable_es_callbacks: false + create_dynamic_annotation annotation_type: 'language', annotated: pm1, set_fields: { language: 'en' }.to_json, disable_es_callbacks: false + create_tipline_request team_id: @team.id, associated: pm1, platform: 'whatsapp', language: 'en', disable_es_callbacks: false + + pm2 = create_project_media team: @team, channel: { main: channel, others: [channel] }, tags: ['foo', 'test'], disable_es_callbacks: false + create_dynamic_annotation annotation_type: 'language', annotated: pm2, set_fields: { language: 'en' }.to_json, disable_es_callbacks: false + create_tipline_request team_id: @team.id, associated: pm2, platform: 'whatsapp', language: 'en', disable_es_callbacks: false + create_tipline_request team_id: @team.id, associated: pm2, platform: 'whatsapp', language: 'en', disable_es_callbacks: false + + pm3 = create_project_media team: @team, channel: { main: 0, others: [0] }, tags: ['test-1'], disable_es_callbacks: false + create_dynamic_annotation annotation_type: 'language', annotated: pm3, set_fields: { language: 'en' }.to_json, disable_es_callbacks: false + create_tipline_request team_id: @team.id, associated: pm3, platform: 'whatsapp', language: 'en', disable_es_callbacks: false + + pm4 = create_project_media team: @team, channel: { main: channel, others: [channel] }, tags: ['test-2'], disable_es_callbacks: false + create_dynamic_annotation annotation_type: 'language', annotated: pm4, set_fields: { language: 'pt' }.to_json, disable_es_callbacks: false + create_tipline_request team_id: @team.id, associated: pm4, platform: 'whatsapp', language: 'pt', disable_es_callbacks: false + + pm5 = create_project_media team: @team, channel: { main: channel, others: [channel] }, disable_es_callbacks: false + create_dynamic_annotation annotation_type: 'language', annotated: pm5, set_fields: { language: 'en' }.to_json, disable_es_callbacks: false + create_tipline_request team_id: @team.id, associated: pm4, platform: 'whatsapp', language: 'en', disable_es_callbacks: false + + sleep 3 + + object = TeamStatistics.new(@team, 'past_week', 'en', 'whatsapp') + expected = [{ id: 'foo', label: 'foo', value: 3 }, { id: 'test', label: 'test', value: 2 }, { id: 'bar', label: 'bar', value: 1 }] + assert_equal expected, object.top_media_tags + end + end +end diff --git a/test/models/bot/alegre_2_test.rb b/test/models/bot/alegre_2_test.rb index 29e1056640..efe63f0d47 100644 --- a/test/models/bot/alegre_2_test.rb +++ b/test/models/bot/alegre_2_test.rb @@ -315,7 +315,7 @@ def teardown WebMock.stub_request(:post, 'http://alegre.test/text/similarity/').to_return(body: 'success') WebMock.stub_request(:delete, 'http://alegre.test/text/similarity/').to_return(body: { success: true }.to_json) WebMock.stub_request(:delete, 'http://alegre.test/image/similarity/').to_return(body: { success: true }.to_json) - WebMock.stub_request(:post, 'http://alegre.test/text/similarity/search/').to_return(body: { success: true }.to_json) + WebMock.stub_request(:post, 'http://alegre.test/similarity/sync/text').to_return(body: { success: true }.to_json) WebMock.stub_request(:post, 'http://alegre.test/image/ocr/').to_return(body: { text: 'Foo bar' }.to_json) WebMock.stub_request(:post, 'http://alegre.test/similarity/sync/image').to_return(body: { result: [ diff --git a/test/models/bot/alegre_3_test.rb b/test/models/bot/alegre_3_test.rb index 5a7a90cfe5..1d9b74cec2 100644 --- a/test/models/bot/alegre_3_test.rb +++ b/test/models/bot/alegre_3_test.rb @@ -149,7 +149,7 @@ def teardown WebMock.disable_net_connect! allow: /#{CheckConfig.get('elasticsearch_host')}|#{CheckConfig.get('storage_endpoint')}/ WebMock.stub_request(:post, 'http://alegre/text/similarity/').to_return(body: 'success') WebMock.stub_request(:delete, 'http://alegre/text/similarity/').to_return(body: {success: true}.to_json) - WebMock.stub_request(:post, 'http://alegre/text/similarity/search/').to_return(body: {success: true}.to_json) + WebMock.stub_request(:post, 'http://alegre/similarity/sync/text').to_return(body: {success: true}.to_json) WebMock.stub_request(:post, 'http://alegre/audio/similarity/').to_return(body: { "success": true }.to_json) diff --git a/test/models/bot/alegre_test.rb b/test/models/bot/alegre_test.rb index 8f5ddcb94a..f15e9b5d08 100644 --- a/test/models/bot/alegre_test.rb +++ b/test/models/bot/alegre_test.rb @@ -44,7 +44,8 @@ def teardown WebMock.stub_request(:post, 'http://alegre/text/langid/').to_return(body: 'bad JSON response') WebMock.stub_request(:post, 'http://alegre/text/langid/').to_return(body: 'bad JSON response') WebMock.stub_request(:post, 'http://alegre/text/similarity/').to_return(body: 'success') - WebMock.stub_request(:post, 'http://alegre/text/similarity/search/').to_return(body: 'success') + WebMock.stub_request(:post, 'http://alegre/similarity/sync/text').to_return(body: 'success') + WebMock.stub_request(:post, 'http://alegre/similarity/async/text').to_return(body: 'success') WebMock.disable_net_connect! allow: /#{CheckConfig.get('elasticsearch_host')}|#{CheckConfig.get('storage_endpoint')}/ Bot::Alegre.any_instance.stubs(:get_language).raises(RuntimeError) assert_nothing_raised do @@ -102,6 +103,7 @@ def teardown test "should unarchive item after running" do WebMock.stub_request(:delete, 'http://alegre/text/similarity/').to_return(body: {success: true}.to_json) + WebMock.stub_request(:post, 'http://alegre/similarity/async/text').to_return(body: {results: []}.to_json) stub_configs({ 'alegre_host' => 'http://alegre', 'alegre_token' => 'test' }) do WebMock.stub_request(:delete, 'http://alegre/text/similarity/').to_return(status: 200, body: '{}') pm = create_project_media @@ -213,7 +215,7 @@ def teardown test "should index report data" do WebMock.stub_request(:delete, 'http://alegre:3100/text/similarity/').to_return(body: {success: true}.to_json) - WebMock.stub_request(:post, 'http://alegre:3100/text/similarity/').to_return(body: {}.to_json) + WebMock.stub_request(:post, 'http://alegre:3100/similarity/sync/text').to_return(body: {}.to_json) pm = create_project_media team: @team assert_nothing_raised do publish_report(pm) @@ -222,7 +224,7 @@ def teardown test "should use OCR data for similarity matching" do WebMock.stub_request(:post, 'http://alegre:3100/text/langid/').with(body: { text: 'Foo bar' }.to_json).to_return(status: 200, body: '{}') - WebMock.stub_request(:post, 'http://alegre:3100/text/similarity/').to_return(status: 200, body: '{}') + WebMock.stub_request(:post, 'http://alegre:3100/similarity/sync/text').to_return(status: 200, body: '{}') pm = create_project_media team: @team pm2 = create_project_media team: @team Bot::Alegre.stubs(:get_items_with_similar_description).returns({ pm2.id => {:score=>0.9, :context=>{"team_id"=>@team.id, "field"=>"original_description", "project_media_id"=>pm2.id, "has_custom_id"=>true}, :model=>"elasticsearch"} }) @@ -270,7 +272,7 @@ def teardown # This test to reproduce errbit error CHECK-1218 test "should match to existing parent" do WebMock.stub_request(:post, 'http://alegre:3100/text/langid/').with(body: { text: 'Foo bar' }.to_json).to_return(status: 200, body: '{}') - WebMock.stub_request(:post, 'http://alegre:3100/text/similarity/').to_return(status: 200, body: '{}') + WebMock.stub_request(:post, 'http://alegre:3100/similarity/sync/text').to_return(status: 200, body: '{}') pm_s = create_project_media team: @team pm = create_project_media team: @team pm2 = create_project_media team: @team @@ -288,7 +290,7 @@ def teardown test "should use transcription data for similarity matching" do WebMock.stub_request(:post, 'http://alegre:3100/text/langid/').with(body: { text: 'Foo bar' }.to_json).to_return(status: 200, body: '{}') WebMock.stub_request(:delete, 'http://alegre:3100/text/similarity/').to_return(status: 200, body: '{}') - WebMock.stub_request(:post, 'http://alegre:3100/text/similarity/').to_return(status: 200, body: '{}') + WebMock.stub_request(:post, 'http://alegre:3100/similarity/sync/text').to_return(status: 200, body: '{}') json_schema = { type: 'object', required: ['job_name'], @@ -313,7 +315,7 @@ def teardown end test "should check existing relationship before create a new one" do - WebMock.stub_request(:post, 'http://alegre:3100/text/similarity/').to_return(status: 200, body: '{}') + WebMock.stub_request(:post, 'http://alegre:3100/similarity/sync/text').to_return(status: 200, body: '{}') WebMock.stub_request(:post, 'http://alegre:3100/text/langid/').with(body: { text: 'Foo bar' }.to_json).to_return(status: 200, body: '{}') pm = create_project_media team: @team pm2 = create_project_media team: @team diff --git a/test/models/bot/alegre_v2_test.rb b/test/models/bot/alegre_v2_test.rb index f99d11b117..79e7ac4a3d 100644 --- a/test/models/bot/alegre_v2_test.rb +++ b/test/models/bot/alegre_v2_test.rb @@ -37,7 +37,7 @@ def teardown tpm = TemporaryProjectMedia.new tpm.type = k assert_equal tpm.media.type, v - [:is_blank?, :is_link?, :is_text?, :is_image?, :is_video?, :is_audio?].each do |meth| + [:is_blank?, :is_link?, :is_text?, :is_image?, :is_video?, :is_audio?, :is_uploaded_media?].each do |meth| assert_equal [true, false].include?(tpm.send(meth)), true end end @@ -484,6 +484,15 @@ def teardown assert_equal JSON.parse(Bot::Alegre.delete(pm1).to_json), JSON.parse(response.to_json) end + test "should return false and log error during delete request" do + pm1 = create_project_media team: @team, media: create_uploaded_audio + Bot::Alegre.stubs(:request_delete).raises(StandardError) + Rails.logger.expects(:error).with("[Alegre Bot] Exception on Delete for ProjectMedia ##{pm1.id}: Bot::Alegre::Error - StandardError").returns(nil) + CheckSentry.expects(:notify).with(instance_of(Bot::Alegre::Error), bot: "alegre", project_media: pm1, params: {}, field: nil).returns(false) + result = Bot::Alegre.delete(pm1) + assert_equal false, result + end + test "should get_items" do pm1 = create_project_media team: @team, media: create_uploaded_audio response = { diff --git a/test/models/bot/fetch_2_test.rb b/test/models/bot/fetch_2_test.rb index a457b66bf4..3c2ef02d61 100644 --- a/test/models/bot/fetch_2_test.rb +++ b/test/models/bot/fetch_2_test.rb @@ -39,7 +39,7 @@ def setup WebMock.stub_request(:get, 'http://fetch:8000/claim_reviews?end_time=2017-08-11&include_raw=false&offset=0&per_page=100&service=test&start_time=2017-08-06').to_return(body: [@claim_review].to_json) WebMock.stub_request(:post, 'http://fetch:8000/subscribe').with(body: { service: 'foo', url: 'http://check:3100/api/webhooks/fetch?team=fetch&token=test', language: nil }.to_json).to_return(body: '{}') WebMock.stub_request(:delete, 'http://fetch:8000/subscribe').with(body: { service: 'test', url: 'http://check:3100/api/webhooks/fetch?team=fetch&token=test'}.to_json).to_return(body: '{}') - WebMock.stub_request(:post, 'http://alegre:3100/text/similarity/').to_return(body: {}.to_json) + WebMock.stub_request(:post, 'http://alegre:3100/similarity/sync/text').to_return(body: {}.to_json) WebMock.stub_request(:delete, 'http://alegre:3100/text/similarity/').to_return(body: {}.to_json) create_verification_status_stuff diff --git a/test/models/bot/fetch_test.rb b/test/models/bot/fetch_test.rb index 83bf681611..33728ef056 100644 --- a/test/models/bot/fetch_test.rb +++ b/test/models/bot/fetch_test.rb @@ -44,7 +44,7 @@ def setup WebMock.stub_request(:get, 'http://fetch:8000/claim_reviews?end_time=2017-08-11&include_raw=false&offset=0&per_page=100&service=test&start_time=2017-08-06').to_return(body: [@claim_review].to_json) WebMock.stub_request(:post, 'http://fetch:8000/subscribe').with(body: { service: 'foo', url: 'http://check:3100/api/webhooks/fetch?team=fetch&token=test', language: nil }.to_json).to_return(body: '{}') WebMock.stub_request(:delete, 'http://fetch:8000/subscribe').with(body: { service: 'test', url: 'http://check:3100/api/webhooks/fetch?team=fetch&token=test'}.to_json).to_return(body: '{}') - WebMock.stub_request(:post, 'http://alegre:3100/text/similarity/').to_return(body: {}.to_json) + WebMock.stub_request(:post, 'http://alegre:3100/similarity/sync/text').to_return(body: {}.to_json) WebMock.stub_request(:delete, 'http://alegre:3100/text/similarity/').to_return(body: {}.to_json) create_verification_status_stuff diff --git a/test/models/bot/smooch_3_test.rb b/test/models/bot/smooch_3_test.rb index acef2b3e66..843b40602d 100644 --- a/test/models/bot/smooch_3_test.rb +++ b/test/models/bot/smooch_3_test.rb @@ -74,8 +74,8 @@ def teardown long_text = [] 15.times{ long_text << random_string } # messages contain the following: - # 1). long text( > min_number_of_words_for_tipline_submit_shortcut) - # 2). short text (< min_number_of_words_for_tipline_submit_shortcut) + # 1). long text( > min_number_of_words_for_tipline_long_text) + # 2). short text (< min_number_of_words_for_tipline_long_text) # 3). link # 4). 2 medias # Result: created four items (one claim, one link and two items of type image) @@ -191,13 +191,146 @@ def teardown end end + test "should bundle message with link and text" do + uid = random_string + payload = { + trigger: 'message:appUser', + app: { + '_id': @app_id + }, + version: 'v1.1', + messages: [message], + appUser: { + '_id': random_string, + 'conversationStarted': true + } + } + Sidekiq::Testing.fake! do + # 1) Send link and short text + message = { + '_id': random_string, + authorId: uid, + type: 'text', + source: { type: "whatsapp" }, + text: "#{@link_url} short text", + } + payload[:messages] = [message] + Bot::Smooch.run(payload.to_json) + sleep 1 + assert_difference 'ProjectMedia.count' do + assert_no_difference 'Claim.count' do + assert_difference 'Link.count' do + Sidekiq::Worker.drain_all + end + end + end + # Clean up created items to start other cases with same link + ProjectMedia.last.destroy + Link.last.destroy + # 2) Send link with long text + long_text = [] + 15.times{ long_text << random_string } + link_long_text = @link_url.concat(' ').concat(long_text.join(' ')) + message = { + '_id': random_string, + authorId: uid, + type: 'text', + source: { type: "whatsapp" }, + text: link_long_text, + } + payload[:messages] = [message] + Bot::Smooch.run(payload.to_json) + sleep 1 + pm_id = ProjectMedia.last.id + assert_difference 'ProjectMedia.count', 2 do + assert_difference 'Claim.count' do + assert_difference 'Link.count' do + assert_difference 'Relationship.count' do + Sidekiq::Worker.drain_all + end + end + end + end + l1 = Link.last + c1 = Claim.last + pm_l1 = l1.project_medias.last + pm_c1 = c1.project_medias.last + r = Relationship.last + assert_equal [pm_l1.id, pm_c1.id].sort, [r.source_id, r.target_id].sort + # 3) Same message multiple times (re-send message in step 2) + message['_id'] = random_string + payload[:messages] = [message] + Bot::Smooch.run(payload.to_json) + sleep 1 + assert_no_difference 'ProjectMedia.count' do + assert_no_difference 'Relationship.count' do + assert_difference 'TiplineRequest.count', 2 do + Sidekiq::Worker.drain_all + end + end + end + assert_equal 2, pm_l1.tipline_requests.count + assert_equal 2, pm_c1.tipline_requests.count + # 4) Send different messages with the same link + long_text2 = [] + 15.times{ long_text2 << random_string } + link_long_text2 = long_text2.join(' ').concat(' ').concat(@link_url) + message = { + '_id': random_string, + authorId: uid, + type: 'text', + source: { type: "whatsapp" }, + text: link_long_text2, + } + payload[:messages] = [message] + Bot::Smooch.run(payload.to_json) + sleep 1 + assert_difference 'ProjectMedia.count' do + assert_difference 'Relationship.count' do + assert_difference 'Claim.count' do + assert_no_difference 'Link.count' do + Sidekiq::Worker.drain_all + end + end + end + end + pm = ProjectMedia.last + r = Relationship.last + assert_equal [pm_l1.id, pm.id].sort, [r.source_id, r.target_id].sort + # 5) Send two messages with the same text but different links + link_long_text3 = @link_url_2.concat(' ').concat(long_text.join(' ')).concat(' ').concat(random_string) + message = { + '_id': random_string, + authorId: uid, + type: 'text', + source: { type: 'whatsapp' }, + text: link_long_text3, + } + payload[:messages] = [message] + Bot::Smooch.run(payload.to_json) + sleep 1 + assert_difference 'ProjectMedia.count', 2 do + assert_difference 'Relationship.count' do + assert_difference 'Link.count' do + assert_difference 'Claim.count' do + Sidekiq::Worker.drain_all + end + end + end + end + pm = ProjectMedia.last + r = Relationship.last + assert_equal pm.id, r.target_id + end + end + test "should force relationship between media and caption text" do long_text = [] 15.times{ long_text << random_string } caption = long_text.join(' ') # messages contain the following: - # 1). media with long text( > min_number_of_words_for_tipline_submit_shortcut) - # 2). media with short text (< min_number_of_words_for_tipline_submit_shortcut) + # 1). media with long text( > min_number_of_words_for_tipline_long_text) + # 2). media with short text (< min_number_of_words_for_tipline_long_text) # Result: created three items and one relationship (one claim for caption and two items of type image) last_id = ProjectMedia.last.id Sidekiq::Testing.fake! do @@ -250,7 +383,7 @@ def teardown claim_item = ProjectMedia.joins(:media).where('medias.type' => 'Claim').last assert_equal caption, claim_item.media.quote r = Relationship.last - assert_equal Relationship.suggested_type, r.relationship_type + assert_equal Relationship.confirmed_type, r.relationship_type assert_equal claim_item.id, r.target_id assert_equal 1, claim_item.tipline_requests.count end diff --git a/test/models/bot/smooch_6_test.rb b/test/models/bot/smooch_6_test.rb index 27cc6fd8e6..52590c01ba 100644 --- a/test/models/bot/smooch_6_test.rb +++ b/test/models/bot/smooch_6_test.rb @@ -139,6 +139,8 @@ def send_message_outside_24_hours_window(template, pm = nil) test "should submit query without details on tipline bot v2" do WebMock.stub_request(:post, /\/text\/similarity\/search\//).to_return(body: {}.to_json) # For explainers + WebMock.stub_request(:post, /\/similarity\/async\/text/).to_return(body: {}.to_json) # For explainers + WebMock.stub_request(:post, /\/similarity\/sync\/text/).to_return(body: {}.to_json) # For explainers claim = 'This is a test claim' send_message 'hello', '1', '1', random_string, random_string, claim, random_string, random_string, '1' assert_saved_query_type 'default_requests' @@ -208,6 +210,8 @@ def send_message_outside_24_hours_window(template, pm = nil) end test "should submit query with details on tipline bot v2" do + WebMock.stub_request(:post, /\/similarity\/sync\/text/).to_return(body: {}.to_json) # For explainers + WebMock.stub_request(:post, /\/similarity\/async\/text/).to_return(body: {}.to_json) # For explainers WebMock.stub_request(:post, /\/text\/similarity\/search\//).to_return(body: {}.to_json) # For explainers claim = 'This is a test claim' send_message 'hello', '1', '1', random_string, '2', random_string, claim, '1' @@ -285,7 +289,7 @@ def send_message_outside_24_hours_window(template, pm = nil) end test "should submit query and handle search error on tipline bot v2" do - WebMock.stub_request(:post, /\/text\/similarity\/search\//).to_return(body: {}.to_json) # For explainers + WebMock.stub_request(:post, /\/similarity\/sync\/text/).to_return(body: {}.to_json) # For explainers CheckSearch.any_instance.stubs(:medias).raises(StandardError) Sidekiq::Testing.inline! do send_message 'hello', '1', '1', 'Foo bar', '1' @@ -384,7 +388,7 @@ def send_message_outside_24_hours_window(template, pm = nil) ProjectMedia.any_instance.stubs(:report_status).returns('published') ProjectMedia.any_instance.stubs(:analysis_published_article_url).returns(random_url) Bot::Alegre.stubs(:get_merged_similar_items).returns({ create_project_media.id => { score: 0.9 } }) - WebMock.stub_request(:post, /\/text\/similarity\/search\//).to_return(body: {}.to_json) # For explainers + WebMock.stub_request(:post, /\/similarity\/sync\/text/).to_return(body: {}.to_json) # For explainers Sidekiq::Testing.inline! do send_message 'hello', '1', '1', "Foo bar foo bar #{url} foo bar", '1' end @@ -664,7 +668,7 @@ def send_message_outside_24_hours_window(template, pm = nil) test "should not duplicate messages when saving" do @team.set_languages ['en'] @team.save! - message_text = 'not_a_url' #Not a URL, not media, and not longer than 'min_number_of_words_for_tipline_submit_shortcut' + message_text = 'not_a_url' #Not a URL, not media, and not longer than 'min_number_of_words_for_tipline_long_text' send_message message_text, '1', message_text, '1' assert_state 'search' Sidekiq::Worker.drain_all @@ -693,7 +697,7 @@ def send_message_outside_24_hours_window(template, pm = nil) pm = create_project_media team: @team publish_report(pm, {}, nil, { language: 'pt', use_visual_card: false }) Bot::Smooch.stubs(:get_search_results).returns([pm]) - WebMock.stub_request(:post, /\/text\/similarity\/search\//).to_return(body: {}.to_json) # For explainers + WebMock.stub_request(:post, /\/similarity\/sync\/text/).to_return(body: {}.to_json) # For explainers Sidekiq::Testing.inline! do send_message 'hello', '1', '1', 'Foo bar', '1' end @@ -807,9 +811,9 @@ def send_message_outside_24_hours_window(template, pm = nil) test 'should process menu option using NLU' do # Mock any call to Alegre like `POST /text/similarity/` with a "text" parameter that contains "want" - Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/text/similarity/' && z[:text] =~ /want/ }.returns(true) + Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/similarity/sync/text' && z[:text] =~ /want/ }.returns(true) # Mock any call to Alegre like `GET /text/similarity/` with a "text" parameter that does not contain "want" - Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/text/similarity/search/' && (z[:text] =~ /want/).nil? }.returns({ 'result' => [] }) + Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/similarity/sync/text' && (z[:text] =~ /want/).nil? }.returns({ 'result' => [] }) # Enable NLU and add a couple of keywords for the newsletter menu option nlu = SmoochNlu.new(@team.slug) @@ -822,9 +826,9 @@ def send_message_outside_24_hours_window(template, pm = nil) subscription_option_id = @installation.get_smooch_workflows[0]['smooch_state_main']['smooch_menu_options'][2]['smooch_menu_option_id'] # Mock a call to Alegre like `GET /text/similarity/` with a "text" parameter that contains "want" - Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/text/similarity/search/' && z[:text] =~ /want/ }.returns({ 'result' => [ - { '_score' => 0.9, '_source' => { 'context' => { 'menu_option_id' => subscription_option_id } } }, - { '_score' => 0.2, '_source' => { 'context' => { 'menu_option_id' => query_option_id } } } + Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/similarity/sync/text' && z[:text] =~ /want/ }.returns({ 'result' => [ + { 'score' => 0.9, 'context' => { 'menu_option_id' => subscription_option_id } }, + { 'score' => 0.2, 'context' => { 'menu_option_id' => query_option_id } } ]}) # Sending a message about the newsletter should take to the newsletter state, as per configurations done above @@ -836,9 +840,9 @@ def send_message_outside_24_hours_window(template, pm = nil) assert_state 'main' # Mock a call to Alegre like `GET /text/similarity/` with a "text" parameter that contains "want" - Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/text/similarity/search/' && z[:text] =~ /want/ }.returns({ 'result' => [ - { '_score' => 0.96, '_source' => { 'context' => { 'menu_option_id' => subscription_option_id } } }, - { '_score' => 0.91, '_source' => { 'context' => { 'menu_option_id' => query_option_id } } } + Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/similarity/sync/text' && z[:text] =~ /want/ }.returns({ 'result' => [ + { 'score' => 0.96, 'context' => { 'menu_option_id' => subscription_option_id } }, + { 'score' => 0.91, 'context' => { 'menu_option_id' => query_option_id } } ]}) # Sending a message that returns more than one option (disambiguation) @@ -875,9 +879,9 @@ def send_message_outside_24_hours_window(template, pm = nil) Sidekiq::Testing.fake! do WebMock.disable_net_connect! allow: /#{CheckConfig.get('elasticsearch_host')}|#{CheckConfig.get('storage_endpoint')}/ # Mock any call to Alegre like `POST /text/similarity/` with a "text" parameter that contains "who are you" - Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/text/similarity/' && z[:text] =~ /who are you/ }.returns(true) + Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/similarity/sync/text' && z[:text] =~ /who are you/ }.returns(true) # Mock any call to Alegre like `GET /text/similarity/` with a "text" parameter that does not contain "who are you" - Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/text/similarity/search/' && (z[:text] =~ /who are you/).nil? }.returns({ 'result' => [] }) + Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/similarity/sync/text' && (z[:text] =~ /who are you/).nil? }.returns({ 'result' => [] }) # Enable NLU and add a couple of keywords to a new "About Us" resource nlu = SmoochNlu.new(@team.slug) @@ -887,9 +891,9 @@ def send_message_outside_24_hours_window(template, pm = nil) r.add_keyword('who are you') # Mock a call to Alegre like `GET /text/similarity/` with a "text" parameter that contains "who are you" - Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/text/similarity/search/' && z[:text] =~ /who are you/ }.returns({ 'result' => [ - { '_score' => 0.9, '_source' => { 'context' => { 'resource_id' => 0 } } }, - { '_score' => 0.8, '_source' => { 'context' => { 'resource_id' => r.id } } } + Bot::Alegre.stubs(:request).with{ |x, y, z| x == 'post' && y == '/similarity/sync/text' && z[:text] =~ /who are you/ }.returns({ 'result' => [ + { '_score' => 0.9, 'context' => { 'resource_id' => 0 } }, + { '_score' => 0.8, 'context' => { 'resource_id' => r.id } } ]}) # Sending a message asking about the tipline should send a resource, as per configurations done above @@ -965,13 +969,16 @@ def send_message_outside_24_hours_window(template, pm = nil) end test "should get dynamic resource on tipline bot v2" do - WebMock.stub_request(:get, /googleapis\.com\/civicinfo/).to_return(body: { pollingLocations: [{ address: {} }], earlyVoteSites: [{ address: {} }] }.to_json) - @resource.content_type = 'dynamic' - @resource.save! - send_message 'hello', '1', '4' - assert_state 'resource_waiting_for_user_input' - send_message '972 Mission St San Francisco CA' - assert_state 'waiting_for_message' + civic_api_mocked_data = { pollingLocations: [{ address: {}, startDate: '2024-11-01', endDate: '2024-11-05', pollingHours: '7am - 5pm' }], earlyVoteSites: [{ address: {}, startDate: '2024-11-01', endDate: '2024-11-05', pollingHours: '7am - 5pm' }] } + WebMock.stub_request(:get, /googleapis\.com\/civicinfo/).to_return(body: civic_api_mocked_data.to_json) + stub_configs({ 'google_api_key' => random_string }) do + @resource.content_type = 'dynamic' + @resource.save! + send_message 'hello', '1', '4' + assert_state 'resource_waiting_for_user_input' + send_message '972 Mission St San Francisco CA' + assert_state 'waiting_for_message' + end end test "should not get dynamic resource on tipline bot v2 if resource is not available anymore" do @@ -986,11 +993,30 @@ def send_message_outside_24_hours_window(template, pm = nil) test "should not get dynamic resource on tipline bot v2 if external API returns an error" do WebMock.stub_request(:get, /googleapis\.com\/civicinfo/).to_return(body: { pollingLocations: 'Some error' }.to_json) - @resource.content_type = 'dynamic' - @resource.save! - send_message 'hello', '1', '4' - assert_state 'resource_waiting_for_user_input' - send_message '972 Mission St San Francisco CA' - assert_state 'waiting_for_message' + stub_configs({ 'google_api_key' => random_string }) do + @resource.content_type = 'dynamic' + @resource.save! + send_message 'hello', '1', '4' + assert_state 'resource_waiting_for_user_input' + send_message '972 Mission St San Francisco CA' + assert_state 'waiting_for_message' + end + end + + test "should unsubscribe user when clicking on newsletter button" do + Sidekiq::Testing.inline! do + # Create subscription + ts = create_tipline_subscription team_id: @team.id, uid: @uid + assert_not_nil TiplineSubscription.find_by_id(ts.id) + + # "Send" newsletter + message_id = random_string + response = OpenStruct.new(body: OpenStruct.new({ message: OpenStruct.new(id: message_id) })) + Bot::Smooch.save_smooch_response(response, nil, Time.now.to_i, 'newsletter', 'en') + + # Click on "Unsubscribe" button + send_message_to_smooch_bot('Unsubscribe', @uid, { 'quotedMessage' => { 'content' => { '_id' => message_id } } }) + assert_nil TiplineSubscription.find_by_id(ts.id) + end end end diff --git a/test/models/bot/smooch_7_test.rb b/test/models/bot/smooch_7_test.rb index 4fd46ac40e..8aee9c2afe 100644 --- a/test/models/bot/smooch_7_test.rb +++ b/test/models/bot/smooch_7_test.rb @@ -413,6 +413,8 @@ def teardown sleep 2 # Wait for ElasticSearch to index content assert_equal [pm1.id, pm2.id, pm3.id], Bot::Smooch.search_for_similar_published_fact_checks('text', 'Foo Bar', [t.id]).to_a.map(&:id) + # Calling wiht skip_cache true + assert_equal [pm1.id, pm2.id, pm3.id], Bot::Smooch.search_for_similar_published_fact_checks('text', 'Foo Bar', [t.id], nil, nil, nil, true).to_a.map(&:id) end test "should store media" do @@ -600,7 +602,8 @@ def teardown end test "should include claim_description_content in smooch search" do - WebMock.stub_request(:post, 'http://alegre:3100/text/similarity/').to_return(body: {}.to_json) + WebMock.stub_request(:post, 'http://alegre:3100/similarity/async/image').to_return(body: {}.to_json) + WebMock.stub_request(:post, 'http://alegre:3100/similarity/sync/text').to_return(body: {}.to_json) RequestStore.store[:skip_cached_field_update] = false t = create_team m = create_uploaded_image diff --git a/test/models/explainer_test.rb b/test/models/explainer_test.rb index a74124c711..7d191e1e7b 100644 --- a/test/models/explainer_test.rb +++ b/test/models/explainer_test.rb @@ -83,6 +83,7 @@ def setup end test "should create tag texts when setting tags" do + WebMock.stub_request(:post, /\/similarity\/async\/text/).to_return(body: {}.to_json) # For explainers Sidekiq::Testing.inline! do assert_difference 'TagText.count' do create_explainer tags: ['foo'] @@ -99,12 +100,12 @@ def setup } # Index two paragraphs and title when the explainer is created - Bot::Alegre.stubs(:request).with('post', '/text/similarity/', anything).times(3) + Bot::Alegre.stubs(:request).with('post', '/similarity/async/text', anything).times(3) Bot::Alegre.stubs(:request).with('delete', '/text/similarity/', anything).never ex = create_explainer description: description # Update the index when paragraphs change - Bot::Alegre.stubs(:request).with('post', '/text/similarity/', anything).times(2) + Bot::Alegre.stubs(:request).with('post', '/similarity/async/text', anything).times(2) Bot::Alegre.stubs(:request).with('delete', '/text/similarity/', anything).once ex = Explainer.find(ex.id) ex.description = 'Now this is the only paragraph' diff --git a/test/models/fact_check_test.rb b/test/models/fact_check_test.rb index 9e2a887f0c..88b8a70d0c 100644 --- a/test/models/fact_check_test.rb +++ b/test/models/fact_check_test.rb @@ -690,4 +690,42 @@ def setup assert_equal 'unstarted', pm.reload.status end end + + test "should reset report when fact-check is detached" do + RequestStore.store[:skip_cached_field_update] = false + create_report_design_annotation_type + Sidekiq::Testing.inline! do + pm = create_project_media + cd = create_claim_description(project_media: pm) + fc = create_fact_check claim_description: cd, title: 'Foo' + r = pm.get_dynamic_annotation('report_design') + publish_report(pm, {}, r) + assert_equal 'published', pm.reload.report_status + assert_equal 'Foo', pm.reload.report_text_title + + cd.project_media = nil # Remove the claim/fact-check from the item + cd.save! + assert_equal 'paused', pm.reload.report_status + assert_equal '', pm.reload.report_text_title + + cd.project_media = pm # Re-add the claim/fact-check to the item + cd.save! + assert_equal 'paused', pm.reload.report_status + assert_equal 'Foo', pm.reload.report_text_title + end + end + + test "should reset cached fields when fact-check is detached" do + RequestStore.store[:skip_cached_field_update] = false + Sidekiq::Testing.inline! do + pm = create_project_media + cd = create_claim_description project_media: pm + fc = create_fact_check claim_description: cd, title: 'Foo' + assert_equal fc.id, pm.reload.fact_check_id + + cd.project_media = nil # Remove the claim/fact-check from the item + cd.save! + assert_nil pm.reload.fact_check_id + end + end end diff --git a/test/models/media_test.rb b/test/models/media_test.rb index 123c2734e2..711f8f74ca 100644 --- a/test/models/media_test.rb +++ b/test/models/media_test.rb @@ -604,6 +604,18 @@ def setup end end + test 'should validate url length' do + pender_url = CheckConfig.get('pender_url_private') + '/api/medias' + url = "#{random_url}?params=#{random_string(2000)}" + response = { type: 'media', data: { url: url, type: 'item', title: "Foo \u0000 bar" } } + WebMock.stub_request(:get, pender_url).with({ query: { url: url } }).to_return(body: response.to_json) + assert_no_difference 'Link.count' do + assert_raises ActiveRecord::RecordInvalid do + create_media url: url + end + end + end + test "should have uuid" do m = create_media assert_equal m.id, m.uuid diff --git a/test/models/project_media_5_test.rb b/test/models/project_media_5_test.rb index 709aa4450b..ef8ba5a0d2 100644 --- a/test/models/project_media_5_test.rb +++ b/test/models/project_media_5_test.rb @@ -174,15 +174,6 @@ def setup assert pm.is_link? end - test "checks truthfulness of is_uploaded_image?" do - u = create_user - t = create_team - pm = create_project_media team: t - pm.media.type = "UploadedImage" - pm.media.save! - assert pm.is_uploaded_image? - end - test "checks truthfulness of is_image?" do u = create_user t = create_team diff --git a/test/models/project_media_7_test.rb b/test/models/project_media_7_test.rb index c96ca0a7cc..73553e8b81 100644 --- a/test/models/project_media_7_test.rb +++ b/test/models/project_media_7_test.rb @@ -67,4 +67,52 @@ def setup assert_equal 'Claim', pm_claim.media.type assert_equal 'This is a claim.', pm_claim.media.quote end + + test "should not create duplicate media from original claim URL as Link" do + setup_elasticsearch + + # Mock Pender response for Link + link_url = 'https://example.com' + pender_url = CheckConfig.get('pender_url_private') + '/api/medias' + link_response = { + type: 'media', + data: { + url: link_url, + type: 'item' + } + }.to_json + WebMock.stub_request(:get, pender_url).with(query: { url: link_url }).to_return(body: link_response) + + t = create_team + create_project team: t + + assert_raise RuntimeError do + 2.times { create_project_media(team: t, set_original_claim: link_url) } + end + end + + test "should create duplicate media from original claim URL as UploadedImage" do + Tempfile.create(['test_image', '.jpg']) do |file| + file.write(File.read(File.join(Rails.root, 'test', 'data', 'rails.png'))) + file.rewind + image_url = "http://example.com/#{file.path.split('/').last}" + WebMock.stub_request(:get, image_url).to_return(body: file.read, headers: { 'Content-Type' => 'image/jpeg' }) + + t = create_team + create_project team: t + + assert_difference 'ProjectMedia.count', 2 do + 2.times { create_project_media(team: t, set_original_claim: image_url) } + end + end + end + + test "should create duplicate media from original claim URL as Claim" do + t = create_team + create_project team: t + + assert_difference 'ProjectMedia.count', 2 do + 2.times { create_project_media(team: t, set_original_claim: 'This is a claim.') } + end + end end diff --git a/test/models/relationship_2_test.rb b/test/models/relationship_2_test.rb index 50ec8c3f68..4e8d976f70 100644 --- a/test/models/relationship_2_test.rb +++ b/test/models/relationship_2_test.rb @@ -2,12 +2,15 @@ class Relationship2Test < ActiveSupport::TestCase def setup - super - Sidekiq::Testing.inline! + Sidekiq::Testing.fake! @team = create_team @project = create_project team: @team end + def teardown + User.current = Team.current = nil + end + test "should create relationship" do assert_difference 'Relationship.count' do create_relationship @@ -54,6 +57,7 @@ def setup end test "should destroy relationships when project media is destroyed" do + Sidekiq::Testing.inline! pm = create_project_media team: @team pm2 = create_project_media team: @team pm3 = create_project_media team: @team @@ -140,6 +144,7 @@ def setup end test "should archive or restore medias when source is archived or restored" do + Sidekiq::Testing.inline! RequestStore.store[:skip_delete_for_ever] = true s = create_project_media project: @project t1 = create_project_media project: @project @@ -159,6 +164,7 @@ def setup end test "should delete medias when source is deleted" do + Sidekiq::Testing.inline! s = create_project_media project: @project t1 = create_project_media project: @project t2 = create_project_media project: @project @@ -205,6 +211,7 @@ def setup end test "should have versions" do + Sidekiq::Testing.inline! with_versioning do u = create_user is_admin: true t = create_team @@ -243,6 +250,7 @@ def setup end test "should propagate change if source and target are swapped" do + Sidekiq::Testing.inline! u = create_user is_admin: true t = create_team with_current_user_and_team(u, t) do @@ -342,6 +350,7 @@ def setup test "should cache the name of who confirmed a similar item and store confirmation information" do RequestStore.store[:skip_cached_field_update] = false + Sidekiq::Testing.inline! t = create_team u = create_user is_admin: true pm1 = create_project_media team: t @@ -361,4 +370,122 @@ def setup r.destroy! assert_queries(0, '=') { assert_nil pm2.confirmed_as_similar_by_name } end + + test "should move fact-check from child to parent when creating relationship if child has a fact-check but parent does not" do + t = create_team + child = create_project_media team: t + create_claim_description project_media: child + parent = create_project_media team: t + relationship = nil + + # No report for any of them: No failure + assert_difference 'Relationship.count' do + assert_nothing_raised do + relationship = create_relationship source: parent, target: child, relationship_type: Relationship.confirmed_type + end + end + relationship.destroy! + + # Child has a published report, but parent doesn't: No failure; claim/fact-check/report should be moved from the child to the parent + child = ProjectMedia.find(child.id) + parent = ProjectMedia.find(parent.id) + report = publish_report(child) + claim = child.reload.claim_description + assert_not_nil report + assert_not_nil claim + assert_not_nil child.reload.claim_description + assert_not_nil child.reload.get_dynamic_annotation('report_design') + assert_nil parent.reload.claim_description + assert_nil parent.reload.get_dynamic_annotation('report_design') + assert_difference 'Relationship.count' do + assert_nothing_raised do + relationship = create_relationship source: parent, target: child, relationship_type: Relationship.confirmed_type + end + end + assert_not_nil parent.reload.claim_description + assert_not_nil parent.reload.get_dynamic_annotation('report_design') + assert_equal claim, parent.reload.claim_description + assert_equal report, parent.reload.get_dynamic_annotation('report_design') + assert_nil child.reload.claim_description + assert_nil child.reload.get_dynamic_annotation('report_design') + relationship.destroy! + + # Child has a published report, and parent has one too: Failure + child = ProjectMedia.find(child.id) + parent = ProjectMedia.find(parent.id) + publish_report(child) + assert_no_difference 'Relationship.count' do + assert_raises 'ActiveRecord::RecordInvalid' do + create_relationship source: parent, target: child, relationship_type: Relationship.confirmed_type + end + end + end + + test "should belong to only one media cluster" do + t = create_team + pm1 = create_project_media team: t + pm2 = create_project_media team: t + pm3 = create_project_media team: t + pm4 = create_project_media team: t + + # Create a relationship between two items + assert_difference 'Relationship.count' do + assert_nothing_raised do + create_relationship source_id: pm1.id, target_id: pm2.id + end + end + + # If an item is already a child, it can't be a child in another relationship + assert_no_difference 'Relationship.count' do + assert_raises ActiveRecord::StatementInvalid do + create_relationship source_id: pm3.id, target_id: pm2.id + end + end + + # If an item is already a child, it can't be a parent in another relationship + assert_no_difference 'Relationship.count' do + assert_raises ActiveRecord::StatementInvalid do + create_relationship source_id: pm2.id, target_id: pm3.id + end + end + + # If an item is already a parent, it can't be a child in another relationship - move targets to new relationship + assert_equal 1, Relationship.where(source_id: pm1.id).count + assert_equal 0, Relationship.where(source_id: pm3.id).count + create_relationship source_id: pm3.id, target_id: pm1.id + assert_equal 0, Relationship.where(source_id: pm1.id).count + assert_equal 2, Relationship.where(source_id: pm3.id).count + + # If an item is already a parent, it can still have another child + assert_difference 'Relationship.count' do + assert_nothing_raised do + create_relationship source_id: pm3.id, target_id: pm4.id + end + end + end + + # If we're trying to create a relationship between C (target_id) and B (source_id), but there is already a relationship between A (source_id) and B (target_id), + # then, instead, create the relationship between A (source_id) and C (target_id) (so, if A's cluster contains B, then C comes in and our algorithm says C is similar + # to B, it is added to A's cluster). Exception: If the relationship between A (source_id) and B (target_id) is a suggestion, we should not create any relationship + # at all when trying to create a relationship between C (target_id) and B (source_id) (regardless if it’s a suggestion or a confirmed match) - but we should log that case. + test "should add to existing media cluster" do + t = create_team + a = create_project_media team: t + b = create_project_media team: t + c = create_project_media team: t + Relationship.create_unless_exists(a.id, b.id, Relationship.confirmed_type) + Relationship.create_unless_exists(b.id, c.id, Relationship.confirmed_type) + assert !Relationship.where(source: b, target: c).exists? + assert Relationship.where(source: a, target: b).exists? + assert Relationship.where(source: a, target: c).exists? + + a = create_project_media team: t + b = create_project_media team: t + c = create_project_media team: t + Relationship.create_unless_exists(a.id, b.id, Relationship.suggested_type) + Relationship.create_unless_exists(b.id, c.id, Relationship.confirmed_type) + assert !Relationship.where(source: b, target: c).exists? + assert Relationship.where(source: a, target: b).exists? + assert !Relationship.where(source: a, target: c).exists? + end end diff --git a/test/models/relationship_test.rb b/test/models/relationship_test.rb index 74d847a46b..f52b1e3372 100644 --- a/test/models/relationship_test.rb +++ b/test/models/relationship_test.rb @@ -36,8 +36,10 @@ def setup test "should update sources_count and parent_id for confirmed item" do setup_elasticsearch t = create_team - pm_s = create_project_media team: t - pm_t = create_project_media team: t + pm_s = create_project_media team: t, disable_es_callbacks: false + pm_t = create_project_media team: t, disable_es_callbacks: false + pm_t2 = create_project_media team: t, disable_es_callbacks: false + pm_t3 = create_project_media team: t, disable_es_callbacks: false r = create_relationship source_id: pm_s.id, target_id: pm_t.id, relationship_type: Relationship.suggested_type sleep 2 es_t = $repository.find(get_es_id(pm_t)) @@ -47,11 +49,36 @@ def setup # Confirm item r.relationship_type = Relationship.confirmed_type r.save! - sleep 2 - es_t = $repository.find(get_es_id(pm_t)) - assert_equal r.source_id, es_t['parent_id'] - assert_equal pm_t.reload.sources_count, es_t['sources_count'] - assert_equal 1, pm_t.reload.sources_count + r2 = create_relationship source_id: pm_s.id, target_id: pm_t2.id, relationship_type: Relationship.confirmed_type + r3 = create_relationship source_id: pm_s.id, target_id: pm_t3.id, relationship_type: Relationship.suggested_type + sleep 1 + es_s = $repository.find(get_es_id(pm_s)) + assert_equal pm_s.id, es_s['parent_id'] + assert_equal pm_s.reload.sources_count, es_s['sources_count'] + assert_equal 0, pm_s.reload.sources_count + [pm_t, pm_t2].each do |pm| + es = $repository.find(get_es_id(pm)) + assert_equal pm_s.id, es['parent_id'] + assert_equal pm.reload.sources_count, es['sources_count'] + assert_equal 1, pm.reload.sources_count + end + # Verify parent_id after pin another item + r2.source_id = pm_t2.id + r2.target_id = pm_s.id + r2.disable_es_callbacks = false + r2.save! + sleep 1 + es_t2 = $repository.find(get_es_id(pm_t2)) + assert_equal pm_t2.id, es_t2['parent_id'] + assert_equal pm_t2.reload.sources_count, es_t2['sources_count'] + assert_equal 0, pm_t2.reload.sources_count + [pm_s, pm_t].each do |pm| + es = $repository.find(get_es_id(pm)) + assert_equal pm_t2.id, es['parent_id'] + assert_equal pm.reload.sources_count, es['sources_count'] + assert_equal 1, pm.reload.sources_count + end + # Verify destory r.destroy! es_t = $repository.find(get_es_id(pm_t)) assert_equal pm_t.id, es_t['parent_id'] @@ -199,6 +226,55 @@ def setup end end + test "should move explainer to source after pin item or match items" do + t = create_team + e = create_explainer team: t + e_s = create_explainer team: t + e_t = create_explainer team: t + source = create_project_media team: t + target = create_project_media team: t + u = create_user + create_team_user team: t, user: u, role: 'admin' + with_versioning do + with_current_user_and_team(u, t) do + source.explainers << e + source.explainers << e_s + target.explainers << e + target.explainers << e_t + end + end + assert_equal 2, source.explainer_items.count + assert_equal 2, target.explainer_items.count + sv_count = Version.from_partition(t.id).where(event_type: 'create_explaineritem', associated_id: source.id).count + tv_count = Version.from_partition(t.id).where(event_type: 'create_explaineritem', associated_id: target.id).count + assert_equal 2, sv_count + assert_equal 2, tv_count + r = create_relationship source_id: source.id, target_id: target.id, relationship_type: Relationship.confirmed_type + assert_equal 3, source.explainer_items.count + assert_equal 0, target.explainer_items.count + sv_count = Version.from_partition(t.id).where(event_type: 'create_explaineritem', associated_id: source.id).count + tv_count = Version.from_partition(t.id).where(event_type: 'create_explaineritem', associated_id: target.id).count + assert_equal 4, sv_count + assert_equal 0, tv_count + # Pin target item + r.source_id = target.id + r.target_id = source.id + r.save! + assert_equal 0, source.explainer_items.count + assert_equal 3, target.explainer_items.count + sv_count = Version.from_partition(t.id).where(event_type: 'create_explaineritem', associated_id: source.id).count + tv_count = Version.from_partition(t.id).where(event_type: 'create_explaineritem', associated_id: target.id).count + assert_equal 0, sv_count + assert_equal 4, tv_count + # should not move for similar item + pm2_s = create_project_media team: t + pm2 = create_project_media team: t + e2 = create_explainer team: t + pm2.explainers << e + r2 = create_relationship source_id: pm2_s.id, target_id: pm2.id, relationship_type: Relationship.suggested_type + assert_equal 1, pm2.explainer_items.count + end + test "should not attempt to update source count if source does not exist" do r = create_relationship relationship_type: Relationship.confirmed_type r.source.delete @@ -228,15 +304,17 @@ def setup t = create_team pm1 = create_project_media team: t pm2 = create_project_media team: t - assert_nothing_raised do + assert_difference 'Relationship.count' do create_relationship source_id: pm1.id, target_id: pm2.id end - assert_raises 'ActiveRecord::RecordNotUnique' do + assert_no_difference 'Relationship.count' do create_relationship source_id: pm2.id, target_id: pm1.id end pm3 = create_project_media - assert_raises 'ActiveRecord::RecordInvalid' do - create_relationship source_id: pm3.id, target_id: pm3.id + assert_no_difference 'Relationship.count' do + assert_raises ActiveRecord::RecordInvalid do + create_relationship source_id: pm3.id, target_id: pm3.id + end end end @@ -331,21 +409,23 @@ def setup end test "should move similar items when export item" do + Sidekiq::Testing.fake! t = create_team - pm = create_project_media team: t + pm1 = create_project_media team: t pm2 = create_project_media team: t - pm_1 = create_project_media team: t - pm_2 = create_project_media team: t - pm2_1 = create_project_media team: t - # Add pm_1 & pm_2 as a similar item to pm - create_relationship source_id: pm.id, target_id: pm_1.id, relationship_type: Relationship.suggested_type - create_relationship source_id: pm.id, target_id: pm_2.id, relationship_type: Relationship.suggested_type - # Add pm_1 & pm2_1 as a similar item to pm2 - create_relationship source_id: pm2.id, target_id: pm_1.id, relationship_type: Relationship.suggested_type - create_relationship source_id: pm.id, target_id: pm2_1.id, relationship_type: Relationship.suggested_type - # Expose pm to pm2 - create_relationship source_id: pm.id, target_id: pm2.id, relationship_type: Relationship.confirmed_type - assert_equal 3, Relationship.where(source_id: pm.id, relationship_type: Relationship.suggested_type).count + pm1a = create_project_media team: t + pm1b = create_project_media team: t + pm2a = create_project_media team: t + pm2b = create_project_media team: t + # Add pm1a & pm1b as a similar items to pm1 + create_relationship source_id: pm1.id, target_id: pm1a.id, relationship_type: Relationship.suggested_type + create_relationship source_id: pm1.id, target_id: pm1b.id, relationship_type: Relationship.suggested_type + # Add pm2a & pm2b as a similar items to pm2 + create_relationship source_id: pm2.id, target_id: pm2a.id, relationship_type: Relationship.suggested_type + create_relationship source_id: pm2.id, target_id: pm2b.id, relationship_type: Relationship.suggested_type + # Export pm1 to pm2 + create_relationship source_id: pm1.id, target_id: pm2.id, relationship_type: Relationship.confirmed_type + assert_equal 4, Relationship.where(source_id: pm1.id, relationship_type: Relationship.suggested_type).count assert_equal 0, Relationship.where(source_id: pm2.id, relationship_type: Relationship.suggested_type).count end end diff --git a/test/models/request_test.rb b/test/models/request_test.rb index 354bd184ec..565e8fab40 100644 --- a/test/models/request_test.rb +++ b/test/models/request_test.rb @@ -116,7 +116,7 @@ def setup end test "should send text request to Alegre" do - Bot::Alegre.stubs(:request).returns(true) + Bot::Alegre.stubs(:request).returns({'result' => []}) assert_nothing_raised do create_request(media: create_claim_media) end @@ -124,7 +124,6 @@ def setup end test "should send media request to Alegre" do - Bot::Alegre.stubs(:request).returns(true) assert_nothing_raised do create_request(media: create_uploaded_image) end @@ -132,14 +131,14 @@ def setup end test "should attach to similar text long" do - Bot::Alegre.stubs(:request).returns(true) f = create_feed + Bot::Alegre.stubs(:request).with('post', '/similarity/async/text', anything).returns({}) m1 = Media.create! type: 'Claim', quote: 'Foo bar foo bar' r1 = create_request media: m1, feed: f m2 = Media.create! type: 'Claim', quote: 'Foo bar foo bar 2' r2 = create_request media: m2, feed: f - response = { 'result' => [{ '_source' => { 'context' => { 'request_id' => r1.id } } }] } - Bot::Alegre.stubs(:request).with('post', '/text/similarity/search/', { text: 'Foo bar foo bar 2', models: [::Bot::Alegre::ELASTICSEARCH_MODEL, ::Bot::Alegre::MEAN_TOKENS_MODEL], per_model_threshold: {::Bot::Alegre::ELASTICSEARCH_MODEL => 0.85, ::Bot::Alegre::MEAN_TOKENS_MODEL => 0.9}, limit: 20, context: { feed_id: f.id } }).returns(response) + response = { 'result' => [{ 'context' => { 'request_id' => r1.id } }] } + Bot::Alegre.stubs(:request).with('post', '/similarity/sync/text', { text: 'Foo bar foo bar 2', models: [::Bot::Alegre::ELASTICSEARCH_MODEL, ::Bot::Alegre::MEAN_TOKENS_MODEL], per_model_threshold: {::Bot::Alegre::ELASTICSEARCH_MODEL => 0.85, ::Bot::Alegre::MEAN_TOKENS_MODEL => 0.9}, limit: 20, context: { feed_id: f.id } }).returns(response) r2.attach_to_similar_request! #Alegre should be called with ES and vector model for request with 4 or more words assert_equal r1, r2.reload.similar_to_request @@ -148,14 +147,14 @@ def setup end test "should attach to similar text short" do - Bot::Alegre.stubs(:request).returns(true) f = create_feed + Bot::Alegre.stubs(:request).with('post', '/similarity/async/text', anything).returns({}) m1 = Media.create! type: 'Claim', quote: 'Foo bar foo bar' r1 = create_request media: m1, feed: f m2 = Media.create! type: 'Claim', quote: 'Foo bar 2' r2 = create_request media: m2, feed: f - response = { 'result' => [{ '_source' => { 'context' => { 'request_id' => r1.id } } }] } - Bot::Alegre.stubs(:request).with('post', '/text/similarity/search/', { text: 'Foo bar 2', models: [::Bot::Alegre::MEAN_TOKENS_MODEL], per_model_threshold: {::Bot::Alegre::MEAN_TOKENS_MODEL => 0.9}, limit: 20, context: { feed_id: f.id } }).returns(response) + response = { 'result' => [{ 'context' => { 'request_id' => r1.id } }] } + Bot::Alegre.stubs(:request).with('post', '/similarity/sync/text', { text: 'Foo bar 2', models: [::Bot::Alegre::MEAN_TOKENS_MODEL], per_model_threshold: {::Bot::Alegre::MEAN_TOKENS_MODEL => 0.9}, limit: 20, context: { feed_id: f.id } }).returns(response) r2.attach_to_similar_request! #Alegre should only be called with vector models for 2 or 3 word request assert_equal r1, r2.reload.similar_to_request @@ -164,7 +163,7 @@ def setup end test "should not attach to similar text short" do - Bot::Alegre.stubs(:request).returns(true) + Bot::Alegre.stubs(:request).returns({'result' => []}) f = create_feed m1 = Media.create! type: 'Claim', quote: 'Foo bar foo bar' r1 = create_request media: m1, feed: f @@ -193,7 +192,7 @@ def setup # end test "should attach to similar link" do - Bot::Alegre.stubs(:request).returns(true) + Bot::Alegre.stubs(:request).returns({'result' => []}) f = create_feed m = create_valid_media create_request request_type: 'text', media: m diff --git a/test/models/team_test.rb b/test/models/team_test.rb index 12023ccad7..6635867cd8 100644 --- a/test/models/team_test.rb +++ b/test/models/team_test.rb @@ -1130,9 +1130,41 @@ def setup assert_equal 3, t.reload.medias_count end + test "should return an existing empty data report structure if no statistics exist" do + t = create_team + + assert_equal( + [{"ID" => "-", + "Platform" => "-", + "Language" => t.default_language, + "Month" => "-", + "WhatsApp conversations" => "-", + "Business Conversations" => "-", + "Service Conversations" => "-", + "Unique users" => "-", + "Returning users" => "-", + "Published reports" => "-", + "Positive searches" => "-", + "Negative searches" => "-", + "Positive feedback" => "-", + "Negative feedback" => "-", + "Reports sent to users" => "-", + "Unique users who received a report" => "-", + "Average (median) response time" => "-", + "Current subscribers" => "-", + "Unique newsletters sent" => "-", + "Total newsletters sent" => "-", + "Total newsletters delivered" => "-", + "Newsletter subscriptions" => "-", + "Newsletter cancellations" => "-", + "Org" => t.name}], + t.data_report + ) + end + test "should default to Rails cache for data report if monthly team statistics not present" do t = create_team - assert_nil t.data_report + assert_equal t.data_report.dig(0, "Month"), '-' Rails.cache.write("data:report:#{t.id}", [{ 'Month' => 'Jan 2022', 'Search' => 1, 'Foo' => 2 }]) assert_equal([{ 'Month' => '1. Jan 2022', 'Search' => 1, 'Foo' => 2 }], t.data_report) @@ -1140,7 +1172,7 @@ def setup test "should return data report with chronologically ordered items, preferring the MonthlyTeamStatistics when present" do t = create_team(name: 'Test team') - assert_nil t.data_report + assert_equal t.data_report.dig(0, "Month"), '-' Rails.cache.write("data:report:#{t.id}", [{ 'Month' => 'Jan 2022', 'Unique users' => 200 }]) diff --git a/test/workers/generic_worker_test.rb b/test/workers/generic_worker_test.rb index baf671cecd..6958809b06 100644 --- a/test/workers/generic_worker_test.rb +++ b/test/workers/generic_worker_test.rb @@ -3,6 +3,7 @@ class GenericWorkerTest < ActiveSupport::TestCase def setup require 'sidekiq/testing' + WebMock.disable_net_connect! allow: /http:\/\/bot|#{CheckConfig.get('elasticsearch_host')}|#{CheckConfig.get('storage_endpoint')}/ Sidekiq::Worker.clear_all end @@ -35,9 +36,15 @@ def teardown project_media_id = pm.id tags_json = ['one', 'two'].to_json - assert_difference "Tag.where(annotation_type: 'tag').count", difference = 2 do + assert_difference "Tag.where(annotation_type: 'tag').count", 2 do GenericWorker.perform_async('Tag', 'create_project_media_tags', project_media_id, tags_json, user_id: pm.user_id) end + tags_json = [''].to_json + assert_nothing_raised do + assert_no_difference "Tag.where(annotation_type: 'tag').count" do + GenericWorker.perform_async('Tag', 'create_project_media_tags', project_media_id, tags_json, user_id: pm.user_id) + end + end end test "should schedule a job, without raising an error, for a method that takes no parameters" do diff --git a/test/workers/reindex_alegre_workspace_test.rb b/test/workers/reindex_alegre_workspace_test.rb index afa42eb9f2..a294946fb1 100644 --- a/test/workers/reindex_alegre_workspace_test.rb +++ b/test/workers/reindex_alegre_workspace_test.rb @@ -25,7 +25,7 @@ def setup @tbi.save Bot::Alegre.stubs(:get_alegre_tbi).returns(TeamBotInstallation.new) Sidekiq::Testing.inline! - Bot::Alegre.stubs(:request).with('post', '/text/bulk_similarity/', anything).returns("done") + Bot::Alegre.stubs(:request).with('post', '/similarity/async/text', anything).returns("done") end def teardown @@ -58,16 +58,20 @@ def teardown test "checks alegre package in get_request_doc" do package = { - :doc_id=>Bot::Alegre.item_doc_id(@pm, "title"), - :text=>"Some text", - :context=>{ - :team_id=>@pm.team_id, - :project_media_id=>@pm.id, - :has_custom_id=>true, - :temporary_media=>false, - :field=>"title" + :doc=>{ + :doc_id=>Bot::Alegre.item_doc_id(@pm, "title"), + :text=>"Some text", + :context=>{ + :team_id=>@pm.team_id, + :project_media_id=>@pm.id, + :has_custom_id=>true, + :temporary_media=>false, + :field=>"title" + }, + :models=>["elasticsearch"], + :requires_callback=>true }, - :models=>["elasticsearch"] + :type=>"text" } response = ReindexAlegreWorkspace.new.get_request_doc(@pm, "title", "Some text") assert_equal package, response @@ -93,18 +97,21 @@ def teardown test "tests the parallel request" do package = { - :doc_id=>Bot::Alegre.item_doc_id(@pm, "title"), - :text=>"Some text", - :context=>{ - :team_id=>@pm.team_id, - :project_media_id=>@pm.id, - :has_custom_id=>true, - :temporary_media=>false, - :field=>"title" + :doc=>{ + :doc_id=>Bot::Alegre.item_doc_id(@pm, "title"), + :text=>"Some text", + :context=>{ + :team_id=>@pm.team_id, + :project_media_id=>@pm.id, + :has_custom_id=>true, + :temporary_media=>false, + :field=>"title" + }, + :models=>["elasticsearch"] }, - :models=>["elasticsearch"] + :type=>"text" } - response = ReindexAlegreWorkspace.new.check_for_write(1.upto(30).collect{|x| package}, "a", @team.id, true, 1) + response = ReindexAlegreWorkspace.new.check_for_write(1.upto(30).collect{|x| package}, "a", @team.id, true) assert_equal Array, response.class end