Fixes and changes for the data dashboard backend (#2159)

Some fixes and changes for the data dashboard backend:

 - CV2-5812: Expand the "matched media" data point to include all tipline request types.
 - CV2-5813: Use the default language when an explainer is created without language (includes migration rake task).
 - CV2-5842: Use updated_at instead of created_at for article tags analytics.
 - CV2-5835: For messages analytics, include only the states sent and received.
 - CV2-5836: Use the top_media_tags method from CheckDataPoints lib in the TeamStatistics lib.

app/models/explainer.rb

@@ -7,10 +7,10 @@ class Explainer < ApplicationRecord
   has_many :explainer_items, dependent: :destroy
   has_many :project_medias, through: :explainer_items
 
-  before_validation :set_team
+  before_validation :set_team, :set_language
   validates_format_of :url, with: URI.regexp, allow_blank: true, allow_nil: true
   validates_presence_of :team, :title, :description
-  validate :language_in_allowed_values, unless: proc { |e| e.language.blank? }
+  validate :language_in_allowed_values
 
   after_save :update_paragraphs_in_alegre
   after_update :detach_explainer_if_trashed
@@ -137,8 +137,13 @@ def set_team
     self.team ||= Team.current
   end
 
+  def set_language
+    default_language = self.team&.get_language || 'und'
+    self.language ||= default_language
+  end
+
   def language_in_allowed_values
-    allowed_languages = self.team.get_languages || ['en']
+    allowed_languages = self.team&.get_languages || ['en']
     allowed_languages << 'und'
     errors.add(:language, I18n.t(:"errors.messages.invalid_article_language_value")) unless allowed_languages.include?(self.language)
   end

lib/check_data_points.rb

@@ -6,7 +6,7 @@ class << self
     # Number of tipline messages
     def tipline_messages(team_id, start_date, end_date, granularity = nil, platform = nil, language = nil)
       start_date, end_date = parse_start_end_dates(start_date, end_date)
-      query = TiplineMessage.where(team_id: team_id, created_at: start_date..end_date)
+      query = TiplineMessage.where(team_id: team_id, created_at: start_date..end_date, state: ['sent', 'received'])
       query_based_on_granularity(query, platform, language, granularity)
     end
 

lib/tasks/migrate/20241216215050_set_language_for_explainers.rake

@@ -0,0 +1,18 @@
+namespace :check do
+  namespace :migrate do
+    task set_language_for_explainers: :environment do
+      started = Time.now.to_i
+      query = Explainer.where(language: nil)
+      n = query.count
+      i = 0
+      query.find_each do |explainer|
+        i += 1
+        language = explainer.team&.get_language || 'und'
+        explainer.update_column(:language, language)
+        puts "[#{Time.now}] [#{i}/#{n}] Setting language for explainer ##{explainer.id} as #{language}"
+      end
+      minutes = ((Time.now.to_i - started) / 60).to_i
+      puts "[#{Time.now}] Done in #{minutes} minutes. Number of explainers without language: #{query.count}"
+    end
+  end
+end

lib/team_statistics.rb

@@ -83,10 +83,10 @@ def top_articles_tags
       FROM (
         SELECT unnest(fcs.tags) AS tag FROM fact_checks fcs
           INNER JOIN claim_descriptions cds ON fcs.claim_description_id = cds.id
-          WHERE cds.team_id = :team_id AND fcs.created_at BETWEEN :start_date AND :end_date AND fcs.language IN (:language)
+          WHERE cds.team_id = :team_id AND fcs.updated_at BETWEEN :start_date AND :end_date AND fcs.language IN (:language)
         UNION ALL
         SELECT unnest(explainers.tags) AS tag FROM explainers
-          WHERE explainers.team_id = :team_id AND explainers.created_at BETWEEN :start_date AND :end_date AND explainers.language IN (:language)
+          WHERE explainers.team_id = :team_id AND explainers.updated_at BETWEEN :start_date AND :end_date AND explainers.language IN (:language)
       ) AS all_tags
       GROUP BY tag
       ORDER BY tag_count DESC
@@ -198,7 +198,7 @@ def top_requested_media_clusters
   # FIXME: The "demand" is across languages and platforms
   def top_media_tags
     tags = {}
-    clusters = CheckDataPoints.top_clusters(@team.id, @start_date, @end_date, 5, 'last_seen', @language || @all_languages, 'language', @platform)
+    clusters = CheckDataPoints.top_media_tags(@team.id, @start_date, @end_date, 20, 'last_seen', @language || @all_languages, 'language', @platform)
     clusters.each do |pm_id, demand|
       item = ProjectMedia.find(pm_id)
       item.tags_as_sentence.split(',').map(&:strip).each do |tag|
@@ -218,7 +218,7 @@ def number_of_articles_sent
   end
 
   def number_of_matched_results_by_article_type
-    query = TiplineRequest.where(team_id: @team.id, smooch_request_type: ['relevant_search_result_requests', 'irrelevant_search_result_requests', 'timeout_search_requests'], created_at: @start_date..@end_date)
+    query = TiplineRequest.where(team_id: @team.id, created_at: @start_date..@end_date)
     query = query.where(platform: @platform) unless @platform.blank?
     query = query.where(language: @language) unless @language.blank?
     { 'FactCheck' => query.joins(project_media: { claim_description: :fact_check }).count, 'Explainer' => query.joins(project_media: :explainers).count }

test/lib/team_statistics_test.rb

@@ -2,6 +2,7 @@
 
 class TeamStatisticsTest < ActiveSupport::TestCase
   def setup
+    Explainer.delete_all
     @team = create_team
     @team.set_languages = ['en', 'pt']
     @team.save!
@@ -52,15 +53,15 @@ def teardown
       create_fact_check(tags: ['foo', 'bar'], language: 'en', rating: 'false', claim_description: create_claim_description(project_media: create_project_media(team: @team)))
       create_fact_check(tags: ['foo', 'bar'], claim_description: create_claim_description(project_media: create_project_media(team: team)))
       exp = create_explainer team: @team, language: 'en', tags: ['foo']
-      create_explainer team: @team, tags: ['foo', 'bar']
+      create_explainer team: @team, tags: ['foo', 'bar'], language: 'pt'
       create_explainer language: 'en', team: team, tags: ['foo', 'bar']
     end
 
     travel_to Time.parse('2024-01-02') do
       create_fact_check(tags: ['bar'], report_status: 'published', rating: 'verified', language: 'en', claim_description: create_claim_description(project_media: create_project_media(team: @team)))
       create_fact_check(tags: ['foo', 'bar'], claim_description: create_claim_description(project_media: create_project_media(team: team)))
       create_explainer team: @team, language: 'en', tags: ['foo']
-      create_explainer team: @team, tags: ['foo', 'bar']
+      create_explainer team: @team, tags: ['foo', 'bar'], language: 'pt'
       create_explainer language: 'en', team: team, tags: ['foo', 'bar']
       exp.updated_at = Time.now
       exp.save!

test/models/explainer_test.rb

@@ -5,6 +5,10 @@ def setup
     Explainer.delete_all
   end
 
+  def teardown
+    User.current = Team.current = nil
+  end
+
   test "should create explainer" do
     assert_difference 'Explainer.count' do
       create_explainer
@@ -159,4 +163,9 @@ def setup
     models_thresholds = Explainer.get_alegre_models_and_thresholds(ex.team_id)
     assert_kind_of Hash, models_thresholds
   end
+
+  test "should set default language when language is not set" do
+    ex = create_explainer language: nil
+    assert_equal 'en', ex.reload.language
+  end
 end