Fixes and changes for the data dashboard backend (#2121)

Some fixes and changes to the data dashboard backend:

- Fixing the date range: the "start" is set for the beginning of the day while the "end" is set for the end of the day.
- Fixing sorting for the "top" fields, in order to be sure that the ones with highest values come first.
- Changing the format for the "top" fields: instead of hash key/value, it's now an array of objects with fields id, label and value.

References: CV2-4111.

lib/check_data_points.rb

@@ -127,8 +127,8 @@ def new_users(team_id, start_date, end_date)
 
     def parse_start_end_dates(start_date, end_date)
       # date format is `2023-08-23`
-      start_date = Time.parse(start_date)
-      end_date = Time.parse(end_date)
+      start_date = Time.parse(start_date).beginning_of_day
+      end_date = Time.parse(end_date).end_of_day
       raise 'End date should be greater than start date' if start_date > end_date
       return start_date, end_date
     end

lib/team_statistics.rb

@@ -15,7 +15,7 @@ def initialize(team, period, language, platform = nil)
     end
 
     range = time_range.to_a
-    @start_date, @end_date = range.first, range.last
+    @start_date, @end_date = range.first.to_datetime.beginning_of_day, range.last.to_datetime.end_of_day
     @start_date_str, @end_date_str = @start_date.strftime('%Y-%m-%d'), @end_date.strftime('%Y-%m-%d')
 
     @platform = platform
@@ -67,13 +67,14 @@ def number_of_fact_checks_by_rating
 
   # FIXME: Only fact-checks for now (need to add explainers) and the "demand" is across languages and platforms
   def top_articles_sent
-    data = {}
+    data = []
     clusters = CheckDataPoints.top_clusters(@team.id, @start_date, @end_date, 5, 'last_seen', @language || @all_languages, 'fc_language')
     clusters.each do |pm_id, demand|
       item = ProjectMedia.find(pm_id)
-      data[item.fact_check_title || item.title] = demand
+      title = item.fact_check_title || item.title
+      data << { id: item.id, label: title, value: demand }
     end
-    data
+    data.sort_by{ |object| object[:value] }.reverse
   end
 
   def top_articles_tags
@@ -94,11 +95,11 @@ def top_articles_tags
 
     language = @language ? [@language] : @all_languages
     result = ActiveRecord::Base.connection.execute(ApplicationRecord.sanitize_sql_for_assignment([sql, team_id: @team.id, start_date: @start_date, end_date: @end_date, language: language]))
-    data = {}
+    data = []
     result.each do |row|
-      data[row['tag']] = row['tag_count'].to_i
+      data << { id: row['tag'], label: row['tag'], value: row['tag_count'].to_i }
     end
-    data.sort.reverse.to_h
+    data.sort_by{ |object| object[:value] }.reverse
   end
 
   # For tiplines
@@ -185,27 +186,29 @@ def number_of_media_received_by_media_type
 
   # FIXME: The "demand" is across languages and platforms
   def top_requested_media_clusters
-    data = {}
+    data = []
     clusters = CheckDataPoints.top_clusters(@team.id, @start_date, @end_date, 5, 'last_seen', @language || @all_languages, 'request_language', @platform)
     clusters.each do |pm_id, demand|
       item = ProjectMedia.find(pm_id)
-      data[item.title] = demand
+      data << { id: item.id, label: item.title, value: demand }
     end
-    data
+    data.sort_by{ |object| object[:value] }.reverse
   end
 
   # FIXME: The "demand" is across languages and platforms
   def top_media_tags
-    data = {}
+    tags = {}
     clusters = CheckDataPoints.top_clusters(@team.id, @start_date, @end_date, 5, 'last_seen', @language || @all_languages, 'language', @platform)
     clusters.each do |pm_id, demand|
       item = ProjectMedia.find(pm_id)
       item.tags_as_sentence.split(',').map(&:strip).each do |tag|
-        data[tag] ||= 0
-        data[tag] += demand
+        tags[tag] ||= 0
+        tags[tag] += demand
       end
     end
-    data.sort_by{ |_key, value| value }.reverse.first(5).to_h
+    data = []
+    tags.each { |tag, value| data << { id: tag, label: tag, value: value } }
+    data.sort_by{ |object| object[:value] }.reverse
   end
 
   # For both articles and tiplines
@@ -233,7 +236,7 @@ def time_range
     }[@period.to_sym]
     from = Time.now.ago(ago) unless ago.nil?
     from = Time.now.beginning_of_year if @period.to_s == 'year_to_date'
-    from.to_datetime..Time.now.to_datetime
+    from.to_datetime.beginning_of_day..Time.now.to_datetime.end_of_day
   end
 
   def fact_checks_base_query(timestamp_field = :created_at, group_by_day = false)

test/lib/team_statistics_test.rb

@@ -76,7 +76,7 @@ def teardown
       assert_equal 2, object.number_of_fact_checks_created
       assert_equal 1, object.number_of_published_fact_checks
       assert_equal({ 'false' => 1, 'verified' => 1 }, object.number_of_fact_checks_by_rating)
-      assert_equal({ 'foo' => 3, 'bar' => 2 }, object.top_articles_tags)
+      assert_equal([{ id: 'foo', label: 'foo', value: 3 }, { id: 'bar', label: 'bar', value: 2 }], object.top_articles_tags)
     end
   end
 
@@ -96,7 +96,7 @@ def teardown
     sleep 2
 
     object = TeamStatistics.new(@team, 'past_week', 'en')
-    expected = { 'Foo' => 2, 'Bar' => 1 }
+    expected = [{ id: pm2.id, label: 'Foo', value: 2 }, { id: pm1.id, label: 'Bar', value: 1 }]
     assert_equal expected, object.top_articles_sent
   end
 
@@ -168,7 +168,7 @@ def teardown
       sleep 3
 
       object = TeamStatistics.new(@team, 'past_week', 'en', 'whatsapp')
-      expected = { 'Foo' => 2, 'Bar' => 1 }
+      expected = [{ id: pm2.id, label: 'Foo', value: 2 }, { id: pm1.id, label: 'Bar', value: 1 }]
       assert_equal expected, object.top_requested_media_clusters
     end
   end
@@ -204,7 +204,7 @@ def teardown
       sleep 3
 
       object = TeamStatistics.new(@team, 'past_week', 'en', 'whatsapp')
-      expected = { 'foo' => 3, 'test' => 2, 'bar' => 1 }
+      expected = [{ id: 'foo', label: 'foo', value: 3 }, { id: 'test', label: 'test', value: 2 }, { id: 'bar', label: 'bar', value: 1 }]
       assert_equal expected, object.top_media_tags
     end
   end