From 4b11024302ae799f9f8455264d97fa73d4245de8 Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Wed, 13 Sep 2023 16:19:08 -0700 Subject: [PATCH 1/3] CV2-3589 add language as a parameter for import_claim_reviews --- app/models/bot/fetch.rb | 6 +++--- lib/tasks/data/fetch.rake | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/app/models/bot/fetch.rb b/app/models/bot/fetch.rb index 2375a88a0e..aaf8142a6d 100644 --- a/app/models/bot/fetch.rb +++ b/app/models/bot/fetch.rb @@ -94,7 +94,7 @@ def self.setup_service(installation, previous_services, new_services, language=n end end previous_services.each { |previous_service| self.call_fetch_api(:delete, 'subscribe', { service: previous_service, url: self.webhook_url(team) }) } - Bot::Fetch::Import.delay_for(1.second, { queue: 'fetch', retry: 0 }).import_claim_reviews(installation.id) unless new_services.blank? + Bot::Fetch::Import.delay_for(1.second, { queue: 'fetch', retry: 0 }).import_claim_reviews(installation.id, language, false, nil) unless new_services.blank? end def self.is_service_supported?(service) @@ -148,7 +148,7 @@ def self.call_fetch_api(verb, endpoint, params = {}) # Mandatory fields in the imported ClaimReview: claim_review_headline, claim_review_url, created_at and id class Import - def self.import_claim_reviews(installation_id, force = false, maximum = nil) + def self.import_claim_reviews(installation_id, language = nil, force = false, maximum = nil) installation = TeamBotInstallation.find(installation_id) RequestStore.store[:skip_notifications] = true RequestStore.store[:skip_cached_field_update] = false @@ -171,7 +171,7 @@ def self.import_claim_reviews(installation_id, force = false, maximum = nil) (from.to_i..to.to_i).step(step.days).each do |current_timestamp| from2 = Time.at(current_timestamp) to2 = from2 + step.days - Bot::Fetch.get_claim_reviews({ service: service_name, start_time: from2.strftime('%Y-%m-%d'), end_time: to2.strftime('%Y-%m-%d')}).each do |claim_review| + Bot::Fetch.get_claim_reviews({ service: service_name, start_time: from2.strftime('%Y-%m-%d'), end_time: to2.strftime('%Y-%m-%d'), language: language}).each do |claim_review| next if !maximum.nil? && total >= maximum self.import_claim_review(claim_review, team.id, user.id, status_fallback, status_mapping, auto_publish_reports, force) total += 1 diff --git a/lib/tasks/data/fetch.rake b/lib/tasks/data/fetch.rake index f3264042d3..bf73d88883 100644 --- a/lib/tasks/data/fetch.rake +++ b/lib/tasks/data/fetch.rake @@ -137,7 +137,8 @@ namespace :check do services.each { |service| Bot::Fetch.call_fetch_api(:delete, 'subscribe', { service: service, url: Bot::Fetch.webhook_url(team) }) } services.each { |service| Bot::Fetch.call_fetch_api(:post, 'subscribe', { service: service, url: Bot::Fetch.webhook_url(team) }) } tbi = TeamBotInstallation.where(user_id: BotUser.find_by_login('fetch').id, team_id: team.id).last.id - Bot::Fetch::Import.delay(retry: 0).import_claim_reviews(tbi, force) + #dont support languages here, its just going to get too messy via the rake syntax. + Bot::Fetch::Import.delay(retry: 0).import_claim_reviews(tbi, nil, force) end end @@ -172,7 +173,7 @@ namespace :check do end Bot::Fetch.set_service(team.slug, services, 'undetermined', status_mapping) tbi = TeamBotInstallation.where(user_id: BotUser.find_by_login('fetch').id, team_id: team.id).last.id - Bot::Fetch::Import.import_claim_reviews(tbi, true, size) + Bot::Fetch::Import.import_claim_reviews(tbi, nil, true, size) puts "A sample of #{size} articles were imported from Fetch service(s) #{services.join(', ')} to https://checkmedia.org/#{team.slug}/imported-fact-checks." end end From 142869bf98d9229662e37b50d969653d3996d88f Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Wed, 13 Sep 2023 18:11:45 -0700 Subject: [PATCH 2/3] CV2-3589 refactor to not uselessly explode all the tests and rewrite them with a meaningless null value --- app/models/bot/fetch.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/models/bot/fetch.rb b/app/models/bot/fetch.rb index aaf8142a6d..60628b77fd 100644 --- a/app/models/bot/fetch.rb +++ b/app/models/bot/fetch.rb @@ -171,7 +171,9 @@ def self.import_claim_reviews(installation_id, language = nil, force = false, ma (from.to_i..to.to_i).step(step.days).each do |current_timestamp| from2 = Time.at(current_timestamp) to2 = from2 + step.days - Bot::Fetch.get_claim_reviews({ service: service_name, start_time: from2.strftime('%Y-%m-%d'), end_time: to2.strftime('%Y-%m-%d'), language: language}).each do |claim_review| + params = { service: service_name, start_time: from2.strftime('%Y-%m-%d'), end_time: to2.strftime('%Y-%m-%d'), language: language} + params[:language] = language if !language.nil? + Bot::Fetch.get_claim_reviews(params).each do |claim_review| next if !maximum.nil? && total >= maximum self.import_claim_review(claim_review, team.id, user.id, status_fallback, status_mapping, auto_publish_reports, force) total += 1 From 7a77932fd45b4e4cf170d2f6aa9dbde3e226c71f Mon Sep 17 00:00:00 2001 From: Devin Gaffney Date: Thu, 14 Sep 2023 05:40:35 -0700 Subject: [PATCH 3/3] fix small oversight --- app/models/bot/fetch.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/bot/fetch.rb b/app/models/bot/fetch.rb index 60628b77fd..7153a3a999 100644 --- a/app/models/bot/fetch.rb +++ b/app/models/bot/fetch.rb @@ -171,7 +171,7 @@ def self.import_claim_reviews(installation_id, language = nil, force = false, ma (from.to_i..to.to_i).step(step.days).each do |current_timestamp| from2 = Time.at(current_timestamp) to2 = from2 + step.days - params = { service: service_name, start_time: from2.strftime('%Y-%m-%d'), end_time: to2.strftime('%Y-%m-%d'), language: language} + params = { service: service_name, start_time: from2.strftime('%Y-%m-%d'), end_time: to2.strftime('%Y-%m-%d')} params[:language] = language if !language.nil? Bot::Fetch.get_claim_reviews(params).each do |claim_review| next if !maximum.nil? && total >= maximum