SacImports NAV1: create mailing list subscriptions (HIT-753)

app/domain/sac_imports/csv_source/nav1.rb

@@ -38,10 +38,10 @@ class SacImports::CsvSource
     :opt_in_die_alpen_physisch, # "OPT_IN_Die_Alpen_physisch",
     :opt_in_die_alpen_digital, # "OPT_IN_Die_Alpen_digital",
     :opt_in_fundraising, # "OPT_IN_Fundraising",
-    :opt_in_sektionsbulletin_physisch_opt_in, # "OPT_IN_Sektionsbulletin_physisch",
-    :opt_in_sektionsbulletin_digital_opt_in, # "OPT_IN_Sektionsbulletin_digital",
-    :opt_out_die_alpen_physisch, # "OPT_OUT_Die_Alpen_physisch",
-    :opt_out_die_alpen_digital, # "OPT_OUT_Die_Alpen_digital",
+    :opt_in_sektionsbulletin_physisch, # "OPT_IN_Sektionsbulletin_physisch",
+    :opt_in_sektionsbulletin_digital, # "OPT_IN_Sektionsbulletin_digital",
+    :opt_out_sektionsbulletin_physisch, # "OPT_OUT_Sektionsbulletin_physisch",
+    :opt_out_sektionsbulletin_digital, # "OPT_OUT_Sektionsbulletin_digital",
     :termination_reason # "Austrittsgrund"
   )
 end

app/domain/sac_imports/log_counts.rb

@@ -31,7 +31,7 @@ def calculate_deltas(scopes, labeled_scopes, &block)
         prepare_definitions(labeled_scopes, tally_by_type: false)
 
       model_counts_before = definitions.each_with_object({}) do |d, counts|
-        counts[d] = {total: d.scope.count}
+        counts[d] = {total: d.scope.respond_to?(:call) ? d.scope.call : d.scope.count}
         # also calculate counts by type if enabled and the model is an STI baseclass
         if d.tally_by_type && d.model.attribute_names.include?(d.model.inheritance_column) &&
             d.model.table_name == d.model.send(:undecorated_table_name, d.model.name)
@@ -45,7 +45,7 @@ def calculate_deltas(scopes, labeled_scopes, &block)
 
       definitions.each do |d|
         total_before = model_counts_before[d][:total]
-        total_after = d.scope.count
+        total_after = d.scope.respond_to?(:call) ? d.scope.call : d.scope.count
         delta = total_after - total_before
         data << [d.label, total_before, total_after, delta]
 

app/domain/sac_imports/nav1_subscriptions_importer.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+#  Copyright (c) 2024, Schweizer Alpen-Club. This file is part of
+#  hitobito_sac_cas and licensed under the Affero General Public License version 3
+#  or later. See the COPYING file at the top-level directory or at
+#  https://github.com/hitobito/hitobito_sac_cas
+
+module SacImports
+  class Nav1SubscriptionsImporter
+    include LogCounts
+
+    REPORT_HEADERS = [
+      :navision_membership_number,
+      :navision_name,
+      :status,
+      :warnings,
+      :errors
+    ]
+
+    attr_reader :list_die_alpen_digital, :list_die_alpen_paper, :list_fundraising
+
+    def initialize(output: $stdout)
+      @output = output
+      @source_file = CsvSource.new(:NAV1)
+      @csv_report = CsvReport.new(:"nav1-3_subscriptions", REPORT_HEADERS, output:)
+      @list_die_alpen_paper = MailingList
+        .find_by!(internal_key: SacCas::MAILING_LIST_DIE_ALPEN_PAPER_INTERNAL_KEY)
+      @list_die_alpen_digital = MailingList
+        .find_by!(internal_key: SacCas::MAILING_LIST_DIE_ALPEN_DIGITAL_INTERNAL_KEY)
+      @list_fundraising = MailingList
+        .find_by!(internal_key: SacCas::MAILING_LIST_SPENDENAUFRUFE_INTERNAL_KEY)
+
+      # warm up memoized data otherwise each thread will create a new one
+      bulletin_list_paper_by_group_id
+      bulletin_list_digital_by_group_id
+      people_by_id
+    end
+
+    def create
+      data = @source_file.rows
+
+      @csv_report.log("The file contains #{data.size} rows.")
+
+      # log_counts_delta(@csv_report,
+      #   "Die Alpen Paper receiver" => -> { MailingLists::Subscribers.new(list_die_alpen_paper).people.size },
+      #   "Die Alpen Digital receiver" => -> { MailingLists::Subscribers.new(list_die_alpen_digital).people.size },
+      #   "Fundraising receiver count" => -> { MailingLists::Subscribers.new(list_fundraising).people.size },
+      #   "Bulletin Paper receiver count" => -> { receiver_numbers_for_lists(bulletin_list_paper_by_group_id.values) },
+      #   "Bulletin Digital receiver count" => -> { receiver_numbers_for_lists(bulletin_list_digital_by_group_id.values) }) do
+      log_counts_delta(@csv_report,
+        Subscription,
+        "Die Alpen Paper subscription" => Subscription.where(subscriber_type: "Person", mailing_list_id: list_die_alpen_paper.id, excluded: false),
+        "Die Alpen Paper exclusion" => Subscription.where(subscriber_type: "Person", mailing_list_id: list_die_alpen_paper.id, excluded: true),
+        "Die Alpen Digital subscription" => Subscription.where(subscriber_type: "Person", mailing_list_id: list_die_alpen_digital.id, excluded: false),
+        "Fundraising subscription" => Subscription.where(subscriber_type: "Person", mailing_list_id: list_fundraising.id, excluded: false),
+        "Bulletin Paper subscriptions" => Subscription.joins(:mailing_list).where(subscriber_type: "Person", mailing_list: {internal_key: SacCas::MAILING_LIST_SEKTIONSBULLETIN_PAPER_INTERNAL_KEY}, excluded: false),
+        "Bulletin Paper exclusions" => Subscription.joins(:mailing_list).where(subscriber_type: "Person", mailing_list: {internal_key: SacCas::MAILING_LIST_SEKTIONSBULLETIN_PAPER_INTERNAL_KEY}, excluded: true),
+        "Bulletin Digital subscriptions" => Subscription.joins(:mailing_list).where(subscriber_type: "Person", mailing_list: {internal_key: SacCas::MAILING_LIST_SEKTIONSBULLETIN_DIGITAL_INTERNAL_KEY}, excluded: false)) do
+        progress = Progress.new(data.size, title: "NAV1 Subscriptions Import", output: @output)
+
+        data.each do |row|
+          # Parallel.map(data, in_threads: Etc.nprocessors) do |row|
+          progress.step
+          process_row(row)
+        end
+      end
+      # end
+
+      @csv_report.finalize
+    end
+
+    private
+
+    def process_row(row)
+      People::SubscriptionEntry.new(row, @csv_report, people_by_id, list_die_alpen_paper,
+        list_die_alpen_digital, list_fundraising, bulletin_list_paper_by_group_id,
+        bulletin_list_digital_by_group_id).create
+    end
+
+    def people_by_id
+      @people ||= Person.select(:id).index_by(&:id)
+    end
+
+    def bulletin_list_paper_by_group_id
+      @bulletin_list_paper_by_group_id ||= MailingList
+        .where(internal_key: SacCas::MAILING_LIST_SEKTIONSBULLETIN_PAPER_INTERNAL_KEY)
+        .index_by(&:group_id)
+    end
+
+    def bulletin_list_digital_by_group_id
+      @bulletin_list_digital_by_group_id ||= MailingList
+        .where(internal_key: SacCas::MAILING_LIST_SEKTIONSBULLETIN_DIGITAL_INTERNAL_KEY)
+        .index_by(&:group_id)
+    end
+
+    def receiver_numbers_for_lists(lists)
+      lists.sum do |list|
+        MailingLists::Subscribers.new(list).people.size
+      end
+    end
+  end
+end

app/domain/sac_imports/people/subscription_entry.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+#  Copyright (c) 2024, Schweizer Alpen-Club. This file is part of
+#  hitobito_sac_cas and licensed under the Affero General Public License version 3
+#  or later. See the COPYING file at the top-level directory or at
+#  https://github.com/hitobito/hitobito_sac_cas
+
+module SacImports::People
+  class SubscriptionEntry
+    attr_reader :row, :csv_report, :people_by_id, :list_die_alpen_paper, :list_die_alpen_digital,
+      :list_fundraising, :bulletin_list_paper_by_group_id, :bulletin_list_digital_by_group_id
+
+    def initialize(row, csv_report, people_by_id, list_die_alpen_paper, list_die_alpen_digital,
+      list_fundraising, bulletin_list_paper_by_group_id, bulletin_list_digital_by_group_id)
+      @row = row
+      @csv_report = csv_report
+      @people_by_id = people_by_id
+      @list_die_alpen_paper = list_die_alpen_paper
+      @list_die_alpen_digital = list_die_alpen_digital
+      @list_fundraising = list_fundraising
+      @bulletin_list_paper_by_group_id = bulletin_list_paper_by_group_id
+      @bulletin_list_digital_by_group_id = bulletin_list_digital_by_group_id
+    end
+
+    def create
+      return report(row, error: "Person not found") unless person
+
+      subscribe_fundraising
+      subscribe_die_alpen
+      subscribe_bulletin
+    end
+
+    def person_id = parse_id(row.navision_id)
+
+    def bulletin_digital_opt_in_group_ids = parse_ids(row.opt_in_sektionsbulletin_digital)
+
+    def bulletin_digital_opt_out_group_ids = parse_ids(row.opt_out_sektionsbulletin_digital)
+
+    def bulletin_paper_opt_in_group_ids = parse_ids(row.opt_in_sektionsbulletin_physisch)
+
+    def bulletin_paper_opt_out_group_ids = parse_ids(row.opt_out_sektionsbulletin_physisch)
+
+    private
+
+    def subscribe?(field)
+      value = row.send(field)
+      ["0", "1"].include?(value) || raise("Invalid value #{value.inspect} for #{field}")
+      value == "1"
+    end
+
+    def already_subscribed?(list)
+      MailingLists::Subscribers.new(list_die_alpen_paper, Person.where(id: person.id))
+        .people.include?(person)
+    end
+
+    def subscribe(list)
+      sub = Subscription.where(subscriber: person, mailing_list: list).first_or_initialize
+      sub.update!(excluded: false)
+    end
+
+    def unsubscribe(list)
+      sub = Subscription.where(subscriber: person, mailing_list: list).first_or_initialize
+      sub.update!(excluded: true)
+    end
+
+    def subscribe_fundraising
+      # fundraisig is opt-in, so simply subscribe if the field is set
+      subscribe(list_fundraising) if subscribe?(:opt_in_fundraising)
+    end
+
+    def subscribe_die_alpen
+      # die alpen digital is opt-in, so simply subscribe if the field is set
+      subscribe(list_die_alpen_digital) if subscribe?(:opt_in_die_alpen_digital)
+
+      # die alpen paper is opt-out, so we subscribe or unsubscribe depending on the field
+      if subscribe?(:opt_in_die_alpen_physisch)
+        subscribe(list_die_alpen_paper)
+      else
+        unsubscribe(list_die_alpen_paper)
+      end
+    end
+
+    def process_bulletin_subscriptions(action, label, group_ids, lists_by_group_id)
+      group_ids.each do |group_id|
+        list = lists_by_group_id[group_id]
+        next report(row, error: "List #{label} not found for group #{group_id}") unless list
+
+        if block_given?
+          send(action, list) if yield(list)
+        else
+          send(action, list)
+        end
+      end
+    end
+
+    def subscribe_bulletin
+      # bulletin paper is opt-out.
+      # Subscribe all sections in bulletin_paper_opt_in_group_ids,
+      # and unsubscribe all sections in bulletin_paper_opt_out_group_ids.
+      process_bulletin_subscriptions(
+        :subscribe, "bulletin paper",
+        bulletin_paper_opt_in_group_ids, bulletin_list_paper_by_group_id
+      ) # { |list| !already_subscribed?(list) }
+      process_bulletin_subscriptions(
+        :unsubscribe, "bulletin paper",
+        bulletin_paper_opt_out_group_ids, bulletin_list_paper_by_group_id
+      ) # { |list| already_subscribed?(list) }
+
+      # bulletin digital is opt-in. Subscribe all sections in bulletin_digital_opt_in_group_ids.
+      # Unsubscribe is not necessary because it is opt-in.
+      process_bulletin_subscriptions(
+        :subscribe, "bulletin digital",
+        bulletin_digital_opt_in_group_ids, bulletin_list_digital_by_group_id
+      )
+    end
+
+    def person = people_by_id[person_id]
+
+    def subscriptions = Person::Subscriptions.new(person)
+
+    def parse_ids(string)
+      string
+        .presence
+        &.split(";")
+        &.map(&method(:parse_id)) || []
+    end
+
+    def parse_id(string)
+      return nil if string.blank?
+      Integer(string.gsub(/^0+/, ""))
+    end
+
+    def report(row, warning: nil, error: nil)
+      csv_report.add_row({
+        navision_membership_number: row.navision_id,
+        navision_name: "#{row.first_name} #{row.last_name}",
+        errors: error,
+        warnings: warning,
+        status: if error.present?
+                  "error"
+                else
+                  warning.present? ? "warning" : "success"
+                end
+      })
+    end
+  end
+end

app/domain/sac_imports/sac_sections/group_entry.rb

@@ -165,15 +165,15 @@ def setup_mailing_lists
       setup_bulletin_paper if row.has_bulletin_paper == "1"
     end
 
-    def setup_mailing_list(internal_key, name, subscribable_for, filter_chain = {})
+    def setup_mailing_list(internal_key, name, subscribable_for, mode, filter_chain = {})
       return if group.mailing_lists.find_by(internal_key:)
 
       group.mailing_lists.create!(
         internal_key:,
         name:,
         filter_chain:,
         subscribable_for:,
-        subscribable_mode: "opt_in",
+        subscribable_mode: mode,
         subscriptions: [
           Subscription.new(
             subscriber: group,
@@ -189,13 +189,13 @@ def setup_mailing_list(internal_key, name, subscribable_for, filter_chain = {})
     def setup_bulletin_digital
       internal_key = SacCas::MAILING_LIST_SEKTIONSBULLETIN_DIGITAL_INTERNAL_KEY
       name = "Sektionsbulletin digital"
-      setup_mailing_list(internal_key, name, "anyone")
+      setup_mailing_list(internal_key, name, "anyone", "opt_in")
     end
 
     def setup_bulletin_paper
       internal_key = SacCas::MAILING_LIST_SEKTIONSBULLETIN_PAPER_INTERNAL_KEY
       name = "Sektionsbulletin physisch"
-      setup_mailing_list(internal_key, name, "configured", filter_chain)
+      setup_mailing_list(internal_key, name, "configured", "opt_out", filter_chain)
     end
 
     def filter_chain

lib/tasks/sac_imports.rake

@@ -67,6 +67,7 @@ namespace :sac_imports do
     "nav2a-3_families",
     "wso21-1_people",
     "nav2b-2_non_membership_roles",
+    "nav1-3_subscriptions",
     :update_sac_family_address,
     "nav17-1_event_kinds",
     :cleanup,
@@ -93,6 +94,12 @@ namespace :sac_imports do
     SacImports::MembershipYearsReport.new.create
   end
 
+  desc "NAV1 Imports subscriptions from Navision"
+  task "nav1-3_subscriptions": :setup do
+    SacImports::Nav1SubscriptionsImporter.new.create
+    Rake::Task["sac_imports:dump_database"].execute(dump_name: "nav1-3_subscriptions")
+  end
+
   desc "Import people from WSO2"
   task "wso21-1_people": :setup do
     SacImports::Wso2PeopleImporter.new.create

spec/domain/sac_imports/sac_sections/group_entry_spec.rb

@@ -56,7 +56,7 @@
       expect(list.name).to eq("Sektionsbulletin physisch")
       expect(list.internal_key).to eq SacCas::MAILING_LIST_SEKTIONSBULLETIN_PAPER_INTERNAL_KEY
       expect(list.subscribable_for).to eq("configured")
-      expect(list.subscribable_mode).to eq("opt_in")
+      expect(list.subscribable_mode).to eq("opt_out")
       expect(list.filter_chain.to_hash).to eq(
         "invoice_receiver" =>
            {"stammsektion" => "true", "zusatzsektion" => "true", "group_id" => "42"}