Add nav18 events importer (#1367)

Co-authored-by: z <[email protected]>

app/domain/sac_imports/csv_source.rb

@@ -16,6 +16,7 @@ class SacImports::CsvSource
     NAV3: Nav3,
     NAV6: Nav6,
     NAV17: Nav17,
+    NAV18: Nav18,
     WSO21: Wso2,
     CHIMP_1: Chimp,
     CHIMP_2: Chimp,

app/domain/sac_imports/csv_source/nav18.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+#  Copyright (c) 2024, Schweizer Alpen-Club. This file is part of
+#  hitobito_sac_cas and licensed under the Affero General Public License version 3
+#  or later. See the COPYING file at the top-level directory or at
+#  https://github.com/hitobito/hitobito_sac_cas
+
+class SacImports::CsvSource
+  # Event::Course
+  # !!! DO NOT CHANGE THE ORDER OF THE KEYS !!!
+  # they must match the order of the columns in the CSV files
+  Nav18 = Data.define(
+    :name_de, # Name_DE
+    :name_fr, # Name_FR
+    :name_it, # Name_IT
+    :kind, # Kursart
+    :number, # Kursnummer
+    :state, # Status
+    :description_de, # Beschreibung_DE
+    :description_fr, # Beschreibung_FR
+    :description_it, # Beschreibung_IT
+    :contact_id, # Kontaktperson
+    :location, # Ort_Adresse
+    :cost_center, # Kostenstelle
+    :cost_unit, # Kostenträger
+    :minimum_age, # Mindestalter
+    :maximum_age, # Maximalalter
+    :minimum_participants, # Minimale_TN_Zahl
+    :maximum_participants, # Maximale_TN_Zahl
+    :ideal_class_size, # Ideale_Klassengrösse
+    :maximum_class_size, # Maximale_Klassengrösse
+    :training_days, # Ausbildungstage
+    :season, # Saison
+    :accommodation, # Unterkunft
+    :reserve_accommodation, # Unterkunft_reservieren_durch_SAC_reservieren_durch_SAC
+    :meals, # Verpflegung
+    :globally_visible, # Sichtbarkeit
+    :language, # Sprache
+    :annual, # Jährlich_wiederkehrend
+    :start_point_of_time, # Kursbeginn
+    :application_opening_at, # Anmeldebeginn
+    :application_closing_at, # Anmeldeschluss
+    :application_conditions_de, # Aufnahmebedingungen_DE
+    :application_conditions_fr, # Aufnahmebedingungen_FR
+    :application_conditions_it, # Aufnahmebedingungen_IT
+    :external_applications, # Externe_Anmeldungen
+    :participations_visible, # Teilnehmersichtbarkeit
+    :priorization, # Priorisierung
+    :automatic_assignment, # Automatische_Zuteilung
+    :signature, # Unterschift_erforderlich
+    :signature_confirmation, # Zweitunterschrift_erforderlich
+    :signature_confirmation_text, # Zweitunterschrift
+    :applications_cancelable, # Abmeldung_möglich
+    :display_booking_info, # Anzeige_Anmeldestand
+    :price_member, # Mitgliederpreis
+    :price_regular, # Normalpreis
+    :price_subsidized, # Subventionierter_Preis
+    :price_js_active_member, # J_S_A_Mitgliederpreis
+    :price_js_active_regular, # J_S_A_Normalpreis
+    :price_js_passive_member, # J_S_P_Mitgliederpreis
+    :price_js_passive_regular, # J_S_P_Normalpreis
+    :brief_description_de, # Kurzbeschreibung_DE
+    :brief_description_fr, # Kurzbeschreibung_FR
+    :brief_description_it, # Kurzbeschreibung_IT
+    :specialities_de, # Besonderes_DE
+    :specialities_fr, # Besonderes_FR
+    :specialities_it, # Besonderes_IT
+    :similar_tours_de, # Vergleichstouren_DE
+    :similar_tours_fr, # Vergleichstouren_FR
+    :similar_tours_it, # Vergleichstouren_IT
+    :program_de, # Programm_DE
+    :program_fr, # Programm_FR
+    :program_it, # Programm_IT
+    :link_participants, # Link_Teilnehmer
+    :link_leaders, # Link_Kurskader
+    :link_survey, # Link_Umfrage
+    :book_discount_code, # Rabattcode_Buchversand
+    :canceled_reason, # Annulationsgrund
+    :nav19_number, # NAV19_Kurs (selbes wie Kursnummer)
+    :date_label, # NAV19_Bezeichnung
+    :date_location, # NAV19_Ort
+    :date_start_at, # NAV19_Von
+    :date_finish_at # NAV19_Bis
+  )
+end

app/domain/sac_imports/events/event_entry.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+#  Copyright (c) 2024, Schweizer Alpen-Club. This file is part of
+#  hitobito_sac_cas and licensed under the Affero General Public License version 3
+#  or later. See the COPYING file at the top-level directory or at
+#  https://github.com/hitobito/hitobito_sac_cas.
+
+module SacImports
+  module Events
+    class EventEntry
+      LOCALES = [:de, :fr, :it].freeze
+      ATTRS_TRANSLATED = [:name, :description, :application_conditions, :brief_description,
+        :specialities, :similar_tours, :program]
+      ATTRS_REGULAR = [:number, :state, :contact_id, :location, :minimum_age, :maximum_age,
+        :minimum_participants, :maximum_participants, :ideal_class_size, :maximum_class_size,
+        :training_days, :season, :accommodation, :meals, :language, :start_point_of_time,
+        :application_opening_at, :application_closing_at, :signature_confirmation_text,
+        :price_member, :price_regular, :price_subsidized, :price_js_active_member,
+        :price_js_active_regular, :price_js_passive_member, :price_js_passive_regular,
+        :link_participants, :link_leaders, :link_survey, :book_discount_code]
+      ATTRS_BOOLEAN = [:reserve_accommodation, :globally_visible, :annual, :external_applications,
+        :participations_visible, :priorization, :automatic_assignment, :signature,
+        :signature_confirmation, :applications_cancelable, :display_booking_info]
+      ATTRS_BELONGS_TO = [:kind, :cost_center, :cost_unit]
+
+      attr_reader :row, :associations, :warnings
+
+      delegate :valid?, :errors, to: :event
+
+      def initialize(row, associations)
+        @row = row
+        @associations = associations
+        @warnings = []
+        build_event
+      end
+
+      def import!
+        event.save!
+      end
+
+      def error_messages
+        errors.full_messages.join(", ")
+      end
+
+      def event
+        @event ||= Event::Course.find_or_initialize_by(number: row.number)
+      end
+
+      def build_event
+        event.attributes = regular_attrs
+        event.attributes = boolean_attrs
+        event.attributes = belongs_to_attrs
+        event.canceled_reason = canceled_reason
+        LOCALES.each do |locale|
+          event.attributes = translated_attrs(locale)
+        end
+        event.groups = [associations.fetch(:groups).fetch(:root)]
+        build_date
+        normalize_event
+      end
+
+      def regular_attrs
+        ATTRS_REGULAR.each_with_object({}) do |attr, hash|
+          hash[attr] = value(attr)
+        end
+      end
+
+      def translated_attrs(locale)
+        ATTRS_TRANSLATED.each_with_object({locale: locale}) do |attr, hash|
+          val = value(:"#{attr}_#{locale}")
+          hash[attr] = strip_paragraph(val) unless val.nil?
+        end
+      end
+
+      def belongs_to_attrs
+        ATTRS_BELONGS_TO.each_with_object({}) do |attr, hash|
+          hash[:"#{attr}_id"] = association_id(attr, value(attr))
+        end
+      end
+
+      def association_id(attr, value)
+        return nil if value.nil?
+
+        associations.fetch(attr.to_s.pluralize.to_sym).fetch(value) do
+          @warnings << "#{attr} with value #{value} couldn't be found"
+          nil
+        end
+      end
+
+      def boolean_attrs
+        ATTRS_BOOLEAN.each_with_object({}) do |attr, hash|
+          hash[attr] = value(attr) == "1"
+        end
+      end
+
+      def build_date
+        event.dates.find_or_initialize_by(
+          label: value(:date_label),
+          location: value(:date_location),
+          start_at: value(:date_start_at),
+          finish_at: value(:date_finish_at)
+        )
+      end
+
+      def canceled_reason
+        canceled_reason_value = value(:canceled_reason)
+        if canceled_reason_value == "not_applicable"
+          "weather"
+        else
+          canceled_reason_value
+        end
+      end
+
+      def normalize_event
+      end
+
+      def value(attr)
+        row.public_send(attr)
+      end
+
+      def strip_paragraph(text)
+        match = text.match(/\A<p>(.*?)<\/p>\z/m)
+        if match && text.scan("<p>").count == 1
+          match[1]
+        else
+          text
+        end
+      end
+    end
+  end
+end

app/domain/sac_imports/nav18_events_importer.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+#  Copyright (c) 2024, Schweizer Alpen-Club. This file is part of
+#  hitobito_sac_cas and licensed under the Affero General Public License version 3
+#  or later. See the COPYING file at the top-level directory or at
+#  https://github.com/hitobito/hitobito_sac_cas.
+
+module SacImports
+  class Nav18EventsImporter
+    include LogCounts
+
+    REPORT_HEADERS = [
+      :number,
+      :name_de,
+      :status,
+      :errors
+    ]
+
+    def initialize(output: $stdout, import_spec_fixture: false)
+      @output = output
+      # spec fixture includes all sections and it's public data
+      @import_spec_fixture = import_spec_fixture
+      @source_file = source_file
+      @csv_report = SacImports::CsvReport.new("nav18-events", REPORT_HEADERS, output:)
+    end
+
+    def create
+      @csv_report.log("The file contains #{@source_file.lines_count} rows.")
+      progress = Progress.new(@source_file.lines_count, title: "NAV18 Events")
+
+      log_counts_delta(@csv_report, Event.unscoped) do
+        @source_file.rows do |row|
+          progress.step
+          process_row(row)
+        end
+      end
+
+      @csv_report.finalize
+    end
+
+    private
+
+    def source_file
+      if @import_spec_fixture
+        CsvSource.new(:NAV18, source_dir: spec_fixture_dir)
+      else
+        CsvSource.new(:NAV18)
+      end
+    end
+
+    def spec_fixture_dir
+      Pathname.new(HitobitoSacCas::Wagon.root.join("spec", "fixtures", "files", "sac_imports_src"))
+    end
+
+    def process_row(row)
+      entry = Events::EventEntry.new(row, associations)
+      entry.import! if entry.valid?
+      report_warnings(entry)
+      report_errors(entry)
+    end
+
+    def report_errors(entry)
+      return if entry.errors.blank?
+
+      @output.puts("#{entry.row.name_de} (#{entry.row.number}): ❌ #{entry.error_messages}")
+      @csv_report.add_row(
+        number: entry.row.number,
+        name_de: entry.row.name_de,
+        status: "error",
+        errors: entry.error_messages
+      )
+    end
+
+    def report_warnings(entry)
+      return if entry.warnings.blank?
+
+      @csv_report.add_row(
+        number: entry.row.number,
+        name_de: entry.row.name_de,
+        status: "warning",
+        errors: entry.warnings.join(", ")
+      )
+    end
+
+    def associations
+      @associations ||= {
+        kinds: Event::Kind::Translation.where(locale: :de).pluck(:short_name, :event_kind_id).to_h,
+        cost_centers: CostCenter.pluck(:code, :id).to_h,
+        cost_units: CostUnit.pluck(:code, :id).to_h,
+        groups: {root: Group.root}
+      }
+    end
+  end
+end

app/models/concerns/events/courses/state.rb

@@ -32,7 +32,7 @@ module Events::Courses::State
 
     self.possible_states = SAC_COURSE_STATES.keys.collect(&:to_s)
 
-    validate :assert_valid_state_change, if: :state_changed?
+    validate :assert_valid_state_change, if: :state_changed?, on: :update
 
     before_create :set_default_state
     before_save :adjust_application_state, if: :application_closing_at_changed?
@@ -90,7 +90,8 @@ def assert_valid_state_change
   end
 
   def set_default_state
-    self.state = :created
+    # Explicitly call self[:state].blank? because self.state is overridden in youth wagon to return first possible state if nil.
+    self.state = :created if self[:state].blank?
   end
 
   def state_changed_to?(new_state)

lib/tasks/sac_imports.rake

@@ -71,6 +71,7 @@ namespace :sac_imports do
     "chimp-subscriptions",
     :update_sac_family_address,
     "nav17-1_event_kinds",
+    "nav18-1_events",
     :cleanup,
     :check_data_quality
   ] do
@@ -161,6 +162,12 @@ namespace :sac_imports do
     Rake::Task["sac_imports:dump_database"].execute(dump_name: "nav17-event-kinds")
   end
 
+  desc "Imports events"
+  task "nav18-1_events": :setup do
+    SacImports::Nav18EventsImporter.new.create
+    Rake::Task["sac_imports:dump_database"].execute(dump_name: "nav18-events")
+  end
+
   desc "NAV1 Imports subscriptions from Navision"
   task "chimp-subscriptions": :setup do
     SacImports::ChimpImporter.new.create

spec/controllers/event/courses/state_controller_spec.rb

@@ -51,6 +51,7 @@
 
       it "does not update state if step makes event invalid" do
         course = Fabricate(:sac_course)
+        puts "created"
 
         put :update, params: {group_id: group.id, id: course.id, state: "application_open"}