From 39608f9b56e9085857956abd0aded67ff6f88fc4 Mon Sep 17 00:00:00 2001 From: z Date: Fri, 6 Dec 2024 16:54:09 +0100 Subject: [PATCH] Add NAV17 importer for event kinds (#1348) --- app/domain/sac_imports/csv_source.rb | 3 +- app/domain/sac_imports/csv_source/nav17.rb | 43 ++++++ app/domain/sac_imports/events/kind_entry.rb | 127 ++++++++++++++++++ .../sac_imports/nav17_event_kinds_importer.rb | 96 +++++++++++++ lib/tasks/sac_imports.rake | 7 + .../nav17_event_kinds_importer_spec.rb | 108 +++++++++++++++ .../files/sac_imports_src/NAV17_fixture.csv | 9 ++ .../qualification_kind_translations.yml | 7 + 8 files changed, 399 insertions(+), 1 deletion(-) create mode 100644 app/domain/sac_imports/csv_source/nav17.rb create mode 100644 app/domain/sac_imports/events/kind_entry.rb create mode 100644 app/domain/sac_imports/nav17_event_kinds_importer.rb create mode 100644 spec/domain/sac_imports/nav17_event_kinds_importer_spec.rb create mode 100644 spec/fixtures/files/sac_imports_src/NAV17_fixture.csv diff --git a/app/domain/sac_imports/csv_source.rb b/app/domain/sac_imports/csv_source.rb index 3d059e142..b8a4478cd 100644 --- a/app/domain/sac_imports/csv_source.rb +++ b/app/domain/sac_imports/csv_source.rb @@ -15,6 +15,7 @@ class SacImports::CsvSource NAV2b: Nav2b, NAV3: Nav3, NAV6: Nav6, + NAV17: Nav17, WSO21: Wso2 }.freeze @@ -33,7 +34,7 @@ def rows(filter: nil) next unless filter.blank? || filter_match?(row, filter) if block_given? - yield rows + yield row else data << row end diff --git a/app/domain/sac_imports/csv_source/nav17.rb b/app/domain/sac_imports/csv_source/nav17.rb new file mode 100644 index 000000000..7bd138ec7 --- /dev/null +++ b/app/domain/sac_imports/csv_source/nav17.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +# Copyright (c) 2024, Schweizer Alpen-Club. This file is part of +# hitobito_sac_cas and licensed under the Affero General Public License version 3 +# or later. See the COPYING file at the top-level directory or at +# https://github.com/hitobito/hitobito_sac_cas + +class SacImports::CsvSource + # Event::Kinds + # !!! DO NOT CHANGE THE ORDER OF THE KEYS !!! + # they must match the order of the columns in the CSV files + Nav17 = Data.define( + :label_de, # Bezeichnung_DE + :label_fr, # Bezeichnung_FR + :label_it, # Bezeichnung_IT + :short_name, # Kurzname + :kind_category, # Kurskategorie + :general_information_de, # Standardbeschreibung_DE + :general_information_fr, # Standardbeschreibung_FR + :general_information_it, # Standardbeschreibung_IT + :application_conditions_de, # Aufnahmebedingungen_DE + :application_conditions_fr, # Aufnahmebedingungen_FR + :application_conditions_it, # Aufnahmebedingungen_IT + :level, # Kursstufe + :cost_center, # Kostenstelle + :cost_unit, # Kostenträger + :course_compensation_categories, # Vergütungskategorien + :minimum_age, # Mindestalter + :maximum_age, # Maximalalter + :minimum_participants, # Minimale_TN_Zahl + :maximum_participants, # Maximale_TN_Zahl + :ideal_class_size, # Ideale_Klassengrösse + :maximum_class_size, # Maximale_Klassengrösse + :training_days, # Ausbildungstage + :season, # Saison + :accommodation, # Unterkunft + :reserve_accommodation, # Unterkunft_reservieren_durch_SAC + :section_may_create, # Von_Sektion_erstellbar + :precondition, # Vorbedingungen + :qualification, # Qualifiziert_für + :prolongation # Verlängert + ) +end diff --git a/app/domain/sac_imports/events/kind_entry.rb b/app/domain/sac_imports/events/kind_entry.rb new file mode 100644 index 000000000..287d3cfdd --- /dev/null +++ b/app/domain/sac_imports/events/kind_entry.rb @@ -0,0 +1,127 @@ +# frozen_string_literal: true + +# Copyright (c) 2024, Schweizer Alpen-Club. This file is part of +# hitobito_sac_cas and licensed under the Affero General Public License version 3 +# or later. See the COPYING file at the top-level directory or at +# https://github.com/hitobito/hitobito_sac_cas. + +module SacImports + module Events + class KindEntry + LOCALES = [:de, :fr, :it].freeze + ATTRS_TRANSLATED = [:label, :general_information, :application_conditions] + ATTRS_REGULAR = [:short_name, :minimum_age, :maximum_age, :minimum_participants, + :maximum_participants, :ideal_class_size, :maximum_class_size, :training_days, + :season, :accommodation] + ATTRS_BOOLEAN = [:reserve_accommodation, :section_may_create] + ATTRS_BELONGS_TO = [:kind_category, :level, :cost_center, :cost_unit] + + attr_reader :row, :associations, :warnings + + delegate :valid?, :errors, to: :kind + + def initialize(row, associations) + @row = row + @associations = associations + @warnings = [] + build_kind + end + + def import! + kind.save! + end + + def error_messages + errors.full_messages.join(", ") + end + + def kind + @kind ||= Event::Kind.find_or_initialize_by(short_name: row.short_name) + end + + def build_kind + kind.attributes = regular_attrs + kind.attributes = boolean_attrs + kind.attributes = belongs_to_attrs + LOCALES.each do |locale| + kind.attributes = translated_attrs(locale) + end + kind.course_compensation_category_ids = select_course_compensation_category_ids + build_kind_qualification_kinds + normalize_kind + end + + def regular_attrs + ATTRS_REGULAR.each_with_object({}) do |attr, hash| + hash[attr] = value(attr) + end + end + + def translated_attrs(locale) + ATTRS_TRANSLATED.each_with_object({locale: locale}) do |attr, hash| + val = value(:"#{attr}_#{locale}") + hash[attr] = strip_paragraph(val) unless val.nil? + end + end + + def belongs_to_attrs + ATTRS_BELONGS_TO.each_with_object({}) do |attr, hash| + hash[:"#{attr}_id"] = association_id(attr, value(attr)) + end + end + + def association_id(attr, value) + return nil if value.nil? + + associations.fetch(attr.to_s.pluralize.to_sym).fetch(value) do + @warnings << "#{attr} with value #{value} couldn't be found" + nil + end + end + + def boolean_attrs + ATTRS_BOOLEAN.each_with_object({}) do |attr, hash| + hash[attr] = value(attr) == "1" + end + end + + def select_course_compensation_category_ids + row.course_compensation_categories.to_s.split(",").map do |category| + association_id(:course_compensation_category, category.strip) + end.compact.uniq + end + + def build_kind_qualification_kinds + Event::KindQualificationKind::CATEGORIES.each do |category| + row.public_send(category).to_s.split(",").each do |quali_kind| + quali_kind_id = association_id(:qualification_kind, quali_kind.strip) + next unless quali_kind_id + + kind.event_kind_qualification_kinds.find_or_initialize_by( + qualification_kind_id: quali_kind_id, + category: category, + role: "participant" + ) + end + end + end + + def normalize_kind + kind.maximum_age = nil if kind.maximum_age&.zero? + end + + def value(attr) + row.public_send(attr) + end + + def strip_paragraph(text) + match = text.match(/\A

(.*?)<\/p>\z/m) + if match && text.scan("

").count == 1 + match[1] + else + text + end + end + end + end +end diff --git a/app/domain/sac_imports/nav17_event_kinds_importer.rb b/app/domain/sac_imports/nav17_event_kinds_importer.rb new file mode 100644 index 000000000..39fd3619b --- /dev/null +++ b/app/domain/sac_imports/nav17_event_kinds_importer.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +# Copyright (c) 2024, Schweizer Alpen-Club. This file is part of +# hitobito_sac_cas and licensed under the Affero General Public License version 3 +# or later. See the COPYING file at the top-level directory or at +# https://github.com/hitobito/hitobito_sac_cas. + +module SacImports + class Nav17EventKindsImporter + include LogCounts + + REPORT_HEADERS = [ + :short_name, + :label, + :status, + :errors + ] + + def initialize(output: $stdout, import_spec_fixture: false) + @output = output + # spec fixture includes all sections and it's public data + @import_spec_fixture = import_spec_fixture + @source_file = source_file + @csv_report = SacImports::CsvReport.new("nav17-event-kinds", REPORT_HEADERS, output:) + end + + def create + @csv_report.log("The file contains #{@source_file.lines_count} rows.") + progress = Progress.new(@source_file.lines_count, title: "NAV17 Event Kinds") + + log_counts_delta(@csv_report, Event::Kind.unscoped) do + @source_file.rows do |row| + progress.step + process_row(row) + end + end + + @csv_report.finalize + end + + private + + def source_file + if @import_spec_fixture + CsvSource.new(:NAV17, source_dir: spec_fixture_dir) + else + CsvSource.new(:NAV17) + end + end + + def spec_fixture_dir + Pathname.new(HitobitoSacCas::Wagon.root.join("spec", "fixtures", "files", "sac_imports_src")) + end + + def process_row(row) + entry = Events::KindEntry.new(row, associations) + entry.import! if entry.valid? + report_warnings(entry) + report_errors(entry) + end + + def report_errors(entry) + return if entry.errors.blank? + + @output.puts("#{entry.row.label_de} (#{entry.row.short_name}): ❌ #{entry.error_messages}") + @csv_report.add_row( + short_name: entry.row.short_name, + label: entry.row.label_de, + status: "error", + errors: entry.error_messages + ) + end + + def report_warnings(entry) + return if entry.warnings.blank? + + @csv_report.add_row( + short_name: entry.row.short_name, + label: entry.row.label_de, + status: "warning", + errors: entry.warnings.join(", ") + ) + end + + def associations + @associations ||= { + kind_categories: Event::KindCategory.pluck(:order, :id).to_h.transform_keys(&:to_s), + qualification_kinds: QualificationKind.pluck(:label, :id).to_h, + levels: Event::Level.pluck(:code, :id).to_h.transform_keys(&:to_s), + cost_centers: CostCenter.pluck(:code, :id).to_h, + cost_units: CostUnit.pluck(:code, :id).to_h, + course_compensation_categories: CourseCompensationCategory.pluck(:short_name, :id).to_h + } + end + end +end diff --git a/lib/tasks/sac_imports.rake b/lib/tasks/sac_imports.rake index 0673408db..bd940a3e0 100644 --- a/lib/tasks/sac_imports.rake +++ b/lib/tasks/sac_imports.rake @@ -68,6 +68,7 @@ namespace :sac_imports do "wso21-1_people", "nav2b-2_non_membership_roles", :update_sac_family_address, + "nav17-1_event_kinds", :cleanup, :check_data_quality ] do @@ -146,6 +147,12 @@ namespace :sac_imports do Rake::Task["sac_imports:dump_database"].execute(dump_name: "nav5-huts") end + desc "Imports event kinds" + task "nav17-1_event_kinds": :setup do + SacImports::Nav17EventKindsImporter.new.create + Rake::Task["sac_imports:dump_database"].execute(dump_name: "nav17-event-kinds") + end + desc "Run cleanup tasks" task cleanup: :setup do SacImports::Cleanup.new.run diff --git a/spec/domain/sac_imports/nav17_event_kinds_importer_spec.rb b/spec/domain/sac_imports/nav17_event_kinds_importer_spec.rb new file mode 100644 index 000000000..dd13eb460 --- /dev/null +++ b/spec/domain/sac_imports/nav17_event_kinds_importer_spec.rb @@ -0,0 +1,108 @@ +# frozen_string_literal: true + +# Copyright (c) 2024, Schweizer Alpen-Club. This file is part of +# hitobito_sac_cas and licensed under the Affero General Public License version 3 +# or later. See the COPYING file at the top-level directory or at +# https://github.com/hitobito/hitobito_sac_cas. + +require "spec_helper" + +describe SacImports::Nav17EventKindsImporter do + let(:sac_imports_src) { file_fixture("sac_imports_src").expand_path } + let(:output) { double(puts: nil, print: nil) } + let(:report) { described_class.new(output: output) } + let(:report_file) { Rails.root.join("log", "sac_imports", "nav17-event-kinds_2024-01-23-1142.csv") } + let(:report_headers) { + %w[short_name label status errors] + } + let(:csv_report) { CSV.read(report_file, col_sep: ";") } + + before do + File.delete(report_file) if File.exist?(report_file) + stub_const("SacImports::CsvSource::SOURCE_DIR", sac_imports_src) + + load(HitobitoSacCas::Wagon.root.join("db", "seeds", "course_master_data.rb")) + seed_cost_centers + seed_cost_units + seed_event_kind_categories + seed_event_levels + seed_course_compensation_categories + end + + it "creates report for entries in source file" do + expected_output = [] + expected_output << "Alpinwandern (S6570): ❌ Kurskategorie muss ausgefüllt werden, Saison ist kein gültiger Wert" + + expect(output).to receive(:puts).with("The file contains 9 rows.") + expected_output.flatten.each do |output_line| + expect(output).to receive(:puts).with(output_line) + end + expect(output).to receive(:puts).with("\n\n\nReport generated in 0.0 minutes.") + expect(output).to receive(:puts).with("Thank you for flying with SAC Imports.") + expect(output).to receive(:puts).with("Report written to #{report_file}") + + travel_to DateTime.new(2024, 1, 23, 10, 42) + + expect { report.create } + .to change { Event::Kind.count }.by(8).and \ + change { Event::KindQualificationKind.count }.by(3) + + kind = Event::Kind.find_by(short_name: "S1460") + expect(kind.attributes.symbolize_keys).to include( + short_name: "S1460", + label: "Alpine Umwelt", + general_information: nil, + application_conditions: "gute allgemeine Kondition für eine Wintertour in den Bergen mit Schneeschuhen", + minimum_age: 18, + minimum_participants: 4, + maximum_participants: 8, + ideal_class_size: 4, + maximum_class_size: 8, + maximum_age: nil, + season: "winter", + accommodation: "hut", + training_days: 0.2e1, + reserve_accommodation: false, + section_may_create: false + ) + expect(kind.kind_category.label).to eq("Diverse Kurse Winter") + expect(kind.level.code).to eq(1) + expect(kind.cost_center.code).to eq("2100026") + expect(kind.cost_unit.code).to eq("A1300") + I18n.with_locale(:fr) do + expect(kind.label).to eq("Environnement alpin") + expect(kind.application_conditions).to eq("bonne condition physique générale pour une course d’hiver en raquettes en montagne") + end + I18n.with_locale(:it) do + expect(kind.label).to eq("Environnement alpin") # fallback to fr + end + + kind = Event::Kind.find_by(short_name: "S5750") + expect(kind.event_kind_qualification_kinds.size).to eq(2) + kqk = kind.event_kind_qualification_kinds.first + expect(kqk.qualification_kind.label).to be_in(["Ski Leiter", "Snowboard Leiter"]) + expect(kqk.category).to eq("prolongation") + expect(kqk.role).to eq("participant") + + kind = Event::Kind.find_by(short_name: "S6560") + expect(kind.section_may_create).to eq(true) + expect(kind.event_kind_qualification_kinds.size).to eq(1) + kqk = kind.event_kind_qualification_kinds.first + expect(kqk.qualification_kind.label).to eq("Ski Leiter") + expect(kqk.category).to eq("qualification") + expect(kqk.role).to eq("participant") + + kind = Event::Kind.find_by(short_name: "S5780") + expect(kind.course_compensation_categories).to be_blank + + expect(File.exist?(report_file)).to be_truthy + + expect(csv_report.size).to eq(3) + expect(csv_report.first).to eq(report_headers) + expect(csv_report[1]).to eq(["S6570", "Alpinwandern", "warning", "kind_category with value 6990 couldn't be found, course_compensation_category with value HON-KAT-X couldn't be found"]) + expect(csv_report[2]).to eq(["S6570", "Alpinwandern", "error", "Kurskategorie muss ausgefüllt werden, Saison ist kein gültiger Wert"]) + + File.delete(report_file) + expect(File.exist?(report_file)).to be_falsey + end +end diff --git a/spec/fixtures/files/sac_imports_src/NAV17_fixture.csv b/spec/fixtures/files/sac_imports_src/NAV17_fixture.csv new file mode 100644 index 000000000..a19112a3d --- /dev/null +++ b/spec/fixtures/files/sac_imports_src/NAV17_fixture.csv @@ -0,0 +1,9 @@ +Alpine Umwelt,Environnement alpin,NULL,S1460,1300,NULL,NULL,NULL,

gute allgemeine Kondition für eine Wintertour in den Bergen mit Schneeschuhen

,

bonne condition physique générale pour une course d’hiver en raquettes en montagne

,NULL,1,2100026,A1300,"HO-0001,HON-KAT-I,KP-ADMIN-KAT I-IV,KP-REISE/MATERIAL,SPÖ-0001",18,0,4,8,4,8,2.00,winter,hut,0,0,NULL,NULL,NULL +Alpine Umwelt,Environnement alpin,NULL,S5760,5600,NULL,NULL,NULL,"

Gute allgemeine Kondition für Touren im Hochgebirge, Trittsicherheit. Wir werden grösstenteils auf Wanderwegen bleiben, betreten aber von Zeit zu Zeit für Beobachtungen unwegsames Gelände.

","

bonne condition physique générale pour des courses de haute montagne, pied sûr. Nous resterons en grande partie sur des sentiers pédestres, mais parcourrons de temps à autre des terrains dépourvus de chemins

",NULL,1,2100026,A5600,"HO-0001,HO-0004,HON-KAT-I,KP-ADMIN-KAT I-IV,KP-REISE/MATERIAL,KV-0001,SPD-0001,SPÖ-0001,SPP-0001",18,0,4,8,4,8,2.00,summer,hut,0,0,NULL,NULL,NULL +Alpinwandern,Randonnée alpine,NULL,S5750,5400,NULL,NULL,NULL,"

Kondition für bis 5-7-stündige Wanderungen bis Schwierigkeitsgrad T4, Trittsicherheit, kein Schwindelgefühl

","

condition physique pour des randonnées jusqu’à 5-7 heures, jusqu’au degré de difficulté T4?; pied sûr?; pas de vertige

",NULL,2,2100022,A5400,"HO-0001,HO-0004,HON-KAT-I,HON-KAT-I,KP-ADMIN-KAT I-IV,KP-REISE/MATERIAL,KV-0001,SPD-0001,SPÖ-0001,SPP-0001",18,0,5,18,6,7,5.00,summer,pension,0,0,NULL,NULL,"Ski Leiter,Snowboard Leiter" +Alpinwandern,Randonnée alpine,NULL,S6570,6990,NULL,NULL,NULL,

aktive/r SAC-Tourenleiter/in Sommer (Unterschrift SAC-Tourenchefin)

,

activité actuelle de chef/fe de courses du CAS été (signature du préposé aux courses)

,NULL,5,2100025,A6500,"HO-0001,HON-KAT-X,KP-ADMIN-KAT I-IV,KP-REISE/MATERIAL,KV-0001,SPÖ-0001",18,0,5,12,6,7,2.00,spring,hut,0,1,NULL,NULL,NULL +Alpinwandern T5,Randonnée alpine T5,NULL,S6560,6500,NULL,NULL,NULL,"

Anforderungen: Tourenleiter/in Alpinwandern oder Sommer 1, Sommer 2; Kondition für bis 8-stündige Wanderungen bis Schwierigkeitsgrad T5+; Leitungserfahrung bis T4; gute Trittsicherheit, kein Schwindelgefühl, Affinität zu technischen Hilfsmitteln, gute Selbsteinschätzung

","

Exigences: cchef·fe de courses randonnée alpine ou été 1, été 2; condition physique pour une randonnée de huit heures allant jusqu’à un niveau de difficulté T5+; expérience chef· fe de courses jusqu’au niveau T4; le pied très sûr, pas de vertige, affinité avec les outils techniques, bonne auto-évaluation

",NULL,5,2100025,A6500,"HON-KAT-V,HON-KAT-I,KP-ADMIN-KAT I-IV,KP-REISE/MATERIAL",18,0,5,6,6,6,3.00,summer,pension,0,1,NULL,Ski Leiter,NULL +Alpinwandern: Schnupperkurs,Randonnée alpine: cours de découverte,NULL,S5745,5400,NULL,NULL,NULL,"

Kondition für bis zu 5-stündige Wanderungen bis Schwierigkeitsgrad T3, Trittsicherheit, kein Schwindelgefühl

","

condition physique pour des randonnées jusqu’à 5 heures, jusqu’au degré de difficulté T3?; pied sûr?; pas de vertige

",NULL,1,2100022,A5400,"HO-0001,HO-0002,HO-0004,HON-KAT-II,HON-KAT-II,KP-ADMIN-KAT I-IV,KP-REISE/MATERIAL,KV-0001,SPD-0001,SPÖ-0001,SPP-0001",18,0,5,12,6,6,2.00,summer,hut,0,0,NULL,NULL,NULL +Bergsteigen Sommer,Alpinisme été,NULL,S6530,6500,NULL,NULL,NULL,

aktive/r SAC-Tourenleiter/in Sommer (Unterschrift SAC-Tourenchef)

,

activité actuelle de chef/fe de courses du CAS été (signature du préposé aux courses)

,NULL,5,2100025,A6500,"HO-0001,HON-KAT-I,KP-ADMIN-KAT I-IV,KP-REISE/MATERIAL,KV-0001,SPÖ-0001,SPP-0001",18,0,5,12,6,7,3.00,summer,hut,0,0,NULL,NULL,NULL +Bergwandern,Randonnée en montagne,NULL,S5740,5300,NULL,NULL,NULL,"

Kondition für bis 5-stündige Wanderungen auf weiss-rot-weiss markierten Wegen bis max. Grad T3 nach der SAC-Berg- und Alpinwanderskala.

","

bonne condition physique pour randonnées jusqu’à 5 heures sur sentiers balisés blanc-rouge-blanc, jusqu’au degré T3 max. selon l’échelle CAS pour la randonnée en montagne et alpine

",NULL,2,2100022,A5300,"HO-0001,HON-KAT-I,HON-KAT-II,KP-ADMIN-KAT I-IV,KP-REISE/MATERIAL,KV-0001,SPD-0001,SPÖ-0001,SPP-0001",18,0,5,16,8,8,5.00,summer,pension,0,1,NULL,NULL,NULL +Biwakieren,NULL,NULL,S5780,5600,NULL,NULL,NULL,,NULL,NULL,2,2100022,A5600,NULL,18,0,5,12,6,7,2.00,summer,bivouac,0,0,NULL,NULL,NULL diff --git a/spec/fixtures/qualification_kind_translations.yml b/spec/fixtures/qualification_kind_translations.yml index 3564c8d2a..37bae2ef2 100644 --- a/spec/fixtures/qualification_kind_translations.yml +++ b/spec/fixtures/qualification_kind_translations.yml @@ -10,3 +10,10 @@ ski_leader_de: label: Ski Leiter created_at: <%= Time.zone.now %> updated_at: <%= Time.zone.now %> + +snowboard_leader_de: + qualification_kind_id: <%= ActiveRecord::FixtureSet.identify(:snowboard_leader) %> + locale: de + label: Snowboard Leiter + created_at: <%= Time.zone.now %> + updated_at: <%= Time.zone.now %>