From 7c831d1106b9a00acc63236848ee244340bc2d2d Mon Sep 17 00:00:00 2001 From: jaume Date: Fri, 15 Mar 2024 22:34:55 +0100 Subject: [PATCH] Added new parser for the new commerzbank csv file --- .../finances/importer/importers/__init__.py | 3 +- .../importers/commerz_bank_2024_en.py | 73 ++++++++++++++ .../tests/resources/commerz_bank_2024.csv | 13 +++ .../importer/tests/tests_importers.py | 98 ++++++++++++++++++- 4 files changed, 185 insertions(+), 2 deletions(-) create mode 100644 server/finances/importer/importers/commerz_bank_2024_en.py create mode 100644 server/finances/importer/tests/resources/commerz_bank_2024.csv diff --git a/server/finances/importer/importers/__init__.py b/server/finances/importer/importers/__init__.py index 74a868c8..292ec9d8 100644 --- a/server/finances/importer/importers/__init__.py +++ b/server/finances/importer/importers/__init__.py @@ -1,5 +1,5 @@ from finances.importer.importers import (caixa_bank, caixa_enginyers, edenred, - n26, qif, commerz_bank) + n26, qif, commerz_bank, commerz_bank_2024_en) FORMAT_LIST = { # caixa_bank.CaixaBankAccount.key: caixa_bank.CaixaBankAccount, @@ -7,6 +7,7 @@ caixa_bank.CaixaBankCard.key: caixa_bank.CaixaBankCard, caixa_bank.CaixaBankCard2020.key: caixa_bank.CaixaBankCard2020, commerz_bank.CommerzBank.key: commerz_bank.CommerzBank, + commerz_bank_2024_en.CommerzBank2024en.key: commerz_bank_2024_en.CommerzBank2024en, caixa_enginyers.CaixaEnginyersAccount.key: caixa_enginyers.CaixaEnginyersAccount, caixa_enginyers.CaixaEnginyersCredit.key: caixa_enginyers.CaixaEnginyersCredit, edenred.TicketRestaurant.key: edenred.TicketRestaurant, diff --git a/server/finances/importer/importers/commerz_bank_2024_en.py b/server/finances/importer/importers/commerz_bank_2024_en.py new file mode 100644 index 00000000..776f358f --- /dev/null +++ b/server/finances/importer/importers/commerz_bank_2024_en.py @@ -0,0 +1,73 @@ +import re +from .abstract import AbstractImporter, MapLocaleDateMixin +from ..parsers import CsvSourceFile + +DATE_REGEX = re.compile('(\d{4})\-(\d{2})\-(\d{2})T\d{2}:\d{2}:\d{2}') + +class CommerzBank2024en(AbstractImporter, MapLocaleDateMixin): + LOCAL_DATE_FORMAT = ["%d.%m.%Y"] + + key = "commerz-bank-2024-en" + file_regex = "^[A-Z]{2}(?:[ ]?[0-9]){18,20}_" + + _discard = 1 + + _mapping = { + 'movement_name': 8, + 'date': 10, + 'date_value': 1, + 'value': 4, + 'details': 9 + } + + def _creator(self, file_name): + return CsvSourceFile(file_name, self._discard, delimiter=';') + + def split_message(self, message): + if message.startswith('/'): + sub_msg = [ x + for x in message.split("//") + if x.startswith("Kartenzahlung") + ] + if len(sub_msg)>0: + message = sub_msg[0] + + if '//' in message: + message = message[:message.index('//')] + + date_match = DATE_REGEX.search(message) + end_index = message.find("End-To-End") + if date_match is not None: + index = date_match.start() + movement_name = message[:index] + date_groups = date_match.groups() + date_info = "{2}.{1}.{0}".format(*date_groups) + slash_index = movement_name.find('/') + details = None + if slash_index>0: + details = movement_name[slash_index+2:].strip() + movement_name = movement_name[:slash_index] + + return movement_name.strip(), details, date_info + + elif end_index>0: + return message[:end_index].strip(), None, None + return message, None, None + + + def build(self, row): + movement_name, details, new_date = self.split_message(row[3]) + + if len(movement_name)>250: + movement_name = movement_name[:250] + + if new_date is None: + new_date = row[0] + + row.extend([movement_name, details, new_date]) + #print(len(row)) + row = self.map_locale_date(row) + #print(row) + data = super(CommerzBank2024en, self).build(row) + data.value = float(data.value.replace(',', '.')) + return data \ No newline at end of file diff --git a/server/finances/importer/tests/resources/commerz_bank_2024.csv b/server/finances/importer/tests/resources/commerz_bank_2024.csv new file mode 100644 index 00000000..d1fb091a --- /dev/null +++ b/server/finances/importer/tests/resources/commerz_bank_2024.csv @@ -0,0 +1,13 @@ +Booking date;Value date;Transaction type;Booking text;Amount;Currency;Account IBAN;Category +23.04.2024;23.04.2024;Interest/charges;AUSGABE EINER DEBITKARTE ENTGELT ERSATZKARTE GIROCARD;-15;EUR;DE13344532223457; +12.03.2024;12.03.2024;Debit;Stadtwerke Ingolstadt Energie GmbH E-Mobility Rechnung Nr. 56315 zu Ve rtrag 704860, Kundennummer 718838 End-To-End Reference: A13092014.719833.704860.1546523 Mandate Reference: E-716758-714869-1 ID Of Ordering Party: DE09ZZZ987000575308 SEPA-CORE-DIRECT DEBIT subsequent;-20,55;EUR;DE13344532223457; +10.03.2024;10.03.2024;Debit;Stadtapotheke Stephan Kurzeder e.K. 2024-03-10T13:15:16 KFN 1 VJ 2812 Kartenzahlung;-24,93;EUR;DE13344532223457; +04.03.2024;04.03.2024;Debit;85053 EDEKA FANDERL//SOMEPLACE/DE 2024-03-01T16:20:09 KFN 0 VJ 2712 Kartenzahlung;-20,12;EUR;DE13344532223457; +01.03.2024;01.03.2024;Debit;M001-MEDIA MARKT//SOMEPLACE/DE 2024-02-29T13:31:45 KFN 0 VJ 2712 Kartenzahlung;-45,99;EUR;DE13344532223457; +29.02.2024;29.02.2024;Debit;BAUHAUS SOMEPLACE - DANKE 270217190097028271201056880 ELV6512 0607 27.02 17.19 ME0 End-To-End Reference: 27021719009702827120105688065120607 Mandate Reference: G512060770672402271719 ID Of Ordering Party: DE16ZZZ00000020245 SEPA-CORE-DIRECT DEBIT subsequent;-22,35;EUR;DE13344532223457; +19.02.2024;19.02.2024;Debit;FLUGHAFEN MUENCHEN GMBH GIR 6912847 2024-02-18T19:23:05 KFN 0 VJ 2712 Kartenzahlung;-8;EUR;DE13344532223457; +14.02.2024;14.02.2024;Interest/charges;Datenschutzhinweis ab 28.02.2024: Wir übermitteln Überweisungsdaten an den Zahlungsdienstleister des Empfängers (ZDE). Eingeschaltete Dienstleister können erforderliche Prüfungen zur Verhinderung von Zah- lungsverkehrsbetrug vornehmen. Der ZDE kann dem Empfänger die Über- weisungsdaten zur Verfügung stel- len (auch IBAN). Bei grenzüberschr. Überweisungen und Eilüberweisungen im Inland können die Daten in ge- meinsamer Verantwortung mit SWIFT verarbeitet werden. Zur System- sicherheit speichert SWIFT die Daten vorübergehend in Rechenzen- zentren in der EU, Schweiz und USA. Informationen und Vertrag über die gemeinsame Verantwortung zum SWIFT- Transaktionsverarbeitungsdienst: https://www.commerzbank.de/ hinweise/rechtliche-hinweise/;0;EUR;DE13344532223457; +10.07.2023;08.07.2023;Cash deposit/withdrawal;Bargeldauszahlung Commerzbank 00210074/Rathausplatz/I 2023-07-08T12:07:49 KFN 0 VJ 2312;-120;EUR;DE1020399412309; +07.07.2023;07.07.2023;Debit;/MISTRAL-SO//ZZ1ECOPJJKKNNVT6ZH//CHA N//SFA//USTRD//Kartenzahlung ARAL Some place Stra-e 7 2023-07-06T21:30:51 KFN 0 VJ 2312 Kartenzahlung;-15,11;EUR;DE1632145743634236; +30.06.2023;30.06.2023;Interest/charges;Periodic balance statement Account 201527900 EUR Bank Code 721 400 52 from 31.03.2023 to 30.06.2023 Balance after closing 1.283,80 EUR re.: approval of the balancing;0;EUR;DE1632145743634236; +20.06.2023;20.06.2023;Transfer;Jaume Singla Valls End-To-End Reference: e089219898642eab0f6d43654524cba Customer Reference: NSCT2306377775670000000000000000002;500;EUR;DE1632145743634236; \ No newline at end of file diff --git a/server/finances/importer/tests/tests_importers.py b/server/finances/importer/tests/tests_importers.py index eb1c068f..b6e4bd41 100644 --- a/server/finances/importer/tests/tests_importers.py +++ b/server/finances/importer/tests/tests_importers.py @@ -5,7 +5,7 @@ from finances.core.models import RawDataSource from finances.management.models import Filter, FilterConditionals, Tag -from ..importers import caixa_bank, caixa_enginyers, n26, commerz_bank +from ..importers import caixa_bank, caixa_enginyers, n26, commerz_bank, commerz_bank_2024_en from ..models import IMPORT_STATUS, StatusReport, StatusReportRow from .classes.importer import SAMPLE_DATA, TestAccount @@ -243,6 +243,102 @@ def test_filters(self): self.assertEqual(rds.tags.count(), 1) +class CommerzBank2024en(TransactionTestCase): + def setUp(self): + self.subject = commerz_bank_2024_en.CommerzBank2024en('cb', PATH+"/resources/commerz_bank_2024.csv", 'test') + + def check_movement(self, date, movement_name, value): + query_test = RawDataSource.objects.filter(date=date) + self.assertEqual(query_test.count(), 1) + test_value = query_test.first() + self.assertEqual(test_value.value, value) + self.assertEqual(test_value.movement_name, movement_name) + + + def test_basic(self): + self.subject.run() + self.assertEqual(StatusReport.objects.all().count(), 1) + print(StatusReport.objects.first().description) + self.assertEqual(RawDataSource.objects.all().count(), 12) + + self.check_movement( + date='2024-04-23', + value=-15.00, + movement_name="AUSGABE EINER DEBITKARTE ENTGELT ERSATZKARTE GIROCARD" + ) + + self.check_movement( + date='2024-03-12', + value=-20.55, + movement_name="Stadtwerke Ingolstadt Energie GmbH E-Mobility Rechnung Nr. 56315 zu Ve rtrag 704860, Kundennummer 718838" + ) + + self.check_movement( + date= '2024-03-10', + value= -24.93, + movement_name= "Stadtapotheke Stephan Kurzeder e.K." + ) + self.check_movement( + date= '2024-03-04', + value= -20.12, + movement_name= "85053 EDEKA FANDERL" + ) + self.check_movement( + date= '2024-03-01', + value= -45.99, + movement_name= "M001-MEDIA MARKT" + ) + self.check_movement( + date= '2024-02-29', + value= -22.35, + movement_name= "BAUHAUS SOMEPLACE - DANKE 270217190097028271201056880 ELV6512 0607 27.02 17.19 ME0" + ) + self.check_movement( + date= '2024-02-18', + value= -8.00, + movement_name= "FLUGHAFEN MUENCHEN GMBH GIR 6912847" + ) + self.check_movement( + date= '2024-02-14', + value= 0.00, + movement_name= "Datenschutzhinweis ab 28.02.2024: Wir übermitteln Überweisungsdaten an den Zahlungsdienstleister des Empfängers (ZDE). Eingeschaltete Dienstleister können erforderliche Prüfungen zur Verhinderung von Zah- lungsverkehrsbetrug vornehmen. Der ZDE kann d" + ) + self.check_movement( + date= '2023-07-08', + value= -120.00, + movement_name= "Bargeldauszahlung Commerzbank 00210074" + ) + self.check_movement( + date= '2023-07-06', + value= -15.11, + movement_name= "Kartenzahlung ARAL Some place Stra-e 7" + ) + self.check_movement( + date= '2023-06-30', + value= 0.00, + movement_name= "Periodic balance statement Account 201527900 EUR Bank Code 721 400 52 from 31.03.2023 to 30.06.2023 Balance after closing 1.283,80 EUR re.: approval of the balancing" + ) + self.check_movement( + date= '2023-06-20', + value= 500.00, + movement_name= "Jaume Singla Valls" + ) + + def test_filters(self): + t1 = Tag(name="Test tag") + t1.save() + f = Filter( + tag=t1, + type_conditional=FilterConditionals.GREATER, + conditional="0") + f.save() + + self.subject.run() + self.subject.apply_filters() + rds = RawDataSource.objects.filter(date="2023-06-20").first() + self.assertEqual(rds.tags.count(), 1) + + class N26Test(TransactionTestCase): def setUp(self): self.subject = n26.Number26('n26', PATH + "/resources/n26_es.csv", 'test')