adding commerz-bank importer

swift-server/app-tests/app-test.swift

@@ -6,7 +6,12 @@ import XCTest
 
 @testable import App
 
-class TestError: Error {}
+class TestError: Error {
+    let message: String?
+    init(message: String? = nil) {
+        self.message = message
+    }
+}
 
 func createUser(
 	app: Application, username: String, email: String, password: String,

swift-server/app-tests/importers/importer-caixa-enginyers.tests.swift

@@ -5,48 +5,13 @@ import XCTest
 
 @testable import App
 
-final class CaixaEnginyersImporterTests: XCTestCase {
-	var importerService: NewImportService!
-	var bankTransactionService: BankTransactionService!
-	var statusReportsService: StatusReportsService!
-	var caixaEnginyersImporter: CaixaEnginyersAccountImporter!
-	var group: UserGroup!
-	var app: Application?
-
-	func getTestFile(file: String) -> String {
-		let pwd = URL(fileURLWithPath: #file).pathComponents
-			.prefix(while: { $0 != "importers" }).joined(separator: "/").dropFirst()
-		return "\(pwd)/\(file)"
-	}
-
-	override func setUp() async throws {
-		let app = try await Application.make(.testing)
-		try await configure(app)
-		self.app = app
-
-		self.group = UserGroup(name: "Test User Group")
-		try await self.group.save(on: app.db)
-
-		caixaEnginyersImporter = CaixaEnginyersAccountImporter()
-		let testParsers: [ParserFactory] = [
-			caixaEnginyersImporter, CaixaEnginiersCreditImporter(),
-		]
-		importerService = NewImportService(parsers: testParsers)
-		bankTransactionService = BankTransactionService()
-		statusReportsService = StatusReportsService()
-	}
-
-	override func tearDown() async throws {
-		try await self.app?.asyncShutdown()
-		self.app = nil
-	}
-
-	func getDb() throws -> Database {
-		guard let app = app else {
-			throw TestError()
-		}
-		return app.db
-	}
+final class CaixaEnginyersImporterTests: BaseImporterTests {
+
+    override func getParsers() throws -> [any ParserFactory] {
+        return [
+            CaixaEnginyersAccountImporter(), CaixaEnginiersCreditImporter(),
+        ]
+    }
 
 	func testCaixaEnginyersAccountImport() async throws {
 		let groupOwnerId = try self.group.requireID()

swift-server/app-tests/importers/importer-commerz-bank.swift

@@ -0,0 +1,57 @@
+import XCTest
+@testable import App
+
+final class CommerzBankEnImporterTests: BaseImporterTests {
+    override func getParsers() throws -> [any ParserFactory] {
+        return [CommerzBankEnImporter()]
+    }
+    func testInsertingDataInCommerzBank() async throws {
+        let groupOwnerId = try self.group.requireID()
+        let db = try getDb()
+
+        let filePath = getTestFile(file: "test_files/commerz_bank.CSV")
+
+        try await importerService.importFromFile(on: db, groupOwnerId: groupOwnerId, key: "commerz-bank/en", fileName: "CommerzBank", filePath: filePath)
+
+
+        let reports = try await statusReportsService.getAll(
+            on: db, groupIds: [groupOwnerId])
+        XCTAssertEqual(reports.list.count, 1)
+        XCTAssertEqual(reports.list.first?.status, "OK")
+        XCTAssertEqual(reports.list.first?.description, "")
+        XCTAssertNil(reports.list.first?.context)
+
+        let transactions = try await bankTransactionService.getAll(
+            on: db, groupIds: [groupOwnerId])
+        XCTAssertEqual(transactions.list.count, 5)
+
+        let queryTest = transactions.list.first(where: { $0.date == DateOnly("2019-05-02") })
+        XCTAssertEqual(queryTest?.value, 256.01)
+        XCTAssertEqual(queryTest?.movementName, "Concept and more concepts")
+
+        let queryTest2 = transactions.list.first(where: { $0.date == DateOnly("2020-01-09") })
+        XCTAssertEqual(queryTest2?.value, -25)
+        XCTAssertEqual(queryTest2?.movementName, "Commerzbank 0321554")
+
+        let queryTest3 = transactions.list.first(where: { $0.date == DateOnly("2020-01-07") })
+        XCTAssertEqual(queryTest3?.movementName, "Backerei Sipl GmbH Fil.35 GIR 69036")
+
+        let queryTest4 = transactions.list.first(where: { $0.date == DateOnly("2020-02-09") })
+        XCTAssertEqual(queryTest4?.movementName, "ARAL Some address")
+
+        let queryTest5 = transactions.list.first(where: { $0.date == DateOnly("2020-02-07") })
+        XCTAssertEqual(queryTest5?.movementName, "BACKSTUBE WUENSCHE GMBH")
+    }
+}
+
+final class CommerzBankUnitTests: XCTestCase {
+    func testSplitMessage() throws {
+        let importerService = CommerzBankEnImporter()
+
+        let (msg, details, date) = try importerService.splitMessage("Kartenzahlung ARAL Some address 2020-02-09T21:13:19 KFN 1 VJ 2442 Kartenzahlung")
+        XCTAssertEqual(msg, "ARAL Some address")
+
+
+    }
+}
+

swift-server/app-tests/importers/importer-helper.swift

@@ -1,4 +1,6 @@
-import Foundation
+import XCTest
+import XCTVapor
+import Fluent
 
 @testable import App
 
@@ -68,3 +70,49 @@ class TestDynamicImporter: ParserFactory {
 		}
 	}
 }
+
+class BaseImporterTests: XCTestCase {
+    var importerService: NewImportService!
+    var bankTransactionService: BankTransactionService!
+    var statusReportsService: StatusReportsService!
+    var group: UserGroup!
+    var app: Application?
+
+    func getTestFile(file: String) -> String {
+        let pwd = URL(fileURLWithPath: #file).pathComponents
+            .prefix(while: { $0 != "app-tests" }).joined(separator: "/").dropFirst()
+        return "\(pwd)/\(file)"
+    }
+
+    func getParsers() throws -> [ParserFactory] {
+        throw TestError(message: "A class must overwrite getParsers")
+    }
+
+    override func setUp() async throws {
+        let app = try await Application.make(.testing)
+        try await configure(app)
+        self.app = app
+
+        self.group = UserGroup(name: "Test User Group")
+        try await self.group.save(on: app.db)
+
+        let testParsers: [ParserFactory] = try getParsers()
+        importerService = NewImportService(parsers: testParsers)
+        bankTransactionService = BankTransactionService()
+        statusReportsService = StatusReportsService()
+    }
+
+    override func tearDown() async throws {
+        try await self.app?.asyncShutdown()
+        self.app = nil
+    }
+
+    func getDb() throws -> Database {
+        guard let app = app else {
+            throw TestError()
+        }
+        return app.db
+    }
+
+
+}

swift-server/app-tests/importers/importer-n26.tests.swift

@@ -5,44 +5,11 @@ import XCTest
 
 @testable import App
 
-final class N26ImporterTests: XCTestCase {
-	var importerService: NewImportService!
-	var bankTransactionService: BankTransactionService!
-	var statusReportsService: StatusReportsService!
-	var group: UserGroup!
-	var app: Application?
-
-	func getTestFile(file: String) -> String {
-		let pwd = URL(fileURLWithPath: #file).pathComponents
-			.prefix(while: { $0 != "importers" }).joined(separator: "/").dropFirst()
-		return "\(pwd)/\(file)"
-	}
-
-	override func setUp() async throws {
-		let app = try await Application.make(.testing)
-		try await configure(app)
-		self.app = app
-
-		self.group = UserGroup(name: "Test User Group")
-		try await self.group.save(on: app.db)
-
-		let testParsers: [ParserFactory] = [N26Importer()]
-		importerService = NewImportService(parsers: testParsers)
-		bankTransactionService = BankTransactionService()
-		statusReportsService = StatusReportsService()
-	}
-
-	override func tearDown() async throws {
-		try await self.app?.asyncShutdown()
-		self.app = nil
-	}
-
-	func getDb() throws -> Database {
-		guard let app = app else {
-			throw TestError()
-		}
-		return app.db
-	}
+final class N26ImporterTests: BaseImporterTests {
+
+    override func getParsers() throws -> [any ParserFactory] {
+        return [N26Importer()]
+    }
 
 	func testN26Import() async throws {
 		let groupOwnerId = try self.group.requireID()

swift-server/app-tests/importers/importer-service.tests.swift

@@ -5,43 +5,13 @@ import XCTest
 
 @testable import App
 
-final class ImporterServiceTests: XCTestCase {
-	var importerService: NewImportService!
-	var bankTransactionService: BankTransactionService!
-	var statusReportsService: StatusReportsService!
-	var group: UserGroup!
-	var app: Application?
-
-	override func setUp() async throws {
-		// let testParsers: [ParserFactory] = [TestBasicImporter(), N26Importer(), CommerzBankEnImporter()]
-
-		let app = try await Application.make(.testing)
-		try await configure(app)
-		self.app = app
-
-		self.group = UserGroup(name: "Test User Group")
-		try await self.group.save(on: app.db)
-
-		let testParsers: [ParserFactory] = [
-			TestBasicImporter(),
-			TestBasicImporter(key: "test-invalid-data", data: [["a": "b"]]),
-		]
-		importerService = NewImportService(parsers: testParsers)
-		bankTransactionService = BankTransactionService()
-		statusReportsService = StatusReportsService()
-	}
-
-	override func tearDown() async throws {
-		try await self.app?.asyncShutdown()
-		self.app = nil
-	}
-
-	func getDb() throws -> Database {
-		guard let app = app else {
-			throw TestError()
-		}
-		return app.db
-	}
+final class ImporterServiceTests: BaseImporterTests {
+    override func getParsers() throws -> [any ParserFactory] {
+        return [
+            TestBasicImporter(),
+            TestBasicImporter(key: "test-invalid-data", data: [["a": "b"]]),
+        ]
+    }
 
 	func testImportEverythingFine() async throws {
 		let groupOwnerId = try self.group.requireID()

swift-server/app/importers/importers/commerz-bank.importer.swift

@@ -0,0 +1,92 @@
+import Foundation
+import Vapor
+import CSV
+
+class CommerzBankEnImporter: ParserFactory {
+	let DATE_REGEX = try! NSRegularExpression(pattern: "(?<year>\\d{4})-(?<month>\\d{2})-(?<day>\\d{2})T\\d{2}:\\d{2}:\\d{2}", options: [])
+    let transformHelper: TransformHelper<[String?]>
+    let key: String = "commerz-bank/en"
+    let fileRegex: String = "Umsaetze_KtoNr.*\\.CSV"
+
+    init() {
+        let fieldsMap = FieldsMap<Int>(
+            movementName: 9,
+            date: 11,
+            dateValue: 1,
+            details: 10, value: 4
+        )
+        self.transformHelper = TransformHelper(fieldsMap, dateFormat: "dd.MM.yyyy")
+    }
+
+    func splitMessage(_ msg: String) throws -> (String, String?, String?){
+        var message = msg
+
+        if message.hasPrefix("Auszahlung") {
+            message.removeFirst("Auszahlung".count)
+        } else if message.hasPrefix("Kartenzahlung") {
+            message.removeFirst("Kartenzahlung".count)
+        }
+
+        let dateMatch = DATE_REGEX.firstMatch(in: message, options: [], range: NSRange(location: 0, length: message.count))
+        if let dateMatch = dateMatch {
+            let range = Range(dateMatch.range(at: 0), in: message)
+            guard let range = range else {
+                throw Exception(.E10000)
+            }
+            var movementName = String(message[..<range.lowerBound])
+            let dayRange = Range(dateMatch.range(withName: "day"), in: message)
+            let monthRange = Range(dateMatch.range(withName: "month"), in: message)
+            let yearRange = Range(dateMatch.range(withName: "year"), in: message)
+            let dateInfo = "\(message[dayRange!]).\(message[monthRange!]).\(message[yearRange!])"
+            var details : String?
+            if let slashIndex = movementName.firstIndex(of: "/") {
+                details = String(movementName[slashIndex...]).trimmingCharacters(in: .whitespaces)
+                movementName.removeSubrange(slashIndex...)
+            }
+
+            return (movementName.trimmingCharacters(in: .whitespaces), details, dateInfo)
+        } else if let strRange = message.range(of: "End-to-End-Ref") {
+            let splitMsg = message[..<strRange.lowerBound]
+            return (splitMsg.trimmingCharacters(in: .whitespaces), nil, nil)
+        }
+
+        return (message.trimmingCharacters(in: .whitespaces), nil, nil)
+    }
+
+    func create(filePath: String) -> AsyncThrowingStream<PartialBankTransaction, Error> {
+        AsyncThrowingStream { continuation in
+            Task {
+                do {
+                    let csv = try parseCsv(filePath: filePath)
+
+                    var lineCounter = 0
+                    while let row = csv.next() {
+                        lineCounter += 1
+                        do {
+                            var mappedData : [String?] = row
+
+                            let (movementName, details, newDate) = try splitMessage(mappedData[3] ?? "")
+
+                            mappedData.append(movementName)
+                            mappedData.append(details)
+                            if let newDate = newDate {
+                                mappedData.append(newDate)
+                            } else {
+                                mappedData.append(row[0])
+                            }
+                            let transaction = try self.transformHelper.map(mappedData)
+                            continuation.yield(transaction)
+                        } catch {
+                            throw Exception(.E10011, context: ["line": lineCounter], cause: error)
+                        }
+                    }
+
+                    continuation.finish()
+                } catch {
+                    continuation.finish(throwing: error)
+                }
+            }
+        }
+    }
+}
+

todo.md

@@ -39,12 +39,14 @@
   * [ ] Fix ui
 * [ ] Imports
   * [x] Import caixa enginyers
-  * [ ] Import n26
-  * [ ] Import commerzbank
+  * [x] Import n26
+  * [x] Import commerzbank
   * [ ] Import qif
+  * [ ] Crud api
   * [ ] Call to apply rules
 * [ ] Bank transactions
   * [x] get bank transactions data
+  * [] add/remove label
 * [ ] labels
   * [x] get labels
   * [ ] create label