diff --git a/jobs/payment-jobs/poetry.lock b/jobs/payment-jobs/poetry.lock index e3398d1f4..cab220502 100644 --- a/jobs/payment-jobs/poetry.lock +++ b/jobs/payment-jobs/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -2021,9 +2021,9 @@ werkzeug = "3.0.3" [package.source] type = "git" -url = "https://github.com/bcgov/sbc-pay.git" -reference = "feature/22263" -resolved_reference = "86f23886af96d7f361165ae32ff71396d1bf2183" +url = "https://github.com/seeker25/sbc-pay.git" +reference = "21519" +resolved_reference = "468f2b01d32a4f9dd0c43a649203a0b5bf713f50" subdirectory = "pay-api" [[package]] @@ -3174,4 +3174,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.12" -content-hash = "9dea67bbde426d21457ec8325937cdc37b09ad9dae99454fedf751914576f06b" +content-hash = "9e7d3948bdc3f84687c150ac4034b11992b08382a3198254065a392e85a9a13a" diff --git a/jobs/payment-jobs/pyproject.toml b/jobs/payment-jobs/pyproject.toml index 2b43b8db2..7514b60b8 100644 --- a/jobs/payment-jobs/pyproject.toml +++ b/jobs/payment-jobs/pyproject.toml @@ -7,7 +7,7 @@ readme = "README.md" [tool.poetry.dependencies] python = "^3.12" -pay-api = {git = "https://github.com/bcgov/sbc-pay.git", subdirectory = "pay-api", branch = "feature/22263"} +pay-api = {git = "https://github.com/seeker25/sbc-pay.git", branch = "21519", subdirectory = "pay-api"} flask = "^3.0.2" flask-sqlalchemy = "^3.1.1" sqlalchemy = "^2.0.28" diff --git a/jobs/payment-jobs/tasks/ap_task.py b/jobs/payment-jobs/tasks/ap_task.py index 9e0b22e0c..0ddad3b2d 100644 --- a/jobs/payment-jobs/tasks/ap_task.py +++ b/jobs/payment-jobs/tasks/ap_task.py @@ -14,7 +14,7 @@ """Task to create AP file for FAS refunds and Disbursement via EFT for non-government orgs without a GL.""" import time -from datetime import date, datetime, timezone +from datetime import date, datetime, timedelta, timezone from typing import List from flask import current_app @@ -29,15 +29,17 @@ from pay_api.models import EjvHeader as EjvHeaderModel from pay_api.models import EjvLink as EjvLinkModel from pay_api.models import Invoice as InvoiceModel +from pay_api.models import Receipt as ReceiptModel from pay_api.models import Refund as RefundModel from pay_api.models import RoutingSlip as RoutingSlipModel from pay_api.models import db from pay_api.utils.enums import ( - DisbursementStatus, EFTShortnameRefundStatus, EjvFileType, EJVLinkType, RoutingSlipStatus) + DisbursementStatus, EFTShortnameRefundStatus, EjvFileType, EJVLinkType, InvoiceStatus, PaymentMethod, + RoutingSlipStatus) +from sqlalchemy import Date, cast from tasks.common.cgi_ap import CgiAP from tasks.common.dataclasses import APLine -from tasks.ejv_partner_distribution_task import EjvPartnerDistributionTask class ApTask(CgiAP): @@ -166,13 +168,52 @@ def _create_routing_slip_refund_file(cls): # pylint:disable=too-many-locals, to cls._create_file_and_upload(ap_content) + @classmethod + def get_invoices_for_disbursement(cls, partner): + """Return invoices for disbursement. Used by EJV and AP.""" + disbursement_date = datetime.now(tz=timezone.utc).replace(tzinfo=None) \ + - timedelta(days=current_app.config.get('DISBURSEMENT_DELAY_IN_DAYS')) + invoices: List[InvoiceModel] = db.session.query(InvoiceModel) \ + .filter(InvoiceModel.invoice_status_code == InvoiceStatus.PAID.value) \ + .filter( + InvoiceModel.payment_method_code.notin_([PaymentMethod.INTERNAL.value, + PaymentMethod.DRAWDOWN.value, + PaymentMethod.EFT.value])) \ + .filter((InvoiceModel.disbursement_status_code.is_(None)) | + (InvoiceModel.disbursement_status_code == DisbursementStatus.ERRORED.value)) \ + .filter(~InvoiceModel.receipts.any(cast(ReceiptModel.receipt_date, Date) >= disbursement_date.date())) \ + .filter(InvoiceModel.corp_type_code == partner.code) \ + .all() + current_app.logger.info(invoices) + return invoices + + @classmethod + def get_invoices_for_refund_reversal(cls, partner): + """Return invoices for refund reversal.""" + # REFUND_REQUESTED for credit card payments, CREDITED for AR and REFUNDED for other payments. + refund_inv_statuses = (InvoiceStatus.REFUNDED.value, InvoiceStatus.REFUND_REQUESTED.value, + InvoiceStatus.CREDITED.value) + + invoices: List[InvoiceModel] = db.session.query(InvoiceModel) \ + .filter(InvoiceModel.invoice_status_code.in_(refund_inv_statuses)) \ + .filter( + InvoiceModel.payment_method_code.notin_([PaymentMethod.INTERNAL.value, + PaymentMethod.DRAWDOWN.value, + PaymentMethod.EFT.value])) \ + .filter(InvoiceModel.disbursement_status_code == DisbursementStatus.COMPLETED.value) \ + .filter(InvoiceModel.corp_type_code == partner.code) \ + .all() + current_app.logger.info(invoices) + return invoices + @classmethod def _create_non_gov_disbursement_file(cls): # pylint:disable=too-many-locals """Create AP file for disbursement for non government entities without a GL code via EFT and upload to CGI.""" cls.ap_type = EjvFileType.NON_GOV_DISBURSEMENT bca_partner = CorpTypeModel.find_by_code('BCA') - total_invoices: List[InvoiceModel] = EjvPartnerDistributionTask().get_invoices_for_disbursement(bca_partner) + \ - EjvPartnerDistributionTask().get_invoices_for_refund_reversal(bca_partner) + # TODO these two functions need to be reworked when we onboard BCA again. + total_invoices: List[InvoiceModel] = cls.get_invoices_for_disbursement(bca_partner) + \ + cls.get_invoices_for_refund_reversal(bca_partner) current_app.logger.info(f'Found {len(total_invoices)} to disburse.') if not total_invoices: @@ -228,10 +269,10 @@ def _create_non_gov_disbursement_file(cls): # pylint:disable=too-many-locals @classmethod def _create_file_and_upload(cls, ap_content): - file_path_with_name, trg_file_path = cls.create_inbox_and_trg_files(ap_content) - cls.upload(ap_content, cls.get_file_name(), file_path_with_name, trg_file_path) + file_path_with_name, trg_file_path, file_name = cls.create_inbox_and_trg_files(ap_content) + cls.upload(ap_content, file_name, file_path_with_name, trg_file_path) db.session.commit() - # Add a sleep to prevent collision on file name. + # Sleep to prevent collision on file name. time.sleep(1) @classmethod diff --git a/jobs/payment-jobs/tasks/common/cgi_ejv.py b/jobs/payment-jobs/tasks/common/cgi_ejv.py index 90700bef8..41804afee 100644 --- a/jobs/payment-jobs/tasks/common/cgi_ejv.py +++ b/jobs/payment-jobs/tasks/common/cgi_ejv.py @@ -148,7 +148,8 @@ def get_trg_suffix(cls): def create_inbox_and_trg_files(cls, ejv_content): """Create inbox and trigger files.""" file_path: str = tempfile.gettempdir() - file_path_with_name = f'{file_path}/{cls.get_file_name()}' + file_name = cls.get_file_name() + file_path_with_name = f'{file_path}/{file_name}' trg_file_path = f'{file_path_with_name}.{cls.get_trg_suffix()}' with open(file_path_with_name, 'a+', encoding='utf-8') as jv_file: jv_file.write(ejv_content) @@ -157,4 +158,4 @@ def create_inbox_and_trg_files(cls, ejv_content): with open(trg_file_path, 'a+', encoding='utf-8') as trg_file: trg_file.write('') trg_file.close() - return file_path_with_name, trg_file_path + return file_path_with_name, trg_file_path, file_name diff --git a/jobs/payment-jobs/tasks/common/dataclasses.py b/jobs/payment-jobs/tasks/common/dataclasses.py index 8c03d2fb1..2e260971a 100644 --- a/jobs/payment-jobs/tasks/common/dataclasses.py +++ b/jobs/payment-jobs/tasks/common/dataclasses.py @@ -12,15 +12,41 @@ # See the License for the specific language governing permissions and # limitations under the License. """Common dataclasses for tasks, dataclasses allow for cleaner code with autocompletion in vscode.""" +from decimal import Decimal + from dataclasses import dataclass from typing import List, Optional from dataclass_wizard import JSONWizard +from pay_api.models import DistributionCode as DistributionCodeModel from pay_api.models import Invoice as InvoiceModel +from pay_api.models import PartnerDisbursements as PartnerDisbursementModel from pay_api.models import PaymentLineItem as LineItemModel from pay_api.utils.enums import InvoiceStatus from tasks.common.enums import PaymentDetailsGlStatus +@dataclass +class DisbursementLineItem: + """DTO mapping for disbursement line item.""" + + amount: Decimal + flow_through: str + description_identifier: str + is_reversal: bool + target_type: str + identifier: int + + +@dataclass +class Disbursement: + """DTO mapping for disbursement.""" + + bcreg_distribution_code: DistributionCodeModel + partner_distribution_code: DistributionCodeModel + line_item: DisbursementLineItem + target: InvoiceModel | PartnerDisbursementModel + + @dataclass class RefundData(JSONWizard): """Refund data from order status query.""" diff --git a/jobs/payment-jobs/tasks/ejv_partner_distribution_task.py b/jobs/payment-jobs/tasks/ejv_partner_distribution_task.py index 3bf434559..2ecad9a15 100644 --- a/jobs/payment-jobs/tasks/ejv_partner_distribution_task.py +++ b/jobs/payment-jobs/tasks/ejv_partner_distribution_task.py @@ -17,6 +17,7 @@ from datetime import datetime, timedelta, timezone from typing import List +from decimal import Decimal from flask import current_app from pay_api.models import CorpType as CorpTypeModel from pay_api.models import DistributionCode as DistributionCodeModel @@ -26,14 +27,18 @@ from pay_api.models import EjvLink as EjvLinkModel from pay_api.models import FeeSchedule as FeeScheduleModel from pay_api.models import Invoice as InvoiceModel +from pay_api.models import PartnerDisbursements as PartnerDisbursementsModel from pay_api.models import PaymentLineItem as PaymentLineItemModel from pay_api.models import Receipt as ReceiptModel from pay_api.models import db from pay_api.utils.enums import DisbursementStatus, EjvFileType, EJVLinkType, InvoiceStatus, PaymentMethod -from sqlalchemy import Date, cast, func, select -from sqlalchemy.dialects.postgresql import ARRAY, INTEGER +from sqlalchemy import Date, and_, cast from tasks.common.cgi_ejv import CgiEjv +from tasks.common.dataclasses import Disbursement, DisbursementLineItem + +# Just a warning for this code, there aren't decent unit tests that test this. If you're changing this job, you'll need +# to do a side by side file comparison to previous versions to ensure that the changes are correct. class EjvPartnerDistributionTask(CgiEjv): @@ -44,7 +49,7 @@ def create_ejv_file(cls): """Create JV files and upload to CGI. Steps: - 1. Find all invoices from invoice table for disbursements. + 1. Find all invoices/partial refunds/EFT reversals for disbursements. 2. Group by fee schedule and create JV Header and JV Details. 3. Upload the file to minio for future reference. 4. Upload to sftp for processing. First upload JV file and then a TRG file. @@ -54,167 +59,174 @@ def create_ejv_file(cls): cls._create_ejv_file_for_partner(batch_type='GA') # External ministry @staticmethod - def get_invoices_for_disbursement(partner): - """Return invoices for disbursement. Used by EJV and AP.""" + def get_disbursement_by_distribution_for_partner(partner): + """Return disbursements dataclass for partners.""" + # Internal invoices aren't disbursed to partners, DRAWDOWN is handled by the mainframe. + # EFT is handled by the PartnerDisbursements table. + # ##################################################### Original (Legacy way) - invoice.disbursement_status_code + # Eventually we'll abandon this and use the PartnerDisbursements table for all disbursements. + # We'd need a migration and more changes to move it to the table. + skip_payment_methods = [PaymentMethod.INTERNAL.value, PaymentMethod.DRAWDOWN.value, PaymentMethod.EFT.value] disbursement_date = datetime.now(tz=timezone.utc).replace(tzinfo=None) - \ timedelta(days=current_app.config.get( 'DISBURSEMENT_DELAY_IN_DAYS')) - invoices: List[InvoiceModel] = db.session.query(InvoiceModel) \ - .filter(InvoiceModel.invoice_status_code == InvoiceStatus.PAID.value) \ - .filter( - InvoiceModel.payment_method_code.notin_([PaymentMethod.INTERNAL.value, - PaymentMethod.DRAWDOWN.value, - PaymentMethod.EFT.value])) \ - .filter((InvoiceModel.disbursement_status_code.is_(None)) | - (InvoiceModel.disbursement_status_code == DisbursementStatus.ERRORED.value)) \ - .filter(~InvoiceModel.receipts.any(cast(ReceiptModel.receipt_date, Date) >= disbursement_date.date())) \ + base_query = db.session.query(InvoiceModel, PaymentLineItemModel, DistributionCodeModel) \ + .join(PaymentLineItemModel, PaymentLineItemModel.invoice_id == InvoiceModel.id) \ + .join(DistributionCodeModel, + DistributionCodeModel.distribution_code_id == PaymentLineItemModel.fee_distribution_id) \ + .filter(InvoiceModel.payment_method_code.notin_(skip_payment_methods)) \ .filter(InvoiceModel.corp_type_code == partner.code) \ + .filter(PaymentLineItemModel.total > 0) \ + .filter(DistributionCodeModel.stop_ejv.is_(False) | DistributionCodeModel.stop_ejv.is_(None)) \ + .order_by(DistributionCodeModel.distribution_code_id, PaymentLineItemModel.id) + + transactions = base_query.filter((InvoiceModel.disbursement_status_code.is_(None)) | + (InvoiceModel.disbursement_status_code == DisbursementStatus.ERRORED.value)) \ + .filter(~InvoiceModel.receipts.any(cast(ReceiptModel.receipt_date, Date) >= disbursement_date.date())) \ + .filter(InvoiceModel.invoice_status_code == InvoiceStatus.PAID.value) \ .all() - current_app.logger.info(invoices) - return invoices - @classmethod - def get_invoices_for_refund_reversal(cls, partner): - """Return invoices for refund reversal.""" # REFUND_REQUESTED for credit card payments, CREDITED for AR and REFUNDED for other payments. - refund_inv_statuses = (InvoiceStatus.REFUNDED.value, InvoiceStatus.REFUND_REQUESTED.value, - InvoiceStatus.CREDITED.value) - - invoices: List[InvoiceModel] = db.session.query(InvoiceModel) \ - .filter(InvoiceModel.invoice_status_code.in_(refund_inv_statuses)) \ - .filter( - InvoiceModel.payment_method_code.notin_([PaymentMethod.INTERNAL.value, - PaymentMethod.DRAWDOWN.value, - PaymentMethod.EFT.value])) \ + reversals = base_query.filter(InvoiceModel.invoice_status_code.in_([InvoiceStatus.REFUNDED.value, + InvoiceStatus.REFUND_REQUESTED.value, + InvoiceStatus.CREDITED.value])) \ .filter(InvoiceModel.disbursement_status_code == DisbursementStatus.COMPLETED.value) \ - .filter(InvoiceModel.corp_type_code == partner.code) \ .all() - current_app.logger.info(invoices) - return invoices + + disbursement_rows = [] + distribution_code_totals = {} + for invoice, payment_line_item, distribution_code in transactions + reversals: + distribution_code_totals.setdefault(distribution_code.distribution_code_id, 0) + distribution_code_totals[distribution_code.distribution_code_id] += payment_line_item.total + disbursement_rows.append(Disbursement( + bcreg_distribution_code=distribution_code, + partner_distribution_code=distribution_code.disbursement_distribution_code, + target=invoice, + line_item=DisbursementLineItem( + amount=payment_line_item.total, + flow_through=f'{invoice.id:<110}', + description_identifier=f'#{invoice.id}', + is_reversal=invoice.invoice_status_code in [InvoiceStatus.REFUNDED.value, + InvoiceStatus.REFUND_REQUESTED.value, + InvoiceStatus.CREDITED.value], + target_type=EJVLinkType.INVOICE.value, + identifier=invoice.id + ) + )) + # ################################################################# END OF Legacy way of handling disbursements. + # Partner disbursements - New + # Partial refunds need to be added to here later, although they should be fairly rare as most of them are from + # NRO (NRO is internal, meaning no disbursement needed.) + partner_disbursements = db.session.query(PartnerDisbursementsModel, + PaymentLineItemModel, + DistributionCodeModel) \ + .join(PaymentLineItemModel, and_(PaymentLineItemModel.invoice_id == PartnerDisbursementsModel.target_id, + PartnerDisbursementsModel.target_type == EJVLinkType.INVOICE.value)) \ + .join(DistributionCodeModel, + DistributionCodeModel.distribution_code_id == PaymentLineItemModel.fee_distribution_id) \ + .filter(PartnerDisbursementsModel.status_code == DisbursementStatus.WAITING_FOR_RECEIPT.value) \ + .filter(PartnerDisbursementsModel.partner_code == partner.code) \ + .filter(DistributionCodeModel.stop_ejv.is_(False) | DistributionCodeModel.stop_ejv.is_(None)) \ + .filter(~InvoiceModel.receipts.any(cast(ReceiptModel.receipt_date, Date) >= disbursement_date.date())) \ + .order_by(DistributionCodeModel.distribution_code_id, PaymentLineItemModel.id) \ + .all() + + for partner_disbursement, payment_line_item, distribution_code in partner_disbursements: + suffix = 'PR' if partner_disbursement.target_type == EJVLinkType.PARTIAL_REFUND else '' + flow_through = f'{payment_line_item.invoice_id}-{partner_disbursement.id}' + if suffix != '': + flow_through += f'-{suffix}' + distribution_code_totals.setdefault(distribution_code.distribution_code_id, 0) + distribution_code_totals[distribution_code.distribution_code_id] += partner_disbursement.amount + disbursement_rows.append(Disbursement( + bcreg_distribution_code=distribution_code, + partner_distribution_code=distribution_code.disbursement_distribution_code, + target=partner_disbursement, + line_item=DisbursementLineItem( + amount=partner_disbursement.amount, + flow_through=flow_through, + description_identifier='#' + flow_through, + is_reversal=partner_disbursement.is_reversal, + target_type=partner_disbursement.target_type, + identifier=partner_disbursement.target_id + ) + )) + disbursement_rows.sort(key=lambda x: x.bcreg_distribution_code.distribution_code_id) + return disbursement_rows, distribution_code_totals @classmethod def _create_ejv_file_for_partner(cls, batch_type: str): # pylint:disable=too-many-locals, too-many-statements """Create EJV file for the partner and upload.""" - ejv_content: str = '' - batch_total: float = 0 - control_total: int = 0 + ejv_content, batch_total, control_total = '', Decimal('0'), Decimal('0') today = datetime.now(tz=timezone.utc) disbursement_desc = current_app.config.get('CGI_DISBURSEMENT_DESC'). \ format(today.strftime('%B').upper(), f'{today.day:0>2}')[:100] disbursement_desc = f'{disbursement_desc:<100}' - - # Create a ejv file model record. - ejv_file_model: EjvFileModel = EjvFileModel( + ejv_file_model = EjvFileModel( file_type=EjvFileType.DISBURSEMENT.value, file_ref=cls.get_file_name(), disbursement_status_code=DisbursementStatus.UPLOADED.value ).flush() batch_number = cls.get_batch_number(ejv_file_model.id) - - # Get partner list. Each of the partner will go as a JV Header and transactions as JV Details. - partners = cls._get_partners_by_batch_type(batch_type) - current_app.logger.info(partners) - - # JV Batch Header - batch_header: str = cls.get_batch_header(batch_number, batch_type) - - for partner in partners: - # Find all invoices for the partner to disburse. - # This includes invoices which are not PAID and invoices which are refunded. - payment_invoices = cls.get_invoices_for_disbursement(partner) - refund_reversals = cls.get_invoices_for_refund_reversal(partner) - invoices = payment_invoices + refund_reversals - # If no invoices continue. - if not invoices: + batch_header = cls.get_batch_header(batch_number, batch_type) + effective_date = cls.get_effective_date() + # Each of the partner will go as a JV Header and transactions as JV Details. + for partner in cls._get_partners_by_batch_type(batch_type): + current_app.logger.info(partner) + disbursements, distribution_code_totals = cls.get_disbursement_by_distribution_for_partner(partner) + if not disbursements: continue - effective_date: str = cls.get_effective_date() - # Construct journal name - ejv_header_model: EjvFileModel = EjvHeaderModel( + ejv_header_model = EjvHeaderModel( partner_code=partner.code, disbursement_status_code=DisbursementStatus.UPLOADED.value, ejv_file_id=ejv_file_model.id ).flush() - journal_name: str = cls.get_journal_name(ejv_header_model.id) - - # To populate JV Header and JV Details, group these invoices by the distribution - # and create one JV Header and detail for each. - distribution_code_set = set() - invoice_id_list = [] - for inv in invoices: - invoice_id_list.append(inv.id) - for line_item in inv.payment_line_items: - distribution_code_set.add(line_item.fee_distribution_id) - - for distribution_code_id in list(distribution_code_set): - distribution_code: DistributionCodeModel = DistributionCodeModel.find_by_id(distribution_code_id) - credit_distribution_code: DistributionCodeModel = DistributionCodeModel.find_by_id( - distribution_code.disbursement_distribution_code_id - ) - if credit_distribution_code.stop_ejv: - continue - - line_items = cls._find_line_items_by_invoice_and_distribution(distribution_code_id, invoice_id_list) - - total: float = 0 - for line in line_items: - total += line.total - - batch_total += total - - debit_distribution = cls.get_distribution_string(distribution_code) # Debit from BCREG GL - credit_distribution = cls.get_distribution_string(credit_distribution_code) # Credit to partner GL - - # JV Header - ejv_content = '{}{}'.format(ejv_content, # pylint:disable=consider-using-f-string - cls.get_jv_header(batch_type, cls.get_journal_batch_name(batch_number), - journal_name, total)) - control_total += 1 - - line_number: int = 0 - for line in line_items: - # JV Details - line_number += 1 - # Flow Through add it as the invoice id. - flow_through = f'{line.invoice_id:<110}' - # debit_distribution and credit_distribution stays as is for invoices which are not PAID - # For reversals, we just need to reverse the debit and credit. - is_reversal = InvoiceModel.find_by_id(line.invoice_id).invoice_status_code in \ - (InvoiceStatus.REFUNDED.value, - InvoiceStatus.REFUND_REQUESTED.value, - InvoiceStatus.CREDITED.value) + journal_name = cls.get_journal_name(ejv_header_model.id) + sequence = 1 - invoice_number = f'#{line.invoice_id}' - description = disbursement_desc[:-len(invoice_number)] + invoice_number - description = f'{description[:100]:<100}' + last_distribution_code = None + line_number = 1 + for disbursement in disbursements: + # debit_distribution and credit_distribution stays as is for invoices which are not PAID + if last_distribution_code != disbursement.bcreg_distribution_code.distribution_code_id: + header_total = distribution_code_totals[disbursement.bcreg_distribution_code.distribution_code_id] ejv_content = '{}{}'.format(ejv_content, # pylint:disable=consider-using-f-string - cls.get_jv_line(batch_type, credit_distribution, description, - effective_date, flow_through, journal_name, line.total, - line_number, 'C' if not is_reversal else 'D')) - line_number += 1 + cls.get_jv_header(batch_type, cls.get_journal_batch_name(batch_number), + journal_name, header_total)) control_total += 1 - - # Add a line here for debit too - ejv_content = '{}{}'.format(ejv_content, # pylint:disable=consider-using-f-string - cls.get_jv_line(batch_type, debit_distribution, description, - effective_date, flow_through, journal_name, line.total, - line_number, 'D' if not is_reversal else 'C')) - + last_distribution_code = disbursement.bcreg_distribution_code.distribution_code_id + line_number = 1 + + batch_total += disbursement.line_item.amount + dl = disbursement.line_item + description = disbursement_desc[:-len(dl.description_identifier)] + dl.description_identifier + description = f'{description[:100]:<100}' + for credit_debit_row in range(1, 3): + target_distribution = cls.get_distribution_string( + disbursement.partner_distribution_code if credit_debit_row == 1 else + disbursement.bcreg_distribution_code + ) + # For payment flow, credit the GL partner code, debit the BCREG GL code. + # Reversal is the opposite debit the GL partner code, credit the BCREG GL Code. + credit_debit = 'C' if credit_debit_row == 1 else 'D' + if dl.is_reversal is True: + credit_debit = 'D' if credit_debit == 'C' else 'C' + jv_line = cls.get_jv_line(batch_type, + target_distribution, + description, + effective_date, + f'{dl.flow_through:<110}', + journal_name, + dl.amount, + line_number, + credit_debit) + ejv_content = '{}{}'.format(ejv_content, jv_line) # pylint:disable=consider-using-f-string + line_number += 1 control_total += 1 - sequence = 1 - # Create ejv invoice link records and set invoice status - for inv in invoices: - # Create Ejv file link and flush - link_model = EjvLinkModel(link_id=inv.id, - ejv_header_id=ejv_header_model.id, - disbursement_status_code=DisbursementStatus.UPLOADED.value, - sequence=sequence, - link_type=EJVLinkType.INVOICE.value) - # Set distribution status to invoice - db.session.add(link_model) + cls._update_disbursement_status_and_ejv_link(disbursement, ejv_header_model, sequence) sequence += 1 - inv.disbursement_status_code = DisbursementStatus.UPLOADED.value db.session.flush() @@ -222,32 +234,39 @@ def _create_ejv_file_for_partner(cls, batch_type: str): # pylint:disable=too-ma db.session.rollback() return - # JV Batch Trailer - jv_batch_trailer: str = cls.get_batch_trailer(batch_number, batch_total, batch_type, control_total) - + jv_batch_trailer = cls.get_batch_trailer(batch_number, batch_total, batch_type, control_total) ejv_content = f'{batch_header}{ejv_content}{jv_batch_trailer}' - # Create a file add this content. - file_path_with_name, trg_file_path = cls.create_inbox_and_trg_files(ejv_content) - - # Upload file and trg to FTP - cls.upload(ejv_content, cls.get_file_name(), file_path_with_name, trg_file_path) + file_path_with_name, trg_file_path, file_name = cls.create_inbox_and_trg_files(ejv_content) + cls.upload(ejv_content, file_name, file_path_with_name, trg_file_path) - # commit changes to DB db.session.commit() - # Add a sleep to prevent collision on file name. + # To prevent collision on file name. time.sleep(1) @classmethod - def _find_line_items_by_invoice_and_distribution(cls, distribution_code_id, invoice_id_list) \ - -> List[PaymentLineItemModel]: - """Find and return all payment line items for this distribution.""" - invoice_id_list = select(func.unnest(cast(invoice_id_list, ARRAY(INTEGER)))) - line_items: List[PaymentLineItemModel] = db.session.query(PaymentLineItemModel) \ - .filter(PaymentLineItemModel.invoice_id.in_(invoice_id_list)) \ - .filter(PaymentLineItemModel.total > 0) \ - .filter(PaymentLineItemModel.fee_distribution_id == distribution_code_id) - return line_items + def _update_disbursement_status_and_ejv_link(cls, + disbursement: Disbursement, + ejv_header_model: EjvHeaderModel, + sequence: int): + """Update disbursement status and create EJV Link.""" + if isinstance(disbursement.target, InvoiceModel): + disbursement.target.disbursement_status_code = DisbursementStatus.UPLOADED.value + elif isinstance(disbursement.target, PartnerDisbursementsModel): + # Only EFT is using partner disbursements table for now, eventually we want to move our disbursement + # process over to something similar: Where we have an entire table setup that + # is used to track disbursements, instead of just the three column approach that + # doesn't work when there are multiple reversals etc. + disbursement.target.status_code = DisbursementStatus.UPLOADED.value + disbursement.target.processed_on = datetime.now(tz=timezone.utc) + else: + raise NotImplementedError('Unknown disbursement type') + + db.session.add(EjvLinkModel(link_id=disbursement.line_item.identifier, + link_type=disbursement.line_item.target_type, + ejv_header_id=ejv_header_model.id, + disbursement_status_code=DisbursementStatus.UPLOADED.value, + sequence=sequence)) @classmethod def _get_partners_by_batch_type(cls, batch_type) -> List[CorpTypeModel]: @@ -255,29 +274,25 @@ def _get_partners_by_batch_type(cls, batch_type) -> List[CorpTypeModel]: # CREDIT : Ministry GL code -> disbursement_distribution_code_id on distribution_codes table # DEBIT : BC Registry GL Code -> distribution_code on fee_schedule, starts with 112 bc_reg_client_code = current_app.config.get('CGI_BCREG_CLIENT_CODE') # 112 + # Rule for GA. Credit is 112 and debit is 112. + # Rule for GI. Debit is 112 and credit is not 112. query = db.session.query(DistributionCodeModel.distribution_code_id) \ .filter(DistributionCodeModel.stop_ejv.is_(False) | DistributionCodeModel.stop_ejv.is_(None)) \ .filter(DistributionCodeModel.account_id.is_(None)) \ - .filter(DistributionCodeModel.disbursement_distribution_code_id.is_(None)) - - if batch_type == 'GA': - # Rule for GA. Credit is 112 and debit is 112. - partner_distribution_code_ids: List[int] = query.filter( - DistributionCodeModel.client == bc_reg_client_code - ) - else: - # Rule for GI. Debit is 112 and credit is not 112. - partner_distribution_code_ids: List[int] = query.filter( - DistributionCodeModel.client != bc_reg_client_code - ) + .filter(DistributionCodeModel.disbursement_distribution_code_id.is_(None)) \ + .filter_boolean(batch_type == 'GA', DistributionCodeModel.client == bc_reg_client_code) \ + .filter_boolean(batch_type == 'GI', DistributionCodeModel.client != bc_reg_client_code) # Find all distribution codes who have these partner distribution codes as disbursement. - fee_distribution_codes: List[int] = db.session.query(DistributionCodeModel.distribution_code_id).filter( - DistributionCodeModel.disbursement_distribution_code_id.in_(partner_distribution_code_ids)) - - corp_type_codes: List[str] = db.session.query(FeeScheduleModel.corp_type_code). \ - join(DistributionCodeLinkModel, - DistributionCodeLinkModel.fee_schedule_id == FeeScheduleModel.fee_schedule_id). \ - filter(DistributionCodeLinkModel.distribution_code_id.in_(fee_distribution_codes)) - - return db.session.query(CorpTypeModel).filter(CorpTypeModel.code.in_(corp_type_codes)).all() + partner_distribution_codes = db.session.query(DistributionCodeModel.distribution_code_id).filter( + DistributionCodeModel.disbursement_distribution_code_id.in_(query)) + + corp_type_query = db.session.query(FeeScheduleModel.corp_type_code) \ + .join(DistributionCodeLinkModel, + DistributionCodeLinkModel.fee_schedule_id == FeeScheduleModel.fee_schedule_id) \ + .filter(DistributionCodeLinkModel.distribution_code_id.in_(partner_distribution_codes)) + + result = db.session.query(CorpTypeModel) \ + .filter(CorpTypeModel.has_partner_disbursements.is_(True)) \ + .filter(CorpTypeModel.code.in_(corp_type_query)).all() + return result diff --git a/jobs/payment-jobs/tasks/ejv_payment_task.py b/jobs/payment-jobs/tasks/ejv_payment_task.py index fec001c52..ca90bad01 100644 --- a/jobs/payment-jobs/tasks/ejv_payment_task.py +++ b/jobs/payment-jobs/tasks/ejv_payment_task.py @@ -57,7 +57,6 @@ def _create_ejv_file_for_gov_account(cls, batch_type: str): # pylint:disable=to batch_total: float = 0 control_total: int = 0 - # Create a ejv file model record. ejv_file_model: EjvFileModel = EjvFileModel( file_type=EjvFileType.PAYMENT.value, file_ref=cls.get_file_name(), @@ -65,10 +64,8 @@ def _create_ejv_file_for_gov_account(cls, batch_type: str): # pylint:disable=to ).flush() batch_number = cls.get_batch_number(ejv_file_model.id) - # Get all invoices which should be part of the batch type. account_ids = cls._get_account_ids_for_payment(batch_type) - # JV Batch Header batch_header: str = cls.get_batch_header(batch_number, batch_type) current_app.logger.info('Processing accounts.') @@ -77,20 +74,17 @@ def _create_ejv_file_for_gov_account(cls, batch_type: str): # pylint:disable=to # Find all invoices for the gov account to pay. invoices = cls._get_invoices_for_payment(account_id) pay_account: PaymentAccountModel = PaymentAccountModel.find_by_id(account_id) - # If no invoices continue. if not invoices or not pay_account.billable: continue disbursement_desc = f'{pay_account.name[:100]:<100}' effective_date: str = cls.get_effective_date() - # Construct journal name ejv_header_model: EjvFileModel = EjvHeaderModel( payment_account_id=account_id, disbursement_status_code=DisbursementStatus.UPLOADED.value, ejv_file_id=ejv_file_model.id ).flush() journal_name: str = cls.get_journal_name(ejv_header_model.id) - # Distribution code for the account. debit_distribution_code: DistributionCodeModel = DistributionCodeModel.find_by_active_for_account( account_id ) @@ -125,7 +119,7 @@ def _create_ejv_file_for_gov_account(cls, batch_type: str): # pylint:disable=to total += line.total line_distribution = cls.get_distribution_string(line_distribution_code) flow_through = f'{line.invoice_id:<110}' - # Credit to BCREG GL + # Credit to BCREG GL for a transaction (non-reversal) line_number += 1 control_total += 1 # If it's normal payment then the Line distribution goes as Credit, @@ -135,7 +129,7 @@ def _create_ejv_file_for_gov_account(cls, batch_type: str): # pylint:disable=to line.total, line_number, 'C' if not is_jv_reversal else 'D') - # Debit from GOV ACCOUNT GL + # Debit from GOV ACCOUNT GL for a transaction (non-reversal) line_number += 1 control_total += 1 # If it's normal payment then the Gov account GL goes as Debit, @@ -150,7 +144,7 @@ def _create_ejv_file_for_gov_account(cls, batch_type: str): # pylint:disable=to total += line.service_fees service_fee_distribution = cls.get_distribution_string(service_fee_distribution_code) flow_through = f'{line.invoice_id:<110}' - # Credit to BCREG GL + # Credit to BCREG GL for a transaction (non-reversal) line_number += 1 control_total += 1 account_jv = account_jv + cls.get_jv_line(batch_type, service_fee_distribution, @@ -159,7 +153,7 @@ def _create_ejv_file_for_gov_account(cls, batch_type: str): # pylint:disable=to line.service_fees, line_number, 'C' if not is_jv_reversal else 'D') - # Debit from GOV ACCOUNT GL + # Debit from GOV ACCOUNT GL for a transaction (non-reversal) line_number += 1 control_total += 1 account_jv = account_jv + cls.get_jv_line(batch_type, debit_distribution, description, @@ -168,7 +162,6 @@ def _create_ejv_file_for_gov_account(cls, batch_type: str): # pylint:disable=to line_number, 'D' if not is_jv_reversal else 'C') batch_total += total - # Skip if we have no total from the invoices. if total > 0: # A JV header for each account. control_total += 1 @@ -176,7 +169,6 @@ def _create_ejv_file_for_gov_account(cls, batch_type: str): # pylint:disable=to journal_name, total) + account_jv ejv_content = ejv_content + account_jv - # Create ejv invoice link records and set invoice status current_app.logger.info('Creating ejv invoice link records and setting invoice status.') sequence = 1 for inv in invoices: @@ -187,8 +179,6 @@ def _create_ejv_file_for_gov_account(cls, batch_type: str): # pylint:disable=to sequence=sequence) db.session.add(ejv_invoice_link) sequence += 1 - # Set distribution status to invoice - # Create invoice reference record current_app.logger.debug(f'Creating Invoice Reference for invoice id: {inv.id}') inv_ref = InvoiceReferenceModel( invoice_id=inv.id, @@ -203,23 +193,14 @@ def _create_ejv_file_for_gov_account(cls, batch_type: str): # pylint:disable=to db.session.rollback() return - # JV Batch Trailer batch_trailer: str = cls.get_batch_trailer(batch_number, batch_total, batch_type, control_total) - ejv_content = f'{batch_header}{ejv_content}{batch_trailer}' - - # Create a file add this content. - file_path_with_name, trg_file_path = cls.create_inbox_and_trg_files(ejv_content) - - current_app.logger.info('Uploading to ftp.') - - # Upload file and trg to FTP - cls.upload(ejv_content, cls.get_file_name(), file_path_with_name, trg_file_path) - - # commit changes to DB + file_path_with_name, trg_file_path, file_name = cls.create_inbox_and_trg_files(ejv_content) + current_app.logger.info('Uploading to sftp.') + cls.upload(ejv_content, file_name, file_path_with_name, trg_file_path) db.session.commit() - # Add a sleep to prevent collision on file name. + # Sleep to prevent collision on file name. time.sleep(1) @classmethod diff --git a/jobs/payment-jobs/tests/jobs/test_direct_pay_automated_refund_task.py b/jobs/payment-jobs/tests/jobs/test_direct_pay_automated_refund_task.py index f7dc306d5..6bae89034 100644 --- a/jobs/payment-jobs/tests/jobs/test_direct_pay_automated_refund_task.py +++ b/jobs/payment-jobs/tests/jobs/test_direct_pay_automated_refund_task.py @@ -85,7 +85,8 @@ def payment_status(cls): # pylint: disable=unused-argument; mocks of library me target = 'tasks.direct_pay_automated_refund_task.DirectPayAutomatedRefundTask._query_order_status' monkeypatch.setattr(target, payment_status) - with freeze_time(datetime.datetime.combine(datetime.datetime.utcnow().date(), datetime.time(6, 00))): + with freeze_time(datetime.datetime.combine(datetime.datetime.now(tz=datetime.timezone.utc).date(), + datetime.time(6, 00))): DirectPayAutomatedRefundTask().process_cc_refunds() refund = RefundModel.find_by_invoice_id(invoice.id) assert invoice.invoice_status_code == InvoiceStatus.REFUNDED.value @@ -134,7 +135,8 @@ def payment_status(cls): # pylint: disable=unused-argument; mocks of library me target = 'tasks.direct_pay_automated_refund_task.DirectPayAutomatedRefundTask._query_order_status' monkeypatch.setattr(target, payment_status) - with freeze_time(datetime.datetime.combine(datetime.datetime.utcnow().date(), datetime.time(6, 00))): + with freeze_time(datetime.datetime.combine(datetime.datetime.now(tz=datetime.timezone.utc).date(), + datetime.time(6, 00))): DirectPayAutomatedRefundTask().process_cc_refunds() assert refund.gl_error == 'BAD BAD' assert refund.gl_posted is None diff --git a/jobs/payment-jobs/tests/jobs/test_ejv_partner_distribution_task.py b/jobs/payment-jobs/tests/jobs/test_ejv_partner_distribution_task.py index fa2d0c6b8..d3dcdfb3f 100644 --- a/jobs/payment-jobs/tests/jobs/test_ejv_partner_distribution_task.py +++ b/jobs/payment-jobs/tests/jobs/test_ejv_partner_distribution_task.py @@ -22,8 +22,10 @@ from flask import current_app from freezegun import freeze_time from pay_api.models import CorpType as CorpTypeModel -from pay_api.models import DistributionCode, EjvFile, EjvHeader, EjvLink, FeeSchedule, Invoice, db -from pay_api.utils.enums import CfsAccountStatus, DisbursementStatus, InvoiceStatus, PaymentMethod +from pay_api.models import DistributionCode, EjvFile, EjvHeader, EjvLink, FeeSchedule, Invoice +from pay_api.models import PartnerDisbursements as PartnerDisbursementsModel +from pay_api.models import db +from pay_api.utils.enums import CfsAccountStatus, DisbursementStatus, EJVLinkType, InvoiceStatus, PaymentMethod from tasks.ejv_partner_distribution_task import EjvPartnerDistributionTask @@ -43,6 +45,8 @@ def test_disbursement_for_partners(session, monkeypatch, client_code, batch_type """ monkeypatch.setattr('pysftp.Connection.put', lambda *args, **kwargs: None) corp_type: CorpTypeModel = CorpTypeModel.find_by_code('VS') + corp_type.has_partner_disbursements = True + corp_type.save() pad_account = factory_create_pad_account(auth_account_id='1234', bank_number='001', @@ -76,6 +80,31 @@ def test_disbursement_for_partners(session, monkeypatch, client_code, batch_type factory_payment(invoice_number=inv_ref.invoice_number, payment_status_code='COMPLETED') factory_receipt(invoice_id=invoice.id, receipt_date=datetime.now(tz=timezone.utc)).save() + eft_invoice = factory_invoice(payment_account=pad_account, + corp_type_code=corp_type.code, + total=11.5, + payment_method_code=PaymentMethod.EFT.value, + status_code='PAID') + + factory_payment_line_item(invoice_id=eft_invoice.id, + fee_schedule_id=fee_schedule.fee_schedule_id, + filing_fees=10, + total=10, + service_fees=1.5, + fee_dist_id=fee_distribution.distribution_code_id) + + inv_ref = factory_invoice_reference(invoice_id=eft_invoice.id) + factory_payment(invoice_number=inv_ref.invoice_number, payment_status_code='COMPLETED') + factory_receipt(invoice_id=eft_invoice.id, receipt_date=datetime.now(tz=timezone.utc)).save() + partner_disbursement = PartnerDisbursementsModel( + amount=10, + is_reversal=False, + partner_code=eft_invoice.corp_type_code, + status_code=DisbursementStatus.WAITING_FOR_RECEIPT.value, + target_id=eft_invoice.id, + target_type=EJVLinkType.INVOICE.value + ).save() + EjvPartnerDistributionTask.create_ejv_file() # Lookup invoice and assert disbursement status @@ -101,6 +130,9 @@ def test_disbursement_for_partners(session, monkeypatch, client_code, batch_type assert ejv_file assert ejv_file.disbursement_status_code == DisbursementStatus.UPLOADED.value, f'{batch_type}' + assert partner_disbursement.status_code == DisbursementStatus.UPLOADED.value + assert partner_disbursement.processed_on + # Reverse those payments and assert records. # Set the status of invoice as disbursement completed, so that reversal can kick start. invoice.disbursement_status_code = DisbursementStatus.COMPLETED.value @@ -108,8 +140,13 @@ def test_disbursement_for_partners(session, monkeypatch, client_code, batch_type invoice.invoice_status_code = InvoiceStatus.REFUNDED.value invoice.refund_date = datetime.now(tz=timezone.utc) invoice.save() + partner_disbursement.status = DisbursementStatus.WAITING_FOR_RECEIPT.value + partner_disbursement.is_reversal = True + partner_disbursement.save() EjvPartnerDistributionTask.create_ejv_file() # Lookup invoice and assert disbursement status invoice = Invoice.find_by_id(invoice.id) assert invoice.disbursement_status_code == DisbursementStatus.UPLOADED.value + assert partner_disbursement.status_code == DisbursementStatus.UPLOADED.value + assert partner_disbursement.processed_on diff --git a/pay-api/migrations/versions/2024_10_01_56c4542db0d7_.py b/pay-api/migrations/versions/2024_10_01_56c4542db0d7_.py new file mode 100644 index 000000000..245cb411b --- /dev/null +++ b/pay-api/migrations/versions/2024_10_01_56c4542db0d7_.py @@ -0,0 +1,32 @@ +"""Enable partner disbursements for certain corp types. + +Revision ID: 56c4542db0d7 +Revises: aae01971bd53 +Create Date: 2024-09-17 06:26:15.691631 + +""" +from alembic import op +import sqlalchemy as sa +from pay_api.utils.enums import DisbursementStatus, EJVLinkType + +# revision identifiers, used by Alembic. +# Note you may see foreign keys with distribution_codes_history +# For disbursement_distribution_code_id, service_fee_distribution_code_id +# Please ignore those lines and don't include in migration. + +revision = '56c4542db0d7' +down_revision = 'aae01971bd53' +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute("update corp_types set has_partner_disbursements = 't' where code in ('CSO', 'VS')") + op.execute(f"""insert into partner_disbursements (amount, created_on, partner_code, is_reversal, status_code, target_id, target_type) + select (i.total - i.service_fees) as amount, now() as created_on, i.corp_type_code as partner_code, 'f' as is_reversal, + '{DisbursementStatus.WAITING_FOR_RECEIPT.value}' as status_code, i.id as target_id, '{EJVLinkType.INVOICE.value}' as target_type from invoices i where invoice_status_code in ('APPROVED', 'PAID') + and corp_type_code in ('CSO','VS') and payment_method_code = 'EFT' + """) + +def downgrade(): + op.execute("update corp_types set has_partner_disbursements = 'f' where code in ('CSO', 'VS')") diff --git a/pay-api/migrations/versions/2024_10_01_aae01971bd53_.py b/pay-api/migrations/versions/2024_10_01_aae01971bd53_.py new file mode 100644 index 000000000..3140a2526 --- /dev/null +++ b/pay-api/migrations/versions/2024_10_01_aae01971bd53_.py @@ -0,0 +1,64 @@ +"""Add in has_partner_disbursements column to corp_types, easier for querying, also remove unused columns. +Also add in partner disbursements. + +Revision ID: aae01971bd53 +Revises: 9e5c82dfe9c7 +Create Date: 2024-09-06 10:51:20.058891 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = 'aae01971bd53' +down_revision = '9e5c82dfe9c7' +branch_labels = None +depends_on = None + + +def upgrade(): + with op.batch_alter_table('corp_types', schema=None) as batch_op: + batch_op.add_column(sa.Column('has_partner_disbursements', sa.Boolean(), nullable=True)) + + with op.batch_alter_table('refunds_partial', schema=None) as batch_op: + batch_op.drop_constraint('refunds_partial_disbursement_status_code_fkey', type_='foreignkey') + batch_op.drop_column('disbursement_status_code') + batch_op.drop_column('disbursement_date') + + with op.batch_alter_table('refunds_partial_history', schema=None) as batch_op: + batch_op.drop_constraint('refunds_partial_history_disbursement_status_code_fkey', type_='foreignkey') + batch_op.drop_column('disbursement_status_code') + batch_op.drop_column('disbursement_date') + + op.create_table('partner_disbursements', + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('amount', sa.Numeric(), nullable=False), + sa.Column('created_on', sa.DateTime(), nullable=False), + sa.Column('feedback_on', sa.DateTime(), nullable=True), + sa.Column('partner_code', sa.String(length=50), nullable=False), + sa.Column('processed_on', sa.DateTime(), nullable=True), + sa.Column('is_reversal', sa.Boolean(), nullable=False), + sa.Column('status_code', sa.String(length=25), nullable=False), + sa.Column('target_id', sa.Integer(), nullable=True), + sa.Column('target_type', sa.String(length=50), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + +def downgrade(): + op.drop_table('partner_disbursements') + + with op.batch_alter_table('corp_types', schema=None) as batch_op: + batch_op.drop_column('has_partner_disbursements') + + with op.batch_alter_table('refunds_partial', schema=None) as batch_op: + batch_op.add_column(sa.Column('disbursement_status_code', sa.String(length=20), nullable=True)) + batch_op.add_column(sa.Column('disbursement_date', sa.Date(), nullable=True)) + batch_op.create_foreign_key(None, 'refunds_partial', 'disbursement_status_codes', ['disbursement_status_code'], ['code']) + + with op.batch_alter_table('refunds_partial_history', schema=None) as batch_op: + batch_op.add_column(sa.Column('disbursement_status_code', sa.String(length=20), nullable=True)) + batch_op.add_column(sa.Column('disbursement_date', sa.Date(), nullable=True)) + batch_op.create_foreign_key(None, 'refunds_partial_history', 'disbursement_status_codes', ['disbursement_status_code'], ['code']) + + diff --git a/pay-api/src/pay_api/models/__init__.py b/pay-api/src/pay_api/models/__init__.py index 17182011a..79f96e057 100755 --- a/pay-api/src/pay_api/models/__init__.py +++ b/pay-api/src/pay_api/models/__init__.py @@ -50,6 +50,7 @@ from .line_item_status_code import LineItemStatusCode, LineItemStatusCodeSchema from .non_sufficient_funds import NonSufficientFunds, NonSufficientFundsSchema from .notification_status_code import NotificationStatusCode, NotificationStatusCodeSchema +from .partner_disbursements import PartnerDisbursements from .payment import Payment, PaymentSchema from .payment_account import PaymentAccount, PaymentAccountSchema, PaymentAccountSearchModel # noqa: I001 from .payment_line_item import PaymentLineItem, PaymentLineItemSchema diff --git a/pay-api/src/pay_api/models/corp_type.py b/pay-api/src/pay_api/models/corp_type.py index 587e59cc5..67fe40db1 100644 --- a/pay-api/src/pay_api/models/corp_type.py +++ b/pay-api/src/pay_api/models/corp_type.py @@ -45,6 +45,7 @@ class CorpType(db.Model, CodeTable): 'bcol_staff_fee_code', 'code', 'description', + 'has_partner_disbursements', 'is_online_banking_allowed', 'product' ] @@ -57,6 +58,7 @@ class CorpType(db.Model, CodeTable): bcol_code_no_service_fee = db.Column(db.String(20), nullable=True) bcol_staff_fee_code = db.Column(db.String(20), nullable=True) is_online_banking_allowed = db.Column(Boolean(), default=True) + has_partner_disbursements = db.Column(Boolean(), default=False) batch_type = db.Column(db.String(2), nullable=True) product = db.Column(db.String(20), nullable=True) diff --git a/pay-api/src/pay_api/models/partner_disbursements.py b/pay-api/src/pay_api/models/partner_disbursements.py new file mode 100644 index 000000000..c6c9ddd48 --- /dev/null +++ b/pay-api/src/pay_api/models/partner_disbursements.py @@ -0,0 +1,73 @@ +# Copyright © 2024 Province of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Model to track Partner Disbursements, need this table because invoices can be reversed and applied multiple times. + +This is used in three different distinct flows for EFT only currently (provided the partner disbursements enabled): + +1. Invoice creation - create a Partner Disbursement record +2. Invoice reversal - create a Partner Disbursement reversal record +3. Statement reversal - EFT specific, create a Partner Disbursement reversal record + +""" + +from datetime import datetime, timezone + +from .base_model import BaseModel +from .db import db + + +class PartnerDisbursements(BaseModel): # pylint: disable=too-many-instance-attributes + """This class manages the partner disbursements that should be executed.""" + + __tablename__ = 'partner_disbursements' + # this mapper is used so that new and old versions of the service can be run simultaneously, + # making rolling upgrades easier + # This is used by SQLAlchemy to explicitly define which fields we're interested + # so it doesn't freak out and say it can't map the structure if other fields are present. + # This could occur from a failed deploy or during an upgrade. + # The other option is to tell SQLAlchemy to ignore differences, but that is ambiguous + # and can interfere with Alembic upgrades. + # + # NOTE: please keep mapper names in alpha-order, easier to track that way + # Exception, id is always first, _fields first + __mapper_args__ = { + 'include_properties': [ + 'id', + 'amount', + 'created_on', + 'feedback_on', + 'is_reversal', + 'partner_code', + 'processed_on', + 'status_code', + 'target_id', + 'target_type' + ] + } + + id = db.Column(db.Integer, primary_key=True, autoincrement=True) + amount = db.Column(db.Numeric, nullable=False) + created_on = db.Column('created_on', db.DateTime, nullable=False, default=lambda: datetime.now(tz=timezone.utc)) + feedback_on = db.Column('feedback_on', db.DateTime, nullable=True) + partner_code = db.Column('partner_code', db.String(50), nullable=False) + processed_on = db.Column('processed_on', db.DateTime, nullable=True) + is_reversal = db.Column('is_reversal', db.Boolean(), nullable=False, default=False) + status_code = db.Column('status_code', db.String(25), nullable=False) + target_id = db.Column(db.Integer, nullable=True) + target_type = db.Column(db.String(50), nullable=True) + + @classmethod + def find_by_target(cls, target_id: int, target_type: str): + """Find the Partner Disbursement by target.""" + return cls.query.filter_by(target_id=target_id, target_type=target_type).first() diff --git a/pay-api/src/pay_api/models/refunds_partial.py b/pay-api/src/pay_api/models/refunds_partial.py index 8147ba687..0afd493b2 100644 --- a/pay-api/src/pay_api/models/refunds_partial.py +++ b/pay-api/src/pay_api/models/refunds_partial.py @@ -44,8 +44,6 @@ class RefundsPartial(Audit, Versioned, BaseModel): # pylint: disable=too-many-i 'created_by', 'created_on', 'created_name', - 'disbursement_status_code', - 'disbursement_date', 'payment_line_item_id', 'refund_amount', 'refund_type', @@ -59,8 +57,6 @@ class RefundsPartial(Audit, Versioned, BaseModel): # pylint: disable=too-many-i payment_line_item_id = db.Column(db.Integer, ForeignKey('payment_line_items.id'), nullable=False, index=True) refund_amount = db.Column(db.Numeric(19, 2), nullable=False) refund_type = db.Column(db.String(50), nullable=True) - disbursement_status_code = db.Column(db.String(20), ForeignKey('disbursement_status_codes.code'), nullable=True) - disbursement_date = db.Column(db.DateTime, nullable=True) @define diff --git a/pay-api/src/pay_api/services/eft_refund.py b/pay-api/src/pay_api/services/eft_refund.py index 99510001d..1385c7cef 100644 --- a/pay-api/src/pay_api/services/eft_refund.py +++ b/pay-api/src/pay_api/services/eft_refund.py @@ -4,8 +4,10 @@ from flask import current_app from pay_api.dtos.eft_shortname import EFTShortNameRefundGetRequest, EFTShortNameRefundPatchRequest from pay_api.exceptions import BusinessException, Error +from pay_api.models import CorpType as CorpTypeModel from pay_api.models import Invoice as InvoiceModel from pay_api.models import EFTRefund as EFTRefundModel +from pay_api.models import PartnerDisbursements as PartnerDisbursementsModel from pay_api.models import PaymentAccount from pay_api.models.eft_credit import EFTCredit as EFTCreditModel from pay_api.models.eft_credit_invoice_link import EFTCreditInvoiceLink as EFTCreditInvoiceLinkModel @@ -15,7 +17,8 @@ from pay_api.services.email_service import ShortNameRefundEmailContent, send_email from pay_api.services.eft_short_name_historical import EFTShortnameHistorical as EFTHistoryService from pay_api.utils.enums import ( - EFTCreditInvoiceStatus, EFTHistoricalTypes, EFTShortnameRefundStatus, InvoiceStatus, Role) + DisbursementStatus, EFTCreditInvoiceStatus, EFTHistoricalTypes, EFTShortnameRefundStatus, EJVLinkType, + InvoiceStatus, Role) from pay_api.utils.user_context import user_context from pay_api.utils.util import get_str_by_path @@ -104,7 +107,7 @@ def handle_invoice_refund(invoice: InvoiceModel, case EFTCreditInvoiceStatus.COMPLETED.value: # 4. EFT Credit Link - COMPLETED # (Invoice needs to be reversed and receipt needs to be reversed.) - # reversal_total = Decimal('0') + reversal_total = Decimal('0') for cil in sibling_cils: EFTRefund.return_eft_credit(cil) EFTCreditInvoiceLinkModel( @@ -114,19 +117,21 @@ def handle_invoice_refund(invoice: InvoiceModel, receipt_number=cil.receipt_number, invoice_id=invoice.id, link_group_id=link_group_id).flush() - # if corp_type := CorpTypeModel.find_by_code(invoice.corp_type_code): - # if corp_type.has_partner_disbursements: - # reversal_total += cil.amount + reversal_total += cil.amount - # if reversal_total > 0: - # PartnerDisbursementsModel( - # amount=reversal_total, - # is_reversal=True, - # partner_code=invoice.corp_type_code, - # status_code=DisbursementStatus.WAITING_FOR_JOB.value, - # target_id=invoice.id, - # target_type=EJVLinkType.INVOICE.value - # ).flush() + if reversal_total != invoice.total: + raise BusinessException(Error.EFT_PARTIAL_REFUND) + + if corp_type := CorpTypeModel.find_by_code(invoice.corp_type_code): + if corp_type.has_partner_disbursements and invoice.total - invoice.service_fees > 0: + PartnerDisbursementsModel( + amount=invoice.total - invoice.service_fees, + is_reversal=True, + partner_code=invoice.corp_type_code, + status_code=DisbursementStatus.WAITING_FOR_RECEIPT.value, + target_id=invoice.id, + target_type=EJVLinkType.INVOICE.value + ).flush() current_balance = EFTCreditModel.get_eft_credit_balance(latest_eft_credit.short_name_id) if existing_balance != current_balance: diff --git a/pay-api/src/pay_api/services/eft_service.py b/pay-api/src/pay_api/services/eft_service.py index 2abb9218f..9c16016cb 100644 --- a/pay-api/src/pay_api/services/eft_service.py +++ b/pay-api/src/pay_api/services/eft_service.py @@ -20,6 +20,8 @@ from flask import current_app from sqlalchemy import and_, func + +from pay_api.models import CorpType as CorpTypeModel from pay_api.exceptions import BusinessException from pay_api.models import CfsAccount as CfsAccountModel from pay_api.models import EFTCredit as EFTCreditModel @@ -28,6 +30,7 @@ from pay_api.models import EFTShortnamesHistorical as EFTHistoryModel from pay_api.models import Invoice as InvoiceModel from pay_api.models import InvoiceReference as InvoiceReferenceModel +from pay_api.models import PartnerDisbursements as PartnerDisbursementsModel from pay_api.models import Payment as PaymentModel from pay_api.models import PaymentAccount as PaymentAccountModel from pay_api.models import Receipt as ReceiptModel @@ -35,10 +38,9 @@ from pay_api.models import Statement as StatementModel from pay_api.models import StatementInvoices as StatementInvoicesModel from pay_api.models import db -# from pay_api.models.corp_type import CorpType as CorpTypeModel from pay_api.utils.enums import ( - CfsAccountStatus, EFTCreditInvoiceStatus, InvoiceReferenceStatus, InvoiceStatus, PaymentMethod, PaymentStatus, - PaymentSystem) + CfsAccountStatus, DisbursementStatus, EFTCreditInvoiceStatus, EJVLinkType, InvoiceReferenceStatus, InvoiceStatus, + PaymentMethod, PaymentStatus, PaymentSystem) from pay_api.utils.errors import Error from pay_api.utils.user_context import user_context @@ -80,6 +82,16 @@ def create_invoice(self, payment_account: PaymentAccount, line_items: List[Payme **kwargs) -> None: """Do nothing here, we create invoice references on the create CFS_INVOICES job.""" self.ensure_no_payment_blockers(payment_account) + if corp_type := CorpTypeModel.find_by_code(invoice.corp_type_code): + if corp_type.has_partner_disbursements and invoice.total - invoice.service_fees > 0: + PartnerDisbursementsModel( + amount=invoice.total - invoice.service_fees, + is_reversal=False, + partner_code=invoice.corp_type_code, + status_code=DisbursementStatus.WAITING_FOR_RECEIPT.value, + target_id=invoice.id, + target_type=EJVLinkType.INVOICE.value + ).flush() def complete_post_invoice(self, invoice: Invoice, invoice_reference: InvoiceReference) -> None: """Complete any post invoice activities if needed.""" @@ -141,10 +153,10 @@ def create_invoice_reference(invoice: InvoiceModel, invoice_number: str, @staticmethod def create_receipt(invoice: InvoiceModel, payment: PaymentModel) -> ReceiptModel: """Create a receipt record for an invoice payment.""" - receipt: ReceiptModel = ReceiptModel(receipt_date=payment.payment_date, - receipt_amount=payment.paid_amount, - invoice_id=invoice.id, - receipt_number=payment.receipt_number) + receipt = ReceiptModel(receipt_date=payment.payment_date, + receipt_amount=payment.paid_amount, + invoice_id=invoice.id, + receipt_number=payment.receipt_number) return receipt @staticmethod @@ -201,7 +213,7 @@ def reverse_payment_action(short_name_id: int, statement_id: int): InvoiceStatus.CANCELLED.value] link_group_id = EFTCreditInvoiceLinkModel.get_next_group_link_seq() reversed_credits = 0 - # invoice_disbursements = {} + invoice_disbursements = {} for current_link in credit_invoice_links: invoice = InvoiceModel.find_by_id(current_link.invoice_id) @@ -226,20 +238,24 @@ def reverse_payment_action(short_name_id: int, statement_id: int): invoice_id=invoice.id, link_group_id=link_group_id).flush() - # if corp_type := CorpTypeModel.find_by_code(invoice.corp_type_code): - # if corp_type.has_partner_disbursements and current_link.amount > 0: - # invoice_disbursements.setdefault(invoice, 0) - # invoice_disbursements[invoice] += current_link.amount - - # for invoice, total_amount in invoice_disbursements.items(): - # PartnerDisbursementsModel( - # amount=total_amount, - # is_reversal=True, - # partner_code=invoice.corp_type_code, - # status_code=DisbursementStatus.WAITING_FOR_JOB.value, - # target_id=invoice.id, - # target_type=EJVLinkType.INVOICE.value - # ).flush() + if corp_type := CorpTypeModel.find_by_code(invoice.corp_type_code): + if corp_type.has_partner_disbursements and current_link.amount > 0: + invoice_disbursements.setdefault(invoice, 0) + invoice_disbursements[invoice] += current_link.amount + + for invoice, total_amount in invoice_disbursements.items(): + if total_amount != invoice.total: + raise BusinessException(Error.EFT_PARTIAL_REFUND) + + if total_amount - invoice.service_fees > 0: + PartnerDisbursementsModel( + amount=total_amount - invoice.service_fees, + is_reversal=True, + partner_code=invoice.corp_type_code, + status_code=DisbursementStatus.WAITING_FOR_RECEIPT.value, + target_id=invoice.id, + target_type=EJVLinkType.INVOICE.value + ).flush() statement = StatementModel.find_by_id(statement_id) EFTHistoryService.create_statement_reverse( EFTHistory(short_name_id=short_name_id, diff --git a/pay-api/src/pay_api/services/payment_transaction.py b/pay-api/src/pay_api/services/payment_transaction.py index c6cfecc77..a022147a4 100644 --- a/pay-api/src/pay_api/services/payment_transaction.py +++ b/pay-api/src/pay_api/services/payment_transaction.py @@ -449,7 +449,7 @@ def _update_receipt_details(invoices, payment, receipt_details, transaction_dao) invoice.payment_date = datetime.now(tz=timezone.utc) invoice_reference = InvoiceReference.find_active_reference_by_invoice_id(invoice.id) invoice_reference.status_code = InvoiceReferenceStatus.COMPLETED.value - # TODO If it's not PAD/EFT, publish message. Refactor and move to pay system service later. + # If it's not PAD/EFT, publish message. Refactor and move to pay system service later. if invoice.payment_method_code not in [PaymentMethod.PAD.value, PaymentMethod.EFT.value]: current_app.logger.info(f'Release record for invoice : {invoice.id} ') PaymentTransaction.publish_status(transaction_dao, invoice) diff --git a/pay-api/src/pay_api/utils/enums.py b/pay-api/src/pay_api/utils/enums.py index eae692e90..71311aca2 100644 --- a/pay-api/src/pay_api/utils/enums.py +++ b/pay-api/src/pay_api/utils/enums.py @@ -212,10 +212,12 @@ class DisbursementStatus(Enum): ACKNOWLEDGED = 'ACKNOWLEDGED' COMPLETED = 'COMPLETED' + CANCELLED = 'CANCELLED' ERRORED = 'ERRORED' REVERSED = 'REVERSED' UPLOADED = 'UPLOADED' - WAITING_FOR_JOB = 'WAITING_FOR_JOB' + # Could be waiting for receipt in the job. + WAITING_FOR_RECEIPT = 'WAITING_FOR_RECEIPT' class DisbursementMethod(Enum): @@ -409,17 +411,17 @@ class PaymentDetailsGlStatus(Enum): class QueueSources(Enum): """Queue sources for PAY.""" - PAY_API = 'pay-api' - PAY_JOBS = 'pay-jobs' - PAY_QUEUE = 'pay-queue' - FTP_POLLER = 'ftp-poller' + PAY_API = 'PAY-API' + PAY_JOBS = 'PAY-JOBS' + PAY_QUEUE = 'PAY-QUEUE' + FTP_POLLER = 'FTP-POLLER' class EJVLinkType(Enum): """EJV link types for ejv_link table.""" INVOICE = 'invoice' - REFUND = 'refund' + PARTIAL_REFUND = 'partial_refund' class StatementTemplate(Enum): diff --git a/pay-api/src/pay_api/utils/errors.py b/pay-api/src/pay_api/utils/errors.py index c3ad4756e..cd8b91f94 100644 --- a/pay-api/src/pay_api/utils/errors.py +++ b/pay-api/src/pay_api/utils/errors.py @@ -73,6 +73,7 @@ class Error(Enum): CFS_INVOICES_MISMATCH = 'CFS_INVOICES_MISMATCH', HTTPStatus.BAD_REQUEST + EFT_PARTIAL_REFUND = 'EFT_PARTIAL_REFUND', HTTPStatus.BAD_REQUEST EFT_CREDIT_AMOUNT_UNEXPECTED = 'EFT_CREDIT_AMOUNT_UNEXPECTED', HTTPStatus.BAD_REQUEST EFT_INSUFFICIENT_CREDITS = 'EFT_INSUFFICIENT_CREDITS', HTTPStatus.BAD_REQUEST EFT_PAYMENT_ACTION_ACCOUNT_ID_REQUIRED = 'EFT_PAYMENT_ACTION_ACCOUNT_ID_REQUIRED', HTTPStatus.BAD_REQUEST diff --git a/pay-api/tests/unit/api/test_eft_payment_actions.py b/pay-api/tests/unit/api/test_eft_payment_actions.py index 002ad0b05..5dcb0e922 100755 --- a/pay-api/tests/unit/api/test_eft_payment_actions.py +++ b/pay-api/tests/unit/api/test_eft_payment_actions.py @@ -25,8 +25,10 @@ from dateutil.relativedelta import relativedelta +from pay_api.models import CorpType as CorpTypeModel from pay_api.models import EFTCredit as EFTCreditModel from pay_api.models import EFTShortnames as EFTShortnamesModel +from pay_api.models import PartnerDisbursements from pay_api.models import PaymentAccount as PaymentAccountModel from pay_api.models import Statement as StatementModel from pay_api.services import EftService @@ -66,6 +68,10 @@ def setup_statement_data(account: PaymentAccountModel, invoice_totals: List[Deci total=invoice_total, paid=0).save() factory_statement_invoices(statement_id=statement.id, invoice_id=invoice.id) + corp_type = CorpTypeModel.find_by_code('CP') + corp_type.has_partner_disbursements = True + corp_type.save() + return statement @@ -323,3 +329,7 @@ def test_eft_reverse_payment_action(db, session, client, jwt, app, admin_users_m assert credit_invoice_links[1].amount == credit_invoice_links[0].amount assert credit_invoice_links[1].receipt_number == credit_invoice_links[0].receipt_number assert EFTCreditModel.get_eft_credit_balance(short_name.id) == 100 + + partner_disbursement = PartnerDisbursements.query.first() + assert partner_disbursement.is_reversal is True + assert partner_disbursement.amount == 100 diff --git a/pay-api/tests/unit/services/test_eft_service.py b/pay-api/tests/unit/services/test_eft_service.py index 19acf4625..6f4009e12 100644 --- a/pay-api/tests/unit/services/test_eft_service.py +++ b/pay-api/tests/unit/services/test_eft_service.py @@ -26,6 +26,8 @@ from pay_api.models import EFTCredit as EFTCreditModel from pay_api.models import EFTCreditInvoiceLink as EFTCreditInvoiceLinkModel from pay_api.models import EFTShortnamesHistorical as EFTHistoryModel +from pay_api.models import PartnerDisbursements +from pay_api.models.corp_type import CorpType as CorpTypeModel from pay_api.services.eft_service import EftService from pay_api.services.eft_refund import EFTRefund as EFTRefundService from pay_api.utils.enums import ( @@ -154,7 +156,8 @@ def test_refund_eft_credits_exceed_balance(session): ('2_no_eft_credit_link'), ('3_pending_credit_link'), ('4_completed_credit_link'), - ('5_consolidated_invoice_block') + ('5_consolidated_invoice_block'), + ('6_partial_refund_block'), ]) def test_eft_invoice_refund(session, test_name): """Test various scenarios for eft_invoice_refund.""" @@ -197,7 +200,7 @@ def test_eft_invoice_refund(session, test_name): eft_credit_id=eft_credit.id, status_code=EFTCreditInvoiceStatus.PENDING.value, link_group_id=3).save() - case '4_completed_credit_link' | '5_consolidated_invoice_block': + case '4_completed_credit_link' | '5_consolidated_invoice_block' | '6_partial_refund_block': invoice_reference = factory_invoice_reference(invoice_id=invoice.id, invoice_number='1234').save() if test_name == '5_consolidated_invoice_block': @@ -226,14 +229,28 @@ def test_eft_invoice_refund(session, test_name): eft_credit_id=eft_credit.id, status_code=EFTCreditInvoiceStatus.COMPLETED.value, link_group_id=2).save() + if test_name != '6_partial_refund_block': + cil_6 = factory_eft_credit_invoice_link(invoice_id=invoice.id, + eft_credit_id=eft_credit.id, + status_code=EFTCreditInvoiceStatus.COMPLETED.value, + link_group_id=2).save() + cil_7 = factory_eft_credit_invoice_link(invoice_id=invoice.id, + eft_credit_id=eft_credit.id, + status_code=EFTCreditInvoiceStatus.COMPLETED.value, + link_group_id=2).save() + corp_type = CorpTypeModel.find_by_code('CP') + corp_type.has_partner_disbursements = True + corp_type.save() case _: raise NotImplementedError - if test_name == '5_consolidated_invoice_block': + if test_name in ('5_consolidated_invoice_block', '6_partial_refund_block'): with pytest.raises(BusinessException) as excinfo: invoice.invoice_status_code = eft_service.process_cfs_refund(invoice, payment_account, None) invoice.save() - assert excinfo.value.code == Error.INVALID_CONSOLIDATED_REFUND.name + error = Error.INVALID_CONSOLIDATED_REFUND.value if test_name == '5_consolidated_invoice_block' \ + else Error.EFT_PARTIAL_REFUND.value + assert excinfo.value.code == error[0] return invoice.invoice_status_code = eft_service.process_cfs_refund(invoice, payment_account, None) @@ -272,15 +289,25 @@ def test_eft_invoice_refund(session, test_name): assert cil_3.status_code == EFTCreditInvoiceStatus.COMPLETED.value assert cil_4.status_code == EFTCreditInvoiceStatus.COMPLETED.value assert cil_5.status_code == EFTCreditInvoiceStatus.COMPLETED.value - assert eft_credit.remaining_amount == 4 - assert EFTCreditInvoiceLinkModel.query.count() == 5 + 3 + assert cil_6.status_code == EFTCreditInvoiceStatus.COMPLETED.value + assert cil_7.status_code == EFTCreditInvoiceStatus.COMPLETED.value + assert eft_credit.remaining_amount == 6 pending_refund_count = 0 + amount = 0 for cil in EFTCreditInvoiceLinkModel.query.all(): if cil.status_code == EFTCreditInvoiceStatus.PENDING_REFUND.value: pending_refund_count += 1 - assert pending_refund_count == 3 + amount += cil.amount + assert pending_refund_count == 5 + # 7 originally + assert EFTCreditInvoiceLinkModel.query.count() == 7 + pending_refund_count eft_history = session.query(EFTHistoryModel).one() assert_shortname_refund_history(eft_credit, eft_history, invoice) + assert PartnerDisbursements.query.count() == 1 + partner_disbursement = PartnerDisbursements.query.first() + assert partner_disbursement.is_reversal is True + assert partner_disbursement.partner_code == 'CP' + assert partner_disbursement.amount == amount case _: raise NotImplementedError @@ -291,3 +318,20 @@ def assert_shortname_refund_history(eft_credit, eft_history, invoice): assert eft_history.is_processing is True assert eft_history.amount == invoice.total assert eft_history.transaction_type == EFTHistoricalTypes.INVOICE_REFUND.value + + +def test_eft_partner_disbursement(session): + """Small test to assert if partner disbursement enabled a row is created.""" + payment_account = factory_payment_account(payment_method_code=PaymentMethod.EFT.value) + invoice = factory_invoice(payment_account=payment_account, + status_code=InvoiceStatus.APPROVED.value, + total=5).save() + eft_service.create_invoice(payment_account, 10, invoice) + assert PartnerDisbursements.query.count() == 0 + + corp_type = CorpTypeModel.find_by_code('CP') + corp_type.has_partner_disbursements = True + corp_type.save() + + eft_service.create_invoice(payment_account, 10, invoice) + assert PartnerDisbursements.query.count() == 1 diff --git a/pay-queue/poetry.lock b/pay-queue/poetry.lock index ab49c51dd..1736f437c 100644 --- a/pay-queue/poetry.lock +++ b/pay-queue/poetry.lock @@ -1980,8 +1980,8 @@ werkzeug = "3.0.3" [package.source] type = "git" url = "https://github.com/seeker25/sbc-pay.git" -reference = "21537" -resolved_reference = "aaec435d35acdb06683937827f0b73e38251f897" +reference = "21519" +resolved_reference = "468f2b01d32a4f9dd0c43a649203a0b5bf713f50" subdirectory = "pay-api" [[package]] @@ -2641,13 +2641,13 @@ files = [ [[package]] name = "sentry-sdk" -version = "2.13.0" +version = "2.14.0" description = "Python client for Sentry (https://sentry.io)" optional = false python-versions = ">=3.6" files = [ - {file = "sentry_sdk-2.13.0-py2.py3-none-any.whl", hash = "sha256:6beede8fc2ab4043da7f69d95534e320944690680dd9a963178a49de71d726c6"}, - {file = "sentry_sdk-2.13.0.tar.gz", hash = "sha256:8d4a576f7a98eb2fdb40e13106e41f330e5c79d72a68be1316e7852cf4995260"}, + {file = "sentry_sdk-2.14.0-py2.py3-none-any.whl", hash = "sha256:b8bc3dc51d06590df1291b7519b85c75e2ced4f28d9ea655b6d54033503b5bf4"}, + {file = "sentry_sdk-2.14.0.tar.gz", hash = "sha256:1e0e2eaf6dad918c7d1e0edac868a7bf20017b177f242cefe2a6bcd47955961d"}, ] [package.dependencies] @@ -3104,4 +3104,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.12" -content-hash = "1f549058e6873465c87ef26cd34c5e532b607e0ff6cc77c09707f258168e84da" +content-hash = "33f79252ad025320c5d0dfad84d6402383f5b1394947896e65fa019bb947b008" diff --git a/pay-queue/pyproject.toml b/pay-queue/pyproject.toml index cef3b8a33..ecc3fc94c 100644 --- a/pay-queue/pyproject.toml +++ b/pay-queue/pyproject.toml @@ -16,11 +16,10 @@ minio = "^7.2.5" attrs = "^23.2.0" sqlalchemy = "^2.0.28" itsdangerous = "^2.1.2" -jinja2 = "^3.1.3" protobuf = "4.25.3" launchdarkly-server-sdk = "^8.2.1" cachecontrol = "^0.14.0" -pay-api = {git = "https://github.com/seeker25/sbc-pay.git", subdirectory = "pay-api", branch = "21537"} +pay-api = {git = "https://github.com/seeker25/sbc-pay.git", subdirectory = "pay-api", branch = "21519"} pg8000 = "^1.30.5" diff --git a/pay-queue/src/pay_queue/services/cgi_reconciliations.py b/pay-queue/src/pay_queue/services/cgi_reconciliations.py index 45a389589..c109290c7 100644 --- a/pay-queue/src/pay_queue/services/cgi_reconciliations.py +++ b/pay-queue/src/pay_queue/services/cgi_reconciliations.py @@ -13,7 +13,8 @@ # limitations under the License. """CGI reconciliation file.""" import os -from datetime import datetime +from dataclasses import dataclass +from datetime import datetime, timezone from typing import Dict, List, Optional from flask import current_app @@ -23,6 +24,7 @@ from pay_api.models import EjvLink as EjvLinkModel from pay_api.models import Invoice as InvoiceModel from pay_api.models import InvoiceReference as InvoiceReferenceModel +from pay_api.models import PartnerDisbursements as PartnerDisbursementsModel from pay_api.models import Payment as PaymentModel from pay_api.models import PaymentLineItem as PaymentLineItemModel from pay_api.models import Receipt as ReceiptModel @@ -108,10 +110,10 @@ def _process_ejv_feedback(group_batches) -> bool: # pylint:disable=too-many-loc receipt_number: Optional[str] = None for line in group_batch.splitlines(): # For all these indexes refer the sharepoint docs refer : https://github.com/bcgov/entity/issues/6226 - is_batch_group: bool = line[2:4] == 'BG' - is_batch_header: bool = line[2:4] == 'BH' - is_jv_header: bool = line[2:4] == 'JH' - is_jv_detail: bool = line[2:4] == 'JD' + is_batch_group = line[2:4] == 'BG' + is_batch_header = line[2:4] == 'BH' + is_jv_header = line[2:4] == 'JH' + is_jv_detail = line[2:4] == 'JD' if is_batch_group: batch_number = int(line[15:24]) ejv_file = EjvFileModel.find_by_id(batch_number) @@ -125,7 +127,7 @@ def _process_ejv_feedback(group_batches) -> bool: # pylint:disable=too-many-loc elif is_jv_header: journal_name: str = line[7:17] # {ministry}{ejv_header_model.id:0>8} ejv_header_model_id = int(journal_name[2:]) - ejv_header: EjvHeaderModel = EjvHeaderModel.find_by_id(ejv_header_model_id) + ejv_header = EjvHeaderModel.find_by_id(ejv_header_model_id) ejv_header_return_code = line[271:275] ejv_header.disbursement_status_code = _get_disbursement_status(ejv_header_return_code) ejv_header_error_message = line[275:425] @@ -145,81 +147,126 @@ def _process_ejv_feedback(group_batches) -> bool: # pylint:disable=too-many-loc return has_errors -def _process_jv_details_feedback(ejv_file, has_errors, line, receipt_number): # pylint:disable=too-many-locals - journal_name: str = line[7:17] # {ministry}{ejv_header_model.id:0>8} - ejv_header_model_id = int(journal_name[2:]) +@dataclass +class JVDetailsFeedback: + """JV Details Feedback.""" + + ejv_header_model_id: int + flowthrough: str + journal_name: str + invoice_return_code: str + invoice_return_message: str + line: str + receipt_number: str + invoice: Optional[InvoiceModel] = None + invoice_link: Optional[EjvLinkModel] = None + partner_disbursement: Optional[PartnerDisbursementsModel] = None + + +def _process_jv_details_feedback(ejv_file, has_errors, line, receipt_number) -> bool: + """Process JV Details Feedback.""" + details = _build_jv_details(line, receipt_number) + # If the JV process failed, then mark the GL code against the invoice to be stopped + # for further JV process for the credit GL. + current_app.logger.info('Is Credit or Debit %s - %s', line[104:105], ejv_file.file_type) + credit_or_debit_line = details.line[104:105] + if credit_or_debit_line == 'C' and ejv_file.file_type == EjvFileType.DISBURSEMENT.value: + has_errors = _handle_jv_disbursement_feedback(details, has_errors) + elif credit_or_debit_line == 'D' and ejv_file.file_type == EjvFileType.PAYMENT.value: + has_errors = _handle_jv_payment_feedback(details, has_errors) + return has_errors + + +def _build_jv_details(line, receipt_number) -> JVDetailsFeedback: # Work around for CAS, they said fix the feedback files. line = _fix_invoice_line(line) - invoice_id = int(line[205:315]) + details = JVDetailsFeedback( + journal_name=line[7:17], + ejv_header_model_id=int(line[7:17][2:]), + line=line, + flowthrough=line[205:315].strip(), + invoice_return_code=line[315:319], + invoice_return_message=line[319:469], + receipt_number=receipt_number + ) + if '-' in details.flowthrough: + invoice_id = int(details.flowthrough.split('-')[0]) + partner_disbursement_id = int(details.flowthrough.split('-')[1]) + details.partner_disbursement = PartnerDisbursementsModel.find_by_id(partner_disbursement_id) + else: + invoice_id = int(details.flowthrough) current_app.logger.info('Invoice id - %s', invoice_id) - invoice: InvoiceModel = InvoiceModel.find_by_id(invoice_id) - invoice_link: EjvLinkModel = db.session.query(EjvLinkModel).filter( - EjvLinkModel.ejv_header_id == ejv_header_model_id).filter( + details.invoice = InvoiceModel.find_by_id(invoice_id) + details.invoice_link = db.session.query(EjvLinkModel).filter( + EjvLinkModel.ejv_header_id == details.ejv_header_model_id).filter( EjvLinkModel.link_id == invoice_id).filter( EjvLinkModel.link_type == EJVLinkType.INVOICE.value).one_or_none() - invoice_return_code = line[315:319] - invoice_return_message = line[319:469] - # If the JV process failed, then mark the GL code against the invoice to be stopped - # for further JV process for the credit GL. - current_app.logger.info('Is Credit or Debit %s - %s', line[104:105], ejv_file.file_type) - if line[104:105] == 'C' and ejv_file.file_type == EjvFileType.DISBURSEMENT.value: - disbursement_status = _get_disbursement_status(invoice_return_code) - invoice_link.disbursement_status_code = disbursement_status - invoice_link.message = invoice_return_message.strip() - current_app.logger.info('disbursement_status %s', disbursement_status) - if disbursement_status == DisbursementStatus.ERRORED.value: - has_errors = True - invoice.disbursement_status_code = DisbursementStatus.ERRORED.value - line_items: List[PaymentLineItemModel] = invoice.payment_line_items - for line_item in line_items: - # Line debit distribution - debit_distribution: DistributionCodeModel = DistributionCodeModel \ - .find_by_id(line_item.fee_distribution_id) - credit_distribution: DistributionCodeModel = DistributionCodeModel \ - .find_by_id(debit_distribution.disbursement_distribution_code_id) - credit_distribution.stop_ejv = True - else: - effective_date = datetime.strptime(line[22:30], '%Y%m%d') - _update_invoice_disbursement_status(invoice, effective_date) - - elif line[104:105] == 'D' and ejv_file.file_type == EjvFileType.PAYMENT.value: - # This is for gov account payment JV. - invoice_link.disbursement_status_code = _get_disbursement_status(invoice_return_code) - - invoice_link.message = invoice_return_message.strip() - current_app.logger.info('Invoice ID %s', invoice_id) - inv_ref: InvoiceReferenceModel = InvoiceReferenceModel.find_by_invoice_id_and_status( - invoice_id, InvoiceReferenceStatus.ACTIVE.value) - current_app.logger.info('invoice_link.disbursement_status_code %s', invoice_link.disbursement_status_code) - if invoice_link.disbursement_status_code == DisbursementStatus.ERRORED.value: - has_errors = True - # Cancel the invoice reference. - if inv_ref: - inv_ref.status_code = InvoiceReferenceStatus.CANCELLED.value - # Find the distribution code and set the stop_ejv flag to TRUE - dist_code: DistributionCodeModel = DistributionCodeModel.find_by_active_for_account( - invoice.payment_account_id) - dist_code.stop_ejv = True - elif invoice_link.disbursement_status_code == DisbursementStatus.COMPLETED.value: - # Set the invoice status as REFUNDED if it's a JV reversal, else mark as PAID - effective_date = datetime.strptime(line[22:30], '%Y%m%d') - # No need for credited here as these are just for EJV payments, which are never credited. - is_reversal = invoice.invoice_status_code in ( - InvoiceStatus.REFUNDED.value, InvoiceStatus.REFUND_REQUESTED.value) - _set_invoice_jv_reversal(invoice, effective_date, is_reversal) - - # Mark the invoice reference as COMPLETED, create a receipt - if inv_ref: - inv_ref.status_code = InvoiceReferenceStatus.COMPLETED.value - # Find receipt and add total to it, as single invoice can be multiple rows in the file - if not is_reversal: - receipt = ReceiptModel.find_by_invoice_id_and_receipt_number(invoice_id=invoice_id, - receipt_number=receipt_number) - if receipt: - receipt.receipt_amount += float(line[89:104]) - else: - ReceiptModel(invoice_id=invoice_id, receipt_number=receipt_number, receipt_date=datetime.now(), - receipt_amount=float(line[89:104])).flush() + return details + + +def _handle_jv_disbursement_feedback(details: JVDetailsFeedback, has_errors: bool) -> bool: + disbursement_status = _get_disbursement_status(details.invoice_return_code) + details.invoice_link.disbursement_status_code = disbursement_status + details.invoice_link.message = details.invoice_return_message.strip() + current_app.logger.info('disbursement_status %s', disbursement_status) + if disbursement_status == DisbursementStatus.ERRORED.value: + has_errors = True + if details.partner_disbursement: + details.partner_disbursement.status_code = DisbursementStatus.ERRORED.value + details.partner_disbursement.processed_on = datetime.now(tz=timezone.utc) + details.invoice.disbursement_status_code = DisbursementStatus.ERRORED.value + line_items: List[PaymentLineItemModel] = details.invoice.payment_line_items + for line_item in line_items: + # Line debit distribution + debit_distribution: DistributionCodeModel = DistributionCodeModel \ + .find_by_id(line_item.fee_distribution_id) + credit_distribution: DistributionCodeModel = DistributionCodeModel \ + .find_by_id(debit_distribution.disbursement_distribution_code_id) + credit_distribution.stop_ejv = True + else: + effective_date = datetime.strptime(details.line[22:30], '%Y%m%d') + _update_invoice_disbursement_status(details.invoice, effective_date, details.partner_disbursement) + return has_errors + + +def _handle_jv_payment_feedback(details: JVDetailsFeedback, has_errors: bool) -> bool: + # This is for gov account payment JV. + details.invoice_link.disbursement_status_code = _get_disbursement_status(details.invoice_return_code) + details.invoice_link.message = details.invoice_return_message.strip() + current_app.logger.info('Invoice ID %s', details.invoice.id) + inv_ref = InvoiceReferenceModel.find_by_invoice_id_and_status( + details.invoice.id, InvoiceReferenceStatus.ACTIVE.value) + current_app.logger.info('invoice_link.disbursement_status_code %s', details.invoice_link.disbursement_status_code) + if details.invoice_link.disbursement_status_code == DisbursementStatus.ERRORED.value: + has_errors = True + # Cancel the invoice reference. + if inv_ref: + inv_ref.status_code = InvoiceReferenceStatus.CANCELLED.value + # Find the distribution code and set the stop_ejv flag to TRUE + dist_code = DistributionCodeModel.find_by_active_for_account( + details.invoice.payment_account_id) + dist_code.stop_ejv = True + elif details.invoice_link.disbursement_status_code == DisbursementStatus.COMPLETED.value: + # Set the invoice status as REFUNDED if it's a JV reversal, else mark as PAID + effective_date = datetime.strptime(details.line[22:30], '%Y%m%d') + # No need for credited here as these are just for EJV payments, which are never credited. + is_reversal = details.invoice.invoice_status_code in ( + InvoiceStatus.REFUNDED.value, InvoiceStatus.REFUND_REQUESTED.value) + _set_invoice_jv_reversal(details.invoice, effective_date, is_reversal) + + # Mark the invoice reference as COMPLETED, create a receipt + if inv_ref: + inv_ref.status_code = InvoiceReferenceStatus.COMPLETED.value + # Find receipt and add total to it, as single invoice can be multiple rows in the file + if not is_reversal: + receipt = ReceiptModel.find_by_invoice_id_and_receipt_number(invoice_id=details.invoice.id, + receipt_number=details.receipt_number) + if receipt: + receipt.receipt_amount += float(details.line[89:104]) + else: + ReceiptModel(invoice_id=details.invoice.id, receipt_number=details.receipt_number, + receipt_date=datetime.now(tz=timezone.utc), + receipt_amount=float(details.line[89:104])).flush() return has_errors @@ -243,13 +290,27 @@ def _fix_invoice_line(line): return line -def _update_invoice_disbursement_status(invoice, effective_date: datetime): +def _update_partner_disbursement(partner_disbursement, status_code, effective_date): + """Update the partner disbursement status.""" + if partner_disbursement is None: + return + partner_disbursement.status_code = status_code + partner_disbursement.processed_on = datetime.now(tz=timezone.utc) + partner_disbursement.feedback_on = effective_date + + +def _update_invoice_disbursement_status(invoice: InvoiceModel, + effective_date: datetime, + partner_disbursement: PartnerDisbursementsModel): """Update status to reversed if its a refund, else to completed.""" + # Look up partner disbursements table and update the status. if invoice.invoice_status_code in (InvoiceStatus.REFUNDED.value, InvoiceStatus.REFUND_REQUESTED.value, InvoiceStatus.CREDITED.value): + _update_partner_disbursement(partner_disbursement, DisbursementStatus.REVERSED.value, effective_date) invoice.disbursement_status_code = DisbursementStatus.REVERSED.value invoice.disbursement_reversal_date = effective_date else: + _update_partner_disbursement(partner_disbursement, DisbursementStatus.COMPLETED.value, effective_date) invoice.disbursement_status_code = DisbursementStatus.COMPLETED.value invoice.disbursement_date = effective_date @@ -373,7 +434,7 @@ def _process_ap_header_routing_slips(line) -> bool: def _process_ap_header_non_gov_disbursement(line, ejv_file: EjvFileModel) -> bool: has_errors = False invoice_id = line[19:69].strip() - invoice: InvoiceModel = InvoiceModel.find_by_id(invoice_id) + invoice = InvoiceModel.find_by_id(invoice_id) ap_header_return_code = line[414:418] ap_header_error_message = line[418:568] disbursement_status = _get_disbursement_status(ap_header_return_code) @@ -392,7 +453,7 @@ def _process_ap_header_non_gov_disbursement(line, ejv_file: EjvFileModel) -> boo level='error') else: # TODO - Fix this on BC Assessment launch, so the effective date reads from the feedback. - _update_invoice_disbursement_status(invoice, effective_date=datetime.now()) + _update_invoice_disbursement_status(invoice, effective_date=datetime.now(), partner_disbursement=None) if invoice.invoice_status_code != InvoiceStatus.PAID.value: refund = RefundModel.find_by_invoice_id(invoice.id) refund.gl_posted = datetime.now() diff --git a/pay-queue/tests/conftest.py b/pay-queue/tests/conftest.py index cab7598f2..c257f1432 100644 --- a/pay-queue/tests/conftest.py +++ b/pay-queue/tests/conftest.py @@ -19,7 +19,7 @@ from flask_migrate import Migrate, upgrade from google.api_core.exceptions import NotFound from google.cloud import pubsub -from pay_api import db as _db +from pay_api.models import db as _db from pay_api.services.gcp_queue import GcpQueue from sqlalchemy import event, text from sqlalchemy_utils import create_database, database_exists, drop_database @@ -77,6 +77,7 @@ def session(db, app): # pylint: disable=redefined-outer-name, invalid-name sess = db._make_scoped_session(dict(bind=conn)) # pylint: disable=protected-access # Establish SAVEPOINT (http://docs.sqlalchemy.org/en/latest/orm/session_transaction.html#using-savepoint) nested = sess.begin_nested() + old_session = db.session db.session = sess db.session.commit = nested.commit db.session.rollback = nested.rollback @@ -102,6 +103,8 @@ def restart_savepoint(sess2, trans): # pylint: disable=unused-variable finally: db.session.remove() transaction.rollback() + event.remove(sess, 'after_transaction_end', restart_savepoint) + db.session = old_session @pytest.fixture(scope='session', autouse=True) diff --git a/pay-queue/tests/integration/test_cgi_reconciliations.py b/pay-queue/tests/integration/test_cgi_reconciliations.py index ff44929c8..5feea76a9 100644 --- a/pay-queue/tests/integration/test_cgi_reconciliations.py +++ b/pay-queue/tests/integration/test_cgi_reconciliations.py @@ -16,7 +16,7 @@ Test-Suite to ensure that the Payment Reconciliation queue service is working as expected. """ -from datetime import datetime +from datetime import datetime, timezone from pay_api.models import DistributionCode as DistributionCodeModel from pay_api.models import EjvFile as EjvFileModel @@ -25,9 +25,9 @@ from pay_api.models import FeeSchedule as FeeScheduleModel from pay_api.models import Invoice as InvoiceModel from pay_api.models import InvoiceReference as InvoiceReferenceModel +from pay_api.models import PartnerDisbursements as PartnerDisbursementsModel from pay_api.models import Payment as PaymentModel from pay_api.models import PaymentAccount as PaymentAccountModel -from pay_api.models import PaymentLineItem as PaymentLineItemModel from pay_api.models import Receipt as ReceiptModel from pay_api.models import Refund as RefundModel from pay_api.models import RoutingSlip as RoutingSlipModel @@ -36,6 +36,7 @@ CfsAccountStatus, DisbursementStatus, EjvFileType, EJVLinkType, InvoiceReferenceStatus, InvoiceStatus, PaymentMethod, PaymentStatus, RoutingSlipStatus) from sbc_common_components.utils.enums import QueueMessageTypes +from sqlalchemy import text from tests.integration.utils import add_file_event_to_queue_and_process @@ -53,25 +54,29 @@ def test_successful_partner_ejv_reconciliations(session, app, client): # 4. Create a CFS settlement file, and verify the records cfs_account_number = '1234' partner_code = 'VS' - fee_schedule: FeeScheduleModel = FeeScheduleModel.find_by_filing_type_and_corp_type( + fee_schedule = FeeScheduleModel.find_by_filing_type_and_corp_type( corp_type_code=partner_code, filing_type_code='WILLSEARCH' ) - pay_account: PaymentAccountModel = factory_create_pad_account(status=CfsAccountStatus.ACTIVE.value, - account_number=cfs_account_number) - invoice: InvoiceModel = factory_invoice(payment_account=pay_account, total=100, service_fees=10.0, - corp_type_code='VS', - payment_method_code=PaymentMethod.ONLINE_BANKING.value, - status_code=InvoiceStatus.PAID.value) + pay_account = factory_create_pad_account(status=CfsAccountStatus.ACTIVE.value, + account_number=cfs_account_number) + invoice = factory_invoice(payment_account=pay_account, total=100, service_fees=10.0, + corp_type_code='VS', + payment_method_code=PaymentMethod.ONLINE_BANKING.value, + status_code=InvoiceStatus.PAID.value) + eft_invoice = factory_invoice(payment_account=pay_account, total=100, service_fees=10.0, + corp_type_code='VS', + payment_method_code=PaymentMethod.EFT.value, + status_code=InvoiceStatus.PAID.value) invoice_id = invoice.id - line_item: PaymentLineItemModel = factory_payment_line_item( + line_item = factory_payment_line_item( invoice_id=invoice.id, filing_fees=90.0, service_fees=10.0, total=90.0, fee_schedule_id=fee_schedule.fee_schedule_id ) - dist_code: DistributionCodeModel = DistributionCodeModel.find_by_id(line_item.fee_distribution_id) + dist_code = DistributionCodeModel.find_by_id(line_item.fee_distribution_id) # Check if the disbursement distribution is present for this. if not dist_code.disbursement_distribution_code_id: - disbursement_distribution_code: DistributionCodeModel = factory_distribution(name='Disbursement') + disbursement_distribution_code = factory_distribution(name='Disbursement') dist_code.disbursement_distribution_code_id = disbursement_distribution_code.distribution_code_id dist_code.save() @@ -79,42 +84,58 @@ def test_successful_partner_ejv_reconciliations(session, app, client): factory_invoice_reference( invoice_id=invoice.id, invoice_number=invoice_number, status_code=InvoiceReferenceStatus.COMPLETED.value ) + factory_invoice_reference( + invoice_id=eft_invoice.id, invoice_number='1234567899', status_code=InvoiceReferenceStatus.COMPLETED.value + ) invoice.invoice_status_code = InvoiceStatus.SETTLEMENT_SCHEDULED.value invoice = invoice.save() - # Now create JV records. - # Create EJV File model + partner_disbursement = PartnerDisbursementsModel( + amount=10, + is_reversal=False, + partner_code=eft_invoice.corp_type_code, + status_code=DisbursementStatus.WAITING_FOR_RECEIPT.value, + target_id=eft_invoice.id, + target_type=EJVLinkType.INVOICE.value + ).save() + + eft_flowthrough = f'{eft_invoice.id}-{partner_disbursement.id}' + file_ref = f'INBOX.{datetime.now()}' - ejv_file: EjvFileModel = EjvFileModel(file_ref=file_ref, - disbursement_status_code=DisbursementStatus.UPLOADED.value).save() + ejv_file = EjvFileModel(file_ref=file_ref, + disbursement_status_code=DisbursementStatus.UPLOADED.value).save() ejv_file_id = ejv_file.id - ejv_header: EjvHeaderModel = EjvHeaderModel(disbursement_status_code=DisbursementStatus.UPLOADED.value, - ejv_file_id=ejv_file.id, - partner_code=partner_code, - payment_account_id=pay_account.id).save() + ejv_header = EjvHeaderModel(disbursement_status_code=DisbursementStatus.UPLOADED.value, + ejv_file_id=ejv_file.id, + partner_code=partner_code, + payment_account_id=pay_account.id).save() ejv_header_id = ejv_header.id EjvLinkModel( link_id=invoice.id, link_type=EJVLinkType.INVOICE.value, ejv_header_id=ejv_header.id, disbursement_status_code=DisbursementStatus.UPLOADED.value ).save() + EjvLinkModel( + link_id=eft_invoice.id, link_type=EJVLinkType.INVOICE.value, + ejv_header_id=ejv_header.id, disbursement_status_code=DisbursementStatus.UPLOADED.value + ).save() + ack_file_name = f'ACK.{file_ref}' - with open(ack_file_name, 'a+') as jv_file: + with open(ack_file_name, 'a+', encoding='utf-8') as jv_file: jv_file.write('') jv_file.close() - # Now upload the ACK file to minio and publish message. upload_to_minio(str.encode(''), ack_file_name) add_file_event_to_queue_and_process(client, ack_file_name, QueueMessageTypes.CGI_ACK_MESSAGE_TYPE.value) - # Query EJV File and assert the status is changed ejv_file = EjvFileModel.find_by_id(ejv_file_id) # Now upload a feedback file and check the status. # Just create feedback file to mock the real feedback file. + # Has legacy and added in PartnerDisbursements rows. feedback_content = f'GABG...........00000000{ejv_file_id}...\n' \ f'..BH...0000.................................................................................' \ f'.....................................................................CGI\n' \ @@ -125,23 +146,35 @@ def test_successful_partner_ejv_reconciliations(session, app, client): f'.......................................................................CGI\n' \ f'..JD...FI0000000{ejv_header_id}0000120230529................................................' \ f'...........000000000090.00D.................................................................' \ - f'...................................{invoice_id} ' \ + f'..................................#{invoice_id} ' \ f' 0000........................' \ f'............................................................................................' \ f'..................................CGI\n' \ f'..JD...FI0000000{ejv_header_id}0000220230529................................................' \ f'...........000000000090.00C.................................................................' \ - f'...................................{invoice_id} ' \ + f'..................................#{invoice_id} ' \ f' 0000........................' \ f'............................................................................................' \ f'..................................CGI\n' \ - f'..BT.......FI0000000{ejv_header_id}000000000000002000000000090.000000.......................' \ + f'..JD...FI0000000{ejv_header_id}0000120230529................................................' \ + f'...........000000000090.00D.................................................................' \ + f'..................................#{eft_flowthrough} ' \ + f' 0000........................' \ + f'............................................................................................' \ + f'..................................CGI\n' \ + f'..JD...FI0000000{ejv_header_id}0000220230529................................................' \ + f'...........000000000090.00C.................................................................' \ + f'..................................#{eft_flowthrough} ' \ + f' 0000........................' \ + f'............................................................................................' \ + f'..................................CGI\n' \ + f'..BT.......FI0000000{ejv_header_id}000000000000002000000000180.000000.......................' \ f'............................................................................................' \ f'...................................CGI' feedback_file_name = f'FEEDBACK.{file_ref}' - with open(feedback_file_name, 'a+') as jv_file: + with open(feedback_file_name, 'a+', encoding='utf-8') as jv_file: jv_file.write(feedback_content) jv_file.close() @@ -156,6 +189,9 @@ def test_successful_partner_ejv_reconciliations(session, app, client): assert ejv_file.disbursement_status_code == DisbursementStatus.COMPLETED.value invoice = InvoiceModel.find_by_id(invoice_id) assert invoice.disbursement_status_code == DisbursementStatus.COMPLETED.value + assert partner_disbursement.status_code == DisbursementStatus.COMPLETED.value + assert partner_disbursement.feedback_on + assert partner_disbursement.processed_on def test_failed_partner_ejv_reconciliations(session, app, client): @@ -166,25 +202,29 @@ def test_failed_partner_ejv_reconciliations(session, app, client): # 4. Create a CFS settlement file, and verify the records cfs_account_number = '1234' partner_code = 'VS' - fee_schedule: FeeScheduleModel = FeeScheduleModel.find_by_filing_type_and_corp_type( + fee_schedule = FeeScheduleModel.find_by_filing_type_and_corp_type( corp_type_code=partner_code, filing_type_code='WILLSEARCH' ) - pay_account: PaymentAccountModel = factory_create_pad_account(status=CfsAccountStatus.ACTIVE.value, - account_number=cfs_account_number) - invoice: InvoiceModel = factory_invoice(payment_account=pay_account, total=100, service_fees=10.0, - corp_type_code='VS', - payment_method_code=PaymentMethod.ONLINE_BANKING.value, - status_code=InvoiceStatus.PAID.value) + pay_account = factory_create_pad_account(status=CfsAccountStatus.ACTIVE.value, + account_number=cfs_account_number) + invoice = factory_invoice(payment_account=pay_account, total=100, service_fees=10.0, + corp_type_code='VS', + payment_method_code=PaymentMethod.ONLINE_BANKING.value, + status_code=InvoiceStatus.PAID.value) + eft_invoice = factory_invoice(payment_account=pay_account, total=100, service_fees=10.0, + corp_type_code='VS', + payment_method_code=PaymentMethod.EFT.value, + status_code=InvoiceStatus.PAID.value) invoice_id = invoice.id - line_item: PaymentLineItemModel = factory_payment_line_item( + line_item = factory_payment_line_item( invoice_id=invoice.id, filing_fees=90.0, service_fees=10.0, total=90.0, fee_schedule_id=fee_schedule.fee_schedule_id ) - dist_code: DistributionCodeModel = DistributionCodeModel.find_by_id(line_item.fee_distribution_id) + dist_code = DistributionCodeModel.find_by_id(line_item.fee_distribution_id) # Check if the disbursement distribution is present for this. if not dist_code.disbursement_distribution_code_id: - disbursement_distribution_code: DistributionCodeModel = factory_distribution(name='Disbursement') + disbursement_distribution_code = factory_distribution(name='Disbursement') dist_code.disbursement_distribution_code_id = disbursement_distribution_code.distribution_code_id dist_code.save() disbursement_distribution_code_id = dist_code.disbursement_distribution_code_id @@ -193,20 +233,32 @@ def test_failed_partner_ejv_reconciliations(session, app, client): factory_invoice_reference( invoice_id=invoice.id, invoice_number=invoice_number, status_code=InvoiceReferenceStatus.COMPLETED.value ) + factory_invoice_reference( + invoice_id=eft_invoice.id, invoice_number='1234567899', status_code=InvoiceReferenceStatus.COMPLETED.value + ) invoice.invoice_status_code = InvoiceStatus.SETTLEMENT_SCHEDULED.value invoice = invoice.save() - # Now create JV records. - # Create EJV File model + partner_disbursement = PartnerDisbursementsModel( + amount=10, + is_reversal=False, + partner_code=eft_invoice.corp_type_code, + status_code=DisbursementStatus.WAITING_FOR_RECEIPT.value, + target_id=eft_invoice.id, + target_type=EJVLinkType.INVOICE.value + ).save() + + eft_flowthrough = f'{eft_invoice.id}-{partner_disbursement.id}' + file_ref = f'INBOX{datetime.now()}' - ejv_file: EjvFileModel = EjvFileModel(file_ref=file_ref, - disbursement_status_code=DisbursementStatus.UPLOADED.value).save() + ejv_file = EjvFileModel(file_ref=file_ref, + disbursement_status_code=DisbursementStatus.UPLOADED.value).save() ejv_file_id = ejv_file.id - ejv_header: EjvHeaderModel = EjvHeaderModel(disbursement_status_code=DisbursementStatus.UPLOADED.value, - ejv_file_id=ejv_file.id, - partner_code=partner_code, - payment_account_id=pay_account.id).save() + ejv_header = EjvHeaderModel(disbursement_status_code=DisbursementStatus.UPLOADED.value, + ejv_file_id=ejv_file.id, + partner_code=partner_code, + payment_account_id=pay_account.id).save() ejv_header_id = ejv_header.id EjvLinkModel( link_id=invoice.id, @@ -214,9 +266,14 @@ def test_failed_partner_ejv_reconciliations(session, app, client): ejv_header_id=ejv_header.id, disbursement_status_code=DisbursementStatus.UPLOADED.value ).save() + EjvLinkModel( + link_id=eft_invoice.id, link_type=EJVLinkType.INVOICE.value, + ejv_header_id=ejv_header.id, disbursement_status_code=DisbursementStatus.UPLOADED.value + ).save() + ack_file_name = f'ACK.{file_ref}' - with open(ack_file_name, 'a+') as jv_file: + with open(ack_file_name, 'a+', encoding='utf-8') as jv_file: jv_file.write('') jv_file.close() @@ -229,6 +286,7 @@ def test_failed_partner_ejv_reconciliations(session, app, client): # Now upload a feedback file and check the status. # Just create feedback file to mock the real feedback file. + # Has legacy flow and PartnerDisbursements entries feedback_content = f'GABG...........00000000{ejv_file_id}...\n' \ f'..BH...1111TESTERRORMESSAGE................................................................' \ f'......................................................................CGI\n' \ @@ -239,23 +297,35 @@ def test_failed_partner_ejv_reconciliations(session, app, client): f'...........................................................................CGI\n' \ f'..JD...FI0000000{ejv_header_id}00001.......................................................' \ f'............000000000090.00D...............................................................' \ - f'.....................................{invoice_id} ' \ + f'....................................#{invoice_id} ' \ f' 1111TESTERRORMESSAGE....' \ f'...........................................................................................' \ f'.......................................CGI\n' \ f'..JD...FI0000000{ejv_header_id}00002.......................................................' \ f'............000000000090.00C...............................................................' \ - f'.....................................{invoice_id} ' \ + f'....................................#{invoice_id} ' \ f' 1111TESTERRORMESSAGE....' \ f'...........................................................................................' \ f'.......................................CGI\n' \ - f'..BT...........FI0000000{ejv_header_id}000000000000002000000000090.001111TESTERRORMESSAGE..' \ + f'..JD...FI0000000{ejv_header_id}00001.......................................................' \ + f'............000000000090.00D...............................................................' \ + f'....................................#{eft_flowthrough} ' \ + f' 1111TESTERRORMESSAGE.' \ + f'...........................................................................................' \ + f'..........................................CGI\n' \ + f'..JD...FI0000000{ejv_header_id}00002.......................................................' \ + f'............000000000090.00C...............................................................' \ + f'....................................#{eft_flowthrough} ' \ + f' 1111TESTERRORMESSAGE.' \ + f'...........................................................................................' \ + f'..........................................CGI\n' \ + f'..BT...........FI0000000{ejv_header_id}000000000000002000000000180.001111TESTERRORMESSAGE..' \ f'...........................................................................................' \ f'.........................................CGI\n' feedback_file_name = f'FEEDBACK.{file_ref}' - with open(feedback_file_name, 'a+') as jv_file: + with open(feedback_file_name, 'a+', encoding='utf-8') as jv_file: jv_file.write(feedback_content) jv_file.close() @@ -272,6 +342,8 @@ def test_failed_partner_ejv_reconciliations(session, app, client): assert invoice.disbursement_status_code == DisbursementStatus.ERRORED.value disbursement_distribution_code = DistributionCodeModel.find_by_id(disbursement_distribution_code_id) assert disbursement_distribution_code.stop_ejv + assert partner_disbursement.status_code == DisbursementStatus.ERRORED.value + assert partner_disbursement.processed_on def test_successful_partner_reversal_ejv_reconciliations(session, app, client): @@ -283,25 +355,29 @@ def test_successful_partner_reversal_ejv_reconciliations(session, app, client): # 5. Assert that the payment to partner account is reversed. cfs_account_number = '1234' partner_code = 'VS' - fee_schedule: FeeScheduleModel = FeeScheduleModel.find_by_filing_type_and_corp_type( + fee_schedule = FeeScheduleModel.find_by_filing_type_and_corp_type( corp_type_code=partner_code, filing_type_code='WILLSEARCH' ) - pay_account: PaymentAccountModel = factory_create_pad_account(status=CfsAccountStatus.ACTIVE.value, - account_number=cfs_account_number) - invoice: InvoiceModel = factory_invoice(payment_account=pay_account, total=100, service_fees=10.0, - corp_type_code='VS', - payment_method_code=PaymentMethod.ONLINE_BANKING.value, - status_code=InvoiceStatus.PAID.value) + pay_account = factory_create_pad_account(status=CfsAccountStatus.ACTIVE.value, + account_number=cfs_account_number) + invoice = factory_invoice(payment_account=pay_account, total=100, service_fees=10.0, + corp_type_code='VS', + payment_method_code=PaymentMethod.ONLINE_BANKING.value, + status_code=InvoiceStatus.PAID.value) + eft_invoice = factory_invoice(payment_account=pay_account, total=100, service_fees=10.0, + corp_type_code='VS', + payment_method_code=PaymentMethod.EFT.value, + status_code=InvoiceStatus.PAID.value) invoice_id = invoice.id - line_item: PaymentLineItemModel = factory_payment_line_item( + line_item = factory_payment_line_item( invoice_id=invoice.id, filing_fees=90.0, service_fees=10.0, total=90.0, fee_schedule_id=fee_schedule.fee_schedule_id ) - dist_code: DistributionCodeModel = DistributionCodeModel.find_by_id(line_item.fee_distribution_id) + dist_code = DistributionCodeModel.find_by_id(line_item.fee_distribution_id) # Check if the disbursement distribution is present for this. if not dist_code.disbursement_distribution_code_id: - disbursement_distribution_code: DistributionCodeModel = factory_distribution(name='Disbursement') + disbursement_distribution_code = factory_distribution(name='Disbursement') dist_code.disbursement_distribution_code_id = disbursement_distribution_code.distribution_code_id dist_code.save() @@ -309,31 +385,51 @@ def test_successful_partner_reversal_ejv_reconciliations(session, app, client): factory_invoice_reference( invoice_id=invoice.id, invoice_number=invoice_number, status_code=InvoiceReferenceStatus.COMPLETED.value ) + factory_invoice_reference( + invoice_id=eft_invoice.id, invoice_number='1234567899', status_code=InvoiceReferenceStatus.COMPLETED.value + ) invoice.invoice_status_code = InvoiceStatus.REFUND_REQUESTED.value - # Set as disbursement complete. invoice.disbursement_status_code = DisbursementStatus.COMPLETED.value invoice = invoice.save() - # Now create JV records. - # Create EJV File model + eft_invoice.invoice_status_code = InvoiceStatus.REFUND_REQUESTED.value + eft_invoice.disbursement_status_code = DisbursementStatus.COMPLETED.value + eft_invoice = eft_invoice.save() + + partner_disbursement = PartnerDisbursementsModel( + amount=10, + is_reversal=True, + partner_code=eft_invoice.corp_type_code, + status_code=DisbursementStatus.WAITING_FOR_RECEIPT.value, + target_id=eft_invoice.id, + target_type=EJVLinkType.INVOICE.value + ).save() + + eft_flowthrough = f'{eft_invoice.id}-{partner_disbursement.id}' + file_ref = f'INBOX.{datetime.now()}' - ejv_file: EjvFileModel = EjvFileModel(file_ref=file_ref, - disbursement_status_code=DisbursementStatus.UPLOADED.value).save() + ejv_file = EjvFileModel(file_ref=file_ref, + disbursement_status_code=DisbursementStatus.UPLOADED.value).save() ejv_file_id = ejv_file.id - ejv_header: EjvHeaderModel = EjvHeaderModel(disbursement_status_code=DisbursementStatus.UPLOADED.value, - ejv_file_id=ejv_file.id, - partner_code=partner_code, - payment_account_id=pay_account.id).save() + ejv_header = EjvHeaderModel(disbursement_status_code=DisbursementStatus.UPLOADED.value, + ejv_file_id=ejv_file.id, + partner_code=partner_code, + payment_account_id=pay_account.id).save() ejv_header_id = ejv_header.id EjvLinkModel( link_id=invoice.id, link_type=EJVLinkType.INVOICE.value, ejv_header_id=ejv_header.id, disbursement_status_code=DisbursementStatus.UPLOADED.value ).save() + EjvLinkModel( + link_id=eft_invoice.id, link_type=EJVLinkType.INVOICE.value, + ejv_header_id=ejv_header.id, disbursement_status_code=DisbursementStatus.UPLOADED.value + ).save() + ack_file_name = f'ACK.{file_ref}' - with open(ack_file_name, 'a+') as jv_file: + with open(ack_file_name, 'a+', encoding='utf-8') as jv_file: jv_file.write('') jv_file.close() @@ -346,6 +442,7 @@ def test_successful_partner_reversal_ejv_reconciliations(session, app, client): # Now upload a feedback file and check the status. # Just create feedback file to mock the real feedback file. + # Has legacy flow and PartnerDisbursements entries feedback_content = f'GABG...........00000000{ejv_file_id}...\n' \ f'..BH...0000.................................................................................' \ f'.....................................................................CGI\n' \ @@ -356,65 +453,77 @@ def test_successful_partner_reversal_ejv_reconciliations(session, app, client): f'.......................................................................CGI\n' \ f'..JD...FI0000000{ejv_header_id}0000120230529................................................' \ f'...........000000000090.00C.................................................................' \ - f'...................................{invoice_id} ' \ + f'..................................#{invoice_id} ' \ + f' 0000........................' \ + f'............................................................................................' \ + f'..................................CGI\n' \ + f'..JD...FI0000000{ejv_header_id}0000220230529................................................' \ + f'...........000000000090.00D.................................................................' \ + f'..................................#{invoice_id} ' \ + f' 0000........................' \ + f'............................................................................................' \ + f'..................................CGI\n' \ + f'..JD...FI0000000{ejv_header_id}0000120230529................................................' \ + f'...........000000000090.00C.................................................................' \ + f'...................................{eft_flowthrough} ' \ f' 0000........................' \ f'............................................................................................' \ f'..................................CGI\n' \ f'..JD...FI0000000{ejv_header_id}0000220230529................................................' \ f'...........000000000090.00D.................................................................' \ - f'...................................{invoice_id} ' \ + f'...................................{eft_flowthrough} ' \ f' 0000........................' \ f'............................................................................................' \ f'..................................CGI\n' \ - f'..BT.......FI0000000{ejv_header_id}000000000000002000000000090.000000.......................' \ + f'..BT.......FI0000000{ejv_header_id}000000000000002000000000180.000000.......................' \ f'............................................................................................' \ f'...................................CGI' feedback_file_name = f'FEEDBACK.{file_ref}' - with open(feedback_file_name, 'a+') as jv_file: + with open(feedback_file_name, 'a+', encoding='utf-8') as jv_file: jv_file.write(feedback_content) jv_file.close() - # Now upload the ACK file to minio and publish message. with open(feedback_file_name, 'rb') as f: upload_to_minio(f.read(), feedback_file_name) add_file_event_to_queue_and_process(client, feedback_file_name, QueueMessageTypes.CGI_FEEDBACK_MESSAGE_TYPE.value) - # Query EJV File and assert the status is changed ejv_file = EjvFileModel.find_by_id(ejv_file_id) assert ejv_file.disbursement_status_code == DisbursementStatus.COMPLETED.value invoice = InvoiceModel.find_by_id(invoice_id) assert invoice.disbursement_status_code == DisbursementStatus.REVERSED.value assert invoice.disbursement_reversal_date == datetime(2023, 5, 29) + assert partner_disbursement.status_code == DisbursementStatus.REVERSED.value + assert partner_disbursement.feedback_on + assert partner_disbursement.processed_on -def test_succesful_payment_ejv_reconciliations(session, app, client): +def test_successful_payment_ejv_reconciliations(session, app, client): """Test Reconciliations worker.""" # 1. Create EJV payment accounts # 2. Create invoice and related records # 3. Create a feedback file and assert status - corp_type = 'BEN' filing_type = 'BCINC' # Find fee schedule which have service fees. - fee_schedule: FeeScheduleModel = FeeScheduleModel.find_by_filing_type_and_corp_type(corp_type, filing_type) + fee_schedule = FeeScheduleModel.find_by_filing_type_and_corp_type(corp_type, filing_type) # Create a service fee distribution code service_fee_dist_code = factory_distribution(name='service fee', client='112', reps_centre='99999', service_line='99999', - stob='9999', project_code='9999999') + stob='9999', project_code='9999998') service_fee_dist_code.save() - dist_code: DistributionCodeModel = DistributionCodeModel.find_by_active_for_fee_schedule( + dist_code = DistributionCodeModel.find_by_active_for_fee_schedule( fee_schedule.fee_schedule_id) # Update fee dist code to match the requirement. dist_code.client = '112' dist_code.responsibility_centre = '22222' dist_code.service_line = '33333' dist_code.stob = '4444' - dist_code.project_code = '5555555' + dist_code.project_code = '5555559' dist_code.service_fee_distribution_code_id = service_fee_dist_code.distribution_code_id dist_code.save() @@ -429,9 +538,9 @@ def test_succesful_payment_ejv_reconciliations(session, app, client): # Now create JV records. # Create EJV File model file_ref = f'INBOX.{datetime.now()}' - ejv_file: EjvFileModel = EjvFileModel(file_ref=file_ref, - disbursement_status_code=DisbursementStatus.UPLOADED.value, - file_type=EjvFileType.PAYMENT.value).save() + ejv_file = EjvFileModel(file_ref=file_ref, + disbursement_status_code=DisbursementStatus.UPLOADED.value, + file_type=EjvFileType.PAYMENT.value).save() ejv_file_id = ejv_file.id feedback_content = f'GABG...........00000000{ejv_file_id}...\n' \ @@ -447,15 +556,15 @@ def test_succesful_payment_ejv_reconciliations(session, app, client): inv = factory_invoice(payment_account=jv_acc, corp_type_code=corp_type, total=inv_total_amount, status_code=InvoiceStatus.APPROVED.value, payment_method_code=None) factory_invoice_reference(inv.id, status_code=InvoiceReferenceStatus.ACTIVE.value) - line: PaymentLineItemModel = factory_payment_line_item(invoice_id=inv.id, - fee_schedule_id=fee_schedule.fee_schedule_id, - filing_fees=100, - total=100, - service_fees=1.5, - fee_dist_id=dist_code.distribution_code_id) + line = factory_payment_line_item(invoice_id=inv.id, + fee_schedule_id=fee_schedule.fee_schedule_id, + filing_fees=100, + total=100, + service_fees=1.5, + fee_dist_id=dist_code.distribution_code_id) inv_ids.append(inv.id) - ejv_header: EjvHeaderModel = EjvHeaderModel(disbursement_status_code=DisbursementStatus.UPLOADED.value, - ejv_file_id=ejv_file.id, payment_account_id=jv_acc.id).save() + ejv_header = EjvHeaderModel(disbursement_status_code=DisbursementStatus.UPLOADED.value, + ejv_file_id=ejv_file.id, payment_account_id=jv_acc.id).save() EjvLinkModel(link_id=inv.id, link_type=EJVLinkType.INVOICE.value, ejv_header_id=ejv_header.id, disbursement_status_code=DisbursementStatus.UPLOADED.value @@ -499,7 +608,7 @@ def test_succesful_payment_ejv_reconciliations(session, app, client): f'......................................................................CGI' ack_file_name = f'ACK.{file_ref}' - with open(ack_file_name, 'a+') as jv_file: + with open(ack_file_name, 'a+', encoding='utf-8') as jv_file: jv_file.write('') jv_file.close() @@ -512,7 +621,7 @@ def test_succesful_payment_ejv_reconciliations(session, app, client): feedback_file_name = f'FEEDBACK.{file_ref}' - with open(feedback_file_name, 'a+') as jv_file: + with open(feedback_file_name, 'a+', encoding='utf-8') as jv_file: jv_file.write(feedback_content) jv_file.close() @@ -527,11 +636,11 @@ def test_succesful_payment_ejv_reconciliations(session, app, client): assert ejv_file.disbursement_status_code == DisbursementStatus.COMPLETED.value # Assert invoice and receipt records for inv_id in inv_ids: - invoice: InvoiceModel = InvoiceModel.find_by_id(inv_id) + invoice = InvoiceModel.find_by_id(inv_id) assert invoice.disbursement_status_code is None assert invoice.invoice_status_code == InvoiceStatus.PAID.value assert invoice.payment_date == datetime(2023, 5, 29) - invoice_ref: InvoiceReferenceModel = InvoiceReferenceModel.find_by_invoice_id_and_status( + invoice_ref = InvoiceReferenceModel.find_by_invoice_id_and_status( inv_id, InvoiceReferenceStatus.COMPLETED.value ) assert invoice_ref @@ -541,38 +650,43 @@ def test_succesful_payment_ejv_reconciliations(session, app, client): # Assert payment records for jv_account_id in jv_account_ids: account = PaymentAccountModel.find_by_id(jv_account_id) - payment: PaymentModel = PaymentModel.search_account_payments(auth_account_id=account.auth_account_id, - payment_status=PaymentStatus.COMPLETED.value, - page=1, limit=100)[0] + payment = PaymentModel.search_account_payments(auth_account_id=account.auth_account_id, + payment_status=PaymentStatus.COMPLETED.value, + page=1, limit=100)[0] assert len(payment) == 1 assert payment[0][0].paid_amount == inv_total_amount -def test_succesful_payment_reversal_ejv_reconciliations(session, app, client): +def test_successful_payment_reversal_ejv_reconciliations(session, app, client): """Test Reconciliations worker.""" # 1. Create EJV payment accounts # 2. Create invoice and related records # 3. Create a feedback file and assert status + corp_type = 'CP' + filing_type = 'OTFDR' - corp_type = 'BEN' - filing_type = 'BCINC' + InvoiceModel.query.delete() + # Reset the sequence, because the unit test is only dealing with 1 character for the invoice id. + # This becomes more apparent when running unit tests in parallel. + db.session.execute(text('ALTER SEQUENCE invoices_id_seq RESTART WITH 1')) + db.session.commit() # Find fee schedule which have service fees. - fee_schedule: FeeScheduleModel = FeeScheduleModel.find_by_filing_type_and_corp_type(corp_type, filing_type) + fee_schedule = FeeScheduleModel.find_by_filing_type_and_corp_type(corp_type, filing_type) # Create a service fee distribution code service_fee_dist_code = factory_distribution(name='service fee', client='112', reps_centre='99999', service_line='99999', stob='9999', project_code='9999999') service_fee_dist_code.save() - dist_code: DistributionCodeModel = DistributionCodeModel.find_by_active_for_fee_schedule( + dist_code = DistributionCodeModel.find_by_active_for_fee_schedule( fee_schedule.fee_schedule_id) # Update fee dist code to match the requirement. dist_code.client = '112' dist_code.responsibility_centre = '22222' dist_code.service_line = '33333' dist_code.stob = '4444' - dist_code.project_code = '5555555' + dist_code.project_code = '5555557' dist_code.service_fee_distribution_code_id = service_fee_dist_code.distribution_code_id dist_code.save() @@ -584,10 +698,10 @@ def test_succesful_payment_reversal_ejv_reconciliations(session, app, client): # Now create JV records. # Create EJV File model - file_ref = f'INBOX.{datetime.now()}' - ejv_file: EjvFileModel = EjvFileModel(file_ref=file_ref, - disbursement_status_code=DisbursementStatus.UPLOADED.value, - file_type=EjvFileType.PAYMENT.value).save() + file_ref = f'INBOX.{datetime.now(tz=timezone.utc)}' + ejv_file = EjvFileModel(file_ref=file_ref, + disbursement_status_code=DisbursementStatus.UPLOADED.value, + file_type=EjvFileType.PAYMENT.value).save() ejv_file_id = ejv_file.id feedback_content = f'GABG...........00000000{ejv_file_id}...\n' \ @@ -601,17 +715,18 @@ def test_succesful_payment_reversal_ejv_reconciliations(session, app, client): for jv_acc in jv_accounts: jv_account_ids.append(jv_acc.id) inv = factory_invoice(payment_account=jv_acc, corp_type_code=corp_type, total=inv_total_amount, - status_code=InvoiceStatus.REFUND_REQUESTED.value, payment_method_code=None) + status_code=InvoiceStatus.REFUND_REQUESTED.value, payment_method_code=None + ) factory_invoice_reference(inv.id, status_code=InvoiceReferenceStatus.ACTIVE.value) - line: PaymentLineItemModel = factory_payment_line_item(invoice_id=inv.id, - fee_schedule_id=fee_schedule.fee_schedule_id, - filing_fees=100, - total=100, - service_fees=1.5, - fee_dist_id=dist_code.distribution_code_id) + line = factory_payment_line_item(invoice_id=inv.id, + fee_schedule_id=fee_schedule.fee_schedule_id, + filing_fees=100, + total=100, + service_fees=1.5, + fee_dist_id=dist_code.distribution_code_id) inv_ids.append(inv.id) - ejv_header: EjvHeaderModel = EjvHeaderModel(disbursement_status_code=DisbursementStatus.UPLOADED.value, - ejv_file_id=ejv_file.id, payment_account_id=jv_acc.id).save() + ejv_header = EjvHeaderModel(disbursement_status_code=DisbursementStatus.UPLOADED.value, + ejv_file_id=ejv_file.id, payment_account_id=jv_acc.id).save() EjvLinkModel( link_id=inv.id, link_type=EJVLinkType.INVOICE.value, @@ -655,11 +770,10 @@ def test_succesful_payment_reversal_ejv_reconciliations(session, app, client): f'......................................................................CGI' ack_file_name = f'ACK.{file_ref}' - with open(ack_file_name, 'a+') as jv_file: + with open(ack_file_name, 'a+', encoding='utf-8') as jv_file: jv_file.write('') jv_file.close() - # Now upload the ACK file to minio and publish message. upload_to_minio(str.encode(''), ack_file_name) add_file_event_to_queue_and_process(client, ack_file_name, QueueMessageTypes.CGI_ACK_MESSAGE_TYPE.value) @@ -668,7 +782,7 @@ def test_succesful_payment_reversal_ejv_reconciliations(session, app, client): feedback_file_name = f'FEEDBACK.{file_ref}' - with open(feedback_file_name, 'a+') as jv_file: + with open(feedback_file_name, 'a+', encoding='utf-8') as jv_file: jv_file.write(feedback_content) jv_file.close() @@ -683,11 +797,11 @@ def test_succesful_payment_reversal_ejv_reconciliations(session, app, client): assert ejv_file.disbursement_status_code == DisbursementStatus.COMPLETED.value # Assert invoice and receipt records for inv_id in inv_ids: - invoice: InvoiceModel = InvoiceModel.find_by_id(inv_id) + invoice = InvoiceModel.find_by_id(inv_id) assert invoice.disbursement_status_code is None assert invoice.invoice_status_code == InvoiceStatus.REFUNDED.value assert invoice.refund_date == datetime(2023, 5, 29) - invoice_ref: InvoiceReferenceModel = InvoiceReferenceModel.find_by_invoice_id_and_status( + invoice_ref = InvoiceReferenceModel.find_by_invoice_id_and_status( inv_id, InvoiceReferenceStatus.COMPLETED.value ) assert invoice_ref @@ -695,9 +809,9 @@ def test_succesful_payment_reversal_ejv_reconciliations(session, app, client): # Assert payment records for jv_account_id in jv_account_ids: account = PaymentAccountModel.find_by_id(jv_account_id) - payment: PaymentModel = PaymentModel.search_account_payments(auth_account_id=account.auth_account_id, - payment_status=PaymentStatus.COMPLETED.value, - page=1, limit=100)[0] + payment = PaymentModel.search_account_payments(auth_account_id=account.auth_account_id, + payment_status=PaymentStatus.COMPLETED.value, + page=1, limit=100)[0] assert len(payment) == 1 assert payment[0][0].paid_amount == inv_total_amount @@ -735,15 +849,15 @@ def test_successful_refund_reconciliations(session, app, client): # Now create AP records. # Create EJV File model file_ref = f'INBOX.{datetime.now()}' - ejv_file: EjvFileModel = EjvFileModel(file_ref=file_ref, - file_type=EjvFileType.REFUND.value, - disbursement_status_code=DisbursementStatus.UPLOADED.value).save() + ejv_file = EjvFileModel(file_ref=file_ref, + file_type=EjvFileType.REFUND.value, + disbursement_status_code=DisbursementStatus.UPLOADED.value).save() ejv_file_id = ejv_file.id # Upload an acknowledgement file ack_file_name = f'ACK.{file_ref}' - with open(ack_file_name, 'a+') as jv_file: + with open(ack_file_name, 'a+', encoding='utf-8') as jv_file: jv_file.write('') jv_file.close() @@ -815,7 +929,7 @@ def test_successful_refund_reconciliations(session, app, client): feedback_file_name = f'FEEDBACK.{file_ref}' - with open(feedback_file_name, 'a+') as jv_file: + with open(feedback_file_name, 'a+', encoding='utf-8') as jv_file: jv_file.write(feedback_content) jv_file.close() @@ -866,15 +980,15 @@ def test_failed_refund_reconciliations(session, app, client): # Now create AP records. # Create EJV File model file_ref = f'INBOX.{datetime.now()}' - ejv_file: EjvFileModel = EjvFileModel(file_ref=file_ref, - file_type=EjvFileType.REFUND.value, - disbursement_status_code=DisbursementStatus.UPLOADED.value).save() + ejv_file = EjvFileModel(file_ref=file_ref, + file_type=EjvFileType.REFUND.value, + disbursement_status_code=DisbursementStatus.UPLOADED.value).save() ejv_file_id = ejv_file.id # Upload an acknowledgement file ack_file_name = f'ACK.{file_ref}' - with open(ack_file_name, 'a+') as jv_file: + with open(ack_file_name, 'a+', encoding='utf-8') as jv_file: jv_file.write('') jv_file.close() @@ -947,7 +1061,7 @@ def test_failed_refund_reconciliations(session, app, client): feedback_file_name = f'FEEDBACK.{file_ref}' - with open(feedback_file_name, 'a+') as jv_file: + with open(feedback_file_name, 'a+', encoding='utf-8') as jv_file: jv_file.write(feedback_content) jv_file.close() @@ -1001,13 +1115,13 @@ def test_successful_ap_disbursement(session, app, client): factory_refund(invoice_id=refund_invoice.id) file_ref = f'INBOX.{datetime.now()}' - ejv_file: EjvFileModel = EjvFileModel(file_ref=file_ref, - file_type=EjvFileType.NON_GOV_DISBURSEMENT.value, - disbursement_status_code=DisbursementStatus.UPLOADED.value).save() + ejv_file = EjvFileModel(file_ref=file_ref, + file_type=EjvFileType.NON_GOV_DISBURSEMENT.value, + disbursement_status_code=DisbursementStatus.UPLOADED.value).save() ejv_file_id = ejv_file.id - ejv_header: EjvHeaderModel = EjvHeaderModel(disbursement_status_code=DisbursementStatus.UPLOADED.value, - ejv_file_id=ejv_file.id, payment_account_id=account.id).save() + ejv_header = EjvHeaderModel(disbursement_status_code=DisbursementStatus.UPLOADED.value, + ejv_file_id=ejv_file.id, payment_account_id=account.id).save() EjvLinkModel( link_id=invoice.id, link_type=EJVLinkType.INVOICE.value, @@ -1021,7 +1135,7 @@ def test_successful_ap_disbursement(session, app, client): ack_file_name = f'ACK.{file_ref}' - with open(ack_file_name, 'a+') as jv_file: + with open(ack_file_name, 'a+', encoding='utf-8') as jv_file: jv_file.write('') jv_file.close() @@ -1092,7 +1206,7 @@ def test_successful_ap_disbursement(session, app, client): feedback_file_name = f'FEEDBACK.{file_ref}' - with open(feedback_file_name, 'a+') as jv_file: + with open(feedback_file_name, 'a+', encoding='utf-8') as jv_file: jv_file.write(feedback_content) jv_file.close() @@ -1147,13 +1261,13 @@ def test_failure_ap_disbursement(session, app, client): factory_refund(invoice_id=refund_invoice.id) file_ref = f'INBOX.{datetime.now()}' - ejv_file: EjvFileModel = EjvFileModel(file_ref=file_ref, - file_type=EjvFileType.NON_GOV_DISBURSEMENT.value, - disbursement_status_code=DisbursementStatus.UPLOADED.value).save() + ejv_file = EjvFileModel(file_ref=file_ref, + file_type=EjvFileType.NON_GOV_DISBURSEMENT.value, + disbursement_status_code=DisbursementStatus.UPLOADED.value).save() ejv_file_id = ejv_file.id - ejv_header: EjvHeaderModel = EjvHeaderModel(disbursement_status_code=DisbursementStatus.UPLOADED.value, - ejv_file_id=ejv_file.id, payment_account_id=account.id).save() + ejv_header = EjvHeaderModel(disbursement_status_code=DisbursementStatus.UPLOADED.value, + ejv_file_id=ejv_file.id, payment_account_id=account.id).save() EjvLinkModel( link_id=invoice.id, link_type=EJVLinkType.INVOICE.value, @@ -1167,7 +1281,7 @@ def test_failure_ap_disbursement(session, app, client): ack_file_name = f'ACK.{file_ref}' - with open(ack_file_name, 'a+') as jv_file: + with open(ack_file_name, 'a+', encoding='utf-8') as jv_file: jv_file.write('') jv_file.close() @@ -1241,7 +1355,7 @@ def test_failure_ap_disbursement(session, app, client): feedback_file_name = f'FEEDBACK.{file_ref}' - with open(feedback_file_name, 'a+') as jv_file: + with open(feedback_file_name, 'a+', encoding='utf-8') as jv_file: jv_file.write(feedback_content) jv_file.close()