From 6fcdc1c2e5c283003241f93f7b9fe441e3b06482 Mon Sep 17 00:00:00 2001 From: Anthony Delosa Date: Mon, 10 Jun 2024 21:33:45 +1000 Subject: [PATCH] Support for Expanded IPM parameter file extraction. --- cardutil/cli/mci_ipm_param_to_csv.py | 49 ++- cardutil/config.py | 444 ++++++++++++++++++------- cardutil/mciipm.py | 90 +++-- docs/source/cli.rst | 14 +- tests/cli/test_mci_ipm_encode.py | 2 +- tests/cli/test_mci_ipm_param_to_csv.py | 17 +- tests/cli/test_mci_ipm_to_csv.py | 4 +- tests/test_mciipm.py | 52 ++- 8 files changed, 503 insertions(+), 169 deletions(-) diff --git a/cardutil/cli/mci_ipm_param_to_csv.py b/cardutil/cli/mci_ipm_param_to_csv.py index 1a51993..c856425 100644 --- a/cardutil/cli/mci_ipm_param_to_csv.py +++ b/cardutil/cli/mci_ipm_param_to_csv.py @@ -28,24 +28,49 @@ def cli_run(**kwargs): def cli_parser(): - parser = argparse.ArgumentParser(prog='mci_ipm_param_to_csv', description='Mastercard IPM parameter file to CSV') - parser.add_argument('in_filename', help='IPM Parameter file to process') - parser.add_argument('table_id', help='Parameter table to extract') - parser.add_argument('-o', '--out-filename') - parser.add_argument('--in-encoding') - parser.add_argument('--out-encoding') - parser.add_argument('--debug', action='store_true') - parser.add_argument('--no1014blocking', action='store_true') - parser.add_argument('--config-file', help='File containing cardutil configuration - JSON format') + parser = argparse.ArgumentParser( + prog="mci_ipm_param_to_csv", description="Mastercard IPM parameter file to CSV" + ) + parser.add_argument("in_filename", help="IPM Parameter file to process") + parser.add_argument("table_id", help="Parameter table to extract") + parser.add_argument("-o", "--out-filename") + parser.add_argument("--in-encoding") + parser.add_argument("--out-encoding") + parser.add_argument("--debug", action="store_true") + parser.add_argument("--no1014blocking", action="store_true") + parser.add_argument("--expanded", action="store_true") + parser.add_argument( + "--config-file", help="File containing cardutil configuration - JSON format" + ) add_version(parser) return parser -def mci_ipm_param_to_csv(in_param, out_csv, table_id, config=None, in_encoding=None, no1014blocking=False, **_): +def mci_ipm_param_to_csv( + in_param, + out_csv, + table_id, + config=None, + in_encoding=None, + no1014blocking=False, + expanded=False, + **_ +): blocked = not no1014blocking - vbs_in = mciipm.IpmParamReader(in_param, table_id, param_config=config, blocked=blocked, encoding=in_encoding) - csv_writer = csv.DictWriter(out_csv, fieldnames=config[table_id].keys(), extrasaction="ignore", lineterminator="\n") + vbs_in = mciipm.IpmParamReader( + in_param, + table_id, + param_config=config, + blocked=blocked, + encoding=in_encoding, + expanded=expanded, + ) + fieldnames = ["table_id", "effective_timestamp", "active_inactive_code"] + fieldnames.extend(config[table_id].keys()) + csv_writer = csv.DictWriter( + out_csv, fieldnames=fieldnames, extrasaction="ignore", lineterminator="\n" + ) csv_writer.writeheader() csv_writer.writerows(vbs_in) diff --git a/cardutil/config.py b/cardutil/config.py index a592900..4562a93 100644 --- a/cardutil/config.py +++ b/cardutil/config.py @@ -89,19 +89,20 @@ mci_parameter_tables ==================== -Provides configuration required to extract IPM parameter extracts +Provides configuration required to extract IPM parameter extracts. + +Expanded file format should be used - not compressed. +The effective timestamp, table_id and active inactive code should not +be included here - they are automatically included in the extract. .. code-block:: json { "IP0006T1": { - "effective_timestamp": {"start": 1, "end": 10}, - "active_inactive_code": {"start": 7, "end": 8}, - "table_id": {"start": 8, "end": 11}, - "card_program_id": {"start": 11, "end": 14}, - "data_element_id": {"start": 14, "end": 17}, - "data_element_name": {"start": 17, "end": 74}, - "data_element_format": {"start": 74, "end": 77} + "card_program_id": {"start": 19, "end": 22}, + "data_element_id": {"start": 22, "end": 25}, + "data_element_name": {"start": 25, "end": 82}, + "data_element_format": {"start": 82, "end": 85} }, "IP0040T1": {} } @@ -110,121 +111,340 @@ config = { "bit_config": { - "1": {"field_name": "Bitmap secondary", "field_type": "FIXED", "field_length": 8}, - "2": {"field_name": "PAN", "field_type": "LLVAR", "field_length": 0}, # "field_processor": "PAN"}, - "3": {"field_name": "Processing code", "field_type": "FIXED", "field_length": 6}, - "4": {"field_name": "Amount transaction", "field_type": "FIXED", "field_length": 12, - "field_python_type": "long"}, - "5": {"field_name": "Amount, Reconciliation", "field_type": "FIXED", "field_length": 12, - "field_python_type": "long"}, - "6": {"field_name": "Amount, Cardholder billing", "field_type": "FIXED", "field_length": 12, - "field_python_type": "long"}, - "9": {"field_name": "Conversion rate, Reconciliation", "field_type": "FIXED", "field_length": 8, - "field_python_type": "long"}, - "10": {"field_name": "Conversion rate, Cardholder billing", "field_type": "FIXED", "field_length": 8, - "field_python_type": "long"}, - "12": {"field_name": "Date/Time local transaction", "field_type": "FIXED", "field_length": 12, - "field_python_type": "datetime", "field_date_format": "%y%m%d%H%M%S"}, - "14": {"field_name": "Expiration date", "field_type": "FIXED", "field_length": 4}, - "22": {"field_name": "Point of service data code", "field_type": "FIXED", "field_length": 12}, - "23": {"field_name": "Card sequence number", "field_type": "FIXED", "field_length": 3}, + "1": { + "field_name": "Bitmap secondary", + "field_type": "FIXED", + "field_length": 8, + }, + "2": { + "field_name": "PAN", + "field_type": "LLVAR", + "field_length": 0, + }, # "field_processor": "PAN"}, + "3": { + "field_name": "Processing code", + "field_type": "FIXED", + "field_length": 6, + }, + "4": { + "field_name": "Amount transaction", + "field_type": "FIXED", + "field_length": 12, + "field_python_type": "long", + }, + "5": { + "field_name": "Amount, Reconciliation", + "field_type": "FIXED", + "field_length": 12, + "field_python_type": "long", + }, + "6": { + "field_name": "Amount, Cardholder billing", + "field_type": "FIXED", + "field_length": 12, + "field_python_type": "long", + }, + "9": { + "field_name": "Conversion rate, Reconciliation", + "field_type": "FIXED", + "field_length": 8, + "field_python_type": "long", + }, + "10": { + "field_name": "Conversion rate, Cardholder billing", + "field_type": "FIXED", + "field_length": 8, + "field_python_type": "long", + }, + "12": { + "field_name": "Date/Time local transaction", + "field_type": "FIXED", + "field_length": 12, + "field_python_type": "datetime", + "field_date_format": "%y%m%d%H%M%S", + }, + "14": { + "field_name": "Expiration date", + "field_type": "FIXED", + "field_length": 4, + }, + "22": { + "field_name": "Point of service data code", + "field_type": "FIXED", + "field_length": 12, + }, + "23": { + "field_name": "Card sequence number", + "field_type": "FIXED", + "field_length": 3, + }, "24": {"field_name": "Function code", "field_type": "FIXED", "field_length": 3}, - "25": {"field_name": "Message reason code", "field_type": "FIXED", "field_length": 4}, - "26": {"field_name": "Card acceptor business code", "field_type": "FIXED", "field_length": 4, - "field_python_type": "int"}, - "30": {"field_name": "Amounts, original", "field_type": "FIXED", "field_length": 24}, - "31": {"field_name": "Acquirer reference data", "field_type": "LLVAR", "field_length": 23}, - "32": {"field_name": "Acquiring institution ID code", "field_type": "LLVAR", "field_length": 0}, - "33": {"field_name": "Forwarding institution ID code", "field_type": "LLVAR", "field_length": 0}, - "37": {"field_name": "Retrieval reference number", "field_type": "FIXED", "field_length": 12}, + "25": { + "field_name": "Message reason code", + "field_type": "FIXED", + "field_length": 4, + }, + "26": { + "field_name": "Card acceptor business code", + "field_type": "FIXED", + "field_length": 4, + "field_python_type": "int", + }, + "30": { + "field_name": "Amounts, original", + "field_type": "FIXED", + "field_length": 24, + }, + "31": { + "field_name": "Acquirer reference data", + "field_type": "LLVAR", + "field_length": 23, + }, + "32": { + "field_name": "Acquiring institution ID code", + "field_type": "LLVAR", + "field_length": 0, + }, + "33": { + "field_name": "Forwarding institution ID code", + "field_type": "LLVAR", + "field_length": 0, + }, + "37": { + "field_name": "Retrieval reference number", + "field_type": "FIXED", + "field_length": 12, + }, "38": {"field_name": "Approval code", "field_type": "FIXED", "field_length": 6}, "40": {"field_name": "Service code", "field_type": "FIXED", "field_length": 3}, - "41": {"field_name": "Card acceptor terminal ID", "field_type": "FIXED", "field_length": 8}, - "42": {"field_name": "Card acceptor Id", "field_type": "FIXED", "field_length": 15}, - "43": {"field_name": "Card acceptor name/location", "field_type": "LLVAR", "field_length": 0, - "field_processor": "DE43", - "field_processor_config": r"(?P.+?) *\\(?P.+?) *\\(?P.+?) *\\" - r"(?P.{10})(?P.{3})(?P\S{3})$"}, - "48": {"field_name": "Additional data", "field_type": "LLLVAR", "field_length": 0, "field_processor": "PDS"}, - "49": {"field_name": "Currency code, Transaction", "field_type": "FIXED", "field_length": 3}, - "50": {"field_name": "Currency code, Reconciliation", "field_type": "FIXED", "field_length": 3}, - "51": {"field_name": "Currency code, Cardholder billing", "field_type": "FIXED", "field_length": 3}, - "54": {"field_name": "Amounts, additional", "field_type": "LLLVAR", "field_length": 0}, - "55": {"field_name": "ICC system related data", "field_type": "LLLVAR", "field_length": 255, - "field_processor": "ICC"}, - "62": {"field_name": "Additional data 2", "field_type": "LLLVAR", "field_length": 0, "field_processor": "PDS"}, - "63": {"field_name": "Transaction lifecycle Id", "field_type": "LLLVAR", "field_length": 16}, - "71": {"field_name": "Message number", "field_type": "FIXED", "field_length": 8, "field_python_type": "int"}, + "41": { + "field_name": "Card acceptor terminal ID", + "field_type": "FIXED", + "field_length": 8, + }, + "42": { + "field_name": "Card acceptor Id", + "field_type": "FIXED", + "field_length": 15, + }, + "43": { + "field_name": "Card acceptor name/location", + "field_type": "LLVAR", + "field_length": 0, + "field_processor": "DE43", + "field_processor_config": r"(?P.+?) *\\(?P.+?) *\\(?P.+?) *\\" + r"(?P.{10})(?P.{3})(?P\S{3})$", + }, + "48": { + "field_name": "Additional data", + "field_type": "LLLVAR", + "field_length": 0, + "field_processor": "PDS", + }, + "49": { + "field_name": "Currency code, Transaction", + "field_type": "FIXED", + "field_length": 3, + }, + "50": { + "field_name": "Currency code, Reconciliation", + "field_type": "FIXED", + "field_length": 3, + }, + "51": { + "field_name": "Currency code, Cardholder billing", + "field_type": "FIXED", + "field_length": 3, + }, + "54": { + "field_name": "Amounts, additional", + "field_type": "LLLVAR", + "field_length": 0, + }, + "55": { + "field_name": "ICC system related data", + "field_type": "LLLVAR", + "field_length": 255, + "field_processor": "ICC", + }, + "62": { + "field_name": "Additional data 2", + "field_type": "LLLVAR", + "field_length": 0, + "field_processor": "PDS", + }, + "63": { + "field_name": "Transaction lifecycle Id", + "field_type": "LLLVAR", + "field_length": 16, + }, + "71": { + "field_name": "Message number", + "field_type": "FIXED", + "field_length": 8, + "field_python_type": "int", + }, "72": {"field_name": "Data record", "field_type": "LLLVAR", "field_length": 0}, "73": {"field_name": "Date, Action", "field_type": "FIXED", "field_length": 6}, - "93": {"field_name": "Transaction destination institution ID", "field_type": "LLVAR", "field_length": 0}, - "94": {"field_name": "Transaction originator institution ID", "field_type": "LLVAR", "field_length": 0}, - "95": {"field_name": "Card issuer reference data", "field_type": "LLVAR", "field_length": 10}, - "100": {"field_name": "Receiving institution ID", "field_type": "LLVAR", "field_length": 11}, - "111": {"field_name": "Amount, currency conversion assignment", "field_type": "LLLVAR", "field_length": 0}, - "123": {"field_name": "Additional data 3", "field_type": "LLLVAR", "field_length": 0, "field_processor": "PDS"}, - "124": {"field_name": "Additional data 4", "field_type": "LLLVAR", "field_length": 0, "field_processor": "PDS"}, - "125": {"field_name": "Additional data 5", "field_type": "LLLVAR", "field_length": 0, "field_processor": "PDS"}, - "127": {"field_name": "Network data", "field_type": "LLLVAR", "field_length": 0}}, - + "93": { + "field_name": "Transaction destination institution ID", + "field_type": "LLVAR", + "field_length": 0, + }, + "94": { + "field_name": "Transaction originator institution ID", + "field_type": "LLVAR", + "field_length": 0, + }, + "95": { + "field_name": "Card issuer reference data", + "field_type": "LLVAR", + "field_length": 10, + }, + "100": { + "field_name": "Receiving institution ID", + "field_type": "LLVAR", + "field_length": 11, + }, + "111": { + "field_name": "Amount, currency conversion assignment", + "field_type": "LLLVAR", + "field_length": 0, + }, + "123": { + "field_name": "Additional data 3", + "field_type": "LLLVAR", + "field_length": 0, + "field_processor": "PDS", + }, + "124": { + "field_name": "Additional data 4", + "field_type": "LLLVAR", + "field_length": 0, + "field_processor": "PDS", + }, + "125": { + "field_name": "Additional data 5", + "field_type": "LLLVAR", + "field_length": 0, + "field_processor": "PDS", + }, + "127": { + "field_name": "Network data", + "field_type": "LLLVAR", + "field_length": 0, + }, + }, "output_data_elements": [ - "MTI", "DE2", "DE3", "DE4", "DE12", "DE14", "DE22", "DE23", "DE24", "DE25", "DE26", - "DE30", "DE31", "DE33", "DE37", "DE38", "DE40", "DE41", "DE42", "DE48", "DE49", - "DE50", "DE63", "DE71", "DE73", "DE93", "DE94", "DE95", "DE100", "PDS0023", - "PDS0052", "PDS0122", "PDS0148", "PDS0158", "PDS0165", "DE43_NAME", "DE43_SUBURB", - "DE43_POSTCODE", "ICC_DATA" + "MTI", + "DE2", + "DE3", + "DE4", + "DE12", + "DE14", + "DE22", + "DE23", + "DE24", + "DE25", + "DE26", + "DE30", + "DE31", + "DE33", + "DE37", + "DE38", + "DE40", + "DE41", + "DE42", + "DE48", + "DE49", + "DE50", + "DE63", + "DE71", + "DE73", + "DE93", + "DE94", + "DE95", + "DE100", + "PDS0023", + "PDS0052", + "PDS0122", + "PDS0148", + "PDS0158", + "PDS0165", + "DE43_NAME", + "DE43_SUBURB", + "DE43_POSTCODE", + "ICC_DATA", ], "mci_parameter_tables": { "IP0006T1": { - "effective_timestamp": {"start": 1, "end": 10}, - "active_inactive_code": {"start": 7, "end": 8}, - "table_id": {"start": 8, "end": 11}, - "card_program_id": {"start": 11, "end": 14}, - "data_element_id": {"start": 14, "end": 17}, - "data_element_name": {"start": 17, "end": 74}, - "data_element_format": {"start": 74, "end": 77} + "card_program_id": {"start": 19, "end": 22}, + "data_element_id": {"start": 22, "end": 25}, + "data_element_name": {"start": 25, "end": 82}, + "data_element_format": {"start": 82, "end": 85}, + "data_element_minimum_length": {"start": 85, "end": 88}, + "data_element_mastercard_maximum_length": {"start": 88, "end": 91}, + "data_element_iso_maximum_length": {"start": 91, "end": 94}, + "de_lll_size": {"start": 94, "end": 95}, + "data_element_subfields": {"start": 95, "end": 97}, }, "IP0040T1": { - "effective_timestamp": {"start": 1, "end": 7}, - "active_inactive_code": {"start": 7, "end": 8}, - "table_id": {"start": 8, "end": 11}, - "low_range": {"start": 11, "end": 30}, - "gcms_product": {"start": 30, "end": 33}, - "high_range": {"start": 33, "end": 52}, - "card_program_identifier": {"start": 52, "end": 55}, - "card_program_priority": {"start": 55, "end": 57}, - "member_id": {"start": 57, "end": 68}, - "product_type": {"start": 68, "end": 69}, - "endpoint": {"start": 69, "end": 76}, - "card_country_alpha": {"start": 76, "end": 79}, - "card_country_numeric": {"start": 79, "end": 82}, - "card_region": {"start": 82, "end": 83}, - "product_class": {"start": 83, "end": 86}, - "tran_routing_ind": {"start": 86, "end": 87}, - "first_present_reassign_ind": {"start": 87, "end": 88}, - "product_reassign_switch": {"start": 88, "end": 89}, - "pwcb_optin_switch": {"start": 89, "end": 90}, - "licenced_product_id": {"start": 90, "end": 93}, - "mapping_service_ind": {"start": 93, "end": 94}, - "alm_participation_ind": {"start": 94, "end": 95}, - "alm_activation_date": {"start": 95, "end": 101}, - "cardholder_billing_currency_default": {"start": 101, "end": 104}, - "cardholder_billing_currency_default_exponent": {"start": 104, "end": 105}, - "cardholder_bill_primary_currency": {"start": 105, "end": 133}, - "chip_to_magstripe_conversion_service_indicator": {"start": 133, "end": 134}, - "floor_exp_date": {"start": 134, "end": 140}, - "co_brand_participation_switch": {"start": 140, "end": 141}, - "spend_control_switch": {"start": 141, "end": 142}, - "merchant_cleansing_service_participation": {"start": 142, "end": 145}, - "merchant_cleansing_activation_date": {"start": 145, "end": 151}, - "paypass_enabled_indicator": {"start": 151, "end": 152}, - "rate_type_indicator": {"start": 152, "end": 153}, - "psn_route_indicator": {"start": 153, "end": 154}, - "cash_back_wo_purchase_ind": {"start": 154, "end": 155} + "issuer_account_range_low": {"start": 19, "end": 38}, + "gcms_product_id": {"start": 38, "end": 41}, + "issuer_account_range_high": {"start": 41, "end": 60}, + "card_program_identifier": {"start": 60, "end": 63}, + "issuer_card_program_identifier_priority_code": {"start": 63, "end": 65}, + "member_id": {"start": 65, "end": 76}, + "product_type_id": {"start": 76, "end": 77}, + "endpoint": {"start": 77, "end": 84}, + "card_country_alpha": {"start": 84, "end": 87}, + "card_country_numeric": {"start": 87, "end": 90}, + "region": {"start": 90, "end": 91}, + "product_class": {"start": 91, "end": 94}, + "transaction_routing_indicator": {"start": 94, "end": 95}, + "first_presentment_reassignment_switch": {"start": 95, "end": 96}, + "product_reassignment_switch": {"start": 96, "end": 97}, + "pwcb_opt_in_switch": {"start": 97, "end": 98}, + "licenced_product_id": {"start": 98, "end": 101}, + "mapping_service_ind": {"start": 101, "end": 102}, + "alm_participation_ind": {"start": 102, "end": 103}, + "alm_activation_date": {"start": 103, "end": 109}, + "cardholder_billing_currency_default": {"start": 109, "end": 112}, + "cardholder_billing_currency_exponent_default": {"start": 112, "end": 113}, + "cardholder_bill_primary_currency": {"start": 113, "end": 141}, + "chip_to_magnetic_conversion_service_indicator": {"start": 141, "end": 142}, + "floor_expiration_date": {"start": 142, "end": 148}, + "co_brand_participation_switch": {"start": 148, "end": 149}, + "spend_control_switch": {"start": 149, "end": 150}, + "merchant_cleansing_service_participation": {"start": 150, "end": 153}, + "merchant_cleansing_activation_date": {"start": 153, "end": 159}, + "paypass_enabled_indicator": {"start": 159, "end": 160}, + "regulated_rate_type_indicator": {"start": 160, "end": 161}, + "psn_route_indicator": {"start": 161, "end": 162}, + "cash_back_without_purchase_indicator": {"start": 162, "end": 163}, + "repower_reload_participation_indicator": {"start": 164, "end": 165}, + "moneysend_indicator": {"start": 165, "end": 166}, + "durban_regulated_rate_indicator": {"start": 166, "end": 167}, + "cash_access_only_participating_indicator": {"start": 167, "end": 168}, + "authentication_indicator": {"start": 168, "end": 169}, }, "IP0075T1": { - "mcc_code": {"start": 12, "end": 16}, - "cab_code": {"start": 16, "end": 20} - } - } + "card_acceptor_business_code_mcc": {"start": 19, "end": 24}, + "card_acceptor_business_cab_program": {"start": 24, "end": 28}, + "card_acceptor_business_cab_program_life_cycle_indicator": { + "start": 28, + "end": 29, + }, + "card_acceptor_business_cab_type": {"start": 29, "end": 30}, + "card_acceptor_business_cab_life_cycle_indicator": {"start": 30, "end": 31}, + }, + "IP0095T1": { + "card_program_identifier": {"start": 19, "end": 22}, + "business_service_arrangement_type": {"start": 22, "end": 23}, + "business_service_id_code": {"start": 23, "end": 29}, + "interchange_rate_designator_ird": {"start": 29, "end": 31}, + "card_acceptor_business_cab_program": {"start": 31, "end": 35}, + "life_cycle_indicator": {"start": 35, "end": 36}, + }, + }, } diff --git a/cardutil/mciipm.py b/cardutil/mciipm.py index 801be16..69da3cd 100644 --- a/cardutil/mciipm.py +++ b/cardutil/mciipm.py @@ -379,11 +379,25 @@ class IpmParamReader(VbsReader): _IP0000T1_TABLE_ID = slice(19, 27) _IP0000T1_TABLE_SUB_ID = slice(243, 246) - # table type for all except IP0000T1 - get this 3 letter code from self.table_keys - _TABLE_SUB_ID = slice(8, 11) - - def __init__(self, param_file: typing.BinaryIO, table_id: str, encoding: str = None, param_config: dict = None, - **kwargs): + # compressed table type for all except IP0000T1 - get this 3 letter code from self.table_keys + _C_EFF_TIMESTAMP = slice(0, 7) + _C_ACTIVE_INACTIVE_CODE = slice(7, 8) + _C_TABLE_SUB_ID = slice(8, 11) + + # expanded table common fields + _X_EFF_TIMESTAMP = slice(0, 10) + _X_ACTIVE_INACTIVE_CODE = slice(10, 11) + _X_TABLE_ID = slice(11, 19) + + def __init__( + self, + param_file: typing.BinaryIO, + table_id: str, + encoding: str = None, + param_config: dict = None, + expanded: bool = False, + **kwargs, + ): """ Create a new IpmParamReader @@ -396,6 +410,7 @@ def __init__(self, param_file: typing.BinaryIO, table_id: str, encoding: str = N self.param_config = param_config if param_config else config.config.get('mci_parameter_tables') self.table_id = table_id self.table_index = dict() + self.expanded = expanded super(IpmParamReader, self).__init__(param_file, **kwargs) # check if config available for table id @@ -422,18 +437,49 @@ def __init__(self, param_file: typing.BinaryIO, table_id: str, encoding: str = N def __next__(self) -> dict: while True: record = super(IpmParamReader, self).__next__() - record_table_id = self.table_index.get(record[self._TABLE_SUB_ID].decode(self.encoding)) + if self.expanded: + record_table_id = record[self._X_TABLE_ID].decode(self.encoding) + record_effective_timestamp = record[self._X_EFF_TIMESTAMP].decode( + self.encoding + ) + record_active_inactive_code = record[ + self._X_ACTIVE_INACTIVE_CODE + ].decode(self.encoding) + else: + record_table_id = self.table_index.get( + record[self._C_TABLE_SUB_ID].decode(self.encoding) + ) + record_effective_timestamp = record[self._C_EFF_TIMESTAMP].decode( + self.encoding + ) + record_active_inactive_code = record[ + self._C_ACTIVE_INACTIVE_CODE + ].decode(self.encoding) + + LOGGER.debug(f"{record_table_id=}, {record=}") if record_table_id == self.table_id: - record_dict = dict() + record_dict = { + "table_id": record_table_id, + "effective_timestamp": record_effective_timestamp, + "active_inactive_code": record_active_inactive_code, + } for field in self.param_config[record_table_id]: record_dict[field] = self._get_param_field(record, field) return record_dict def _get_param_field(self, record, field): - table_id = self.table_index.get(record[self._TABLE_SUB_ID].decode(self.encoding)) + field_offset = 0 + if self.expanded: + record_table_id = record[self._X_TABLE_ID].decode(self.encoding) + else: + record_table_id = self.table_index.get( + record[self._C_TABLE_SUB_ID].decode(self.encoding) + ) + field_offset = -8 # all fields should be offset by this value return record[ - self.param_config[ - table_id][field]["start"]:self.param_config[table_id][field]["end"]].decode(self.encoding) + self.param_config[record_table_id][field]["start"] + field_offset: + self.param_config[record_table_id][field]["end"] + field_offset + ].decode(self.encoding) class VbsWriter(object): @@ -662,14 +708,19 @@ def vbs_bytes_to_list(vbs_bytes: bytes, **kwargs) -> list: def ipm_info(input_data: typing.BinaryIO) -> dict: """ Use this function to inspect an IPM file and provide details + :param input_data: The file like object of IPM data - :return: a dictionary containing file information - { - "isValidIPM": True, - "reason": "If not valid, describes the reason" - "isBlocked": True, - "encoding": "latin1", - } + :return: a dictionary containing file information: + + .. code-block:: text + + { + "isValidIPM": True, + "reason": "If not valid, describes the reason" + "isBlocked": True, + "encoding": "latin1", + } + """ output = {"isValidIPM": False} @@ -686,7 +737,8 @@ def ipm_info(input_data: typing.BinaryIO) -> dict: length_bytes = sample_data[:4] record_length = struct.unpack(">I", length_bytes)[0] if record_length > 1000: - output["reason"] = f"First IPM record has large record size ({record_length}) which usually indicates a file issue" + output["reason"] = (f"First IPM record has large record size ({record_length}) which" + f" usually indicates a file issue") return output # check the bitmap to make sure it has a valid bit config @@ -720,7 +772,7 @@ def block_1014_check(sample_data): def bitmap_check(bitmap: bytes) -> (bool, str): - LOGGER.debug(hexdump.hexdump(bitmap,result='return')) + LOGGER.debug(hexdump.hexdump(bitmap, result='return')) bitarray = BitArray.BitArray() bitarray.frombytes(bitmap) bits = bitarray.tolist() diff --git a/docs/source/cli.rst b/docs/source/cli.rst index 89d2786..de4f73c 100644 --- a/docs/source/cli.rst +++ b/docs/source/cli.rst @@ -157,7 +157,7 @@ Extracts parameter tables from the IPM parameter extracts files usage: mci_ipm_param_to_csv [-h] [-o OUT_FILENAME] [--in-encoding IN_ENCODING] [--out-encoding OUT_ENCODING] - [--no1014blocking] + [--no1014blocking] [--expanded] [--config-file CONFIG_FILE] [--version] in_filename table_id @@ -173,6 +173,7 @@ Extracts parameter tables from the IPM parameter extracts files --in-encoding IN_ENCODING --out-encoding OUT_ENCODING --no1014blocking + --expanded --config-file CONFIG_FILE File containing cardutil configuration - JSON format --version show program's version number and exit @@ -210,13 +211,10 @@ See :py:mod:`cardutil.config`. ], "mci_parameter_tables": { "IP0006T1": { - "effective_timestamp": {"start": 1, "end": 10}, - "active_inactive_code": {"start": 7, "end": 8}, - "table_id": {"start": 8, "end": 11}, - "card_program_id": {"start": 11, "end": 14}, - "data_element_id": {"start": 14, "end": 17}, - "data_element_name": {"start": 17, "end": 74}, - "data_element_format": {"start": 74, "end": 77} + "card_program_id": {"start": 19, "end": 22}, + "data_element_id": {"start": 22, "end": 25}, + "data_element_name": {"start": 25, "end": 82}, + "data_element_format": {"start": 82, "end": 85} } } } diff --git a/tests/cli/test_mci_ipm_encode.py b/tests/cli/test_mci_ipm_encode.py index 31fbc2d..6113195 100644 --- a/tests/cli/test_mci_ipm_encode.py +++ b/tests/cli/test_mci_ipm_encode.py @@ -48,7 +48,7 @@ def test_mci_ipm_encode_input_params(self): with tempfile.NamedTemporaryFile(mode='wb', delete=False) as out_ipm: out_ipm.write( b'\x00\x00\x00\x1a' - b'\xf0\xf1\xf0\xf0' # mti(4) + b'\xf0\xf1\xf0\xf0' # mti(4) b'\x80\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' # bitmap(16) b'\xf0\xf1\xf0\xf0\xf0\xf0' # data(6) ) diff --git a/tests/cli/test_mci_ipm_param_to_csv.py b/tests/cli/test_mci_ipm_param_to_csv.py index ea3bc17..8db249c 100644 --- a/tests/cli/test_mci_ipm_param_to_csv.py +++ b/tests/cli/test_mci_ipm_param_to_csv.py @@ -17,7 +17,7 @@ def test_mci_ipm_param_to_csv_parser(self): self.assertEqual( args, {'in_filename': 'file1.ipm', 'out_filename': None, 'in_encoding': None, 'table_id': 'IP0000T1', - 'out_encoding': None, 'no1014blocking': False, 'config_file': None, 'debug': False}) + 'out_encoding': None, 'no1014blocking': False, 'config_file': None, 'debug': False, 'expanded': False}) def test_ipm_to_csv_input_params(self): """ @@ -71,14 +71,17 @@ def test_extract_ip0040t1(self): csv_records = test_csv_data.split('\n') # check the header - config_header_keys = list(MCI_PARAMETER_CONFIG.get('IP0040T1').keys()) + config_header_keys = ["table_id", "effective_timestamp", "active_inactive_code"] + config_header_keys.extend(MCI_PARAMETER_CONFIG.get('IP0040T1').keys()) csv_header_keys = csv_records[0].split(',') + print(csv_header_keys) self.assertEqual(config_header_keys, csv_header_keys) # check the record extracted - self.assertEqual(csv_records[1], '711114,A,036,5116545113000000000,MCC,5116545113999999999,MCC,02,00000001527,' - '1,0084563,AUS,036,C,MCC, ,N,N,Y,MCC, ,N,000000,036,2,' - '0000000000000000000000000001, ,000000,N,Y, ,000000,N,N,N,N') + print(csv_records[1]) + self.assertEqual(csv_records[1], 'IP0040T1,1711114,A,5116545113000000000,MCC,5116545113999999999,' + 'MCC,02,00000001527,1,0084563,AUS,036,C,MCC, ,N,N,Y,MCC, ,N,000000,036,2,' + '0000000000000000000000000001, ,000000,N,Y, ,000000,N,N,N,N,,,,,') def test_extract_ip0040t1_no_records(self): param_file_data = [ @@ -102,7 +105,9 @@ def test_extract_ip0040t1_no_records(self): self.assertEqual(len(csv_records), 2, msg="Only 2 record returned -- header, dummy") # check the header - config_header_keys = list(MCI_PARAMETER_CONFIG.get('IP0040T1').keys()) + config_header_keys = ["table_id", "effective_timestamp", "active_inactive_code"] + config_header_keys.extend(MCI_PARAMETER_CONFIG.get('IP0040T1').keys()) + csv_header_keys = csv_records[0].split(',') self.assertEqual(config_header_keys, csv_header_keys) diff --git a/tests/cli/test_mci_ipm_to_csv.py b/tests/cli/test_mci_ipm_to_csv.py index e0bda92..d91e9ba 100644 --- a/tests/cli/test_mci_ipm_to_csv.py +++ b/tests/cli/test_mci_ipm_to_csv.py @@ -112,7 +112,7 @@ def test_ipm_to_csv_exception_max_reclen(self): f = io.StringIO() with contextlib.redirect_stdout(f): result = mci_ipm_to_csv.cli_run(in_filename=in_ipm_name, out_encoding='ascii') - output = f.getvalue() #.splitlines() + output = f.getvalue() # .splitlines() os.remove(in_ipm_name) os.remove(in_ipm_name + '.csv') print(output) @@ -124,7 +124,7 @@ def test_ipm_to_csv_exception_bad_encoding(self): b'\xf0\xf1\xf0\xf0' # mti (cp037) b'\x80\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' b'nXmXlX\xFF\xFF\x00\x00' - ) + ) with tempfile.NamedTemporaryFile(mode='w+b', delete=False) as in_ipm: in_ipm.write(in_ipm_data) diff --git a/tests/test_mciipm.py b/tests/test_mciipm.py index fa3e6db..ae9ca9e 100644 --- a/tests/test_mciipm.py +++ b/tests/test_mciipm.py @@ -335,10 +335,44 @@ def test_ipm_param_reader(self): test_param_vbs.write_many(param_file_data) test_param_stream.seek(0) - reader = IpmParamReader(test_param_stream, table_id='IP0040T1') + reader = IpmParamReader(test_param_stream, table_id="IP0040T1") - for record in reader: - print(record) + output = list(reader) + print(output) + + self.assertEqual(1, len(output)) + + def test_ipm_param_reader_expanded(self): + """ + parameter files can be expanded. This test reads an expanded record + """ + param_file_data = [ + b"2011101414AIP0000T1IP0000T1 TABLE LIST " + + 188 * b"." + + b"001", + b"2014101414AIP0000T1IP0040T1 ACCOUNT RANGE TABLE " + + 188 * b"." + + b"036", + b"TRAILER RECORD IP0000T1 00000218 ", + b"........xxx....", # dummy record + b"2024012414AIP0040T15417750570000000000MPL5417751329999999999MCC" + b"010000000177510080140USA8401MPL NYNMPL7N0000008402" + b"0000000000000000000000000000 000000NN 000000NNNN0NUNN0N N ", + ] + + with io.BytesIO() as test_param_stream: + with VbsWriter(test_param_stream, blocked=True) as test_param_vbs: + test_param_vbs.write_many(param_file_data) + + test_param_stream.seek(0) + reader = IpmParamReader( + test_param_stream, table_id="IP0040T1", expanded=True + ) + + output = list(reader) + print(output) + + self.assertEqual(1, len(output)) class MciIpmInfoTestCase(unittest.TestCase): @@ -411,12 +445,12 @@ def test_ipm_info_vbs_1014(self): def test_ipm_info_blocked_2028(self): data = io.BytesIO( - b'\x00\x00\x00\xff' + # length(4) - b'1234' + # mti(4) - b'\x70' + (b'\x00' * 15) + # Bitmap(16) - (b' ' * 988) + # data(988) - b'\x40\x40' + # block(2) - (b' ' * 1012) + b'\x40\x40' # record 2 with marker + b'\x00\x00\x00\xff' + # length(4) + b'1234' + # mti(4) + b'\x70' + (b'\x00' * 15) + # Bitmap(16) + (b' ' * 988) + # data(988) + b'\x40\x40' + # block(2) + (b' ' * 1012) + b'\x40\x40' # record 2 with marker ) # data = io.BytesIO(b'\x00\x00\x00\xff' + (b' ' * 1008) + b'\x40\x40' + (b' ' * 1012) + b'\x40\x40') info = ipm_info(data)