From ad31e02616786480cfbadeb39fcaf357e68b4133 Mon Sep 17 00:00:00 2001 From: marination Date: Thu, 30 Mar 2023 21:03:03 +0530 Subject: [PATCH 01/17] feat: Store Party bank details in party records (Customer/Supplier/Employee/Shareholder) --- .../bank_transaction/bank_transaction.json | 29 +++++++++++++++-- .../bank_transaction/bank_transaction.py | 11 +++++++ .../doctype/shareholder/shareholder.json | 32 +++++++++++++++++-- erpnext/buying/doctype/supplier/supplier.json | 30 ++++++++++++++++- .../selling/doctype/customer/customer.json | 25 ++++++++++++++- erpnext/setup/doctype/employee/employee.json | 15 +++++++-- 6 files changed, 132 insertions(+), 10 deletions(-) diff --git a/erpnext/accounts/doctype/bank_transaction/bank_transaction.json b/erpnext/accounts/doctype/bank_transaction/bank_transaction.json index 768d2f0fa4..1543fdb894 100644 --- a/erpnext/accounts/doctype/bank_transaction/bank_transaction.json +++ b/erpnext/accounts/doctype/bank_transaction/bank_transaction.json @@ -33,7 +33,11 @@ "unallocated_amount", "party_section", "party_type", - "party" + "party", + "column_break_3czf", + "bank_party_name", + "bank_party_no", + "bank_party_iban" ], "fields": [ { @@ -202,11 +206,30 @@ "fieldtype": "Data", "label": "Transaction Type", "length": 50 + }, + { + "fieldname": "column_break_3czf", + "fieldtype": "Column Break" + }, + { + "fieldname": "bank_party_name", + "fieldtype": "Data", + "label": "Party Name (Bank Statement)" + }, + { + "fieldname": "bank_party_no", + "fieldtype": "Data", + "label": "Party Account No. (Bank Statement)" + }, + { + "fieldname": "bank_party_iban", + "fieldtype": "Data", + "label": "Party IBAN (Bank Statement)" } ], "is_submittable": 1, "links": [], - "modified": "2022-05-29 18:36:50.475964", + "modified": "2023-03-30 15:30:46.485683", "modified_by": "Administrator", "module": "Accounts", "name": "Bank Transaction", @@ -260,4 +283,4 @@ "states": [], "title_field": "bank_account", "track_changes": 1 -} +} \ No newline at end of file diff --git a/erpnext/accounts/doctype/bank_transaction/bank_transaction.py b/erpnext/accounts/doctype/bank_transaction/bank_transaction.py index fcbaf329f5..676c71910b 100644 --- a/erpnext/accounts/doctype/bank_transaction/bank_transaction.py +++ b/erpnext/accounts/doctype/bank_transaction/bank_transaction.py @@ -8,6 +8,17 @@ from erpnext.controllers.status_updater import StatusUpdater class BankTransaction(StatusUpdater): + # TODO + # On BT save: + # - Match by account no/iban in Customer/Supplier/Employee + # - Match by Party Name + # - If match found, set party type and party name. + + # On submit/update after submit + # - Create/Update a Bank Party Map record + # - User can edit after submit. + # - If changes in party/party name after submit, edit bank party map (which should edit all transactions with same account no/iban/bank party name) + def after_insert(self): self.unallocated_amount = abs(flt(self.withdrawal) - flt(self.deposit)) diff --git a/erpnext/accounts/doctype/shareholder/shareholder.json b/erpnext/accounts/doctype/shareholder/shareholder.json index e94aea94b7..2be2a2fc42 100644 --- a/erpnext/accounts/doctype/shareholder/shareholder.json +++ b/erpnext/accounts/doctype/shareholder/shareholder.json @@ -1,4 +1,5 @@ { + "actions": [], "autoname": "naming_series:", "creation": "2017-12-25 16:50:53.878430", "doctype": "DocType", @@ -19,7 +20,11 @@ "contact_html", "section_break_3", "share_balance", - "contact_list" + "contact_list", + "bank_details_section", + "bank_account_no", + "column_break_tyo0", + "iban" ], "fields": [ { @@ -109,13 +114,33 @@ "hidden": 1, "label": "Contact List", "read_only": 1 + }, + { + "fieldname": "bank_details_section", + "fieldtype": "Section Break", + "label": "Bank Details" + }, + { + "fieldname": "bank_account_no", + "fieldtype": "Data", + "label": "Bank Account No" + }, + { + "fieldname": "column_break_tyo0", + "fieldtype": "Column Break" + }, + { + "fieldname": "iban", + "fieldtype": "Data", + "label": "IBAN" } ], - "modified": "2019-11-17 23:24:11.395882", + "links": [], + "modified": "2023-03-30 16:00:55.087823", "modified_by": "Administrator", "module": "Accounts", "name": "Shareholder", - "name_case": "Title Case", + "naming_rule": "By \"Naming Series\" field", "owner": "Administrator", "permissions": [ { @@ -158,6 +183,7 @@ "search_fields": "folio_no", "sort_field": "modified", "sort_order": "DESC", + "states": [], "title_field": "title", "track_changes": 1 } \ No newline at end of file diff --git a/erpnext/buying/doctype/supplier/supplier.json b/erpnext/buying/doctype/supplier/supplier.json index 1bf7f589e2..a80dcfe538 100644 --- a/erpnext/buying/doctype/supplier/supplier.json +++ b/erpnext/buying/doctype/supplier/supplier.json @@ -52,6 +52,11 @@ "supplier_primary_address", "primary_address", "accounting_tab", + "bank_details_section", + "bank_account_no", + "column_break_n8mz", + "iban", + "section_break_ow3k", "payment_terms", "accounts", "settings_tab", @@ -445,6 +450,29 @@ { "fieldname": "column_break_59", "fieldtype": "Column Break" + }, + { + "fieldname": "bank_details_section", + "fieldtype": "Section Break", + "label": "Bank Details" + }, + { + "fieldname": "bank_account_no", + "fieldtype": "Data", + "label": "Bank Account No" + }, + { + "fieldname": "column_break_n8mz", + "fieldtype": "Column Break" + }, + { + "fieldname": "iban", + "fieldtype": "Data", + "label": "IBAN" + }, + { + "fieldname": "section_break_ow3k", + "fieldtype": "Section Break" } ], "icon": "fa fa-user", @@ -457,7 +485,7 @@ "link_fieldname": "party" } ], - "modified": "2023-02-18 11:05:50.592270", + "modified": "2023-03-30 15:50:40.241257", "modified_by": "Administrator", "module": "Buying", "name": "Supplier", diff --git a/erpnext/selling/doctype/customer/customer.json b/erpnext/selling/doctype/customer/customer.json index c133cd3152..5dc6a72da8 100644 --- a/erpnext/selling/doctype/customer/customer.json +++ b/erpnext/selling/doctype/customer/customer.json @@ -61,6 +61,10 @@ "tax_category", "tax_withholding_category", "accounting_tab", + "bank_details_section", + "bank_account_no", + "column_break_xtwg", + "iban", "credit_limit_section", "payment_terms", "credit_limits", @@ -555,6 +559,25 @@ { "fieldname": "column_break_54", "fieldtype": "Column Break" + }, + { + "fieldname": "bank_details_section", + "fieldtype": "Section Break", + "label": "Bank Details" + }, + { + "fieldname": "bank_account_no", + "fieldtype": "Data", + "label": "Bank Account No" + }, + { + "fieldname": "column_break_xtwg", + "fieldtype": "Column Break" + }, + { + "fieldname": "iban", + "fieldtype": "Data", + "label": "IBAN" } ], "icon": "fa fa-user", @@ -568,7 +591,7 @@ "link_fieldname": "party" } ], - "modified": "2023-02-18 11:04:46.343527", + "modified": "2023-03-30 15:45:44.387975", "modified_by": "Administrator", "module": "Selling", "name": "Customer", diff --git a/erpnext/setup/doctype/employee/employee.json b/erpnext/setup/doctype/employee/employee.json index 99693d9091..6cb4292226 100644 --- a/erpnext/setup/doctype/employee/employee.json +++ b/erpnext/setup/doctype/employee/employee.json @@ -78,7 +78,9 @@ "salary_mode", "bank_details_section", "bank_name", + "column_break_heye", "bank_ac_no", + "iban", "personal_details", "marital_status", "family_background", @@ -804,17 +806,26 @@ { "fieldname": "column_break_104", "fieldtype": "Column Break" + }, + { + "fieldname": "column_break_heye", + "fieldtype": "Column Break" + }, + { + "depends_on": "eval:doc.salary_mode == 'Bank'", + "fieldname": "iban", + "fieldtype": "Data", + "label": "IBAN" } ], "icon": "fa fa-user", "idx": 24, "image_field": "image", "links": [], - "modified": "2022-09-13 10:27:14.579197", + "modified": "2023-03-30 15:57:05.174592", "modified_by": "Administrator", "module": "Setup", "name": "Employee", - "name_case": "Title Case", "naming_rule": "By \"Naming Series\" field", "owner": "Administrator", "permissions": [ From e7745033dfc819bae534cc78a2987210c1fbcdf9 Mon Sep 17 00:00:00 2001 From: marination Date: Fri, 31 Mar 2023 16:11:00 +0530 Subject: [PATCH 02/17] feat: Party auto-matcher from Bank Transaction data - Created Bank Party Mapper - Created class to auto match by account/iban or party name/description(fuzzy) - Automatch and set in transaction or create mapper - `rapidfuzz` introduced --- .../doctype/bank_party_mapper/__init__.py | 0 .../bank_party_mapper/bank_party_mapper.js | 8 + .../bank_party_mapper/bank_party_mapper.json | 80 +++++++++ .../bank_party_mapper/bank_party_mapper.py | 9 + .../test_bank_party_mapper.py | 9 + .../bank_transaction/auto_match_party.py | 157 ++++++++++++++++++ .../bank_transaction/bank_transaction.json | 14 +- .../bank_transaction/bank_transaction.py | 35 +++- pyproject.toml | 1 + 9 files changed, 301 insertions(+), 12 deletions(-) create mode 100644 erpnext/accounts/doctype/bank_party_mapper/__init__.py create mode 100644 erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.js create mode 100644 erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.json create mode 100644 erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.py create mode 100644 erpnext/accounts/doctype/bank_party_mapper/test_bank_party_mapper.py create mode 100644 erpnext/accounts/doctype/bank_transaction/auto_match_party.py diff --git a/erpnext/accounts/doctype/bank_party_mapper/__init__.py b/erpnext/accounts/doctype/bank_party_mapper/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.js b/erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.js new file mode 100644 index 0000000000..d11d8040b2 --- /dev/null +++ b/erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.js @@ -0,0 +1,8 @@ +// Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and contributors +// For license information, please see license.txt + +// frappe.ui.form.on("Bank Party Mapper", { +// refresh(frm) { + +// }, +// }); diff --git a/erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.json b/erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.json new file mode 100644 index 0000000000..4f5f6bfa88 --- /dev/null +++ b/erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.json @@ -0,0 +1,80 @@ +{ + "actions": [], + "creation": "2023-03-31 10:48:20.249481", + "default_view": "List", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "party_type", + "party", + "column_break_wbna", + "bank_party_name_desc", + "bank_party_account_number", + "bank_party_iban" + ], + "fields": [ + { + "fieldname": "bank_party_account_number", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Party Account No. (Bank Statement)" + }, + { + "fieldname": "bank_party_iban", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Party IBAN (Bank Statement)" + }, + { + "fieldname": "column_break_wbna", + "fieldtype": "Column Break" + }, + { + "fieldname": "party_type", + "fieldtype": "Link", + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Party Type", + "options": "DocType" + }, + { + "fieldname": "party", + "fieldtype": "Dynamic Link", + "in_list_view": 1, + "in_standard_filter": 1, + "label": "Party", + "options": "party_type" + }, + { + "fieldname": "bank_party_name_desc", + "fieldtype": "Small Text", + "label": "Party Name/Desc (Bank Statement)" + } + ], + "index_web_pages_for_search": 1, + "links": [], + "modified": "2023-04-03 10:11:31.384383", + "modified_by": "Administrator", + "module": "Accounts", + "name": "Bank Party Mapper", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [], + "track_changes": 1 +} \ No newline at end of file diff --git a/erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.py b/erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.py new file mode 100644 index 0000000000..d3a9a5e586 --- /dev/null +++ b/erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.py @@ -0,0 +1,9 @@ +# Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and contributors +# For license information, please see license.txt + +# import frappe +from frappe.model.document import Document + + +class BankPartyMapper(Document): + pass diff --git a/erpnext/accounts/doctype/bank_party_mapper/test_bank_party_mapper.py b/erpnext/accounts/doctype/bank_party_mapper/test_bank_party_mapper.py new file mode 100644 index 0000000000..c05b23f1a5 --- /dev/null +++ b/erpnext/accounts/doctype/bank_party_mapper/test_bank_party_mapper.py @@ -0,0 +1,9 @@ +# Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and Contributors +# See license.txt + +# import frappe +from frappe.tests.utils import FrappeTestCase + + +class TestBankPartyMapper(FrappeTestCase): + pass diff --git a/erpnext/accounts/doctype/bank_transaction/auto_match_party.py b/erpnext/accounts/doctype/bank_transaction/auto_match_party.py new file mode 100644 index 0000000000..7354fa0928 --- /dev/null +++ b/erpnext/accounts/doctype/bank_transaction/auto_match_party.py @@ -0,0 +1,157 @@ +import frappe +from rapidfuzz import fuzz, process + + +class AutoMatchParty: + def __init__(self, **kwargs) -> None: + self.__dict__.update(kwargs) + + def get(self, key): + return self.__dict__.get(key, None) + + def match(self): + result = AutoMatchbyAccountIBAN( + bank_party_account_number=self.bank_party_account_number, + bank_party_iban=self.bank_party_iban, + deposit=self.deposit, + ).match() + + if not result: + result = AutoMatchbyPartyDescription( + bank_party_name=self.bank_party_name, description=self.description, deposit=self.deposit + ).match() + + return result + + +class AutoMatchbyAccountIBAN: + def __init__(self, **kwargs) -> None: + self.__dict__.update(kwargs) + + def get(self, key): + return self.__dict__.get(key, None) + + def match(self): + if not (self.bank_party_account_number or self.bank_party_iban): + return None + + result = self.match_account_in_bank_party_mapper() + if not result: + result = self.match_account_in_party() + + return result + + def match_account_in_bank_party_mapper(self): + filter_field = ( + "bank_party_account_number" if self.bank_party_account_number else "bank_party_iban" + ) + result = frappe.db.get_value( + "Bank Party Mapper", + filters={filter_field: self.get(filter_field)}, + fieldname=["party_type", "party"], + ) + if result: + party_type, party = result + return (party_type, party, None) + + return result + + def match_account_in_party(self): + # If not check if there is a match in Customer/Supplier/Employee + filter_field = "bank_account_no" if self.bank_party_account_number else "iban" + transaction_field = ( + "bank_party_account_number" if self.bank_party_account_number else "bank_party_iban" + ) + result = None + + parties = ["Supplier", "Employee", "Customer"] # most -> least likely to receive + if self.deposit > 0: + parties = ["Customer", "Supplier", "Employee"] # most -> least likely to pay + + for party in parties: + party_name = frappe.db.get_value( + party, filters={filter_field: self.get(transaction_field)}, fieldname=["name"] + ) + if party_name: + result = (party, party_name, {transaction_field: self.get(transaction_field)}) + break + + return result + + +class AutoMatchbyPartyDescription: + def __init__(self, **kwargs) -> None: + self.__dict__.update(kwargs) + + def get(self, key): + return self.__dict__.get(key, None) + + def match(self): + # Match by Customer, Supplier or Employee Name + # search bank party mapper by party and then description + # fuzzy search by customer/supplier & employee + if not (self.bank_party_name or self.description): + return None + + result = self.match_party_name_desc_in_bank_party_mapper() + + if not result: + result = self.match_party_name_desc_in_party() + + return result + + def match_party_name_desc_in_bank_party_mapper(self): + """Check if match exists for party name or description in Bank Party Mapper""" + result = None + # TODO: or filters + if self.bank_party_name: + result = frappe.db.get_value( + "Bank Party Mapper", + filters={"bank_party_name_desc": self.bank_party_name}, + fieldname=["party_type", "party"], + ) + + if not result and self.description: + result = frappe.db.get_value( + "Bank Party Mapper", + filters={"bank_party_name_desc": self.description}, + fieldname=["party_type", "party"], + ) + + result = result + (None,) if result else result + + return result + + def match_party_name_desc_in_party(self): + """Fuzzy search party name and/or description against parties in the system""" + result = None + + parties = ["Supplier", "Employee", "Customer"] # most-least likely to receive + if frappe.utils.flt(self.deposit) > 0.0: + parties = ["Customer", "Supplier", "Employee"] # most-least likely to pay + + for party in parties: + name_field = party.lower() + "_name" + filters = {"status": "Active"} if party == "Employee" else {"disabled": 0} + + names = frappe.get_all(party, filters=filters, pluck=name_field) + + for field in ["bank_party_name", "description"]: + if not result and self.get(field): + result = self.fuzzy_search_and_return_result(party, names, field) + if result: + break + + return result + + def fuzzy_search_and_return_result(self, party, names, field): + result = process.extractOne(query=self.get(field), choices=names, scorer=fuzz.token_set_ratio) + + if result: + party_name, score, index = result + if score > 75: + return (party, party_name, {"bank_party_name_desc": self.get(field)}) + else: + return None + + return result diff --git a/erpnext/accounts/doctype/bank_transaction/bank_transaction.json b/erpnext/accounts/doctype/bank_transaction/bank_transaction.json index 1543fdb894..4139a9f6d5 100644 --- a/erpnext/accounts/doctype/bank_transaction/bank_transaction.json +++ b/erpnext/accounts/doctype/bank_transaction/bank_transaction.json @@ -36,7 +36,7 @@ "party", "column_break_3czf", "bank_party_name", - "bank_party_no", + "bank_party_account_number", "bank_party_iban" ], "fields": [ @@ -216,20 +216,20 @@ "fieldtype": "Data", "label": "Party Name (Bank Statement)" }, - { - "fieldname": "bank_party_no", - "fieldtype": "Data", - "label": "Party Account No. (Bank Statement)" - }, { "fieldname": "bank_party_iban", "fieldtype": "Data", "label": "Party IBAN (Bank Statement)" + }, + { + "fieldname": "bank_party_account_number", + "fieldtype": "Data", + "label": "Party Account No. (Bank Statement)" } ], "is_submittable": 1, "links": [], - "modified": "2023-03-30 15:30:46.485683", + "modified": "2023-03-31 10:45:30.671309", "modified_by": "Administrator", "module": "Accounts", "name": "Bank Transaction", diff --git a/erpnext/accounts/doctype/bank_transaction/bank_transaction.py b/erpnext/accounts/doctype/bank_transaction/bank_transaction.py index 676c71910b..745450423b 100644 --- a/erpnext/accounts/doctype/bank_transaction/bank_transaction.py +++ b/erpnext/accounts/doctype/bank_transaction/bank_transaction.py @@ -9,11 +9,6 @@ from erpnext.controllers.status_updater import StatusUpdater class BankTransaction(StatusUpdater): # TODO - # On BT save: - # - Match by account no/iban in Customer/Supplier/Employee - # - Match by Party Name - # - If match found, set party type and party name. - # On submit/update after submit # - Create/Update a Bank Party Map record # - User can edit after submit. @@ -22,6 +17,12 @@ class BankTransaction(StatusUpdater): def after_insert(self): self.unallocated_amount = abs(flt(self.withdrawal) - flt(self.deposit)) + def on_update(self): + if self.party_type and self.party: + return + + self.auto_set_party() + def on_submit(self): self.clear_linked_payment_entries() self.set_status() @@ -157,6 +158,30 @@ class BankTransaction(StatusUpdater): payment_entry.payment_document, payment_entry.payment_entry, clearance_date, self ) + def auto_set_party(self): + # TODO: check if enabled + from erpnext.accounts.doctype.bank_transaction.auto_match_party import AutoMatchParty + + result = AutoMatchParty( + bank_party_account_number=self.bank_party_account_number, + bank_party_iban=self.bank_party_iban, + bank_party_name=self.bank_party_name, + description=self.description, + deposit=self.deposit, + ).match() + + if result: + self.party_type, self.party, mapper = result + + if not mapper: + return + + mapper_doc = frappe.get_doc( + {"doctype": "Bank Party Mapper", "party_type": self.party_type, "party": self.party} + ) + mapper_doc.update(mapper) + mapper_doc.insert() + @frappe.whitelist() def get_doctypes_for_bank_reconciliation(): diff --git a/pyproject.toml b/pyproject.toml index 0718e5b4a1..e5bc884729 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ dependencies = [ "pycountry~=20.7.3", "Unidecode~=1.2.0", "barcodenumber~=0.5.0", + "rapidfuzz~=2.15.0", # integration dependencies "gocardless-pro~=1.22.0", From 3a898289b0bcf867263e1b94a90846a1bcaa756a Mon Sep 17 00:00:00 2001 From: marination Date: Mon, 3 Apr 2023 16:11:00 +0530 Subject: [PATCH 03/17] chore: Single query with or filter to search Party Mapper by name/desc --- .../bank_transaction/auto_match_party.py | 36 +++++++++++-------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/erpnext/accounts/doctype/bank_transaction/auto_match_party.py b/erpnext/accounts/doctype/bank_transaction/auto_match_party.py index 7354fa0928..01b674340a 100644 --- a/erpnext/accounts/doctype/bank_transaction/auto_match_party.py +++ b/erpnext/accounts/doctype/bank_transaction/auto_match_party.py @@ -1,4 +1,5 @@ import frappe +from frappe.utils import flt from rapidfuzz import fuzz, process @@ -65,7 +66,7 @@ class AutoMatchbyAccountIBAN: result = None parties = ["Supplier", "Employee", "Customer"] # most -> least likely to receive - if self.deposit > 0: + if flt(self.deposit) > 0: parties = ["Customer", "Supplier", "Employee"] # most -> least likely to pay for party in parties: @@ -103,22 +104,27 @@ class AutoMatchbyPartyDescription: def match_party_name_desc_in_bank_party_mapper(self): """Check if match exists for party name or description in Bank Party Mapper""" result = None - # TODO: or filters + or_filters = [] + if self.bank_party_name: - result = frappe.db.get_value( - "Bank Party Mapper", - filters={"bank_party_name_desc": self.bank_party_name}, - fieldname=["party_type", "party"], - ) + or_filters.append(["bank_party_name_desc", self.bank_party_name]) - if not result and self.description: - result = frappe.db.get_value( - "Bank Party Mapper", - filters={"bank_party_name_desc": self.description}, - fieldname=["party_type", "party"], - ) + if self.description: + or_filters.append(["bank_party_name_desc", self.description]) - result = result + (None,) if result else result + mapper_res = frappe.get_all( + "Bank Party Mapper", + or_filters=or_filters, + fields=["party_type", "party"], + limit_page_length=1, + ) + if mapper_res: + mapper_res = mapper_res[0] + result = ( + mapper_res["party_type"], + mapper_res["party"], + None, + ) return result @@ -127,7 +133,7 @@ class AutoMatchbyPartyDescription: result = None parties = ["Supplier", "Employee", "Customer"] # most-least likely to receive - if frappe.utils.flt(self.deposit) > 0.0: + if flt(self.deposit) > 0.0: parties = ["Customer", "Supplier", "Employee"] # most-least likely to pay for party in parties: From 37c1331aba62ee85a79286d58a71461bb483be54 Mon Sep 17 00:00:00 2001 From: marination Date: Tue, 4 Apr 2023 14:03:35 +0530 Subject: [PATCH 04/17] fix: Don't set description as key in Mapper doc if matched by description - Description is volatile and will keep changing - It will lead to multiple Bank Party Mapper docs for the same party that will never be referenced again - Parts of the descripton keep changing which is why it will never match a mapper record - If matched by desc, dont create mapper record. --- .../doctype/bank_transaction/auto_match_party.py | 12 +++++++++--- .../doctype/bank_transaction/bank_transaction.py | 6 +++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/erpnext/accounts/doctype/bank_transaction/auto_match_party.py b/erpnext/accounts/doctype/bank_transaction/auto_match_party.py index 01b674340a..3fbdc5f36b 100644 --- a/erpnext/accounts/doctype/bank_transaction/auto_match_party.py +++ b/erpnext/accounts/doctype/bank_transaction/auto_match_party.py @@ -107,10 +107,10 @@ class AutoMatchbyPartyDescription: or_filters = [] if self.bank_party_name: - or_filters.append(["bank_party_name_desc", self.bank_party_name]) + or_filters.append({"bank_party_name_desc": self.bank_party_name}) if self.description: - or_filters.append(["bank_party_name_desc", self.description]) + or_filters.append({"bank_party_name_desc": self.description}) mapper_res = frappe.get_all( "Bank Party Mapper", @@ -156,7 +156,13 @@ class AutoMatchbyPartyDescription: if result: party_name, score, index = result if score > 75: - return (party, party_name, {"bank_party_name_desc": self.get(field)}) + # Dont set description as a key in Bank Party Mapper due to its volatility + mapper = {"bank_party_name_desc": self.get(field)} if field == "bank_party_name" else None + return ( + party, + party_name, + mapper, + ) else: return None diff --git a/erpnext/accounts/doctype/bank_transaction/bank_transaction.py b/erpnext/accounts/doctype/bank_transaction/bank_transaction.py index 745450423b..118705d0d3 100644 --- a/erpnext/accounts/doctype/bank_transaction/bank_transaction.py +++ b/erpnext/accounts/doctype/bank_transaction/bank_transaction.py @@ -18,9 +18,6 @@ class BankTransaction(StatusUpdater): self.unallocated_amount = abs(flt(self.withdrawal) - flt(self.deposit)) def on_update(self): - if self.party_type and self.party: - return - self.auto_set_party() def on_submit(self): @@ -162,6 +159,9 @@ class BankTransaction(StatusUpdater): # TODO: check if enabled from erpnext.accounts.doctype.bank_transaction.auto_match_party import AutoMatchParty + if self.party_type and self.party: + return + result = AutoMatchParty( bank_party_account_number=self.bank_party_account_number, bank_party_iban=self.bank_party_iban, From 27ce789023c98f4cd99086db9f5fd8d1f3ee3a30 Mon Sep 17 00:00:00 2001 From: marination Date: Tue, 4 Apr 2023 19:27:01 +0530 Subject: [PATCH 05/17] feat: Manually Update/Correct Party in Bank Transaction - On updating bank trans.n party after submit, the corresponding mapper doc will be updated too - The mapper doc in turn will update all linked bank transactions that do not have this updated value - Added Bank Party Mapper hidden link in Bank Transaction - Rename field in BPM to `Party Name` as it does not hold description data - If a BT matches with a BPM record, link that record in the BT --- .../bank_party_mapper/bank_party_mapper.js | 12 ++++---- .../bank_party_mapper/bank_party_mapper.json | 10 +++---- .../bank_party_mapper/bank_party_mapper.py | 19 ++++++++++-- .../bank_transaction/auto_match_party.py | 30 ++++++++----------- .../bank_transaction/bank_transaction.json | 15 ++++++++-- .../bank_transaction/bank_transaction.py | 19 ++++++++++++ 6 files changed, 73 insertions(+), 32 deletions(-) diff --git a/erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.js b/erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.js index d11d8040b2..b13e46a0e7 100644 --- a/erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.js +++ b/erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.js @@ -1,8 +1,10 @@ // Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and contributors // For license information, please see license.txt -// frappe.ui.form.on("Bank Party Mapper", { -// refresh(frm) { - -// }, -// }); +frappe.ui.form.on("Bank Party Mapper", { + refresh(frm) { + if (!frm.is_new()) { + frm.set_intro(__("Please avoid editing unless you are absolutely certain.")); + } + }, +}); diff --git a/erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.json b/erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.json index 4f5f6bfa88..ddc79f24a0 100644 --- a/erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.json +++ b/erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.json @@ -9,7 +9,7 @@ "party_type", "party", "column_break_wbna", - "bank_party_name_desc", + "bank_party_name", "bank_party_account_number", "bank_party_iban" ], @@ -47,14 +47,14 @@ "options": "party_type" }, { - "fieldname": "bank_party_name_desc", - "fieldtype": "Small Text", - "label": "Party Name/Desc (Bank Statement)" + "fieldname": "bank_party_name", + "fieldtype": "Data", + "label": "Party Name (Bank Statement)" } ], "index_web_pages_for_search": 1, "links": [], - "modified": "2023-04-03 10:11:31.384383", + "modified": "2023-04-04 14:27:23.450456", "modified_by": "Administrator", "module": "Accounts", "name": "Bank Party Mapper", diff --git a/erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.py b/erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.py index d3a9a5e586..02e36b08fe 100644 --- a/erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.py +++ b/erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.py @@ -1,9 +1,24 @@ # Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and contributors # For license information, please see license.txt -# import frappe +import frappe from frappe.model.document import Document class BankPartyMapper(Document): - pass + def on_update(self): + self.update_party_in_linked_transactions() + + def update_party_in_linked_transactions(self): + if self.is_new(): + return + + # Set updated party values in other linked bank transactions + bank_transaction = frappe.qb.DocType("Bank Transaction") + + frappe.qb.update(bank_transaction).set("party_type", self.party_type).set( + "party", self.party + ).where( + (bank_transaction.bank_party_mapper == self.name) + & ((bank_transaction.party_type != self.party_type) | (bank_transaction.party != self.party)) + ).run() diff --git a/erpnext/accounts/doctype/bank_transaction/auto_match_party.py b/erpnext/accounts/doctype/bank_transaction/auto_match_party.py index 3fbdc5f36b..9707f79a2a 100644 --- a/erpnext/accounts/doctype/bank_transaction/auto_match_party.py +++ b/erpnext/accounts/doctype/bank_transaction/auto_match_party.py @@ -49,11 +49,11 @@ class AutoMatchbyAccountIBAN: result = frappe.db.get_value( "Bank Party Mapper", filters={filter_field: self.get(filter_field)}, - fieldname=["party_type", "party"], + fieldname=["party_type", "party", "name"], ) if result: - party_type, party = result - return (party_type, party, None) + party_type, party, mapper_name = result + return (party_type, party, {"mapper_name": mapper_name}) return result @@ -89,33 +89,29 @@ class AutoMatchbyPartyDescription: def match(self): # Match by Customer, Supplier or Employee Name - # search bank party mapper by party and then description + # search bank party mapper by party # fuzzy search by customer/supplier & employee if not (self.bank_party_name or self.description): return None - result = self.match_party_name_desc_in_bank_party_mapper() + result = self.match_party_name_in_bank_party_mapper() if not result: result = self.match_party_name_desc_in_party() return result - def match_party_name_desc_in_bank_party_mapper(self): - """Check if match exists for party name or description in Bank Party Mapper""" + def match_party_name_in_bank_party_mapper(self): + """Check if match exists for party name in Bank Party Mapper""" result = None - or_filters = [] - if self.bank_party_name: - or_filters.append({"bank_party_name_desc": self.bank_party_name}) - - if self.description: - or_filters.append({"bank_party_name_desc": self.description}) + if not self.bank_party_name: + return mapper_res = frappe.get_all( "Bank Party Mapper", - or_filters=or_filters, - fields=["party_type", "party"], + filters={"bank_party_name": self.bank_party_name}, + fields=["party_type", "party", "name"], limit_page_length=1, ) if mapper_res: @@ -123,7 +119,7 @@ class AutoMatchbyPartyDescription: result = ( mapper_res["party_type"], mapper_res["party"], - None, + {"mapper_name": mapper_res["name"]}, ) return result @@ -157,7 +153,7 @@ class AutoMatchbyPartyDescription: party_name, score, index = result if score > 75: # Dont set description as a key in Bank Party Mapper due to its volatility - mapper = {"bank_party_name_desc": self.get(field)} if field == "bank_party_name" else None + mapper = {"bank_party_name": self.get(field)} if field == "bank_party_name" else None return ( party, party_name, diff --git a/erpnext/accounts/doctype/bank_transaction/bank_transaction.json b/erpnext/accounts/doctype/bank_transaction/bank_transaction.json index 4139a9f6d5..d3dc5b5845 100644 --- a/erpnext/accounts/doctype/bank_transaction/bank_transaction.json +++ b/erpnext/accounts/doctype/bank_transaction/bank_transaction.json @@ -37,7 +37,8 @@ "column_break_3czf", "bank_party_name", "bank_party_account_number", - "bank_party_iban" + "bank_party_iban", + "bank_party_mapper" ], "fields": [ { @@ -214,7 +215,7 @@ { "fieldname": "bank_party_name", "fieldtype": "Data", - "label": "Party Name (Bank Statement)" + "label": "Party Name/Account Holder (Bank Statement)" }, { "fieldname": "bank_party_iban", @@ -225,11 +226,19 @@ "fieldname": "bank_party_account_number", "fieldtype": "Data", "label": "Party Account No. (Bank Statement)" + }, + { + "fieldname": "bank_party_mapper", + "fieldtype": "Link", + "hidden": 1, + "label": "Bank Party Mapper", + "options": "Bank Party Mapper", + "read_only": 1 } ], "is_submittable": 1, "links": [], - "modified": "2023-03-31 10:45:30.671309", + "modified": "2023-04-04 15:47:20.620006", "modified_by": "Administrator", "module": "Accounts", "name": "Bank Transaction", diff --git a/erpnext/accounts/doctype/bank_transaction/bank_transaction.py b/erpnext/accounts/doctype/bank_transaction/bank_transaction.py index 118705d0d3..a93882a5b3 100644 --- a/erpnext/accounts/doctype/bank_transaction/bank_transaction.py +++ b/erpnext/accounts/doctype/bank_transaction/bank_transaction.py @@ -35,6 +35,8 @@ class BankTransaction(StatusUpdater): self.update_allocations() self._saving_flag = False + self.set_in_bank_party_mapper() + def on_cancel(self): self.clear_linked_payment_entries(for_cancel=True) self.set_status(update=True) @@ -176,11 +178,28 @@ class BankTransaction(StatusUpdater): if not mapper: return + if mapper.get("mapper_name"): + # Transaction matched with a Bank party Mapper record + self.bank_party_mapper = mapper.get("mapper_name") # Link mapper to Bank Transaction + return + mapper_doc = frappe.get_doc( {"doctype": "Bank Party Mapper", "party_type": self.party_type, "party": self.party} ) mapper_doc.update(mapper) mapper_doc.insert() + self.bank_party_mapper = mapper_doc.name # Link mapper to Bank Transaction + + def set_in_bank_party_mapper(self): + """Set in Bank Party Mapper if Party Type & Party are manually changed after submit.""" + doc_before_update = self.get_doc_before_save() + party_type_changed = self.party_type and (doc_before_update.party_type != self.party_type) + party_changed = self.party and (doc_before_update.party != self.party) + + if (party_type_changed or party_changed) and self.bank_party_mapper: + mapper_doc = frappe.get_doc("Bank Party Mapper", self.bank_party_mapper) + mapper_doc.update({"party_type": self.party_type, "party": self.party}) + mapper_doc.save() @frappe.whitelist() From 33604550ce237c23d2d17345e2c041fc183a9b54 Mon Sep 17 00:00:00 2001 From: marination Date: Tue, 4 Apr 2023 19:42:25 +0530 Subject: [PATCH 06/17] chore: Perform automatch on submit - misc: Clearer naming --- .../doctype/bank_transaction/bank_transaction.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/erpnext/accounts/doctype/bank_transaction/bank_transaction.py b/erpnext/accounts/doctype/bank_transaction/bank_transaction.py index a93882a5b3..6a094915b7 100644 --- a/erpnext/accounts/doctype/bank_transaction/bank_transaction.py +++ b/erpnext/accounts/doctype/bank_transaction/bank_transaction.py @@ -17,13 +17,12 @@ class BankTransaction(StatusUpdater): def after_insert(self): self.unallocated_amount = abs(flt(self.withdrawal) - flt(self.deposit)) - def on_update(self): - self.auto_set_party() - def on_submit(self): self.clear_linked_payment_entries() self.set_status() + self.auto_set_party() + _saving_flag = False # nosemgrep: frappe-semgrep-rules.rules.frappe-modifying-but-not-comitting @@ -35,7 +34,7 @@ class BankTransaction(StatusUpdater): self.update_allocations() self._saving_flag = False - self.set_in_bank_party_mapper() + self.update_automatch_bank_party_mapper() def on_cancel(self): self.clear_linked_payment_entries(for_cancel=True) @@ -190,8 +189,8 @@ class BankTransaction(StatusUpdater): mapper_doc.insert() self.bank_party_mapper = mapper_doc.name # Link mapper to Bank Transaction - def set_in_bank_party_mapper(self): - """Set in Bank Party Mapper if Party Type & Party are manually changed after submit.""" + def update_automatch_bank_party_mapper(self): + """Update Bank Party Mapper if Party Type & Party are manually changed after submit.""" doc_before_update = self.get_doc_before_save() party_type_changed = self.party_type and (doc_before_update.party_type != self.party_type) party_changed = self.party and (doc_before_update.party != self.party) From aea4315435cecd8c531eda5c4bf305b4792ca692 Mon Sep 17 00:00:00 2001 From: marination Date: Tue, 4 Apr 2023 19:56:21 +0530 Subject: [PATCH 07/17] chore: Make auto matching party configurable - Checkbox in Accounts settings "Enable Automatic Party Matching" - Check before invoking automatching methods - misc: Remove TODO comments --- .../accounts_settings/accounts_settings.json | 18 ++++++++++++++++-- .../bank_transaction/bank_transaction.py | 13 ++++--------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/erpnext/accounts/doctype/accounts_settings/accounts_settings.json b/erpnext/accounts/doctype/accounts_settings/accounts_settings.json index c0eed18ad1..259612e0e4 100644 --- a/erpnext/accounts/doctype/accounts_settings/accounts_settings.json +++ b/erpnext/accounts/doctype/accounts_settings/accounts_settings.json @@ -59,7 +59,9 @@ "frozen_accounts_modifier", "report_settings_sb", "tab_break_dpet", - "show_balance_in_coa" + "show_balance_in_coa", + "banking_tab", + "enable_party_matching" ], "fields": [ { @@ -368,6 +370,18 @@ "fieldname": "book_tax_discount_loss", "fieldtype": "Check", "label": "Book Tax Loss on Early Payment Discount" + }, + { + "fieldname": "banking_tab", + "fieldtype": "Tab Break", + "label": "Banking" + }, + { + "default": "0", + "description": "Auto match and set the Party in Bank Transactions", + "fieldname": "enable_party_matching", + "fieldtype": "Check", + "label": "Enable Automatic Party Matching" } ], "icon": "icon-cog", @@ -375,7 +389,7 @@ "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2023-03-28 09:50:20.375233", + "modified": "2023-04-04 16:20:41.330039", "modified_by": "Administrator", "module": "Accounts", "name": "Accounts Settings", diff --git a/erpnext/accounts/doctype/bank_transaction/bank_transaction.py b/erpnext/accounts/doctype/bank_transaction/bank_transaction.py index 6a094915b7..25385a03ca 100644 --- a/erpnext/accounts/doctype/bank_transaction/bank_transaction.py +++ b/erpnext/accounts/doctype/bank_transaction/bank_transaction.py @@ -8,12 +8,6 @@ from erpnext.controllers.status_updater import StatusUpdater class BankTransaction(StatusUpdater): - # TODO - # On submit/update after submit - # - Create/Update a Bank Party Map record - # - User can edit after submit. - # - If changes in party/party name after submit, edit bank party map (which should edit all transactions with same account no/iban/bank party name) - def after_insert(self): self.unallocated_amount = abs(flt(self.withdrawal) - flt(self.deposit)) @@ -21,7 +15,8 @@ class BankTransaction(StatusUpdater): self.clear_linked_payment_entries() self.set_status() - self.auto_set_party() + if frappe.db.get_single_value("Accounts Settings", "enable_party_matching"): + self.auto_set_party() _saving_flag = False @@ -34,7 +29,8 @@ class BankTransaction(StatusUpdater): self.update_allocations() self._saving_flag = False - self.update_automatch_bank_party_mapper() + if frappe.db.get_single_value("Accounts Settings", "enable_party_matching"): + self.update_automatch_bank_party_mapper() def on_cancel(self): self.clear_linked_payment_entries(for_cancel=True) @@ -157,7 +153,6 @@ class BankTransaction(StatusUpdater): ) def auto_set_party(self): - # TODO: check if enabled from erpnext.accounts.doctype.bank_transaction.auto_match_party import AutoMatchParty if self.party_type and self.party: From d7bc1928045d4e89e9acc61b6a71da25537b19a5 Mon Sep 17 00:00:00 2001 From: marination Date: Wed, 5 Apr 2023 15:28:47 +0530 Subject: [PATCH 08/17] fix: Match by both Account No and IBAN & other cleanups - A BT could have both account and iban, and a Supplier could have only IBAN set - In this case, matching by either (only account) gives no match - Match by Account OR IBAN, use `or_filters` - If matched, set both account no. and IBAN in Bank Party Mapper - Explain AutoMatchParty - Add type hints to return values - Use `set_value` to set values in BT after matching since its an after submit event --- .../bank_transaction/auto_match_party.py | 96 +++++++++++++------ .../bank_transaction/bank_transaction.py | 28 +++--- 2 files changed, 82 insertions(+), 42 deletions(-) diff --git a/erpnext/accounts/doctype/bank_transaction/auto_match_party.py b/erpnext/accounts/doctype/bank_transaction/auto_match_party.py index 9707f79a2a..2a2aff1eef 100644 --- a/erpnext/accounts/doctype/bank_transaction/auto_match_party.py +++ b/erpnext/accounts/doctype/bank_transaction/auto_match_party.py @@ -1,16 +1,40 @@ +from typing import Tuple, Union + import frappe from frappe.utils import flt from rapidfuzz import fuzz, process class AutoMatchParty: + """ + Matches by Account/IBAN and then by Party Name/Description sequentially. + Returns when a result is obtained. + + Result (if present) is of the form: (Party Type, Party, Mapper,) + + Mapper(if present) is one of the forms: + 1. {"mapper_name": }: Indicates that an existing Bank Party Mapper matched against + the transaction and the same must be linked in the Bank Transaction. + + 2. {"bank_party_account_number": , "bank_party_iban": } : Indicates that a match was + found in Customer/Supplier/Employee by account details. A Bank Party Mapper is now created + mapping the Party to the Account No./IBAN + + 3. {"bank_party_name": }: Indicates that a match was found in + Customer/Supplier/Employee by party name. A Bank Party Mapper is now created mapping the Party + to the Party Name (Bank Statement). If matched by Description, no mapper is created as + description is not a static key. + + Mapper data is used either to create a new Bank Party Mapper or link an existing mapper to a transaction. + """ + def __init__(self, **kwargs) -> None: self.__dict__.update(kwargs) def get(self, key): return self.__dict__.get(key, None) - def match(self): + def match(self) -> Union[Tuple, None]: result = AutoMatchbyAccountIBAN( bank_party_account_number=self.bank_party_account_number, bank_party_iban=self.bank_party_iban, @@ -42,27 +66,30 @@ class AutoMatchbyAccountIBAN: return result - def match_account_in_bank_party_mapper(self): - filter_field = ( - "bank_party_account_number" if self.bank_party_account_number else "bank_party_iban" - ) - result = frappe.db.get_value( + def match_account_in_bank_party_mapper(self) -> Union[Tuple, None]: + """Check for a IBAN/Account No. match in Bank Party Mapper""" + result = None + or_filters = {} + if self.bank_party_account_number: + or_filters["bank_party_account_number"] = self.bank_party_account_number + + if self.bank_party_iban: + or_filters["bank_party_iban"] = self.bank_party_iban + + mapper = frappe.db.get_all( "Bank Party Mapper", - filters={filter_field: self.get(filter_field)}, - fieldname=["party_type", "party", "name"], + or_filters=or_filters, + fields=["party_type", "party", "name"], + limit_page_length=1, ) - if result: - party_type, party, mapper_name = result - return (party_type, party, {"mapper_name": mapper_name}) + if mapper: + data = mapper[0] + return (data["party_type"], data["party"], {"mapper_name": data["name"]}) return result - def match_account_in_party(self): - # If not check if there is a match in Customer/Supplier/Employee - filter_field = "bank_account_no" if self.bank_party_account_number else "iban" - transaction_field = ( - "bank_party_account_number" if self.bank_party_account_number else "bank_party_iban" - ) + def match_account_in_party(self) -> Union[Tuple, None]: + """Check if there is a IBAN/Account No. match in Customer/Supplier/Employee""" result = None parties = ["Supplier", "Employee", "Customer"] # most -> least likely to receive @@ -70,11 +97,26 @@ class AutoMatchbyAccountIBAN: parties = ["Customer", "Supplier", "Employee"] # most -> least likely to pay for party in parties: - party_name = frappe.db.get_value( - party, filters={filter_field: self.get(transaction_field)}, fieldname=["name"] + or_filters = {} + if self.bank_party_account_number: + acc_no_field = "bank_ac_no" if party == "Employee" else "bank_account_no" + or_filters[acc_no_field] = self.bank_party_account_number + + if self.bank_party_iban: + or_filters["iban"] = self.bank_party_iban + + party_result = frappe.db.get_all( + party, or_filters=or_filters, pluck="name", limit_page_length=1 ) - if party_name: - result = (party, party_name, {transaction_field: self.get(transaction_field)}) + if party_result: + result = ( + party, + party_result[0], + { + "bank_party_account_number": self.get("bank_party_account_number"), + "bank_party_iban": self.get("bank_party_iban"), + }, + ) break return result @@ -87,7 +129,7 @@ class AutoMatchbyPartyDescription: def get(self, key): return self.__dict__.get(key, None) - def match(self): + def match(self) -> Union[Tuple, None]: # Match by Customer, Supplier or Employee Name # search bank party mapper by party # fuzzy search by customer/supplier & employee @@ -101,10 +143,9 @@ class AutoMatchbyPartyDescription: return result - def match_party_name_in_bank_party_mapper(self): + def match_party_name_in_bank_party_mapper(self) -> Union[Tuple, None]: """Check if match exists for party name in Bank Party Mapper""" result = None - if not self.bank_party_name: return @@ -116,7 +157,7 @@ class AutoMatchbyPartyDescription: ) if mapper_res: mapper_res = mapper_res[0] - result = ( + return ( mapper_res["party_type"], mapper_res["party"], {"mapper_name": mapper_res["name"]}, @@ -124,10 +165,9 @@ class AutoMatchbyPartyDescription: return result - def match_party_name_desc_in_party(self): + def match_party_name_desc_in_party(self) -> Union[Tuple, None]: """Fuzzy search party name and/or description against parties in the system""" result = None - parties = ["Supplier", "Employee", "Customer"] # most-least likely to receive if flt(self.deposit) > 0.0: parties = ["Customer", "Supplier", "Employee"] # most-least likely to pay @@ -146,7 +186,7 @@ class AutoMatchbyPartyDescription: return result - def fuzzy_search_and_return_result(self, party, names, field): + def fuzzy_search_and_return_result(self, party, names, field) -> Union[Tuple, None]: result = process.extractOne(query=self.get(field), choices=names, scorer=fuzz.token_set_ratio) if result: diff --git a/erpnext/accounts/doctype/bank_transaction/bank_transaction.py b/erpnext/accounts/doctype/bank_transaction/bank_transaction.py index 25385a03ca..cf3a6e648a 100644 --- a/erpnext/accounts/doctype/bank_transaction/bank_transaction.py +++ b/erpnext/accounts/doctype/bank_transaction/bank_transaction.py @@ -167,22 +167,22 @@ class BankTransaction(StatusUpdater): ).match() if result: - self.party_type, self.party, mapper = result + party_type, party, mapper = result + to_update = {"party_type": party_type, "party": party} - if not mapper: - return + if mapper and mapper.get("mapper_name"): + # Transaction matched with an existing Bank party Mapper record + to_update["bank_party_mapper"] = mapper.get("mapper_name") + elif mapper: + # Make new Mapper record to remember match + mapper_doc = frappe.get_doc( + {"doctype": "Bank Party Mapper", "party_type": party_type, "party": party} + ) + mapper_doc.update(mapper) + mapper_doc.insert() + to_update["bank_party_mapper"] = mapper_doc.name - if mapper.get("mapper_name"): - # Transaction matched with a Bank party Mapper record - self.bank_party_mapper = mapper.get("mapper_name") # Link mapper to Bank Transaction - return - - mapper_doc = frappe.get_doc( - {"doctype": "Bank Party Mapper", "party_type": self.party_type, "party": self.party} - ) - mapper_doc.update(mapper) - mapper_doc.insert() - self.bank_party_mapper = mapper_doc.name # Link mapper to Bank Transaction + frappe.db.set_value("Bank Transaction", self.name, field=to_update) def update_automatch_bank_party_mapper(self): """Update Bank Party Mapper if Party Type & Party are manually changed after submit.""" From 7ed8f59dc89f02562a56d1281219506214a440b3 Mon Sep 17 00:00:00 2001 From: marination Date: Mon, 10 Apr 2023 22:11:00 +0530 Subject: [PATCH 09/17] test: Match by Account No, IBAN, Party Name, Desc and match correction --- .../bank_transaction/test_auto_match_party.py | 193 ++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 erpnext/accounts/doctype/bank_transaction/test_auto_match_party.py diff --git a/erpnext/accounts/doctype/bank_transaction/test_auto_match_party.py b/erpnext/accounts/doctype/bank_transaction/test_auto_match_party.py new file mode 100644 index 0000000000..86181c865d --- /dev/null +++ b/erpnext/accounts/doctype/bank_transaction/test_auto_match_party.py @@ -0,0 +1,193 @@ +# Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and Contributors +# License: GNU General Public License v3. See license.txt + +import frappe +from frappe.tests.utils import FrappeTestCase +from frappe.utils import nowdate + +from erpnext.accounts.doctype.bank_transaction.test_bank_transaction import create_bank_account + + +class TestAutoMatchParty(FrappeTestCase): + @classmethod + def setUpClass(cls): + create_bank_account() + frappe.db.set_single_value("Accounts Settings", "enable_party_matching", 1) + return super().setUpClass() + + @classmethod + def tearDownClass(cls): + frappe.db.set_single_value("Accounts Settings", "enable_party_matching", 0) + + def test_match_by_account_number(self): + """Test if transaction matches with existing (Bank Party Mapper) or new match.""" + create_supplier_for_match(account_no="000000003716541159") + doc = create_bank_transaction( + withdrawal=1200, + transaction_id="562213b0ca1bf838dab8f2c6a39bbc3b", + account_no="000000003716541159", + iban="DE02000000003716541159", + ) + + self.assertEqual(doc.party_type, "Supplier") + self.assertEqual(doc.party, "John Doe & Co.") + self.assertTrue(doc.bank_party_mapper) + + # Check if Bank Party Mapper is created to remember mapping + bank_party_mapper = frappe.get_doc("Bank Party Mapper", doc.bank_party_mapper) + self.assertEqual(bank_party_mapper.party, "John Doe & Co.") + self.assertEqual(bank_party_mapper.bank_party_account_number, "000000003716541159") + self.assertEqual(bank_party_mapper.bank_party_iban, "DE02000000003716541159") + + # Check if created mapping is used for quick match + doc_2 = create_bank_transaction( + withdrawal=500, + transaction_id="602413b8ji8bf838fub8f2c6a39bah7y", + account_no="000000003716541159", + ) + self.assertEqual(doc_2.party, "John Doe & Co.") + self.assertEqual(doc_2.bank_party_mapper, bank_party_mapper.name) + + def test_match_by_iban(self): + create_supplier_for_match(iban="DE02000000003716541159") + doc = create_bank_transaction( + withdrawal=1200, + transaction_id="c5455a224602afaa51592a9d9250600d", + account_no="000000003716541159", + iban="DE02000000003716541159", + ) + + self.assertEqual(doc.party_type, "Supplier") + self.assertEqual(doc.party, "John Doe & Co.") + self.assertTrue(doc.bank_party_mapper) + + bank_party_mapper = frappe.get_doc("Bank Party Mapper", doc.bank_party_mapper) + self.assertEqual(bank_party_mapper.party, "John Doe & Co.") + self.assertEqual(bank_party_mapper.bank_party_account_number, "000000003716541159") + self.assertEqual(bank_party_mapper.bank_party_iban, "DE02000000003716541159") + + def test_match_by_party_name(self): + create_supplier_for_match(supplier_name="Jackson Ella W.") + doc = create_bank_transaction( + withdrawal=1200, + transaction_id="1f6f661f347ff7b1ea588665f473adb1", + party_name="Ella Jackson", + iban="DE04000000003716545346", + ) + self.assertEqual(doc.party_type, "Supplier") + self.assertEqual(doc.party, "Jackson Ella W.") + self.assertTrue(doc.bank_party_mapper) + + bank_party_mapper = frappe.get_doc("Bank Party Mapper", doc.bank_party_mapper) + self.assertEqual(bank_party_mapper.party, "Jackson Ella W.") + self.assertEqual(bank_party_mapper.bank_party_name, "Ella Jackson") + self.assertEqual(bank_party_mapper.bank_party_iban, None) + + # Check if created mapping is used for quick match + doc_2 = create_bank_transaction( + withdrawal=500, + transaction_id="578313b8ji8bf838fub8f2c6a39bah7y", + party_name="Ella Jackson", + account_no="000000004316531152", + ) + self.assertEqual(doc_2.party, "Jackson Ella W.") + self.assertEqual(doc_2.bank_party_mapper, bank_party_mapper.name) + + def test_match_by_description(self): + create_supplier_for_match(supplier_name="Microsoft") + doc = create_bank_transaction( + description="Auftraggeber: microsoft payments Buchungstext: msft ..e3006b5hdy. ref. j375979555927627/5536", + withdrawal=1200, + transaction_id="8df880a2d09c3bed3fea358ca5168c5a", + party_name="", + ) + self.assertEqual(doc.party_type, "Supplier") + self.assertEqual(doc.party, "Microsoft") + self.assertFalse(doc.bank_party_mapper) + + def test_correct_match_after_submit(self): + """Correct wrong mapping after submit. Test impact.""" + # Similar named suppliers + create_supplier_for_match(supplier_name="Amazon") + create_supplier_for_match(supplier_name="Amazing Co.") + + # Bank Transactions actually from "Amazon" that match better with "Amazing Co." + doc = create_bank_transaction( + description="visa06323202 amzn.com/bill 7,88eur1,5324711959 90.22. 1,62 87861003", + withdrawal=24.85, + transaction_id="3a1da4ee2dc5a980138d36ef5297cbd9", + party_name="Amazn Co.", + ) + doc_2 = create_bank_transaction( + description="visa61268005 amzn.com/bill 22,345eur1,7654711959 89.23. 1,64 61268005", + withdrawal=80, + transaction_id="584314e459b00f792bfd569267efba6e", + party_name="Amazn Co.", + ) + + self.assertEqual(doc.party_type, "Supplier") + self.assertEqual(doc.party, "Amazing Co.") + self.assertTrue(doc.bank_party_mapper) + self.assertTrue(doc_2.bank_party_mapper, doc.bank_party_mapper) + + bank_party_mapper = frappe.get_doc("Bank Party Mapper", doc.bank_party_mapper) + self.assertEqual(bank_party_mapper.party, "Amazing Co.") + self.assertEqual(bank_party_mapper.bank_party_name, "Amazn Co.") + + # User corrects the value after submit to "Amazon" + doc.party = "Amazon" + doc.save() + bank_party_mapper.reload() + doc_2.reload() + + # Mapping is edited and all transactions with this mapping are updated + self.assertEqual(bank_party_mapper.party, "Amazon") + self.assertEqual(bank_party_mapper.bank_party_name, "Amazn Co.") + self.assertEqual(doc_2.party, "Amazon") + + +def create_supplier_for_match(supplier_name="John Doe & Co.", account_no=None, iban=None): + if frappe.db.exists("Supplier", supplier_name): + frappe.db.set_value("Supplier", supplier_name, {"bank_account_no": account_no, "iban": iban}) + return + + frappe.get_doc( + { + "doctype": "Supplier", + "supplier_name": supplier_name, + "supplier_group": "Services", + "supplier_type": "Company", + "bank_account_no": account_no, + "iban": iban, + } + ).insert() + + +def create_bank_transaction( + description=None, + withdrawal=0, + deposit=0, + transaction_id=None, + party_name=None, + account_no=None, + iban=None, +): + doc = frappe.get_doc( + { + "doctype": "Bank Transaction", + "description": description or "1512567 BG/000002918 OPSKATTUZWXXX AT776000000098709837 Herr G", + "date": nowdate(), + "withdrawal": withdrawal, + "deposit": deposit, + "currency": "INR", + "bank_account": "Checking Account - Citi Bank", + "transaction_id": transaction_id, + "bank_party_name": party_name, + "bank_party_account_number": account_no, + "bank_party_iban": iban, + } + ).insert() + doc.submit() + doc.reload() + + return doc From 430b247dfc0e4f681fe47d540fd24cd0b40f76d1 Mon Sep 17 00:00:00 2001 From: marination Date: Tue, 11 Apr 2023 01:33:08 +0530 Subject: [PATCH 10/17] fix: Remove bank details fields from Shareholder --- .../doctype/shareholder/shareholder.json | 27 ++----------------- 1 file changed, 2 insertions(+), 25 deletions(-) diff --git a/erpnext/accounts/doctype/shareholder/shareholder.json b/erpnext/accounts/doctype/shareholder/shareholder.json index 2be2a2fc42..e80b05720e 100644 --- a/erpnext/accounts/doctype/shareholder/shareholder.json +++ b/erpnext/accounts/doctype/shareholder/shareholder.json @@ -20,11 +20,7 @@ "contact_html", "section_break_3", "share_balance", - "contact_list", - "bank_details_section", - "bank_account_no", - "column_break_tyo0", - "iban" + "contact_list" ], "fields": [ { @@ -114,29 +110,10 @@ "hidden": 1, "label": "Contact List", "read_only": 1 - }, - { - "fieldname": "bank_details_section", - "fieldtype": "Section Break", - "label": "Bank Details" - }, - { - "fieldname": "bank_account_no", - "fieldtype": "Data", - "label": "Bank Account No" - }, - { - "fieldname": "column_break_tyo0", - "fieldtype": "Column Break" - }, - { - "fieldname": "iban", - "fieldtype": "Data", - "label": "IBAN" } ], "links": [], - "modified": "2023-03-30 16:00:55.087823", + "modified": "2023-04-10 22:02:20.406087", "modified_by": "Administrator", "module": "Accounts", "name": "Shareholder", From dbf7a479b66bc31890b4a957564c7dfdfad49517 Mon Sep 17 00:00:00 2001 From: marination Date: Tue, 9 May 2023 20:36:07 +0530 Subject: [PATCH 11/17] fix: Use existing bank fields to match by bank account no/IBAN - Remove newly added fields in Party doctypes to store bank details - Use Bank Account's fields to match against account no/iban - For employee, if Bank Account does not exist, find in Employee doctype against account no/iban --- .../bank_transaction/auto_match_party.py | 15 ++++++++-- erpnext/buying/doctype/supplier/supplier.json | 30 +------------------ .../selling/doctype/customer/customer.json | 25 +--------------- 3 files changed, 14 insertions(+), 56 deletions(-) diff --git a/erpnext/accounts/doctype/bank_transaction/auto_match_party.py b/erpnext/accounts/doctype/bank_transaction/auto_match_party.py index 2a2aff1eef..e43d9beb05 100644 --- a/erpnext/accounts/doctype/bank_transaction/auto_match_party.py +++ b/erpnext/accounts/doctype/bank_transaction/auto_match_party.py @@ -99,15 +99,24 @@ class AutoMatchbyAccountIBAN: for party in parties: or_filters = {} if self.bank_party_account_number: - acc_no_field = "bank_ac_no" if party == "Employee" else "bank_account_no" - or_filters[acc_no_field] = self.bank_party_account_number + or_filters["bank_account_no"] = self.bank_party_account_number if self.bank_party_iban: or_filters["iban"] = self.bank_party_iban party_result = frappe.db.get_all( - party, or_filters=or_filters, pluck="name", limit_page_length=1 + "Bank Account", or_filters=or_filters, pluck="name", limit_page_length=1 ) + + if party == "Employee" and not party_result: + # Search in Bank Accounts first for Employee, and then Employee record + if "bank_account_no" in or_filters: + or_filters["bank_ac_no"] = or_filters.pop("bank_account_no") + + party_result = frappe.db.get_all( + party, or_filters=or_filters, pluck="name", limit_page_length=1 + ) + if party_result: result = ( party, diff --git a/erpnext/buying/doctype/supplier/supplier.json b/erpnext/buying/doctype/supplier/supplier.json index a80dcfe538..f0097898f6 100644 --- a/erpnext/buying/doctype/supplier/supplier.json +++ b/erpnext/buying/doctype/supplier/supplier.json @@ -52,11 +52,6 @@ "supplier_primary_address", "primary_address", "accounting_tab", - "bank_details_section", - "bank_account_no", - "column_break_n8mz", - "iban", - "section_break_ow3k", "payment_terms", "accounts", "settings_tab", @@ -450,29 +445,6 @@ { "fieldname": "column_break_59", "fieldtype": "Column Break" - }, - { - "fieldname": "bank_details_section", - "fieldtype": "Section Break", - "label": "Bank Details" - }, - { - "fieldname": "bank_account_no", - "fieldtype": "Data", - "label": "Bank Account No" - }, - { - "fieldname": "column_break_n8mz", - "fieldtype": "Column Break" - }, - { - "fieldname": "iban", - "fieldtype": "Data", - "label": "IBAN" - }, - { - "fieldname": "section_break_ow3k", - "fieldtype": "Section Break" } ], "icon": "fa fa-user", @@ -485,7 +457,7 @@ "link_fieldname": "party" } ], - "modified": "2023-03-30 15:50:40.241257", + "modified": "2023-05-09 15:34:13.408932", "modified_by": "Administrator", "module": "Buying", "name": "Supplier", diff --git a/erpnext/selling/doctype/customer/customer.json b/erpnext/selling/doctype/customer/customer.json index 5dc6a72da8..72a1594b2d 100644 --- a/erpnext/selling/doctype/customer/customer.json +++ b/erpnext/selling/doctype/customer/customer.json @@ -61,10 +61,6 @@ "tax_category", "tax_withholding_category", "accounting_tab", - "bank_details_section", - "bank_account_no", - "column_break_xtwg", - "iban", "credit_limit_section", "payment_terms", "credit_limits", @@ -559,25 +555,6 @@ { "fieldname": "column_break_54", "fieldtype": "Column Break" - }, - { - "fieldname": "bank_details_section", - "fieldtype": "Section Break", - "label": "Bank Details" - }, - { - "fieldname": "bank_account_no", - "fieldtype": "Data", - "label": "Bank Account No" - }, - { - "fieldname": "column_break_xtwg", - "fieldtype": "Column Break" - }, - { - "fieldname": "iban", - "fieldtype": "Data", - "label": "IBAN" } ], "icon": "fa fa-user", @@ -591,7 +568,7 @@ "link_fieldname": "party" } ], - "modified": "2023-03-30 15:45:44.387975", + "modified": "2023-05-09 15:38:40.255193", "modified_by": "Administrator", "module": "Selling", "name": "Customer", From 4a14e9ea4e8f131913f7a0ede725db9a7603e85a Mon Sep 17 00:00:00 2001 From: marination Date: Wed, 17 May 2023 14:23:44 +0530 Subject: [PATCH 12/17] fix: Tests --- .../bank_transaction/auto_match_party.py | 2 +- .../bank_transaction/test_auto_match_party.py | 40 ++++++++++++++++--- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/erpnext/accounts/doctype/bank_transaction/auto_match_party.py b/erpnext/accounts/doctype/bank_transaction/auto_match_party.py index e43d9beb05..79d52c6565 100644 --- a/erpnext/accounts/doctype/bank_transaction/auto_match_party.py +++ b/erpnext/accounts/doctype/bank_transaction/auto_match_party.py @@ -105,7 +105,7 @@ class AutoMatchbyAccountIBAN: or_filters["iban"] = self.bank_party_iban party_result = frappe.db.get_all( - "Bank Account", or_filters=or_filters, pluck="name", limit_page_length=1 + "Bank Account", or_filters=or_filters, pluck="party", limit_page_length=1 ) if party == "Employee" and not party_result: diff --git a/erpnext/accounts/doctype/bank_transaction/test_auto_match_party.py b/erpnext/accounts/doctype/bank_transaction/test_auto_match_party.py index 86181c865d..2f94516ac2 100644 --- a/erpnext/accounts/doctype/bank_transaction/test_auto_match_party.py +++ b/erpnext/accounts/doctype/bank_transaction/test_auto_match_party.py @@ -146,22 +146,50 @@ class TestAutoMatchParty(FrappeTestCase): self.assertEqual(doc_2.party, "Amazon") -def create_supplier_for_match(supplier_name="John Doe & Co.", account_no=None, iban=None): - if frappe.db.exists("Supplier", supplier_name): - frappe.db.set_value("Supplier", supplier_name, {"bank_account_no": account_no, "iban": iban}) +def create_supplier_for_match(supplier_name="John Doe & Co.", iban=None, account_no=None): + if frappe.db.exists("Supplier", {"supplier_name": supplier_name}): + # Update related Bank Account details + if not (iban or account_no): + return + + frappe.db.set_value( + dt="Bank Account", + dn={"party": supplier_name}, + field={"iban": iban, "bank_account_no": account_no}, + ) return - frappe.get_doc( + # Create Supplier and Bank Account for the same + supplier = frappe.get_doc( { "doctype": "Supplier", "supplier_name": supplier_name, "supplier_group": "Services", "supplier_type": "Company", - "bank_account_no": account_no, - "iban": iban, } ).insert() + if not frappe.db.exists("Bank", "TestBank"): + frappe.get_doc( + { + "doctype": "Bank", + "bank_name": "TestBank", + } + ).insert(ignore_if_duplicate=True) + + if not frappe.db.exists("Bank Account", supplier.name + " - " + "TestBank"): + frappe.get_doc( + { + "doctype": "Bank Account", + "account_name": supplier.name, + "bank": "TestBank", + "iban": iban, + "bank_account_no": account_no, + "party_type": "Supplier", + "party": supplier.name, + } + ).insert() + def create_bank_transaction( description=None, From 4364fb9628854a27942304e55c26f62d916cd46c Mon Sep 17 00:00:00 2001 From: marination Date: Wed, 17 May 2023 19:45:03 +0530 Subject: [PATCH 13/17] feat: Optional Fuzzy Matching & Skip Matches for multiple similar matches - Fuzzy matching can be enabled optionally in the settings - If a query gets multiple matches with the same score, do not set a party as it is an extremely close call - misc: Add 'cancelled' status to Bank transaction - Test for skipping matching with extremely close matches --- .../accounts_settings/accounts_settings.json | 13 +++- .../bank_transaction/auto_match_party.py | 77 ++++++++++++++----- .../bank_transaction/bank_transaction.json | 4 +- .../bank_transaction/test_auto_match_party.py | 18 +++++ 4 files changed, 88 insertions(+), 24 deletions(-) diff --git a/erpnext/accounts/doctype/accounts_settings/accounts_settings.json b/erpnext/accounts/doctype/accounts_settings/accounts_settings.json index ff07de3467..05f1169d96 100644 --- a/erpnext/accounts/doctype/accounts_settings/accounts_settings.json +++ b/erpnext/accounts/doctype/accounts_settings/accounts_settings.json @@ -64,7 +64,8 @@ "tab_break_dpet", "show_balance_in_coa", "banking_tab", - "enable_party_matching" + "enable_party_matching", + "enable_fuzzy_matching" ], "fields": [ { @@ -404,6 +405,14 @@ "fieldname": "enable_party_matching", "fieldtype": "Check", "label": "Enable Automatic Party Matching" + }, + { + "default": "0", + "depends_on": "enable_party_matching", + "description": "Approximately match the description/party name against parties", + "fieldname": "enable_fuzzy_matching", + "fieldtype": "Check", + "label": "Enable Fuzzy Matching" } ], "icon": "icon-cog", @@ -411,7 +420,7 @@ "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2023-04-21 13:11:37.130743", + "modified": "2023-05-17 12:20:04.107641", "modified_by": "Administrator", "module": "Accounts", "name": "Accounts Settings", diff --git a/erpnext/accounts/doctype/bank_transaction/auto_match_party.py b/erpnext/accounts/doctype/bank_transaction/auto_match_party.py index 79d52c6565..753f0c1171 100644 --- a/erpnext/accounts/doctype/bank_transaction/auto_match_party.py +++ b/erpnext/accounts/doctype/bank_transaction/auto_match_party.py @@ -35,13 +35,15 @@ class AutoMatchParty: return self.__dict__.get(key, None) def match(self) -> Union[Tuple, None]: + result = None result = AutoMatchbyAccountIBAN( bank_party_account_number=self.bank_party_account_number, bank_party_iban=self.bank_party_iban, deposit=self.deposit, ).match() - if not result: + fuzzy_matching_enabled = frappe.db.get_single_value("Accounts Settings", "enable_fuzzy_matching") + if not result and fuzzy_matching_enabled: result = AutoMatchbyPartyDescription( bank_party_name=self.bank_party_name, description=self.description, deposit=self.deposit ).match() @@ -184,31 +186,66 @@ class AutoMatchbyPartyDescription: for party in parties: name_field = party.lower() + "_name" filters = {"status": "Active"} if party == "Employee" else {"disabled": 0} - names = frappe.get_all(party, filters=filters, pluck=name_field) for field in ["bank_party_name", "description"]: - if not result and self.get(field): - result = self.fuzzy_search_and_return_result(party, names, field) - if result: - break + if not self.get(field): + continue + + result, skip = self.fuzzy_search_and_return_result(party, names, field) + if result or skip: + break + + if result or skip: + # We skip if: + # If it was hard to distinguish between close matches and so match is None + # OR if the right match was found + break return result def fuzzy_search_and_return_result(self, party, names, field) -> Union[Tuple, None]: - result = process.extractOne(query=self.get(field), choices=names, scorer=fuzz.token_set_ratio) + skip = False - if result: - party_name, score, index = result - if score > 75: - # Dont set description as a key in Bank Party Mapper due to its volatility - mapper = {"bank_party_name": self.get(field)} if field == "bank_party_name" else None - return ( - party, - party_name, - mapper, - ) - else: - return None + result = process.extract(query=self.get(field), choices=names, scorer=fuzz.token_set_ratio) + party_name, skip = self.process_fuzzy_result(result) - return result + if not party_name: + return None, skip + + # Dont set description as a key in Bank Party Mapper due to its volatility + mapper = {"bank_party_name": self.get(field)} if field == "bank_party_name" else None + return ( + party, + party_name, + mapper, + ), skip + + def process_fuzzy_result(self, result: Union[list, None]): + """ + If there are multiple valid close matches return None as result may be faulty. + Return the result only if one accurate match stands out. + + Returns: Result, Skip (whether or not to continue matching) + """ + PARTY, SCORE, CUTOFF = 0, 1, 80 + + if not result or not len(result): + return None, False + + first_result = result[0] + + if len(result) == 1: + return (result[0][PARTY] if first_result[SCORE] > CUTOFF else None), True + + second_result = result[1] + + if first_result[SCORE] > CUTOFF: + # If multiple matches with the same score, return None but discontinue matching + # Matches were found but were too closes to distinguish between + if first_result[SCORE] == second_result[SCORE]: + return None, True + + return first_result[PARTY], True + else: + return None, False diff --git a/erpnext/accounts/doctype/bank_transaction/bank_transaction.json b/erpnext/accounts/doctype/bank_transaction/bank_transaction.json index d3dc5b5845..e7de71aef4 100644 --- a/erpnext/accounts/doctype/bank_transaction/bank_transaction.json +++ b/erpnext/accounts/doctype/bank_transaction/bank_transaction.json @@ -68,7 +68,7 @@ "fieldtype": "Select", "in_standard_filter": 1, "label": "Status", - "options": "\nPending\nSettled\nUnreconciled\nReconciled" + "options": "\nPending\nSettled\nUnreconciled\nReconciled\nCancelled" }, { "fieldname": "bank_account", @@ -238,7 +238,7 @@ ], "is_submittable": 1, "links": [], - "modified": "2023-04-04 15:47:20.620006", + "modified": "2023-05-17 14:56:10.547480", "modified_by": "Administrator", "module": "Accounts", "name": "Bank Transaction", diff --git a/erpnext/accounts/doctype/bank_transaction/test_auto_match_party.py b/erpnext/accounts/doctype/bank_transaction/test_auto_match_party.py index 2f94516ac2..8c6dc9d766 100644 --- a/erpnext/accounts/doctype/bank_transaction/test_auto_match_party.py +++ b/erpnext/accounts/doctype/bank_transaction/test_auto_match_party.py @@ -13,11 +13,13 @@ class TestAutoMatchParty(FrappeTestCase): def setUpClass(cls): create_bank_account() frappe.db.set_single_value("Accounts Settings", "enable_party_matching", 1) + frappe.db.set_single_value("Accounts Settings", "enable_fuzzy_matching", 1) return super().setUpClass() @classmethod def tearDownClass(cls): frappe.db.set_single_value("Accounts Settings", "enable_party_matching", 0) + frappe.db.set_single_value("Accounts Settings", "enable_fuzzy_matching", 0) def test_match_by_account_number(self): """Test if transaction matches with existing (Bank Party Mapper) or new match.""" @@ -145,6 +147,22 @@ class TestAutoMatchParty(FrappeTestCase): self.assertEqual(bank_party_mapper.bank_party_name, "Amazn Co.") self.assertEqual(doc_2.party, "Amazon") + def test_skip_match_if_multiple_close_results(self): + create_supplier_for_match(supplier_name="Adithya Medical & General Stores") + create_supplier_for_match(supplier_name="Adithya Medical And General Stores") + + doc = create_bank_transaction( + description="Paracetamol Consignment, SINV-0009", + withdrawal=24.85, + transaction_id="3a1da4ee2dc5a980138d56ef3460cbd9", + party_name="Adithya Medical & General", + ) + + # Mapping is skipped as both Supplier names have the same match score + self.assertEqual(doc.party_type, None) + self.assertEqual(doc.party, None) + self.assertFalse(doc.bank_party_mapper) + def create_supplier_for_match(supplier_name="John Doe & Co.", iban=None, account_no=None): if frappe.db.exists("Supplier", {"supplier_name": supplier_name}): From 752a92bd8be6d25325b01e4780f1b37b2195da87 Mon Sep 17 00:00:00 2001 From: marination Date: Tue, 6 Jun 2023 18:59:07 +0530 Subject: [PATCH 14/17] chore: Remove Bank Party Mapper implementation - Matching by Acc No/IBAN can easily happen with Bank Accounts. It's not a tedious query - Historical lookups for Party Name/Desc match are very tricky. The user could have manually set a match and we would not know. Also this leaves the Bank Party Mapper only useful for Party Name/Desc lookups, which feels excessive. - We want to reduce the number of places the same data is stored and reduce confusion - The Party Name/Desc will optionally happen fuzzily, or not at all - There will be no Mapper lookups --- .../doctype/bank_party_mapper/__init__.py | 0 .../bank_party_mapper/bank_party_mapper.js | 10 -- .../bank_party_mapper/bank_party_mapper.json | 80 ----------- .../bank_party_mapper/bank_party_mapper.py | 24 ---- .../test_bank_party_mapper.py | 9 -- .../bank_transaction/auto_match_party.py | 135 ++++-------------- .../bank_transaction/bank_transaction.json | 13 +- .../bank_transaction/bank_transaction.py | 34 +---- .../bank_transaction/test_auto_match_party.py | 79 ---------- 9 files changed, 37 insertions(+), 347 deletions(-) delete mode 100644 erpnext/accounts/doctype/bank_party_mapper/__init__.py delete mode 100644 erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.js delete mode 100644 erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.json delete mode 100644 erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.py delete mode 100644 erpnext/accounts/doctype/bank_party_mapper/test_bank_party_mapper.py diff --git a/erpnext/accounts/doctype/bank_party_mapper/__init__.py b/erpnext/accounts/doctype/bank_party_mapper/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.js b/erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.js deleted file mode 100644 index b13e46a0e7..0000000000 --- a/erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.js +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and contributors -// For license information, please see license.txt - -frappe.ui.form.on("Bank Party Mapper", { - refresh(frm) { - if (!frm.is_new()) { - frm.set_intro(__("Please avoid editing unless you are absolutely certain.")); - } - }, -}); diff --git a/erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.json b/erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.json deleted file mode 100644 index ddc79f24a0..0000000000 --- a/erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.json +++ /dev/null @@ -1,80 +0,0 @@ -{ - "actions": [], - "creation": "2023-03-31 10:48:20.249481", - "default_view": "List", - "doctype": "DocType", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "party_type", - "party", - "column_break_wbna", - "bank_party_name", - "bank_party_account_number", - "bank_party_iban" - ], - "fields": [ - { - "fieldname": "bank_party_account_number", - "fieldtype": "Data", - "in_list_view": 1, - "label": "Party Account No. (Bank Statement)" - }, - { - "fieldname": "bank_party_iban", - "fieldtype": "Data", - "in_list_view": 1, - "label": "Party IBAN (Bank Statement)" - }, - { - "fieldname": "column_break_wbna", - "fieldtype": "Column Break" - }, - { - "fieldname": "party_type", - "fieldtype": "Link", - "in_list_view": 1, - "in_standard_filter": 1, - "label": "Party Type", - "options": "DocType" - }, - { - "fieldname": "party", - "fieldtype": "Dynamic Link", - "in_list_view": 1, - "in_standard_filter": 1, - "label": "Party", - "options": "party_type" - }, - { - "fieldname": "bank_party_name", - "fieldtype": "Data", - "label": "Party Name (Bank Statement)" - } - ], - "index_web_pages_for_search": 1, - "links": [], - "modified": "2023-04-04 14:27:23.450456", - "modified_by": "Administrator", - "module": "Accounts", - "name": "Bank Party Mapper", - "owner": "Administrator", - "permissions": [ - { - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "share": 1, - "write": 1 - } - ], - "sort_field": "modified", - "sort_order": "DESC", - "states": [], - "track_changes": 1 -} \ No newline at end of file diff --git a/erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.py b/erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.py deleted file mode 100644 index 02e36b08fe..0000000000 --- a/erpnext/accounts/doctype/bank_party_mapper/bank_party_mapper.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and contributors -# For license information, please see license.txt - -import frappe -from frappe.model.document import Document - - -class BankPartyMapper(Document): - def on_update(self): - self.update_party_in_linked_transactions() - - def update_party_in_linked_transactions(self): - if self.is_new(): - return - - # Set updated party values in other linked bank transactions - bank_transaction = frappe.qb.DocType("Bank Transaction") - - frappe.qb.update(bank_transaction).set("party_type", self.party_type).set( - "party", self.party - ).where( - (bank_transaction.bank_party_mapper == self.name) - & ((bank_transaction.party_type != self.party_type) | (bank_transaction.party != self.party)) - ).run() diff --git a/erpnext/accounts/doctype/bank_party_mapper/test_bank_party_mapper.py b/erpnext/accounts/doctype/bank_party_mapper/test_bank_party_mapper.py deleted file mode 100644 index c05b23f1a5..0000000000 --- a/erpnext/accounts/doctype/bank_party_mapper/test_bank_party_mapper.py +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and Contributors -# See license.txt - -# import frappe -from frappe.tests.utils import FrappeTestCase - - -class TestBankPartyMapper(FrappeTestCase): - pass diff --git a/erpnext/accounts/doctype/bank_transaction/auto_match_party.py b/erpnext/accounts/doctype/bank_transaction/auto_match_party.py index 753f0c1171..5d94a08f2f 100644 --- a/erpnext/accounts/doctype/bank_transaction/auto_match_party.py +++ b/erpnext/accounts/doctype/bank_transaction/auto_match_party.py @@ -10,22 +10,7 @@ class AutoMatchParty: Matches by Account/IBAN and then by Party Name/Description sequentially. Returns when a result is obtained. - Result (if present) is of the form: (Party Type, Party, Mapper,) - - Mapper(if present) is one of the forms: - 1. {"mapper_name": }: Indicates that an existing Bank Party Mapper matched against - the transaction and the same must be linked in the Bank Transaction. - - 2. {"bank_party_account_number": , "bank_party_iban": } : Indicates that a match was - found in Customer/Supplier/Employee by account details. A Bank Party Mapper is now created - mapping the Party to the Account No./IBAN - - 3. {"bank_party_name": }: Indicates that a match was found in - Customer/Supplier/Employee by party name. A Bank Party Mapper is now created mapping the Party - to the Party Name (Bank Statement). If matched by Description, no mapper is created as - description is not a static key. - - Mapper data is used either to create a new Bank Party Mapper or link an existing mapper to a transaction. + Result (if present) is of the form: (Party Type, Party,) """ def __init__(self, **kwargs) -> None: @@ -44,7 +29,7 @@ class AutoMatchParty: fuzzy_matching_enabled = frappe.db.get_single_value("Accounts Settings", "enable_fuzzy_matching") if not result and fuzzy_matching_enabled: - result = AutoMatchbyPartyDescription( + result = AutoMatchbyPartyNameDescription( bank_party_name=self.bank_party_name, description=self.description, deposit=self.deposit ).match() @@ -62,50 +47,16 @@ class AutoMatchbyAccountIBAN: if not (self.bank_party_account_number or self.bank_party_iban): return None - result = self.match_account_in_bank_party_mapper() - if not result: - result = self.match_account_in_party() - - return result - - def match_account_in_bank_party_mapper(self) -> Union[Tuple, None]: - """Check for a IBAN/Account No. match in Bank Party Mapper""" - result = None - or_filters = {} - if self.bank_party_account_number: - or_filters["bank_party_account_number"] = self.bank_party_account_number - - if self.bank_party_iban: - or_filters["bank_party_iban"] = self.bank_party_iban - - mapper = frappe.db.get_all( - "Bank Party Mapper", - or_filters=or_filters, - fields=["party_type", "party", "name"], - limit_page_length=1, - ) - if mapper: - data = mapper[0] - return (data["party_type"], data["party"], {"mapper_name": data["name"]}) - + result = self.match_account_in_party() return result def match_account_in_party(self) -> Union[Tuple, None]: """Check if there is a IBAN/Account No. match in Customer/Supplier/Employee""" result = None - - parties = ["Supplier", "Employee", "Customer"] # most -> least likely to receive - if flt(self.deposit) > 0: - parties = ["Customer", "Supplier", "Employee"] # most -> least likely to pay + parties = get_parties_in_order(self.deposit) + or_filters = self.get_or_filters() for party in parties: - or_filters = {} - if self.bank_party_account_number: - or_filters["bank_account_no"] = self.bank_party_account_number - - if self.bank_party_iban: - or_filters["iban"] = self.bank_party_iban - party_result = frappe.db.get_all( "Bank Account", or_filters=or_filters, pluck="party", limit_page_length=1 ) @@ -123,17 +74,23 @@ class AutoMatchbyAccountIBAN: result = ( party, party_result[0], - { - "bank_party_account_number": self.get("bank_party_account_number"), - "bank_party_iban": self.get("bank_party_iban"), - }, ) break return result + def get_or_filters(self) -> dict: + or_filters = {} + if self.bank_party_account_number: + or_filters["bank_account_no"] = self.bank_party_account_number -class AutoMatchbyPartyDescription: + if self.bank_party_iban: + or_filters["iban"] = self.bank_party_iban + + return or_filters + + +class AutoMatchbyPartyNameDescription: def __init__(self, **kwargs) -> None: self.__dict__.update(kwargs) @@ -141,52 +98,21 @@ class AutoMatchbyPartyDescription: return self.__dict__.get(key, None) def match(self) -> Union[Tuple, None]: - # Match by Customer, Supplier or Employee Name - # search bank party mapper by party # fuzzy search by customer/supplier & employee if not (self.bank_party_name or self.description): return None - result = self.match_party_name_in_bank_party_mapper() - - if not result: - result = self.match_party_name_desc_in_party() - - return result - - def match_party_name_in_bank_party_mapper(self) -> Union[Tuple, None]: - """Check if match exists for party name in Bank Party Mapper""" - result = None - if not self.bank_party_name: - return - - mapper_res = frappe.get_all( - "Bank Party Mapper", - filters={"bank_party_name": self.bank_party_name}, - fields=["party_type", "party", "name"], - limit_page_length=1, - ) - if mapper_res: - mapper_res = mapper_res[0] - return ( - mapper_res["party_type"], - mapper_res["party"], - {"mapper_name": mapper_res["name"]}, - ) - + result = self.match_party_name_desc_in_party() return result def match_party_name_desc_in_party(self) -> Union[Tuple, None]: """Fuzzy search party name and/or description against parties in the system""" result = None - parties = ["Supplier", "Employee", "Customer"] # most-least likely to receive - if flt(self.deposit) > 0.0: - parties = ["Customer", "Supplier", "Employee"] # most-least likely to pay + parties = get_parties_in_order(self.deposit) for party in parties: - name_field = party.lower() + "_name" filters = {"status": "Active"} if party == "Employee" else {"disabled": 0} - names = frappe.get_all(party, filters=filters, pluck=name_field) + names = frappe.get_all(party, filters=filters, pluck=party.lower() + "_name") for field in ["bank_party_name", "description"]: if not self.get(field): @@ -197,8 +123,7 @@ class AutoMatchbyPartyDescription: break if result or skip: - # We skip if: - # If it was hard to distinguish between close matches and so match is None + # Skip If: It was hard to distinguish between close matches and so match is None # OR if the right match was found break @@ -206,19 +131,15 @@ class AutoMatchbyPartyDescription: def fuzzy_search_and_return_result(self, party, names, field) -> Union[Tuple, None]: skip = False - result = process.extract(query=self.get(field), choices=names, scorer=fuzz.token_set_ratio) party_name, skip = self.process_fuzzy_result(result) if not party_name: return None, skip - # Dont set description as a key in Bank Party Mapper due to its volatility - mapper = {"bank_party_name": self.get(field)} if field == "bank_party_name" else None return ( party, party_name, - mapper, ), skip def process_fuzzy_result(self, result: Union[list, None]): @@ -226,7 +147,7 @@ class AutoMatchbyPartyDescription: If there are multiple valid close matches return None as result may be faulty. Return the result only if one accurate match stands out. - Returns: Result, Skip (whether or not to continue matching) + Returns: Result, Skip (whether or not to discontinue matching) """ PARTY, SCORE, CUTOFF = 0, 1, 80 @@ -234,18 +155,24 @@ class AutoMatchbyPartyDescription: return None, False first_result = result[0] - if len(result) == 1: - return (result[0][PARTY] if first_result[SCORE] > CUTOFF else None), True + return (first_result[PARTY] if first_result[SCORE] > CUTOFF else None), True second_result = result[1] - if first_result[SCORE] > CUTOFF: # If multiple matches with the same score, return None but discontinue matching - # Matches were found but were too closes to distinguish between + # Matches were found but were too close to distinguish between if first_result[SCORE] == second_result[SCORE]: return None, True return first_result[PARTY], True else: return None, False + + +def get_parties_in_order(deposit: float) -> list: + parties = ["Supplier", "Employee", "Customer"] # most -> least likely to receive + if flt(deposit) > 0: + parties = ["Customer", "Supplier", "Employee"] # most -> least likely to pay + + return parties diff --git a/erpnext/accounts/doctype/bank_transaction/bank_transaction.json b/erpnext/accounts/doctype/bank_transaction/bank_transaction.json index e7de71aef4..b32022e6fd 100644 --- a/erpnext/accounts/doctype/bank_transaction/bank_transaction.json +++ b/erpnext/accounts/doctype/bank_transaction/bank_transaction.json @@ -37,8 +37,7 @@ "column_break_3czf", "bank_party_name", "bank_party_account_number", - "bank_party_iban", - "bank_party_mapper" + "bank_party_iban" ], "fields": [ { @@ -226,19 +225,11 @@ "fieldname": "bank_party_account_number", "fieldtype": "Data", "label": "Party Account No. (Bank Statement)" - }, - { - "fieldname": "bank_party_mapper", - "fieldtype": "Link", - "hidden": 1, - "label": "Bank Party Mapper", - "options": "Bank Party Mapper", - "read_only": 1 } ], "is_submittable": 1, "links": [], - "modified": "2023-05-17 14:56:10.547480", + "modified": "2023-06-06 13:58:12.821411", "modified_by": "Administrator", "module": "Accounts", "name": "Bank Transaction", diff --git a/erpnext/accounts/doctype/bank_transaction/bank_transaction.py b/erpnext/accounts/doctype/bank_transaction/bank_transaction.py index 04c3013a00..f82337fbd7 100644 --- a/erpnext/accounts/doctype/bank_transaction/bank_transaction.py +++ b/erpnext/accounts/doctype/bank_transaction/bank_transaction.py @@ -29,9 +29,6 @@ class BankTransaction(StatusUpdater): self.update_allocations() self._saving_flag = False - if frappe.db.get_single_value("Accounts Settings", "enable_party_matching"): - self.update_automatch_bank_party_mapper() - def on_cancel(self): self.clear_linked_payment_entries(for_cancel=True) self.set_status(update=True) @@ -167,33 +164,10 @@ class BankTransaction(StatusUpdater): ).match() if result: - party_type, party, mapper = result - to_update = {"party_type": party_type, "party": party} - - if mapper and mapper.get("mapper_name"): - # Transaction matched with an existing Bank party Mapper record - to_update["bank_party_mapper"] = mapper.get("mapper_name") - elif mapper: - # Make new Mapper record to remember match - mapper_doc = frappe.get_doc( - {"doctype": "Bank Party Mapper", "party_type": party_type, "party": party} - ) - mapper_doc.update(mapper) - mapper_doc.insert() - to_update["bank_party_mapper"] = mapper_doc.name - - frappe.db.set_value("Bank Transaction", self.name, field=to_update) - - def update_automatch_bank_party_mapper(self): - """Update Bank Party Mapper if Party Type & Party are manually changed after submit.""" - doc_before_update = self.get_doc_before_save() - party_type_changed = self.party_type and (doc_before_update.party_type != self.party_type) - party_changed = self.party and (doc_before_update.party != self.party) - - if (party_type_changed or party_changed) and self.bank_party_mapper: - mapper_doc = frappe.get_doc("Bank Party Mapper", self.bank_party_mapper) - mapper_doc.update({"party_type": self.party_type, "party": self.party}) - mapper_doc.save() + party_type, party = result + frappe.db.set_value( + "Bank Transaction", self.name, field={"party_type": party_type, "party": party} + ) @frappe.whitelist() diff --git a/erpnext/accounts/doctype/bank_transaction/test_auto_match_party.py b/erpnext/accounts/doctype/bank_transaction/test_auto_match_party.py index 8c6dc9d766..a498545f2a 100644 --- a/erpnext/accounts/doctype/bank_transaction/test_auto_match_party.py +++ b/erpnext/accounts/doctype/bank_transaction/test_auto_match_party.py @@ -22,7 +22,6 @@ class TestAutoMatchParty(FrappeTestCase): frappe.db.set_single_value("Accounts Settings", "enable_fuzzy_matching", 0) def test_match_by_account_number(self): - """Test if transaction matches with existing (Bank Party Mapper) or new match.""" create_supplier_for_match(account_no="000000003716541159") doc = create_bank_transaction( withdrawal=1200, @@ -33,22 +32,6 @@ class TestAutoMatchParty(FrappeTestCase): self.assertEqual(doc.party_type, "Supplier") self.assertEqual(doc.party, "John Doe & Co.") - self.assertTrue(doc.bank_party_mapper) - - # Check if Bank Party Mapper is created to remember mapping - bank_party_mapper = frappe.get_doc("Bank Party Mapper", doc.bank_party_mapper) - self.assertEqual(bank_party_mapper.party, "John Doe & Co.") - self.assertEqual(bank_party_mapper.bank_party_account_number, "000000003716541159") - self.assertEqual(bank_party_mapper.bank_party_iban, "DE02000000003716541159") - - # Check if created mapping is used for quick match - doc_2 = create_bank_transaction( - withdrawal=500, - transaction_id="602413b8ji8bf838fub8f2c6a39bah7y", - account_no="000000003716541159", - ) - self.assertEqual(doc_2.party, "John Doe & Co.") - self.assertEqual(doc_2.bank_party_mapper, bank_party_mapper.name) def test_match_by_iban(self): create_supplier_for_match(iban="DE02000000003716541159") @@ -61,12 +44,6 @@ class TestAutoMatchParty(FrappeTestCase): self.assertEqual(doc.party_type, "Supplier") self.assertEqual(doc.party, "John Doe & Co.") - self.assertTrue(doc.bank_party_mapper) - - bank_party_mapper = frappe.get_doc("Bank Party Mapper", doc.bank_party_mapper) - self.assertEqual(bank_party_mapper.party, "John Doe & Co.") - self.assertEqual(bank_party_mapper.bank_party_account_number, "000000003716541159") - self.assertEqual(bank_party_mapper.bank_party_iban, "DE02000000003716541159") def test_match_by_party_name(self): create_supplier_for_match(supplier_name="Jackson Ella W.") @@ -78,22 +55,6 @@ class TestAutoMatchParty(FrappeTestCase): ) self.assertEqual(doc.party_type, "Supplier") self.assertEqual(doc.party, "Jackson Ella W.") - self.assertTrue(doc.bank_party_mapper) - - bank_party_mapper = frappe.get_doc("Bank Party Mapper", doc.bank_party_mapper) - self.assertEqual(bank_party_mapper.party, "Jackson Ella W.") - self.assertEqual(bank_party_mapper.bank_party_name, "Ella Jackson") - self.assertEqual(bank_party_mapper.bank_party_iban, None) - - # Check if created mapping is used for quick match - doc_2 = create_bank_transaction( - withdrawal=500, - transaction_id="578313b8ji8bf838fub8f2c6a39bah7y", - party_name="Ella Jackson", - account_no="000000004316531152", - ) - self.assertEqual(doc_2.party, "Jackson Ella W.") - self.assertEqual(doc_2.bank_party_mapper, bank_party_mapper.name) def test_match_by_description(self): create_supplier_for_match(supplier_name="Microsoft") @@ -107,46 +68,6 @@ class TestAutoMatchParty(FrappeTestCase): self.assertEqual(doc.party, "Microsoft") self.assertFalse(doc.bank_party_mapper) - def test_correct_match_after_submit(self): - """Correct wrong mapping after submit. Test impact.""" - # Similar named suppliers - create_supplier_for_match(supplier_name="Amazon") - create_supplier_for_match(supplier_name="Amazing Co.") - - # Bank Transactions actually from "Amazon" that match better with "Amazing Co." - doc = create_bank_transaction( - description="visa06323202 amzn.com/bill 7,88eur1,5324711959 90.22. 1,62 87861003", - withdrawal=24.85, - transaction_id="3a1da4ee2dc5a980138d36ef5297cbd9", - party_name="Amazn Co.", - ) - doc_2 = create_bank_transaction( - description="visa61268005 amzn.com/bill 22,345eur1,7654711959 89.23. 1,64 61268005", - withdrawal=80, - transaction_id="584314e459b00f792bfd569267efba6e", - party_name="Amazn Co.", - ) - - self.assertEqual(doc.party_type, "Supplier") - self.assertEqual(doc.party, "Amazing Co.") - self.assertTrue(doc.bank_party_mapper) - self.assertTrue(doc_2.bank_party_mapper, doc.bank_party_mapper) - - bank_party_mapper = frappe.get_doc("Bank Party Mapper", doc.bank_party_mapper) - self.assertEqual(bank_party_mapper.party, "Amazing Co.") - self.assertEqual(bank_party_mapper.bank_party_name, "Amazn Co.") - - # User corrects the value after submit to "Amazon" - doc.party = "Amazon" - doc.save() - bank_party_mapper.reload() - doc_2.reload() - - # Mapping is edited and all transactions with this mapping are updated - self.assertEqual(bank_party_mapper.party, "Amazon") - self.assertEqual(bank_party_mapper.bank_party_name, "Amazn Co.") - self.assertEqual(doc_2.party, "Amazon") - def test_skip_match_if_multiple_close_results(self): create_supplier_for_match(supplier_name="Adithya Medical & General Stores") create_supplier_for_match(supplier_name="Adithya Medical And General Stores") From eb1db5eaa3e3d28351b4f906b7e91b03da7ae83c Mon Sep 17 00:00:00 2001 From: marination Date: Wed, 7 Jun 2023 11:54:16 +0530 Subject: [PATCH 15/17] chore: Remove instances of `bank_party_mapper` and use `new_doc` --- .../bank_transaction/test_auto_match_party.py | 49 ++++++++----------- 1 file changed, 20 insertions(+), 29 deletions(-) diff --git a/erpnext/accounts/doctype/bank_transaction/test_auto_match_party.py b/erpnext/accounts/doctype/bank_transaction/test_auto_match_party.py index a498545f2a..36ef1fca07 100644 --- a/erpnext/accounts/doctype/bank_transaction/test_auto_match_party.py +++ b/erpnext/accounts/doctype/bank_transaction/test_auto_match_party.py @@ -66,7 +66,6 @@ class TestAutoMatchParty(FrappeTestCase): ) self.assertEqual(doc.party_type, "Supplier") self.assertEqual(doc.party, "Microsoft") - self.assertFalse(doc.bank_party_mapper) def test_skip_match_if_multiple_close_results(self): create_supplier_for_match(supplier_name="Adithya Medical & General Stores") @@ -82,7 +81,6 @@ class TestAutoMatchParty(FrappeTestCase): # Mapping is skipped as both Supplier names have the same match score self.assertEqual(doc.party_type, None) self.assertEqual(doc.party, None) - self.assertFalse(doc.bank_party_mapper) def create_supplier_for_match(supplier_name="John Doe & Co.", iban=None, account_no=None): @@ -99,35 +97,26 @@ def create_supplier_for_match(supplier_name="John Doe & Co.", iban=None, account return # Create Supplier and Bank Account for the same - supplier = frappe.get_doc( - { - "doctype": "Supplier", - "supplier_name": supplier_name, - "supplier_group": "Services", - "supplier_type": "Company", - } - ).insert() + supplier = frappe.new_doc("Supplier") + supplier.supplier_name = supplier_name + supplier.supplier_group = "Services" + supplier.supplier_type = "Company" + supplier.insert() if not frappe.db.exists("Bank", "TestBank"): - frappe.get_doc( - { - "doctype": "Bank", - "bank_name": "TestBank", - } - ).insert(ignore_if_duplicate=True) + bank = frappe.new_doc("Bank") + bank.bank_name = "TestBank" + bank.insert(ignore_if_duplicate=True) if not frappe.db.exists("Bank Account", supplier.name + " - " + "TestBank"): - frappe.get_doc( - { - "doctype": "Bank Account", - "account_name": supplier.name, - "bank": "TestBank", - "iban": iban, - "bank_account_no": account_no, - "party_type": "Supplier", - "party": supplier.name, - } - ).insert() + bank_account = frappe.new_doc("Bank Account") + bank_account.account_name = supplier.name + bank_account.bank = "TestBank" + bank_account.iban = iban + bank_account.bank_account_no = account_no + bank_account.party_type = "Supplier" + bank_account.party = supplier.name + bank_account.insert() def create_bank_transaction( @@ -139,7 +128,8 @@ def create_bank_transaction( account_no=None, iban=None, ): - doc = frappe.get_doc( + doc = frappe.new_doc("Bank Transaction") + doc.update( { "doctype": "Bank Transaction", "description": description or "1512567 BG/000002918 OPSKATTUZWXXX AT776000000098709837 Herr G", @@ -153,7 +143,8 @@ def create_bank_transaction( "bank_party_account_number": account_no, "bank_party_iban": iban, } - ).insert() + ) + doc.insert() doc.submit() doc.reload() From 0d125885836f0ae5015195401dd41653314fddfe Mon Sep 17 00:00:00 2001 From: Anand Baburajan Date: Tue, 20 Jun 2023 12:06:27 +0530 Subject: [PATCH 16/17] fix: date and finance book fixes in fixed asset register (#35751) * fix: handle finance books properly and show all assets by default in fixed asset register * chore: rename value to depr amount * chore: get asset value for correct fb properly * chore: rename include_default_book_entries to include_default_book_assets --- .../fixed_asset_register.js | 119 +++++++------- .../fixed_asset_register.py | 146 +++++++++++------- 2 files changed, 147 insertions(+), 118 deletions(-) diff --git a/erpnext/assets/report/fixed_asset_register/fixed_asset_register.js b/erpnext/assets/report/fixed_asset_register/fixed_asset_register.js index 4f7b836107..b788a32d6a 100644 --- a/erpnext/assets/report/fixed_asset_register/fixed_asset_register.js +++ b/erpnext/assets/report/fixed_asset_register/fixed_asset_register.js @@ -19,56 +19,6 @@ frappe.query_reports["Fixed Asset Register"] = { options: "\nIn Location\nDisposed", default: 'In Location' }, - { - "fieldname":"filter_based_on", - "label": __("Period Based On"), - "fieldtype": "Select", - "options": ["Fiscal Year", "Date Range"], - "default": "Fiscal Year", - "reqd": 1 - }, - { - "fieldname":"from_date", - "label": __("Start Date"), - "fieldtype": "Date", - "default": frappe.datetime.add_months(frappe.datetime.nowdate(), -12), - "depends_on": "eval: doc.filter_based_on == 'Date Range'", - "reqd": 1 - }, - { - "fieldname":"to_date", - "label": __("End Date"), - "fieldtype": "Date", - "default": frappe.datetime.nowdate(), - "depends_on": "eval: doc.filter_based_on == 'Date Range'", - "reqd": 1 - }, - { - "fieldname":"from_fiscal_year", - "label": __("Start Year"), - "fieldtype": "Link", - "options": "Fiscal Year", - "default": frappe.defaults.get_user_default("fiscal_year"), - "depends_on": "eval: doc.filter_based_on == 'Fiscal Year'", - "reqd": 1 - }, - { - "fieldname":"to_fiscal_year", - "label": __("End Year"), - "fieldtype": "Link", - "options": "Fiscal Year", - "default": frappe.defaults.get_user_default("fiscal_year"), - "depends_on": "eval: doc.filter_based_on == 'Fiscal Year'", - "reqd": 1 - }, - { - "fieldname":"date_based_on", - "label": __("Date Based On"), - "fieldtype": "Select", - "options": ["Purchase Date", "Available For Use Date"], - "default": "Purchase Date", - "reqd": 1 - }, { fieldname:"asset_category", label: __("Asset Category"), @@ -89,22 +39,67 @@ frappe.query_reports["Fixed Asset Register"] = { default: "--Select a group--", reqd: 1 }, - { - fieldname:"finance_book", - label: __("Finance Book"), - fieldtype: "Link", - options: "Finance Book", - depends_on: "eval: doc.filter_by_finance_book == 1", - }, - { - fieldname:"filter_by_finance_book", - label: __("Filter by Finance Book"), - fieldtype: "Check" - }, { fieldname:"only_existing_assets", label: __("Only existing assets"), fieldtype: "Check" }, + { + fieldname:"finance_book", + label: __("Finance Book"), + fieldtype: "Link", + options: "Finance Book", + }, + { + "fieldname": "include_default_book_assets", + "label": __("Include Default Book Assets"), + "fieldtype": "Check", + "default": 1 + }, + { + "fieldname":"filter_based_on", + "label": __("Period Based On"), + "fieldtype": "Select", + "options": ["--Select a period--", "Fiscal Year", "Date Range"], + "default": "--Select a period--", + }, + { + "fieldname":"from_date", + "label": __("Start Date"), + "fieldtype": "Date", + "default": frappe.datetime.add_months(frappe.datetime.nowdate(), -12), + "depends_on": "eval: doc.filter_based_on == 'Date Range'", + }, + { + "fieldname":"to_date", + "label": __("End Date"), + "fieldtype": "Date", + "default": frappe.datetime.nowdate(), + "depends_on": "eval: doc.filter_based_on == 'Date Range'", + }, + { + "fieldname":"from_fiscal_year", + "label": __("Start Year"), + "fieldtype": "Link", + "options": "Fiscal Year", + "default": frappe.defaults.get_user_default("fiscal_year"), + "depends_on": "eval: doc.filter_based_on == 'Fiscal Year'", + }, + { + "fieldname":"to_fiscal_year", + "label": __("End Year"), + "fieldtype": "Link", + "options": "Fiscal Year", + "default": frappe.defaults.get_user_default("fiscal_year"), + "depends_on": "eval: doc.filter_based_on == 'Fiscal Year'", + }, + { + "fieldname":"date_based_on", + "label": __("Date Based On"), + "fieldtype": "Select", + "options": ["Purchase Date", "Available For Use Date"], + "default": "Purchase Date", + "depends_on": "eval: doc.filter_based_on == 'Date Range' || doc.filter_based_on == 'Fiscal Year'", + }, ] }; diff --git a/erpnext/assets/report/fixed_asset_register/fixed_asset_register.py b/erpnext/assets/report/fixed_asset_register/fixed_asset_register.py index 984b3fd982..cb61914c25 100644 --- a/erpnext/assets/report/fixed_asset_register/fixed_asset_register.py +++ b/erpnext/assets/report/fixed_asset_register/fixed_asset_register.py @@ -2,9 +2,11 @@ # For license information, please see license.txt +from itertools import chain + import frappe from frappe import _ -from frappe.query_builder.functions import Sum +from frappe.query_builder.functions import IfNull, Sum from frappe.utils import cstr, flt, formatdate, getdate from erpnext.accounts.report.financial_statements import ( @@ -13,7 +15,6 @@ from erpnext.accounts.report.financial_statements import ( validate_fiscal_year, ) from erpnext.assets.doctype.asset.asset import get_asset_value_after_depreciation -from erpnext.assets.doctype.asset.depreciation import get_depreciation_accounts def execute(filters=None): @@ -64,11 +65,9 @@ def get_conditions(filters): def get_data(filters): - data = [] conditions = get_conditions(filters) - depreciation_amount_map = get_finance_book_value_map(filters) pr_supplier_map = get_purchase_receipt_supplier_map() pi_supplier_map = get_purchase_invoice_supplier_map() @@ -102,20 +101,27 @@ def get_data(filters): ] assets_record = frappe.db.get_all("Asset", filters=conditions, fields=fields) - assets_linked_to_fb = None + assets_linked_to_fb = get_assets_linked_to_fb(filters) - if filters.filter_by_finance_book: - assets_linked_to_fb = frappe.db.get_all( - doctype="Asset Finance Book", - filters={"finance_book": filters.finance_book or ("is", "not set")}, - pluck="parent", - ) + company_fb = frappe.get_cached_value("Company", filters.company, "default_finance_book") + + if filters.include_default_book_assets and company_fb: + finance_book = company_fb + elif filters.finance_book: + finance_book = filters.finance_book + else: + finance_book = None + + depreciation_amount_map = get_asset_depreciation_amount_map(filters, finance_book) for asset in assets_record: if assets_linked_to_fb and asset.asset_id not in assets_linked_to_fb: continue - asset_value = get_asset_value_after_depreciation(asset.asset_id, filters.finance_book) + asset_value = get_asset_value_after_depreciation( + asset.asset_id, finance_book + ) or get_asset_value_after_depreciation(asset.asset_id) + row = { "asset_id": asset.asset_id, "asset_name": asset.asset_name, @@ -126,7 +132,7 @@ def get_data(filters): or pi_supplier_map.get(asset.purchase_invoice), "gross_purchase_amount": asset.gross_purchase_amount, "opening_accumulated_depreciation": asset.opening_accumulated_depreciation, - "depreciated_amount": get_depreciation_amount_of_asset(asset, depreciation_amount_map, filters), + "depreciated_amount": get_depreciation_amount_of_asset(asset, depreciation_amount_map), "available_for_use_date": asset.available_for_use_date, "location": asset.location, "asset_category": asset.asset_category, @@ -140,14 +146,23 @@ def get_data(filters): def prepare_chart_data(data, filters): labels_values_map = {} - date_field = frappe.scrub(filters.date_based_on) + if filters.filter_based_on not in ("Date Range", "Fiscal Year"): + filters_filter_based_on = "Date Range" + date_field = "purchase_date" + filters_from_date = min(data, key=lambda a: a.get(date_field)).get(date_field) + filters_to_date = max(data, key=lambda a: a.get(date_field)).get(date_field) + else: + filters_filter_based_on = filters.filter_based_on + date_field = frappe.scrub(filters.date_based_on) + filters_from_date = filters.from_date + filters_to_date = filters.to_date period_list = get_period_list( filters.from_fiscal_year, filters.to_fiscal_year, - filters.from_date, - filters.to_date, - filters.filter_based_on, + filters_from_date, + filters_to_date, + filters_filter_based_on, "Monthly", company=filters.company, ignore_fiscal_year=True, @@ -184,59 +199,78 @@ def prepare_chart_data(data, filters): } -def get_depreciation_amount_of_asset(asset, depreciation_amount_map, filters): - if asset.calculate_depreciation: - depr_amount = depreciation_amount_map.get(asset.asset_id) or 0.0 - else: - depr_amount = get_manual_depreciation_amount_of_asset(asset, filters) +def get_assets_linked_to_fb(filters): + afb = frappe.qb.DocType("Asset Finance Book") - return flt(depr_amount, 2) - - -def get_finance_book_value_map(filters): - date = filters.to_date if filters.filter_based_on == "Date Range" else filters.year_end_date - - return frappe._dict( - frappe.db.sql( - """ Select - ads.asset, SUM(depreciation_amount) - FROM `tabAsset Depreciation Schedule` ads, `tabDepreciation Schedule` ds - WHERE - ds.parent = ads.name - AND ifnull(ads.finance_book, '')=%s - AND ads.docstatus=1 - AND ds.parentfield='depreciation_schedule' - AND ds.schedule_date<=%s - AND ds.journal_entry IS NOT NULL - GROUP BY ads.asset""", - (cstr(filters.finance_book or ""), date), - ) + query = frappe.qb.from_(afb).select( + afb.parent, ) + if filters.include_default_book_assets: + company_fb = frappe.get_cached_value("Company", filters.company, "default_finance_book") -def get_manual_depreciation_amount_of_asset(asset, filters): + if filters.finance_book and company_fb and cstr(filters.finance_book) != cstr(company_fb): + frappe.throw( + _("To use a different finance book, please uncheck 'Include Default Book Entries'") + ) + + query = query.where( + (afb.finance_book.isin([cstr(filters.finance_book), cstr(company_fb), ""])) + | (afb.finance_book.isnull()) + ) + else: + query = query.where( + (afb.finance_book.isin([cstr(filters.finance_book), ""])) | (afb.finance_book.isnull()) + ) + + assets_linked_to_fb = list(chain(*query.run(as_list=1))) + + return assets_linked_to_fb + + +def get_depreciation_amount_of_asset(asset, depreciation_amount_map): + return depreciation_amount_map.get(asset.asset_id) or 0.0 + + +def get_asset_depreciation_amount_map(filters, finance_book): date = filters.to_date if filters.filter_based_on == "Date Range" else filters.year_end_date - (_, _, depreciation_expense_account) = get_depreciation_accounts(asset) - + asset = frappe.qb.DocType("Asset") gle = frappe.qb.DocType("GL Entry") + aca = frappe.qb.DocType("Asset Category Account") + company = frappe.qb.DocType("Company") - result = ( + query = ( frappe.qb.from_(gle) - .select(Sum(gle.debit)) - .where(gle.against_voucher == asset.asset_id) - .where(gle.account == depreciation_expense_account) + .join(asset) + .on(gle.against_voucher == asset.name) + .join(aca) + .on((aca.parent == asset.asset_category) & (aca.company_name == asset.company)) + .join(company) + .on(company.name == asset.company) + .select(asset.name.as_("asset"), Sum(gle.debit).as_("depreciation_amount")) + .where( + gle.account == IfNull(aca.depreciation_expense_account, company.depreciation_expense_account) + ) .where(gle.debit != 0) .where(gle.is_cancelled == 0) - .where(gle.posting_date <= date) - ).run() + .where(asset.docstatus == 1) + .groupby(asset.name) + ) - if result and result[0] and result[0][0]: - depr_amount = result[0][0] + if finance_book: + query = query.where( + (gle.finance_book.isin([cstr(finance_book), ""])) | (gle.finance_book.isnull()) + ) else: - depr_amount = 0 + query = query.where((gle.finance_book.isin([""])) | (gle.finance_book.isnull())) - return depr_amount + if filters.filter_based_on in ("Date Range", "Fiscal Year"): + query = query.where(gle.posting_date <= date) + + asset_depr_amount_map = query.run() + + return dict(asset_depr_amount_map) def get_purchase_receipt_supplier_map(): From df090cbe873d827f5fee64feee20a68b6eab15ce Mon Sep 17 00:00:00 2001 From: Anand Baburajan Date: Tue, 20 Jun 2023 13:03:15 +0530 Subject: [PATCH 17/17] chore: minor typo in fixed asset register (#35801) chore: renaming entries to assets --- .../report/fixed_asset_register/fixed_asset_register.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/erpnext/assets/report/fixed_asset_register/fixed_asset_register.py b/erpnext/assets/report/fixed_asset_register/fixed_asset_register.py index cb61914c25..f810819b4f 100644 --- a/erpnext/assets/report/fixed_asset_register/fixed_asset_register.py +++ b/erpnext/assets/report/fixed_asset_register/fixed_asset_register.py @@ -210,9 +210,7 @@ def get_assets_linked_to_fb(filters): company_fb = frappe.get_cached_value("Company", filters.company, "default_finance_book") if filters.finance_book and company_fb and cstr(filters.finance_book) != cstr(company_fb): - frappe.throw( - _("To use a different finance book, please uncheck 'Include Default Book Entries'") - ) + frappe.throw(_("To use a different finance book, please uncheck 'Include Default Book Assets'")) query = query.where( (afb.finance_book.isin([cstr(filters.finance_book), cstr(company_fb), ""]))