From 2535d5edd0658844a74177da3a3a4bb6abe3f1a2 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Tue, 14 Jun 2022 18:20:33 +0530 Subject: [PATCH] perf: GLE reposting with progress and chunking (#31343) If stock voucher count goes >1000 then fetching all gles and reposting them all at once requires much more memory and can cause crash. - This PR ensures that GLE reposting is done in chunks of 100 vouchers. - This PR also starts keeping track of how many such chunks were processed so in future progress is resumed in event of timeout. --- erpnext/accounts/test/test_utils.py | 2 +- erpnext/accounts/utils.py | 84 ++++++++++++------- .../repost_item_valuation.json | 13 ++- .../repost_item_valuation.py | 1 + .../test_repost_item_valuation.py | 32 +++++++ 5 files changed, 99 insertions(+), 33 deletions(-) diff --git a/erpnext/accounts/test/test_utils.py b/erpnext/accounts/test/test_utils.py index 77c40bae2d..882cd694a3 100644 --- a/erpnext/accounts/test/test_utils.py +++ b/erpnext/accounts/test/test_utils.py @@ -62,8 +62,8 @@ class TestUtils(unittest.TestCase): stock_entry = {"item": item, "to_warehouse": "_Test Warehouse - _TC", "qty": 1, "rate": 10} se1 = make_stock_entry(posting_date="2022-01-01", **stock_entry) - se2 = make_stock_entry(posting_date="2022-02-01", **stock_entry) se3 = make_stock_entry(posting_date="2022-03-01", **stock_entry) + se2 = make_stock_entry(posting_date="2022-02-01", **stock_entry) for doc in (se1, se2, se3): vouchers.append((doc.doctype, doc.name)) diff --git a/erpnext/accounts/utils.py b/erpnext/accounts/utils.py index 65e05410aa..2d86dead2f 100644 --- a/erpnext/accounts/utils.py +++ b/erpnext/accounts/utils.py @@ -2,8 +2,9 @@ # License: GNU General Public License v3. See license.txt +import itertools from json import loads -from typing import List, Tuple +from typing import TYPE_CHECKING, List, Optional, Tuple import frappe import frappe.defaults @@ -22,6 +23,9 @@ from erpnext.accounts.doctype.accounting_dimension.accounting_dimension import g from erpnext.stock import get_warehouse_account_map from erpnext.stock.utils import get_stock_value_on +if TYPE_CHECKING: + from erpnext.stock.doctype.repost_item_valuation.repost_item_valuation import RepostItemValuation + class FiscalYearError(frappe.ValidationError): pass @@ -31,6 +35,9 @@ class PaymentEntryUnlinkError(frappe.ValidationError): pass +GL_REPOSTING_CHUNK = 100 + + @frappe.whitelist() def get_fiscal_year( date=None, fiscal_year=None, label="Date", verbose=1, company=None, as_dict=False @@ -1120,7 +1127,11 @@ def update_gl_entries_after( def repost_gle_for_stock_vouchers( - stock_vouchers, posting_date, company=None, warehouse_account=None + stock_vouchers: List[Tuple[str, str]], + posting_date: str, + company: Optional[str] = None, + warehouse_account=None, + repost_doc: Optional["RepostItemValuation"] = None, ): from erpnext.accounts.general_ledger import toggle_debit_credit_if_negative @@ -1128,40 +1139,50 @@ def repost_gle_for_stock_vouchers( if not stock_vouchers: return - def _delete_gl_entries(voucher_type, voucher_no): - frappe.db.sql( - """delete from `tabGL Entry` - where voucher_type=%s and voucher_no=%s""", - (voucher_type, voucher_no), - ) - - stock_vouchers = sort_stock_vouchers_by_posting_date(stock_vouchers) - if not warehouse_account: warehouse_account = get_warehouse_account_map(company) + stock_vouchers = sort_stock_vouchers_by_posting_date(stock_vouchers) + if repost_doc and repost_doc.gl_reposting_index: + # Restore progress + stock_vouchers = stock_vouchers[cint(repost_doc.gl_reposting_index) :] + precision = get_field_precision(frappe.get_meta("GL Entry").get_field("debit")) or 2 - gle = get_voucherwise_gl_entries(stock_vouchers, posting_date) - for idx, (voucher_type, voucher_no) in enumerate(stock_vouchers): - existing_gle = gle.get((voucher_type, voucher_no), []) - voucher_obj = frappe.get_doc(voucher_type, voucher_no) - # Some transactions post credit as negative debit, this is handled while posting GLE - # but while comparing we need to make sure it's flipped so comparisons are accurate - expected_gle = toggle_debit_credit_if_negative(voucher_obj.get_gl_entries(warehouse_account)) - if expected_gle: - if not existing_gle or not compare_existing_and_expected_gle( - existing_gle, expected_gle, precision - ): - _delete_gl_entries(voucher_type, voucher_no) - voucher_obj.make_gl_entries(gl_entries=expected_gle, from_repost=True) - else: - _delete_gl_entries(voucher_type, voucher_no) + stock_vouchers_iterator = iter(stock_vouchers) - if idx % 20 == 0: - # Commit every 20 documents to avoid losing progress - # and reducing memory usage - frappe.db.commit() + while stock_vouchers_chunk := list(itertools.islice(stock_vouchers_iterator, GL_REPOSTING_CHUNK)): + gle = get_voucherwise_gl_entries(stock_vouchers_chunk, posting_date) + + for voucher_type, voucher_no in stock_vouchers_chunk: + existing_gle = gle.get((voucher_type, voucher_no), []) + voucher_obj = frappe.get_doc(voucher_type, voucher_no) + # Some transactions post credit as negative debit, this is handled while posting GLE + # but while comparing we need to make sure it's flipped so comparisons are accurate + expected_gle = toggle_debit_credit_if_negative(voucher_obj.get_gl_entries(warehouse_account)) + if expected_gle: + if not existing_gle or not compare_existing_and_expected_gle( + existing_gle, expected_gle, precision + ): + _delete_gl_entries(voucher_type, voucher_no) + voucher_obj.make_gl_entries(gl_entries=expected_gle, from_repost=True) + else: + _delete_gl_entries(voucher_type, voucher_no) + frappe.db.commit() + + if repost_doc: + repost_doc.db_set( + "gl_reposting_index", + cint(repost_doc.gl_reposting_index) + GL_REPOSTING_CHUNK, + ) + + +def _delete_gl_entries(voucher_type, voucher_no): + frappe.db.sql( + """delete from `tabGL Entry` + where voucher_type=%s and voucher_no=%s""", + (voucher_type, voucher_no), + ) def sort_stock_vouchers_by_posting_date( @@ -1175,6 +1196,9 @@ def sort_stock_vouchers_by_posting_date( .select(sle.voucher_type, sle.voucher_no, sle.posting_date, sle.posting_time, sle.creation) .where((sle.is_cancelled == 0) & (sle.voucher_no.isin(voucher_nos))) .groupby(sle.voucher_type, sle.voucher_no) + .orderby(sle.posting_date) + .orderby(sle.posting_time) + .orderby(sle.creation) ).run(as_dict=True) sorted_vouchers = [(sle.voucher_type, sle.voucher_no) for sle in sles] diff --git a/erpnext/stock/doctype/repost_item_valuation/repost_item_valuation.json b/erpnext/stock/doctype/repost_item_valuation/repost_item_valuation.json index 156f77f5ac..e093933829 100644 --- a/erpnext/stock/doctype/repost_item_valuation/repost_item_valuation.json +++ b/erpnext/stock/doctype/repost_item_valuation/repost_item_valuation.json @@ -25,7 +25,8 @@ "items_to_be_repost", "affected_transactions", "distinct_item_and_warehouse", - "current_index" + "current_index", + "gl_reposting_index" ], "fields": [ { @@ -181,12 +182,20 @@ "label": "Affected Transactions", "no_copy": 1, "read_only": 1 + }, + { + "default": "0", + "fieldname": "gl_reposting_index", + "fieldtype": "Int", + "hidden": 1, + "label": "GL reposting index", + "read_only": 1 } ], "index_web_pages_for_search": 1, "is_submittable": 1, "links": [], - "modified": "2022-04-18 14:08:08.821602", + "modified": "2022-06-13 12:20:22.182322", "modified_by": "Administrator", "module": "Stock", "name": "Repost Item Valuation", diff --git a/erpnext/stock/doctype/repost_item_valuation/repost_item_valuation.py b/erpnext/stock/doctype/repost_item_valuation/repost_item_valuation.py index 328afc86be..ea24b47a29 100644 --- a/erpnext/stock/doctype/repost_item_valuation/repost_item_valuation.py +++ b/erpnext/stock/doctype/repost_item_valuation/repost_item_valuation.py @@ -192,6 +192,7 @@ def repost_gl_entries(doc): directly_dependent_transactions + list(repost_affected_transaction), doc.posting_date, doc.company, + repost_doc=doc, ) diff --git a/erpnext/stock/doctype/repost_item_valuation/test_repost_item_valuation.py b/erpnext/stock/doctype/repost_item_valuation/test_repost_item_valuation.py index 3184f69aa4..3c74619b46 100644 --- a/erpnext/stock/doctype/repost_item_valuation/test_repost_item_valuation.py +++ b/erpnext/stock/doctype/repost_item_valuation/test_repost_item_valuation.py @@ -2,10 +2,14 @@ # See license.txt +from unittest.mock import MagicMock, call + import frappe from frappe.tests.utils import FrappeTestCase from frappe.utils import nowdate +from frappe.utils.data import today +from erpnext.accounts.utils import repost_gle_for_stock_vouchers from erpnext.controllers.stock_controller import create_item_wise_repost_entries from erpnext.stock.doctype.item.test_item import make_item from erpnext.stock.doctype.purchase_receipt.test_purchase_receipt import make_purchase_receipt @@ -193,3 +197,31 @@ class TestRepostItemValuation(FrappeTestCase): [["a", "b"], ["c", "d"]], sorted(frappe.parse_json(frappe.as_json(set([("a", "b"), ("c", "d")])))), ) + + def test_gl_repost_progress(self): + from erpnext.accounts import utils + + # lower numbers to simplify test + orig_chunk_size = utils.GL_REPOSTING_CHUNK + utils.GL_REPOSTING_CHUNK = 1 + self.addCleanup(setattr, utils, "GL_REPOSTING_CHUNK", orig_chunk_size) + + doc = frappe.new_doc("Repost Item Valuation") + doc.db_set = MagicMock() + + vouchers = [] + company = "_Test Company with perpetual inventory" + posting_date = today() + + for _ in range(3): + se = make_stock_entry(company=company, qty=1, rate=2, target="Stores - TCP1") + vouchers.append((se.doctype, se.name)) + + repost_gle_for_stock_vouchers(stock_vouchers=vouchers, posting_date=posting_date, repost_doc=doc) + self.assertIn(call("gl_reposting_index", 1), doc.db_set.mock_calls) + doc.db_set.reset_mock() + + doc.gl_reposting_index = 1 + repost_gle_for_stock_vouchers(stock_vouchers=vouchers, posting_date=posting_date, repost_doc=doc) + + self.assertNotIn(call("gl_reposting_index", 1), doc.db_set.mock_calls)