Merge pull request #30726 from ankush/dependent_gle_reposting

fix: dependent gle reposting

erpnext/accounts/test/test_utils.py

@@ -1,10 +1,17 @@
 import unittest
 
+import frappe
 from frappe.test_runner import make_test_objects
 
 from erpnext.accounts.party import get_party_shipping_address
-from erpnext.accounts.utils import get_future_stock_vouchers, get_voucherwise_gl_entries
+from erpnext.accounts.utils import (
+	get_future_stock_vouchers,
+	get_voucherwise_gl_entries,
+	sort_stock_vouchers_by_posting_date,
+)
+from erpnext.stock.doctype.item.test_item import make_item
 from erpnext.stock.doctype.purchase_receipt.test_purchase_receipt import make_purchase_receipt
+from erpnext.stock.doctype.stock_entry.stock_entry_utils import make_stock_entry
 
 
 class TestUtils(unittest.TestCase):
@@ -47,6 +54,25 @@ class TestUtils(unittest.TestCase):
 			msg="get_voucherwise_gl_entries not returning expected GLes",
 		)
 
+	def test_stock_voucher_sorting(self):
+		vouchers = []
+
+		item = make_item().name
+
+		stock_entry = {"item": item, "to_warehouse": "_Test Warehouse - _TC", "qty": 1, "rate": 10}
+
+		se1 = make_stock_entry(posting_date="2022-01-01", **stock_entry)
+		se2 = make_stock_entry(posting_date="2022-02-01", **stock_entry)
+		se3 = make_stock_entry(posting_date="2022-03-01", **stock_entry)
+
+		for doc in (se1, se2, se3):
+			vouchers.append((doc.doctype, doc.name))
+
+		vouchers.append(("Stock Entry", "Wat"))
+
+		sorted_vouchers = sort_stock_vouchers_by_posting_date(list(reversed(vouchers)))
+		self.assertEqual(sorted_vouchers, vouchers)
+
 
 ADDRESS_RECORDS = [
 	{

erpnext/accounts/utils.py

@@ -3,6 +3,7 @@
 
 
 from json import loads
+from typing import List, Tuple
 
 import frappe
 import frappe.defaults
@@ -1122,6 +1123,9 @@ def update_gl_entries_after(
 def repost_gle_for_stock_vouchers(
 	stock_vouchers, posting_date, company=None, warehouse_account=None
 ):
+	if not stock_vouchers:
+		return
+
 	def _delete_gl_entries(voucher_type, voucher_no):
 		frappe.db.sql(
 			"""delete from `tabGL Entry`
@@ -1129,6 +1133,8 @@ def repost_gle_for_stock_vouchers(
 			(voucher_type, voucher_no),
 		)
 
+	stock_vouchers = sort_stock_vouchers_by_posting_date(stock_vouchers)
+
 	if not warehouse_account:
 		warehouse_account = get_warehouse_account_map(company)
 
@@ -1149,6 +1155,27 @@ def repost_gle_for_stock_vouchers(
 			_delete_gl_entries(voucher_type, voucher_no)
 
 
+def sort_stock_vouchers_by_posting_date(
+	stock_vouchers: List[Tuple[str, str]]
+) -> List[Tuple[str, str]]:
+	sle = frappe.qb.DocType("Stock Ledger Entry")
+	voucher_nos = [v[1] for v in stock_vouchers]
+
+	sles = (
+		frappe.qb.from_(sle)
+		.select(sle.voucher_type, sle.voucher_no, sle.posting_date, sle.posting_time, sle.creation)
+		.where((sle.is_cancelled == 0) & (sle.voucher_no.isin(voucher_nos)))
+		.groupby(sle.voucher_type, sle.voucher_no)
+	).run(as_dict=True)
+	sorted_vouchers = [(sle.voucher_type, sle.voucher_no) for sle in sles]
+
+	unknown_vouchers = set(stock_vouchers) - set(sorted_vouchers)
+	if unknown_vouchers:
+		sorted_vouchers.extend(unknown_vouchers)
+
+	return sorted_vouchers
+
+
 def get_future_stock_vouchers(
 	posting_date, posting_time, for_warehouses=None, for_items=None, company=None
 ):

erpnext/stock/doctype/bin/bin.py

@@ -4,8 +4,8 @@
 
 import frappe
 from frappe.model.document import Document
-from frappe.query_builder import Case
+from frappe.query_builder import Case, Order
-from frappe.query_builder.functions import Coalesce, Sum
+from frappe.query_builder.functions import Coalesce, CombineDatetime, Sum
 from frappe.utils import flt
 
 
@@ -121,24 +121,23 @@ def update_qty(bin_name, args):
 
 	bin_details = get_bin_details(bin_name)
 	# actual qty is already updated by processing current voucher
-	actual_qty = bin_details.actual_qty
+	actual_qty = bin_details.actual_qty or 0.0
+	sle = frappe.qb.DocType("Stock Ledger Entry")
 
 	# actual qty is not up to date in case of backdated transaction
 	if future_sle_exists(args):
-		actual_qty = (
+		last_sle_qty = (
-			frappe.db.get_value(
+			frappe.qb.from_(sle)
-				"Stock Ledger Entry",
+			.select(sle.qty_after_transaction)
-				filters={
+			.where((sle.item_code == args.get("item_code")) & (sle.warehouse == args.get("warehouse")))
-					"item_code": args.get("item_code"),
+			.orderby(CombineDatetime(sle.posting_date, sle.posting_time), order=Order.desc)
-					"warehouse": args.get("warehouse"),
+			.orderby(sle.creation, order=Order.desc)
-					"is_cancelled": 0,
+			.run()
-				},
-				fieldname="qty_after_transaction",
-				order_by="posting_date desc, posting_time desc, creation desc",
-			)
-			or 0.0
 		)
 
+		if last_sle_qty:
+			actual_qty = last_sle_qty[0][0]
+
 	ordered_qty = flt(bin_details.ordered_qty) + flt(args.get("ordered_qty"))
 	reserved_qty = flt(bin_details.reserved_qty) + flt(args.get("reserved_qty"))
 	indented_qty = flt(bin_details.indented_qty) + flt(args.get("indented_qty"))

erpnext/stock/doctype/repost_item_valuation/repost_item_valuation.json

@@ -23,6 +23,7 @@
   "error_section",
   "error_log",
   "items_to_be_repost",
+  "affected_transactions",
   "distinct_item_and_warehouse",
   "current_index"
  ],
@@ -172,12 +173,20 @@
    "no_copy": 1,
    "print_hide": 1,
    "read_only": 1
+  },
+  {
+   "fieldname": "affected_transactions",
+   "fieldtype": "Code",
+   "hidden": 1,
+   "label": "Affected Transactions",
+   "no_copy": 1,
+   "read_only": 1
   }
  ],
  "index_web_pages_for_search": 1,
  "is_submittable": 1,
  "links": [],
- "modified": "2022-03-30 07:22:48.520266",
+ "modified": "2022-04-18 14:08:08.821602",
  "modified_by": "Administrator",
  "module": "Stock",
  "name": "Repost Item Valuation",

erpnext/stock/doctype/repost_item_valuation/repost_item_valuation.py

@@ -6,11 +6,14 @@ from frappe import _
 from frappe.model.document import Document
 from frappe.utils import cint, get_link_to_form, get_weekday, now, nowtime
 from frappe.utils.user import get_users_with_role
-from rq.timeouts import JobTimeoutException
 
 import erpnext
-from erpnext.accounts.utils import update_gl_entries_after
+from erpnext.accounts.utils import get_future_stock_vouchers, repost_gle_for_stock_vouchers
-from erpnext.stock.stock_ledger import get_items_to_be_repost, repost_future_sle
+from erpnext.stock.stock_ledger import (
+	get_affected_transactions,
+	get_items_to_be_repost,
+	repost_future_sle,
+)
 
 
 class RepostItemValuation(Document):
@@ -129,12 +132,12 @@ def repost(doc):
 
 		doc.set_status("Completed")
 
-	except (Exception, JobTimeoutException):
+	except Exception:
 		frappe.db.rollback()
 		traceback = frappe.get_traceback()
 		frappe.log_error(traceback)
 
-		message = frappe.message_log.pop()
+		message = frappe.message_log.pop() if frappe.message_log else ""
 		if traceback:
 			message += "<br>" + "Traceback: <br>" + traceback
 		frappe.db.set_value(doc.doctype, doc.name, "error_log", message)
@@ -170,6 +173,7 @@ def repost_sl_entries(doc):
 			],
 			allow_negative_stock=doc.allow_negative_stock,
 			via_landed_cost_voucher=doc.via_landed_cost_voucher,
+			doc=doc,
 		)
 
 
@@ -177,27 +181,46 @@ def repost_gl_entries(doc):
 	if not cint(erpnext.is_perpetual_inventory_enabled(doc.company)):
 		return
 
+	# directly modified transactions
+	directly_dependent_transactions = _get_directly_dependent_vouchers(doc)
+	repost_affected_transaction = get_affected_transactions(doc)
+	repost_gle_for_stock_vouchers(
+		directly_dependent_transactions + list(repost_affected_transaction),
+		doc.posting_date,
+		doc.company,
+	)
+
+
+def _get_directly_dependent_vouchers(doc):
+	"""Get stock vouchers that are directly affected by reposting
+	i.e. any one item-warehouse is present in the stock transaction"""
+
+	items = set()
+	warehouses = set()
+
 	if doc.based_on == "Transaction":
 		ref_doc = frappe.get_doc(doc.voucher_type, doc.voucher_no)
 		doc_items, doc_warehouses = ref_doc.get_items_and_warehouses()
+		items.update(doc_items)
+		warehouses.update(doc_warehouses)
 
 		sles = get_items_to_be_repost(doc.voucher_type, doc.voucher_no)
-		sle_items = [sle.item_code for sle in sles]
+		sle_items = {sle.item_code for sle in sles}
-		sle_warehouse = [sle.warehouse for sle in sles]
+		sle_warehouses = {sle.warehouse for sle in sles}
-
+		items.update(sle_items)
-		items = list(set(doc_items).union(set(sle_items)))
+		warehouses.update(sle_warehouses)
-		warehouses = list(set(doc_warehouses).union(set(sle_warehouse)))
 	else:
-		items = [doc.item_code]
+		items.add(doc.item_code)
-		warehouses = [doc.warehouse]
+		warehouses.add(doc.warehouse)
 
-	update_gl_entries_after(
+	affected_vouchers = get_future_stock_vouchers(
-		doc.posting_date,
+		posting_date=doc.posting_date,
-		doc.posting_time,
+		posting_time=doc.posting_time,
-		for_warehouses=warehouses,
+		for_warehouses=list(warehouses),
-		for_items=items,
+		for_items=list(items),
 		company=doc.company,
 	)
+	return affected_vouchers
 
 
 def notify_error_to_stock_managers(doc, traceback):

erpnext/stock/doctype/repost_item_valuation/test_repost_item_valuation.py

@@ -186,3 +186,10 @@ class TestRepostItemValuation(FrappeTestCase):
 		riv.db_set("status", "Skipped")
 		riv.reload()
 		riv.cancel()  # it should cancel now
+
+	def test_queue_progress_serialization(self):
+		# Make sure set/tuple -> list behaviour is retained.
+		self.assertEqual(
+			[["a", "b"], ["c", "d"]],
+			sorted(frappe.parse_json(frappe.as_json(set([("a", "b"), ("c", "d")])))),
+		)

erpnext/stock/doctype/stock_ledger_entry/test_stock_ledger_entry.py

@@ -2,7 +2,7 @@
 # See license.txt
 
 import json
-from operator import itemgetter
+from datetime import timedelta
 from uuid import uuid4
 
 import frappe
@@ -10,6 +10,7 @@ from frappe.core.page.permission_manager.permission_manager import reset
 from frappe.custom.doctype.property_setter.property_setter import make_property_setter
 from frappe.tests.utils import FrappeTestCase
 from frappe.utils import add_days, today
+from frappe.utils.data import add_to_date
 
 from erpnext.accounts.doctype.gl_entry.gl_entry import rename_gle_sle_docs
 from erpnext.stock.doctype.delivery_note.test_delivery_note import create_delivery_note
@@ -1067,6 +1068,64 @@ class TestStockLedgerEntry(FrappeTestCase):
 		receipt2 = make_stock_entry(item_code=item, target=warehouse, qty=15, rate=15)
 		self.assertSLEs(receipt2, [{"stock_queue": [[5, 15]], "stock_value_difference": 175}])
 
+	def test_dependent_gl_entry_reposting(self):
+		def _get_stock_credit(doc):
+			return frappe.db.get_value(
+				"GL Entry",
+				{
+					"voucher_no": doc.name,
+					"voucher_type": doc.doctype,
+					"is_cancelled": 0,
+					"account": "Stock In Hand - TCP1",
+				},
+				"sum(credit)",
+			)
+
+		def _day(days):
+			return add_to_date(date=today(), days=days)
+
+		item = make_item().name
+		A = "Stores - TCP1"
+		B = "Work In Progress - TCP1"
+		C = "Finished Goods - TCP1"
+
+		make_stock_entry(item_code=item, to_warehouse=A, qty=5, rate=10, posting_date=_day(0))
+		make_stock_entry(item_code=item, from_warehouse=A, to_warehouse=B, qty=5, posting_date=_day(1))
+		depdendent_consumption = make_stock_entry(
+			item_code=item, from_warehouse=B, qty=5, posting_date=_day(2)
+		)
+		self.assertEqual(50, _get_stock_credit(depdendent_consumption))
+
+		# backdated receipt - should trigger GL repost of all previous stock entries
+		bd_receipt = make_stock_entry(
+			item_code=item, to_warehouse=A, qty=5, rate=20, posting_date=_day(-1)
+		)
+		self.assertEqual(100, _get_stock_credit(depdendent_consumption))
+
+		# cancelling receipt should reset it back
+		bd_receipt.cancel()
+		self.assertEqual(50, _get_stock_credit(depdendent_consumption))
+
+		bd_receipt2 = make_stock_entry(
+			item_code=item, to_warehouse=A, qty=2, rate=20, posting_date=_day(-2)
+		)
+		# total as per FIFO -> 2 * 20 + 3 * 10 = 70
+		self.assertEqual(70, _get_stock_credit(depdendent_consumption))
+
+		# transfer WIP material to final destination and consume it all
+		depdendent_consumption.cancel()
+		make_stock_entry(item_code=item, from_warehouse=B, to_warehouse=C, qty=5, posting_date=_day(3))
+		final_consumption = make_stock_entry(
+			item_code=item, from_warehouse=C, qty=5, posting_date=_day(4)
+		)
+		# exact amount gets consumed
+		self.assertEqual(70, _get_stock_credit(final_consumption))
+
+		# cancel original backdated receipt - should repost A -> B -> C
+		bd_receipt2.cancel()
+		# original amount
+		self.assertEqual(50, _get_stock_credit(final_consumption))
+
 
 def create_repack_entry(**args):
 	args = frappe._dict(args)

erpnext/stock/doctype/stock_reconciliation/test_stock_reconciliation.py

@@ -10,7 +10,7 @@ from frappe.tests.utils import FrappeTestCase, change_settings
 from frappe.utils import add_days, cstr, flt, nowdate, nowtime, random_string
 
 from erpnext.accounts.utils import get_stock_and_account_balance
-from erpnext.stock.doctype.item.test_item import create_item
+from erpnext.stock.doctype.item.test_item import create_item, make_item
 from erpnext.stock.doctype.purchase_receipt.test_purchase_receipt import make_purchase_receipt
 from erpnext.stock.doctype.serial_no.serial_no import get_serial_nos
 from erpnext.stock.doctype.stock_reconciliation.stock_reconciliation import (
@@ -31,6 +31,7 @@ class TestStockReconciliation(FrappeTestCase):
 
 	def tearDown(self):
 		frappe.flags.dont_execute_stock_reposts = None
+		frappe.local.future_sle = {}
 
 	def test_reco_for_fifo(self):
 		self._test_reco_sle_gle("FIFO")
@@ -311,9 +312,8 @@ class TestStockReconciliation(FrappeTestCase):
 		SR4		| Reco	|	0	|	6	(posting date: today-1) [backdated]
 		PR3		| PR	|	1	|	7	(posting date: today) # can't post future PR
 		"""
-		item_code = "Backdated-Reco-Item"
+		item_code = make_item().name
 		warehouse = "_Test Warehouse - _TC"
-		create_item(item_code)
 
 		pr1 = make_purchase_receipt(
 			item_code=item_code, warehouse=warehouse, qty=10, rate=100, posting_date=add_days(nowdate(), -3)
@@ -395,9 +395,8 @@ class TestStockReconciliation(FrappeTestCase):
 		from erpnext.stock.doctype.delivery_note.test_delivery_note import create_delivery_note
 		from erpnext.stock.stock_ledger import NegativeStockError
 
-		item_code = "Backdated-Reco-Item"
+		item_code = make_item().name
 		warehouse = "_Test Warehouse - _TC"
-		create_item(item_code)
 
 		pr1 = make_purchase_receipt(
 			item_code=item_code, warehouse=warehouse, qty=10, rate=100, posting_date=add_days(nowdate(), -2)
@@ -444,9 +443,8 @@ class TestStockReconciliation(FrappeTestCase):
 		from erpnext.stock.doctype.delivery_note.test_delivery_note import create_delivery_note
 		from erpnext.stock.stock_ledger import NegativeStockError
 
-		item_code = "Backdated-Reco-Cancellation-Item"
+		item_code = make_item().name
 		warehouse = "_Test Warehouse - _TC"
-		create_item(item_code)
 
 		sr = create_stock_reconciliation(
 			item_code=item_code,
@@ -487,9 +485,8 @@ class TestStockReconciliation(FrappeTestCase):
 		frappe.flags.dont_execute_stock_reposts = True
 		frappe.db.rollback()
 
-		item_code = "Backdated-Reco-Cancellation-Item"
+		item_code = make_item().name
 		warehouse = "_Test Warehouse - _TC"
-		create_item(item_code)
 
 		sr = create_stock_reconciliation(
 			item_code=item_code, warehouse=warehouse, qty=10, rate=100, posting_date=add_days(nowdate(), 10)

erpnext/stock/stock_ledger.py

@@ -3,7 +3,7 @@
 
 import copy
 import json
-from typing import Optional
+from typing import Optional, Set, Tuple
 
 import frappe
 from frappe import _
@@ -214,6 +214,7 @@ def repost_future_sle(
 		args = get_items_to_be_repost(voucher_type, voucher_no, doc)
 
 	distinct_item_warehouses = get_distinct_item_warehouse(args, doc)
+	affected_transactions = get_affected_transactions(doc)
 
 	i = get_current_index(doc) or 0
 	while i < len(args):
@@ -231,6 +232,7 @@ def repost_future_sle(
 			allow_negative_stock=allow_negative_stock,
 			via_landed_cost_voucher=via_landed_cost_voucher,
 		)
+		affected_transactions.update(obj.affected_transactions)
 
 		distinct_item_warehouses[
 			(args[i].get("item_code"), args[i].get("warehouse"))
@@ -250,10 +252,14 @@ def repost_future_sle(
 		i += 1
 
 		if doc and i % 2 == 0:
-			update_args_in_repost_item_valuation(doc, i, args, distinct_item_warehouses)
+			update_args_in_repost_item_valuation(
+				doc, i, args, distinct_item_warehouses, affected_transactions
+			)
 
 	if doc and args:
-		update_args_in_repost_item_valuation(doc, i, args, distinct_item_warehouses)
+		update_args_in_repost_item_valuation(
+			doc, i, args, distinct_item_warehouses, affected_transactions
+		)
 
 
 def validate_item_warehouse(args):
@@ -263,20 +269,22 @@ def validate_item_warehouse(args):
 			frappe.throw(_(validation_msg))
 
 
-def update_args_in_repost_item_valuation(doc, index, args, distinct_item_warehouses):
+def update_args_in_repost_item_valuation(
-	frappe.db.set_value(
+	doc, index, args, distinct_item_warehouses, affected_transactions
-		doc.doctype,
+):
-		doc.name,
+	doc.db_set(
 		{
 			"items_to_be_repost": json.dumps(args, default=str),
 			"distinct_item_and_warehouse": json.dumps(
 				{str(k): v for k, v in distinct_item_warehouses.items()}, default=str
 			),
 			"current_index": index,
-		},
+			"affected_transactions": frappe.as_json(affected_transactions),
+		}
 	)
 
-	frappe.db.commit()
+	if not frappe.flags.in_test:
+		frappe.db.commit()
 
 	frappe.publish_realtime(
 		"item_reposting_progress",
@@ -313,6 +321,14 @@ def get_distinct_item_warehouse(args=None, doc=None):
 	return distinct_item_warehouses
 
 
+def get_affected_transactions(doc) -> Set[Tuple[str, str]]:
+	if not doc.affected_transactions:
+		return set()
+
+	transactions = frappe.parse_json(doc.affected_transactions)
+	return {tuple(transaction) for transaction in transactions}
+
+
 def get_current_index(doc=None):
 	if doc and doc.current_index:
 		return doc.current_index
@@ -360,6 +376,7 @@ class update_entries_after(object):
 
 		self.new_items_found = False
 		self.distinct_item_warehouses = args.get("distinct_item_warehouses", frappe._dict())
+		self.affected_transactions: Set[Tuple[str, str]] = set()
 
 		self.data = frappe._dict()
 		self.initialize_previous_data(self.args)
@@ -518,6 +535,7 @@ class update_entries_after(object):
 
 		# previous sle data for this warehouse
 		self.wh_data = self.data[sle.warehouse]
+		self.affected_transactions.add((sle.voucher_type, sle.voucher_no))
 
 		if (sle.serial_no and not self.via_landed_cost_voucher) or not cint(self.allow_negative_stock):
 			# validate negative stock for serialized items, fifo valuation