From d80ca523a43e9d1073dc008ead7d140fb0811a89 Mon Sep 17 00:00:00 2001 From: Rohit Waghchaure Date: Wed, 7 Feb 2024 21:56:21 +0530 Subject: [PATCH 1/3] perf: new column posting datetime in SLE to optimize stock ledger related queries --- .../report/gross_profit/gross_profit.py | 2 +- erpnext/accounts/utils.py | 43 +------ erpnext/manufacturing/doctype/bom/bom.py | 3 +- .../work_order_summary/work_order_summary.py | 2 +- erpnext/patches.txt | 1 + ...te_posting_datetime_and_dropped_indexes.py | 19 +++ .../purchase_receipt/test_purchase_receipt.py | 89 +++++++++++++ .../test_serial_and_batch_bundle.py | 3 +- .../stock/doctype/stock_entry/stock_entry.py | 8 +- .../stock_ledger_entry.json | 10 +- .../stock_ledger_entry/stock_ledger_entry.py | 12 +- .../test_stock_ledger_entry.py | 2 +- .../stock_reservation_entry.py | 4 +- .../incorrect_stock_value_report.py | 4 +- .../product_bundle_balance.py | 4 +- .../report/stock_balance/stock_balance.py | 4 +- .../stock/report/stock_ledger/stock_ledger.py | 2 +- erpnext/stock/stock_ledger.py | 117 ++++++++---------- erpnext/stock/utils.py | 17 ++- 19 files changed, 216 insertions(+), 130 deletions(-) create mode 100644 erpnext/patches/v14_0/update_posting_datetime_and_dropped_indexes.py diff --git a/erpnext/accounts/report/gross_profit/gross_profit.py b/erpnext/accounts/report/gross_profit/gross_profit.py index e4efefe7f5..7162aef8f2 100644 --- a/erpnext/accounts/report/gross_profit/gross_profit.py +++ b/erpnext/accounts/report/gross_profit/gross_profit.py @@ -975,7 +975,7 @@ class GrossProfitGenerator(object): & (sle.is_cancelled == 0) ) .orderby(sle.item_code) - .orderby(sle.warehouse, sle.posting_date, sle.posting_time, sle.creation, order=Order.desc) + .orderby(sle.warehouse, sle.posting_datetime, sle.creation, order=Order.desc) .run(as_dict=True) ) diff --git a/erpnext/accounts/utils.py b/erpnext/accounts/utils.py index 64bc39a77b..157cfdd7f7 100644 --- a/erpnext/accounts/utils.py +++ b/erpnext/accounts/utils.py @@ -982,46 +982,6 @@ def get_currency_precision(): return precision -def get_stock_rbnb_difference(posting_date, company): - stock_items = frappe.db.sql_list( - """select distinct item_code - from `tabStock Ledger Entry` where company=%s""", - company, - ) - - pr_valuation_amount = frappe.db.sql( - """ - select sum(pr_item.valuation_rate * pr_item.qty * pr_item.conversion_factor) - from `tabPurchase Receipt Item` pr_item, `tabPurchase Receipt` pr - where pr.name = pr_item.parent and pr.docstatus=1 and pr.company=%s - and pr.posting_date <= %s and pr_item.item_code in (%s)""" - % ("%s", "%s", ", ".join(["%s"] * len(stock_items))), - tuple([company, posting_date] + stock_items), - )[0][0] - - pi_valuation_amount = frappe.db.sql( - """ - select sum(pi_item.valuation_rate * pi_item.qty * pi_item.conversion_factor) - from `tabPurchase Invoice Item` pi_item, `tabPurchase Invoice` pi - where pi.name = pi_item.parent and pi.docstatus=1 and pi.company=%s - and pi.posting_date <= %s and pi_item.item_code in (%s)""" - % ("%s", "%s", ", ".join(["%s"] * len(stock_items))), - tuple([company, posting_date] + stock_items), - )[0][0] - - # Balance should be - stock_rbnb = flt(pr_valuation_amount, 2) - flt(pi_valuation_amount, 2) - - # Balance as per system - stock_rbnb_account = "Stock Received But Not Billed - " + frappe.get_cached_value( - "Company", company, "abbr" - ) - sys_bal = get_balance_on(stock_rbnb_account, posting_date, in_account_currency=False) - - # Amount should be credited - return flt(stock_rbnb) + flt(sys_bal) - - def get_held_invoices(party_type, party): """ Returns a list of names Purchase Invoices for the given party that are on hold @@ -1428,8 +1388,7 @@ def sort_stock_vouchers_by_posting_date( .select(sle.voucher_type, sle.voucher_no, sle.posting_date, sle.posting_time, sle.creation) .where((sle.is_cancelled == 0) & (sle.voucher_no.isin(voucher_nos))) .groupby(sle.voucher_type, sle.voucher_no) - .orderby(sle.posting_date) - .orderby(sle.posting_time) + .orderby(sle.posting_datetime) .orderby(sle.creation) ).run(as_dict=True) sorted_vouchers = [(sle.voucher_type, sle.voucher_no) for sle in sles] diff --git a/erpnext/manufacturing/doctype/bom/bom.py b/erpnext/manufacturing/doctype/bom/bom.py index 6f3520618b..27c8493ab5 100644 --- a/erpnext/manufacturing/doctype/bom/bom.py +++ b/erpnext/manufacturing/doctype/bom/bom.py @@ -1071,8 +1071,7 @@ def get_valuation_rate(data): frappe.qb.from_(sle) .select(sle.valuation_rate) .where((sle.item_code == item_code) & (sle.valuation_rate > 0) & (sle.is_cancelled == 0)) - .orderby(sle.posting_date, order=frappe.qb.desc) - .orderby(sle.posting_time, order=frappe.qb.desc) + .orderby(sle.posting_datetime, order=frappe.qb.desc) .orderby(sle.creation, order=frappe.qb.desc) .limit(1) ).run(as_dict=True) diff --git a/erpnext/manufacturing/report/work_order_summary/work_order_summary.py b/erpnext/manufacturing/report/work_order_summary/work_order_summary.py index 97f30ef62e..8d3770805e 100644 --- a/erpnext/manufacturing/report/work_order_summary/work_order_summary.py +++ b/erpnext/manufacturing/report/work_order_summary/work_order_summary.py @@ -58,7 +58,7 @@ def get_data(filters): query_filters["creation"] = ("between", [filters.get("from_date"), filters.get("to_date")]) data = frappe.get_all( - "Work Order", fields=fields, filters=query_filters, order_by="planned_start_date asc", debug=1 + "Work Order", fields=fields, filters=query_filters, order_by="planned_start_date asc" ) res = [] diff --git a/erpnext/patches.txt b/erpnext/patches.txt index 4ead7e71f0..3ff4151101 100644 --- a/erpnext/patches.txt +++ b/erpnext/patches.txt @@ -357,3 +357,4 @@ erpnext.patches.v15_0.create_advance_payment_status erpnext.patches.v14_0.migrate_gl_to_payment_ledger erpnext.stock.doctype.delivery_note.patches.drop_unused_return_against_index # 2023-12-20 erpnext.patches.v14_0.set_maintain_stock_for_bom_item +erpnext.patches.v14_0.update_posting_datetime_and_dropped_indexes \ No newline at end of file diff --git a/erpnext/patches/v14_0/update_posting_datetime_and_dropped_indexes.py b/erpnext/patches/v14_0/update_posting_datetime_and_dropped_indexes.py new file mode 100644 index 0000000000..6ec3f84200 --- /dev/null +++ b/erpnext/patches/v14_0/update_posting_datetime_and_dropped_indexes.py @@ -0,0 +1,19 @@ +import frappe + + +def execute(): + frappe.db.sql( + """ + UPDATE `tabStock Ledger Entry` + SET posting_datetime = timestamp(posting_date, posting_time) + """ + ) + + drop_indexes() + + +def drop_indexes(): + if not frappe.db.has_index("tabStock Ledger Entry", "posting_sort_index"): + return + + frappe.db.sql_ddl("ALTER TABLE `tabStock Ledger Entry` DROP INDEX `posting_sort_index`") diff --git a/erpnext/stock/doctype/purchase_receipt/test_purchase_receipt.py b/erpnext/stock/doctype/purchase_receipt/test_purchase_receipt.py index 2d209220de..18e24f4326 100644 --- a/erpnext/stock/doctype/purchase_receipt/test_purchase_receipt.py +++ b/erpnext/stock/doctype/purchase_receipt/test_purchase_receipt.py @@ -2317,6 +2317,95 @@ class TestPurchaseReceipt(FrappeTestCase): serial_no_status = frappe.db.get_value("Serial No", sn, "status") self.assertTrue(serial_no_status != "Active") + def test_sle_qty_after_transaction(self): + item = make_item( + "_Test Item Qty After Transaction", + properties={"is_stock_item": 1, "valuation_method": "FIFO"}, + ).name + + posting_date = today() + posting_time = nowtime() + + # Step 1: Create Purchase Receipt + pr = make_purchase_receipt( + item_code=item, + qty=1, + rate=100, + posting_date=posting_date, + posting_time=posting_time, + do_not_save=1, + ) + + for i in range(9): + pr.append( + "items", + { + "item_code": item, + "qty": 1, + "rate": 100, + "warehouse": pr.items[0].warehouse, + "cost_center": pr.items[0].cost_center, + "expense_account": pr.items[0].expense_account, + "uom": pr.items[0].uom, + "stock_uom": pr.items[0].stock_uom, + "conversion_factor": pr.items[0].conversion_factor, + }, + ) + + self.assertEqual(len(pr.items), 10) + pr.save() + pr.submit() + + data = frappe.get_all( + "Stock Ledger Entry", + fields=["qty_after_transaction", "creation", "posting_datetime"], + filters={"voucher_no": pr.name, "is_cancelled": 0}, + order_by="creation", + ) + + for index, d in enumerate(data): + self.assertEqual(d.qty_after_transaction, 1 + index) + + # Step 2: Create Purchase Receipt + pr = make_purchase_receipt( + item_code=item, + qty=1, + rate=100, + posting_date=posting_date, + posting_time=posting_time, + do_not_save=1, + ) + + for i in range(9): + pr.append( + "items", + { + "item_code": item, + "qty": 1, + "rate": 100, + "warehouse": pr.items[0].warehouse, + "cost_center": pr.items[0].cost_center, + "expense_account": pr.items[0].expense_account, + "uom": pr.items[0].uom, + "stock_uom": pr.items[0].stock_uom, + "conversion_factor": pr.items[0].conversion_factor, + }, + ) + + self.assertEqual(len(pr.items), 10) + pr.save() + pr.submit() + + data = frappe.get_all( + "Stock Ledger Entry", + fields=["qty_after_transaction", "creation", "posting_datetime"], + filters={"voucher_no": pr.name, "is_cancelled": 0}, + order_by="creation", + ) + + for index, d in enumerate(data): + self.assertEqual(d.qty_after_transaction, 11 + index) + def prepare_data_for_internal_transfer(): from erpnext.accounts.doctype.sales_invoice.test_sales_invoice import create_internal_supplier diff --git a/erpnext/stock/doctype/serial_and_batch_bundle/test_serial_and_batch_bundle.py b/erpnext/stock/doctype/serial_and_batch_bundle/test_serial_and_batch_bundle.py index 88b262a8c6..b932c1371d 100644 --- a/erpnext/stock/doctype/serial_and_batch_bundle/test_serial_and_batch_bundle.py +++ b/erpnext/stock/doctype/serial_and_batch_bundle/test_serial_and_batch_bundle.py @@ -5,7 +5,7 @@ import json import frappe from frappe.tests.utils import FrappeTestCase, change_settings -from frappe.utils import add_days, add_to_date, flt, nowdate, nowtime, today +from frappe.utils import flt, nowtime, today from erpnext.stock.doctype.item.test_item import make_item from erpnext.stock.doctype.serial_and_batch_bundle.serial_and_batch_bundle import ( @@ -191,6 +191,7 @@ class TestSerialandBatchBundle(FrappeTestCase): doc.flags.ignore_links = True doc.flags.ignore_validate = True doc.submit() + doc.reload() bundle_doc = make_serial_batch_bundle( { diff --git a/erpnext/stock/doctype/stock_entry/stock_entry.py b/erpnext/stock/doctype/stock_entry/stock_entry.py index 276b2f4f3b..fb417458f7 100644 --- a/erpnext/stock/doctype/stock_entry/stock_entry.py +++ b/erpnext/stock/doctype/stock_entry/stock_entry.py @@ -1899,6 +1899,7 @@ class StockEntry(StockController): return id = create_serial_and_batch_bundle( + self, row, frappe._dict( { @@ -2169,7 +2170,7 @@ class StockEntry(StockController): "to_warehouse": "", "qty": qty, "item_name": item.item_name, - "serial_and_batch_bundle": create_serial_and_batch_bundle(row, item, "Outward"), + "serial_and_batch_bundle": create_serial_and_batch_bundle(self, row, item, "Outward"), "description": item.description, "stock_uom": item.stock_uom, "expense_account": item.expense_account, @@ -2547,6 +2548,7 @@ class StockEntry(StockController): row = frappe._dict({"serial_nos": serial_nos[0 : cint(d.qty)]}) id = create_serial_and_batch_bundle( + self, row, frappe._dict( { @@ -3070,7 +3072,7 @@ def get_stock_entry_data(work_order): return data -def create_serial_and_batch_bundle(row, child, type_of_transaction=None): +def create_serial_and_batch_bundle(parent_doc, row, child, type_of_transaction=None): item_details = frappe.get_cached_value( "Item", child.item_code, ["has_serial_no", "has_batch_no"], as_dict=1 ) @@ -3088,6 +3090,8 @@ def create_serial_and_batch_bundle(row, child, type_of_transaction=None): "item_code": child.item_code, "warehouse": child.warehouse, "type_of_transaction": type_of_transaction, + "posting_date": parent_doc.posting_date, + "posting_time": parent_doc.posting_time, } ) diff --git a/erpnext/stock/doctype/stock_ledger_entry/stock_ledger_entry.json b/erpnext/stock/doctype/stock_ledger_entry/stock_ledger_entry.json index be379940ca..3a094f1e8f 100644 --- a/erpnext/stock/doctype/stock_ledger_entry/stock_ledger_entry.json +++ b/erpnext/stock/doctype/stock_ledger_entry/stock_ledger_entry.json @@ -11,6 +11,7 @@ "warehouse", "posting_date", "posting_time", + "posting_datetime", "is_adjustment_entry", "auto_created_serial_and_batch_bundle", "column_break_6", @@ -100,7 +101,6 @@ "oldfieldtype": "Date", "print_width": "100px", "read_only": 1, - "search_index": 1, "width": "100px" }, { @@ -253,7 +253,6 @@ "options": "Company", "print_width": "150px", "read_only": 1, - "search_index": 1, "width": "150px" }, { @@ -348,6 +347,11 @@ "fieldname": "auto_created_serial_and_batch_bundle", "fieldtype": "Check", "label": "Auto Created Serial and Batch Bundle" + }, + { + "fieldname": "posting_datetime", + "fieldtype": "Datetime", + "label": "Posting Datetime" } ], "hide_toolbar": 1, @@ -356,7 +360,7 @@ "in_create": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2023-11-14 16:47:39.791967", + "modified": "2024-02-07 09:18:13.999231", "modified_by": "Administrator", "module": "Stock", "name": "Stock Ledger Entry", diff --git a/erpnext/stock/doctype/stock_ledger_entry/stock_ledger_entry.py b/erpnext/stock/doctype/stock_ledger_entry/stock_ledger_entry.py index 04441f0e8b..a3e51ca60d 100644 --- a/erpnext/stock/doctype/stock_ledger_entry/stock_ledger_entry.py +++ b/erpnext/stock/doctype/stock_ledger_entry/stock_ledger_entry.py @@ -51,6 +51,7 @@ class StockLedgerEntry(Document): item_code: DF.Link | None outgoing_rate: DF.Currency posting_date: DF.Date | None + posting_datetime: DF.Datetime | None posting_time: DF.Time | None project: DF.Link | None qty_after_transaction: DF.Float @@ -92,6 +93,12 @@ class StockLedgerEntry(Document): self.validate_with_last_transaction_posting_time() self.validate_inventory_dimension_negative_stock() + def set_posting_datetime(self): + from erpnext.stock.utils import get_combine_datetime + + self.posting_datetime = get_combine_datetime(self.posting_date, self.posting_time) + self.db_set("posting_datetime", self.posting_datetime) + def validate_inventory_dimension_negative_stock(self): if self.is_cancelled: return @@ -162,6 +169,7 @@ class StockLedgerEntry(Document): return inv_dimension_dict def on_submit(self): + self.set_posting_datetime() self.check_stock_frozen_date() # Added to handle few test cases where serial_and_batch_bundles are not required @@ -332,9 +340,7 @@ class StockLedgerEntry(Document): def on_doctype_update(): - frappe.db.add_index( - "Stock Ledger Entry", fields=["posting_date", "posting_time"], index_name="posting_sort_index" - ) frappe.db.add_index("Stock Ledger Entry", ["voucher_no", "voucher_type"]) frappe.db.add_index("Stock Ledger Entry", ["batch_no", "item_code", "warehouse"]) frappe.db.add_index("Stock Ledger Entry", ["warehouse", "item_code"], "item_warehouse") + frappe.db.add_index("Stock Ledger Entry", ["posting_datetime", "creation"]) diff --git a/erpnext/stock/doctype/stock_ledger_entry/test_stock_ledger_entry.py b/erpnext/stock/doctype/stock_ledger_entry/test_stock_ledger_entry.py index c0999532d0..8d82c36741 100644 --- a/erpnext/stock/doctype/stock_ledger_entry/test_stock_ledger_entry.py +++ b/erpnext/stock/doctype/stock_ledger_entry/test_stock_ledger_entry.py @@ -1077,7 +1077,7 @@ class TestStockLedgerEntry(FrappeTestCase, StockTestMixin): frappe.qb.from_(sle) .select("qty_after_transaction") .where((sle.item_code == item) & (sle.warehouse == warehouse) & (sle.is_cancelled == 0)) - .orderby(CombineDatetime(sle.posting_date, sle.posting_time)) + .orderby(sle.posting_datetime) .orderby(sle.creation) ).run(pluck=True) diff --git a/erpnext/stock/doctype/stock_reservation_entry/stock_reservation_entry.py b/erpnext/stock/doctype/stock_reservation_entry/stock_reservation_entry.py index 7e03ac3357..26fe8e1787 100644 --- a/erpnext/stock/doctype/stock_reservation_entry/stock_reservation_entry.py +++ b/erpnext/stock/doctype/stock_reservation_entry/stock_reservation_entry.py @@ -7,7 +7,7 @@ import frappe from frappe import _ from frappe.model.document import Document from frappe.query_builder.functions import Sum -from frappe.utils import cint, flt +from frappe.utils import cint, flt, nowdate, nowtime from erpnext.stock.utils import get_or_make_bin, get_stock_balance @@ -866,6 +866,8 @@ def get_ssb_bundle_for_voucher(sre: dict) -> object: bundle = frappe.new_doc("Serial and Batch Bundle") bundle.type_of_transaction = "Outward" bundle.voucher_type = "Delivery Note" + bundle.posting_date = nowdate() + bundle.posting_time = nowtime() for field in ("item_code", "warehouse", "has_serial_no", "has_batch_no"): setattr(bundle, field, sre[field]) diff --git a/erpnext/stock/report/incorrect_stock_value_report/incorrect_stock_value_report.py b/erpnext/stock/report/incorrect_stock_value_report/incorrect_stock_value_report.py index e4f657ca70..da958a8b0f 100644 --- a/erpnext/stock/report/incorrect_stock_value_report/incorrect_stock_value_report.py +++ b/erpnext/stock/report/incorrect_stock_value_report/incorrect_stock_value_report.py @@ -5,7 +5,7 @@ import frappe from frappe import _ from frappe.query_builder import Field -from frappe.query_builder.functions import CombineDatetime, Min +from frappe.query_builder.functions import Min from frappe.utils import add_days, getdate, today import erpnext @@ -75,7 +75,7 @@ def get_data(report_filters): & (sle.company == report_filters.company) & (sle.is_cancelled == 0) ) - .orderby(CombineDatetime(sle.posting_date, sle.posting_time), sle.creation) + .orderby(sle.posting_datetime, sle.creation) ).run(as_dict=True) for d in data: diff --git a/erpnext/stock/report/product_bundle_balance/product_bundle_balance.py b/erpnext/stock/report/product_bundle_balance/product_bundle_balance.py index 9e75201bd1..dd79e7fcaf 100644 --- a/erpnext/stock/report/product_bundle_balance/product_bundle_balance.py +++ b/erpnext/stock/report/product_bundle_balance/product_bundle_balance.py @@ -213,13 +213,11 @@ def get_stock_ledger_entries(filters, items): query = ( frappe.qb.from_(sle) - .force_index("posting_sort_index") .left_join(sle2) .on( (sle.item_code == sle2.item_code) & (sle.warehouse == sle2.warehouse) - & (sle.posting_date < sle2.posting_date) - & (sle.posting_time < sle2.posting_time) + & (sle.posting_datetime < sle2.posting_datetime) & (sle.name < sle2.name) ) .select(sle.item_code, sle.warehouse, sle.qty_after_transaction, sle.company) diff --git a/erpnext/stock/report/stock_balance/stock_balance.py b/erpnext/stock/report/stock_balance/stock_balance.py index 269323810b..500affa51e 100644 --- a/erpnext/stock/report/stock_balance/stock_balance.py +++ b/erpnext/stock/report/stock_balance/stock_balance.py @@ -8,7 +8,7 @@ from typing import Any, Dict, List, Optional, TypedDict import frappe from frappe import _ from frappe.query_builder import Order -from frappe.query_builder.functions import Coalesce, CombineDatetime +from frappe.query_builder.functions import Coalesce from frappe.utils import add_days, cint, date_diff, flt, getdate from frappe.utils.nestedset import get_descendants_of @@ -300,7 +300,7 @@ class StockBalanceReport(object): item_table.item_name, ) .where((sle.docstatus < 2) & (sle.is_cancelled == 0)) - .orderby(CombineDatetime(sle.posting_date, sle.posting_time)) + .orderby(sle.posting_datetime) .orderby(sle.creation) .orderby(sle.actual_qty) ) diff --git a/erpnext/stock/report/stock_ledger/stock_ledger.py b/erpnext/stock/report/stock_ledger/stock_ledger.py index e59f2fe644..13ebfe4a7d 100644 --- a/erpnext/stock/report/stock_ledger/stock_ledger.py +++ b/erpnext/stock/report/stock_ledger/stock_ledger.py @@ -283,7 +283,7 @@ def get_stock_ledger_entries(filters, items): frappe.qb.from_(sle) .select( sle.item_code, - CombineDatetime(sle.posting_date, sle.posting_time).as_("date"), + sle.posting_datetime.as_("date"), sle.warehouse, sle.posting_date, sle.posting_time, diff --git a/erpnext/stock/stock_ledger.py b/erpnext/stock/stock_ledger.py index e88b1921fa..75fdf3b3d9 100644 --- a/erpnext/stock/stock_ledger.py +++ b/erpnext/stock/stock_ledger.py @@ -9,8 +9,9 @@ from typing import Optional, Set, Tuple import frappe from frappe import _, scrub from frappe.model.meta import get_field_precision -from frappe.query_builder.functions import CombineDatetime, Sum +from frappe.query_builder.functions import Sum from frappe.utils import ( + add_to_date, cint, cstr, flt, @@ -33,6 +34,7 @@ from erpnext.stock.doctype.stock_reservation_entry.stock_reservation_entry impor get_sre_reserved_serial_nos_details, ) from erpnext.stock.utils import ( + get_combine_datetime, get_incoming_outgoing_rate_for_cancel, get_incoming_rate, get_or_make_bin, @@ -95,6 +97,7 @@ def make_sl_entries(sl_entries, allow_negative_stock=False, via_landed_cost_vouc sle_doc = make_entry(sle, allow_negative_stock, via_landed_cost_voucher) args = sle_doc.as_dict() + args["posting_datetime"] = get_combine_datetime(args.posting_date, args.posting_time) if sle.get("voucher_type") == "Stock Reconciliation": # preserve previous_qty_after_transaction for qty reposting @@ -616,12 +619,14 @@ class update_entries_after(object): self.process_sle(sle) def get_sle_against_current_voucher(self): - self.args["time_format"] = "%H:%i:%s" + self.args["posting_datetime"] = get_combine_datetime( + self.args.posting_date, self.args.posting_time + ) return frappe.db.sql( """ select - *, timestamp(posting_date, posting_time) as "timestamp" + *, posting_datetime as "timestamp" from `tabStock Ledger Entry` where @@ -629,11 +634,10 @@ class update_entries_after(object): and warehouse = %(warehouse)s and is_cancelled = 0 and ( - posting_date = %(posting_date)s and - time_format(posting_time, %(time_format)s) = time_format(%(posting_time)s, %(time_format)s) + posting_datetime = %(posting_datetime)s ) order by - creation ASC + posting_datetime ASC, creation ASC for update """, self.args, @@ -1401,9 +1405,14 @@ def get_previous_sle_of_current_voucher(args, operator="<", exclude_current_vouc args["time_format"] = "%H:%i:%s" if not args.get("posting_date"): - args["posting_date"] = "1900-01-01" - if not args.get("posting_time"): - args["posting_time"] = "00:00" + args["posting_datetime"] = "1900-01-01 00:00:00" + + if not args.get("posting_datetime"): + args["posting_datetime"] = get_combine_datetime(args["posting_date"], args["posting_time"]) + + if operator == "<=": + # Add 1 second to handle millisecond for less than and equal to condition + args["posting_datetime"] = add_to_date(args["posting_datetime"], seconds=1) voucher_condition = "" if exclude_current_voucher: @@ -1412,23 +1421,20 @@ def get_previous_sle_of_current_voucher(args, operator="<", exclude_current_vouc sle = frappe.db.sql( """ - select *, timestamp(posting_date, posting_time) as "timestamp" + select *, posting_datetime as "timestamp" from `tabStock Ledger Entry` where item_code = %(item_code)s and warehouse = %(warehouse)s and is_cancelled = 0 {voucher_condition} and ( - posting_date < %(posting_date)s or - ( - posting_date = %(posting_date)s and - time_format(posting_time, %(time_format)s) {operator} time_format(%(posting_time)s, %(time_format)s) - ) + posting_datetime {operator} %(posting_datetime)s ) - order by timestamp(posting_date, posting_time) desc, creation desc + order by posting_datetime desc, creation desc limit 1 for update""".format( - operator=operator, voucher_condition=voucher_condition + operator=operator, + voucher_condition=voucher_condition, ), args, as_dict=1, @@ -1469,9 +1475,7 @@ def get_stock_ledger_entries( extra_cond=None, ): """get stock ledger entries filtered by specific posting datetime conditions""" - conditions = " and timestamp(posting_date, posting_time) {0} timestamp(%(posting_date)s, %(posting_time)s)".format( - operator - ) + conditions = " and posting_datetime {0} %(posting_datetime)s".format(operator) if previous_sle.get("warehouse"): conditions += " and warehouse = %(warehouse)s" elif previous_sle.get("warehouse_condition"): @@ -1497,9 +1501,11 @@ def get_stock_ledger_entries( ) if not previous_sle.get("posting_date"): - previous_sle["posting_date"] = "1900-01-01" - if not previous_sle.get("posting_time"): - previous_sle["posting_time"] = "00:00" + previous_sle["posting_datetime"] = "1900-01-01 00:00:00" + else: + previous_sle["posting_datetime"] = get_combine_datetime( + previous_sle["posting_date"], previous_sle["posting_time"] + ) if operator in (">", "<=") and previous_sle.get("name"): conditions += " and name!=%(name)s" @@ -1509,12 +1515,12 @@ def get_stock_ledger_entries( return frappe.db.sql( """ - select *, timestamp(posting_date, posting_time) as "timestamp" + select *, posting_datetime as "timestamp" from `tabStock Ledger Entry` where item_code = %%(item_code)s and is_cancelled = 0 %(conditions)s - order by timestamp(posting_date, posting_time) %(order)s, creation %(order)s + order by posting_datetime %(order)s, creation %(order)s %(limit)s %(for_update)s""" % { "conditions": conditions, @@ -1540,7 +1546,7 @@ def get_sle_by_voucher_detail_no(voucher_detail_no, excluded_sle=None): "posting_date", "posting_time", "voucher_detail_no", - "timestamp(posting_date, posting_time) as timestamp", + "posting_datetime as timestamp", ], as_dict=1, ) @@ -1552,13 +1558,10 @@ def get_batch_incoming_rate( sle = frappe.qb.DocType("Stock Ledger Entry") - timestamp_condition = CombineDatetime(sle.posting_date, sle.posting_time) < CombineDatetime( - posting_date, posting_time - ) + timestamp_condition = sle.posting_datetime < get_combine_datetime(posting_date, posting_time) if creation: timestamp_condition |= ( - CombineDatetime(sle.posting_date, sle.posting_time) - == CombineDatetime(posting_date, posting_time) + sle.posting_datetime == get_combine_datetime(posting_date, posting_time) ) & (sle.creation < creation) batch_details = ( @@ -1639,7 +1642,7 @@ def get_valuation_rate( AND valuation_rate >= 0 AND is_cancelled = 0 AND NOT (voucher_no = %s AND voucher_type = %s) - order by posting_date desc, posting_time desc, name desc limit 1""", + order by posting_datetime desc, name desc limit 1""", (item_code, warehouse, voucher_no, voucher_type), ): return flt(last_valuation_rate[0][0]) @@ -1699,6 +1702,7 @@ def update_qty_in_future_sle(args, allow_negative_stock=False): qty_shift = args.actual_qty args["time_format"] = "%H:%i:%s" + args["posting_datetime"] = get_combine_datetime(args["posting_date"], args["posting_time"]) # find difference/shift in qty caused by stock reconciliation if args.voucher_type == "Stock Reconciliation": @@ -1708,8 +1712,6 @@ def update_qty_in_future_sle(args, allow_negative_stock=False): next_stock_reco_detail = get_next_stock_reco(args) if next_stock_reco_detail: detail = next_stock_reco_detail[0] - - # add condition to update SLEs before this date & time datetime_limit_condition = get_datetime_limit_condition(detail) frappe.db.sql( @@ -1722,13 +1724,9 @@ def update_qty_in_future_sle(args, allow_negative_stock=False): and voucher_no != %(voucher_no)s and is_cancelled = 0 and ( - posting_date > %(posting_date)s or - ( - posting_date = %(posting_date)s and - time_format(posting_time, %(time_format)s) > time_format(%(posting_time)s, %(time_format)s) - ) + posting_datetime > %(posting_datetime)s ) - {datetime_limit_condition} + {datetime_limit_condition} """, args, ) @@ -1785,20 +1783,11 @@ def get_next_stock_reco(kwargs): & (sle.voucher_no != kwargs.get("voucher_no")) & (sle.is_cancelled == 0) & ( - ( - CombineDatetime(sle.posting_date, sle.posting_time) - > CombineDatetime(kwargs.get("posting_date"), kwargs.get("posting_time")) - ) - | ( - ( - CombineDatetime(sle.posting_date, sle.posting_time) - == CombineDatetime(kwargs.get("posting_date"), kwargs.get("posting_time")) - ) - & (sle.creation > kwargs.get("creation")) - ) + sle.posting_datetime + >= get_combine_datetime(kwargs.get("posting_date"), kwargs.get("posting_time")) ) ) - .orderby(CombineDatetime(sle.posting_date, sle.posting_time)) + .orderby(sle.posting_datetime) .orderby(sle.creation) .limit(1) ) @@ -1810,11 +1799,13 @@ def get_next_stock_reco(kwargs): def get_datetime_limit_condition(detail): + posting_datetime = get_combine_datetime(detail.posting_date, detail.posting_time) + return f""" and - (timestamp(posting_date, posting_time) < timestamp('{detail.posting_date}', '{detail.posting_time}') + (posting_datetime < '{posting_datetime}' or ( - timestamp(posting_date, posting_time) = timestamp('{detail.posting_date}', '{detail.posting_time}') + posting_datetime = '{posting_datetime}' and creation < '{detail.creation}' ) )""" @@ -1888,10 +1879,10 @@ def get_future_sle_with_negative_qty(args): item_code = %(item_code)s and warehouse = %(warehouse)s and voucher_no != %(voucher_no)s - and timestamp(posting_date, posting_time) >= timestamp(%(posting_date)s, %(posting_time)s) + and posting_datetime >= %(posting_datetime)s and is_cancelled = 0 and qty_after_transaction < 0 - order by timestamp(posting_date, posting_time) asc + order by posting_datetime asc limit 1 """, args, @@ -1904,20 +1895,20 @@ def get_future_sle_with_negative_batch_qty(args): """ with batch_ledger as ( select - posting_date, posting_time, voucher_type, voucher_no, - sum(actual_qty) over (order by posting_date, posting_time, creation) as cumulative_total + posting_date, posting_time, posting_datetime, voucher_type, voucher_no, + sum(actual_qty) over (order by posting_datetime, creation) as cumulative_total from `tabStock Ledger Entry` where item_code = %(item_code)s and warehouse = %(warehouse)s and batch_no=%(batch_no)s and is_cancelled = 0 - order by posting_date, posting_time, creation + order by posting_datetime, creation ) select * from batch_ledger where cumulative_total < 0.0 - and timestamp(posting_date, posting_time) >= timestamp(%(posting_date)s, %(posting_time)s) + and posting_datetime >= %(posting_datetime)s limit 1 """, args, @@ -2059,6 +2050,7 @@ def is_internal_transfer(sle): def get_stock_value_difference(item_code, warehouse, posting_date, posting_time, voucher_no=None): table = frappe.qb.DocType("Stock Ledger Entry") + posting_datetime = get_combine_datetime(posting_date, posting_time) query = ( frappe.qb.from_(table) @@ -2067,10 +2059,7 @@ def get_stock_value_difference(item_code, warehouse, posting_date, posting_time, (table.is_cancelled == 0) & (table.item_code == item_code) & (table.warehouse == warehouse) - & ( - (table.posting_date < posting_date) - | ((table.posting_date == posting_date) & (table.posting_time <= posting_time)) - ) + & (table.posting_datetime <= posting_datetime) ) ) diff --git a/erpnext/stock/utils.py b/erpnext/stock/utils.py index 9eac172aa7..82ff5b86da 100644 --- a/erpnext/stock/utils.py +++ b/erpnext/stock/utils.py @@ -8,7 +8,7 @@ from typing import Dict, Optional import frappe from frappe import _ from frappe.query_builder.functions import CombineDatetime, IfNull, Sum -from frappe.utils import cstr, flt, get_link_to_form, nowdate, nowtime +from frappe.utils import cstr, flt, get_link_to_form, get_time, getdate, nowdate, nowtime import erpnext from erpnext.stock.doctype.serial_and_batch_bundle.serial_and_batch_bundle import ( @@ -647,3 +647,18 @@ def _update_item_info(scan_result: Dict[str, Optional[str]]) -> Dict[str, Option ): scan_result.update(item_info) return scan_result + + +def get_combine_datetime(posting_date, posting_time): + import datetime + + if isinstance(posting_date, str): + posting_date = getdate(posting_date) + + if isinstance(posting_time, str): + posting_time = get_time(posting_time) + + if isinstance(posting_time, datetime.timedelta): + posting_time = (datetime.datetime.min + posting_time).time() + + return datetime.datetime.combine(posting_date, posting_time) From f04676aaed69d669e43099077c85d93fedbf7e07 Mon Sep 17 00:00:00 2001 From: Rohit Waghchaure Date: Thu, 15 Feb 2024 18:08:14 +0530 Subject: [PATCH 2/3] test: test cases to test clash timestamp entries --- erpnext/patches.txt | 2 +- .../test_stock_ledger_entry.py | 84 +++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) diff --git a/erpnext/patches.txt b/erpnext/patches.txt index 3ff4151101..1f47ad5c9e 100644 --- a/erpnext/patches.txt +++ b/erpnext/patches.txt @@ -263,6 +263,7 @@ execute:frappe.rename_doc("Report", "TDS Payable Monthly", "Tax Withholding Deta [post_model_sync] execute:frappe.delete_doc_if_exists('Workspace', 'ERPNext Integrations Settings') +erpnext.patches.v14_0.update_posting_datetime_and_dropped_indexes erpnext.patches.v14_0.rename_ongoing_status_in_sla_documents erpnext.patches.v14_0.delete_shopify_doctypes erpnext.patches.v14_0.delete_healthcare_doctypes @@ -357,4 +358,3 @@ erpnext.patches.v15_0.create_advance_payment_status erpnext.patches.v14_0.migrate_gl_to_payment_ledger erpnext.stock.doctype.delivery_note.patches.drop_unused_return_against_index # 2023-12-20 erpnext.patches.v14_0.set_maintain_stock_for_bom_item -erpnext.patches.v14_0.update_posting_datetime_and_dropped_indexes \ No newline at end of file diff --git a/erpnext/stock/doctype/stock_ledger_entry/test_stock_ledger_entry.py b/erpnext/stock/doctype/stock_ledger_entry/test_stock_ledger_entry.py index 8d82c36741..26c5c6400b 100644 --- a/erpnext/stock/doctype/stock_ledger_entry/test_stock_ledger_entry.py +++ b/erpnext/stock/doctype/stock_ledger_entry/test_stock_ledger_entry.py @@ -2,6 +2,7 @@ # See license.txt import json +import time from uuid import uuid4 import frappe @@ -1154,6 +1155,89 @@ class TestStockLedgerEntry(FrappeTestCase, StockTestMixin): except Exception as e: self.fail("Double processing of qty for clashing timestamp.") + def test_previous_sle_with_clashed_timestamp(self): + + item = make_item().name + warehouse = "_Test Warehouse - _TC" + + reciept1 = make_stock_entry( + item_code=item, + to_warehouse=warehouse, + qty=100, + rate=10, + posting_date="2021-01-01", + posting_time="02:00:00", + ) + + time.sleep(3) + + reciept2 = make_stock_entry( + item_code=item, + to_warehouse=warehouse, + qty=5, + posting_date="2021-01-01", + rate=10, + posting_time="02:00:00.1234", + ) + + sle = frappe.get_all( + "Stock Ledger Entry", + filters={"voucher_no": reciept1.name}, + fields=["qty_after_transaction", "actual_qty"], + ) + self.assertEqual(sle[0].qty_after_transaction, 100) + self.assertEqual(sle[0].actual_qty, 100) + + sle = frappe.get_all( + "Stock Ledger Entry", + filters={"voucher_no": reciept2.name}, + fields=["qty_after_transaction", "actual_qty"], + ) + self.assertEqual(sle[0].qty_after_transaction, 105) + self.assertEqual(sle[0].actual_qty, 5) + + def test_backdated_sle_with_same_timestamp(self): + + item = make_item().name + warehouse = "_Test Warehouse - _TC" + + reciept1 = make_stock_entry( + item_code=item, + to_warehouse=warehouse, + qty=5, + posting_date="2021-01-01", + rate=10, + posting_time="02:00:00.1234", + ) + + time.sleep(3) + + # backdated entry with same timestamp but different ms part + reciept2 = make_stock_entry( + item_code=item, + to_warehouse=warehouse, + qty=100, + rate=10, + posting_date="2021-01-01", + posting_time="02:00:00", + ) + + sle = frappe.get_all( + "Stock Ledger Entry", + filters={"voucher_no": reciept1.name}, + fields=["qty_after_transaction", "actual_qty"], + ) + self.assertEqual(sle[0].qty_after_transaction, 105) + self.assertEqual(sle[0].actual_qty, 5) + + sle = frappe.get_all( + "Stock Ledger Entry", + filters={"voucher_no": reciept2.name}, + fields=["qty_after_transaction", "actual_qty"], + ) + self.assertEqual(sle[0].qty_after_transaction, 100) + self.assertEqual(sle[0].actual_qty, 100) + @change_settings("System Settings", {"float_precision": 3, "currency_precision": 2}) def test_transfer_invariants(self): """Extact stock value should be transferred.""" From a73ba2c0d26b9d27bb5a75bc6c9739e49035f266 Mon Sep 17 00:00:00 2001 From: Rohit Waghchaure Date: Wed, 21 Feb 2024 17:32:02 +0530 Subject: [PATCH 3/3] chore: remove microsecond from posting_datetime --- erpnext/stock/doctype/stock_entry/test_stock_entry.py | 10 ++++------ .../stock_ledger_entry/test_stock_ledger_entry.py | 4 ++-- erpnext/stock/stock_ledger.py | 9 +-------- erpnext/stock/utils.py | 2 +- .../test_subcontracting_receipt.py | 4 ---- 5 files changed, 8 insertions(+), 21 deletions(-) diff --git a/erpnext/stock/doctype/stock_entry/test_stock_entry.py b/erpnext/stock/doctype/stock_entry/test_stock_entry.py index 99c050afa2..9d1a3f7e62 100644 --- a/erpnext/stock/doctype/stock_entry/test_stock_entry.py +++ b/erpnext/stock/doctype/stock_entry/test_stock_entry.py @@ -1611,24 +1611,22 @@ class TestStockEntry(FrappeTestCase): item_code = "Test Negative Item - 001" item_doc = create_item(item_code=item_code, is_stock_item=1, valuation_rate=10) - make_stock_entry( + se1 = make_stock_entry( item_code=item_code, posting_date=add_days(today(), -3), posting_time="00:00:00", - purpose="Material Receipt", + target="_Test Warehouse - _TC", qty=10, to_warehouse="_Test Warehouse - _TC", - do_not_save=True, ) - make_stock_entry( + se2 = make_stock_entry( item_code=item_code, posting_date=today(), posting_time="00:00:00", - purpose="Material Receipt", + source="_Test Warehouse - _TC", qty=8, from_warehouse="_Test Warehouse - _TC", - do_not_save=True, ) sr_doc = create_stock_reconciliation( diff --git a/erpnext/stock/doctype/stock_ledger_entry/test_stock_ledger_entry.py b/erpnext/stock/doctype/stock_ledger_entry/test_stock_ledger_entry.py index 26c5c6400b..40a2d5a566 100644 --- a/erpnext/stock/doctype/stock_ledger_entry/test_stock_ledger_entry.py +++ b/erpnext/stock/doctype/stock_ledger_entry/test_stock_ledger_entry.py @@ -1227,7 +1227,7 @@ class TestStockLedgerEntry(FrappeTestCase, StockTestMixin): filters={"voucher_no": reciept1.name}, fields=["qty_after_transaction", "actual_qty"], ) - self.assertEqual(sle[0].qty_after_transaction, 105) + self.assertEqual(sle[0].qty_after_transaction, 5) self.assertEqual(sle[0].actual_qty, 5) sle = frappe.get_all( @@ -1235,7 +1235,7 @@ class TestStockLedgerEntry(FrappeTestCase, StockTestMixin): filters={"voucher_no": reciept2.name}, fields=["qty_after_transaction", "actual_qty"], ) - self.assertEqual(sle[0].qty_after_transaction, 100) + self.assertEqual(sle[0].qty_after_transaction, 105) self.assertEqual(sle[0].actual_qty, 100) @change_settings("System Settings", {"float_precision": 3, "currency_precision": 2}) diff --git a/erpnext/stock/stock_ledger.py b/erpnext/stock/stock_ledger.py index e8bd3fcbb1..2ae6c197a1 100644 --- a/erpnext/stock/stock_ledger.py +++ b/erpnext/stock/stock_ledger.py @@ -11,7 +11,6 @@ from frappe import _, scrub from frappe.model.meta import get_field_precision from frappe.query_builder.functions import Sum from frappe.utils import ( - add_to_date, cint, cstr, flt, @@ -637,7 +636,7 @@ class update_entries_after(object): posting_datetime = %(posting_datetime)s ) order by - posting_datetime ASC, creation ASC + creation ASC for update """, self.args, @@ -1403,17 +1402,12 @@ class update_entries_after(object): def get_previous_sle_of_current_voucher(args, operator="<", exclude_current_voucher=False): """get stock ledger entries filtered by specific posting datetime conditions""" - args["time_format"] = "%H:%i:%s" if not args.get("posting_date"): args["posting_datetime"] = "1900-01-01 00:00:00" if not args.get("posting_datetime"): args["posting_datetime"] = get_combine_datetime(args["posting_date"], args["posting_time"]) - if operator == "<=": - # Add 1 second to handle millisecond for less than and equal to condition - args["posting_datetime"] = add_to_date(args["posting_datetime"], seconds=1) - voucher_condition = "" if exclude_current_voucher: voucher_no = args.get("voucher_no") @@ -1701,7 +1695,6 @@ def update_qty_in_future_sle(args, allow_negative_stock=False): datetime_limit_condition = "" qty_shift = args.actual_qty - args["time_format"] = "%H:%i:%s" args["posting_datetime"] = get_combine_datetime(args["posting_date"], args["posting_time"]) # find difference/shift in qty caused by stock reconciliation diff --git a/erpnext/stock/utils.py b/erpnext/stock/utils.py index 5dc9dd45ab..93e2fa46c7 100644 --- a/erpnext/stock/utils.py +++ b/erpnext/stock/utils.py @@ -671,4 +671,4 @@ def get_combine_datetime(posting_date, posting_time): if isinstance(posting_time, datetime.timedelta): posting_time = (datetime.datetime.min + posting_time).time() - return datetime.datetime.combine(posting_date, posting_time) + return datetime.datetime.combine(posting_date, posting_time).replace(microsecond=0) diff --git a/erpnext/subcontracting/doctype/subcontracting_receipt/test_subcontracting_receipt.py b/erpnext/subcontracting/doctype/subcontracting_receipt/test_subcontracting_receipt.py index 5523c318a5..0450038d80 100644 --- a/erpnext/subcontracting/doctype/subcontracting_receipt/test_subcontracting_receipt.py +++ b/erpnext/subcontracting/doctype/subcontracting_receipt/test_subcontracting_receipt.py @@ -643,10 +643,6 @@ class TestSubcontractingReceipt(FrappeTestCase): ) scr = make_subcontracting_receipt(sco.name) scr.save() - for row in scr.supplied_items: - self.assertNotEqual(row.rate, 300.00) - self.assertFalse(row.serial_and_batch_bundle) - scr.submit() scr.reload()