From d168a71cfb2279b690e5b24d51ca2168091eccfb Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Wed, 14 Nov 2018 18:58:30 +0530 Subject: [PATCH] perf(patch): Use INSERT with ON DUPLICATE KEY UPDATE update_total_qty_field patch was using UPDATE statement with CASE Now use INSERT INTO with ON DUPLICATE KEY UPDATE clause which allows this query to use indexes and update multiple rows per query. --- .../patches/v11_0/update_total_qty_field.py | 35 ++++++++++++------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/erpnext/patches/v11_0/update_total_qty_field.py b/erpnext/patches/v11_0/update_total_qty_field.py index 5c7663d5aa..8f086992b6 100644 --- a/erpnext/patches/v11_0/update_total_qty_field.py +++ b/erpnext/patches/v11_0/update_total_qty_field.py @@ -9,12 +9,12 @@ def execute(): frappe.reload_doc('stock', 'doctype', 'purchase_receipt') frappe.reload_doc('accounts', 'doctype', 'sales_invoice') frappe.reload_doc('accounts', 'doctype', 'purchase_invoice') - + doctypes = ["Sales Order", "Sales Invoice", "Delivery Note",\ "Purchase Order", "Purchase Invoice", "Purchase Receipt", "Quotation", "Supplier Quotation"] for doctype in doctypes: - total_qty = frappe.db.sql(''' + total_qty = frappe.db.sql(''' SELECT parent, SUM(qty) as qty FROM @@ -22,14 +22,25 @@ def execute(): GROUP BY parent ''' % (doctype), as_dict = True) - when_then = [] - for d in total_qty: - when_then.append(""" - when dt.name = '{0}' then {1} - """.format(frappe.db.escape(d.get("parent")), d.get("qty"))) + # Query to update total_qty might become too big, Update in batches + # batch_size is chosen arbitrarily, Don't try too hard to reason about it + batch_size = 100000 + for i in range(0, len(total_qty), batch_size): + batch_transactions = total_qty[i:i + batch_size] - if when_then: - frappe.db.sql(''' - UPDATE - `tab%s` dt SET dt.total_qty = CASE %s ELSE 0.0 END - ''' % (doctype, " ".join(when_then))) \ No newline at end of file + # UPDATE with CASE for some reason cannot use PRIMARY INDEX, + # causing all rows to be examined, leading to a very slow update + + # UPDATE with WHERE clause uses PRIMARY INDEX, but will lead to too many queries + + # INSERT with ON DUPLICATE KEY UPDATE uses PRIMARY INDEX + # and can perform multiple updates per query + # This is probably never used anywhere else as of now, but should be + values = [] + for d in batch_transactions: + values.append("('{}', {})".format(d.parent, d.qty)) + conditions = ",".join(values) + frappe.db.sql(""" + INSERT INTO `tab{}` (name, total_qty) VALUES {} + ON DUPLICATE KEY UPDATE name = VALUES(name), total_qty = VALUES(total_qty) + """.format(doctype, conditions))