Optimization related to large number of items (#13606)

erpnext/selling/doctype/sales_order/sales_order.js

@@ -65,6 +65,10 @@ frappe.ui.form.on("Sales Order Item", {
 });
 
 erpnext.selling.SalesOrderController = erpnext.selling.SellingController.extend({
+	onload: function(doc, dt, dn) {
+		this._super();
+	},
+
 	refresh: function(doc, dt, dn) {
 		var me = this;
 		this._super();

erpnext/stock/doctype/item/item.json

@@ -140,7 +140,7 @@
    "remember_last_selected_value": 0,
    "report_hide": 0,
    "reqd": 0,
-   "search_index": 0,
+   "search_index": 1,
    "set_only_once": 1,
    "translatable": 0,
    "unique": 0
@@ -236,7 +236,7 @@
    "remember_last_selected_value": 0,
    "report_hide": 0,
    "reqd": 1,
-   "search_index": 0,
+   "search_index": 1,
    "set_only_once": 0,
    "translatable": 0,
    "unique": 0
@@ -3063,7 +3063,7 @@
    "remember_last_selected_value": 0,
    "report_hide": 0,
    "reqd": 0,
-   "search_index": 1,
+   "search_index": 0,
    "set_only_once": 0,
    "translatable": 0,
    "unique": 0
@@ -3561,7 +3561,7 @@
  "issingle": 0,
  "istable": 0,
  "max_attachments": 1,
- "modified": "2018-03-06 10:21:48.715529",
+ "modified": "2018-04-11 12:21:48.715529",
  "modified_by": "Administrator",
  "module": "Stock",
  "name": "Item",

erpnext/stock/report/stock_balance/stock_balance.js

@@ -9,7 +9,7 @@ frappe.query_reports["Stock Balance"] = {
 			"fieldtype": "Date",
 			"width": "80",
 			"reqd": 1,
-			"default": frappe.sys_defaults.year_start_date,
+			"default": frappe.datetime.add_months(frappe.datetime.get_today(), -1),
 		},
 		{
 			"fieldname":"to_date",
@@ -31,7 +31,12 @@ frappe.query_reports["Stock Balance"] = {
 			"label": __("Item"),
 			"fieldtype": "Link",
 			"width": "80",
-			"options": "Item"
+			"options": "Item",
+			"get_query": function() {
+				return {
+					query: "erpnext.controllers.queries.item_query"
+				}
+			}
 		},
 		{
 			"fieldname": "warehouse",

erpnext/stock/report/stock_balance/stock_balance.py

@@ -4,7 +4,8 @@
 from __future__ import unicode_literals
 import frappe
 from frappe import _
-from frappe.utils import flt, cint, getdate
+from frappe.utils import flt, cint, getdate, now
+from erpnext.stock.report.stock_ledger.stock_ledger import get_item_group_condition
 
 def execute(filters=None):
 	if not filters: filters = {}
@@ -12,10 +13,11 @@ def execute(filters=None):
 	validate_filters(filters)
 
 	columns = get_columns()
-	item_map = get_item_details(filters)
+	items = get_items(filters)
-	item_reorder_detail_map = get_item_reorder_details(filters)
+	sle = get_stock_ledger_entries(filters, items)
-	iwb_map = get_item_warehouse_map(filters)
+	iwb_map = get_item_warehouse_map(filters, sle)
-	
+	item_map = get_item_details(items, sle, filters)
+	item_reorder_detail_map = get_item_reorder_details(item_map.keys())
 
 	data = []
 	for (company, item, warehouse) in sorted(iwb_map):
@@ -88,21 +90,9 @@ def get_conditions(filters):
 	else:
 		frappe.throw(_("'To Date' is required"))
 
-	if filters.get("item_group"):		
-		ig_details = frappe.db.get_value("Item Group", filters.get("item_group"), 
-			["lft", "rgt"], as_dict=1)
-			
-		if ig_details:
-			conditions += """ 
-				and exists (select name from `tabItem Group` ig 
-				where ig.lft >= %s and ig.rgt <= %s and item.item_group = ig.name)
-			""" % (ig_details.lft, ig_details.rgt)
-		
-	if filters.get("item_code"):
-		conditions += " and sle.item_code = '%s'" % frappe.db.escape(filters.get("item_code"), percent=False)
-
 	if filters.get("warehouse"):
-		warehouse_details = frappe.db.get_value("Warehouse", filters.get("warehouse"), ["lft", "rgt"], as_dict=1)
+		warehouse_details = frappe.db.get_value("Warehouse",
+			filters.get("warehouse"), ["lft", "rgt"], as_dict=1)
 		if warehouse_details:
 			conditions += " and exists (select name from `tabWarehouse` wh \
 				where wh.lft >= %s and wh.rgt <= %s and sle.warehouse = wh.name)"%(warehouse_details.lft,
@@ -110,30 +100,29 @@ def get_conditions(filters):
 
 	return conditions
 
-def get_stock_ledger_entries(filters):
+def get_stock_ledger_entries(filters, items):
-	conditions = get_conditions(filters)
+	item_conditions_sql = ''
+	if items:
+		item_conditions_sql = ' and sle.item_code in ({})'\
+			.format(', '.join(['"' + frappe.db.escape(i) + '"' for i in items]))
 
-	join_table_query = ""
+	conditions = get_conditions(filters)
-	if filters.get("item_group"):
-		join_table_query = "inner join `tabItem` item on item.name = sle.item_code"
 
 	return frappe.db.sql("""
 		select
 			sle.item_code, warehouse, sle.posting_date, sle.actual_qty, sle.valuation_rate,
 			sle.company, sle.voucher_type, sle.qty_after_transaction, sle.stock_value_difference
 		from
-			`tabStock Ledger Entry` sle force index (posting_sort_index) %s
+			`tabStock Ledger Entry` sle force index (posting_sort_index)
-		where sle.docstatus < 2 %s 
+		where sle.docstatus < 2 %s %s
 		order by sle.posting_date, sle.posting_time, sle.name""" %
-		(join_table_query, conditions), as_dict=1)
+		(item_conditions_sql, conditions), as_dict=1)
 
-def get_item_warehouse_map(filters):
+def get_item_warehouse_map(filters, sle):
 	iwb_map = {}
 	from_date = getdate(filters.get("from_date"))
 	to_date = getdate(filters.get("to_date"))
 
-	sle = get_stock_ledger_entries(filters)
-
 	for d in sle:
 		key = (d.company, d.item_code, d.warehouse)
 		if key not in iwb_map:
@@ -191,20 +180,33 @@ def filter_items_with_no_transactions(iwb_map):
 
 	return iwb_map
 
-def get_item_details(filters):
+def get_items(filters):
-	condition = ''
+	conditions = []
-	value = ()
 	if filters.get("item_code"):
-		condition = "where item_code=%s"
+		conditions.append("item.name=%(item_code)s")
-		value = (filters.get("item_code"),)
+	else:
+		if filters.get("brand"):
+			conditions.append("item.brand=%(brand)s")
+		if filters.get("item_group"):
+			conditions.append(get_item_group_condition(filters.get("item_group")))
 
-	items = frappe.db.sql("""
+	items = []
-		select name, item_name, stock_uom, item_group, brand, description
+	if conditions:
-		from tabItem
+		items = frappe.db.sql_list("""select name from `tabItem` item where {}"""
-		{condition}
+			.format(" and ".join(conditions)), filters)
-	""".format(condition=condition), value, as_dict=1)
+	return items
 
-	item_details = dict((d.name , d) for d in items)
+def get_item_details(items, sle, filters):
+	item_details = {}
+	if not items:
+		items = list(set([d.item_code for d in sle]))
+
+	for item in frappe.db.sql("""
+		select name, item_name, description, item_group, brand, stock_uom
+		from `tabItem`
+		where name in ({0})
+		""".format(', '.join(['"' + frappe.db.escape(i) + '"' for i in items])), as_dict=1):
+			item_details.setdefault(item.name, item)
 
 	if filters.get('show_variant_attributes', 0) == 1:
 		variant_values = get_variant_values_for(item_details.keys())
@@ -212,18 +214,12 @@ def get_item_details(filters):
 
 	return item_details
 
-def get_item_reorder_details(filters):
+def get_item_reorder_details(items):
-	condition = ''
-	value = ()
-	if filters.get("item_code"):
-		condition = "where parent=%s"
-		value = (filters.get("item_code"),)
-
 	item_reorder_details = frappe.db.sql("""
 		select parent, warehouse, warehouse_reorder_qty, warehouse_reorder_level
 		from `tabItem Reorder`
-		{condition}
+		where parent in ({0})
-	""".format(condition=condition), value, as_dict=1)
+	""".format(', '.join(['"' + frappe.db.escape(i) + '"' for i in items])), as_dict=1)
 
 	return dict((d.parent + d.warehouse, d) for d in item_reorder_details)
 
@@ -233,12 +229,10 @@ def validate_filters(filters):
 		if sle_count > 500000:
 			frappe.throw(_("Please set filter based on Item or Warehouse"))
 
-
 def get_variants_attributes():
 	'''Return all item variant attributes.'''
 	return [i.name for i in frappe.get_all('Item Attribute')]
 
-
 def get_variant_values_for(items):
 	'''Returns variant values for items.'''
 	attribute_map = {}

erpnext/stock/report/stock_ledger/stock_ledger.js

@@ -35,7 +35,12 @@ frappe.query_reports["Stock Ledger"] = {
 			"fieldname":"item_code",
 			"label": __("Item"),
 			"fieldtype": "Link",
-			"options": "Item"
+			"options": "Item",
+			"get_query": function() {
+				return {
+					query: "erpnext.controllers.queries.item_query"
+				}
+			}
 		},
 		{
 			"fieldname":"item_group",

erpnext/stock/report/stock_ledger/stock_ledger.py

@@ -7,13 +7,12 @@ from frappe import _
 
 def execute(filters=None):
 	columns = get_columns()
-	item_conditions = get_item_conditions(filters)
+	items = get_items(filters)
-	item_details = get_item_details(filters, item_conditions)
+	sl_entries = get_stock_ledger_entries(filters, items)
-	sl_entries = get_stock_ledger_entries(filters, item_conditions, item_details)
+	item_details = get_item_details(items, sl_entries)
 	opening_row = get_opening_balance(filters, columns)
 
 	data = []
-
 	if opening_row:
 		data.append(opening_row)
 
@@ -53,12 +52,12 @@ def get_columns():
 
 	return columns
 
-def get_stock_ledger_entries(filters, item_conditions, item_details):
+def get_stock_ledger_entries(filters, items):
 	item_conditions_sql = ''
-	if item_conditions:
-		items = ['"' + frappe.db.escape(i) + '"' for i in item_details.keys()]
 	if items:
-			item_conditions_sql = 'and sle.item_code in ({})'.format(', '.join(items))
+		item_conditions_sql = 'and sle.item_code in ({})'\
+			.format(', '.join(['"' + frappe.db.escape(i) + '"' for i in items]))
+
 	return frappe.db.sql("""select concat_ws(" ", posting_date, posting_time) as date,
 			item_code, warehouse, actual_qty, qty_after_transaction, incoming_rate, valuation_rate,
 			stock_value, voucher_type, voucher_no, batch_no, serial_no, company, project
@@ -73,25 +72,35 @@ def get_stock_ledger_entries(filters, item_conditions, item_details):
 			item_conditions_sql = item_conditions_sql
 		), filters, as_dict=1)
 
-def get_item_details(filters, item_conditions):
+def get_items(filters):
-	item_details = {}
-	for item in frappe.db.sql("""select name, item_name, description, item_group,
-			brand, stock_uom from `tabItem` item {item_conditions}"""\
-			.format(item_conditions=item_conditions), filters, as_dict=1):
-		item_details.setdefault(item.name, item)
-
-	return item_details
-
-def get_item_conditions(filters):
 	conditions = []
 	if filters.get("item_code"):
 		conditions.append("item.name=%(item_code)s")
+	else:
 		if filters.get("brand"):
 			conditions.append("item.brand=%(brand)s")
 		if filters.get("item_group"):
 			conditions.append(get_item_group_condition(filters.get("item_group")))
 
-	return "where {}".format(" and ".join(conditions)) if conditions else ""
+	items = []
+	if conditions:
+		items = frappe.db.sql_list("""select name from `tabItem` item where {}"""
+			.format(" and ".join(conditions)), filters)
+	return items
+
+def get_item_details(items, sl_entries):
+	item_details = {}
+	if not items:
+		items = list(set([d.item_code for d in sl_entries]))
+
+	for item in frappe.db.sql("""
+		select name, item_name, description, item_group, brand, stock_uom
+		from `tabItem`
+		where name in ({0})
+		""".format(', '.join(['"' + frappe.db.escape(i) + '"' for i in items])), as_dict=1):
+			item_details.setdefault(item.name, item)
+
+	return item_details
 
 def get_sle_conditions(filters):
 	conditions = []

erpnext/stock/report/stock_projected_qty/stock_projected_qty.js

@@ -19,7 +19,12 @@ frappe.query_reports["Stock Projected Qty"] = {
 			"fieldname":"item_code",
 			"label": __("Item"),
 			"fieldtype": "Link",
-			"options": "Item"
+			"options": "Item",
+			"get_query": function() {
+				return {
+					query: "erpnext.controllers.queries.item_query"
+				}
+			}
 		},
 		{
 			"fieldname":"brand",