feat: Make search index fields configurable

- Move indexing logic to separate file
- Add more validation logic for 'search index fields' field

erpnext/e_commerce/doctype/e_commerce_settings/e_commerce_settings.py

@@ -6,8 +6,9 @@ import frappe
 from frappe.utils import cint, comma_and
 from frappe import _, msgprint
 from frappe.model.document import Document
-from frappe.utils import cint
+from frappe.utils import get_datetime, get_datetime_str, now_datetime, unique
-from frappe.utils import get_datetime, get_datetime_str, now_datetime
+
+from erpnext.e_commerce.website_item_indexing import create_website_items_index, ALLOWED_INDEXABLE_FIELDS_SET
 
 class ShoppingCartSetupError(frappe.ValidationError): pass
 
@@ -54,13 +55,26 @@ class ECommerceSettings(Document):
 
 	def validate_search_index_fields(self):
 		if not self.search_index_fields:
-			return 
+			return
-		
-		# Clean up
-		fields = self.search_index_fields.replace(' ', '')
-		fields = fields.strip(',')
 
-		self.search_index_fields = fields
+		# Clean up
+		# Remove whitespaces
+		fields = self.search_index_fields.replace(' ', '')
+		# Remove extra ',' and remove duplicates
+		fields = unique(fields.strip(',').split(','))
+
+		# All fields should be indexable
+		if not (set(fields).issubset(ALLOWED_INDEXABLE_FIELDS_SET)):
+			invalid_fields = list(set(fields).difference(ALLOWED_INDEXABLE_FIELDS_SET))
+			num_invalid_fields = len(invalid_fields)
+			invalid_fields = comma_and(invalid_fields)
+
+			if num_invalid_fields > 1:
+				frappe.throw(_("{0} are not valid options for Search Index Field.").format(frappe.bold(invalid_fields)))
+			else:
+				frappe.throw(_("{0} is not a valid option for Search Index Field.").format(frappe.bold(invalid_fields)))
+
+		self.search_index_fields = ','.join(fields)
 
 	def validate_exchange_rates_exist(self):
 		"""check if exchange rates exist for all Price List currencies (to company's currency)"""
@@ -113,6 +127,15 @@ class ECommerceSettings(Document):
 	def get_shipping_rules(self, shipping_territory):
 		return self.get_name_from_territory(shipping_territory, "shipping_rules", "shipping_rule")
 
+	def on_change(self):
+		old_doc = self.get_doc_before_save()
+		old_fields = old_doc.search_index_fields
+		new_fields = self.search_index_fields
+
+		# if search index fields get changed
+		if not (new_fields == old_fields):
+			create_website_items_index()
+
 def validate_cart_settings(doc, method):
 	frappe.get_doc("E Commerce Settings", "E Commerce Settings").run_method("validate")
 

erpnext/e_commerce/doctype/website_item/website_item.py

@@ -17,7 +17,7 @@ from erpnext.setup.doctype.item_group.item_group import (get_parent_item_groups,
 from erpnext.e_commerce.doctype.item_review.item_review import get_item_reviews
 
 # SEARCH 
-from erpnext.templates.pages.product_search import (
+from erpnext.e_commerce.website_item_indexing import (
 	insert_item_to_index, 
 	update_index_for_item, 
 	delete_item_from_index

erpnext/e_commerce/website_item_indexing.py

@@ -0,0 +1,167 @@
+# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# License: GNU General Public License v3. See license.txt
+
+from __future__ import unicode_literals
+
+import frappe
+import redis
+
+from redisearch import (
+        Client, AutoCompleter, Query,
+        Suggestion, IndexDefinition, 
+        TextField, TagField,
+        Document
+	)
+
+# GLOBAL CONSTANTS
+WEBSITE_ITEM_INDEX = 'website_items_index'
+WEBSITE_ITEM_KEY_PREFIX = 'website_item:'
+WEBSITE_ITEM_NAME_AUTOCOMPLETE = 'website_items_name_dict'
+
+ALLOWED_INDEXABLE_FIELDS_SET = {
+	'item_code',
+	'item_name',
+	'item_group',
+	'brand',
+	'description',
+	'web_long_description'
+}
+
+def create_website_items_index():
+	'''Creates Index Definition'''
+	# CREATE index
+	client = Client(WEBSITE_ITEM_INDEX, port=13000)
+
+	# DROP if already exists
+	try:
+		client.drop_index()
+	except:
+		pass
+
+	
+	idx_def = IndexDefinition([WEBSITE_ITEM_KEY_PREFIX])
+
+	# Based on e-commerce settings
+	idx_fields = frappe.db.get_single_value(
+		'E Commerce Settings', 
+		'search_index_fields'
+	).split(',')
+
+	if 'web_item_name' in idx_fields:
+		idx_fields.remove('web_item_name')
+	
+	idx_fields = list(map(to_search_field, idx_fields))
+
+	client.create_index(
+		[TextField("web_item_name", sortable=True)] + idx_fields,
+		definition=idx_def
+	)
+
+	reindex_all_web_items()
+	define_autocomplete_dictionary()
+
+def to_search_field(field):
+	if field == "tags":
+		return TagField("tags", separator=",")
+
+	return TextField(field)
+
+def insert_item_to_index(website_item_doc):
+	# Insert item to index
+	key = get_cache_key(website_item_doc.name)
+	r = redis.Redis("localhost", 13000)
+	web_item = create_web_item_map(website_item_doc)
+	r.hset(key, mapping=web_item)
+	insert_to_name_ac(website_item_doc.web_item_name, website_item_doc.name)
+
+def insert_to_name_ac(web_name, doc_name):
+	ac = AutoCompleter(WEBSITE_ITEM_NAME_AUTOCOMPLETE, port=13000)
+	ac.add_suggestions(Suggestion(web_name, payload=doc_name))
+
+def create_web_item_map(website_item_doc):
+	fields_to_index = get_fields_indexed()
+
+	web_item = {}
+
+	for f in fields_to_index:
+		web_item[f] = website_item_doc.get(f) or ''
+
+	return web_item
+	
+def update_index_for_item(website_item_doc):
+	# Reinsert to Cache
+	insert_item_to_index(website_item_doc)
+	define_autocomplete_dictionary()
+	# TODO: Only reindex updated items
+	create_website_items_index()
+
+def delete_item_from_index(website_item_doc):
+	r = redis.Redis("localhost", 13000)
+	key = get_cache_key(website_item_doc.name)
+	
+	try:
+		r.delete(key)
+	except:
+		return False
+
+	# TODO: Also delete autocomplete suggestion
+	return True
+
+def define_autocomplete_dictionary():
+	print("Defining ac dict...")
+	# AC for name
+	# TODO: AC for category
+
+	r = redis.Redis("localhost", 13000)
+	ac = AutoCompleter(WEBSITE_ITEM_NAME_AUTOCOMPLETE, port=13000)
+
+	try:
+		r.delete(WEBSITE_ITEM_NAME_AUTOCOMPLETE)
+	except:
+		return False
+	
+	items = frappe.get_all(
+		'Website Item', 
+		fields=['web_item_name'], 
+		filters={"published": True}
+	)
+
+	for item in items:
+		print("adding suggestion: " + item.web_item_name)
+		ac.add_suggestions(Suggestion(item.web_item_name))
+
+	return True
+
+def reindex_all_web_items():
+	items = frappe.get_all(
+		'Website Item', 
+		fields=get_fields_indexed(), 
+		filters={"published": True}
+	)
+
+	r = redis.Redis("localhost", 13000)
+	for item in items:
+		web_item = create_web_item_map(item)
+		key = get_cache_key(item.name)
+		print(key, web_item)
+		r.hset(key, mapping=web_item)
+
+def get_cache_key(name):
+	name = frappe.scrub(name)
+	return f"{WEBSITE_ITEM_KEY_PREFIX}{name}"
+
+def get_fields_indexed():
+	fields_to_index = frappe.db.get_single_value(
+		'E Commerce Settings', 
+		'search_index_fields'
+	).split(',')
+
+	mandatory_fields = ['name', 'web_item_name', 'route', 'thumbnail']
+	fields_to_index = fields_to_index + mandatory_fields
+
+	return fields_to_index
+
+# TODO: Remove later
+# # Figure out a way to run this at startup
+define_autocomplete_dictionary()
+create_website_items_index()

erpnext/templates/pages/product_search.py

@@ -1,8 +1,6 @@
-# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
+# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
 # License: GNU General Public License v3. See license.txt
 
-from __future__ import unicode_literals
-
 import frappe
 from frappe.utils import cint, cstr, nowdate
 
@@ -10,17 +8,8 @@ from erpnext.setup.doctype.item_group.item_group import get_item_for_list_in_htm
 from erpnext.e_commerce.shopping_cart.product_info import set_product_info_for_website
 
 # For SEARCH -------
-import redis
+from redisearch import AutoCompleter, Client, Query
-from redisearch import (
+from erpnext.e_commerce.website_item_indexing import WEBSITE_ITEM_INDEX, WEBSITE_ITEM_NAME_AUTOCOMPLETE
-	Client, AutoCompleter, Query,
-	Suggestion, IndexDefinition, 
-	TextField, TagField,
-	Document
-	)
-
-WEBSITE_ITEM_INDEX = 'website_items_index'
-WEBSITE_ITEM_KEY_PREFIX = 'website_item:'
-WEBSITE_ITEM_NAME_AUTOCOMPLETE = 'website_items_name_dict'
 # -----------------
 
 no_cache = 1
@@ -94,111 +83,3 @@ def search(query):
 
 def convert_to_dict(redis_search_doc):
 	return redis_search_doc.__dict__
-
-def create_website_items_index():
-	'''Creates Index Definition'''
-	# CREATE index
-	client = Client(WEBSITE_ITEM_INDEX, port=13000)
-
-	# DROP if already exists
-	try:
-		client.drop_index()
-	except:
-		pass
-
-	
-	idx_def = IndexDefinition([WEBSITE_ITEM_KEY_PREFIX])
-
-	client.create_index(
-		[TextField("web_item_name", sortable=True), TagField("tags")],
-		definition=idx_def
-	)
-
-	reindex_all_web_items()
-
-def insert_item_to_index(website_item_doc):
-	# Insert item to index
-	key = get_cache_key(website_item_doc.name)
-	r = redis.Redis("localhost", 13000)
-	web_item = create_web_item_map(website_item_doc)
-	r.hset(key, mapping=web_item)
-	insert_to_name_ac(website_item_doc.web_item_name, website_item_doc.name)
-
-def insert_to_name_ac(web_name, doc_name):
-	ac = AutoCompleter(WEBSITE_ITEM_NAME_AUTOCOMPLETE, port=13000)
-	ac.add_suggestions(Suggestion(web_name, payload=doc_name))
-
-def create_web_item_map(website_item_doc):
-	web_item = {}
-	web_item["web_item_name"] = website_item_doc.web_item_name
-	web_item["route"] = website_item_doc.route
-	web_item["thumbnail"] = website_item_doc.thumbnail or ''
-	web_item["description"] = website_item_doc.description or ''
-
-	return web_item
-
-def update_index_for_item(website_item_doc):
-	# Reinsert to Cache
-	insert_item_to_index(website_item_doc)
-	define_autocomplete_dictionary()
-	# TODO: Only reindex updated items
-	create_website_items_index()
-
-def delete_item_from_index(website_item_doc):
-	r = redis.Redis("localhost", 13000)
-	key = get_cache_key(website_item_doc.name)
-
-	try:
-		r.delete(key)
-	except:
-		return False
-
-	# TODO: Also delete autocomplete suggestion
-	return True
-
-def define_autocomplete_dictionary():
-	# AC for name
-	# TODO: AC for category
-
-	r = redis.Redis("localhost", 13000)
-	ac = AutoCompleter(WEBSITE_ITEM_NAME_AUTOCOMPLETE, port=13000)
-
-	try:
-		r.delete(WEBSITE_ITEM_NAME_AUTOCOMPLETE)
-	except:
-		return False
-
-	items = frappe.get_all(
-		'Website Item',
-		fields=['web_item_name'],
-		filters={"published": True}
-	)
-
-	for item in items:
-		print("adding suggestion: " + item.web_item_name)
-		ac.add_suggestions(Suggestion(item.web_item_name))
-
-	return True
-
-def reindex_all_web_items():
-	items = frappe.get_all(
-		'Website Item',
-		fields=['web_item_name', 'name', 'route', 'thumbnail', 'description'],
-		filters={"published": True}
-	)
-
-	r = redis.Redis("localhost", 13000)
-	for item in items:
-		web_item = create_web_item_map(item)
-		key = get_cache_key(item.name)
-		print(key, web_item)
-		r.hset(key, mapping=web_item)
-
-def get_cache_key(name):
-	name = frappe.scrub(name)
-	return f"{WEBSITE_ITEM_KEY_PREFIX}{name}"
-
-# TODO: Remove later
-# Figure out a way to run this at startup
-define_autocomplete_dictionary()
-create_website_items_index()