From 8e55c95eccbb6e14c118deef3f1a6fa0f4f2e4bf Mon Sep 17 00:00:00 2001 From: Hussain Nagaria Date: Mon, 26 Apr 2021 07:01:06 +0530 Subject: [PATCH] feat: Make search index fields configurable - Move indexing logic to separate file - Add more validation logic for 'search index fields' field --- .../e_commerce_settings.py | 39 +++- .../doctype/website_item/website_item.py | 2 +- erpnext/e_commerce/website_item_indexing.py | 167 ++++++++++++++++++ erpnext/templates/pages/product_search.py | 125 +------------ 4 files changed, 202 insertions(+), 131 deletions(-) create mode 100644 erpnext/e_commerce/website_item_indexing.py diff --git a/erpnext/e_commerce/doctype/e_commerce_settings/e_commerce_settings.py b/erpnext/e_commerce/doctype/e_commerce_settings/e_commerce_settings.py index 1fd3bfac81..a028a5f86f 100644 --- a/erpnext/e_commerce/doctype/e_commerce_settings/e_commerce_settings.py +++ b/erpnext/e_commerce/doctype/e_commerce_settings/e_commerce_settings.py @@ -6,8 +6,9 @@ import frappe from frappe.utils import cint, comma_and from frappe import _, msgprint from frappe.model.document import Document -from frappe.utils import cint -from frappe.utils import get_datetime, get_datetime_str, now_datetime +from frappe.utils import get_datetime, get_datetime_str, now_datetime, unique + +from erpnext.e_commerce.website_item_indexing import create_website_items_index, ALLOWED_INDEXABLE_FIELDS_SET class ShoppingCartSetupError(frappe.ValidationError): pass @@ -54,13 +55,26 @@ class ECommerceSettings(Document): def validate_search_index_fields(self): if not self.search_index_fields: - return - - # Clean up - fields = self.search_index_fields.replace(' ', '') - fields = fields.strip(',') + return - self.search_index_fields = fields + # Clean up + # Remove whitespaces + fields = self.search_index_fields.replace(' ', '') + # Remove extra ',' and remove duplicates + fields = unique(fields.strip(',').split(',')) + + # All fields should be indexable + if not (set(fields).issubset(ALLOWED_INDEXABLE_FIELDS_SET)): + invalid_fields = list(set(fields).difference(ALLOWED_INDEXABLE_FIELDS_SET)) + num_invalid_fields = len(invalid_fields) + invalid_fields = comma_and(invalid_fields) + + if num_invalid_fields > 1: + frappe.throw(_("{0} are not valid options for Search Index Field.").format(frappe.bold(invalid_fields))) + else: + frappe.throw(_("{0} is not a valid option for Search Index Field.").format(frappe.bold(invalid_fields))) + + self.search_index_fields = ','.join(fields) def validate_exchange_rates_exist(self): """check if exchange rates exist for all Price List currencies (to company's currency)""" @@ -113,6 +127,15 @@ class ECommerceSettings(Document): def get_shipping_rules(self, shipping_territory): return self.get_name_from_territory(shipping_territory, "shipping_rules", "shipping_rule") + def on_change(self): + old_doc = self.get_doc_before_save() + old_fields = old_doc.search_index_fields + new_fields = self.search_index_fields + + # if search index fields get changed + if not (new_fields == old_fields): + create_website_items_index() + def validate_cart_settings(doc, method): frappe.get_doc("E Commerce Settings", "E Commerce Settings").run_method("validate") diff --git a/erpnext/e_commerce/doctype/website_item/website_item.py b/erpnext/e_commerce/doctype/website_item/website_item.py index bb0af5224c..58d4f246c6 100644 --- a/erpnext/e_commerce/doctype/website_item/website_item.py +++ b/erpnext/e_commerce/doctype/website_item/website_item.py @@ -17,7 +17,7 @@ from erpnext.setup.doctype.item_group.item_group import (get_parent_item_groups, from erpnext.e_commerce.doctype.item_review.item_review import get_item_reviews # SEARCH -from erpnext.templates.pages.product_search import ( +from erpnext.e_commerce.website_item_indexing import ( insert_item_to_index, update_index_for_item, delete_item_from_index diff --git a/erpnext/e_commerce/website_item_indexing.py b/erpnext/e_commerce/website_item_indexing.py new file mode 100644 index 0000000000..0e48a2d7e6 --- /dev/null +++ b/erpnext/e_commerce/website_item_indexing.py @@ -0,0 +1,167 @@ +# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# License: GNU General Public License v3. See license.txt + +from __future__ import unicode_literals + +import frappe +import redis + +from redisearch import ( + Client, AutoCompleter, Query, + Suggestion, IndexDefinition, + TextField, TagField, + Document + ) + +# GLOBAL CONSTANTS +WEBSITE_ITEM_INDEX = 'website_items_index' +WEBSITE_ITEM_KEY_PREFIX = 'website_item:' +WEBSITE_ITEM_NAME_AUTOCOMPLETE = 'website_items_name_dict' + +ALLOWED_INDEXABLE_FIELDS_SET = { + 'item_code', + 'item_name', + 'item_group', + 'brand', + 'description', + 'web_long_description' +} + +def create_website_items_index(): + '''Creates Index Definition''' + # CREATE index + client = Client(WEBSITE_ITEM_INDEX, port=13000) + + # DROP if already exists + try: + client.drop_index() + except: + pass + + + idx_def = IndexDefinition([WEBSITE_ITEM_KEY_PREFIX]) + + # Based on e-commerce settings + idx_fields = frappe.db.get_single_value( + 'E Commerce Settings', + 'search_index_fields' + ).split(',') + + if 'web_item_name' in idx_fields: + idx_fields.remove('web_item_name') + + idx_fields = list(map(to_search_field, idx_fields)) + + client.create_index( + [TextField("web_item_name", sortable=True)] + idx_fields, + definition=idx_def + ) + + reindex_all_web_items() + define_autocomplete_dictionary() + +def to_search_field(field): + if field == "tags": + return TagField("tags", separator=",") + + return TextField(field) + +def insert_item_to_index(website_item_doc): + # Insert item to index + key = get_cache_key(website_item_doc.name) + r = redis.Redis("localhost", 13000) + web_item = create_web_item_map(website_item_doc) + r.hset(key, mapping=web_item) + insert_to_name_ac(website_item_doc.web_item_name, website_item_doc.name) + +def insert_to_name_ac(web_name, doc_name): + ac = AutoCompleter(WEBSITE_ITEM_NAME_AUTOCOMPLETE, port=13000) + ac.add_suggestions(Suggestion(web_name, payload=doc_name)) + +def create_web_item_map(website_item_doc): + fields_to_index = get_fields_indexed() + + web_item = {} + + for f in fields_to_index: + web_item[f] = website_item_doc.get(f) or '' + + return web_item + +def update_index_for_item(website_item_doc): + # Reinsert to Cache + insert_item_to_index(website_item_doc) + define_autocomplete_dictionary() + # TODO: Only reindex updated items + create_website_items_index() + +def delete_item_from_index(website_item_doc): + r = redis.Redis("localhost", 13000) + key = get_cache_key(website_item_doc.name) + + try: + r.delete(key) + except: + return False + + # TODO: Also delete autocomplete suggestion + return True + +def define_autocomplete_dictionary(): + print("Defining ac dict...") + # AC for name + # TODO: AC for category + + r = redis.Redis("localhost", 13000) + ac = AutoCompleter(WEBSITE_ITEM_NAME_AUTOCOMPLETE, port=13000) + + try: + r.delete(WEBSITE_ITEM_NAME_AUTOCOMPLETE) + except: + return False + + items = frappe.get_all( + 'Website Item', + fields=['web_item_name'], + filters={"published": True} + ) + + for item in items: + print("adding suggestion: " + item.web_item_name) + ac.add_suggestions(Suggestion(item.web_item_name)) + + return True + +def reindex_all_web_items(): + items = frappe.get_all( + 'Website Item', + fields=get_fields_indexed(), + filters={"published": True} + ) + + r = redis.Redis("localhost", 13000) + for item in items: + web_item = create_web_item_map(item) + key = get_cache_key(item.name) + print(key, web_item) + r.hset(key, mapping=web_item) + +def get_cache_key(name): + name = frappe.scrub(name) + return f"{WEBSITE_ITEM_KEY_PREFIX}{name}" + +def get_fields_indexed(): + fields_to_index = frappe.db.get_single_value( + 'E Commerce Settings', + 'search_index_fields' + ).split(',') + + mandatory_fields = ['name', 'web_item_name', 'route', 'thumbnail'] + fields_to_index = fields_to_index + mandatory_fields + + return fields_to_index + +# TODO: Remove later +# # Figure out a way to run this at startup +define_autocomplete_dictionary() +create_website_items_index() diff --git a/erpnext/templates/pages/product_search.py b/erpnext/templates/pages/product_search.py index 010e7a6832..d935a7a744 100644 --- a/erpnext/templates/pages/product_search.py +++ b/erpnext/templates/pages/product_search.py @@ -1,8 +1,6 @@ -# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors +# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors # License: GNU General Public License v3. See license.txt -from __future__ import unicode_literals - import frappe from frappe.utils import cint, cstr, nowdate @@ -10,17 +8,8 @@ from erpnext.setup.doctype.item_group.item_group import get_item_for_list_in_htm from erpnext.e_commerce.shopping_cart.product_info import set_product_info_for_website # For SEARCH ------- -import redis -from redisearch import ( - Client, AutoCompleter, Query, - Suggestion, IndexDefinition, - TextField, TagField, - Document - ) - -WEBSITE_ITEM_INDEX = 'website_items_index' -WEBSITE_ITEM_KEY_PREFIX = 'website_item:' -WEBSITE_ITEM_NAME_AUTOCOMPLETE = 'website_items_name_dict' +from redisearch import AutoCompleter, Client, Query +from erpnext.e_commerce.website_item_indexing import WEBSITE_ITEM_INDEX, WEBSITE_ITEM_NAME_AUTOCOMPLETE # ----------------- no_cache = 1 @@ -94,111 +83,3 @@ def search(query): def convert_to_dict(redis_search_doc): return redis_search_doc.__dict__ - -def create_website_items_index(): - '''Creates Index Definition''' - # CREATE index - client = Client(WEBSITE_ITEM_INDEX, port=13000) - - # DROP if already exists - try: - client.drop_index() - except: - pass - - - idx_def = IndexDefinition([WEBSITE_ITEM_KEY_PREFIX]) - - client.create_index( - [TextField("web_item_name", sortable=True), TagField("tags")], - definition=idx_def - ) - - reindex_all_web_items() - -def insert_item_to_index(website_item_doc): - # Insert item to index - key = get_cache_key(website_item_doc.name) - r = redis.Redis("localhost", 13000) - web_item = create_web_item_map(website_item_doc) - r.hset(key, mapping=web_item) - insert_to_name_ac(website_item_doc.web_item_name, website_item_doc.name) - -def insert_to_name_ac(web_name, doc_name): - ac = AutoCompleter(WEBSITE_ITEM_NAME_AUTOCOMPLETE, port=13000) - ac.add_suggestions(Suggestion(web_name, payload=doc_name)) - -def create_web_item_map(website_item_doc): - web_item = {} - web_item["web_item_name"] = website_item_doc.web_item_name - web_item["route"] = website_item_doc.route - web_item["thumbnail"] = website_item_doc.thumbnail or '' - web_item["description"] = website_item_doc.description or '' - - return web_item - -def update_index_for_item(website_item_doc): - # Reinsert to Cache - insert_item_to_index(website_item_doc) - define_autocomplete_dictionary() - # TODO: Only reindex updated items - create_website_items_index() - -def delete_item_from_index(website_item_doc): - r = redis.Redis("localhost", 13000) - key = get_cache_key(website_item_doc.name) - - try: - r.delete(key) - except: - return False - - # TODO: Also delete autocomplete suggestion - return True - -def define_autocomplete_dictionary(): - # AC for name - # TODO: AC for category - - r = redis.Redis("localhost", 13000) - ac = AutoCompleter(WEBSITE_ITEM_NAME_AUTOCOMPLETE, port=13000) - - try: - r.delete(WEBSITE_ITEM_NAME_AUTOCOMPLETE) - except: - return False - - items = frappe.get_all( - 'Website Item', - fields=['web_item_name'], - filters={"published": True} - ) - - for item in items: - print("adding suggestion: " + item.web_item_name) - ac.add_suggestions(Suggestion(item.web_item_name)) - - return True - -def reindex_all_web_items(): - items = frappe.get_all( - 'Website Item', - fields=['web_item_name', 'name', 'route', 'thumbnail', 'description'], - filters={"published": True} - ) - - r = redis.Redis("localhost", 13000) - for item in items: - web_item = create_web_item_map(item) - key = get_cache_key(item.name) - print(key, web_item) - r.hset(key, mapping=web_item) - -def get_cache_key(name): - name = frappe.scrub(name) - return f"{WEBSITE_ITEM_KEY_PREFIX}{name}" - -# TODO: Remove later -# Figure out a way to run this at startup -define_autocomplete_dictionary() -create_website_items_index()