feat: Make search index fields configurable
- Move indexing logic to separate file - Add more validation logic for 'search index fields' field
This commit is contained in:
Hussain Nagaria
marination
parent
3160187825
commit
8e55c95ecc
@ -6,8 +6,9 @@ import frappe
|
||||
from frappe.utils import cint, comma_and
|
||||
from frappe import _, msgprint
|
||||
from frappe.model.document import Document
|
||||
from frappe.utils import cint
|
||||
from frappe.utils import get_datetime, get_datetime_str, now_datetime
|
||||
from frappe.utils import get_datetime, get_datetime_str, now_datetime, unique
|
||||
|
||||
from erpnext.e_commerce.website_item_indexing import create_website_items_index, ALLOWED_INDEXABLE_FIELDS_SET
|
||||
|
||||
class ShoppingCartSetupError(frappe.ValidationError): pass
|
||||
|
||||
@ -54,13 +55,26 @@ class ECommerceSettings(Document):
|
||||
|
||||
def validate_search_index_fields(self):
|
||||
if not self.search_index_fields:
|
||||
return
|
||||
|
||||
# Clean up
|
||||
fields = self.search_index_fields.replace(' ', '')
|
||||
fields = fields.strip(',')
|
||||
return
|
||||
|
||||
self.search_index_fields = fields
|
||||
# Clean up
|
||||
# Remove whitespaces
|
||||
fields = self.search_index_fields.replace(' ', '')
|
||||
# Remove extra ',' and remove duplicates
|
||||
fields = unique(fields.strip(',').split(','))
|
||||
|
||||
# All fields should be indexable
|
||||
if not (set(fields).issubset(ALLOWED_INDEXABLE_FIELDS_SET)):
|
||||
invalid_fields = list(set(fields).difference(ALLOWED_INDEXABLE_FIELDS_SET))
|
||||
num_invalid_fields = len(invalid_fields)
|
||||
invalid_fields = comma_and(invalid_fields)
|
||||
|
||||
if num_invalid_fields > 1:
|
||||
frappe.throw(_("{0} are not valid options for Search Index Field.").format(frappe.bold(invalid_fields)))
|
||||
else:
|
||||
frappe.throw(_("{0} is not a valid option for Search Index Field.").format(frappe.bold(invalid_fields)))
|
||||
|
||||
self.search_index_fields = ','.join(fields)
|
||||
|
||||
def validate_exchange_rates_exist(self):
|
||||
"""check if exchange rates exist for all Price List currencies (to company's currency)"""
|
||||
@ -113,6 +127,15 @@ class ECommerceSettings(Document):
|
||||
def get_shipping_rules(self, shipping_territory):
|
||||
return self.get_name_from_territory(shipping_territory, "shipping_rules", "shipping_rule")
|
||||
|
||||
def on_change(self):
|
||||
old_doc = self.get_doc_before_save()
|
||||
old_fields = old_doc.search_index_fields
|
||||
new_fields = self.search_index_fields
|
||||
|
||||
# if search index fields get changed
|
||||
if not (new_fields == old_fields):
|
||||
create_website_items_index()
|
||||
|
||||
def validate_cart_settings(doc, method):
|
||||
frappe.get_doc("E Commerce Settings", "E Commerce Settings").run_method("validate")
|
||||
|
||||
|
||||
@ -17,7 +17,7 @@ from erpnext.setup.doctype.item_group.item_group import (get_parent_item_groups,
|
||||
from erpnext.e_commerce.doctype.item_review.item_review import get_item_reviews
|
||||
|
||||
# SEARCH
|
||||
from erpnext.templates.pages.product_search import (
|
||||
from erpnext.e_commerce.website_item_indexing import (
|
||||
insert_item_to_index,
|
||||
update_index_for_item,
|
||||
delete_item_from_index
|
||||
|
||||
167
erpnext/e_commerce/website_item_indexing.py
Normal file
167
erpnext/e_commerce/website_item_indexing.py
Normal file
@ -0,0 +1,167 @@
|
||||
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
|
||||
# License: GNU General Public License v3. See license.txt
|
||||
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import frappe
|
||||
import redis
|
||||
|
||||
from redisearch import (
|
||||
Client, AutoCompleter, Query,
|
||||
Suggestion, IndexDefinition,
|
||||
TextField, TagField,
|
||||
Document
|
||||
)
|
||||
|
||||
# GLOBAL CONSTANTS
|
||||
WEBSITE_ITEM_INDEX = 'website_items_index'
|
||||
WEBSITE_ITEM_KEY_PREFIX = 'website_item:'
|
||||
WEBSITE_ITEM_NAME_AUTOCOMPLETE = 'website_items_name_dict'
|
||||
|
||||
ALLOWED_INDEXABLE_FIELDS_SET = {
|
||||
'item_code',
|
||||
'item_name',
|
||||
'item_group',
|
||||
'brand',
|
||||
'description',
|
||||
'web_long_description'
|
||||
}
|
||||
|
||||
def create_website_items_index():
|
||||
'''Creates Index Definition'''
|
||||
# CREATE index
|
||||
client = Client(WEBSITE_ITEM_INDEX, port=13000)
|
||||
|
||||
# DROP if already exists
|
||||
try:
|
||||
client.drop_index()
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
idx_def = IndexDefinition([WEBSITE_ITEM_KEY_PREFIX])
|
||||
|
||||
# Based on e-commerce settings
|
||||
idx_fields = frappe.db.get_single_value(
|
||||
'E Commerce Settings',
|
||||
'search_index_fields'
|
||||
).split(',')
|
||||
|
||||
if 'web_item_name' in idx_fields:
|
||||
idx_fields.remove('web_item_name')
|
||||
|
||||
idx_fields = list(map(to_search_field, idx_fields))
|
||||
|
||||
client.create_index(
|
||||
[TextField("web_item_name", sortable=True)] + idx_fields,
|
||||
definition=idx_def
|
||||
)
|
||||
|
||||
reindex_all_web_items()
|
||||
define_autocomplete_dictionary()
|
||||
|
||||
def to_search_field(field):
|
||||
if field == "tags":
|
||||
return TagField("tags", separator=",")
|
||||
|
||||
return TextField(field)
|
||||
|
||||
def insert_item_to_index(website_item_doc):
|
||||
# Insert item to index
|
||||
key = get_cache_key(website_item_doc.name)
|
||||
r = redis.Redis("localhost", 13000)
|
||||
web_item = create_web_item_map(website_item_doc)
|
||||
r.hset(key, mapping=web_item)
|
||||
insert_to_name_ac(website_item_doc.web_item_name, website_item_doc.name)
|
||||
|
||||
def insert_to_name_ac(web_name, doc_name):
|
||||
ac = AutoCompleter(WEBSITE_ITEM_NAME_AUTOCOMPLETE, port=13000)
|
||||
ac.add_suggestions(Suggestion(web_name, payload=doc_name))
|
||||
|
||||
def create_web_item_map(website_item_doc):
|
||||
fields_to_index = get_fields_indexed()
|
||||
|
||||
web_item = {}
|
||||
|
||||
for f in fields_to_index:
|
||||
web_item[f] = website_item_doc.get(f) or ''
|
||||
|
||||
return web_item
|
||||
|
||||
def update_index_for_item(website_item_doc):
|
||||
# Reinsert to Cache
|
||||
insert_item_to_index(website_item_doc)
|
||||
define_autocomplete_dictionary()
|
||||
# TODO: Only reindex updated items
|
||||
create_website_items_index()
|
||||
|
||||
def delete_item_from_index(website_item_doc):
|
||||
r = redis.Redis("localhost", 13000)
|
||||
key = get_cache_key(website_item_doc.name)
|
||||
|
||||
try:
|
||||
r.delete(key)
|
||||
except:
|
||||
return False
|
||||
|
||||
# TODO: Also delete autocomplete suggestion
|
||||
return True
|
||||
|
||||
def define_autocomplete_dictionary():
|
||||
print("Defining ac dict...")
|
||||
# AC for name
|
||||
# TODO: AC for category
|
||||
|
||||
r = redis.Redis("localhost", 13000)
|
||||
ac = AutoCompleter(WEBSITE_ITEM_NAME_AUTOCOMPLETE, port=13000)
|
||||
|
||||
try:
|
||||
r.delete(WEBSITE_ITEM_NAME_AUTOCOMPLETE)
|
||||
except:
|
||||
return False
|
||||
|
||||
items = frappe.get_all(
|
||||
'Website Item',
|
||||
fields=['web_item_name'],
|
||||
filters={"published": True}
|
||||
)
|
||||
|
||||
for item in items:
|
||||
print("adding suggestion: " + item.web_item_name)
|
||||
ac.add_suggestions(Suggestion(item.web_item_name))
|
||||
|
||||
return True
|
||||
|
||||
def reindex_all_web_items():
|
||||
items = frappe.get_all(
|
||||
'Website Item',
|
||||
fields=get_fields_indexed(),
|
||||
filters={"published": True}
|
||||
)
|
||||
|
||||
r = redis.Redis("localhost", 13000)
|
||||
for item in items:
|
||||
web_item = create_web_item_map(item)
|
||||
key = get_cache_key(item.name)
|
||||
print(key, web_item)
|
||||
r.hset(key, mapping=web_item)
|
||||
|
||||
def get_cache_key(name):
|
||||
name = frappe.scrub(name)
|
||||
return f"{WEBSITE_ITEM_KEY_PREFIX}{name}"
|
||||
|
||||
def get_fields_indexed():
|
||||
fields_to_index = frappe.db.get_single_value(
|
||||
'E Commerce Settings',
|
||||
'search_index_fields'
|
||||
).split(',')
|
||||
|
||||
mandatory_fields = ['name', 'web_item_name', 'route', 'thumbnail']
|
||||
fields_to_index = fields_to_index + mandatory_fields
|
||||
|
||||
return fields_to_index
|
||||
|
||||
# TODO: Remove later
|
||||
# # Figure out a way to run this at startup
|
||||
define_autocomplete_dictionary()
|
||||
create_website_items_index()
|
||||
@ -1,8 +1,6 @@
|
||||
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
|
||||
# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
|
||||
# License: GNU General Public License v3. See license.txt
|
||||
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import frappe
|
||||
from frappe.utils import cint, cstr, nowdate
|
||||
|
||||
@ -10,17 +8,8 @@ from erpnext.setup.doctype.item_group.item_group import get_item_for_list_in_htm
|
||||
from erpnext.e_commerce.shopping_cart.product_info import set_product_info_for_website
|
||||
|
||||
# For SEARCH -------
|
||||
import redis
|
||||
from redisearch import (
|
||||
Client, AutoCompleter, Query,
|
||||
Suggestion, IndexDefinition,
|
||||
TextField, TagField,
|
||||
Document
|
||||
)
|
||||
|
||||
WEBSITE_ITEM_INDEX = 'website_items_index'
|
||||
WEBSITE_ITEM_KEY_PREFIX = 'website_item:'
|
||||
WEBSITE_ITEM_NAME_AUTOCOMPLETE = 'website_items_name_dict'
|
||||
from redisearch import AutoCompleter, Client, Query
|
||||
from erpnext.e_commerce.website_item_indexing import WEBSITE_ITEM_INDEX, WEBSITE_ITEM_NAME_AUTOCOMPLETE
|
||||
# -----------------
|
||||
|
||||
no_cache = 1
|
||||
@ -94,111 +83,3 @@ def search(query):
|
||||
|
||||
def convert_to_dict(redis_search_doc):
|
||||
return redis_search_doc.__dict__
|
||||
|
||||
def create_website_items_index():
|
||||
'''Creates Index Definition'''
|
||||
# CREATE index
|
||||
client = Client(WEBSITE_ITEM_INDEX, port=13000)
|
||||
|
||||
# DROP if already exists
|
||||
try:
|
||||
client.drop_index()
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
idx_def = IndexDefinition([WEBSITE_ITEM_KEY_PREFIX])
|
||||
|
||||
client.create_index(
|
||||
[TextField("web_item_name", sortable=True), TagField("tags")],
|
||||
definition=idx_def
|
||||
)
|
||||
|
||||
reindex_all_web_items()
|
||||
|
||||
def insert_item_to_index(website_item_doc):
|
||||
# Insert item to index
|
||||
key = get_cache_key(website_item_doc.name)
|
||||
r = redis.Redis("localhost", 13000)
|
||||
web_item = create_web_item_map(website_item_doc)
|
||||
r.hset(key, mapping=web_item)
|
||||
insert_to_name_ac(website_item_doc.web_item_name, website_item_doc.name)
|
||||
|
||||
def insert_to_name_ac(web_name, doc_name):
|
||||
ac = AutoCompleter(WEBSITE_ITEM_NAME_AUTOCOMPLETE, port=13000)
|
||||
ac.add_suggestions(Suggestion(web_name, payload=doc_name))
|
||||
|
||||
def create_web_item_map(website_item_doc):
|
||||
web_item = {}
|
||||
web_item["web_item_name"] = website_item_doc.web_item_name
|
||||
web_item["route"] = website_item_doc.route
|
||||
web_item["thumbnail"] = website_item_doc.thumbnail or ''
|
||||
web_item["description"] = website_item_doc.description or ''
|
||||
|
||||
return web_item
|
||||
|
||||
def update_index_for_item(website_item_doc):
|
||||
# Reinsert to Cache
|
||||
insert_item_to_index(website_item_doc)
|
||||
define_autocomplete_dictionary()
|
||||
# TODO: Only reindex updated items
|
||||
create_website_items_index()
|
||||
|
||||
def delete_item_from_index(website_item_doc):
|
||||
r = redis.Redis("localhost", 13000)
|
||||
key = get_cache_key(website_item_doc.name)
|
||||
|
||||
try:
|
||||
r.delete(key)
|
||||
except:
|
||||
return False
|
||||
|
||||
# TODO: Also delete autocomplete suggestion
|
||||
return True
|
||||
|
||||
def define_autocomplete_dictionary():
|
||||
# AC for name
|
||||
# TODO: AC for category
|
||||
|
||||
r = redis.Redis("localhost", 13000)
|
||||
ac = AutoCompleter(WEBSITE_ITEM_NAME_AUTOCOMPLETE, port=13000)
|
||||
|
||||
try:
|
||||
r.delete(WEBSITE_ITEM_NAME_AUTOCOMPLETE)
|
||||
except:
|
||||
return False
|
||||
|
||||
items = frappe.get_all(
|
||||
'Website Item',
|
||||
fields=['web_item_name'],
|
||||
filters={"published": True}
|
||||
)
|
||||
|
||||
for item in items:
|
||||
print("adding suggestion: " + item.web_item_name)
|
||||
ac.add_suggestions(Suggestion(item.web_item_name))
|
||||
|
||||
return True
|
||||
|
||||
def reindex_all_web_items():
|
||||
items = frappe.get_all(
|
||||
'Website Item',
|
||||
fields=['web_item_name', 'name', 'route', 'thumbnail', 'description'],
|
||||
filters={"published": True}
|
||||
)
|
||||
|
||||
r = redis.Redis("localhost", 13000)
|
||||
for item in items:
|
||||
web_item = create_web_item_map(item)
|
||||
key = get_cache_key(item.name)
|
||||
print(key, web_item)
|
||||
r.hset(key, mapping=web_item)
|
||||
|
||||
def get_cache_key(name):
|
||||
name = frappe.scrub(name)
|
||||
return f"{WEBSITE_ITEM_KEY_PREFIX}{name}"
|
||||
|
||||
# TODO: Remove later
|
||||
# Figure out a way to run this at startup
|
||||
define_autocomplete_dictionary()
|
||||
create_website_items_index()
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user