Shiloh/brotherton-erpnext
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #30522 from marination/redisearch-app-install

Browse Source 
feat: Redisearch with consent
This commit is contained in:
Marica 2022-04-04 19:24:06 +05:30 committed by GitHub
commit 3c3aca9568
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 122 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
erpnext/e_commerce/doctype/e_commerce_settings/e_commerce_settings.json
View File

@ -47,7 +47,7 @@
"item_search_settings_section", "item_search_settings_section",
"redisearch_warning", "redisearch_warning",
"search_index_fields", "search_index_fields",
"show_categories_in_search_autocomplete", "is_redisearch_enabled",
"is_redisearch_loaded", "is_redisearch_loaded",
"shop_by_category_section", "shop_by_category_section",
"slideshow", "slideshow",
@ -293,6 +293,7 @@
"fieldname": "search_index_fields", "fieldname": "search_index_fields",
"fieldtype": "Small Text", "fieldtype": "Small Text",
"label": "Search Index Fields", "label": "Search Index Fields",
"mandatory_depends_on": "is_redisearch_enabled",
"read_only_depends_on": "eval:!doc.is_redisearch_loaded" "read_only_depends_on": "eval:!doc.is_redisearch_loaded"
}, },
{ {
@ -301,13 +302,6 @@
"fieldtype": "Section Break", "fieldtype": "Section Break",
"label": "Item Search Settings" "label": "Item Search Settings"
}, },
{
"default": "1",
"fieldname": "show_categories_in_search_autocomplete",
"fieldtype": "Check",
"label": "Show Categories in Search Autocomplete",
"read_only_depends_on": "eval:!doc.is_redisearch_loaded"
},
{ {
"default": "0", "default": "0",
"fieldname": "is_redisearch_loaded", "fieldname": "is_redisearch_loaded",
@ -365,12 +359,19 @@
"fieldname": "show_price_in_quotation", "fieldname": "show_price_in_quotation",
"fieldtype": "Check", "fieldtype": "Check",
"label": "Show Price in Quotation" "label": "Show Price in Quotation"
},
{
"default": "0",
"fieldname": "is_redisearch_enabled",
"fieldtype": "Check",
"label": "Enable Redisearch",
"read_only_depends_on": "eval:!doc.is_redisearch_loaded"
} }
], ],
"index_web_pages_for_search": 1, "index_web_pages_for_search": 1,
"issingle": 1, "issingle": 1,
"links": [], "links": [],
"modified": "2021-09-02 14:02:44.785824", "modified": "2022-04-01 18:35:56.106756",
"modified_by": "Administrator", "modified_by": "Administrator",
"module": "E-commerce", "module": "E-commerce",
"name": "E Commerce Settings", "name": "E Commerce Settings",
@ -389,5 +390,6 @@
], ],
"sort_field": "modified", "sort_field": "modified",
"sort_order": "DESC", "sort_order": "DESC",
"states": [],
"track_changes": 1 "track_changes": 1
} }

17
erpnext/e_commerce/doctype/e_commerce_settings/e_commerce_settings.py
View File

@ -9,6 +9,7 @@ from frappe.utils import comma_and, flt, unique
from erpnext.e_commerce.redisearch_utils import ( from erpnext.e_commerce.redisearch_utils import (
create_website_items_index, create_website_items_index,
define_autocomplete_dictionary,
get_indexable_web_fields, get_indexable_web_fields,
is_search_module_loaded, is_search_module_loaded,
) )
@ -21,6 +22,8 @@ class ShoppingCartSetupError(frappe.ValidationError):
class ECommerceSettings(Document): class ECommerceSettings(Document):
def onload(self): def onload(self):
self.get("__onload").quotation_series = frappe.get_meta("Quotation").get_options("naming_series") self.get("__onload").quotation_series = frappe.get_meta("Quotation").get_options("naming_series")
# flag >> if redisearch is installed and loaded
self.is_redisearch_loaded = is_search_module_loaded() self.is_redisearch_loaded = is_search_module_loaded()
def validate(self): def validate(self):
@ -34,6 +37,20 @@ class ECommerceSettings(Document):
frappe.clear_document_cache("E Commerce Settings", "E Commerce Settings") frappe.clear_document_cache("E Commerce Settings", "E Commerce Settings")
self.is_redisearch_enabled_pre_save = frappe.db.get_single_value(
"E Commerce Settings", "is_redisearch_enabled"
)
def after_save(self):
self.create_redisearch_indexes()
def create_redisearch_indexes(self):
# if redisearch is enabled (value changed) create indexes and dictionary
value_changed = self.is_redisearch_enabled != self.is_redisearch_enabled_pre_save
if self.is_redisearch_loaded and self.is_redisearch_enabled and value_changed:
define_autocomplete_dictionary()
create_website_items_index()
def validate_field_filters(self): def validate_field_filters(self):
if not (self.enable_field_filters and self.filter_fields): if not (self.enable_field_filters and self.filter_fields):
return return

124
erpnext/e_commerce/redisearch_utils.py
View File

@ -1,8 +1,12 @@
# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors # Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
# License: GNU General Public License v3. See license.txt # License: GNU General Public License v3. See license.txt
import json
import frappe import frappe
from frappe import _
from frappe.utils.redis_wrapper import RedisWrapper from frappe.utils.redis_wrapper import RedisWrapper
from redis import ResponseError
from redisearch import AutoCompleter, Client, IndexDefinition, Suggestion, TagField, TextField from redisearch import AutoCompleter, Client, IndexDefinition, Suggestion, TagField, TextField
WEBSITE_ITEM_INDEX = "website_items_index" WEBSITE_ITEM_INDEX = "website_items_index"
@ -22,6 +26,12 @@ def get_indexable_web_fields():
return [df.fieldname for df in valid_fields] return [df.fieldname for df in valid_fields]
def is_redisearch_enabled():
"Return True only if redisearch is loaded and enabled."
is_redisearch_enabled = frappe.db.get_single_value("E Commerce Settings", "is_redisearch_enabled")
return is_search_module_loaded() and is_redisearch_enabled
def is_search_module_loaded(): def is_search_module_loaded():
try: try:
cache = frappe.cache() cache = frappe.cache()
@ -32,14 +42,14 @@ def is_search_module_loaded():
) )
return "search" in parsed_output return "search" in parsed_output
except Exception: except Exception:
return False return False # handling older redis versions
def if_redisearch_loaded(function): def if_redisearch_enabled(function):
"Decorator to check if Redisearch is loaded." "Decorator to check if Redisearch is enabled."
def wrapper(*args, **kwargs): def wrapper(*args, **kwargs):
if is_search_module_loaded(): if is_redisearch_enabled():
func = function(*args, **kwargs) func = function(*args, **kwargs)
return func return func
return return
@ -51,22 +61,25 @@ def make_key(key):
return "{0}|{1}".format(frappe.conf.db_name, key).encode("utf-8") return "{0}|{1}".format(frappe.conf.db_name, key).encode("utf-8")
@if_redisearch_loaded @if_redisearch_enabled
def create_website_items_index(): def create_website_items_index():
"Creates Index Definition." "Creates Index Definition."
# CREATE index # CREATE index
client = Client(make_key(WEBSITE_ITEM_INDEX), conn=frappe.cache()) client = Client(make_key(WEBSITE_ITEM_INDEX), conn=frappe.cache())
# DROP if already exists
try: try:
client.drop_index() client.drop_index() # drop if already exists
except Exception: except ResponseError:
# will most likely raise a ResponseError if index does not exist
# ignore and create index
pass pass
except Exception:
raise_redisearch_error()
idx_def = IndexDefinition([make_key(WEBSITE_ITEM_KEY_PREFIX)]) idx_def = IndexDefinition([make_key(WEBSITE_ITEM_KEY_PREFIX)])
# Based on e-commerce settings # Index fields mentioned in e-commerce settings
idx_fields = frappe.db.get_single_value("E Commerce Settings", "search_index_fields") idx_fields = frappe.db.get_single_value("E Commerce Settings", "search_index_fields")
idx_fields = idx_fields.split(",") if idx_fields else [] idx_fields = idx_fields.split(",") if idx_fields else []
@ -91,20 +104,20 @@ def to_search_field(field):
return TextField(field) return TextField(field)
@if_redisearch_loaded @if_redisearch_enabled
def insert_item_to_index(website_item_doc): def insert_item_to_index(website_item_doc):
# Insert item to index # Insert item to index
key = get_cache_key(website_item_doc.name) key = get_cache_key(website_item_doc.name)
cache = frappe.cache() cache = frappe.cache()
web_item = create_web_item_map(website_item_doc) web_item = create_web_item_map(website_item_doc)
for k, v in web_item.items(): for field, value in web_item.items():
super(RedisWrapper, cache).hset(make_key(key), k, v) super(RedisWrapper, cache).hset(make_key(key), field, value)
insert_to_name_ac(website_item_doc.web_item_name, website_item_doc.name) insert_to_name_ac(website_item_doc.web_item_name, website_item_doc.name)
@if_redisearch_loaded @if_redisearch_enabled
def insert_to_name_ac(web_name, doc_name): def insert_to_name_ac(web_name, doc_name):
ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=frappe.cache()) ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=frappe.cache())
ac.add_suggestions(Suggestion(web_name, payload=doc_name)) ac.add_suggestions(Suggestion(web_name, payload=doc_name))
@ -114,20 +127,20 @@ def create_web_item_map(website_item_doc):
fields_to_index = get_fields_indexed() fields_to_index = get_fields_indexed()
web_item = {} web_item = {}
for f in fields_to_index: for field in fields_to_index:
web_item[f] = website_item_doc.get(f) or "" web_item[field] = website_item_doc.get(field) or ""
return web_item return web_item
@if_redisearch_loaded @if_redisearch_enabled
def update_index_for_item(website_item_doc): def update_index_for_item(website_item_doc):
# Reinsert to Cache # Reinsert to Cache
insert_item_to_index(website_item_doc) insert_item_to_index(website_item_doc)
define_autocomplete_dictionary() define_autocomplete_dictionary()
@if_redisearch_loaded @if_redisearch_enabled
def delete_item_from_index(website_item_doc): def delete_item_from_index(website_item_doc):
cache = frappe.cache() cache = frappe.cache()
key = get_cache_key(website_item_doc.name) key = get_cache_key(website_item_doc.name)
@ -135,13 +148,13 @@ def delete_item_from_index(website_item_doc):
try: try:
cache.delete(key) cache.delete(key)
except Exception: except Exception:
return False raise_redisearch_error()
delete_from_ac_dict(website_item_doc) delete_from_ac_dict(website_item_doc)
return True return True
@if_redisearch_loaded @if_redisearch_enabled
def delete_from_ac_dict(website_item_doc): def delete_from_ac_dict(website_item_doc):
"""Removes this items's name from autocomplete dictionary""" """Removes this items's name from autocomplete dictionary"""
cache = frappe.cache() cache = frappe.cache()
@ -149,40 +162,60 @@ def delete_from_ac_dict(website_item_doc):
name_ac.delete(website_item_doc.web_item_name) name_ac.delete(website_item_doc.web_item_name)
@if_redisearch_loaded @if_redisearch_enabled
def define_autocomplete_dictionary(): def define_autocomplete_dictionary():
"""Creates an autocomplete search dictionary for `name`. """
Also creats autocomplete dictionary for `categories` if Defines/Redefines an autocomplete search dictionary for Website Item Name.
checked in E Commerce Settings""" Also creats autocomplete dictionary for Published Item Groups.
"""
cache = frappe.cache() cache = frappe.cache()
name_ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=cache) item_ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=cache)
cat_ac = AutoCompleter(make_key(WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE), conn=cache) item_group_ac = AutoCompleter(make_key(WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE), conn=cache)
ac_categories = frappe.db.get_single_value(
"E Commerce Settings", "show_categories_in_search_autocomplete"
)
# Delete both autocomplete dicts # Delete both autocomplete dicts
try: try:
cache.delete(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE)) cache.delete(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE))
cache.delete(make_key(WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE)) cache.delete(make_key(WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE))
except Exception: except Exception:
return False raise_redisearch_error()
create_items_autocomplete_dict(autocompleter=item_ac)
create_item_groups_autocomplete_dict(autocompleter=item_group_ac)
@if_redisearch_enabled
def create_items_autocomplete_dict(autocompleter):
"Add items as suggestions in Autocompleter."
items = frappe.get_all( items = frappe.get_all(
"Website Item", fields=["web_item_name", "item_group"], filters={"published": 1} "Website Item", fields=["web_item_name", "item_group"], filters={"published": 1}
) )
for item in items: for item in items:
name_ac.add_suggestions(Suggestion(item.web_item_name)) autocompleter.add_suggestions(Suggestion(item.web_item_name))
if ac_categories and item.item_group:
cat_ac.add_suggestions(Suggestion(item.item_group))
return True
@if_redisearch_loaded @if_redisearch_enabled
def create_item_groups_autocomplete_dict(autocompleter):
"Add item groups with weightage as suggestions in Autocompleter."
published_item_groups = frappe.get_all(
"Item Group", fields=["name", "route", "weightage"], filters={"show_in_website": 1}
)
if not published_item_groups:
return
for item_group in published_item_groups:
payload = json.dumps({"name": item_group.name, "route": item_group.route})
autocompleter.add_suggestions(
Suggestion(
string=item_group.name,
score=frappe.utils.flt(item_group.weightage) or 1.0,
payload=payload, # additional info that can be retrieved later
)
)
@if_redisearch_enabled
def reindex_all_web_items(): def reindex_all_web_items():
items = frappe.get_all("Website Item", fields=get_fields_indexed(), filters={"published": True}) items = frappe.get_all("Website Item", fields=get_fields_indexed(), filters={"published": True})
@ -191,8 +224,8 @@ def reindex_all_web_items():
web_item = create_web_item_map(item) web_item = create_web_item_map(item)
key = make_key(get_cache_key(item.name)) key = make_key(get_cache_key(item.name))
for k, v in web_item.items(): for field, value in web_item.items():
super(RedisWrapper, cache).hset(key, k, v) super(RedisWrapper, cache).hset(key, field, value)
def get_cache_key(name): def get_cache_key(name):
@ -210,7 +243,12 @@ def get_fields_indexed():
return fields_to_index return fields_to_index
# TODO: Remove later def raise_redisearch_error():
# # Figure out a way to run this at startup "Create an Error Log and raise error."
define_autocomplete_dictionary() traceback = frappe.get_traceback()
create_website_items_index() log = frappe.log_error(traceback, frappe._("Redisearch Error"))
log_link = frappe.utils.get_link_to_form("Error Log", log.name)
frappe.throw(
msg=_("Something went wrong. Check {0}").format(log_link), title=_("Redisearch Error")
)

20
erpnext/templates/pages/product_search.py
View File

@ -1,6 +1,8 @@
# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors # Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors
# License: GNU General Public License v3. See license.txt # License: GNU General Public License v3. See license.txt
import json
import frappe import frappe
from frappe.utils import cint, cstr from frappe.utils import cint, cstr
from redisearch import AutoCompleter, Client, Query from redisearch import AutoCompleter, Client, Query
@ -9,7 +11,7 @@ from erpnext.e_commerce.redisearch_utils import (
WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE, WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE,
WEBSITE_ITEM_INDEX, WEBSITE_ITEM_INDEX,
WEBSITE_ITEM_NAME_AUTOCOMPLETE, WEBSITE_ITEM_NAME_AUTOCOMPLETE,
is_search_module_loaded, is_redisearch_enabled,
make_key, make_key,
) )
from erpnext.e_commerce.shopping_cart.product_info import set_product_info_for_website from erpnext.e_commerce.shopping_cart.product_info import set_product_info_for_website
@ -74,8 +76,8 @@ def search(query):
def product_search(query, limit=10, fuzzy_search=True): def product_search(query, limit=10, fuzzy_search=True):
search_results = {"from_redisearch": True, "results": []} search_results = {"from_redisearch": True, "results": []}
if not is_search_module_loaded(): if not is_redisearch_enabled():
# Redisearch module not loaded # Redisearch module not enabled
search_results["from_redisearch"] = False search_results["from_redisearch"] = False
search_results["results"] = get_product_data(query, 0, limit) search_results["results"] = get_product_data(query, 0, limit)
return search_results return search_results
@ -86,6 +88,8 @@ def product_search(query, limit=10, fuzzy_search=True):
red = frappe.cache() red = frappe.cache()
query = clean_up_query(query) query = clean_up_query(query)
# TODO: Check perf/correctness with Suggestions & Query vs only Query
# TODO: Use Levenshtein Distance in Query (max=3)
ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=red) ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=red)
client = Client(make_key(WEBSITE_ITEM_INDEX), conn=red) client = Client(make_key(WEBSITE_ITEM_INDEX), conn=red)
suggestions = ac.get_suggestions( suggestions = ac.get_suggestions(
@ -121,8 +125,8 @@ def convert_to_dict(redis_search_doc):
def get_category_suggestions(query): def get_category_suggestions(query):
search_results = {"results": []} search_results = {"results": []}
if not is_search_module_loaded(): if not is_redisearch_enabled():
# Redisearch module not loaded, query db # Redisearch module not enabled, query db
categories = frappe.db.get_all( categories = frappe.db.get_all(
"Item Group", "Item Group",
filters={"name": ["like", "%{0}%".format(query)], "show_in_website": 1}, filters={"name": ["like", "%{0}%".format(query)], "show_in_website": 1},
@ -135,8 +139,10 @@ def get_category_suggestions(query):
return search_results return search_results
ac = AutoCompleter(make_key(WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE), conn=frappe.cache()) ac = AutoCompleter(make_key(WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE), conn=frappe.cache())
suggestions = ac.get_suggestions(query, num=10) suggestions = ac.get_suggestions(query, num=10, with_payloads=True)
search_results["results"] = [s.string for s in suggestions] results = [json.loads(s.payload) for s in suggestions]
search_results["results"] = results
return search_results return search_results