From 4a38ce659d4da7400fd219060cbcdd77ce6ccf1b Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Mon, 22 Aug 2022 00:23:22 +0530 Subject: [PATCH] refactor!: drop redisearch incr: replace text and tag fields incr: use rediswrapper's make key incr: indexDefinition from redis incr: replace index creation incr: replace AutoCompleter incr: replace product search ac incr: replace client querying fix: broken redisearch load test fix: pass actual query to get suggestion --- erpnext/e_commerce/redisearch_utils.py | 58 ++++++++++++----------- erpnext/templates/pages/product_search.py | 21 ++++---- pyproject.toml | 1 - 3 files changed, 41 insertions(+), 39 deletions(-) diff --git a/erpnext/e_commerce/redisearch_utils.py b/erpnext/e_commerce/redisearch_utils.py index 1f649c7b48..87ca9bd83d 100644 --- a/erpnext/e_commerce/redisearch_utils.py +++ b/erpnext/e_commerce/redisearch_utils.py @@ -7,7 +7,9 @@ import frappe from frappe import _ from frappe.utils.redis_wrapper import RedisWrapper from redis import ResponseError -from redisearch import AutoCompleter, Client, IndexDefinition, Suggestion, TagField, TextField +from redis.commands.search.field import TagField, TextField +from redis.commands.search.indexDefinition import IndexDefinition +from redis.commands.search.suggestion import Suggestion WEBSITE_ITEM_INDEX = "website_items_index" WEBSITE_ITEM_KEY_PREFIX = "website_item:" @@ -35,12 +37,9 @@ def is_redisearch_enabled(): def is_search_module_loaded(): try: cache = frappe.cache() - out = cache.execute_command("MODULE LIST") - - parsed_output = " ".join( - (" ".join([frappe.as_unicode(s) for s in o if not isinstance(s, int)]) for o in out) - ) - return "search" in parsed_output + for module in cache.module_list(): + if module.get(b"name") == b"search": + return True except Exception: return False # handling older redis versions @@ -58,18 +57,18 @@ def if_redisearch_enabled(function): def make_key(key): - return "{0}|{1}".format(frappe.conf.db_name, key).encode("utf-8") + return frappe.cache().make_key(key) @if_redisearch_enabled def create_website_items_index(): "Creates Index Definition." - # CREATE index - client = Client(make_key(WEBSITE_ITEM_INDEX), conn=frappe.cache()) + redis = frappe.cache() + index = redis.ft(WEBSITE_ITEM_INDEX) try: - client.drop_index() # drop if already exists + index.dropindex() # drop if already exists except ResponseError: # will most likely raise a ResponseError if index does not exist # ignore and create index @@ -86,9 +85,10 @@ def create_website_items_index(): if "web_item_name" in idx_fields: idx_fields.remove("web_item_name") - idx_fields = list(map(to_search_field, idx_fields)) + idx_fields = [to_search_field(f) for f in idx_fields] - client.create_index( + # TODO: sortable? + index.create_index( [TextField("web_item_name", sortable=True)] + idx_fields, definition=idx_def, ) @@ -119,8 +119,8 @@ def insert_item_to_index(website_item_doc): @if_redisearch_enabled def insert_to_name_ac(web_name, doc_name): - ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=frappe.cache()) - ac.add_suggestions(Suggestion(web_name, payload=doc_name)) + ac = frappe.cache().ft() + ac.sugadd(WEBSITE_ITEM_NAME_AUTOCOMPLETE, Suggestion(web_name, payload=doc_name)) def create_web_item_map(website_item_doc): @@ -157,9 +157,8 @@ def delete_item_from_index(website_item_doc): @if_redisearch_enabled def delete_from_ac_dict(website_item_doc): """Removes this items's name from autocomplete dictionary""" - cache = frappe.cache() - name_ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=cache) - name_ac.delete(website_item_doc.web_item_name) + ac = frappe.cache().ft() + ac.sugdel(website_item_doc.web_item_name) @if_redisearch_enabled @@ -170,8 +169,6 @@ def define_autocomplete_dictionary(): """ cache = frappe.cache() - item_ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=cache) - item_group_ac = AutoCompleter(make_key(WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE), conn=cache) # Delete both autocomplete dicts try: @@ -180,38 +177,43 @@ def define_autocomplete_dictionary(): except Exception: raise_redisearch_error() - create_items_autocomplete_dict(autocompleter=item_ac) - create_item_groups_autocomplete_dict(autocompleter=item_group_ac) + create_items_autocomplete_dict() + create_item_groups_autocomplete_dict() @if_redisearch_enabled -def create_items_autocomplete_dict(autocompleter): +def create_items_autocomplete_dict(): "Add items as suggestions in Autocompleter." + + ac = frappe.cache().ft() items = frappe.get_all( "Website Item", fields=["web_item_name", "item_group"], filters={"published": 1} ) - for item in items: - autocompleter.add_suggestions(Suggestion(item.web_item_name)) + ac.sugadd(WEBSITE_ITEM_NAME_AUTOCOMPLETE, Suggestion(item.web_item_name)) @if_redisearch_enabled -def create_item_groups_autocomplete_dict(autocompleter): +def create_item_groups_autocomplete_dict(): "Add item groups with weightage as suggestions in Autocompleter." + published_item_groups = frappe.get_all( "Item Group", fields=["name", "route", "weightage"], filters={"show_in_website": 1} ) if not published_item_groups: return + ac = frappe.cache().ft() + for item_group in published_item_groups: payload = json.dumps({"name": item_group.name, "route": item_group.route}) - autocompleter.add_suggestions( + ac.sugadd( + WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE, Suggestion( string=item_group.name, score=frappe.utils.flt(item_group.weightage) or 1.0, payload=payload, # additional info that can be retrieved later - ) + ), ) diff --git a/erpnext/templates/pages/product_search.py b/erpnext/templates/pages/product_search.py index 0768cc3fa6..f40fd479f4 100644 --- a/erpnext/templates/pages/product_search.py +++ b/erpnext/templates/pages/product_search.py @@ -5,14 +5,13 @@ import json import frappe from frappe.utils import cint, cstr -from redisearch import AutoCompleter, Client, Query +from redis.commands.search.query import Query from erpnext.e_commerce.redisearch_utils import ( WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE, WEBSITE_ITEM_INDEX, WEBSITE_ITEM_NAME_AUTOCOMPLETE, is_redisearch_enabled, - make_key, ) from erpnext.e_commerce.shopping_cart.product_info import set_product_info_for_website from erpnext.setup.doctype.item_group.item_group import get_item_for_list_in_html @@ -88,15 +87,17 @@ def product_search(query, limit=10, fuzzy_search=True): if not query: return search_results - red = frappe.cache() + redis = frappe.cache() query = clean_up_query(query) # TODO: Check perf/correctness with Suggestions & Query vs only Query # TODO: Use Levenshtein Distance in Query (max=3) - ac = AutoCompleter(make_key(WEBSITE_ITEM_NAME_AUTOCOMPLETE), conn=red) - client = Client(make_key(WEBSITE_ITEM_INDEX), conn=red) - suggestions = ac.get_suggestions( - query, num=limit, fuzzy=fuzzy_search and len(query) > 3 # Fuzzy on length < 3 can be real slow + redisearch = redis.ft(WEBSITE_ITEM_INDEX) + suggestions = redisearch.sugget( + WEBSITE_ITEM_NAME_AUTOCOMPLETE, + query, + num=limit, + fuzzy=fuzzy_search and len(query) > 3, ) # Build a query @@ -106,8 +107,8 @@ def product_search(query, limit=10, fuzzy_search=True): query_string += f"|('{clean_up_query(s.string)}')" q = Query(query_string) + results = redisearch.search(q) - results = client.search(q) search_results["results"] = list(map(convert_to_dict, results.docs)) search_results["results"] = sorted( search_results["results"], key=lambda k: frappe.utils.cint(k["ranking"]), reverse=True @@ -141,8 +142,8 @@ def get_category_suggestions(query): if not query: return search_results - ac = AutoCompleter(make_key(WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE), conn=frappe.cache()) - suggestions = ac.get_suggestions(query, num=10, with_payloads=True) + ac = frappe.cache().ft() + suggestions = ac.sugget(WEBSITE_ITEM_CATEGORY_AUTOCOMPLETE, query, num=10, with_payloads=True) results = [json.loads(s.payload) for s in suggestions] diff --git a/pyproject.toml b/pyproject.toml index 5acfd39272..14684f3491 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,6 @@ dependencies = [ "pycountry~=20.7.3", "python-stdnum~=1.16", "Unidecode~=1.2.0", - "redisearch~=2.1.0", # integration dependencies "gocardless-pro~=1.22.0",