From fb1a40cadaea2152fcce165861464be85c6e32ab Mon Sep 17 00:00:00 2001 From: Rohit Waghchaure Date: Wed, 31 May 2023 14:11:15 +0530 Subject: [PATCH] fix: old data reposting causing low server disk space --- .../repost_item_valuation.js | 6 + .../repost_item_valuation.json | 33 +++- .../repost_item_valuation.py | 15 ++ erpnext/stock/stock_ledger.py | 167 ++++++++++++++++-- 4 files changed, 199 insertions(+), 22 deletions(-) diff --git a/erpnext/stock/doctype/repost_item_valuation/repost_item_valuation.js b/erpnext/stock/doctype/repost_item_valuation/repost_item_valuation.js index 8aec532847..40748ce3f5 100644 --- a/erpnext/stock/doctype/repost_item_valuation/repost_item_valuation.js +++ b/erpnext/stock/doctype/repost_item_valuation/repost_item_valuation.js @@ -59,6 +59,7 @@ frappe.ui.form.on('Repost Item Valuation', { if (frm.doc.status == 'In Progress') { frm.doc.current_index = data.current_index; frm.doc.items_to_be_repost = data.items_to_be_repost; + frm.doc.total_reposting_count = data.total_reposting_count; frm.dashboard.reset(); frm.trigger('show_reposting_progress'); @@ -95,6 +96,11 @@ frappe.ui.form.on('Repost Item Valuation', { var bars = []; let total_count = frm.doc.items_to_be_repost ? JSON.parse(frm.doc.items_to_be_repost).length : 0; + + if (frm.doc?.total_reposting_count) { + total_count = frm.doc.total_reposting_count; + } + let progress = flt(cint(frm.doc.current_index) / total_count * 100, 2) || 0.5; var title = __('Reposting Completed {0}%', [progress]); diff --git a/erpnext/stock/doctype/repost_item_valuation/repost_item_valuation.json b/erpnext/stock/doctype/repost_item_valuation/repost_item_valuation.json index 8a5309c348..1c5b521c29 100644 --- a/erpnext/stock/doctype/repost_item_valuation/repost_item_valuation.json +++ b/erpnext/stock/doctype/repost_item_valuation/repost_item_valuation.json @@ -22,11 +22,15 @@ "amended_from", "error_section", "error_log", + "reposting_info_section", + "reposting_data_file", "items_to_be_repost", - "affected_transactions", "distinct_item_and_warehouse", + "column_break_o1sj", + "total_reposting_count", "current_index", - "gl_reposting_index" + "gl_reposting_index", + "affected_transactions" ], "fields": [ { @@ -191,13 +195,36 @@ "fieldtype": "Int", "hidden": 1, "label": "GL reposting index", + "no_copy": 1, + "read_only": 1 + }, + { + "fieldname": "reposting_info_section", + "fieldtype": "Section Break", + "label": "Reposting Info" + }, + { + "fieldname": "column_break_o1sj", + "fieldtype": "Column Break" + }, + { + "fieldname": "total_reposting_count", + "fieldtype": "Int", + "label": "Total Reposting Count", + "no_copy": 1, + "read_only": 1 + }, + { + "fieldname": "reposting_data_file", + "fieldtype": "Attach", + "label": "Reposting Data File", "read_only": 1 } ], "index_web_pages_for_search": 1, "is_submittable": 1, "links": [], - "modified": "2022-11-28 16:00:05.637440", + "modified": "2023-05-31 12:48:57.138693", "modified_by": "Administrator", "module": "Stock", "name": "Repost Item Valuation", diff --git a/erpnext/stock/doctype/repost_item_valuation/repost_item_valuation.py b/erpnext/stock/doctype/repost_item_valuation/repost_item_valuation.py index d3bcab76ab..d5fc710625 100644 --- a/erpnext/stock/doctype/repost_item_valuation/repost_item_valuation.py +++ b/erpnext/stock/doctype/repost_item_valuation/repost_item_valuation.py @@ -3,6 +3,7 @@ import frappe from frappe import _ +from frappe.desk.form.load import get_attachments from frappe.exceptions import QueryDeadlockError, QueryTimeoutError from frappe.model.document import Document from frappe.query_builder import DocType, Interval @@ -95,6 +96,12 @@ class RepostItemValuation(Document): self.allow_negative_stock = 1 + def on_cancel(self): + self.clear_attachment() + + def on_trash(self): + self.clear_attachment() + def set_company(self): if self.based_on == "Transaction": self.company = frappe.get_cached_value(self.voucher_type, self.voucher_no, "company") @@ -110,6 +117,14 @@ class RepostItemValuation(Document): if write: self.db_set("status", self.status) + def clear_attachment(self): + if attachments := get_attachments(self.doctype, self.name): + attachment = attachments[0] + frappe.delete_doc("File", attachment.name) + + if self.reposting_data_file: + self.db_set("reposting_data_file", None) + def on_submit(self): """During tests reposts are executed immediately. diff --git a/erpnext/stock/stock_ledger.py b/erpnext/stock/stock_ledger.py index 6106809273..12b9641755 100644 --- a/erpnext/stock/stock_ledger.py +++ b/erpnext/stock/stock_ledger.py @@ -6,10 +6,21 @@ import json from typing import Optional, Set, Tuple import frappe -from frappe import _ +from frappe import _, scrub from frappe.model.meta import get_field_precision from frappe.query_builder.functions import CombineDatetime, Sum -from frappe.utils import cint, cstr, flt, get_link_to_form, getdate, now, nowdate +from frappe.utils import ( + cint, + cstr, + flt, + get_link_to_form, + getdate, + gzip_compress, + gzip_decompress, + now, + nowdate, + parse_json, +) import erpnext from erpnext.stock.doctype.bin.bin import update_qty as update_bin_qty @@ -211,14 +222,18 @@ def repost_future_sle( if not args: args = [] # set args to empty list if None to avoid enumerate error + reposting_data = {} + if doc and doc.reposting_data_file: + reposting_data = get_reposting_data(doc.reposting_data_file) + items_to_be_repost = get_items_to_be_repost( - voucher_type=voucher_type, voucher_no=voucher_no, doc=doc + voucher_type=voucher_type, voucher_no=voucher_no, doc=doc, reposting_data=reposting_data ) if items_to_be_repost: args = items_to_be_repost - distinct_item_warehouses = get_distinct_item_warehouse(args, doc) - affected_transactions = get_affected_transactions(doc) + distinct_item_warehouses = get_distinct_item_warehouse(args, doc, reposting_data=reposting_data) + affected_transactions = get_affected_transactions(doc, reposting_data=reposting_data) i = get_current_index(doc) or 0 while i < len(args): @@ -261,6 +276,28 @@ def repost_future_sle( ) +def get_reposting_data(file_path) -> dict: + file_name = frappe.db.get_value( + "File", + { + "file_url": file_path, + "attached_to_field": "reposting_data_file", + }, + "name", + ) + + if not file_name: + return frappe._dict() + + attached_file = frappe.get_doc("File", file_name) + + data = gzip_decompress(attached_file.get_content()) + if data := json.loads(data.decode("utf-8")): + data = data + + return parse_json(data) + + def validate_item_warehouse(args): for field in ["item_code", "warehouse", "posting_date", "posting_time"]: if args.get(field) in [None, ""]: @@ -271,28 +308,107 @@ def validate_item_warehouse(args): def update_args_in_repost_item_valuation( doc, index, args, distinct_item_warehouses, affected_transactions ): - doc.db_set( - { - "items_to_be_repost": json.dumps(args, default=str), - "distinct_item_and_warehouse": json.dumps( - {str(k): v for k, v in distinct_item_warehouses.items()}, default=str - ), - "current_index": index, - "affected_transactions": frappe.as_json(affected_transactions), - } - ) + if not doc.items_to_be_repost: + file_name = "" + if doc.reposting_data_file: + file_name = get_reposting_file_name(doc.doctype, doc.name) + # frappe.delete_doc("File", file_name, ignore_permissions=True, delete_permanently=True) + + doc.reposting_data_file = create_json_gz_file( + { + "items_to_be_repost": args, + "distinct_item_and_warehouse": {str(k): v for k, v in distinct_item_warehouses.items()}, + "affected_transactions": affected_transactions, + }, + doc, + file_name, + ) + + doc.db_set( + { + "current_index": index, + "total_reposting_count": len(args), + "reposting_data_file": doc.reposting_data_file, + } + ) + + else: + doc.db_set( + { + "items_to_be_repost": json.dumps(args, default=str), + "distinct_item_and_warehouse": json.dumps( + {str(k): v for k, v in distinct_item_warehouses.items()}, default=str + ), + "current_index": index, + "affected_transactions": frappe.as_json(affected_transactions), + } + ) if not frappe.flags.in_test: frappe.db.commit() frappe.publish_realtime( "item_reposting_progress", - {"name": doc.name, "items_to_be_repost": json.dumps(args, default=str), "current_index": index}, + { + "name": doc.name, + "items_to_be_repost": json.dumps(args, default=str), + "current_index": index, + "total_reposting_count": len(args), + }, ) -def get_items_to_be_repost(voucher_type=None, voucher_no=None, doc=None): +def get_reposting_file_name(dt, dn): + return frappe.db.get_value( + "File", + { + "attached_to_doctype": dt, + "attached_to_name": dn, + "attached_to_field": "reposting_data_file", + }, + "name", + ) + + +def create_json_gz_file(data, doc, file_name=None) -> str: + encoded_content = frappe.safe_encode(frappe.as_json(data)) + compressed_content = gzip_compress(encoded_content) + + if not file_name: + json_filename = f"{scrub(doc.doctype)}-{scrub(doc.name)}.json.gz" + _file = frappe.get_doc( + { + "doctype": "File", + "file_name": json_filename, + "attached_to_doctype": doc.doctype, + "attached_to_name": doc.name, + "attached_to_field": "reposting_data_file", + "content": compressed_content, + "is_private": 1, + } + ) + _file.save(ignore_permissions=True) + + return _file.file_url + else: + file_doc = frappe.get_doc("File", file_name) + path = file_doc.get_full_path() + + with open(path, "wb") as f: + f.write(compressed_content) + + return doc.reposting_data_file + + +def get_items_to_be_repost(voucher_type=None, voucher_no=None, doc=None, reposting_data=None): + if not reposting_data and doc and doc.reposting_data_file: + reposting_data = get_reposting_data(doc.reposting_data_file) + + if reposting_data and reposting_data.items_to_be_repost: + return reposting_data.items_to_be_repost + items_to_be_repost = [] + if doc and doc.items_to_be_repost: items_to_be_repost = json.loads(doc.items_to_be_repost) or [] @@ -308,8 +424,15 @@ def get_items_to_be_repost(voucher_type=None, voucher_no=None, doc=None): return items_to_be_repost or [] -def get_distinct_item_warehouse(args=None, doc=None): +def get_distinct_item_warehouse(args=None, doc=None, reposting_data=None): + if not reposting_data and doc and doc.reposting_data_file: + reposting_data = get_reposting_data(doc.reposting_data_file) + + if reposting_data and reposting_data.distinct_item_and_warehouse: + return reposting_data.distinct_item_and_warehouse + distinct_item_warehouses = {} + if doc and doc.distinct_item_and_warehouse: distinct_item_warehouses = json.loads(doc.distinct_item_and_warehouse) distinct_item_warehouses = { @@ -324,7 +447,13 @@ def get_distinct_item_warehouse(args=None, doc=None): return distinct_item_warehouses -def get_affected_transactions(doc) -> Set[Tuple[str, str]]: +def get_affected_transactions(doc, reposting_data=None) -> Set[Tuple[str, str]]: + if not reposting_data and doc and doc.reposting_data_file: + reposting_data = get_reposting_data(doc.reposting_data_file) + + if reposting_data and reposting_data.affected_transactions: + return {tuple(transaction) for transaction in reposting_data.affected_transactions} + if not doc.affected_transactions: return set()