Merge pull request #35498 from rohitwaghchaure/fixed-db-binlogs-getting-full

fix: old data reposting causing low server disk space

erpnext/stock/doctype/repost_item_valuation/repost_item_valuation.js

@@ -59,6 +59,7 @@ frappe.ui.form.on('Repost Item Valuation', {
 			if (frm.doc.status == 'In Progress') {
 				frm.doc.current_index = data.current_index;
 				frm.doc.items_to_be_repost = data.items_to_be_repost;
+				frm.doc.total_reposting_count = data.total_reposting_count;
 
 				frm.dashboard.reset();
 				frm.trigger('show_reposting_progress');
@@ -95,6 +96,11 @@ frappe.ui.form.on('Repost Item Valuation', {
 		var bars = [];
 
 		let total_count = frm.doc.items_to_be_repost ? JSON.parse(frm.doc.items_to_be_repost).length : 0;
+
+		if (frm.doc?.total_reposting_count) {
+			total_count = frm.doc.total_reposting_count;
+		}
+
 		let progress = flt(cint(frm.doc.current_index) / total_count * 100, 2) || 0.5;
 		var title = __('Reposting Completed {0}%', [progress]);
 

erpnext/stock/doctype/repost_item_valuation/repost_item_valuation.json

@@ -22,11 +22,15 @@
   "amended_from",
   "error_section",
   "error_log",
+  "reposting_info_section",
+  "reposting_data_file",
   "items_to_be_repost",
-  "affected_transactions",
   "distinct_item_and_warehouse",
+  "column_break_o1sj",
+  "total_reposting_count",
   "current_index",
-  "gl_reposting_index"
+  "gl_reposting_index",
+  "affected_transactions"
  ],
  "fields": [
   {
@@ -191,13 +195,36 @@
    "fieldtype": "Int",
    "hidden": 1,
    "label": "GL reposting index",
+   "no_copy": 1,
+   "read_only": 1
+  },
+  {
+   "fieldname": "reposting_info_section",
+   "fieldtype": "Section Break",
+   "label": "Reposting Info"
+  },
+  {
+   "fieldname": "column_break_o1sj",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "total_reposting_count",
+   "fieldtype": "Int",
+   "label": "Total Reposting Count",
+   "no_copy": 1,
+   "read_only": 1
+  },
+  {
+   "fieldname": "reposting_data_file",
+   "fieldtype": "Attach",
+   "label": "Reposting Data File",
    "read_only": 1
   }
  ],
  "index_web_pages_for_search": 1,
  "is_submittable": 1,
  "links": [],
- "modified": "2022-11-28 16:00:05.637440",
+ "modified": "2023-05-31 12:48:57.138693",
  "modified_by": "Administrator",
  "module": "Stock",
  "name": "Repost Item Valuation",

erpnext/stock/doctype/repost_item_valuation/repost_item_valuation.py

@@ -3,6 +3,7 @@
 
 import frappe
 from frappe import _
+from frappe.desk.form.load import get_attachments
 from frappe.exceptions import QueryDeadlockError, QueryTimeoutError
 from frappe.model.document import Document
 from frappe.query_builder import DocType, Interval
@@ -95,6 +96,12 @@ class RepostItemValuation(Document):
 
 		self.allow_negative_stock = 1
 
+	def on_cancel(self):
+		self.clear_attachment()
+
+	def on_trash(self):
+		self.clear_attachment()
+
 	def set_company(self):
 		if self.based_on == "Transaction":
 			self.company = frappe.get_cached_value(self.voucher_type, self.voucher_no, "company")
@@ -110,6 +117,14 @@ class RepostItemValuation(Document):
 		if write:
 			self.db_set("status", self.status)
 
+	def clear_attachment(self):
+		if attachments := get_attachments(self.doctype, self.name):
+			attachment = attachments[0]
+			frappe.delete_doc("File", attachment.name)
+
+		if self.reposting_data_file:
+			self.db_set("reposting_data_file", None)
+
 	def on_submit(self):
 		"""During tests reposts are executed immediately.
 

erpnext/stock/stock_ledger.py

@@ -6,10 +6,21 @@ import json
 from typing import Optional, Set, Tuple
 
 import frappe
-from frappe import _
+from frappe import _, scrub
 from frappe.model.meta import get_field_precision
 from frappe.query_builder.functions import CombineDatetime, Sum
-from frappe.utils import cint, cstr, flt, get_link_to_form, getdate, now, nowdate
+from frappe.utils import (
+	cint,
+	cstr,
+	flt,
+	get_link_to_form,
+	getdate,
+	gzip_compress,
+	gzip_decompress,
+	now,
+	nowdate,
+	parse_json,
+)
 
 import erpnext
 from erpnext.stock.doctype.bin.bin import update_qty as update_bin_qty
@@ -214,14 +225,18 @@ def repost_future_sle(
 	if not args:
 		args = []  # set args to empty list if None to avoid enumerate error
 
+	reposting_data = {}
+	if doc and doc.reposting_data_file:
+		reposting_data = get_reposting_data(doc.reposting_data_file)
+
 	items_to_be_repost = get_items_to_be_repost(
-		voucher_type=voucher_type, voucher_no=voucher_no, doc=doc
+		voucher_type=voucher_type, voucher_no=voucher_no, doc=doc, reposting_data=reposting_data
 	)
 	if items_to_be_repost:
 		args = items_to_be_repost
 
-	distinct_item_warehouses = get_distinct_item_warehouse(args, doc)
+	distinct_item_warehouses = get_distinct_item_warehouse(args, doc, reposting_data=reposting_data)
-	affected_transactions = get_affected_transactions(doc)
+	affected_transactions = get_affected_transactions(doc, reposting_data=reposting_data)
 
 	i = get_current_index(doc) or 0
 	while i < len(args):
@@ -264,6 +279,28 @@ def repost_future_sle(
 			)
 
 
+def get_reposting_data(file_path) -> dict:
+	file_name = frappe.db.get_value(
+		"File",
+		{
+			"file_url": file_path,
+			"attached_to_field": "reposting_data_file",
+		},
+		"name",
+	)
+
+	if not file_name:
+		return frappe._dict()
+
+	attached_file = frappe.get_doc("File", file_name)
+
+	data = gzip_decompress(attached_file.get_content())
+	if data := json.loads(data.decode("utf-8")):
+		data = data
+
+	return parse_json(data)
+
+
 def validate_item_warehouse(args):
 	for field in ["item_code", "warehouse", "posting_date", "posting_time"]:
 		if args.get(field) in [None, ""]:
@@ -274,6 +311,31 @@ def validate_item_warehouse(args):
 def update_args_in_repost_item_valuation(
 	doc, index, args, distinct_item_warehouses, affected_transactions
 ):
+	if not doc.items_to_be_repost:
+		file_name = ""
+		if doc.reposting_data_file:
+			file_name = get_reposting_file_name(doc.doctype, doc.name)
+			# frappe.delete_doc("File", file_name, ignore_permissions=True, delete_permanently=True)
+
+		doc.reposting_data_file = create_json_gz_file(
+			{
+				"items_to_be_repost": args,
+				"distinct_item_and_warehouse": {str(k): v for k, v in distinct_item_warehouses.items()},
+				"affected_transactions": affected_transactions,
+			},
+			doc,
+			file_name,
+		)
+
+		doc.db_set(
+			{
+				"current_index": index,
+				"total_reposting_count": len(args),
+				"reposting_data_file": doc.reposting_data_file,
+			}
+		)
+
+	else:
 		doc.db_set(
 			{
 				"items_to_be_repost": json.dumps(args, default=str),
@@ -290,12 +352,66 @@ def update_args_in_repost_item_valuation(
 
 	frappe.publish_realtime(
 		"item_reposting_progress",
-		{"name": doc.name, "items_to_be_repost": json.dumps(args, default=str), "current_index": index},
+		{
+			"name": doc.name,
+			"items_to_be_repost": json.dumps(args, default=str),
+			"current_index": index,
+			"total_reposting_count": len(args),
+		},
 	)
 
 
-def get_items_to_be_repost(voucher_type=None, voucher_no=None, doc=None):
+def get_reposting_file_name(dt, dn):
+	return frappe.db.get_value(
+		"File",
+		{
+			"attached_to_doctype": dt,
+			"attached_to_name": dn,
+			"attached_to_field": "reposting_data_file",
+		},
+		"name",
+	)
+
+
+def create_json_gz_file(data, doc, file_name=None) -> str:
+	encoded_content = frappe.safe_encode(frappe.as_json(data))
+	compressed_content = gzip_compress(encoded_content)
+
+	if not file_name:
+		json_filename = f"{scrub(doc.doctype)}-{scrub(doc.name)}.json.gz"
+		_file = frappe.get_doc(
+			{
+				"doctype": "File",
+				"file_name": json_filename,
+				"attached_to_doctype": doc.doctype,
+				"attached_to_name": doc.name,
+				"attached_to_field": "reposting_data_file",
+				"content": compressed_content,
+				"is_private": 1,
+			}
+		)
+		_file.save(ignore_permissions=True)
+
+		return _file.file_url
+	else:
+		file_doc = frappe.get_doc("File", file_name)
+		path = file_doc.get_full_path()
+
+		with open(path, "wb") as f:
+			f.write(compressed_content)
+
+		return doc.reposting_data_file
+
+
+def get_items_to_be_repost(voucher_type=None, voucher_no=None, doc=None, reposting_data=None):
+	if not reposting_data and doc and doc.reposting_data_file:
+		reposting_data = get_reposting_data(doc.reposting_data_file)
+
+	if reposting_data and reposting_data.items_to_be_repost:
+		return reposting_data.items_to_be_repost
+
 	items_to_be_repost = []
+
 	if doc and doc.items_to_be_repost:
 		items_to_be_repost = json.loads(doc.items_to_be_repost) or []
 
@@ -311,8 +427,15 @@ def get_items_to_be_repost(voucher_type=None, voucher_no=None, doc=None):
 	return items_to_be_repost or []
 
 
-def get_distinct_item_warehouse(args=None, doc=None):
+def get_distinct_item_warehouse(args=None, doc=None, reposting_data=None):
+	if not reposting_data and doc and doc.reposting_data_file:
+		reposting_data = get_reposting_data(doc.reposting_data_file)
+
+	if reposting_data and reposting_data.distinct_item_and_warehouse:
+		return reposting_data.distinct_item_and_warehouse
+
 	distinct_item_warehouses = {}
+
 	if doc and doc.distinct_item_and_warehouse:
 		distinct_item_warehouses = json.loads(doc.distinct_item_and_warehouse)
 		distinct_item_warehouses = {
@@ -327,7 +450,13 @@ def get_distinct_item_warehouse(args=None, doc=None):
 	return distinct_item_warehouses
 
 
-def get_affected_transactions(doc) -> Set[Tuple[str, str]]:
+def get_affected_transactions(doc, reposting_data=None) -> Set[Tuple[str, str]]:
+	if not reposting_data and doc and doc.reposting_data_file:
+		reposting_data = get_reposting_data(doc.reposting_data_file)
+
+	if reposting_data and reposting_data.affected_transactions:
+		return {tuple(transaction) for transaction in reposting_data.affected_transactions}
+
 	if not doc.affected_transactions:
 		return set()