fix: old data reposting causing low server disk space
This commit is contained in:
Rohit Waghchaure
parent
ec5d34117b
commit
fb1a40cada
@ -59,6 +59,7 @@ frappe.ui.form.on('Repost Item Valuation', {
|
||||
if (frm.doc.status == 'In Progress') {
|
||||
frm.doc.current_index = data.current_index;
|
||||
frm.doc.items_to_be_repost = data.items_to_be_repost;
|
||||
frm.doc.total_reposting_count = data.total_reposting_count;
|
||||
|
||||
frm.dashboard.reset();
|
||||
frm.trigger('show_reposting_progress');
|
||||
@ -95,6 +96,11 @@ frappe.ui.form.on('Repost Item Valuation', {
|
||||
var bars = [];
|
||||
|
||||
let total_count = frm.doc.items_to_be_repost ? JSON.parse(frm.doc.items_to_be_repost).length : 0;
|
||||
|
||||
if (frm.doc?.total_reposting_count) {
|
||||
total_count = frm.doc.total_reposting_count;
|
||||
}
|
||||
|
||||
let progress = flt(cint(frm.doc.current_index) / total_count * 100, 2) || 0.5;
|
||||
var title = __('Reposting Completed {0}%', [progress]);
|
||||
|
||||
|
||||
@ -22,11 +22,15 @@
|
||||
"amended_from",
|
||||
"error_section",
|
||||
"error_log",
|
||||
"reposting_info_section",
|
||||
"reposting_data_file",
|
||||
"items_to_be_repost",
|
||||
"affected_transactions",
|
||||
"distinct_item_and_warehouse",
|
||||
"column_break_o1sj",
|
||||
"total_reposting_count",
|
||||
"current_index",
|
||||
"gl_reposting_index"
|
||||
"gl_reposting_index",
|
||||
"affected_transactions"
|
||||
],
|
||||
"fields": [
|
||||
{
|
||||
@ -191,13 +195,36 @@
|
||||
"fieldtype": "Int",
|
||||
"hidden": 1,
|
||||
"label": "GL reposting index",
|
||||
"no_copy": 1,
|
||||
"read_only": 1
|
||||
},
|
||||
{
|
||||
"fieldname": "reposting_info_section",
|
||||
"fieldtype": "Section Break",
|
||||
"label": "Reposting Info"
|
||||
},
|
||||
{
|
||||
"fieldname": "column_break_o1sj",
|
||||
"fieldtype": "Column Break"
|
||||
},
|
||||
{
|
||||
"fieldname": "total_reposting_count",
|
||||
"fieldtype": "Int",
|
||||
"label": "Total Reposting Count",
|
||||
"no_copy": 1,
|
||||
"read_only": 1
|
||||
},
|
||||
{
|
||||
"fieldname": "reposting_data_file",
|
||||
"fieldtype": "Attach",
|
||||
"label": "Reposting Data File",
|
||||
"read_only": 1
|
||||
}
|
||||
],
|
||||
"index_web_pages_for_search": 1,
|
||||
"is_submittable": 1,
|
||||
"links": [],
|
||||
"modified": "2022-11-28 16:00:05.637440",
|
||||
"modified": "2023-05-31 12:48:57.138693",
|
||||
"modified_by": "Administrator",
|
||||
"module": "Stock",
|
||||
"name": "Repost Item Valuation",
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
|
||||
import frappe
|
||||
from frappe import _
|
||||
from frappe.desk.form.load import get_attachments
|
||||
from frappe.exceptions import QueryDeadlockError, QueryTimeoutError
|
||||
from frappe.model.document import Document
|
||||
from frappe.query_builder import DocType, Interval
|
||||
@ -95,6 +96,12 @@ class RepostItemValuation(Document):
|
||||
|
||||
self.allow_negative_stock = 1
|
||||
|
||||
def on_cancel(self):
|
||||
self.clear_attachment()
|
||||
|
||||
def on_trash(self):
|
||||
self.clear_attachment()
|
||||
|
||||
def set_company(self):
|
||||
if self.based_on == "Transaction":
|
||||
self.company = frappe.get_cached_value(self.voucher_type, self.voucher_no, "company")
|
||||
@ -110,6 +117,14 @@ class RepostItemValuation(Document):
|
||||
if write:
|
||||
self.db_set("status", self.status)
|
||||
|
||||
def clear_attachment(self):
|
||||
if attachments := get_attachments(self.doctype, self.name):
|
||||
attachment = attachments[0]
|
||||
frappe.delete_doc("File", attachment.name)
|
||||
|
||||
if self.reposting_data_file:
|
||||
self.db_set("reposting_data_file", None)
|
||||
|
||||
def on_submit(self):
|
||||
"""During tests reposts are executed immediately.
|
||||
|
||||
|
||||
@ -6,10 +6,21 @@ import json
|
||||
from typing import Optional, Set, Tuple
|
||||
|
||||
import frappe
|
||||
from frappe import _
|
||||
from frappe import _, scrub
|
||||
from frappe.model.meta import get_field_precision
|
||||
from frappe.query_builder.functions import CombineDatetime, Sum
|
||||
from frappe.utils import cint, cstr, flt, get_link_to_form, getdate, now, nowdate
|
||||
from frappe.utils import (
|
||||
cint,
|
||||
cstr,
|
||||
flt,
|
||||
get_link_to_form,
|
||||
getdate,
|
||||
gzip_compress,
|
||||
gzip_decompress,
|
||||
now,
|
||||
nowdate,
|
||||
parse_json,
|
||||
)
|
||||
|
||||
import erpnext
|
||||
from erpnext.stock.doctype.bin.bin import update_qty as update_bin_qty
|
||||
@ -211,14 +222,18 @@ def repost_future_sle(
|
||||
if not args:
|
||||
args = [] # set args to empty list if None to avoid enumerate error
|
||||
|
||||
reposting_data = {}
|
||||
if doc and doc.reposting_data_file:
|
||||
reposting_data = get_reposting_data(doc.reposting_data_file)
|
||||
|
||||
items_to_be_repost = get_items_to_be_repost(
|
||||
voucher_type=voucher_type, voucher_no=voucher_no, doc=doc
|
||||
voucher_type=voucher_type, voucher_no=voucher_no, doc=doc, reposting_data=reposting_data
|
||||
)
|
||||
if items_to_be_repost:
|
||||
args = items_to_be_repost
|
||||
|
||||
distinct_item_warehouses = get_distinct_item_warehouse(args, doc)
|
||||
affected_transactions = get_affected_transactions(doc)
|
||||
distinct_item_warehouses = get_distinct_item_warehouse(args, doc, reposting_data=reposting_data)
|
||||
affected_transactions = get_affected_transactions(doc, reposting_data=reposting_data)
|
||||
|
||||
i = get_current_index(doc) or 0
|
||||
while i < len(args):
|
||||
@ -261,6 +276,28 @@ def repost_future_sle(
|
||||
)
|
||||
|
||||
|
||||
def get_reposting_data(file_path) -> dict:
|
||||
file_name = frappe.db.get_value(
|
||||
"File",
|
||||
{
|
||||
"file_url": file_path,
|
||||
"attached_to_field": "reposting_data_file",
|
||||
},
|
||||
"name",
|
||||
)
|
||||
|
||||
if not file_name:
|
||||
return frappe._dict()
|
||||
|
||||
attached_file = frappe.get_doc("File", file_name)
|
||||
|
||||
data = gzip_decompress(attached_file.get_content())
|
||||
if data := json.loads(data.decode("utf-8")):
|
||||
data = data
|
||||
|
||||
return parse_json(data)
|
||||
|
||||
|
||||
def validate_item_warehouse(args):
|
||||
for field in ["item_code", "warehouse", "posting_date", "posting_time"]:
|
||||
if args.get(field) in [None, ""]:
|
||||
@ -271,28 +308,107 @@ def validate_item_warehouse(args):
|
||||
def update_args_in_repost_item_valuation(
|
||||
doc, index, args, distinct_item_warehouses, affected_transactions
|
||||
):
|
||||
doc.db_set(
|
||||
{
|
||||
"items_to_be_repost": json.dumps(args, default=str),
|
||||
"distinct_item_and_warehouse": json.dumps(
|
||||
{str(k): v for k, v in distinct_item_warehouses.items()}, default=str
|
||||
),
|
||||
"current_index": index,
|
||||
"affected_transactions": frappe.as_json(affected_transactions),
|
||||
}
|
||||
)
|
||||
if not doc.items_to_be_repost:
|
||||
file_name = ""
|
||||
if doc.reposting_data_file:
|
||||
file_name = get_reposting_file_name(doc.doctype, doc.name)
|
||||
# frappe.delete_doc("File", file_name, ignore_permissions=True, delete_permanently=True)
|
||||
|
||||
doc.reposting_data_file = create_json_gz_file(
|
||||
{
|
||||
"items_to_be_repost": args,
|
||||
"distinct_item_and_warehouse": {str(k): v for k, v in distinct_item_warehouses.items()},
|
||||
"affected_transactions": affected_transactions,
|
||||
},
|
||||
doc,
|
||||
file_name,
|
||||
)
|
||||
|
||||
doc.db_set(
|
||||
{
|
||||
"current_index": index,
|
||||
"total_reposting_count": len(args),
|
||||
"reposting_data_file": doc.reposting_data_file,
|
||||
}
|
||||
)
|
||||
|
||||
else:
|
||||
doc.db_set(
|
||||
{
|
||||
"items_to_be_repost": json.dumps(args, default=str),
|
||||
"distinct_item_and_warehouse": json.dumps(
|
||||
{str(k): v for k, v in distinct_item_warehouses.items()}, default=str
|
||||
),
|
||||
"current_index": index,
|
||||
"affected_transactions": frappe.as_json(affected_transactions),
|
||||
}
|
||||
)
|
||||
|
||||
if not frappe.flags.in_test:
|
||||
frappe.db.commit()
|
||||
|
||||
frappe.publish_realtime(
|
||||
"item_reposting_progress",
|
||||
{"name": doc.name, "items_to_be_repost": json.dumps(args, default=str), "current_index": index},
|
||||
{
|
||||
"name": doc.name,
|
||||
"items_to_be_repost": json.dumps(args, default=str),
|
||||
"current_index": index,
|
||||
"total_reposting_count": len(args),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def get_items_to_be_repost(voucher_type=None, voucher_no=None, doc=None):
|
||||
def get_reposting_file_name(dt, dn):
|
||||
return frappe.db.get_value(
|
||||
"File",
|
||||
{
|
||||
"attached_to_doctype": dt,
|
||||
"attached_to_name": dn,
|
||||
"attached_to_field": "reposting_data_file",
|
||||
},
|
||||
"name",
|
||||
)
|
||||
|
||||
|
||||
def create_json_gz_file(data, doc, file_name=None) -> str:
|
||||
encoded_content = frappe.safe_encode(frappe.as_json(data))
|
||||
compressed_content = gzip_compress(encoded_content)
|
||||
|
||||
if not file_name:
|
||||
json_filename = f"{scrub(doc.doctype)}-{scrub(doc.name)}.json.gz"
|
||||
_file = frappe.get_doc(
|
||||
{
|
||||
"doctype": "File",
|
||||
"file_name": json_filename,
|
||||
"attached_to_doctype": doc.doctype,
|
||||
"attached_to_name": doc.name,
|
||||
"attached_to_field": "reposting_data_file",
|
||||
"content": compressed_content,
|
||||
"is_private": 1,
|
||||
}
|
||||
)
|
||||
_file.save(ignore_permissions=True)
|
||||
|
||||
return _file.file_url
|
||||
else:
|
||||
file_doc = frappe.get_doc("File", file_name)
|
||||
path = file_doc.get_full_path()
|
||||
|
||||
with open(path, "wb") as f:
|
||||
f.write(compressed_content)
|
||||
|
||||
return doc.reposting_data_file
|
||||
|
||||
|
||||
def get_items_to_be_repost(voucher_type=None, voucher_no=None, doc=None, reposting_data=None):
|
||||
if not reposting_data and doc and doc.reposting_data_file:
|
||||
reposting_data = get_reposting_data(doc.reposting_data_file)
|
||||
|
||||
if reposting_data and reposting_data.items_to_be_repost:
|
||||
return reposting_data.items_to_be_repost
|
||||
|
||||
items_to_be_repost = []
|
||||
|
||||
if doc and doc.items_to_be_repost:
|
||||
items_to_be_repost = json.loads(doc.items_to_be_repost) or []
|
||||
|
||||
@ -308,8 +424,15 @@ def get_items_to_be_repost(voucher_type=None, voucher_no=None, doc=None):
|
||||
return items_to_be_repost or []
|
||||
|
||||
|
||||
def get_distinct_item_warehouse(args=None, doc=None):
|
||||
def get_distinct_item_warehouse(args=None, doc=None, reposting_data=None):
|
||||
if not reposting_data and doc and doc.reposting_data_file:
|
||||
reposting_data = get_reposting_data(doc.reposting_data_file)
|
||||
|
||||
if reposting_data and reposting_data.distinct_item_and_warehouse:
|
||||
return reposting_data.distinct_item_and_warehouse
|
||||
|
||||
distinct_item_warehouses = {}
|
||||
|
||||
if doc and doc.distinct_item_and_warehouse:
|
||||
distinct_item_warehouses = json.loads(doc.distinct_item_and_warehouse)
|
||||
distinct_item_warehouses = {
|
||||
@ -324,7 +447,13 @@ def get_distinct_item_warehouse(args=None, doc=None):
|
||||
return distinct_item_warehouses
|
||||
|
||||
|
||||
def get_affected_transactions(doc) -> Set[Tuple[str, str]]:
|
||||
def get_affected_transactions(doc, reposting_data=None) -> Set[Tuple[str, str]]:
|
||||
if not reposting_data and doc and doc.reposting_data_file:
|
||||
reposting_data = get_reposting_data(doc.reposting_data_file)
|
||||
|
||||
if reposting_data and reposting_data.affected_transactions:
|
||||
return {tuple(transaction) for transaction in reposting_data.affected_transactions}
|
||||
|
||||
if not doc.affected_transactions:
|
||||
return set()
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user