Shiloh/brotherton-erpnext
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #30146 from marination/bom-update-tool

Browse Source 
refactor: Add exception handling in background job within BOM Update Tool
This commit is contained in:
Marica 2022-03-31 15:54:34 +05:30 committed by GitHub
commit 96fc6ad590
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 482 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
erpnext/hooks.py
View File

@ -469,7 +469,7 @@ scheduler_events = {
],
"daily_long": [
"erpnext.setup.doctype.email_digest.email_digest.send",
"erpnext.manufacturing.doctype.bom_update_tool.bom_update_tool.update_latest_price_in_all_boms",
"erpnext.manufacturing.doctype.bom_update_tool.bom_update_tool.auto_update_latest_price_in_all_boms",
"erpnext.hr.doctype.leave_ledger_entry.leave_ledger_entry.process_expired_allocation",
"erpnext.hr.utils.generate_leave_encashment",
"erpnext.hr.utils.allocate_earned_leaves",

9
erpnext/manufacturing/doctype/bom/bom.py
View File

@ -697,15 +697,6 @@ class BOM(WebsiteGenerator):
self.scrap_material_cost = total_sm_cost
self.base_scrap_material_cost = base_total_sm_cost
def update_new_bom(self, old_bom, new_bom, rate):
for d in self.get("items"):
if d.bom_no != old_bom:
continue
d.bom_no = new_bom
d.rate = rate
d.amount = (d.stock_qty or d.qty) * rate
def update_exploded_items(self, save=True):
"""Update Flat BOM, following will be correct data"""
self.get_exploded_items()

0
erpnext/manufacturing/doctype/bom_update_log/__init__.py Normal file
View File

8
erpnext/manufacturing/doctype/bom_update_log/bom_update_log.js Normal file
View File

@ -0,0 +1,8 @@
// Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and contributors
// For license information, please see license.txt
frappe.ui.form.on('BOM Update Log', {
// refresh: function(frm) {
// }
});

109
erpnext/manufacturing/doctype/bom_update_log/bom_update_log.json Normal file
View File

@ -0,0 +1,109 @@
{
"actions": [],
"autoname": "BOM-UPDT-LOG-.#####",
"creation": "2022-03-16 14:23:35.210155",
"description": "BOM Update Tool Log with job status maintained",
"doctype": "DocType",
"editable_grid": 1,
"engine": "InnoDB",
"field_order": [
"current_bom",
"new_bom",
"column_break_3",
"update_type",
"status",
"error_log",
"amended_from"
],
"fields": [
{
"fieldname": "current_bom",
"fieldtype": "Link",
"in_list_view": 1,
"in_standard_filter": 1,
"label": "Current BOM",
"options": "BOM"
},
{
"fieldname": "new_bom",
"fieldtype": "Link",
"in_list_view": 1,
"in_standard_filter": 1,
"label": "New BOM",
"options": "BOM"
},
{
"fieldname": "column_break_3",
"fieldtype": "Column Break"
},
{
"fieldname": "update_type",
"fieldtype": "Select",
"in_list_view": 1,
"label": "Update Type",
"options": "Replace BOM\nUpdate Cost"
},
{
"fieldname": "status",
"fieldtype": "Select",
"label": "Status",
"options": "Queued\nIn Progress\nCompleted\nFailed"
},
{
"fieldname": "amended_from",
"fieldtype": "Link",
"label": "Amended From",
"no_copy": 1,
"options": "BOM Update Log",
"print_hide": 1,
"read_only": 1
},
{
"fieldname": "error_log",
"fieldtype": "Link",
"label": "Error Log",
"options": "Error Log"
}
],
"in_create": 1,
"index_web_pages_for_search": 1,
"is_submittable": 1,
"links": [],
"modified": "2022-03-31 12:51:44.885102",
"modified_by": "Administrator",
"module": "Manufacturing",
"name": "BOM Update Log",
"naming_rule": "Expression (old style)",
"owner": "Administrator",
"permissions": [
{
"create": 1,
"delete": 1,
"email": 1,
"export": 1,
"print": 1,
"read": 1,
"report": 1,
"role": "System Manager",
"share": 1,
"submit": 1,
"write": 1
},
{
"create": 1,
"email": 1,
"export": 1,
"print": 1,
"read": 1,
"report": 1,
"role": "Manufacturing Manager",
"share": 1,
"submit": 1,
"write": 1
}
],
"sort_field": "modified",
"sort_order": "DESC",
"states": [],
"track_changes": 1
}

164
erpnext/manufacturing/doctype/bom_update_log/bom_update_log.py Normal file
View File

@ -0,0 +1,164 @@
# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and contributors
# For license information, please see license.txt
from typing import Dict, List, Literal, Optional
import frappe
from frappe import _
from frappe.model.document import Document
from frappe.utils import cstr, flt
from erpnext.manufacturing.doctype.bom_update_tool.bom_update_tool import update_cost
class BOMMissingError(frappe.ValidationError):
pass
class BOMUpdateLog(Document):
def validate(self):
if self.update_type == "Replace BOM":
self.validate_boms_are_specified()
self.validate_same_bom()
self.validate_bom_items()
self.status = "Queued"
def validate_boms_are_specified(self):
if self.update_type == "Replace BOM" and not (self.current_bom and self.new_bom):
frappe.throw(
msg=_("Please mention the Current and New BOM for replacement."),
title=_("Mandatory"),
exc=BOMMissingError,
)
def validate_same_bom(self):
if cstr(self.current_bom) == cstr(self.new_bom):
frappe.throw(_("Current BOM and New BOM can not be same"))
def validate_bom_items(self):
current_bom_item = frappe.db.get_value("BOM", self.current_bom, "item")
new_bom_item = frappe.db.get_value("BOM", self.new_bom, "item")
if current_bom_item != new_bom_item:
frappe.throw(_("The selected BOMs are not for the same item"))
def on_submit(self):
if frappe.flags.in_test:
return
if self.update_type == "Replace BOM":
boms = {"current_bom": self.current_bom, "new_bom": self.new_bom}
frappe.enqueue(
method="erpnext.manufacturing.doctype.bom_update_log.bom_update_log.run_bom_job",
doc=self,
boms=boms,
timeout=40000,
)
else:
frappe.enqueue(
method="erpnext.manufacturing.doctype.bom_update_log.bom_update_log.run_bom_job",
doc=self,
update_type="Update Cost",
timeout=40000,
)
def replace_bom(boms: Dict) -> None:
"""Replace current BOM with new BOM in parent BOMs."""
current_bom = boms.get("current_bom")
new_bom = boms.get("new_bom")
unit_cost = get_new_bom_unit_cost(new_bom)
update_new_bom_in_bom_items(unit_cost, current_bom, new_bom)
frappe.cache().delete_key("bom_children")
parent_boms = get_parent_boms(new_bom)
for bom in parent_boms:
bom_obj = frappe.get_doc("BOM", bom)
# this is only used for versioning and we do not want
# to make separate db calls by using load_doc_before_save
# which proves to be expensive while doing bulk replace
bom_obj._doc_before_save = bom_obj
bom_obj.update_exploded_items()
bom_obj.calculate_cost()
bom_obj.update_parent_cost()
bom_obj.db_update()
if bom_obj.meta.get("track_changes") and not bom_obj.flags.ignore_version:
bom_obj.save_version()
def update_new_bom_in_bom_items(unit_cost: float, current_bom: str, new_bom: str) -> None:
bom_item = frappe.qb.DocType("BOM Item")
(
frappe.qb.update(bom_item)
.set(bom_item.bom_no, new_bom)
.set(bom_item.rate, unit_cost)
.set(bom_item.amount, (bom_item.stock_qty * unit_cost))
.where(
(bom_item.bom_no == current_bom) & (bom_item.docstatus < 2) & (bom_item.parenttype == "BOM")
)
).run()
def get_parent_boms(new_bom: str, bom_list: Optional[List] = None) -> List:
bom_list = bom_list or []
bom_item = frappe.qb.DocType("BOM Item")
parents = (
frappe.qb.from_(bom_item)
.select(bom_item.parent)
.where((bom_item.bom_no == new_bom) & (bom_item.docstatus < 2) & (bom_item.parenttype == "BOM"))
.run(as_dict=True)
)
for d in parents:
if new_bom == d.parent:
frappe.throw(_("BOM recursion: {0} cannot be child of {1}").format(new_bom, d.parent))
bom_list.append(d.parent)
get_parent_boms(d.parent, bom_list)
return list(set(bom_list))
def get_new_bom_unit_cost(new_bom: str) -> float:
bom = frappe.qb.DocType("BOM")
new_bom_unitcost = (
frappe.qb.from_(bom).select(bom.total_cost / bom.quantity).where(bom.name == new_bom).run()
)
return flt(new_bom_unitcost[0][0])
def run_bom_job(
doc: "BOMUpdateLog",
boms: Optional[Dict[str, str]] = None,
update_type: Literal["Replace BOM", "Update Cost"] = "Replace BOM",
) -> None:
try:
doc.db_set("status", "In Progress")
if not frappe.flags.in_test:
frappe.db.commit()
frappe.db.auto_commit_on_many_writes = 1
boms = frappe._dict(boms or {})
if update_type == "Replace BOM":
replace_bom(boms)
else:
update_cost()
doc.db_set("status", "Completed")
except Exception:
frappe.db.rollback()
error_log = frappe.log_error(message=frappe.get_traceback(), title=_("BOM Update Tool Error"))
doc.db_set("status", "Failed")
doc.db_set("error_log", error_log.name)
finally:
frappe.db.auto_commit_on_many_writes = 0
frappe.db.commit() # nosemgrep

13
erpnext/manufacturing/doctype/bom_update_log/bom_update_log_list.js Normal file
View File

@ -0,0 +1,13 @@
frappe.listview_settings['BOM Update Log'] = {
add_fields: ["status"],
get_indicator: function(doc) {
let status_map = {
"Queued": "orange",
"In Progress": "blue",
"Completed": "green",
"Failed": "red"
};
return [__(doc.status), status_map[doc.status], "status,=," + doc.status];
}
};

96
erpnext/manufacturing/doctype/bom_update_log/test_bom_update_log.py Normal file
View File

@ -0,0 +1,96 @@
# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and Contributors
# See license.txt
import frappe
from frappe.tests.utils import FrappeTestCase
from erpnext.manufacturing.doctype.bom_update_log.bom_update_log import (
BOMMissingError,
run_bom_job,
)
from erpnext.manufacturing.doctype.bom_update_tool.bom_update_tool import enqueue_replace_bom
test_records = frappe.get_test_records("BOM")
class TestBOMUpdateLog(FrappeTestCase):
"Test BOM Update Tool Operations via BOM Update Log."
def setUp(self):
bom_doc = frappe.copy_doc(test_records[0])
bom_doc.items[1].item_code = "_Test Item"
bom_doc.insert()
self.boms = frappe._dict(
current_bom="BOM-_Test Item Home Desktop Manufactured-001",
new_bom=bom_doc.name,
)
self.new_bom_doc = bom_doc
def tearDown(self):
frappe.db.rollback()
if self._testMethodName == "test_bom_update_log_completion":
# clear logs and delete BOM created via setUp
frappe.db.delete("BOM Update Log")
self.new_bom_doc.cancel()
self.new_bom_doc.delete()
# explicitly commit and restore to original state
frappe.db.commit() # nosemgrep
def test_bom_update_log_validate(self):
"Test if BOM presence is validated."
with self.assertRaises(BOMMissingError):
enqueue_replace_bom(boms={})
with self.assertRaises(frappe.ValidationError):
enqueue_replace_bom(boms=frappe._dict(current_bom=self.boms.new_bom, new_bom=self.boms.new_bom))
with self.assertRaises(frappe.ValidationError):
enqueue_replace_bom(boms=frappe._dict(current_bom=self.boms.new_bom, new_bom="Dummy BOM"))
def test_bom_update_log_queueing(self):
"Test if BOM Update Log is created and queued."
log = enqueue_replace_bom(
boms=self.boms,
)
self.assertEqual(log.docstatus, 1)
self.assertEqual(log.status, "Queued")
def test_bom_update_log_completion(self):
"Test if BOM Update Log handles job completion correctly."
log = enqueue_replace_bom(
boms=self.boms,
)
# Explicitly commits log, new bom (setUp) and replacement impact.
# Is run via background jobs IRL
run_bom_job(
doc=log,
boms=self.boms,
update_type="Replace BOM",
)
log.reload()
self.assertEqual(log.status, "Completed")
# teardown (undo replace impact) due to commit
boms = frappe._dict(
current_bom=self.boms.new_bom,
new_bom=self.boms.current_bom,
)
log2 = enqueue_replace_bom(
boms=self.boms,
)
run_bom_job( # Explicitly commits
doc=log2,
boms=boms,
update_type="Replace BOM",
)
self.assertEqual(log2.status, "Completed")

47
erpnext/manufacturing/doctype/bom_update_tool/bom_update_tool.js
View File

@ -20,30 +20,67 @@ frappe.ui.form.on('BOM Update Tool', {
refresh: function(frm) {
frm.disable_save();
frm.events.disable_button(frm, "replace");
frm.add_custom_button(__("View BOM Update Log"), () => {
frappe.set_route("List", "BOM Update Log");
});
},
replace: function(frm) {
disable_button: (frm, field, disable=true) => {
frm.get_field(field).input.disabled = disable;
},
current_bom: (frm) => {
if (frm.doc.current_bom && frm.doc.new_bom) {
frm.events.disable_button(frm, "replace", false);
}
},
new_bom: (frm) => {
if (frm.doc.current_bom && frm.doc.new_bom) {
frm.events.disable_button(frm, "replace", false);
}
},
replace: (frm) => {
if (frm.doc.current_bom && frm.doc.new_bom) {
frappe.call({
method: "erpnext.manufacturing.doctype.bom_update_tool.bom_update_tool.enqueue_replace_bom",
freeze: true,
args: {
args: {
boms: {
"current_bom": frm.doc.current_bom,
"new_bom": frm.doc.new_bom
}
},
callback: result => {
if (result && result.message && !result.exc) {
frm.events.confirm_job_start(frm, result.message);
}
}
});
}
},
update_latest_price_in_all_boms: function() {
update_latest_price_in_all_boms: (frm) => {
frappe.call({
method: "erpnext.manufacturing.doctype.bom_update_tool.bom_update_tool.enqueue_update_cost",
freeze: true,
callback: function() {
frappe.msgprint(__("Latest price updated in all BOMs"));
callback: result => {
if (result && result.message && !result.exc) {
frm.events.confirm_job_start(frm, result.message);
}
}
});
},
confirm_job_start: (frm, log_data) => {
let log_link = frappe.utils.get_form_link("BOM Update Log", log_data.name, true);
frappe.msgprint({
"message": __("BOM Updation is queued and may take a few minutes. Check {0} for progress.", [log_link]),
"title": __("BOM Update Initiated"),
"indicator": "blue"
});
}
});

149
erpnext/manufacturing/doctype/bom_update_tool/bom_update_tool.py
View File

@ -1,136 +1,69 @@
# Copyright (c) 2017, Frappe Technologies Pvt. Ltd. and contributors
# Copyright (c) 2022, Frappe Technologies Pvt. Ltd. and contributors
# For license information, please see license.txt
import json
from typing import TYPE_CHECKING, Dict, Literal, Optional, Union
if TYPE_CHECKING:
from erpnext.manufacturing.doctype.bom_update_log.bom_update_log import BOMUpdateLog
import click
import frappe
from frappe import _
from frappe.model.document import Document
from frappe.utils import cstr, flt
from erpnext.manufacturing.doctype.bom.bom import get_boms_in_bottom_up_order
class BOMUpdateTool(Document):
def replace_bom(self):
self.validate_bom()
unit_cost = get_new_bom_unit_cost(self.new_bom)
self.update_new_bom(unit_cost)
frappe.cache().delete_key("bom_children")
bom_list = self.get_parent_boms(self.new_bom)
with click.progressbar(bom_list) as bom_list:
pass
for bom in bom_list:
try:
bom_obj = frappe.get_cached_doc("BOM", bom)
# this is only used for versioning and we do not want
# to make separate db calls by using load_doc_before_save
# which proves to be expensive while doing bulk replace
bom_obj._doc_before_save = bom_obj
bom_obj.update_new_bom(self.current_bom, self.new_bom, unit_cost)
bom_obj.update_exploded_items()
bom_obj.calculate_cost()
bom_obj.update_parent_cost()
bom_obj.db_update()
if bom_obj.meta.get("track_changes") and not bom_obj.flags.ignore_version:
bom_obj.save_version()
except Exception:
frappe.log_error(frappe.get_traceback())
def validate_bom(self):
if cstr(self.current_bom) == cstr(self.new_bom):
frappe.throw(_("Current BOM and New BOM can not be same"))
if frappe.db.get_value("BOM", self.current_bom, "item") != frappe.db.get_value(
"BOM", self.new_bom, "item"
):
frappe.throw(_("The selected BOMs are not for the same item"))
def update_new_bom(self, unit_cost):
frappe.db.sql(
"""update `tabBOM Item` set bom_no=%s,
rate=%s, amount=stock_qty*%s where bom_no = %s and docstatus < 2 and parenttype='BOM'""",
(self.new_bom, unit_cost, unit_cost, self.current_bom),
)
def get_parent_boms(self, bom, bom_list=None):
if bom_list is None:
bom_list = []
data = frappe.db.sql(
"""SELECT DISTINCT parent FROM `tabBOM Item`
WHERE bom_no = %s AND docstatus < 2 AND parenttype='BOM'""",
bom,
)
for d in data:
if self.new_bom == d[0]:
frappe.throw(_("BOM recursion: {0} cannot be child of {1}").format(bom, self.new_bom))
bom_list.append(d[0])
self.get_parent_boms(d[0], bom_list)
return list(set(bom_list))
def get_new_bom_unit_cost(bom):
new_bom_unitcost = frappe.db.sql(
"""SELECT `total_cost`/`quantity`
FROM `tabBOM` WHERE name = %s""",
bom,
)
return flt(new_bom_unitcost[0][0]) if new_bom_unitcost else 0
pass
@frappe.whitelist()
def enqueue_replace_bom(args):
if isinstance(args, str):
args = json.loads(args)
def enqueue_replace_bom(
boms: Optional[Union[Dict, str]] = None, args: Optional[Union[Dict, str]] = None
) -> "BOMUpdateLog":
"""Returns a BOM Update Log (that queues a job) for BOM Replacement."""
boms = boms or args
if isinstance(boms, str):
boms = json.loads(boms)
frappe.enqueue(
"erpnext.manufacturing.doctype.bom_update_tool.bom_update_tool.replace_bom",
args=args,
timeout=40000,
)
frappe.msgprint(_("Queued for replacing the BOM. It may take a few minutes."))
update_log = create_bom_update_log(boms=boms)
return update_log
@frappe.whitelist()
def enqueue_update_cost():
frappe.enqueue(
"erpnext.manufacturing.doctype.bom_update_tool.bom_update_tool.update_cost", timeout=40000
)
frappe.msgprint(
_("Queued for updating latest price in all Bill of Materials. It may take a few minutes.")
)
def enqueue_update_cost() -> "BOMUpdateLog":
"""Returns a BOM Update Log (that queues a job) for BOM Cost Updation."""
update_log = create_bom_update_log(update_type="Update Cost")
return update_log
def update_latest_price_in_all_boms():
def auto_update_latest_price_in_all_boms() -> None:
"""Called via hooks.py."""
if frappe.db.get_single_value("Manufacturing Settings", "update_bom_costs_automatically"):
update_cost()
def replace_bom(args):
frappe.db.auto_commit_on_many_writes = 1
args = frappe._dict(args)
doc = frappe.get_doc("BOM Update Tool")
doc.current_bom = args.current_bom
doc.new_bom = args.new_bom
doc.replace_bom()
frappe.db.auto_commit_on_many_writes = 0
def update_cost():
frappe.db.auto_commit_on_many_writes = 1
def update_cost() -> None:
"""Updates Cost for all BOMs from bottom to top."""
bom_list = get_boms_in_bottom_up_order()
for bom in bom_list:
frappe.get_doc("BOM", bom).update_cost(update_parent=False, from_child_bom=True)
frappe.db.auto_commit_on_many_writes = 0
def create_bom_update_log(
boms: Optional[Dict[str, str]] = None,
update_type: Literal["Replace BOM", "Update Cost"] = "Replace BOM",
) -> "BOMUpdateLog":
"""Creates a BOM Update Log that handles the background job."""
boms = boms or {}
current_bom = boms.get("current_bom")
new_bom = boms.get("new_bom")
return frappe.get_doc(
{
"doctype": "BOM Update Log",
"current_bom": current_bom,
"new_bom": new_bom,
"update_type": update_type,
}
).submit()

15
erpnext/manufacturing/doctype/bom_update_tool/test_bom_update_tool.py
View File

@ -4,6 +4,7 @@
import frappe
from frappe.tests.utils import FrappeTestCase
from erpnext.manufacturing.doctype.bom_update_log.bom_update_log import replace_bom
from erpnext.manufacturing.doctype.bom_update_tool.bom_update_tool import update_cost
from erpnext.manufacturing.doctype.production_plan.test_production_plan import make_bom
from erpnext.stock.doctype.item.test_item import create_item
@ -12,6 +13,8 @@ test_records = frappe.get_test_records("BOM")
class TestBOMUpdateTool(FrappeTestCase):
"Test major functions run via BOM Update Tool."
def test_replace_bom(self):
current_bom = "BOM-_Test Item Home Desktop Manufactured-001"
@ -19,18 +22,16 @@ class TestBOMUpdateTool(FrappeTestCase):
bom_doc.items[1].item_code = "_Test Item"
bom_doc.insert()
update_tool = frappe.get_doc("BOM Update Tool")
update_tool.current_bom = current_bom
update_tool.new_bom = bom_doc.name
update_tool.replace_bom()
boms = frappe._dict(current_bom=current_bom, new_bom=bom_doc.name)
replace_bom(boms)
self.assertFalse(frappe.db.sql("select name from `tabBOM Item` where bom_no=%s", current_bom))
self.assertTrue(frappe.db.sql("select name from `tabBOM Item` where bom_no=%s", bom_doc.name))
# reverse, as it affects other testcases
update_tool.current_bom = bom_doc.name
update_tool.new_bom = current_bom
update_tool.replace_bom()
boms.current_bom = bom_doc.name
boms.new_bom = current_bom
replace_bom(boms)
def test_bom_cost(self):
for item in ["BOM Cost Test Item 1", "BOM Cost Test Item 2", "BOM Cost Test Item 3"]: