From 815c616f18ae1cd6ec1dbd315fa5f0376edef523 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Thu, 9 Nov 2023 15:26:52 +0530 Subject: [PATCH 01/16] chore: remove 'Bulk Transaction Log' doctype --- .../doctype/bulk_transaction_log/__init__.py | 0 .../bulk_transaction_log.js | 30 ------- .../bulk_transaction_log.json | 51 ------------ .../bulk_transaction_log.py | 67 ---------------- .../test_bulk_transaction_log.py | 79 ------------------- 5 files changed, 227 deletions(-) delete mode 100644 erpnext/bulk_transaction/doctype/bulk_transaction_log/__init__.py delete mode 100644 erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.js delete mode 100644 erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.json delete mode 100644 erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.py delete mode 100644 erpnext/bulk_transaction/doctype/bulk_transaction_log/test_bulk_transaction_log.py diff --git a/erpnext/bulk_transaction/doctype/bulk_transaction_log/__init__.py b/erpnext/bulk_transaction/doctype/bulk_transaction_log/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.js b/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.js deleted file mode 100644 index 0073170a85..0000000000 --- a/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.js +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and contributors -// For license information, please see license.txt - -frappe.ui.form.on('Bulk Transaction Log', { - - refresh: function(frm) { - frm.disable_save(); - frm.add_custom_button(__('Retry Failed Transactions'), ()=>{ - frappe.confirm(__("Retry Failing Transactions ?"), ()=>{ - query(frm, 1); - } - ); - }); - } -}); - -function query(frm) { - frappe.call({ - method: "erpnext.bulk_transaction.doctype.bulk_transaction_log.bulk_transaction_log.retry_failing_transaction", - args: { - log_date: frm.doc.log_date - } - }).then((r) => { - if (r.message === "No Failed Records") { - frappe.show_alert(__(r.message), 5); - } else { - frappe.show_alert(__("Retrying Failed Transactions"), 5); - } - }); -} \ No newline at end of file diff --git a/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.json b/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.json deleted file mode 100644 index da42cf1bd4..0000000000 --- a/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "actions": [], - "allow_rename": 1, - "creation": "2021-11-30 13:41:16.343827", - "doctype": "DocType", - "editable_grid": 1, - "engine": "InnoDB", - "field_order": [ - "log_date", - "logger_data" - ], - "fields": [ - { - "fieldname": "log_date", - "fieldtype": "Date", - "label": "Log Date", - "read_only": 1 - }, - { - "fieldname": "logger_data", - "fieldtype": "Table", - "label": "Logger Data", - "options": "Bulk Transaction Log Detail" - } - ], - "index_web_pages_for_search": 1, - "links": [], - "modified": "2022-02-03 17:23:02.935325", - "modified_by": "Administrator", - "module": "Bulk Transaction", - "name": "Bulk Transaction Log", - "owner": "Administrator", - "permissions": [ - { - "create": 1, - "delete": 1, - "email": 1, - "export": 1, - "print": 1, - "read": 1, - "report": 1, - "role": "System Manager", - "share": 1, - "write": 1 - } - ], - "sort_field": "modified", - "sort_order": "DESC", - "states": [], - "track_changes": 1 -} \ No newline at end of file diff --git a/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.py b/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.py deleted file mode 100644 index 0596be4462..0000000000 --- a/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.py +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and contributors -# For license information, please see license.txt - -from datetime import date - -import frappe -from frappe.model.document import Document - -from erpnext.utilities.bulk_transaction import task, update_logger - - -class BulkTransactionLog(Document): - pass - - -@frappe.whitelist() -def retry_failing_transaction(log_date=None): - if not log_date: - log_date = str(date.today()) - btp = frappe.qb.DocType("Bulk Transaction Log Detail") - data = ( - frappe.qb.from_(btp) - .select(btp.transaction_name, btp.from_doctype, btp.to_doctype) - .distinct() - .where(btp.retried != 1) - .where(btp.transaction_status == "Failed") - .where(btp.date == log_date) - ).run(as_dict=True) - - if data: - if len(data) > 10: - frappe.enqueue(job, queue="long", job_name="bulk_retry", data=data, log_date=log_date) - else: - job(data, log_date) - else: - return "No Failed Records" - - -def job(data, log_date): - for d in data: - failed = [] - try: - frappe.db.savepoint("before_creation_of_record") - task(d.transaction_name, d.from_doctype, d.to_doctype) - except Exception as e: - frappe.db.rollback(save_point="before_creation_of_record") - failed.append(e) - update_logger( - d.transaction_name, - e, - d.from_doctype, - d.to_doctype, - status="Failed", - log_date=log_date, - restarted=1, - ) - - if not failed: - update_logger( - d.transaction_name, - None, - d.from_doctype, - d.to_doctype, - status="Success", - log_date=log_date, - restarted=1, - ) diff --git a/erpnext/bulk_transaction/doctype/bulk_transaction_log/test_bulk_transaction_log.py b/erpnext/bulk_transaction/doctype/bulk_transaction_log/test_bulk_transaction_log.py deleted file mode 100644 index c673be89b3..0000000000 --- a/erpnext/bulk_transaction/doctype/bulk_transaction_log/test_bulk_transaction_log.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright (c) 2021, Frappe Technologies Pvt. Ltd. and Contributors -# See license.txt - -import unittest -from datetime import date - -import frappe - -from erpnext.utilities.bulk_transaction import transaction_processing - - -class TestBulkTransactionLog(unittest.TestCase): - def setUp(self): - create_company() - create_customer() - create_item() - - def test_entry_in_log(self): - so_name = create_so() - transaction_processing([{"name": so_name}], "Sales Order", "Sales Invoice") - doc = frappe.get_doc("Bulk Transaction Log", str(date.today())) - for d in doc.get("logger_data"): - if d.transaction_name == so_name: - self.assertEqual(d.transaction_name, so_name) - self.assertEqual(d.transaction_status, "Success") - self.assertEqual(d.from_doctype, "Sales Order") - self.assertEqual(d.to_doctype, "Sales Invoice") - self.assertEqual(d.retried, 0) - - -def create_company(): - if not frappe.db.exists("Company", "_Test Company"): - frappe.get_doc( - { - "doctype": "Company", - "company_name": "_Test Company", - "country": "India", - "default_currency": "INR", - } - ).insert() - - -def create_customer(): - if not frappe.db.exists("Customer", "Bulk Customer"): - frappe.get_doc({"doctype": "Customer", "customer_name": "Bulk Customer"}).insert() - - -def create_item(): - if not frappe.db.exists("Item", "MK"): - frappe.get_doc( - { - "doctype": "Item", - "item_code": "MK", - "item_name": "Milk", - "description": "Milk", - "item_group": "Products", - } - ).insert() - - -def create_so(intent=None): - so = frappe.new_doc("Sales Order") - so.customer = "Bulk Customer" - so.company = "_Test Company" - so.transaction_date = date.today() - - so.set_warehouse = "Finished Goods - _TC" - so.append( - "items", - { - "item_code": "MK", - "delivery_date": date.today(), - "qty": 10, - "rate": 80, - }, - ) - so.insert() - so.submit() - return so.name From e5a8ad54e26d74b5713e7db057860c1262c1b93b Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Thu, 9 Nov 2023 15:29:57 +0530 Subject: [PATCH 02/16] chore: convert child to normal table --- .../bulk_transaction_log_detail.js | 8 ++++++++ .../bulk_transaction_log_detail.json | 18 +++++++++++++++--- .../test_bulk_transaction_log_detail.py | 9 +++++++++ 3 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 erpnext/bulk_transaction/doctype/bulk_transaction_log_detail/bulk_transaction_log_detail.js create mode 100644 erpnext/bulk_transaction/doctype/bulk_transaction_log_detail/test_bulk_transaction_log_detail.py diff --git a/erpnext/bulk_transaction/doctype/bulk_transaction_log_detail/bulk_transaction_log_detail.js b/erpnext/bulk_transaction/doctype/bulk_transaction_log_detail/bulk_transaction_log_detail.js new file mode 100644 index 0000000000..5669601d11 --- /dev/null +++ b/erpnext/bulk_transaction/doctype/bulk_transaction_log_detail/bulk_transaction_log_detail.js @@ -0,0 +1,8 @@ +// Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and contributors +// For license information, please see license.txt + +// frappe.ui.form.on("Bulk Transaction Log Detail", { +// refresh(frm) { + +// }, +// }); diff --git a/erpnext/bulk_transaction/doctype/bulk_transaction_log_detail/bulk_transaction_log_detail.json b/erpnext/bulk_transaction/doctype/bulk_transaction_log_detail/bulk_transaction_log_detail.json index 8262caa020..64518de4c0 100644 --- a/erpnext/bulk_transaction/doctype/bulk_transaction_log_detail/bulk_transaction_log_detail.json +++ b/erpnext/bulk_transaction/doctype/bulk_transaction_log_detail/bulk_transaction_log_detail.json @@ -71,14 +71,26 @@ } ], "index_web_pages_for_search": 1, - "istable": 1, "links": [], - "modified": "2022-02-03 19:57:31.650359", + "modified": "2023-11-09 15:29:33.013547", "modified_by": "Administrator", "module": "Bulk Transaction", "name": "Bulk Transaction Log Detail", "owner": "Administrator", - "permissions": [], + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], "sort_field": "modified", "sort_order": "DESC", "states": [], diff --git a/erpnext/bulk_transaction/doctype/bulk_transaction_log_detail/test_bulk_transaction_log_detail.py b/erpnext/bulk_transaction/doctype/bulk_transaction_log_detail/test_bulk_transaction_log_detail.py new file mode 100644 index 0000000000..5217b601f8 --- /dev/null +++ b/erpnext/bulk_transaction/doctype/bulk_transaction_log_detail/test_bulk_transaction_log_detail.py @@ -0,0 +1,9 @@ +# Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and Contributors +# See license.txt + +# import frappe +from frappe.tests.utils import FrappeTestCase + + +class TestBulkTransactionLogDetail(FrappeTestCase): + pass From c4f8f3613f1173a54b17e3c6ca8ea52469f32e3b Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Thu, 9 Nov 2023 16:07:45 +0530 Subject: [PATCH 03/16] chore: rearrange fields --- .../bulk_transaction_log_detail.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/erpnext/bulk_transaction/doctype/bulk_transaction_log_detail/bulk_transaction_log_detail.json b/erpnext/bulk_transaction/doctype/bulk_transaction_log_detail/bulk_transaction_log_detail.json index 64518de4c0..051dc0af02 100644 --- a/erpnext/bulk_transaction/doctype/bulk_transaction_log_detail/bulk_transaction_log_detail.json +++ b/erpnext/bulk_transaction/doctype/bulk_transaction_log_detail/bulk_transaction_log_detail.json @@ -6,12 +6,12 @@ "editable_grid": 1, "engine": "InnoDB", "field_order": [ + "from_doctype", "transaction_name", "date", "time", "transaction_status", "error_description", - "from_doctype", "to_doctype", "retried" ], @@ -72,7 +72,7 @@ ], "index_web_pages_for_search": 1, "links": [], - "modified": "2023-11-09 15:29:33.013547", + "modified": "2023-11-09 15:33:26.828998", "modified_by": "Administrator", "module": "Bulk Transaction", "name": "Bulk Transaction Log Detail", From ebd74a4e5b68534a4ceab2482618e16643f220d1 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Thu, 9 Nov 2023 16:38:34 +0530 Subject: [PATCH 04/16] refactor: simplify logging logic bulk_transaction --- erpnext/utilities/bulk_transaction.py | 74 +++++---------------------- 1 file changed, 14 insertions(+), 60 deletions(-) diff --git a/erpnext/utilities/bulk_transaction.py b/erpnext/utilities/bulk_transaction.py index fcee265644..fd8e722648 100644 --- a/erpnext/utilities/bulk_transaction.py +++ b/erpnext/utilities/bulk_transaction.py @@ -3,6 +3,7 @@ from datetime import date, datetime import frappe from frappe import _ +from frappe.utils import today @frappe.whitelist() @@ -38,7 +39,7 @@ def job(deserialized_data, from_doctype, to_doctype): except Exception as e: frappe.db.rollback(save_point="before_creation_state") fail_count += 1 - update_logger( + create_log( doc_name, str(frappe.get_traceback()), from_doctype, @@ -47,7 +48,7 @@ def job(deserialized_data, from_doctype, to_doctype): log_date=str(date.today()), ) else: - update_logger( + create_log( doc_name, None, from_doctype, to_doctype, status="Success", log_date=str(date.today()) ) @@ -108,45 +109,18 @@ def task(doc_name, from_doctype, to_doctype): obj.insert(ignore_mandatory=True) -def check_logger_doc_exists(log_date): - return frappe.db.exists("Bulk Transaction Log", log_date) - - -def get_logger_doc(log_date): - return frappe.get_doc("Bulk Transaction Log", log_date) - - -def create_logger_doc(): - log_doc = frappe.new_doc("Bulk Transaction Log") - log_doc.set_new_name(set_name=str(date.today())) - log_doc.log_date = date.today() - - return log_doc - - -def append_data_to_logger(log_doc, doc_name, error, from_doctype, to_doctype, status, restarted): - row = log_doc.append("logger_data", {}) - row.transaction_name = doc_name - row.date = date.today() +def create_log(doc_name, e, from_doctype, to_doctype, status, log_date=None, restarted=0): + transaction_log = frappe.new_doc("Bulk Transaction Log Detail") + transaction_log.transaction_name = doc_name + transaction_log.date = today() now = datetime.now() - row.time = now.strftime("%H:%M:%S") - row.transaction_status = status - row.error_description = str(error) - row.from_doctype = from_doctype - row.to_doctype = to_doctype - row.retried = restarted - - -def update_logger(doc_name, e, from_doctype, to_doctype, status, log_date=None, restarted=0): - if not check_logger_doc_exists(log_date): - log_doc = create_logger_doc() - append_data_to_logger(log_doc, doc_name, e, from_doctype, to_doctype, status, restarted) - log_doc.insert() - else: - log_doc = get_logger_doc(log_date) - if record_exists(log_doc, doc_name, status): - append_data_to_logger(log_doc, doc_name, e, from_doctype, to_doctype, status, restarted) - log_doc.save() + transaction_log.time = now.strftime("%H:%M:%S") + transaction_log.transaction_status = status + transaction_log.error_description = str(e) + transaction_log.from_doctype = from_doctype + transaction_log.to_doctype = to_doctype + transaction_log.retried = restarted + transaction_log.save() def show_job_status(fail_count, deserialized_data_count, to_doctype): @@ -176,23 +150,3 @@ def show_job_status(fail_count, deserialized_data_count, to_doctype): title="Failed", indicator="red", ) - - -def record_exists(log_doc, doc_name, status): - record = mark_retrired_transaction(log_doc, doc_name) - if record and status == "Failed": - return False - elif record and status == "Success": - return True - else: - return True - - -def mark_retrired_transaction(log_doc, doc_name): - record = 0 - for d in log_doc.get("logger_data"): - if d.transaction_name == doc_name and d.transaction_status == "Failed": - frappe.db.set_value("Bulk Transaction Log Detail", d.name, "retried", 1) - record = record + 1 - - return record From ade09bc709ab8bca047abdad5235461a4618bc17 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Thu, 9 Nov 2023 17:16:58 +0530 Subject: [PATCH 05/16] chore: add list view filters --- .../bulk_transaction_log_detail.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/erpnext/bulk_transaction/doctype/bulk_transaction_log_detail/bulk_transaction_log_detail.json b/erpnext/bulk_transaction/doctype/bulk_transaction_log_detail/bulk_transaction_log_detail.json index 051dc0af02..3a5d434a04 100644 --- a/erpnext/bulk_transaction/doctype/bulk_transaction_log_detail/bulk_transaction_log_detail.json +++ b/erpnext/bulk_transaction/doctype/bulk_transaction_log_detail/bulk_transaction_log_detail.json @@ -20,6 +20,7 @@ "fieldname": "transaction_name", "fieldtype": "Dynamic Link", "in_list_view": 1, + "in_standard_filter": 1, "label": "Name", "options": "from_doctype" }, @@ -39,6 +40,7 @@ { "fieldname": "from_doctype", "fieldtype": "Link", + "in_standard_filter": 1, "label": "From Doctype", "options": "DocType", "read_only": 1 @@ -54,6 +56,7 @@ "fieldname": "date", "fieldtype": "Date", "in_list_view": 1, + "in_standard_filter": 1, "label": "Date ", "read_only": 1 }, @@ -70,9 +73,10 @@ "read_only": 1 } ], + "in_create": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2023-11-09 15:33:26.828998", + "modified": "2023-11-09 16:55:31.026064", "modified_by": "Administrator", "module": "Bulk Transaction", "name": "Bulk Transaction Log Detail", From 73090fa1306fd65ff8e290d57486697af5722081 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Thu, 9 Nov 2023 17:40:04 +0530 Subject: [PATCH 06/16] chore: add indexes --- .../bulk_transaction_log_detail.json | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/erpnext/bulk_transaction/doctype/bulk_transaction_log_detail/bulk_transaction_log_detail.json b/erpnext/bulk_transaction/doctype/bulk_transaction_log_detail/bulk_transaction_log_detail.json index 3a5d434a04..fe1a6d9b56 100644 --- a/erpnext/bulk_transaction/doctype/bulk_transaction_log_detail/bulk_transaction_log_detail.json +++ b/erpnext/bulk_transaction/doctype/bulk_transaction_log_detail/bulk_transaction_log_detail.json @@ -22,7 +22,8 @@ "in_list_view": 1, "in_standard_filter": 1, "label": "Name", - "options": "from_doctype" + "options": "from_doctype", + "search_index": 1 }, { "fieldname": "transaction_status", @@ -43,7 +44,8 @@ "in_standard_filter": 1, "label": "From Doctype", "options": "DocType", - "read_only": 1 + "read_only": 1, + "search_index": 1 }, { "fieldname": "to_doctype", @@ -58,7 +60,8 @@ "in_list_view": 1, "in_standard_filter": 1, "label": "Date ", - "read_only": 1 + "read_only": 1, + "search_index": 1 }, { "fieldname": "time", @@ -76,7 +79,7 @@ "in_create": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2023-11-09 16:55:31.026064", + "modified": "2023-11-09 17:44:14.804836", "modified_by": "Administrator", "module": "Bulk Transaction", "name": "Bulk Transaction Log Detail", @@ -99,4 +102,4 @@ "sort_order": "DESC", "states": [], "track_changes": 1 -} \ No newline at end of file +} From b0dfc936a10aef4dfd743959a6cf5e47ea69632e Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Thu, 9 Nov 2023 17:43:04 +0530 Subject: [PATCH 07/16] chore: make `from_doctype` readonly --- .../bulk_transaction_log_detail/bulk_transaction_log_detail.json | 1 + 1 file changed, 1 insertion(+) diff --git a/erpnext/bulk_transaction/doctype/bulk_transaction_log_detail/bulk_transaction_log_detail.json b/erpnext/bulk_transaction/doctype/bulk_transaction_log_detail/bulk_transaction_log_detail.json index fe1a6d9b56..5491f56a1a 100644 --- a/erpnext/bulk_transaction/doctype/bulk_transaction_log_detail/bulk_transaction_log_detail.json +++ b/erpnext/bulk_transaction/doctype/bulk_transaction_log_detail/bulk_transaction_log_detail.json @@ -23,6 +23,7 @@ "in_standard_filter": 1, "label": "Name", "options": "from_doctype", + "read_only": 1, "search_index": 1 }, { From a248b13cc353fcd94b0009c74c24512ea629a376 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Thu, 9 Nov 2023 20:15:33 +0530 Subject: [PATCH 08/16] feat: virtual parent doctype --- .../doctype/bulk_transaction_log/__init__.py | 0 .../bulk_transaction_log.js | 8 ++ .../bulk_transaction_log.json | 80 +++++++++++++++++++ .../bulk_transaction_log.py | 28 +++++++ .../test_bulk_transaction_log.py | 9 +++ 5 files changed, 125 insertions(+) create mode 100644 erpnext/bulk_transaction/doctype/bulk_transaction_log/__init__.py create mode 100644 erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.js create mode 100644 erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.json create mode 100644 erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.py create mode 100644 erpnext/bulk_transaction/doctype/bulk_transaction_log/test_bulk_transaction_log.py diff --git a/erpnext/bulk_transaction/doctype/bulk_transaction_log/__init__.py b/erpnext/bulk_transaction/doctype/bulk_transaction_log/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.js b/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.js new file mode 100644 index 0000000000..07a8009816 --- /dev/null +++ b/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.js @@ -0,0 +1,8 @@ +// Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and contributors +// For license information, please see license.txt + +// frappe.ui.form.on("Bulk Transaction Log", { +// refresh(frm) { + +// }, +// }); diff --git a/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.json b/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.json new file mode 100644 index 0000000000..a874b791ca --- /dev/null +++ b/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.json @@ -0,0 +1,80 @@ +{ + "actions": [], + "allow_copy": 1, + "creation": "2023-11-09 20:14:45.139593", + "doctype": "DocType", + "engine": "InnoDB", + "field_order": [ + "date", + "column_break_bsan", + "log_entries", + "section_break_mdmv", + "succeeded", + "column_break_qryp", + "failed" + ], + "fields": [ + { + "fieldname": "date", + "fieldtype": "Date", + "label": "Date", + "read_only": 1 + }, + { + "fieldname": "log_entries", + "fieldtype": "Int", + "in_list_view": 1, + "label": "Log Entries", + "read_only": 1 + }, + { + "fieldname": "column_break_bsan", + "fieldtype": "Column Break" + }, + { + "fieldname": "section_break_mdmv", + "fieldtype": "Section Break" + }, + { + "fieldname": "succeeded", + "fieldtype": "Int", + "label": "Succeeded", + "read_only": 1 + }, + { + "fieldname": "column_break_qryp", + "fieldtype": "Column Break" + }, + { + "fieldname": "failed", + "fieldtype": "Int", + "label": "Failed", + "read_only": 1 + } + ], + "in_create": 1, + "is_virtual": 1, + "links": [], + "modified": "2023-11-10 11:13:52.733076", + "modified_by": "Administrator", + "module": "Bulk Transaction", + "name": "Bulk Transaction Log", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "sort_field": "modified", + "sort_order": "DESC", + "states": [] +} \ No newline at end of file diff --git a/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.py b/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.py new file mode 100644 index 0000000000..fb9fcf7c52 --- /dev/null +++ b/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.py @@ -0,0 +1,28 @@ +# Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and contributors +# For license information, please see license.txt + +# import frappe +from frappe.model.document import Document + + +class BulkTransactionLog(Document): + def db_insert(self, *args, **kwargs): + pass + + def load_from_db(self): + pass + + def db_update(self): + pass + + @staticmethod + def get_list(args): + pass + + @staticmethod + def get_count(args): + pass + + @staticmethod + def get_stats(args): + pass diff --git a/erpnext/bulk_transaction/doctype/bulk_transaction_log/test_bulk_transaction_log.py b/erpnext/bulk_transaction/doctype/bulk_transaction_log/test_bulk_transaction_log.py new file mode 100644 index 0000000000..01bb615a3e --- /dev/null +++ b/erpnext/bulk_transaction/doctype/bulk_transaction_log/test_bulk_transaction_log.py @@ -0,0 +1,9 @@ +# Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and Contributors +# See license.txt + +# import frappe +from frappe.tests.utils import FrappeTestCase + + +class TestBulkTransactionLog(FrappeTestCase): + pass From af35590549987ab84eadbf08c10f3837242203dc Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Thu, 9 Nov 2023 21:07:01 +0530 Subject: [PATCH 09/16] refactor: add basic functionalities --- .../bulk_transaction_log.js | 15 ++-- .../bulk_transaction_log.py | 71 +++++++++++++++++-- 2 files changed, 75 insertions(+), 11 deletions(-) diff --git a/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.js b/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.js index 07a8009816..218424beca 100644 --- a/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.js +++ b/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.js @@ -1,8 +1,13 @@ // Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and contributors // For license information, please see license.txt -// frappe.ui.form.on("Bulk Transaction Log", { -// refresh(frm) { - -// }, -// }); +frappe.ui.form.on("Bulk Transaction Log", { + refresh(frm) { + frm.add_custom_button(__('Succeeded Entries'), function() { + frappe.set_route('List', 'Bulk Transaction Log Detail', {'date': frm.doc.date, 'transaction_status': "Success"}); + }, __("View")); + frm.add_custom_button(__('Failed Entries'), function() { + frappe.set_route('List', 'Bulk Transaction Log Detail', {'date': frm.doc.date, 'transaction_status': "Failed"}); + }, __("View")); + }, +}); diff --git a/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.py b/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.py index fb9fcf7c52..4febb48a60 100644 --- a/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.py +++ b/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.py @@ -1,8 +1,14 @@ # Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and contributors # For license information, please see license.txt -# import frappe +import frappe +from frappe import qb from frappe.model.document import Document +from frappe.query_builder.functions import Count +from frappe.utils import cint +from pypika import Order + +log_detail = qb.DocType("Bulk Transaction Log Detail") class BulkTransactionLog(Document): @@ -10,14 +16,51 @@ class BulkTransactionLog(Document): pass def load_from_db(self): - pass - - def db_update(self): - pass + succeeded_logs = ( + qb.from_(log_detail) + .select(Count(log_detail.date).as_("count")) + .where((log_detail.date == self.name) & (log_detail.transaction_status == "Success")) + .run() + )[0][0] or 0 + failed_logs = ( + qb.from_(log_detail) + .select(Count(log_detail.date).as_("count")) + .where((log_detail.date == self.name) & (log_detail.transaction_status == "Failed")) + .run() + )[0][0] or 0 + total_logs = succeeded_logs + failed_logs + transaction_log = frappe._dict( + { + "date": self.name, + "count": total_logs, + "succeeded": succeeded_logs, + "failed": failed_logs, + } + ) + super(Document, self).__init__(serialize_transaction_log(transaction_log)) @staticmethod def get_list(args): - pass + limit = cint(args.get("page_length")) or 20 + dates = ( + qb.from_(log_detail) + .select(log_detail.date) + .distinct() + .orderby(log_detail.date, order=Order.desc) + .limit(limit) + .run() + ) + + transaction_logs = ( + qb.from_(log_detail) + .select(log_detail.date.as_("date"), Count(log_detail.date).as_("count")) + .where(log_detail.date.isin(dates)) + .orderby(log_detail.date, order=Order.desc) + .groupby(log_detail.date) + .limit(limit) + .run(as_dict=True) + ) + return [serialize_transaction_log(x) for x in transaction_logs] @staticmethod def get_count(args): @@ -26,3 +69,19 @@ class BulkTransactionLog(Document): @staticmethod def get_stats(args): pass + + def db_update(self, *args, **kwargs): + pass + + def delete(self): + pass + + +def serialize_transaction_log(data): + return frappe._dict( + name=data.date, + date=data.date, + log_entries=data.count, + succeeded=data.succeeded, + failed=data.failed, + ) From 194d70f8a0c93c8e2952cd58cdc5d994414198fb Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Fri, 10 Nov 2023 11:44:25 +0530 Subject: [PATCH 10/16] chore: show retried in list view --- .../bulk_transaction_log_detail.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/erpnext/bulk_transaction/doctype/bulk_transaction_log_detail/bulk_transaction_log_detail.json b/erpnext/bulk_transaction/doctype/bulk_transaction_log_detail/bulk_transaction_log_detail.json index 5491f56a1a..9590325a06 100644 --- a/erpnext/bulk_transaction/doctype/bulk_transaction_log_detail/bulk_transaction_log_detail.json +++ b/erpnext/bulk_transaction/doctype/bulk_transaction_log_detail/bulk_transaction_log_detail.json @@ -73,6 +73,7 @@ { "fieldname": "retried", "fieldtype": "Int", + "in_list_view": 1, "label": "Retried", "read_only": 1 } @@ -80,7 +81,7 @@ "in_create": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2023-11-09 17:44:14.804836", + "modified": "2023-11-10 11:44:10.758342", "modified_by": "Administrator", "module": "Bulk Transaction", "name": "Bulk Transaction Log Detail", @@ -103,4 +104,4 @@ "sort_order": "DESC", "states": [], "track_changes": 1 -} +} \ No newline at end of file From 0aa1636d04bed9fef260aedde4ed6a42b6a1968b Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Fri, 10 Nov 2023 12:20:30 +0530 Subject: [PATCH 11/16] refactor: barebones retry functionality --- .../bulk_transaction_log.js | 8 ++++++ erpnext/utilities/bulk_transaction.py | 28 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.js b/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.js index 218424beca..6e2250052d 100644 --- a/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.js +++ b/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.js @@ -9,5 +9,13 @@ frappe.ui.form.on("Bulk Transaction Log", { frm.add_custom_button(__('Failed Entries'), function() { frappe.set_route('List', 'Bulk Transaction Log Detail', {'date': frm.doc.date, 'transaction_status': "Failed"}); }, __("View")); + if (frm.doc.failed) { + frm.add_custom_button(__('Retry Failed Transactions'), function() { + frappe.call({ + method: "erpnext.utilities.bulk_transaction.retry_failed_transactions", + args: {date: frm.doc.date} + }).then(()=> { }); + }); + } }, }); diff --git a/erpnext/utilities/bulk_transaction.py b/erpnext/utilities/bulk_transaction.py index fd8e722648..a596001583 100644 --- a/erpnext/utilities/bulk_transaction.py +++ b/erpnext/utilities/bulk_transaction.py @@ -29,6 +29,34 @@ def transaction_processing(data, from_doctype, to_doctype): job(deserialized_data, from_doctype, to_doctype) +@frappe.whitelist() +def retry_failed_transactions(date: str | None): + if date: + failed_docs = frappe.db.get_all( + "Bulk Transaction Log Detail", + filters={"date": date, "transaction_status": "Failed", "retried": 0}, + fields=["name", "transaction_name", "from_doctype", "to_doctype"], + ) + if not failed_docs: + frappe.msgprint("There are no Failed transactions") + return + + for log in failed_docs: + try: + frappe.db.savepoint("before_creation_state") + task(log.transaction_name, log.from_doctype, log.to_doctype) + except Exception as e: + frappe.db.rollback(save_point="before_creation_state") + update_log(log.name, "Failed", 1) + else: + update_log(log.name, "Success", 1) + + +def update_log(log_name, status, retried): + frappe.db.set_value("Bulk Transaction Log Detail", log_name, "transaction_status", status) + frappe.db.set_value("Bulk Transaction Log Detail", log_name, "retried", retried) + + def job(deserialized_data, from_doctype, to_doctype): fail_count = 0 for d in deserialized_data: From c3202886901eaecc0ab8b9bf73d8816994e6dc1f Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Fri, 10 Nov 2023 13:45:52 +0530 Subject: [PATCH 12/16] refactor: rollback for retries and UI alerts --- .../bulk_transaction_log.js | 2 +- erpnext/utilities/bulk_transaction.py | 20 +++++++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.js b/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.js index 6e2250052d..3135d41cc1 100644 --- a/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.js +++ b/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.js @@ -12,7 +12,7 @@ frappe.ui.form.on("Bulk Transaction Log", { if (frm.doc.failed) { frm.add_custom_button(__('Retry Failed Transactions'), function() { frappe.call({ - method: "erpnext.utilities.bulk_transaction.retry_failed_transactions", + method: "erpnext.utilities.bulk_transaction.retry", args: {date: frm.doc.date} }).then(()=> { }); }); diff --git a/erpnext/utilities/bulk_transaction.py b/erpnext/utilities/bulk_transaction.py index a596001583..7e73c2fe0b 100644 --- a/erpnext/utilities/bulk_transaction.py +++ b/erpnext/utilities/bulk_transaction.py @@ -3,7 +3,7 @@ from datetime import date, datetime import frappe from frappe import _ -from frappe.utils import today +from frappe.utils import get_link_to_form, today @frappe.whitelist() @@ -30,7 +30,7 @@ def transaction_processing(data, from_doctype, to_doctype): @frappe.whitelist() -def retry_failed_transactions(date: str | None): +def retry(date: str | None): if date: failed_docs = frappe.db.get_all( "Bulk Transaction Log Detail", @@ -38,9 +38,21 @@ def retry_failed_transactions(date: str | None): fields=["name", "transaction_name", "from_doctype", "to_doctype"], ) if not failed_docs: - frappe.msgprint("There are no Failed transactions") - return + frappe.msgprint(_("There are no Failed transactions")) + else: + job = frappe.enqueue( + retry_failed_transactions, + failed_docs=failed_docs, + ) + frappe.msgprint( + _("Job: {0} has been triggered for processing failed transactions").format( + get_link_to_form("RQ Job", job.id) + ) + ) + +def retry_failed_transactions(failed_docs: list | None): + if failed_docs: for log in failed_docs: try: frappe.db.savepoint("before_creation_state") From 73639db9105260d0d1af3cea35caa03523f396e5 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Fri, 10 Nov 2023 15:27:28 +0530 Subject: [PATCH 13/16] chore: resolve linting issue --- .../doctype/bulk_transaction_log/bulk_transaction_log.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.py b/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.py index 4febb48a60..1a078b53d5 100644 --- a/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.py +++ b/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.py @@ -8,14 +8,13 @@ from frappe.query_builder.functions import Count from frappe.utils import cint from pypika import Order -log_detail = qb.DocType("Bulk Transaction Log Detail") - class BulkTransactionLog(Document): def db_insert(self, *args, **kwargs): pass def load_from_db(self): + log_detail = qb.DocType("Bulk Transaction Log Detail") succeeded_logs = ( qb.from_(log_detail) .select(Count(log_detail.date).as_("count")) @@ -41,6 +40,7 @@ class BulkTransactionLog(Document): @staticmethod def get_list(args): + log_detail = qb.DocType("Bulk Transaction Log Detail") limit = cint(args.get("page_length")) or 20 dates = ( qb.from_(log_detail) From 93295bf25b67069936e23d03f5a1c559294bc25f Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Sat, 11 Nov 2023 05:10:16 +0530 Subject: [PATCH 14/16] refactor: support list view filters --- .../bulk_transaction_log.json | 8 +++- .../bulk_transaction_log.py | 42 +++++++++++++------ 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.json b/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.json index a874b791ca..75cb358ff2 100644 --- a/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.json +++ b/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.json @@ -2,6 +2,7 @@ "actions": [], "allow_copy": 1, "creation": "2023-11-09 20:14:45.139593", + "default_view": "List", "doctype": "DocType", "engine": "InnoDB", "field_order": [ @@ -17,6 +18,8 @@ { "fieldname": "date", "fieldtype": "Date", + "in_list_view": 1, + "in_standard_filter": 1, "label": "Date", "read_only": 1 }, @@ -55,7 +58,7 @@ "in_create": 1, "is_virtual": 1, "links": [], - "modified": "2023-11-10 11:13:52.733076", + "modified": "2023-11-11 04:52:49.347376", "modified_by": "Administrator", "module": "Bulk Transaction", "name": "Bulk Transaction Log", @@ -76,5 +79,6 @@ ], "sort_field": "modified", "sort_order": "DESC", - "states": [] + "states": [], + "title_field": "date" } \ No newline at end of file diff --git a/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.py b/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.py index 1a078b53d5..8ae54ddab8 100644 --- a/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.py +++ b/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.py @@ -40,26 +40,33 @@ class BulkTransactionLog(Document): @staticmethod def get_list(args): - log_detail = qb.DocType("Bulk Transaction Log Detail") + filter_date = parse_list_filters(args) limit = cint(args.get("page_length")) or 20 - dates = ( + log_detail = qb.DocType("Bulk Transaction Log Detail") + + dates_query = ( qb.from_(log_detail) .select(log_detail.date) .distinct() .orderby(log_detail.date, order=Order.desc) .limit(limit) - .run() ) + if filter_date: + dates_query = dates_query.where(log_detail.date == filter_date) + dates = dates_query.run() + + transaction_logs = [] + if dates: + transaction_logs_query = ( + qb.from_(log_detail) + .select(log_detail.date.as_("date"), Count(log_detail.date).as_("count")) + .where(log_detail.date.isin(dates)) + .orderby(log_detail.date, order=Order.desc) + .groupby(log_detail.date) + .limit(limit) + ) + transaction_logs = transaction_logs_query.run(as_dict=True) - transaction_logs = ( - qb.from_(log_detail) - .select(log_detail.date.as_("date"), Count(log_detail.date).as_("count")) - .where(log_detail.date.isin(dates)) - .orderby(log_detail.date, order=Order.desc) - .groupby(log_detail.date) - .limit(limit) - .run(as_dict=True) - ) return [serialize_transaction_log(x) for x in transaction_logs] @staticmethod @@ -85,3 +92,14 @@ def serialize_transaction_log(data): succeeded=data.succeeded, failed=data.failed, ) + + +def parse_list_filters(args): + # parse date filter + filter_date = None + for fil in args.get("filters"): + if isinstance(fil, list): + for elem in fil: + if elem == "date": + filter_date = fil[3] + return filter_date From a52a1b49af1c7b72ba85563057f60bb7cf54e81c Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Sat, 11 Nov 2023 05:20:27 +0530 Subject: [PATCH 15/16] refactor: update traceback on retry --- erpnext/utilities/bulk_transaction.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/erpnext/utilities/bulk_transaction.py b/erpnext/utilities/bulk_transaction.py index 7e73c2fe0b..402c305ca9 100644 --- a/erpnext/utilities/bulk_transaction.py +++ b/erpnext/utilities/bulk_transaction.py @@ -59,14 +59,16 @@ def retry_failed_transactions(failed_docs: list | None): task(log.transaction_name, log.from_doctype, log.to_doctype) except Exception as e: frappe.db.rollback(save_point="before_creation_state") - update_log(log.name, "Failed", 1) + update_log(log.name, "Failed", 1, str(frappe.get_traceback())) else: update_log(log.name, "Success", 1) -def update_log(log_name, status, retried): +def update_log(log_name, status, retried, err=None): frappe.db.set_value("Bulk Transaction Log Detail", log_name, "transaction_status", status) frappe.db.set_value("Bulk Transaction Log Detail", log_name, "retried", retried) + if err: + frappe.db.set_value("Bulk Transaction Log Detail", log_name, "error_description", err) def job(deserialized_data, from_doctype, to_doctype): From a393a6b76c35d95e4422fbf239c468ab48e67b71 Mon Sep 17 00:00:00 2001 From: ruthra kumar Date: Mon, 13 Nov 2023 10:39:40 +0530 Subject: [PATCH 16/16] refactor: raise exception on invalid date --- .../doctype/bulk_transaction_log/bulk_transaction_log.js | 2 +- .../doctype/bulk_transaction_log/bulk_transaction_log.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.js b/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.js index 3135d41cc1..dc54d606e7 100644 --- a/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.js +++ b/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.js @@ -14,7 +14,7 @@ frappe.ui.form.on("Bulk Transaction Log", { frappe.call({ method: "erpnext.utilities.bulk_transaction.retry", args: {date: frm.doc.date} - }).then(()=> { }); + }); }); } }, diff --git a/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.py b/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.py index 8ae54ddab8..712caf1f91 100644 --- a/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.py +++ b/erpnext/bulk_transaction/doctype/bulk_transaction_log/bulk_transaction_log.py @@ -15,6 +15,13 @@ class BulkTransactionLog(Document): def load_from_db(self): log_detail = qb.DocType("Bulk Transaction Log Detail") + + has_records = frappe.db.sql( + f"select exists (select * from `tabBulk Transaction Log Detail` where date = '{self.name}');" + )[0][0] + if not has_records: + raise frappe.DoesNotExistError + succeeded_logs = ( qb.from_(log_detail) .select(Count(log_detail.date).as_("count"))