Shiloh/brotherton-erpnext
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

perf: Optimization of gl entry processing logic in period closing voucher

Browse Source
This commit is contained in:
Nabin Hait 2022-07-18 17:59:42 +05:30
parent a8179220be
commit 4caaab32d1
5 changed files with 141 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
erpnext/accounts/doctype/gl_entry/gl_entry.py
View File

@ -42,7 +42,7 @@ class GLEntry(Document):
self.validate_and_set_fiscal_year() self.validate_and_set_fiscal_year()
self.pl_must_have_cost_center() self.pl_must_have_cost_center()
if not self.flags.from_repost: if not self.flags.from_repost and self.voucher_type != "Period Closing Voucher":
self.check_mandatory() self.check_mandatory()
self.validate_cost_center() self.validate_cost_center()
self.check_pl_account() self.check_pl_account()
@ -51,7 +51,7 @@ class GLEntry(Document):
def on_update(self): def on_update(self):
adv_adj = self.flags.adv_adj adv_adj = self.flags.adv_adj
if not self.flags.from_repost: if not self.flags.from_repost and self.voucher_type != "Period Closing Voucher":
self.validate_account_details(adv_adj) self.validate_account_details(adv_adj)
self.validate_dimensions_for_pl_and_bs() self.validate_dimensions_for_pl_and_bs()
self.validate_allowed_dimensions() self.validate_allowed_dimensions()

24
erpnext/accounts/doctype/period_closing_voucher/period_closing_voucher.json
View File

@ -10,10 +10,11 @@
"fiscal_year", "fiscal_year",
"amended_from", "amended_from",
"company", "company",
"cost_center_wise_pnl",
"column_break1", "column_break1",
"closing_account_head", "closing_account_head",
"remarks" "remarks",
"status",
"error_message"
], ],
"fields": [ "fields": [
{ {
@ -86,17 +87,26 @@
"reqd": 1 "reqd": 1
}, },
{ {
"default": "0", "depends_on": "eval:doc.docstatus!=0",
"fieldname": "cost_center_wise_pnl", "fieldname": "status",
"fieldtype": "Check", "fieldtype": "Select",
"label": "Book Cost Center Wise Profit/Loss" "label": "GL Entry Processing Status",
"options": "In Progress\nCompleted\nFailed",
"read_only": 1
},
{
"depends_on": "eval:doc.status=='Failed'",
"fieldname": "error_message",
"fieldtype": "Text",
"label": "Error Message",
"read_only": 1
} }
], ],
"icon": "fa fa-file-text", "icon": "fa fa-file-text",
"idx": 1, "idx": 1,
"is_submittable": 1, "is_submittable": 1,
"links": [], "links": [],
"modified": "2021-05-20 15:27:37.210458", "modified": "2022-07-15 14:51:04.714154",
"modified_by": "Administrator", "modified_by": "Administrator",
"module": "Accounts", "module": "Accounts",
"name": "Period Closing Voucher", "name": "Period Closing Voucher",

191
erpnext/accounts/doctype/period_closing_voucher/period_closing_voucher.py
View File

@ -20,13 +20,26 @@ class PeriodClosingVoucher(AccountsController):
self.validate_posting_date() self.validate_posting_date()
def on_submit(self): def on_submit(self):
self.status = "In Progress"
self.make_gl_entries() self.make_gl_entries()
def on_cancel(self): def on_cancel(self):
self.status = "In Progress"
self.ignore_linked_doctypes = ("GL Entry", "Stock Ledger Entry") self.ignore_linked_doctypes = ("GL Entry", "Stock Ledger Entry")
from erpnext.accounts.general_ledger import make_reverse_gl_entries gle_count = frappe.db.count(
"GL Entry",
make_reverse_gl_entries(voucher_type="Period Closing Voucher", voucher_no=self.name) {
"voucher_type": "Period Closing Voucher",
"voucher_no": self.name,
"is_cancelled": 0
}
)
if gle_count > 5000:
frappe.enqueue(make_reverse_gl_entries, voucher_type="Period Closing Voucher",
voucher_no=self.name, queue="long")
frappe.msgprint(_("The GL Entries will be cancelled in the background, it can take a few minutes."), alert=True)
else:
make_reverse_gl_entries(voucher_type="Period Closing Voucher", voucher_no=self.name)
def validate_account_head(self): def validate_account_head(self):
closing_account_type = frappe.db.get_value("Account", self.closing_account_head, "root_type") closing_account_type = frappe.db.get_value("Account", self.closing_account_head, "root_type")
@ -67,90 +80,85 @@ class PeriodClosingVoucher(AccountsController):
def make_gl_entries(self): def make_gl_entries(self):
gl_entries = self.get_gl_entries() gl_entries = self.get_gl_entries()
if gl_entries: if gl_entries:
from erpnext.accounts.general_ledger import make_gl_entries if len(gl_entries) > 5000:
frappe.enqueue(process_gl_entries, gl_entries=gl_entries, queue="long")
make_gl_entries(gl_entries) frappe.msgprint(_("The GL Entries will be processed in the background, it can take a few minutes."), alert=True)
else:
process_gl_entries(gl_entries)
def get_gl_entries(self): def get_gl_entries(self):
gl_entries = [] gl_entries = []
pl_accounts = self.get_pl_balances()
for acc in pl_accounts: # pl account
for acc in self.get_pl_balances_based_on_dimensions(group_by_account=True):
if flt(acc.bal_in_company_currency): if flt(acc.bal_in_company_currency):
gl_entries.append( gl_entries.append(self.get_gle_for_pl_account(acc))
self.get_gl_dict(
{
"account": acc.account,
"cost_center": acc.cost_center,
"finance_book": acc.finance_book,
"account_currency": acc.account_currency,
"debit_in_account_currency": abs(flt(acc.bal_in_account_currency))
if flt(acc.bal_in_account_currency) < 0
else 0,
"debit": abs(flt(acc.bal_in_company_currency))
if flt(acc.bal_in_company_currency) < 0
else 0,
"credit_in_account_currency": abs(flt(acc.bal_in_account_currency))
if flt(acc.bal_in_account_currency) > 0
else 0,
"credit": abs(flt(acc.bal_in_company_currency))
if flt(acc.bal_in_company_currency) > 0
else 0,
},
item=acc,
)
)
if gl_entries: # closing liability account
gle_for_net_pl_bal = self.get_pnl_gl_entry(pl_accounts) for acc in self.get_pl_balances_based_on_dimensions(group_by_account=False):
gl_entries += gle_for_net_pl_bal if flt(acc.bal_in_company_currency):
gl_entries.append(self.get_gle_for_closing_account(acc))
return gl_entries return gl_entries
def get_pnl_gl_entry(self, pl_accounts): def get_gle_for_pl_account(self, acc):
company_cost_center = frappe.db.get_value("Company", self.company, "cost_center") gl_entry = self.get_gl_dict(
gl_entries = [] {
"account": acc.account,
"cost_center": acc.cost_center,
"finance_book": acc.finance_book,
"account_currency": acc.account_currency,
"debit_in_account_currency": abs(flt(acc.bal_in_account_currency))
if flt(acc.bal_in_account_currency) < 0
else 0,
"debit": abs(flt(acc.bal_in_company_currency))
if flt(acc.bal_in_company_currency) < 0
else 0,
"credit_in_account_currency": abs(flt(acc.bal_in_account_currency))
if flt(acc.bal_in_account_currency) > 0
else 0,
"credit": abs(flt(acc.bal_in_company_currency))
if flt(acc.bal_in_company_currency) > 0
else 0,
},
item=acc,
)
self.update_default_dimensions(gl_entry, acc)
return gl_entry
for acc in pl_accounts: def get_gle_for_closing_account(self, acc):
if flt(acc.bal_in_company_currency): gl_entry = self.get_gl_dict(
cost_center = acc.cost_center if self.cost_center_wise_pnl else company_cost_center {
gl_entry = self.get_gl_dict( "account": self.closing_account_head,
{ "cost_center": acc.cost_center,
"account": self.closing_account_head, "finance_book": acc.finance_book,
"cost_center": cost_center, "account_currency": acc.account_currency,
"finance_book": acc.finance_book, "debit_in_account_currency": abs(flt(acc.bal_in_account_currency))
"account_currency": acc.account_currency, if flt(acc.bal_in_account_currency) > 0
"debit_in_account_currency": abs(flt(acc.bal_in_account_currency)) else 0,
if flt(acc.bal_in_account_currency) > 0 "debit": abs(flt(acc.bal_in_company_currency))
else 0, if flt(acc.bal_in_company_currency) > 0
"debit": abs(flt(acc.bal_in_company_currency)) else 0,
if flt(acc.bal_in_company_currency) > 0 "credit_in_account_currency": abs(flt(acc.bal_in_account_currency))
else 0, if flt(acc.bal_in_account_currency) < 0
"credit_in_account_currency": abs(flt(acc.bal_in_account_currency)) else 0,
if flt(acc.bal_in_account_currency) < 0 "credit": abs(flt(acc.bal_in_company_currency))
else 0, if flt(acc.bal_in_company_currency) < 0
"credit": abs(flt(acc.bal_in_company_currency)) else 0,
if flt(acc.bal_in_company_currency) < 0 },
else 0, item=acc,
}, )
item=acc, self.update_default_dimensions(gl_entry, acc)
) return gl_entry
self.update_default_dimensions(gl_entry) def update_default_dimensions(self, gl_entry, acc):
gl_entries.append(gl_entry)
return gl_entries
def update_default_dimensions(self, gl_entry):
if not self.accounting_dimensions: if not self.accounting_dimensions:
self.accounting_dimensions = get_accounting_dimensions() self.accounting_dimensions = get_accounting_dimensions()
_, default_dimensions = get_dimensions()
for dimension in self.accounting_dimensions: for dimension in self.accounting_dimensions:
gl_entry.update({dimension: default_dimensions.get(self.company, {}).get(dimension)}) gl_entry.update({dimension: acc.get(dimension)})
def get_pl_balances(self): def get_pl_balances_based_on_dimensions(self, group_by_account=False):
"""Get balance for dimension-wise pl accounts""" """Get balance for dimension-wise pl accounts"""
dimension_fields = ["t1.cost_center", "t1.finance_book"] dimension_fields = ["t1.cost_center", "t1.finance_book"]
@ -159,20 +167,47 @@ class PeriodClosingVoucher(AccountsController):
for dimension in self.accounting_dimensions: for dimension in self.accounting_dimensions:
dimension_fields.append("t1.{0}".format(dimension)) dimension_fields.append("t1.{0}".format(dimension))
return frappe.db.sql( if group_by_account:
""" dimension_fields.append("t1.account")
return frappe.db.sql("""
select select
t1.account, t2.account_currency, {dimension_fields}, t2.account_currency,
{dimension_fields},
sum(t1.debit_in_account_currency) - sum(t1.credit_in_account_currency) as bal_in_account_currency, sum(t1.debit_in_account_currency) - sum(t1.credit_in_account_currency) as bal_in_account_currency,
sum(t1.debit) - sum(t1.credit) as bal_in_company_currency sum(t1.debit) - sum(t1.credit) as bal_in_company_currency
from `tabGL Entry` t1, `tabAccount` t2 from `tabGL Entry` t1, `tabAccount` t2
where t1.is_cancelled = 0 and t1.account = t2.name and t2.report_type = 'Profit and Loss' where
and t2.docstatus < 2 and t2.company = %s t1.is_cancelled = 0
and t1.posting_date between %s and %s and t1.account = t2.name
group by t1.account, {dimension_fields} and t2.report_type = 'Profit and Loss'
and t2.docstatus < 2
and t2.company = %s
and t1.posting_date between %s and %s
group by {dimension_fields}
""".format( """.format(
dimension_fields=", ".join(dimension_fields) dimension_fields=", ".join(dimension_fields)
), ),
(self.company, self.get("year_start_date"), self.posting_date), (self.company, self.get("year_start_date"), self.posting_date),
as_dict=1, as_dict=1,
) )
def process_gl_entries(gl_entries):
from erpnext.accounts.general_ledger import make_gl_entries
try:
make_gl_entries(gl_entries, merge_entries=False)
frappe.db.set_value("Period Closing Voucher", gl_entries[0].get("voucher_no"), "status", "Completed")
except Exception as e:
frappe.db.rollback()
frappe.log_error(e)
frappe.db.set_value("Period Closing Voucher", gl_entries[0].get("voucher_no"), "status", "Failed")
def make_reverse_gl_entries(voucher_type, voucher_no):
from erpnext.accounts.general_ledger import make_reverse_gl_entries
try:
make_reverse_gl_entries(voucher_type=voucher_type, voucher_no=voucher_no)
frappe.db.set_value("Period Closing Voucher", voucher_no, "status", "Completed")
except Exception as e:
frappe.db.rollback()
frappe.log_error(e)
frappe.db.set_value("Period Closing Voucher", gl_entries[0].get("voucher_no"), "status", "Failed")

3
erpnext/accounts/doctype/period_closing_voucher/test_period_closing_voucher.py
View File

@ -49,7 +49,7 @@ class TestPeriodClosingVoucher(unittest.TestCase):
expected_gle = ( expected_gle = (
("Cost of Goods Sold - TPC", 0.0, 600.0), ("Cost of Goods Sold - TPC", 0.0, 600.0),
(surplus_account, 600.0, 400.0), (surplus_account, 200.0, 0.0),
("Sales - TPC", 400.0, 0.0), ("Sales - TPC", 400.0, 0.0),
) )
@ -59,7 +59,6 @@ class TestPeriodClosingVoucher(unittest.TestCase):
""", """,
(pcv.name), (pcv.name),
) )
self.assertEqual(pcv_gle, expected_gle) self.assertEqual(pcv_gle, expected_gle)
def test_cost_center_wise_posting(self): def test_cost_center_wise_posting(self):

5
erpnext/accounts/general_ledger.py
View File

@ -168,6 +168,7 @@ def get_cost_center_allocation_data(company, posting_date):
def merge_similar_entries(gl_map, precision=None): def merge_similar_entries(gl_map, precision=None):
merged_gl_map = [] merged_gl_map = []
accounting_dimensions = get_accounting_dimensions() accounting_dimensions = get_accounting_dimensions()
for entry in gl_map: for entry in gl_map:
# if there is already an entry in this account then just add it # if there is already an entry in this account then just add it
# to that entry # to that entry
@ -290,7 +291,6 @@ def save_entries(gl_map, adv_adj, update_outstanding, from_repost=False):
for entry in gl_map: for entry in gl_map:
make_entry(entry, adv_adj, update_outstanding, from_repost) make_entry(entry, adv_adj, update_outstanding, from_repost)
def make_entry(args, adv_adj, update_outstanding, from_repost=False): def make_entry(args, adv_adj, update_outstanding, from_repost=False):
gle = frappe.new_doc("GL Entry") gle = frappe.new_doc("GL Entry")
gle.update(args) gle.update(args)
@ -298,9 +298,10 @@ def make_entry(args, adv_adj, update_outstanding, from_repost=False):
gle.flags.from_repost = from_repost gle.flags.from_repost = from_repost
gle.flags.adv_adj = adv_adj gle.flags.adv_adj = adv_adj
gle.flags.update_outstanding = update_outstanding or "Yes" gle.flags.update_outstanding = update_outstanding or "Yes"
gle.flags.notify_update = False
gle.submit() gle.submit()
if not from_repost: if not from_repost and gle.voucher_type != "Period Closing Voucher":
validate_expense_against_budget(args) validate_expense_against_budget(args)