From 9784d27317cf8e426c366a60438f4b40591124eb Mon Sep 17 00:00:00 2001 From: Nabin Hait Date: Fri, 30 Dec 2016 16:21:35 +0530 Subject: [PATCH] Optimization to reduce GLE reposting time for future stock transactions --- .../doctype/fiscal_year/fiscal_year.py | 2 + erpnext/accounts/doctype/gl_entry/gl_entry.py | 23 +++--- .../period_closing_voucher.py | 4 +- .../test_period_closing_voucher.py | 4 +- .../purchase_invoice/purchase_invoice.py | 6 +- .../doctype/sales_invoice/sales_invoice.py | 6 +- erpnext/accounts/general_ledger.py | 21 ++++-- .../accounts/report/cash_flow/cash_flow.py | 9 ++- .../accounts/report/financial_statements.py | 5 +- erpnext/accounts/utils.py | 75 +++++++++++++------ erpnext/controllers/accounts_controller.py | 2 +- erpnext/controllers/stock_controller.py | 29 ++++--- .../vehicle_expenses/vehicle_expenses.py | 2 +- .../process_payroll/process_payroll.py | 7 +- .../doctype/salary_slip/test_salary_slip.py | 4 +- .../stock_ledger_entry/stock_ledger_entry.py | 7 +- 16 files changed, 128 insertions(+), 78 deletions(-) diff --git a/erpnext/accounts/doctype/fiscal_year/fiscal_year.py b/erpnext/accounts/doctype/fiscal_year/fiscal_year.py index 2ff98da983..517a2a332e 100644 --- a/erpnext/accounts/doctype/fiscal_year/fiscal_year.py +++ b/erpnext/accounts/doctype/fiscal_year/fiscal_year.py @@ -43,11 +43,13 @@ class FiscalYear(Document): def on_update(self): check_duplicate_fiscal_year(self) + frappe.cache().delete_value("fiscal_years") def on_trash(self): global_defaults = frappe.get_doc("Global Defaults") if global_defaults.current_fiscal_year == self.name: frappe.throw(_("You cannot delete Fiscal Year {0}. Fiscal Year {0} is set as default in Global Settings").format(self.name)) + frappe.cache().delete_value("fiscal_years") def validate_overlap(self): existing_fiscal_years = frappe.db.sql("""select name from `tabFiscal Year` diff --git a/erpnext/accounts/doctype/gl_entry/gl_entry.py b/erpnext/accounts/doctype/gl_entry/gl_entry.py index 5773813955..ce60298c17 100644 --- a/erpnext/accounts/doctype/gl_entry/gl_entry.py +++ b/erpnext/accounts/doctype/gl_entry/gl_entry.py @@ -18,22 +18,27 @@ class GLEntry(Document): def validate(self): self.flags.ignore_submit_comment = True self.check_mandatory() - self.pl_must_have_cost_center() - self.check_pl_account() - self.validate_cost_center() - self.validate_party() - self.validate_currency() self.validate_and_set_fiscal_year() + + if not self.flags.from_repost: + self.pl_must_have_cost_center() + self.check_pl_account() + self.validate_cost_center() + self.validate_party() + self.validate_currency() - def on_update_with_args(self, adv_adj, update_outstanding = 'Yes'): - self.validate_account_details(adv_adj) + + def on_update_with_args(self, adv_adj, update_outstanding = 'Yes', from_repost=False): + if not from_repost: + self.validate_account_details(adv_adj) + check_freezing_date(self.posting_date, adv_adj) + validate_frozen_account(self.account, adv_adj) - check_freezing_date(self.posting_date, adv_adj) validate_balance_type(self.account, adv_adj) # Update outstanding amt on against voucher if self.against_voucher_type in ['Journal Entry', 'Sales Invoice', 'Purchase Invoice'] \ - and self.against_voucher and update_outstanding == 'Yes': + and self.against_voucher and update_outstanding == 'Yes' and not from_repost: update_outstanding_amt(self.account, self.party_type, self.party, self.against_voucher_type, self.against_voucher) diff --git a/erpnext/accounts/doctype/period_closing_voucher/period_closing_voucher.py b/erpnext/accounts/doctype/period_closing_voucher/period_closing_voucher.py index 04d4ed7078..03d0918226 100644 --- a/erpnext/accounts/doctype/period_closing_voucher/period_closing_voucher.py +++ b/erpnext/accounts/doctype/period_closing_voucher/period_closing_voucher.py @@ -35,9 +35,9 @@ class PeriodClosingVoucher(AccountsController): def validate_posting_date(self): from erpnext.accounts.utils import get_fiscal_year, validate_fiscal_year - validate_fiscal_year(self.posting_date, self.fiscal_year, label=_("Posting Date"), doc=self) + validate_fiscal_year(self.posting_date, self.fiscal_year, self.company, label=_("Posting Date"), doc=self) - self.year_start_date = get_fiscal_year(self.posting_date, self.fiscal_year)[1] + self.year_start_date = get_fiscal_year(self.posting_date, self.fiscal_year, company=self.company)[1] pce = frappe.db.sql("""select name from `tabPeriod Closing Voucher` where posting_date > %s and fiscal_year = %s and docstatus = 1""", diff --git a/erpnext/accounts/doctype/period_closing_voucher/test_period_closing_voucher.py b/erpnext/accounts/doctype/period_closing_voucher/test_period_closing_voucher.py index d68e29157b..9ef66edc34 100644 --- a/erpnext/accounts/doctype/period_closing_voucher/test_period_closing_voucher.py +++ b/erpnext/accounts/doctype/period_closing_voucher/test_period_closing_voucher.py @@ -11,7 +11,7 @@ from erpnext.accounts.doctype.journal_entry.test_journal_entry import make_journ class TestPeriodClosingVoucher(unittest.TestCase): def test_closing_entry(self): - year_start_date = get_fiscal_year(today())[1] + year_start_date = get_fiscal_year(today(), company="_Test Company")[1] make_journal_entry("_Test Bank - _TC", "Sales - _TC", 400, "_Test Cost Center - _TC", posting_date=now(), submit=True) @@ -70,7 +70,7 @@ class TestPeriodClosingVoucher(unittest.TestCase): "doctype": "Period Closing Voucher", "closing_account_head": "_Test Account Reserves and Surplus - _TC", "company": "_Test Company", - "fiscal_year": get_fiscal_year(today())[0], + "fiscal_year": get_fiscal_year(today(), company="_Test Company")[0], "posting_date": today(), "remarks": "test" }) diff --git a/erpnext/accounts/doctype/purchase_invoice/purchase_invoice.py b/erpnext/accounts/doctype/purchase_invoice/purchase_invoice.py index 14459fa497..c0693d14a6 100644 --- a/erpnext/accounts/doctype/purchase_invoice/purchase_invoice.py +++ b/erpnext/accounts/doctype/purchase_invoice/purchase_invoice.py @@ -302,11 +302,11 @@ class PurchaseInvoice(BuyingController): asset.flags.ignore_validate_update_after_submit = True asset.save() - def make_gl_entries(self, repost_future_gle=True): + def make_gl_entries(self, gl_entries=None, repost_future_gle=True, from_repost=False): if not self.grand_total: return - - gl_entries = self.get_gl_entries() + if not gl_entries: + gl_entries = self.get_gl_entries() if gl_entries: update_outstanding = "No" if (cint(self.is_paid) or self.write_off_account) else "Yes" diff --git a/erpnext/accounts/doctype/sales_invoice/sales_invoice.py b/erpnext/accounts/doctype/sales_invoice/sales_invoice.py index db6d594580..64425c8d88 100644 --- a/erpnext/accounts/doctype/sales_invoice/sales_invoice.py +++ b/erpnext/accounts/doctype/sales_invoice/sales_invoice.py @@ -532,10 +532,12 @@ class SalesInvoice(SellingController): if d.delivery_note and frappe.db.get_value("Delivery Note", d.delivery_note, "docstatus") != 1: throw(_("Delivery Note {0} is not submitted").format(d.delivery_note)) - def make_gl_entries(self, repost_future_gle=True): + def make_gl_entries(self, gl_entries=None, repost_future_gle=True, from_repost=False): if not self.grand_total: return - gl_entries = self.get_gl_entries() + + if not gl_entries: + gl_entries = self.get_gl_entries() if gl_entries: from erpnext.accounts.general_ledger import make_gl_entries diff --git a/erpnext/accounts/general_ledger.py b/erpnext/accounts/general_ledger.py index 392902c935..4c5535d69a 100644 --- a/erpnext/accounts/general_ledger.py +++ b/erpnext/accounts/general_ledger.py @@ -11,12 +11,12 @@ from erpnext.accounts.doctype.budget.budget import validate_expense_against_budg class StockAccountInvalidTransaction(frappe.ValidationError): pass -def make_gl_entries(gl_map, cancel=False, adv_adj=False, merge_entries=True, update_outstanding='Yes'): +def make_gl_entries(gl_map, cancel=False, adv_adj=False, merge_entries=True, update_outstanding='Yes', from_repost=False): if gl_map: if not cancel: gl_map = process_gl_map(gl_map, merge_entries) if gl_map and len(gl_map) > 1: - save_entries(gl_map, adv_adj, update_outstanding) + save_entries(gl_map, adv_adj, update_outstanding, from_repost) else: frappe.throw(_("Incorrect number of General Ledger Entries found. You might have selected a wrong Account in the transaction.")) else: @@ -78,21 +78,26 @@ def check_if_in_list(gle, gl_map): and cstr(e.get('project')) == cstr(gle.get('project')): return e -def save_entries(gl_map, adv_adj, update_outstanding): - validate_account_for_auto_accounting_for_stock(gl_map) +def save_entries(gl_map, adv_adj, update_outstanding, from_repost=False): + if not from_repost: + validate_account_for_auto_accounting_for_stock(gl_map) + round_off_debit_credit(gl_map) for entry in gl_map: - make_entry(entry, adv_adj, update_outstanding) + make_entry(entry, adv_adj, update_outstanding, from_repost) + # check against budget - validate_expense_against_budget(entry) + if not from_repost: + validate_expense_against_budget(entry) -def make_entry(args, adv_adj, update_outstanding): +def make_entry(args, adv_adj, update_outstanding, from_repost=False): args.update({"doctype": "GL Entry"}) gle = frappe.get_doc(args) gle.flags.ignore_permissions = 1 + gle.flags.from_repost = from_repost gle.insert() - gle.run_method("on_update_with_args", adv_adj, update_outstanding) + gle.run_method("on_update_with_args", adv_adj, update_outstanding, from_repost) gle.submit() def validate_account_for_auto_accounting_for_stock(gl_map): diff --git a/erpnext/accounts/report/cash_flow/cash_flow.py b/erpnext/accounts/report/cash_flow/cash_flow.py index 182878af37..7a776f57e1 100644 --- a/erpnext/accounts/report/cash_flow/cash_flow.py +++ b/erpnext/accounts/report/cash_flow/cash_flow.py @@ -10,7 +10,8 @@ from erpnext.accounts.utils import get_fiscal_year def execute(filters=None): - period_list = get_period_list(filters.from_fiscal_year, filters.to_fiscal_year, filters.periodicity) + period_list = get_period_list(filters.from_fiscal_year, filters.to_fiscal_year, + filters.periodicity) operation_accounts = { "section_name": "Operations", @@ -103,7 +104,7 @@ def get_account_type_based_data(company, account_type, period_list, accumulated_ data = {} total = 0 for period in period_list: - start_date = get_start_date(period, accumulated_values) + start_date = get_start_date(period, accumulated_values, company) gl_sum = frappe.db.sql_list(""" select sum(credit) - sum(debit) from `tabGL Entry` @@ -126,10 +127,10 @@ def get_account_type_based_data(company, account_type, period_list, accumulated_ data["total"] = total return data -def get_start_date(period, accumulated_values): +def get_start_date(period, accumulated_values, company): start_date = period["year_start_date"] if accumulated_values: - start_date = get_fiscal_year(period.to_date)[1] + start_date = get_fiscal_year(period.to_date, company=company)[1] return start_date diff --git a/erpnext/accounts/report/financial_statements.py b/erpnext/accounts/report/financial_statements.py index bc4a220faa..c897d1ca61 100644 --- a/erpnext/accounts/report/financial_statements.py +++ b/erpnext/accounts/report/financial_statements.py @@ -3,10 +3,8 @@ from __future__ import unicode_literals import frappe -import math from frappe import _ -from frappe.utils import (flt, getdate, get_first_day, get_last_day, date_diff, - add_months, add_days, formatdate, cint) +from frappe.utils import flt, getdate, get_first_day, add_months, add_days, formatdate def get_period_list(from_fiscal_year, to_fiscal_year, periodicity): """Get a list of dict {"from_date": from_date, "to_date": to_date, "key": key, "label": label} @@ -149,7 +147,6 @@ def calculate_values(accounts_by_name, gl_entries_by_account, period_list, accum def get_date_fiscal_year(date): from erpnext.accounts.utils import get_fiscal_year - return get_fiscal_year(date)[0] def accumulate_values_into_parents(accounts, accounts_by_name, period_list, accumulated_values): diff --git a/erpnext/accounts/utils.py b/erpnext/accounts/utils.py index 8e59d8f99b..187fc24142 100644 --- a/erpnext/accounts/utils.py +++ b/erpnext/accounts/utils.py @@ -20,32 +20,63 @@ def get_fiscal_year(date=None, fiscal_year=None, label="Date", verbose=1, compan return get_fiscal_years(date, fiscal_year, label, verbose, company, as_dict=as_dict)[0] def get_fiscal_years(transaction_date=None, fiscal_year=None, label="Date", verbose=1, company=None, as_dict=False): - # if year start date is 2012-04-01, year end date should be 2013-03-31 (hence subdate) - cond = " disabled = 0" - if fiscal_year: - cond += " and fy.name = %(fiscal_year)s" - else: - cond += " and %(transaction_date)s >= fy.year_start_date and %(transaction_date)s <= fy.year_end_date" + fiscal_years = frappe.cache().hget("fiscal_years", company) or [] + + if not fiscal_years: + # if year start date is 2012-04-01, year end date should be 2013-03-31 (hence subdate) + cond = "" + if fiscal_year: + cond += " and fy.name = {0}".format(frappe.db.escape(fiscal_year)) + if company: + cond += """ + and (not exists (select name + from `tabFiscal Year Company` fyc + where fyc.parent = fy.name) + or exists(select company + from `tabFiscal Year Company` fyc + where fyc.parent = fy.name + and fyc.company=%(company)s) + ) + """ - if company: - cond += """ and (not exists(select name from `tabFiscal Year Company` fyc where fyc.parent = fy.name) - or exists(select company from `tabFiscal Year Company` fyc where fyc.parent = fy.name and fyc.company=%(company)s ))""" + fiscal_years = frappe.db.sql(""" + select + fy.name, fy.year_start_date, fy.year_end_date + from + `tabFiscal Year` fy + where + disabled = 0 {0} + order by + fy.year_start_date desc""".format(cond), { + "company": company + }, as_dict=True) + + frappe.cache().hset("fiscal_years", company, fiscal_years) - fy = frappe.db.sql("""select fy.name, fy.year_start_date, fy.year_end_date from `tabFiscal Year` fy - where %s order by fy.year_start_date desc""" % cond, { - "fiscal_year": fiscal_year, - "transaction_date": transaction_date, - "company": company - }, as_dict=as_dict) + if transaction_date: + transaction_date = getdate(transaction_date) - if not fy: - error_msg = _("""{0} {1} not in any active Fiscal Year. For more details check {2}.""").format(label, formatdate(transaction_date), "https://frappe.github.io/erpnext/user/manual/en/accounts/articles/fiscal-year-error") - if verbose==1: frappe.msgprint(error_msg) - raise FiscalYearError, error_msg - return fy + for fy in fiscal_years: + matched = False + if fiscal_year and fy.name == fiscal_year: + matched = True -def validate_fiscal_year(date, fiscal_year, label=_("Date"), doc=None): - years = [f[0] for f in get_fiscal_years(date, label=label)] + if (transaction_date and getdate(fy.year_start_date) <= transaction_date + and getdate(fy.year_end_date) >= transaction_date): + matched = True + + if matched: + if as_dict: + return (fy,) + else: + return ((fy.name, fy.year_start_date, fy.year_end_date),) + + error_msg = _("""{0} {1} not in any active Fiscal Year.""").format(label, formatdate(transaction_date)) + if verbose==1: frappe.msgprint(error_msg) + raise FiscalYearError, error_msg + +def validate_fiscal_year(date, fiscal_year, company, label=_("Date"), doc=None): + years = [f[0] for f in get_fiscal_years(date, label=label, company=company)] if fiscal_year not in years: if doc: doc.fiscal_year = years[0] diff --git a/erpnext/controllers/accounts_controller.py b/erpnext/controllers/accounts_controller.py index 554529c6f6..da0591134e 100644 --- a/erpnext/controllers/accounts_controller.py +++ b/erpnext/controllers/accounts_controller.py @@ -113,7 +113,7 @@ class AccountsController(TransactionBase): date_field = "transaction_date" if date_field and self.get(date_field): - validate_fiscal_year(self.get(date_field), self.fiscal_year, + validate_fiscal_year(self.get(date_field), self.fiscal_year, self.company, self.meta.get_label(date_field), self) def validate_due_date(self): diff --git a/erpnext/controllers/stock_controller.py b/erpnext/controllers/stock_controller.py index f0c8dbfc16..3e900b2814 100644 --- a/erpnext/controllers/stock_controller.py +++ b/erpnext/controllers/stock_controller.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals import frappe -from frappe.utils import cint, flt, cstr +from frappe.utils import cint, flt, cstr, now from frappe import msgprint, _ import frappe.defaults from erpnext.accounts.utils import get_fiscal_year @@ -15,7 +15,7 @@ class StockController(AccountsController): super(StockController, self).validate() self.validate_inspection() - def make_gl_entries(self, repost_future_gle=True): + def make_gl_entries(self, gl_entries=None, repost_future_gle=True, from_repost=False): if self.docstatus == 2: delete_gl_entries(voucher_type=self.doctype, voucher_no=self.name) @@ -23,8 +23,9 @@ class StockController(AccountsController): warehouse_account = get_warehouse_account() if self.docstatus==1: - gl_entries = self.get_gl_entries(warehouse_account) - make_gl_entries(gl_entries) + if not gl_entries: + gl_entries = self.get_gl_entries(warehouse_account) + make_gl_entries(gl_entries, from_repost=from_repost) if repost_future_gle: items, warehouses = self.get_items_and_warehouses() @@ -224,7 +225,7 @@ class StockController(AccountsController): def make_gl_entries_on_cancel(self, repost_future_gle=True): if frappe.db.sql("""select name from `tabGL Entry` where voucher_type=%s and voucher_no=%s""", (self.doctype, self.name)): - self.make_gl_entries(repost_future_gle) + self.make_gl_entries(repost_future_gle=repost_future_gle) def get_serialized_items(self): serialized_items = [] @@ -308,7 +309,7 @@ def update_gl_entries_after(posting_date, posting_time, for_warehouses=None, for if expected_gle: if not existing_gle or not compare_existing_and_expected_gle(existing_gle, expected_gle): _delete_gl_entries(voucher_type, voucher_no) - voucher_obj.make_gl_entries(repost_future_gle=False) + voucher_obj.make_gl_entries(gl_entries=expected_gle, repost_future_gle=False, from_repost=True) else: _delete_gl_entries(voucher_type, voucher_no) @@ -363,10 +364,14 @@ def get_voucherwise_gl_entries(future_stock_vouchers, posting_date): return gl_entries def get_warehouse_account(): - warehouse_account = frappe._dict() + if not frappe.flags.warehouse_account_map: + warehouse_account = frappe._dict() - for d in frappe.db.sql("""select warehouse, name, account_currency from tabAccount - where account_type = 'Stock' and (warehouse is not null and warehouse != '' - and is_group != 1) and is_group=0 """, as_dict=1): - warehouse_account.setdefault(d.warehouse, d) - return warehouse_account + for d in frappe.db.sql("""select warehouse, name, account_currency from tabAccount + where account_type = 'Stock' and (warehouse is not null and warehouse != '' + and is_group != 1) and is_group=0 """, as_dict=1): + warehouse_account.setdefault(d.warehouse, d) + + frappe.flags.warehouse_account_map = warehouse_account + + return frappe.flags.warehouse_account_map diff --git a/erpnext/fleet_management/report/vehicle_expenses/vehicle_expenses.py b/erpnext/fleet_management/report/vehicle_expenses/vehicle_expenses.py index 717a94f271..a03b7f3e13 100644 --- a/erpnext/fleet_management/report/vehicle_expenses/vehicle_expenses.py +++ b/erpnext/fleet_management/report/vehicle_expenses/vehicle_expenses.py @@ -15,7 +15,7 @@ def execute(filters=None): columns=get_columns() data=get_log_data(filters) chart=get_chart_data(data,period_list) - return columns,data,None,chart + return columns, data, None, chart def get_columns(): columns = [_("License") + ":Link/Vehicle:100", _("Make") + ":data:50", diff --git a/erpnext/hr/doctype/process_payroll/process_payroll.py b/erpnext/hr/doctype/process_payroll/process_payroll.py index 45030cba13..0d9fc7ccdb 100644 --- a/erpnext/hr/doctype/process_payroll/process_payroll.py +++ b/erpnext/hr/doctype/process_payroll/process_payroll.py @@ -284,17 +284,18 @@ class ProcessPayroll(Document): frappe.db.set_value("Salary Slip", ss_obj.name, "journal_entry", jv_name) def set_start_end_dates(self): - self.update(get_start_end_dates(self.payroll_frequency, self.start_date or self.posting_date)) + self.update(get_start_end_dates(self.payroll_frequency, + self.start_date or self.posting_date, self.company)) @frappe.whitelist() -def get_start_end_dates(payroll_frequency, start_date=None): +def get_start_end_dates(payroll_frequency, start_date=None, company=None): '''Returns dict of start and end dates for given payroll frequency based on start_date''' if not payroll_frequency: frappe.throw(_("Please set Payroll Frequency first")) if payroll_frequency == "Monthly" or payroll_frequency == "Bimonthly": - fiscal_year = get_fiscal_year(start_date)[0] + fiscal_year = get_fiscal_year(start_date, company=company)[0] month = "%02d" % getdate(start_date).month m = get_month_details(fiscal_year, month) if payroll_frequency == "Bimonthly": diff --git a/erpnext/hr/doctype/salary_slip/test_salary_slip.py b/erpnext/hr/doctype/salary_slip/test_salary_slip.py index 54faa64053..d1a8d81da8 100644 --- a/erpnext/hr/doctype/salary_slip/test_salary_slip.py +++ b/erpnext/hr/doctype/salary_slip/test_salary_slip.py @@ -134,7 +134,7 @@ class TestSalarySlip(unittest.TestCase): self.assertTrue(email_queue) def test_payroll_frequency(self): - fiscal_year = get_fiscal_year(nowdate())[0] + fiscal_year = get_fiscal_year(nowdate(), company="_Test Company")[0] month = "%02d" % getdate(nowdate()).month m = get_month_details(fiscal_year, month) @@ -185,7 +185,7 @@ class TestSalarySlip(unittest.TestCase): }).insert() def make_holiday_list(self): - fiscal_year = get_fiscal_year(nowdate()) + fiscal_year = get_fiscal_year(nowdate(), company="_Test Company") if not frappe.db.get_value("Holiday List", "Salary Slip Test Holiday List"): holiday_list = frappe.get_doc({ "doctype": "Holiday List", diff --git a/erpnext/stock/doctype/stock_ledger_entry/stock_ledger_entry.py b/erpnext/stock/doctype/stock_ledger_entry/stock_ledger_entry.py index 2caabee7b4..00e3abec71 100644 --- a/erpnext/stock/doctype/stock_ledger_entry/stock_ledger_entry.py +++ b/erpnext/stock/doctype/stock_ledger_entry/stock_ledger_entry.py @@ -27,9 +27,6 @@ class StockLedgerEntry(Document): self.validate_and_set_fiscal_year() self.block_transactions_against_group_warehouse() - from erpnext.accounts.utils import validate_fiscal_year - validate_fiscal_year(self.posting_date, self.fiscal_year, self.meta.get_label("posting_date"), self) - def on_submit(self): self.check_stock_frozen_date() self.actual_amt_check() @@ -117,6 +114,10 @@ class StockLedgerEntry(Document): def validate_and_set_fiscal_year(self): if not self.fiscal_year: self.fiscal_year = get_fiscal_year(self.posting_date, company=self.company)[0] + else: + from erpnext.accounts.utils import validate_fiscal_year + validate_fiscal_year(self.posting_date, self.fiscal_year, self.company, + self.meta.get_label("posting_date"), self) def block_transactions_against_group_warehouse(self): from erpnext.stock.utils import is_group_warehouse