Merge pull request #7440 from nabinhait/gle_repost_optimization

Optimization to reduce GLE reposting time for future stock transactions

erpnext/accounts/doctype/fiscal_year/fiscal_year.py

@@ -43,11 +43,13 @@ class FiscalYear(Document):
 
 	def on_update(self):
 		check_duplicate_fiscal_year(self)
+		frappe.cache().delete_value("fiscal_years")
 	
 	def on_trash(self):
 		global_defaults = frappe.get_doc("Global Defaults")
 		if global_defaults.current_fiscal_year == self.name:
 			frappe.throw(_("You cannot delete Fiscal Year {0}. Fiscal Year {0} is set as default in Global Settings").format(self.name))
+		frappe.cache().delete_value("fiscal_years")
 
 	def validate_overlap(self):
 		existing_fiscal_years = frappe.db.sql("""select name from `tabFiscal Year`

erpnext/accounts/doctype/gl_entry/gl_entry.py

@@ -18,22 +18,27 @@ class GLEntry(Document):
 	def validate(self):
 		self.flags.ignore_submit_comment = True
 		self.check_mandatory()
-		self.pl_must_have_cost_center()
-		self.check_pl_account()
-		self.validate_cost_center()
-		self.validate_party()
-		self.validate_currency()
 		self.validate_and_set_fiscal_year()
 		
-	def on_update_with_args(self, adv_adj, update_outstanding = 'Yes'):
+		if not self.flags.from_repost:
-		self.validate_account_details(adv_adj)
+			self.pl_must_have_cost_center()
+			self.check_pl_account()
+			self.validate_cost_center()
+			self.validate_party()
+			self.validate_currency()
+
+
+	def on_update_with_args(self, adv_adj, update_outstanding = 'Yes', from_repost=False):
+		if not from_repost:
+			self.validate_account_details(adv_adj)
+			check_freezing_date(self.posting_date, adv_adj)
+			
 		validate_frozen_account(self.account, adv_adj)
-		check_freezing_date(self.posting_date, adv_adj)
 		validate_balance_type(self.account, adv_adj)
 
 		# Update outstanding amt on against voucher
 		if self.against_voucher_type in ['Journal Entry', 'Sales Invoice', 'Purchase Invoice'] \
-			and self.against_voucher and update_outstanding == 'Yes':
+			and self.against_voucher and update_outstanding == 'Yes' and not from_repost:
 				update_outstanding_amt(self.account, self.party_type, self.party, self.against_voucher_type,
 					self.against_voucher)
 

erpnext/accounts/doctype/period_closing_voucher/period_closing_voucher.py

@@ -35,9 +35,9 @@ class PeriodClosingVoucher(AccountsController):
 	def validate_posting_date(self):
 		from erpnext.accounts.utils import get_fiscal_year, validate_fiscal_year
 
-		validate_fiscal_year(self.posting_date, self.fiscal_year, label=_("Posting Date"), doc=self)
+		validate_fiscal_year(self.posting_date, self.fiscal_year, self.company, label=_("Posting Date"), doc=self)
 
-		self.year_start_date = get_fiscal_year(self.posting_date, self.fiscal_year)[1]
+		self.year_start_date = get_fiscal_year(self.posting_date, self.fiscal_year, company=self.company)[1]
 
 		pce = frappe.db.sql("""select name from `tabPeriod Closing Voucher`
 			where posting_date > %s and fiscal_year = %s and docstatus = 1""",

erpnext/accounts/doctype/period_closing_voucher/test_period_closing_voucher.py

@@ -11,7 +11,7 @@ from erpnext.accounts.doctype.journal_entry.test_journal_entry import make_journ
 
 class TestPeriodClosingVoucher(unittest.TestCase):
 	def test_closing_entry(self):
-		year_start_date = get_fiscal_year(today())[1]
+		year_start_date = get_fiscal_year(today(), company="_Test Company")[1]
 
 		make_journal_entry("_Test Bank - _TC", "Sales - _TC", 400,
 			"_Test Cost Center - _TC", posting_date=now(), submit=True)
@@ -70,7 +70,7 @@ class TestPeriodClosingVoucher(unittest.TestCase):
 			"doctype": "Period Closing Voucher",
 			"closing_account_head": "_Test Account Reserves and Surplus - _TC",
 			"company": "_Test Company",
-			"fiscal_year": get_fiscal_year(today())[0],
+			"fiscal_year": get_fiscal_year(today(), company="_Test Company")[0],
 			"posting_date": today(),
 			"remarks": "test"
 		})

erpnext/accounts/doctype/purchase_invoice/purchase_invoice.py

@@ -302,11 +302,11 @@ class PurchaseInvoice(BuyingController):
 				asset.flags.ignore_validate_update_after_submit = True
 				asset.save()
 
-	def make_gl_entries(self, repost_future_gle=True):
+	def make_gl_entries(self, gl_entries=None, repost_future_gle=True, from_repost=False):
 		if not self.grand_total:
 			return
-		
+		if not gl_entries:
-		gl_entries = self.get_gl_entries()
+			gl_entries = self.get_gl_entries()
 		
 		if gl_entries:
 			update_outstanding = "No" if (cint(self.is_paid) or self.write_off_account) else "Yes"

erpnext/accounts/doctype/sales_invoice/sales_invoice.py

@@ -532,10 +532,12 @@ class SalesInvoice(SellingController):
 			if d.delivery_note and frappe.db.get_value("Delivery Note", d.delivery_note, "docstatus") != 1:
 				throw(_("Delivery Note {0} is not submitted").format(d.delivery_note))
 
-	def make_gl_entries(self, repost_future_gle=True):
+	def make_gl_entries(self, gl_entries=None, repost_future_gle=True, from_repost=False):
 		if not self.grand_total:
 			return
-		gl_entries = self.get_gl_entries()
+			
+		if not gl_entries:
+			gl_entries = self.get_gl_entries()
 
 		if gl_entries:
 			from erpnext.accounts.general_ledger import make_gl_entries

erpnext/accounts/general_ledger.py

@@ -11,12 +11,12 @@ from erpnext.accounts.doctype.budget.budget import validate_expense_against_budg
 
 class StockAccountInvalidTransaction(frappe.ValidationError): pass
 
-def make_gl_entries(gl_map, cancel=False, adv_adj=False, merge_entries=True, update_outstanding='Yes'):
+def make_gl_entries(gl_map, cancel=False, adv_adj=False, merge_entries=True, update_outstanding='Yes', from_repost=False):
 	if gl_map:
 		if not cancel:
 			gl_map = process_gl_map(gl_map, merge_entries)
 			if gl_map and len(gl_map) > 1:
-				save_entries(gl_map, adv_adj, update_outstanding)
+				save_entries(gl_map, adv_adj, update_outstanding, from_repost)
 			else:
 				frappe.throw(_("Incorrect number of General Ledger Entries found. You might have selected a wrong Account in the transaction."))
 		else:
@@ -78,21 +78,26 @@ def check_if_in_list(gle, gl_map):
 			and cstr(e.get('project')) == cstr(gle.get('project')):
 				return e
 
-def save_entries(gl_map, adv_adj, update_outstanding):
+def save_entries(gl_map, adv_adj, update_outstanding, from_repost=False):
-	validate_account_for_auto_accounting_for_stock(gl_map)
+	if not from_repost:
+		validate_account_for_auto_accounting_for_stock(gl_map)
+		
 	round_off_debit_credit(gl_map)
 
 	for entry in gl_map:
-		make_entry(entry, adv_adj, update_outstanding)
+		make_entry(entry, adv_adj, update_outstanding, from_repost)
-		# check against budget
-		validate_expense_against_budget(entry)
 		
-def make_entry(args, adv_adj, update_outstanding):
+		# check against budget
+		if not from_repost:
+			validate_expense_against_budget(entry)
+
+def make_entry(args, adv_adj, update_outstanding, from_repost=False):
 	args.update({"doctype": "GL Entry"})
 	gle = frappe.get_doc(args)
 	gle.flags.ignore_permissions = 1
+	gle.flags.from_repost = from_repost
 	gle.insert()
-	gle.run_method("on_update_with_args", adv_adj, update_outstanding)
+	gle.run_method("on_update_with_args", adv_adj, update_outstanding, from_repost)
 	gle.submit()
 
 def validate_account_for_auto_accounting_for_stock(gl_map):

erpnext/accounts/report/cash_flow/cash_flow.py

@@ -10,7 +10,8 @@ from erpnext.accounts.utils import get_fiscal_year
 
 
 def execute(filters=None):
-	period_list = get_period_list(filters.from_fiscal_year, filters.to_fiscal_year, filters.periodicity)
+	period_list = get_period_list(filters.from_fiscal_year, filters.to_fiscal_year, 
+		filters.periodicity)
 
 	operation_accounts = {
 		"section_name": "Operations",
@@ -103,7 +104,7 @@ def get_account_type_based_data(company, account_type, period_list, accumulated_
 	data = {}
 	total = 0
 	for period in period_list:
-		start_date = get_start_date(period, accumulated_values)
+		start_date = get_start_date(period, accumulated_values, company)
 		gl_sum = frappe.db.sql_list("""
 			select sum(credit) - sum(debit)
 			from `tabGL Entry`
@@ -126,10 +127,10 @@ def get_account_type_based_data(company, account_type, period_list, accumulated_
 	data["total"] = total
 	return data
 
-def get_start_date(period, accumulated_values):
+def get_start_date(period, accumulated_values, company):
 	start_date = period["year_start_date"]
 	if accumulated_values:
-		start_date = get_fiscal_year(period.to_date)[1]
+		start_date = get_fiscal_year(period.to_date, company=company)[1]
 
 	return start_date
 

erpnext/accounts/report/financial_statements.py

@@ -3,10 +3,8 @@
 
 from __future__ import unicode_literals
 import frappe
-import math
 from frappe import _
-from frappe.utils import (flt, getdate, get_first_day, get_last_day, date_diff,
+from frappe.utils import flt, getdate, get_first_day, add_months, add_days, formatdate
-	add_months, add_days, formatdate, cint)
 
 def get_period_list(from_fiscal_year, to_fiscal_year, periodicity):
 	"""Get a list of dict {"from_date": from_date, "to_date": to_date, "key": key, "label": label}
@@ -149,7 +147,6 @@ def calculate_values(accounts_by_name, gl_entries_by_account, period_list, accum
 				
 def get_date_fiscal_year(date):
 	from erpnext.accounts.utils import get_fiscal_year
-	
 	return get_fiscal_year(date)[0]
 
 def accumulate_values_into_parents(accounts, accounts_by_name, period_list, accumulated_values):

erpnext/accounts/utils.py

@@ -20,32 +20,63 @@ def get_fiscal_year(date=None, fiscal_year=None, label="Date", verbose=1, compan
 	return get_fiscal_years(date, fiscal_year, label, verbose, company, as_dict=as_dict)[0]
 
 def get_fiscal_years(transaction_date=None, fiscal_year=None, label="Date", verbose=1, company=None, as_dict=False):
-	# if year start date is 2012-04-01, year end date should be 2013-03-31 (hence subdate)
+	fiscal_years = frappe.cache().hget("fiscal_years", company) or []
-	cond = " disabled = 0"
-	if fiscal_year:
-		cond += " and fy.name = %(fiscal_year)s"
-	else:
-		cond += " and %(transaction_date)s >= fy.year_start_date and %(transaction_date)s <= fy.year_end_date"
 	
-	if company:
+	if not fiscal_years:		
-		cond += """ and (not exists(select name from `tabFiscal Year Company` fyc where fyc.parent = fy.name)
+		# if year start date is 2012-04-01, year end date should be 2013-03-31 (hence subdate)
-			or exists(select company from `tabFiscal Year Company` fyc where fyc.parent = fy.name and fyc.company=%(company)s ))"""
+		cond = ""
+		if fiscal_year:
+			cond += " and fy.name = {0}".format(frappe.db.escape(fiscal_year))
+		if company:
+			cond += """
+				and (not exists (select name 
+					from `tabFiscal Year Company` fyc 
+					where fyc.parent = fy.name) 
+				or exists(select company 
+					from `tabFiscal Year Company` fyc 
+					where fyc.parent = fy.name 
+					and fyc.company=%(company)s)
+				)
+			"""
 
-	fy = frappe.db.sql("""select fy.name, fy.year_start_date, fy.year_end_date from `tabFiscal Year` fy
+		fiscal_years = frappe.db.sql("""
-		where %s order by fy.year_start_date desc""" % cond, {
+			select 
-			"fiscal_year": fiscal_year,
+				fy.name, fy.year_start_date, fy.year_end_date 
-			"transaction_date": transaction_date,
+			from 
-			"company": company
+				`tabFiscal Year` fy
-		}, as_dict=as_dict)
+			where 
+				disabled = 0 {0}
+			order by 
+				fy.year_start_date desc""".format(cond), {
+				"company": company
+			}, as_dict=True)
 			
-	if not fy:
+		frappe.cache().hset("fiscal_years", company, fiscal_years)
-		error_msg = _("""{0} {1} not in any active Fiscal Year. For more details check {2}.""").format(label, formatdate(transaction_date), "https://frappe.github.io/erpnext/user/manual/en/accounts/articles/fiscal-year-error")
-		if verbose==1: frappe.msgprint(error_msg)
-		raise FiscalYearError, error_msg
-	return fy
 
-def validate_fiscal_year(date, fiscal_year, label=_("Date"), doc=None):
+	if transaction_date:
-	years = [f[0] for f in get_fiscal_years(date, label=label)]
+		transaction_date = getdate(transaction_date)
+
+	for fy in fiscal_years:
+		matched = False
+		if fiscal_year and fy.name == fiscal_year:
+			matched = True
+
+		if (transaction_date and getdate(fy.year_start_date) <= transaction_date 
+			and getdate(fy.year_end_date) >= transaction_date):
+			matched = True
+			
+		if matched:
+			if as_dict:
+				return (fy,)
+			else:
+				return ((fy.name, fy.year_start_date, fy.year_end_date),)
+
+	error_msg = _("""{0} {1} not in any active Fiscal Year.""").format(label, formatdate(transaction_date))
+	if verbose==1: frappe.msgprint(error_msg)
+	raise FiscalYearError, error_msg
+
+def validate_fiscal_year(date, fiscal_year, company, label=_("Date"), doc=None):
+	years = [f[0] for f in get_fiscal_years(date, label=label, company=company)]
 	if fiscal_year not in years:
 		if doc:
 			doc.fiscal_year = years[0]

erpnext/controllers/accounts_controller.py

@@ -113,7 +113,7 @@ class AccountsController(TransactionBase):
 				date_field = "transaction_date"
 
 			if date_field and self.get(date_field):
-				validate_fiscal_year(self.get(date_field), self.fiscal_year,
+				validate_fiscal_year(self.get(date_field), self.fiscal_year, self.company, 
 					self.meta.get_label(date_field), self)
 
 	def validate_due_date(self):

erpnext/controllers/stock_controller.py

@@ -3,7 +3,7 @@
 
 from __future__ import unicode_literals
 import frappe
-from frappe.utils import cint, flt, cstr
+from frappe.utils import cint, flt, cstr, now
 from frappe import msgprint, _
 import frappe.defaults
 from erpnext.accounts.utils import get_fiscal_year
@@ -15,7 +15,7 @@ class StockController(AccountsController):
 		super(StockController, self).validate()
 		self.validate_inspection()
 	
-	def make_gl_entries(self, repost_future_gle=True):
+	def make_gl_entries(self, gl_entries=None, repost_future_gle=True, from_repost=False):
 		if self.docstatus == 2:
 			delete_gl_entries(voucher_type=self.doctype, voucher_no=self.name)
 
@@ -23,8 +23,9 @@ class StockController(AccountsController):
 			warehouse_account = get_warehouse_account()
 
 			if self.docstatus==1:
-				gl_entries = self.get_gl_entries(warehouse_account)
+				if not gl_entries:
-				make_gl_entries(gl_entries)
+					gl_entries = self.get_gl_entries(warehouse_account)
+				make_gl_entries(gl_entries, from_repost=from_repost)
 
 			if repost_future_gle:
 				items, warehouses = self.get_items_and_warehouses()
@@ -224,7 +225,7 @@ class StockController(AccountsController):
 	def make_gl_entries_on_cancel(self, repost_future_gle=True):
 		if frappe.db.sql("""select name from `tabGL Entry` where voucher_type=%s
 			and voucher_no=%s""", (self.doctype, self.name)):
-				self.make_gl_entries(repost_future_gle)
+				self.make_gl_entries(repost_future_gle=repost_future_gle)
 
 	def get_serialized_items(self):
 		serialized_items = []
@@ -308,7 +309,7 @@ def update_gl_entries_after(posting_date, posting_time, for_warehouses=None, for
 		if expected_gle:
 			if not existing_gle or not compare_existing_and_expected_gle(existing_gle, expected_gle):
 				_delete_gl_entries(voucher_type, voucher_no)
-				voucher_obj.make_gl_entries(repost_future_gle=False)
+				voucher_obj.make_gl_entries(gl_entries=expected_gle, repost_future_gle=False, from_repost=True)
 		else:
 			_delete_gl_entries(voucher_type, voucher_no)
 
@@ -363,10 +364,14 @@ def get_voucherwise_gl_entries(future_stock_vouchers, posting_date):
 	return gl_entries
 
 def get_warehouse_account():
-	warehouse_account = frappe._dict()
+	if not frappe.flags.warehouse_account_map:
+		warehouse_account = frappe._dict()
 
-	for d in frappe.db.sql("""select warehouse, name, account_currency from tabAccount
+		for d in frappe.db.sql("""select warehouse, name, account_currency from tabAccount
-		where account_type = 'Stock' and (warehouse is not null and warehouse != ''
+			where account_type = 'Stock' and (warehouse is not null and warehouse != ''
-		and is_group != 1) and is_group=0 """, as_dict=1):
+			and is_group != 1) and is_group=0 """, as_dict=1):
-			warehouse_account.setdefault(d.warehouse, d)
+				warehouse_account.setdefault(d.warehouse, d)
-	return warehouse_account
+		
+		frappe.flags.warehouse_account_map = warehouse_account
+		
+	return frappe.flags.warehouse_account_map

erpnext/fleet_management/report/vehicle_expenses/vehicle_expenses.py

@@ -15,7 +15,7 @@ def execute(filters=None):
 	columns=get_columns()
 	data=get_log_data(filters)
 	chart=get_chart_data(data,period_list)
-	return columns,data,None,chart
+	return columns, data, None, chart
 	
 def get_columns():
 	columns = [_("License") + ":Link/Vehicle:100", _("Make") + ":data:50",

erpnext/hr/doctype/process_payroll/process_payroll.py

@@ -296,17 +296,18 @@ class ProcessPayroll(Document):
 			frappe.db.set_value("Salary Slip", ss_obj.name, "journal_entry", jv_name)
 
 	def set_start_end_dates(self):
-		self.update(get_start_end_dates(self.payroll_frequency, self.start_date or self.posting_date))
+		self.update(get_start_end_dates(self.payroll_frequency, 
+			self.start_date or self.posting_date, self.company))
 
 
 @frappe.whitelist()
-def get_start_end_dates(payroll_frequency, start_date=None):
+def get_start_end_dates(payroll_frequency, start_date=None, company=None):
 	'''Returns dict of start and end dates for given payroll frequency based on start_date'''
 	if not payroll_frequency:
 		frappe.throw(_("Please set Payroll Frequency first"))
 
 	if payroll_frequency == "Monthly" or payroll_frequency == "Bimonthly":
-		fiscal_year = get_fiscal_year(start_date)[0]
+		fiscal_year = get_fiscal_year(start_date, company=company)[0]
 		month = "%02d" % getdate(start_date).month
 		m = get_month_details(fiscal_year, month)
 		if payroll_frequency == "Bimonthly":

erpnext/hr/doctype/salary_slip/test_salary_slip.py

@@ -134,7 +134,7 @@ class TestSalarySlip(unittest.TestCase):
 		self.assertTrue(email_queue)
 
 	def test_payroll_frequency(self):
-		fiscal_year = get_fiscal_year(nowdate())[0]
+		fiscal_year = get_fiscal_year(nowdate(), company="_Test Company")[0]
 		month = "%02d" % getdate(nowdate()).month
 		m = get_month_details(fiscal_year, month)
 
@@ -185,7 +185,7 @@ class TestSalarySlip(unittest.TestCase):
 			}).insert()
 
 	def make_holiday_list(self):
-		fiscal_year = get_fiscal_year(nowdate())
+		fiscal_year = get_fiscal_year(nowdate(), company="_Test Company")
 		if not frappe.db.get_value("Holiday List", "Salary Slip Test Holiday List"):
 			holiday_list = frappe.get_doc({
 				"doctype": "Holiday List",

erpnext/stock/doctype/stock_ledger_entry/stock_ledger_entry.py

@@ -27,9 +27,6 @@ class StockLedgerEntry(Document):
 		self.validate_and_set_fiscal_year()
 		self.block_transactions_against_group_warehouse()
 
-		from erpnext.accounts.utils import validate_fiscal_year
-		validate_fiscal_year(self.posting_date, self.fiscal_year, self.meta.get_label("posting_date"), self)
-
 	def on_submit(self):
 		self.check_stock_frozen_date()
 		self.actual_amt_check()
@@ -117,6 +114,10 @@ class StockLedgerEntry(Document):
 	def validate_and_set_fiscal_year(self):
 		if not self.fiscal_year:
 			self.fiscal_year = get_fiscal_year(self.posting_date, company=self.company)[0]
+		else:
+			from erpnext.accounts.utils import validate_fiscal_year
+			validate_fiscal_year(self.posting_date, self.fiscal_year, self.company, 
+				self.meta.get_label("posting_date"), self)
 
 	def block_transactions_against_group_warehouse(self):
 		from erpnext.stock.utils import is_group_warehouse