Shiloh/brotherton-erpnext
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #3470 from nabinhait/develop

Browse Source 
Performance upgrade in reports and indexing
This commit is contained in:
Nabin Hait 2015-06-14 22:01:30 +05:30
commit dc633fe360
10 changed files with 125 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
erpnext/accounts/doctype/account/account.json
View File

@ -35,7 +35,7 @@
"permlevel": 0,
"read_only": 1,
"reqd": 1,
"search_index": 1
"search_index": 0
},
{
"default": "0",
@ -44,7 +44,7 @@
"label": "Is Group",
"permlevel": 0,
"precision": "",
"search_index": 1
"search_index": 0
},
{
"fieldname": "company",
@ -57,7 +57,7 @@
"permlevel": 0,
"read_only": 1,
"reqd": 1,
"search_index": 1
"search_index": 0
},
{
"fieldname": "root_type",
@ -147,7 +147,8 @@
"label": "Lft",
"permlevel": 0,
"print_hide": 1,
"read_only": 1
"read_only": 1,
"search_index": 1
},
{
"fieldname": "rgt",
@ -156,7 +157,8 @@
"label": "Rgt",
"permlevel": 0,
"print_hide": 1,
"read_only": 1
"read_only": 1,
"search_index": 1
},
{
"fieldname": "old_parent",
@ -171,7 +173,7 @@
"icon": "icon-money",
"idx": 1,
"in_create": 0,
"modified": "2015-05-28 14:10:40.606010",
"modified": "2015-06-14 20:57:55.471334",
"modified_by": "Administrator",
"module": "Accounts",
"name": "Account",

5
erpnext/accounts/doctype/gl_entry/gl_entry.json
View File

@ -48,7 +48,8 @@
"fieldtype": "Dynamic Link",
"label": "Party",
"options": "party_type",
"permlevel": 0
"permlevel": 0,
"search_index": 1
},
{
"fieldname": "cost_center",
@ -192,7 +193,7 @@
"icon": "icon-list",
"idx": 1,
"in_create": 1,
"modified": "2015-04-27 20:32:48.246818",
"modified": "2015-06-14 20:57:19.800276",
"modified_by": "Administrator",
"module": "Accounts",
"name": "GL Entry",

2
erpnext/accounts/doctype/journal_entry/journal_entry.py
View File

@ -544,7 +544,7 @@ def get_against_jv(doctype, txt, searchfield, start, page_len, filters):
and (ifnull(jv_detail.against_invoice, '') = '' and ifnull(jv_detail.against_voucher, '') = ''
and ifnull(jv_detail.against_jv, '') = '' )
and jv.docstatus = 1 and jv.{0} like %s order by jv.name desc limit %s, %s""".format(searchfield),
(filters["account"], cstr(filters["party"]), "%{0}%".format(txt), start, page_len))
(filters.get("account"), cstr(filters.get("party")), "%{0}%".format(txt), start, page_len))
@frappe.whitelist()
def get_outstanding(args):

25
erpnext/accounts/report/accounts_receivable/accounts_receivable.py
View File

@ -51,7 +51,7 @@ class ReceivablePayableReport(object):
currency_precision = get_currency_precision() or 2
dr_or_cr = "debit" if args.get("party_type") == "Customer" else "credit"
voucher_details = self.get_voucher_details()
voucher_details = self.get_voucher_details(args.get("party_type"))
future_vouchers = self.get_entries_after(self.filters.report_date, args.get("party_type"))
@ -153,23 +153,26 @@ class ReceivablePayableReport(object):
return self.party_map
def get_voucher_details(self):
def get_voucher_details(self, party_type):
voucher_details = frappe._dict()
if party_type == "Customer":
for si in frappe.db.sql("""select name, due_date
from `tabSales Invoice` where docstatus=1""", as_dict=1):
voucher_details.setdefault(si.name, si)
for si in frappe.db.sql("""select name, due_date
from `tabSales Invoice` where docstatus=1""", as_dict=1):
voucher_details.setdefault(si.name, si)
for pi in frappe.db.sql("""select name, due_date, bill_no, bill_date
from `tabPurchase Invoice` where docstatus=1""", as_dict=1):
voucher_details.setdefault(pi.name, pi)
if party_type == "Supplier":
for pi in frappe.db.sql("""select name, due_date, bill_no, bill_date
from `tabPurchase Invoice` where docstatus=1""", as_dict=1):
voucher_details.setdefault(pi.name, pi)
return voucher_details
def get_gl_entries(self, party_type):
if not hasattr(self, "gl_entries"):
conditions, values = self.prepare_conditions(party_type)
self.gl_entries = frappe.db.sql("""select * from `tabGL Entry`
self.gl_entries = frappe.db.sql("""select posting_date, account, party_type, party, debit, credit,
voucher_type, voucher_no, against_voucher_type, against_voucher from `tabGL Entry`
where docstatus < 2 and party_type=%s {0} order by posting_date, party"""
.format(conditions), values, as_dict=True)
@ -187,7 +190,7 @@ class ReceivablePayableReport(object):
if self.filters.get(party_type_field):
conditions.append("party=%s")
values.append(self.filters.get(party_type_field))
values.append(self.filters.get(party_type_field))
return " and ".join(conditions), values

48
erpnext/accounts/report/financial_statements.py
View File

@ -83,7 +83,7 @@ def get_data(company, root_type, balance_must_be, period_list, ignore_closing_en
gl_entries_by_account = get_gl_entries(company, period_list[0]["from_date"], period_list[-1]["to_date"],
accounts[0].lft, accounts[0].rgt, ignore_closing_entries=ignore_closing_entries)
calculate_values(accounts, gl_entries_by_account, period_list)
calculate_values(accounts_by_name, gl_entries_by_account, period_list)
accumulate_values_into_parents(accounts, accounts_by_name, period_list)
out = prepare_data(accounts, balance_must_be, period_list)
@ -92,16 +92,14 @@ def get_data(company, root_type, balance_must_be, period_list, ignore_closing_en
return out
def calculate_values(accounts, gl_entries_by_account, period_list):
for d in accounts:
for name in ([d.name] + (d.collapsed_children or [])):
for entry in gl_entries_by_account.get(name, []):
for period in period_list:
entry.posting_date = getdate(entry.posting_date)
# check if posting date is within the period
if entry.posting_date <= period.to_date:
d[period.key] = d.get(period.key, 0.0) + flt(entry.debit) - flt(entry.credit)
def calculate_values(accounts_by_name, gl_entries_by_account, period_list):
for entries in gl_entries_by_account.values():
for entry in entries:
d = accounts_by_name.get(entry.account)
for period in period_list:
# check if posting date is within the period
if entry.posting_date <= period.to_date:
d[period.key] = d.get(period.key, 0.0) + flt(entry.debit) - flt(entry.credit)
def accumulate_values_into_parents(accounts, accounts_by_name, period_list):
@ -159,22 +157,8 @@ def add_total_row(out, balance_must_be, period_list):
out.append({})
def get_accounts(company, root_type):
# root lft, rgt
root_account = frappe.db.sql("""select lft, rgt from `tabAccount`
where company=%s and root_type=%s and ifnull(parent_account, '') = ''
order by lft limit 1""",
(company, root_type), as_dict=True)
if not root_account:
return None
lft, rgt = root_account[0].lft, root_account[0].rgt
accounts = frappe.db.sql("""select * from `tabAccount`
where company=%(company)s and lft >= %(lft)s and rgt <= %(rgt)s order by lft""",
{ "company": company, "lft": lft, "rgt": rgt }, as_dict=True)
return accounts
return frappe.db.sql("""select name, parent_account, lft, rgt, root_type, report_type, account_name from `tabAccount`
where company=%s and root_type=%s order by lft""", (company, root_type), as_dict=True)
def filter_accounts(accounts, depth=10):
parent_children_map = {}
@ -196,14 +180,6 @@ def filter_accounts(accounts, depth=10):
filtered_accounts.append(child)
add_to_list(child.name, level + 1)
else:
# include all children at level lower than the depth
parent_account = accounts_by_name[parent]
parent_account["collapsed_children"] = []
for d in accounts:
if d.lft > parent_account.lft and d.rgt < parent_account.rgt:
parent_account["collapsed_children"].append(d.name)
add_to_list(None, 0)
return filtered_accounts, accounts_by_name
@ -234,7 +210,7 @@ def get_gl_entries(company, from_date, to_date, root_lft, root_rgt, ignore_closi
if from_date:
additional_conditions.append("and posting_date >= %(from_date)s")
gl_entries = frappe.db.sql("""select * from `tabGL Entry`
gl_entries = frappe.db.sql("""select posting_date, account, debit, credit from `tabGL Entry`
where company=%(company)s
{additional_conditions}
and posting_date <= %(to_date)s

12
erpnext/accounts/report/general_ledger/general_ledger.py
View File

@ -66,7 +66,7 @@ def get_gl_entries(filters):
gl_entries = frappe.db.sql("""select posting_date, account, party_type, party,
sum(ifnull(debit, 0)) as debit, sum(ifnull(credit, 0)) as credit,
voucher_type, voucher_no, cost_center, remarks, is_opening, against
voucher_type, voucher_no, cost_center, remarks, against
from `tabGL Entry`
where company=%(company)s {conditions}
{group_by_condition}
@ -91,6 +91,9 @@ def get_conditions(filters):
if filters.get("party"):
conditions.append("party=%(party)s")
if not (filters.get("account") or filters.get("party") or filters.get("group_by_account")):
conditions.append("posting_date >=%(from_date)s")
from frappe.desk.reportview import build_match_conditions
match_conditions = build_match_conditions("GL Entry")
@ -148,14 +151,15 @@ def initialize_gle_map(gl_entries):
def get_accountwise_gle(filters, gl_entries, gle_map):
opening, total_debit, total_credit = 0, 0, 0
from_date, to_date = getdate(filters.from_date), getdate(filters.to_date)
for gle in gl_entries:
amount = flt(gle.debit, 3) - flt(gle.credit, 3)
if gle.posting_date < getdate(filters.from_date):
if (filters.get("account") or filters.get("party") or filters.get("group_by_account")) \
and gle.posting_date < from_date:
gle_map[gle.account].opening += amount
if filters.get("account") or filters.get("party"):
opening += amount
elif gle.posting_date <= getdate(filters.to_date):
elif gle.posting_date <= to_date:
gle_map[gle.account].entries.append(gle)
gle_map[gle.account].total_debit += flt(gle.debit, 3)
gle_map[gle.account].total_credit += flt(gle.credit, 3)

79
erpnext/accounts/report/trial_balance/trial_balance.py
View File

@ -9,7 +9,7 @@ from erpnext.accounts.report.financial_statements import filter_accounts, get_gl
value_fields = ("opening_debit", "opening_credit", "debit", "credit", "closing_debit", "closing_credit")
def execute(filters):
def execute(filters=None):
validate_filters(filters)
data = get_data(filters)
columns = get_columns()
@ -45,8 +45,8 @@ def validate_filters(filters):
filters.to_date = filters.year_end_date
def get_data(filters):
accounts = frappe.db.sql("""select * from `tabAccount` where company=%s order by lft""",
filters.company, as_dict=True)
accounts = frappe.db.sql("""select name, parent_account, account_name, root_type, report_type, lft, rgt
from `tabAccount` where company=%s order by lft""", filters.company, as_dict=True)
if not accounts:
return None
@ -56,17 +56,58 @@ def get_data(filters):
min_lft, max_rgt = frappe.db.sql("""select min(lft), max(rgt) from `tabAccount`
where company=%s""", (filters.company,))[0]
gl_entries_by_account = get_gl_entries(filters.company, None, filters.to_date, min_lft, max_rgt,
gl_entries_by_account = get_gl_entries(filters.company, filters.from_date, filters.to_date, min_lft, max_rgt,
ignore_closing_entries=not flt(filters.with_period_closing_entry))
total_row = calculate_values(accounts, gl_entries_by_account, filters)
opening_balances = get_opening_balances(filters)
total_row = calculate_values(accounts, gl_entries_by_account, opening_balances, filters)
accumulate_values_into_parents(accounts, accounts_by_name)
data = prepare_data(accounts, filters, total_row)
return data
def get_opening_balances(filters):
balance_sheet_opening = get_rootwise_opening_balances(filters, "Balance Sheet")
pl_opening = get_rootwise_opening_balances(filters, "Profit and Loss")
balance_sheet_opening.update(pl_opening)
return balance_sheet_opening
def get_rootwise_opening_balances(filters, report_type):
additional_conditions = " and posting_date >= %(year_start_date)s" \
if report_type == "Profit and Loss" else ""
if not flt(filters.with_period_closing_entry):
additional_conditions += " and ifnull(voucher_type, '')!='Period Closing Voucher'"
gle = frappe.db.sql("""
select
account, sum(ifnull(debit, 0)) as opening_debit, sum(ifnull(credit, 0)) as opening_credit
from `tabGL Entry`
where
company=%(company)s
{additional_conditions}
and posting_date < %(from_date)s
and account in (select name from `tabAccount` where report_type=%(report_type)s)
group by account""".format(additional_conditions=additional_conditions),
{
"company": filters.company,
"from_date": filters.from_date,
"report_type": report_type,
"year_start_date": filters.year_start_date
},
as_dict=True)
opening = frappe._dict()
for d in gle:
opening.setdefault(d.account, d)
return opening
def calculate_values(accounts, gl_entries_by_account, filters):
def calculate_values(accounts, gl_entries_by_account, opening_balances, filters):
init = {
"opening_debit": 0.0,
"opening_credit": 0.0,
@ -88,29 +129,15 @@ def calculate_values(accounts, gl_entries_by_account, filters):
d.update(init.copy())
for entry in gl_entries_by_account.get(d.name, []):
posting_date = getdate(entry.posting_date)
# opening
if posting_date < filters.from_date:
is_valid_opening = (d.root_type in ("Asset", "Liability", "Equity") or
(filters.year_start_date <= posting_date < filters.from_date))
if is_valid_opening:
d["opening_debit"] += flt(entry.debit)
d["opening_credit"] += flt(entry.credit)
elif posting_date <= filters.to_date:
if entry.is_opening == "Yes" and d.root_type in ("Asset", "Liability", "Equity"):
d["opening_debit"] += flt(entry.debit)
d["opening_credit"] += flt(entry.credit)
else:
d["debit"] += flt(entry.debit)
d["credit"] += flt(entry.credit)
d["debit"] += flt(entry.debit)
d["credit"] += flt(entry.credit)
total_row["debit"] += d["debit"]
total_row["credit"] += d["credit"]
# add opening
d["opening_debit"] = opening_balances.get(d.name, {}).get("opening_debit", 0)
d["opening_credit"] = opening_balances.get(d.name, {}).get("opening_credit", 0)
return total_row

4
erpnext/accounts/utils.py
View File

@ -91,7 +91,7 @@ def get_balance_on(account=None, date=None, party_type=None, party=None):
# different filter for group and ledger - improved performance
if acc.is_group:
cond.append("""exists (
select * from `tabAccount` ac where ac.name = gle.account
select name from `tabAccount` ac where ac.name = gle.account
and ac.lft >= %s and ac.rgt <= %s
)""" % (acc.lft, acc.rgt))
else:
@ -397,7 +397,7 @@ def get_outstanding_invoices(amount_query, account, party_type, party):
for d in outstanding_voucher_list:
payment_amount = frappe.db.sql("""
select ifnull(sum(ifnull({amount_query}, 0)), 0)
select ifnull(sum({amount_query}), 0)
from
`tabGL Entry`
where

3
erpnext/patches.txt
View File

@ -165,4 +165,5 @@ erpnext.patches.v5_0.repost_gle_for_jv_with_multiple_party
erpnext.patches.v5_0.portal_fixes
erpnext.patches.v5_0.reset_values_in_tools
execute:frappe.delete_doc("Page", "users")
erpnext.patches.v5_0.update_material_transferred_for_manufacturing_again
erpnext.patches.v5_0.update_material_transferred_for_manufacturing_again
erpnext.patches.v5_0.index_on_account_and_gl_entry

22
erpnext/patches/v5_0/index_on_account_and_gl_entry.py Normal file
View File

@ -0,0 +1,22 @@
from __future__ import unicode_literals
import frappe
def execute():
index_map = {
"Account": ["parent_account", "lft", "rgt"],
"GL Entry": ["posting_date", "account", 'party', "voucher_no"]
}
for dt, indexes in index_map.items():
existing_indexes = [d.Key_name for d in frappe.db.sql("""show index from `tab{0}`
where Column_name != 'name'""".format(dt), as_dict=1)]
for old in existing_indexes:
if old in ("parent", "group_or_ledger", "is_pl_account", "debit_or_credit", "account_name", "company"):
frappe.db.sql("alter table `tab{0}` drop index {1}".format(dt, old))
existing_indexes.remove(old)
for new in indexes:
if new not in existing_indexes:
frappe.db.sql("alter table `tab{0}` add index ({1})".format(dt, new))