feat: Chart of Accounts Importer (#16623)

* bare doctype created for COA utility

* improvise doctype's design

* validation to check no transaction exist for the company

* download file and import coa - client side logic added

* download csv template to create custom chart

* read the custom csv uploaded and parse it to appropriate format

* convert list of list to nested tree structure

* preview the uploaded chart before actual import

* toggle field based on necessity

* tweak create_charts and build_json logic to incorporate COA Import utility

* code cleanify and validation call added

* code enhancement for flexibility and validation added

* show import button only if data is validated

* unset existing data and load new accounts

* disable coa fields if parent_company set, minor improv

* file api fix

* added progress bar

* codacy fixes

* fix: Add account number in template

* fix: TDS account exception handling fix
This commit is contained in:
Zarrar 2019-03-23 10:30:12 +05:30 committed by Deepesh Garg
parent 81f4d1c745
commit 963d62b701
10 changed files with 611 additions and 9 deletions

View File

@ -9,8 +9,8 @@ from unidecode import unidecode
from six import iteritems from six import iteritems
from frappe.utils.nestedset import rebuild_tree from frappe.utils.nestedset import rebuild_tree
def create_charts(company, chart_template=None, existing_company=None): def create_charts(company, chart_template=None, existing_company=None, custom_chart=None):
chart = get_chart(chart_template, existing_company) chart = custom_chart or get_chart(chart_template, existing_company)
if chart: if chart:
accounts = [] accounts = []
@ -40,7 +40,7 @@ def create_charts(company, chart_template=None, existing_company=None):
"report_type": report_type, "report_type": report_type,
"account_number": account_number, "account_number": account_number,
"account_type": child.get("account_type"), "account_type": child.get("account_type"),
"account_currency": frappe.db.get_value('Company', company, "default_currency"), "account_currency": child.get('account_currency') or frappe.db.get_value('Company', company, "default_currency"),
"tax_rate": child.get("tax_rate") "tax_rate": child.get("tax_rate")
}) })
@ -207,9 +207,9 @@ def validate_bank_account(coa, bank_account):
return (bank_account in accounts) return (bank_account in accounts)
@frappe.whitelist() @frappe.whitelist()
def build_tree_from_json(chart_template): def build_tree_from_json(chart_template, chart_data=None):
''' get chart template from its folder and parse the json to be rendered as tree ''' ''' get chart template from its folder and parse the json to be rendered as tree '''
chart = get_chart(chart_template) chart = chart_data or get_chart(chart_template)
# if no template selected, return as it is # if no template selected, return as it is
if not chart: if not chart:

View File

@ -0,0 +1,138 @@
frappe.ui.form.on('Chart of Accounts Importer', {
onload: function (frm) {
frm.set_value("company", "");
frm.set_value("import_file", "");
},
refresh: function (frm) {
// disable default save
frm.disable_save();
// make company mandatory
frm.set_df_property('company', 'reqd', frm.doc.company ? 0 : 1);
frm.set_df_property('import_file_section', 'hidden', frm.doc.company ? 0 : 1);
frm.set_df_property('chart_preview', 'hidden',
$(frm.fields_dict['chart_tree'].wrapper).html()!="" ? 0 : 1);
// Show import button when file is successfully attached
if (frm.page && frm.page.show_import_button) {
create_import_button(frm);
}
// show download template button when company is properly selected
if(frm.doc.company) {
// download the csv template file
frm.add_custom_button(__("Download template"), function () {
let get_template_url = 'erpnext.accounts.doctype.chart_of_accounts_importer.chart_of_accounts_importer.download_template';
open_url_post(frappe.request.url, { cmd: get_template_url, doctype: frm.doc.doctype });
});
} else {
frm.set_value("import_file", "");
}
},
import_file: function (frm) {
if (!frm.doc.import_file) {
frm.page.set_indicator("");
$(frm.fields_dict['chart_tree'].wrapper).empty(); // empty wrapper on removing file
} else {
generate_tree_preview(frm);
validate_csv_data(frm);
}
},
company: function (frm) {
// validate that no Gl Entry record for the company exists.
frappe.call({
method: "erpnext.accounts.doctype.chart_of_accounts_importer.chart_of_accounts_importer.validate_company",
args: {
company: frm.doc.company
},
callback: function(r) {
if(r.message===false) {
frm.set_value("company", "");
frappe.throw(__("Transactions against the company already exist! "));
} else {
frm.trigger("refresh");
}
}
});
}
});
var validate_csv_data = function(frm) {
frappe.call({
method: "erpnext.accounts.doctype.chart_of_accounts_importer.chart_of_accounts_importer.validate_accounts",
args: {file_name: frm.doc.import_file},
callback: function(r) {
if(r.message && r.message[0]===true) {
frm.page["show_import_button"] = true;
frm.page["total_accounts"] = r.message[1];
frm.trigger("refresh");
} else {
frm.page.set_indicator(__('Resolve error and upload again.'), 'orange');
frappe.throw(__(r.message));
}
}
});
};
var create_import_button = function(frm) {
frm.page.set_primary_action(__("Start Import"), function () {
setup_progress_bar(frm);
frappe.call({
method: "erpnext.accounts.doctype.chart_of_accounts_importer.chart_of_accounts_importer.import_coa",
args: {
file_name: frm.doc.import_file,
company: frm.doc.company
},
freeze: true,
callback: function(r) {
if(!r.exc) {
clearInterval(frm.page["interval"]);
frm.page.set_indicator(__('Import Successfull'), 'blue');
frappe.hide_progress();
create_reset_button(frm);
}
}
});
}).addClass('btn btn-primary');
};
var create_reset_button = function(frm) {
frm.page.set_primary_action(__("Reset"), function () {
frm.page.clear_primary_action();
delete frm.page["show_import_button"];
frm.reload_doc();
}).addClass('btn btn-primary');
};
var generate_tree_preview = function(frm) {
let parent = __('All Accounts');
$(frm.fields_dict['chart_tree'].wrapper).empty(); // empty wrapper to load new data
// generate tree structure based on the csv data
new frappe.ui.Tree({
parent: $(frm.fields_dict['chart_tree'].wrapper),
label: parent,
expandable: true,
method: 'erpnext.accounts.doctype.chart_of_accounts_importer.chart_of_accounts_importer.get_coa',
args: {
file_name: frm.doc.import_file,
parent: parent,
doctype: 'Chart of Accounts Importer'
},
onclick: function(node) {
parent = node.value;
}
});
};
var setup_progress_bar = function(frm) {
frm.page["seconds_elapsed"] = 0;
frm.page["execution_time"] = (frm.page["total_accounts"] > 100) ? 100 : frm.page["total_accounts"];
frm.page["interval"] = setInterval(function() {
frm.page["seconds_elapsed"] += 1;
frappe.show_progress(__('Creating Accounts'), frm.page["seconds_elapsed"], frm.page["execution_time"]);
}, 250);
};

View File

@ -0,0 +1,226 @@
{
"allow_copy": 1,
"allow_events_in_timeline": 0,
"allow_guest_to_view": 0,
"allow_import": 0,
"allow_rename": 0,
"beta": 0,
"creation": "2019-02-01 12:24:34.761380",
"custom": 0,
"description": "Import Chart of Accounts from a csv file",
"docstatus": 0,
"doctype": "DocType",
"document_type": "Other",
"editable_grid": 1,
"engine": "InnoDB",
"fields": [
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "company",
"fieldtype": "Link",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 1,
"in_standard_filter": 0,
"label": "Company",
"length": 0,
"no_copy": 0,
"options": "Company",
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "import_file_section",
"fieldtype": "Section Break",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"depends_on": "",
"fieldname": "import_file",
"fieldtype": "Attach",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Attach custom Chart of Accounts file",
"length": 0,
"no_copy": 0,
"options": "",
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 1,
"columns": 0,
"fieldname": "chart_preview",
"fieldtype": "Section Break",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Chart Preview",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "chart_tree",
"fieldtype": "HTML",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Chart Tree",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
}
],
"has_web_view": 0,
"hide_heading": 1,
"hide_toolbar": 1,
"idx": 0,
"image_view": 0,
"in_create": 1,
"is_submittable": 0,
"issingle": 1,
"istable": 0,
"max_attachments": 0,
"modified": "2019-02-04 23:10:30.136807",
"modified_by": "Administrator",
"module": "Accounts",
"name": "Chart of Accounts Importer",
"name_case": "",
"owner": "Administrator",
"permissions": [
{
"amend": 0,
"cancel": 0,
"create": 1,
"delete": 0,
"email": 0,
"export": 0,
"if_owner": 0,
"import": 0,
"permlevel": 0,
"print": 0,
"read": 1,
"report": 0,
"role": "Accounts Manager",
"set_user_permissions": 0,
"share": 1,
"submit": 0,
"write": 1
}
],
"quick_entry": 1,
"read_only": 1,
"read_only_onload": 0,
"show_name_in_global_search": 0,
"sort_field": "",
"sort_order": "DESC",
"track_changes": 0,
"track_seen": 0,
"track_views": 0
}

View File

@ -0,0 +1,199 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and contributors
# For license information, please see license.txt
from __future__ import unicode_literals
import frappe, csv
from frappe import _
from frappe.utils import cstr
from frappe.model.document import Document
from frappe.utils.csvutils import UnicodeWriter
from erpnext.accounts.doctype.account.chart_of_accounts.chart_of_accounts import create_charts, build_tree_from_json
class ChartofAccountsImporter(Document):
pass
@frappe.whitelist()
def validate_company(company):
if frappe.db.get_all('GL Entry', {"company": company}, "name", limit=1):
return False
@frappe.whitelist()
def import_coa(file_name, company):
# delete existing data for accounts
unset_existing_data(company)
# create accounts
forest = build_forest(generate_data_from_csv(file_name))
create_charts(company, custom_chart=forest)
# trigger on_update for company to reset default accounts
set_default_accounts(company)
def generate_data_from_csv(file_name, as_dict=False):
''' read csv file and return the generated nested tree '''
file_doc = frappe.get_doc('File', {"file_url": file_name})
file_path = file_doc.get_full_path()
data = []
with open(file_path, 'r') as in_file:
csv_reader = list(csv.reader(in_file))
headers = csv_reader[1][1:]
del csv_reader[0:2] # delete top row and headers row
for row in csv_reader:
if as_dict:
data.append({frappe.scrub(header): row[index+1] for index, header in enumerate(headers)})
else:
if not row[2]: row[2] = row[1]
data.append(row[1:])
# convert csv data
return data
@frappe.whitelist()
def get_coa(doctype, parent, is_root=False, file_name=None):
''' called by tree view (to fetch node's children) '''
parent = None if parent==_('All Accounts') else parent
forest = build_forest(generate_data_from_csv(file_name))
accounts = build_tree_from_json("", chart_data=forest) # returns alist of dict in a tree render-able form
# filter out to show data for the selected node only
accounts = [d for d in accounts if d['parent_account']==parent]
return accounts
def build_forest(data):
'''
converts list of list into a nested tree
if a = [[1,1], [1,2], [3,2], [4,4], [5,4]]
tree = {
1: {
2: {
3: {}
}
},
4: {
5: {}
}
}
'''
# set the value of nested dictionary
def set_nested(d, path, value):
reduce(lambda d, k: d.setdefault(k, {}), path[:-1], d)[path[-1]] = value
return d
# returns the path of any node in list format
def return_parent(data, child):
for row in data:
account_name, parent_account = row[0:2]
if parent_account == account_name == child:
return [parent_account]
elif account_name == child:
return [child] + return_parent(data, parent_account)
charts_map, paths = {}, []
for i in data:
account_name, _, account_number, is_group, account_type, root_type = i
charts_map[account_name] = {}
if is_group: charts_map[account_name]["is_group"] = is_group
if account_type: charts_map[account_name]["account_type"] = account_type
if root_type: charts_map[account_name]["root_type"] = root_type
if account_number: charts_map[account_name]["account_number"] = account_number
path = return_parent(data, account_name)[::-1]
paths.append(path) # List of path is created
out = {}
for path in paths:
for n, account_name in enumerate(path):
set_nested(out, path[:n+1], charts_map[account_name]) # setting the value of nested dictionary.
return out
@frappe.whitelist()
def download_template():
data = frappe._dict(frappe.local.form_dict)
fields = ["Account Name", "Parent Account", "Account Number", "Is Group", "Account Type", "Root Type"]
writer = UnicodeWriter()
writer.writerow([_('Chart of Accounts Template')])
writer.writerow([_("Column Labels : ")] + fields)
writer.writerow([_("Start entering data from here : ")])
# download csv file
frappe.response['result'] = cstr(writer.getvalue())
frappe.response['type'] = 'csv'
frappe.response['doctype'] = data.get('doctype')
@frappe.whitelist()
def validate_accounts(file_name):
accounts = generate_data_from_csv(file_name, as_dict=True)
accounts_dict = {}
for account in accounts:
accounts_dict.setdefault(account["account_name"], account)
if account["parent_account"] and accounts_dict[account["parent_account"]]:
accounts_dict[account["parent_account"]]["is_group"] = 1
message = validate_root(accounts_dict)
if message: return message
message = validate_account_types(accounts_dict)
if message: return message
return [True, len(accounts)]
def validate_root(accounts):
roots = [accounts[d] for d in accounts if not accounts[d].get('parent_account')]
if len(roots) < 4:
return _("Number of root accounts cannot be less than 4")
for account in roots:
if not account.get("root_type"):
return _("Please enter Root Type for - {0}").format(account.get("account_name"))
elif account.get("root_type") not in ("Asset", "Liability", "Expense", "Income", "Equity"):
return _('Root Type for "{0}" must be one of the Asset, Liability, Income, Expense and Equity').format(account.get("account_name"))
def validate_account_types(accounts):
account_types_for_ledger = ["Cost of Goods Sold", "Depreciation", "Fixed Asset", "Payable", "Receivable", "Stock Adjustment"]
account_types = [accounts[d]["account_type"] for d in accounts if not accounts[d]['is_group']]
missing = list(set(account_types_for_ledger) - set(account_types))
if missing:
return _("Please identify/create Account (Ledger) for type - {0}").format(' , '.join(missing))
account_types_for_group = ["Bank", "Cash", "Stock"]
account_groups = [accounts[d]["account_type"] for d in accounts if accounts[d]['is_group']]
missing = list(set(account_types_for_group) - set(account_groups))
if missing:
return _("Please identify/create Account (Group) for type - {0}").format(' , '.join(missing))
def unset_existing_data(company):
linked = frappe.db.sql('''select fieldname from tabDocField
where fieldtype="Link" and options="Account" and parent="Company"''', as_dict=True)
# remove accounts data from company
update_values = {d.fieldname: '' for d in linked}
frappe.db.set_value('Company', company, update_values, update_values)
# remove accounts data from various doctypes
for doctype in ["Account", "Party Account", "Mode of Payment Account", "Tax Withholding Account",
"Sales Taxes and Charges Template", "Purchase Taxes and Charges Template"]:
frappe.db.sql('''delete from `tab{0}` where `company`="%s"''' # nosec
.format(doctype) % (company))
def set_default_accounts(company):
from erpnext.setup.doctype.company.company import install_country_fixtures
company = frappe.get_doc('Company', company)
company.update({
"default_receivable_account": frappe.db.get_value("Account",
{"company": company.name, "account_type": "Receivable", "is_group": 0}),
"default_payable_account": frappe.db.get_value("Account",
{"company": company.name, "account_type": "Payable", "is_group": 0})
})
company.save()
install_country_fixtures(company.name)
company.create_default_tax_template()

View File

@ -0,0 +1,23 @@
/* eslint-disable */
// rename this file from _test_[name] to test_[name] to activate
// and remove above this line
QUnit.test("test: Chart of Accounts Importer", function (assert) {
let done = assert.async();
// number of asserts
assert.expect(1);
frappe.run_serially([
// insert a new Chart of Accounts Importer
() => frappe.tests.make('Chart of Accounts Importer', [
// values to be set
{key: 'value'}
]),
() => {
assert.equal(cur_frm.doc.key, 'value');
},
() => done()
]);
});

View File

@ -0,0 +1,10 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and Contributors
# See license.txt
from __future__ import unicode_literals
import frappe
import unittest
class TestChartofAccountsImporter(unittest.TestCase):
pass

View File

@ -395,7 +395,9 @@ def set_tax_withholding_category(company):
doc.insert() doc.insert()
except frappe.DuplicateEntryError: except frappe.DuplicateEntryError:
doc = frappe.get_doc("Tax Withholding Category", d.get("name")) doc = frappe.get_doc("Tax Withholding Category", d.get("name"))
doc.append("accounts", accounts[0])
if accounts:
doc.append("accounts", accounts[0])
# if fiscal year don't match with any of the already entered data, append rate row # if fiscal year don't match with any of the already entered data, append rate row
fy_exist = [k for k in doc.get('rates') if k.get('fiscal_year')==fiscal_year] fy_exist = [k for k in doc.get('rates') if k.get('fiscal_year')==fiscal_year]

View File

@ -276,4 +276,5 @@ var disbale_coa_fields = function(frm, bool=true) {
frm.set_df_property("create_chart_of_accounts_based_on", "read_only", bool); frm.set_df_property("create_chart_of_accounts_based_on", "read_only", bool);
frm.set_df_property("chart_of_accounts", "read_only", bool); frm.set_df_property("chart_of_accounts", "read_only", bool);
frm.set_df_property("existing_company", "read_only", bool); frm.set_df_property("existing_company", "read_only", bool);
}; }

View File

@ -97,8 +97,6 @@ class Company(NestedSet):
install_country_fixtures(self.name) install_country_fixtures(self.name)
self.create_default_tax_template() self.create_default_tax_template()
if not frappe.db.get_value("Department", {"company": self.name}): if not frappe.db.get_value("Department", {"company": self.name}):
from erpnext.setup.setup_wizard.operations.install_fixtures import install_post_company_fixtures from erpnext.setup.setup_wizard.operations.install_fixtures import install_post_company_fixtures
install_post_company_fixtures(frappe._dict({'company_name': self.name})) install_post_company_fixtures(frappe._dict({'company_name': self.name}))
@ -335,6 +333,11 @@ class Company(NestedSet):
where doctype='Global Defaults' and field='default_company' where doctype='Global Defaults' and field='default_company'
and value=%s""", self.name) and value=%s""", self.name)
# reset default company
frappe.db.sql("""update `tabSingles` set value=""
where doctype='Chart of Accounts Importer' and field='company'
and value=%s""", self.name)
# delete BOMs # delete BOMs
boms = frappe.db.sql_list("select name from tabBOM where company=%s", self.name) boms = frappe.db.sql_list("select name from tabBOM where company=%s", self.name)
if boms: if boms: