feat(tally): Preprocess master data to generate COA
This commit is contained in:
Aditya Hase
parent
9a84a99ac0
commit
55d465f3ad
@ -3,6 +3,35 @@
|
||||
|
||||
frappe.ui.form.on('Tally Migration', {
|
||||
refresh: function(frm) {
|
||||
|
||||
}
|
||||
if (frm.doc.master_data && frm.doc.day_book) {
|
||||
frm.disable_save();
|
||||
if(frm.doc.status != "In Progress") {
|
||||
frm.page.set_primary_action("Preprocess", () => frm.trigger("preprocess"));
|
||||
}
|
||||
} else {
|
||||
frm.set_value("status", "Attach File");
|
||||
}
|
||||
if (frm.doc.tally_company && frm.doc.erpnext_company) {
|
||||
frm.set_df_property("company_section", "hidden", 0);
|
||||
frm.page.set_primary_action("Start Import", () => frm.trigger("start_import"));
|
||||
}
|
||||
},
|
||||
preprocess: function(frm) {
|
||||
frm.call({
|
||||
doc: frm.doc,
|
||||
method: "preprocess",
|
||||
freeze: true
|
||||
}).then((r) => {
|
||||
frm.set_value("status", "Preprocessing In Progress");
|
||||
});
|
||||
},
|
||||
start_import: function(frm) {
|
||||
frm.call({
|
||||
doc: frm.doc,
|
||||
method: "start_import",
|
||||
freeze: true
|
||||
}).then((r) => {
|
||||
frm.set_value("status", "Import In Progress");
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
@ -14,6 +14,38 @@
|
||||
"editable_grid": 1,
|
||||
"engine": "InnoDB",
|
||||
"fields": [
|
||||
{
|
||||
"allow_bulk_edit": 0,
|
||||
"allow_in_quick_entry": 0,
|
||||
"allow_on_submit": 0,
|
||||
"bold": 0,
|
||||
"collapsible": 0,
|
||||
"columns": 0,
|
||||
"fieldname": "status",
|
||||
"fieldtype": "Data",
|
||||
"hidden": 1,
|
||||
"ignore_user_permissions": 0,
|
||||
"ignore_xss_filter": 0,
|
||||
"in_filter": 0,
|
||||
"in_global_search": 0,
|
||||
"in_list_view": 0,
|
||||
"in_standard_filter": 0,
|
||||
"label": "Status",
|
||||
"length": 0,
|
||||
"no_copy": 0,
|
||||
"permlevel": 0,
|
||||
"precision": "",
|
||||
"print_hide": 0,
|
||||
"print_hide_if_no_value": 0,
|
||||
"read_only": 0,
|
||||
"remember_last_selected_value": 0,
|
||||
"report_hide": 0,
|
||||
"reqd": 0,
|
||||
"search_index": 0,
|
||||
"set_only_once": 0,
|
||||
"translatable": 0,
|
||||
"unique": 0
|
||||
},
|
||||
{
|
||||
"allow_bulk_edit": 0,
|
||||
"allow_in_quick_entry": 0,
|
||||
@ -46,6 +78,40 @@
|
||||
"translatable": 0,
|
||||
"unique": 0
|
||||
},
|
||||
{
|
||||
"allow_bulk_edit": 0,
|
||||
"allow_in_quick_entry": 0,
|
||||
"allow_on_submit": 0,
|
||||
"bold": 0,
|
||||
"collapsible": 0,
|
||||
"columns": 0,
|
||||
"default": "Sundry Creditors",
|
||||
"fieldname": "tally_creditors_account",
|
||||
"fieldtype": "Data",
|
||||
"hidden": 0,
|
||||
"ignore_user_permissions": 0,
|
||||
"ignore_xss_filter": 0,
|
||||
"in_filter": 0,
|
||||
"in_global_search": 0,
|
||||
"in_list_view": 0,
|
||||
"in_standard_filter": 0,
|
||||
"label": "Tally Creditors Account",
|
||||
"length": 0,
|
||||
"no_copy": 0,
|
||||
"options": "",
|
||||
"permlevel": 0,
|
||||
"precision": "",
|
||||
"print_hide": 0,
|
||||
"print_hide_if_no_value": 0,
|
||||
"read_only": 0,
|
||||
"remember_last_selected_value": 0,
|
||||
"report_hide": 0,
|
||||
"reqd": 1,
|
||||
"search_index": 0,
|
||||
"set_only_once": 0,
|
||||
"translatable": 0,
|
||||
"unique": 0
|
||||
},
|
||||
{
|
||||
"allow_bulk_edit": 0,
|
||||
"allow_in_quick_entry": 0,
|
||||
@ -108,6 +174,168 @@
|
||||
"set_only_once": 0,
|
||||
"translatable": 0,
|
||||
"unique": 0
|
||||
},
|
||||
{
|
||||
"allow_bulk_edit": 0,
|
||||
"allow_in_quick_entry": 0,
|
||||
"allow_on_submit": 0,
|
||||
"bold": 0,
|
||||
"collapsible": 0,
|
||||
"columns": 0,
|
||||
"default": "Sundry Debtors",
|
||||
"fieldname": "tally_debtors_account",
|
||||
"fieldtype": "Data",
|
||||
"hidden": 0,
|
||||
"ignore_user_permissions": 0,
|
||||
"ignore_xss_filter": 0,
|
||||
"in_filter": 0,
|
||||
"in_global_search": 0,
|
||||
"in_list_view": 0,
|
||||
"in_standard_filter": 0,
|
||||
"label": "Tally Debtors Account",
|
||||
"length": 0,
|
||||
"no_copy": 0,
|
||||
"permlevel": 0,
|
||||
"precision": "",
|
||||
"print_hide": 0,
|
||||
"print_hide_if_no_value": 0,
|
||||
"read_only": 0,
|
||||
"remember_last_selected_value": 0,
|
||||
"report_hide": 0,
|
||||
"reqd": 1,
|
||||
"search_index": 0,
|
||||
"set_only_once": 0,
|
||||
"translatable": 0,
|
||||
"unique": 0
|
||||
},
|
||||
{
|
||||
"allow_bulk_edit": 0,
|
||||
"allow_in_quick_entry": 0,
|
||||
"allow_on_submit": 0,
|
||||
"bold": 0,
|
||||
"collapsible": 0,
|
||||
"columns": 0,
|
||||
"fieldname": "company_section",
|
||||
"fieldtype": "Section Break",
|
||||
"hidden": 1,
|
||||
"ignore_user_permissions": 0,
|
||||
"ignore_xss_filter": 0,
|
||||
"in_filter": 0,
|
||||
"in_global_search": 0,
|
||||
"in_list_view": 0,
|
||||
"in_standard_filter": 0,
|
||||
"label": "Company Section",
|
||||
"length": 0,
|
||||
"no_copy": 0,
|
||||
"permlevel": 0,
|
||||
"precision": "",
|
||||
"print_hide": 0,
|
||||
"print_hide_if_no_value": 0,
|
||||
"read_only": 0,
|
||||
"remember_last_selected_value": 0,
|
||||
"report_hide": 0,
|
||||
"reqd": 0,
|
||||
"search_index": 0,
|
||||
"set_only_once": 0,
|
||||
"translatable": 0,
|
||||
"unique": 0
|
||||
},
|
||||
{
|
||||
"allow_bulk_edit": 0,
|
||||
"allow_in_quick_entry": 0,
|
||||
"allow_on_submit": 0,
|
||||
"bold": 0,
|
||||
"collapsible": 0,
|
||||
"columns": 0,
|
||||
"fieldname": "tally_company",
|
||||
"fieldtype": "Data",
|
||||
"hidden": 0,
|
||||
"ignore_user_permissions": 0,
|
||||
"ignore_xss_filter": 0,
|
||||
"in_filter": 0,
|
||||
"in_global_search": 0,
|
||||
"in_list_view": 0,
|
||||
"in_standard_filter": 0,
|
||||
"label": "Tally Company",
|
||||
"length": 0,
|
||||
"no_copy": 0,
|
||||
"options": "",
|
||||
"permlevel": 0,
|
||||
"precision": "",
|
||||
"print_hide": 0,
|
||||
"print_hide_if_no_value": 0,
|
||||
"read_only": 1,
|
||||
"remember_last_selected_value": 0,
|
||||
"report_hide": 0,
|
||||
"reqd": 0,
|
||||
"search_index": 0,
|
||||
"set_only_once": 0,
|
||||
"translatable": 0,
|
||||
"unique": 0
|
||||
},
|
||||
{
|
||||
"allow_bulk_edit": 0,
|
||||
"allow_in_quick_entry": 0,
|
||||
"allow_on_submit": 0,
|
||||
"bold": 0,
|
||||
"collapsible": 0,
|
||||
"columns": 0,
|
||||
"fieldname": "column_break_8",
|
||||
"fieldtype": "Column Break",
|
||||
"hidden": 0,
|
||||
"ignore_user_permissions": 0,
|
||||
"ignore_xss_filter": 0,
|
||||
"in_filter": 0,
|
||||
"in_global_search": 0,
|
||||
"in_list_view": 0,
|
||||
"in_standard_filter": 0,
|
||||
"length": 0,
|
||||
"no_copy": 0,
|
||||
"permlevel": 0,
|
||||
"precision": "",
|
||||
"print_hide": 0,
|
||||
"print_hide_if_no_value": 0,
|
||||
"read_only": 0,
|
||||
"remember_last_selected_value": 0,
|
||||
"report_hide": 0,
|
||||
"reqd": 0,
|
||||
"search_index": 0,
|
||||
"set_only_once": 0,
|
||||
"translatable": 0,
|
||||
"unique": 0
|
||||
},
|
||||
{
|
||||
"allow_bulk_edit": 0,
|
||||
"allow_in_quick_entry": 0,
|
||||
"allow_on_submit": 0,
|
||||
"bold": 0,
|
||||
"collapsible": 0,
|
||||
"columns": 0,
|
||||
"fieldname": "erpnext_company",
|
||||
"fieldtype": "Data",
|
||||
"hidden": 0,
|
||||
"ignore_user_permissions": 0,
|
||||
"ignore_xss_filter": 0,
|
||||
"in_filter": 0,
|
||||
"in_global_search": 0,
|
||||
"in_list_view": 0,
|
||||
"in_standard_filter": 0,
|
||||
"label": "ERPNext Company",
|
||||
"length": 0,
|
||||
"no_copy": 0,
|
||||
"options": "",
|
||||
"permlevel": 0,
|
||||
"precision": "",
|
||||
"print_hide": 0,
|
||||
"print_hide_if_no_value": 0,
|
||||
"read_only": 0,
|
||||
"remember_last_selected_value": 0,
|
||||
"report_hide": 0,
|
||||
"reqd": 0,
|
||||
"search_index": 0,
|
||||
"set_only_once": 0,
|
||||
"translatable": 0,
|
||||
"unique": 0
|
||||
}
|
||||
],
|
||||
"has_web_view": 0,
|
||||
@ -120,7 +348,7 @@
|
||||
"issingle": 0,
|
||||
"istable": 0,
|
||||
"max_attachments": 0,
|
||||
"modified": "2019-02-20 12:59:17.746113",
|
||||
"modified": "2019-03-01 15:02:47.992385",
|
||||
"modified_by": "Administrator",
|
||||
"module": "ERPNext Integrations",
|
||||
"name": "Tally Migration",
|
||||
|
||||
@ -4,9 +4,129 @@
|
||||
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import json
|
||||
import re
|
||||
import zipfile
|
||||
import frappe
|
||||
from frappe.model.document import Document
|
||||
from bs4 import BeautifulSoup as bs
|
||||
|
||||
PRIMARY_ACCOUNT = "Primary"
|
||||
|
||||
class TallyMigration(Document):
|
||||
pass
|
||||
def _preprocess(self):
|
||||
company, chart_of_accounts_tree, customers, suppliers = self._process_master_data()
|
||||
self.tally_company = company
|
||||
self.erpnext_company = company
|
||||
self.status = "Preprocessed"
|
||||
self.save()
|
||||
|
||||
def _process_master_data(self):
|
||||
def get_master_collection(master_data):
|
||||
master_file = frappe.get_doc("File", {"file_url": master_data})
|
||||
|
||||
with zipfile.ZipFile(master_file.get_full_path()) as zf:
|
||||
content = zf.read(zf.namelist()[0]).decode("utf-16")
|
||||
|
||||
master = bs(sanitize(emptify(content)), "xml")
|
||||
collection = master.BODY.IMPORTDATA.REQUESTDATA
|
||||
return collection
|
||||
|
||||
def get_company_name(collection):
|
||||
return collection.find_all("REMOTECMPINFO.LIST")[0].REMOTECMPNAME.string
|
||||
|
||||
def get_coa_customers_suppliers(collection):
|
||||
root_type_map = {
|
||||
"Application of Funds (Assets)": "Asset",
|
||||
"Expenses": "Expense",
|
||||
"Income": "Income",
|
||||
"Source of Funds (Liabilities)": "Liability"
|
||||
}
|
||||
roots = set(root_type_map.keys())
|
||||
accounts = list(get_groups(collection.find_all("GROUP"))) + list(get_ledgers(collection.find_all("LEDGER")))
|
||||
children, parents = get_children_and_parent_dict(accounts)
|
||||
group_set = [acc[1] for acc in accounts if acc[2]]
|
||||
children, customers, suppliers = remove_parties(parents, children, group_set)
|
||||
coa = traverse({}, children, roots, roots, group_set)
|
||||
|
||||
for account in coa:
|
||||
coa[account]["root_type"] = root_type_map[account]
|
||||
|
||||
return coa, customers, suppliers
|
||||
|
||||
def get_groups(accounts):
|
||||
for account in accounts:
|
||||
if account["NAME"] in (self.tally_creditors_account, self.tally_debtors_account):
|
||||
yield get_parent(account), account["NAME"], 0
|
||||
else:
|
||||
yield get_parent(account), account["NAME"], 1
|
||||
|
||||
def get_ledgers(accounts):
|
||||
for account in accounts:
|
||||
# If Ledger doesn't have PARENT field then don't create Account
|
||||
# For example "Profit & Loss A/c"
|
||||
if account.PARENT:
|
||||
yield account.PARENT.string, account["NAME"], 0
|
||||
|
||||
def get_parent(account):
|
||||
if account.PARENT:
|
||||
return account.PARENT.string
|
||||
return {
|
||||
("Yes", "No"): "Application of Funds (Assets)",
|
||||
("Yes", "Yes"): "Expenses",
|
||||
("No", "Yes"): "Income",
|
||||
("No", "No"): "Source of Funds (Liabilities)",
|
||||
}[(account.ISDEEMEDPOSITIVE.string, account.ISREVENUE.string)]
|
||||
|
||||
def get_children_and_parent_dict(accounts):
|
||||
children, parents = {}, {}
|
||||
for parent, account, is_group in accounts:
|
||||
children.setdefault(parent, set()).add(account)
|
||||
parents.setdefault(account, set()).add(parent)
|
||||
return children, parents
|
||||
|
||||
def remove_parties(parents, children, group_set):
|
||||
customers, suppliers = set(), set()
|
||||
for account in parents:
|
||||
if self.tally_creditors_account in parents[account]:
|
||||
children.pop(account, None)
|
||||
if account not in group_set:
|
||||
customers.add(account)
|
||||
elif self.tally_debtors_account in parents[account]:
|
||||
children.pop(account, None)
|
||||
if account not in group_set:
|
||||
suppliers.add(account)
|
||||
return children, customers, suppliers
|
||||
|
||||
def traverse(tree, children, accounts, roots, group_set):
|
||||
for account in accounts:
|
||||
if account in group_set or account in roots:
|
||||
if account in children:
|
||||
tree[account] = traverse({}, children, children[account], roots, group_set)
|
||||
else:
|
||||
tree[account] = {"is_group": 1}
|
||||
else:
|
||||
tree[account] = {}
|
||||
return tree
|
||||
|
||||
collection = get_master_collection(self.master_data)
|
||||
|
||||
company = get_company_name(collection)
|
||||
chart_of_accounts_tree, customer_names, supplier_names = get_coa_customers_suppliers(collection)
|
||||
|
||||
return company, chart_of_accounts_tree, customer_names, supplier_names
|
||||
|
||||
def preprocess(self):
|
||||
frappe.enqueue_doc(self.doctype, self.name, "_preprocess")
|
||||
|
||||
def start_import(self):
|
||||
pass
|
||||
|
||||
def sanitize(string):
|
||||
return re.sub("", "", string)
|
||||
|
||||
def emptify(string):
|
||||
string = re.sub(r"<\w+/>", "", string)
|
||||
string = re.sub(r"<([\w.]+)>\s*<\/\1>", "", string)
|
||||
string = re.sub(r"\r\n", "", string)
|
||||
return string
|
||||
|
||||
Loading…
Reference in New Issue
Block a user