From 55d465f3ad15f75e972bacfd602f58927b676b9d Mon Sep 17 00:00:00 2001 From: Aditya Hase Date: Fri, 1 Mar 2019 15:34:47 +0530 Subject: [PATCH] feat(tally): Preprocess master data to generate COA --- .../tally_migration/tally_migration.js | 33 ++- .../tally_migration/tally_migration.json | 230 +++++++++++++++++- .../tally_migration/tally_migration.py | 122 +++++++++- 3 files changed, 381 insertions(+), 4 deletions(-) diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js index 065fcc722e..4652d60bff 100644 --- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js +++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js @@ -3,6 +3,35 @@ frappe.ui.form.on('Tally Migration', { refresh: function(frm) { - - } + if (frm.doc.master_data && frm.doc.day_book) { + frm.disable_save(); + if(frm.doc.status != "In Progress") { + frm.page.set_primary_action("Preprocess", () => frm.trigger("preprocess")); + } + } else { + frm.set_value("status", "Attach File"); + } + if (frm.doc.tally_company && frm.doc.erpnext_company) { + frm.set_df_property("company_section", "hidden", 0); + frm.page.set_primary_action("Start Import", () => frm.trigger("start_import")); + } + }, + preprocess: function(frm) { + frm.call({ + doc: frm.doc, + method: "preprocess", + freeze: true + }).then((r) => { + frm.set_value("status", "Preprocessing In Progress"); + }); + }, + start_import: function(frm) { + frm.call({ + doc: frm.doc, + method: "start_import", + freeze: true + }).then((r) => { + frm.set_value("status", "Import In Progress"); + }); + }, }); diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json index b54e6beb20..7d51850aff 100644 --- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json +++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json @@ -14,6 +14,38 @@ "editable_grid": 1, "engine": "InnoDB", "fields": [ + { + "allow_bulk_edit": 0, + "allow_in_quick_entry": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "fieldname": "status", + "fieldtype": "Data", + "hidden": 1, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "Status", + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "translatable": 0, + "unique": 0 + }, { "allow_bulk_edit": 0, "allow_in_quick_entry": 0, @@ -46,6 +78,40 @@ "translatable": 0, "unique": 0 }, + { + "allow_bulk_edit": 0, + "allow_in_quick_entry": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "default": "Sundry Creditors", + "fieldname": "tally_creditors_account", + "fieldtype": "Data", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "Tally Creditors Account", + "length": 0, + "no_copy": 0, + "options": "", + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 1, + "search_index": 0, + "set_only_once": 0, + "translatable": 0, + "unique": 0 + }, { "allow_bulk_edit": 0, "allow_in_quick_entry": 0, @@ -108,6 +174,168 @@ "set_only_once": 0, "translatable": 0, "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_in_quick_entry": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "default": "Sundry Debtors", + "fieldname": "tally_debtors_account", + "fieldtype": "Data", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "Tally Debtors Account", + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 1, + "search_index": 0, + "set_only_once": 0, + "translatable": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_in_quick_entry": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "fieldname": "company_section", + "fieldtype": "Section Break", + "hidden": 1, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "Company Section", + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "translatable": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_in_quick_entry": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "fieldname": "tally_company", + "fieldtype": "Data", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "Tally Company", + "length": 0, + "no_copy": 0, + "options": "", + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 1, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "translatable": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_in_quick_entry": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "fieldname": "column_break_8", + "fieldtype": "Column Break", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "length": 0, + "no_copy": 0, + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "translatable": 0, + "unique": 0 + }, + { + "allow_bulk_edit": 0, + "allow_in_quick_entry": 0, + "allow_on_submit": 0, + "bold": 0, + "collapsible": 0, + "columns": 0, + "fieldname": "erpnext_company", + "fieldtype": "Data", + "hidden": 0, + "ignore_user_permissions": 0, + "ignore_xss_filter": 0, + "in_filter": 0, + "in_global_search": 0, + "in_list_view": 0, + "in_standard_filter": 0, + "label": "ERPNext Company", + "length": 0, + "no_copy": 0, + "options": "", + "permlevel": 0, + "precision": "", + "print_hide": 0, + "print_hide_if_no_value": 0, + "read_only": 0, + "remember_last_selected_value": 0, + "report_hide": 0, + "reqd": 0, + "search_index": 0, + "set_only_once": 0, + "translatable": 0, + "unique": 0 } ], "has_web_view": 0, @@ -120,7 +348,7 @@ "issingle": 0, "istable": 0, "max_attachments": 0, - "modified": "2019-02-20 12:59:17.746113", + "modified": "2019-03-01 15:02:47.992385", "modified_by": "Administrator", "module": "ERPNext Integrations", "name": "Tally Migration", diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py index f4e064ae1a..4c27d743fb 100644 --- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py +++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py @@ -4,9 +4,129 @@ from __future__ import unicode_literals +import json +import re +import zipfile import frappe from frappe.model.document import Document +from bs4 import BeautifulSoup as bs +PRIMARY_ACCOUNT = "Primary" class TallyMigration(Document): - pass + def _preprocess(self): + company, chart_of_accounts_tree, customers, suppliers = self._process_master_data() + self.tally_company = company + self.erpnext_company = company + self.status = "Preprocessed" + self.save() + + def _process_master_data(self): + def get_master_collection(master_data): + master_file = frappe.get_doc("File", {"file_url": master_data}) + + with zipfile.ZipFile(master_file.get_full_path()) as zf: + content = zf.read(zf.namelist()[0]).decode("utf-16") + + master = bs(sanitize(emptify(content)), "xml") + collection = master.BODY.IMPORTDATA.REQUESTDATA + return collection + + def get_company_name(collection): + return collection.find_all("REMOTECMPINFO.LIST")[0].REMOTECMPNAME.string + + def get_coa_customers_suppliers(collection): + root_type_map = { + "Application of Funds (Assets)": "Asset", + "Expenses": "Expense", + "Income": "Income", + "Source of Funds (Liabilities)": "Liability" + } + roots = set(root_type_map.keys()) + accounts = list(get_groups(collection.find_all("GROUP"))) + list(get_ledgers(collection.find_all("LEDGER"))) + children, parents = get_children_and_parent_dict(accounts) + group_set = [acc[1] for acc in accounts if acc[2]] + children, customers, suppliers = remove_parties(parents, children, group_set) + coa = traverse({}, children, roots, roots, group_set) + + for account in coa: + coa[account]["root_type"] = root_type_map[account] + + return coa, customers, suppliers + + def get_groups(accounts): + for account in accounts: + if account["NAME"] in (self.tally_creditors_account, self.tally_debtors_account): + yield get_parent(account), account["NAME"], 0 + else: + yield get_parent(account), account["NAME"], 1 + + def get_ledgers(accounts): + for account in accounts: + # If Ledger doesn't have PARENT field then don't create Account + # For example "Profit & Loss A/c" + if account.PARENT: + yield account.PARENT.string, account["NAME"], 0 + + def get_parent(account): + if account.PARENT: + return account.PARENT.string + return { + ("Yes", "No"): "Application of Funds (Assets)", + ("Yes", "Yes"): "Expenses", + ("No", "Yes"): "Income", + ("No", "No"): "Source of Funds (Liabilities)", + }[(account.ISDEEMEDPOSITIVE.string, account.ISREVENUE.string)] + + def get_children_and_parent_dict(accounts): + children, parents = {}, {} + for parent, account, is_group in accounts: + children.setdefault(parent, set()).add(account) + parents.setdefault(account, set()).add(parent) + return children, parents + + def remove_parties(parents, children, group_set): + customers, suppliers = set(), set() + for account in parents: + if self.tally_creditors_account in parents[account]: + children.pop(account, None) + if account not in group_set: + customers.add(account) + elif self.tally_debtors_account in parents[account]: + children.pop(account, None) + if account not in group_set: + suppliers.add(account) + return children, customers, suppliers + + def traverse(tree, children, accounts, roots, group_set): + for account in accounts: + if account in group_set or account in roots: + if account in children: + tree[account] = traverse({}, children, children[account], roots, group_set) + else: + tree[account] = {"is_group": 1} + else: + tree[account] = {} + return tree + + collection = get_master_collection(self.master_data) + + company = get_company_name(collection) + chart_of_accounts_tree, customer_names, supplier_names = get_coa_customers_suppliers(collection) + + return company, chart_of_accounts_tree, customer_names, supplier_names + + def preprocess(self): + frappe.enqueue_doc(self.doctype, self.name, "_preprocess") + + def start_import(self): + pass + +def sanitize(string): + return re.sub("", "", string) + +def emptify(string): + string = re.sub(r"<\w+/>", "", string) + string = re.sub(r"<([\w.]+)>\s*<\/\1>", "", string) + string = re.sub(r"\r\n", "", string) + return string