Shiloh/brotherton-erpnext
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: Major changes

Browse Source
This commit is contained in:
Aditya Hase 2019-03-05 19:16:00 +05:30
parent b6e9eb9784
commit ba61be93ae
3 changed files with 491 additions and 656 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js
View File

@ -2,36 +2,49 @@
// For license information, please see license.txt
frappe.ui.form.on('Tally Migration', {
onload: function(frm) {
frappe.realtime.on("tally_migration_progress_update", function (data) {
frm.dashboard.show_progress(data.title, (data.count / data.total) * 100, data.message);
if (data.count == data.total) {
window.setTimeout(title => frm.dashboard.hide_progress(title), 1500, data.title);
}
});
},
refresh: function(frm) {
if (frm.doc.master_data && frm.doc.day_book) {
frm.disable_save();
if(frm.doc.status != "In Progress") {
frm.page.set_primary_action("Preprocess", () => frm.trigger("preprocess"));
if (frm.doc.master_data && !frm.doc.is_master_data_imported) {
if (frm.doc.is_master_data_processed) {
if (frm.doc.status != "Importing Master Data") {
frm.events.add_button(frm, __("Import Master Data"), "import_master_data");
}
} else {
frm.set_value("status", "Attach File");
if (frm.doc.status != "Processing Master Data") {
frm.events.add_button(frm, __("Process Master Data"), "process_master_data");
}
}
}
if (frm.doc.day_book_data && !frm.doc.is_day_book_data_imported) {
if (frm.doc.is_day_book_data_processed) {
if (frm.doc.status != "Importing Day Book Data") {
frm.events.add_button(frm, __("Import Day Book Data"), "import_day_book_data");
}
} else {
if (frm.doc.status != "Processing Day Book Data") {
frm.events.add_button(frm, __("Process Day Book Data"), "process_day_book_data");
}
}
if (frm.doc.tally_company && frm.doc.erpnext_company) {
frm.set_df_property("company_section", "hidden", 0);
frm.page.set_primary_action("Start Import", () => frm.trigger("start_import"));
}
},
preprocess: function(frm) {
frm.call({
add_button: function(frm, label, method) {
frm.add_custom_button(
label,
() => frm.call({
doc: frm.doc,
method: "preprocess",
freeze: true
}).then((r) => {
frm.set_value("status", "Preprocessing In Progress");
});
},
start_import: function(frm) {
frm.call({
doc: frm.doc,
method: "start_import",
freeze: true
}).then((r) => {
frm.set_value("status", "Import In Progress");
});
},
method: method,
freeze: true,
callback: () => {
frm.remove_custom_button(label);
}
})
);
}
});

639
erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json
View File

@ -1,610 +1,219 @@
{
"allow_copy": 0,
"allow_events_in_timeline": 0,
"allow_guest_to_view": 0,
"allow_import": 0,
"allow_rename": 0,
"autoname": "",
"beta": 0,
"beta": 1,
"creation": "2019-02-01 14:27:09.485238",
"custom": 0,
"docstatus": 0,
"doctype": "DocType",
"document_type": "",
"editable_grid": 1,
"engine": "InnoDB",
"field_order": [
"status",
"master_data",
"is_master_data_processed",
"is_master_data_imported",
"column_break_2",
"tally_creditors_account",
"tally_debtors_account",
"company_section",
"tally_company",
"column_break_8",
"erpnext_company",
"processed_files_section",
"chart_of_accounts",
"parties",
"addresses",
"column_break_17",
"uoms",
"items",
"vouchers",
"accounts_section",
"default_warehouse",
"round_off_account",
"column_break_21",
"default_cost_center",
"day_book_section",
"day_book_data",
"column_break_27",
"is_day_book_data_processed",
"is_day_book_data_imported"
],
"fields": [
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "status",
"fieldtype": "Data",
"hidden": 1,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Status",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
"label": "Status"
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "master_data",
"fieldtype": "Attach",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 1,
"in_standard_filter": 0,
"label": "Master Data",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
"label": "Master Data"
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"default": "Sundry Creditors",
"fieldname": "tally_creditors_account",
"fieldtype": "Data",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Tally Creditors Account",
"length": 0,
"no_copy": 0,
"options": "",
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 1,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
"reqd": 1
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "column_break_2",
"fieldtype": "Column Break",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
"fieldtype": "Column Break"
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "day_book",
"fieldtype": "Attach",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 1,
"in_standard_filter": 0,
"label": "Day Book",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"default": "Sundry Debtors",
"fieldname": "tally_debtors_account",
"fieldtype": "Data",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Tally Debtors Account",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 1,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
"reqd": 1
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"depends_on": "is_master_data_processed",
"fieldname": "company_section",
"fieldtype": "Section Break",
"hidden": 1,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Company Section",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
"fieldtype": "Section Break"
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "tally_company",
"fieldtype": "Data",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Tally Company",
"length": 0,
"no_copy": 0,
"options": "",
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 1,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
"read_only": 1
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "column_break_8",
"fieldtype": "Column Break",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
"fieldtype": "Column Break"
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "erpnext_company",
"fieldtype": "Data",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "ERPNext Company",
"length": 0,
"no_copy": 0,
"options": "",
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 1,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
"label": "ERPNext Company"
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "processed_files",
"fieldname": "processed_files_section",
"fieldtype": "Section Break",
"hidden": 1,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Processed Files",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
"label": "Processed Files"
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "chart_of_accounts",
"fieldtype": "Attach",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Chart of Accounts",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
"label": "Chart of Accounts"
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "parties",
"fieldtype": "Attach",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Parties",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
"label": "Parties"
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "addresses",
"fieldtype": "Attach",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Addresses",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
"label": "Addresses"
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "column_break_17",
"fieldtype": "Column Break",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
"fieldtype": "Column Break"
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "uoms",
"fieldtype": "Attach",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "UOMs",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
"label": "UOMs"
},
{
"allow_bulk_edit": 0,
"allow_in_quick_entry": 0,
"allow_on_submit": 0,
"bold": 0,
"collapsible": 0,
"columns": 0,
"fieldname": "items",
"fieldtype": "Attach",
"hidden": 0,
"ignore_user_permissions": 0,
"ignore_xss_filter": 0,
"in_filter": 0,
"in_global_search": 0,
"in_list_view": 0,
"in_standard_filter": 0,
"label": "Items",
"length": 0,
"no_copy": 0,
"permlevel": 0,
"precision": "",
"print_hide": 0,
"print_hide_if_no_value": 0,
"read_only": 0,
"remember_last_selected_value": 0,
"report_hide": 0,
"reqd": 0,
"search_index": 0,
"set_only_once": 0,
"translatable": 0,
"unique": 0
"label": "Items"
},
{
"fieldname": "vouchers",
"fieldtype": "Attach",
"label": "Vouchers"
},
{
"depends_on": "is_master_data_imported",
"fieldname": "accounts_section",
"fieldtype": "Section Break",
"label": "Accounts"
},
{
"fieldname": "default_warehouse",
"fieldtype": "Link",
"label": "Default Warehouse",
"options": "Warehouse"
},
{
"fieldname": "round_off_account",
"fieldtype": "Link",
"label": "Round Off Account",
"options": "Account"
},
{
"fieldname": "column_break_21",
"fieldtype": "Column Break"
},
{
"fieldname": "default_cost_center",
"fieldtype": "Link",
"label": "Default Cost Center",
"options": "Cost Center"
},
{
"fieldname": "is_master_data_processed",
"fieldtype": "Check",
"label": "Is Master Data Processed",
"read_only": 1
},
{
"fieldname": "is_day_book_data_processed",
"fieldtype": "Check",
"label": "Is Day Book Data Processed",
"read_only": 1
},
{
"fieldname": "is_day_book_data_imported",
"fieldtype": "Check",
"label": "Is Day Book Data Imported",
"read_only": 1
},
{
"fieldname": "is_master_data_imported",
"fieldtype": "Check",
"label": "Is Master Data Imported",
"read_only": 1
},
{
"depends_on": "is_master_data_imported",
"fieldname": "day_book_section",
"fieldtype": "Section Break"
},
{
"fieldname": "column_break_27",
"fieldtype": "Column Break"
},
{
"fieldname": "day_book_data",
"fieldtype": "Attach",
"in_list_view": 1,
"label": "Day Book Data"
}
],
"has_web_view": 0,
"hide_heading": 0,
"hide_toolbar": 0,
"idx": 0,
"image_view": 0,
"in_create": 0,
"is_submittable": 0,
"issingle": 0,
"istable": 0,
"max_attachments": 0,
"modified": "2019-03-02 21:51:48.869051",
"modified": "2019-04-29 05:46:54.394967",
"modified_by": "Administrator",
"module": "ERPNext Integrations",
"name": "Tally Migration",
"name_case": "",
"owner": "Administrator",
"permissions": [
{
"amend": 0,
"cancel": 0,
"create": 1,
"delete": 1,
"email": 1,
"export": 1,
"if_owner": 0,
"import": 0,
"permlevel": 0,
"print": 1,
"read": 1,
"report": 1,
"role": "System Manager",
"set_user_permissions": 0,
"share": 1,
"submit": 0,
"write": 1
}
],
"quick_entry": 0,
"read_only": 0,
"read_only_onload": 0,
"show_name_in_global_search": 0,
"sort_field": "modified",
"sort_order": "DESC",
"track_changes": 1,
"track_seen": 0,
"track_views": 0
"track_changes": 1
}

407
erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py
View File

@ -4,86 +4,65 @@
from __future__ import unicode_literals
from decimal import Decimal
import json
import re
import traceback
import zipfile
import frappe
from frappe import _
from frappe.custom.doctype.custom_field.custom_field import create_custom_field
from frappe.model.document import Document
from frappe.model.naming import getseries, revert_series_if_last
from frappe.utils.data import format_datetime
from bs4 import BeautifulSoup as bs
from erpnext import encode_company_abbr
from erpnext.accounts.doctype.account.chart_of_accounts.chart_of_accounts import create_charts
PRIMARY_ACCOUNT = "Primary"
VOUCHER_CHUNK_SIZE = 500
class TallyMigration(Document):
def _preprocess(self):
company, chart_of_accounts_tree, customers, suppliers = self._process_master_data()
parties, addresses = self._process_parties(customers, suppliers)
items, uoms = self._process_stock_items()
self.tally_company = company
self.erpnext_company = company
self.status = "Preprocessed"
def autoname(self):
if not self.name:
self.name = "Tally Migration on " + format_datetime(self.creation)
coa_file = frappe.get_doc({
"doctype": "File",
"file_name": "COA.json",
"attached_to_doctype": self.doctype,
"attached_to_name": self.name,
"content": json.dumps(chart_of_accounts_tree)
}).insert()
self.chart_of_accounts = coa_file.file_url
def get_collection(self, data_file):
def sanitize(string):
return re.sub("", "", string)
parties_file = frappe.get_doc({
"doctype": "File",
"file_name": "Parties.json",
"attached_to_doctype": self.doctype,
"attached_to_name": self.name,
"content": json.dumps(parties)
}).insert()
self.parties = parties_file.file_url
def emptify(string):
string = re.sub(r"<\w+/>", "", string)
string = re.sub(r"<([\w.]+)>\s*<\/\1>", "", string)
string = re.sub(r"\r\n", "", string)
return string
addresses_file = frappe.get_doc({
"doctype": "File",
"file_name": "Addresses.json",
"attached_to_doctype": self.doctype,
"attached_to_name": self.name,
"content": json.dumps(addresses)
}).insert()
self.addresses = addresses_file.file_url
uoms_file = frappe.get_doc({
"doctype": "File",
"file_name": "UOMs.json",
"attached_to_doctype": self.doctype,
"attached_to_name": self.name,
"content": json.dumps(uoms)
}).insert()
self.uoms = uoms_file.file_url
items_file = frappe.get_doc({
"doctype": "File",
"file_name": "Items.json",
"attached_to_doctype": self.doctype,
"attached_to_name": self.name,
"content": json.dumps(items)
}).insert()
self.items = items_file.file_url
self.save()
def get_master_collection(self):
master_file = frappe.get_doc("File", {"file_url": self.master_data})
master_file = frappe.get_doc("File", {"file_url": data_file})
with zipfile.ZipFile(master_file.get_full_path()) as zf:
content = zf.read(zf.namelist()[0]).decode("utf-16")
encoded_content = zf.read(zf.namelist()[0])
try:
content = encoded_content.decode("utf-8-sig")
except UnicodeDecodeError:
content = encoded_content.decode("utf-16")
master = bs(sanitize(emptify(content)), "xml")
collection = master.BODY.IMPORTDATA.REQUESTDATA
return collection
def _process_master_data(self):
def dump_processed_data(self, data):
for key, value in data.items():
f = frappe.get_doc({
"doctype": "File",
"file_name": key + ".json",
"attached_to_doctype": self.doctype,
"attached_to_name": self.name,
"content": json.dumps(value)
}).insert()
setattr(self, key, f.file_url)
def _process_master_data(self):
def get_company_name(collection):
return collection.find_all("REMOTECMPINFO.LIST")[0].REMOTECMPNAME.string
@ -144,11 +123,11 @@ class TallyMigration(Document):
if self.tally_creditors_account in parents[account]:
children.pop(account, None)
if account not in group_set:
customers.add(account)
suppliers.add(account)
elif self.tally_debtors_account in parents[account]:
children.pop(account, None)
if account not in group_set:
suppliers.add(account)
customers.add(account)
return children, customers, suppliers
def traverse(tree, children, accounts, roots, group_set):
@ -162,14 +141,6 @@ class TallyMigration(Document):
tree[account] = {}
return tree
collection = self.get_master_collection()
company = get_company_name(collection)
chart_of_accounts_tree, customer_names, supplier_names = get_coa_customers_suppliers(collection)
return company, chart_of_accounts_tree, customer_names, supplier_names
def _process_parties(self, customers, suppliers):
def get_parties_addresses(collection, customers, suppliers):
parties, addresses = [], []
for account in collection.find_all("LEDGER"):
@ -210,33 +181,49 @@ class TallyMigration(Document):
})
return parties, addresses
collection = self.get_master_collection()
parties, addresses = get_parties_addresses(collection, customers, suppliers)
return parties, addresses
def _process_stock_items(self):
collection = self.get_master_collection()
def get_stock_items_uoms(collection):
uoms = []
for uom in collection.find_all("UNIT"):
uoms.append({"doctype": "UOM", "uom_name": uom.NAME.string})
items = []
for item in collection.find_all("STOCKITEM"):
hsn_code = item.find_all("GSTDETAILS.LIST")[0].HSNCODE
items.append({
"doctype": "Item",
"item_code" : item.NAME.string,
"stock_uom": item.BASEUNITS.string,
"gst_hsn_code": hsn_code.string if hsn_code else None,
"is_stock_item": 0,
"item_group": "All Item Groups",
"item_defaults": [{"company": self.erpnext_company}]
})
return items, uoms
def preprocess(self):
frappe.enqueue_doc(self.doctype, self.name, "_preprocess")
def _start_import(self):
self.publish("Process Master Data", _("Reading Uploaded File"), 1, 5)
collection = self.get_collection(self.master_data)
company = get_company_name(collection)
self.tally_company = company
self.erpnext_company = company
self.publish("Process Master Data", _("Processing Chart of Accounts and Parties"), 2, 5)
chart_of_accounts, customers, suppliers = get_coa_customers_suppliers(collection)
self.publish("Process Master Data", _("Processing Party Addresses"), 3, 5)
parties, addresses = get_parties_addresses(collection, customers, suppliers)
self.publish("Process Master Data", _("Processing Items and UOMs"), 4, 5)
items, uoms = get_stock_items_uoms(collection)
data = {"chart_of_accounts": chart_of_accounts, "parties": parties, "addresses": addresses, "items": items, "uoms": uoms}
self.publish("Process Master Data", _("Done"), 5, 5)
self.dump_processed_data(data)
self.is_master_data_processed = 1
self.status = ""
self.save()
def publish(self, title, message, count, total):
frappe.publish_realtime("tally_migration_progress_update", {"title": title, "message": message, "count": count, "total": total})
def _import_master_data(self):
def create_company_and_coa(coa_file_url):
coa_file = frappe.get_doc("File", {"file_url": coa_file_url})
frappe.local.flags.ignore_chart_of_accounts = True
@ -244,56 +231,282 @@ class TallyMigration(Document):
"doctype": "Company",
"company_name": self.erpnext_company,
"default_currency": "INR",
"enable_perpetual_inventory": 0,
}).insert()
frappe.local.flags.ignore_chart_of_accounts = False
create_charts(company.name, json.loads(coa_file.get_content()))
create_charts(company.name, custom_chart=json.loads(coa_file.get_content()))
company.create_default_warehouses()
def create_parties_addresses(parties_file_url, addresses_file_url):
def create_parties_and_addresses(parties_file_url, addresses_file_url):
parties_file = frappe.get_doc("File", {"file_url": parties_file_url})
for party in json.loads(parties_file.get_content()):
try:
frappe.get_doc(party).insert()
except:
log(party)
self.log(party)
addresses_file = frappe.get_doc("File", {"file_url": addresses_file_url})
for address in json.loads(addresses_file.get_content()):
try:
frappe.get_doc(address).insert(ignore_mandatory=True)
except:
log(address)
try:
gstin = address.pop("gstin", None)
frappe.get_doc(address).insert(ignore_mandatory=True)
self.log({"address": address, "message": "Invalid GSTIN: {}. Address was created without GSTIN".format(gstin)})
except:
self.log(address)
def create_items_uoms(items_file_url, uoms_file_url):
uoms_file = frappe.get_doc("File", {"file_url": uoms_file_url})
for uom in json.loads(uoms_file.get_content()):
if not frappe.db.exists(uom):
try:
frappe.get_doc(uom).insert()
except:
log(uom)
self.log(uom)
items_file = frappe.get_doc("File", {"file_url": items_file_url})
for item in json.loads(items_file.get_content()):
try:
frappe.get_doc(item).insert()
except:
log(item)
self.log(item)
self.publish("Import Master Data", _("Creating Company and Importing Chart of Accounts"), 1, 4)
create_company_and_coa(self.chart_of_accounts)
create_parties_addresses(self.parties, self.addresses)
self.publish("Import Master Data", _("Importing Parties and Addresses"), 2, 4)
create_parties_and_addresses(self.parties, self.addresses)
self.publish("Import Master Data", _("Importing Items and UOMs"), 3, 4)
create_items_uoms(self.items, self.uoms)
self.publish("Import Master Data", _("Done"), 4, 4)
self.status = ""
self.is_master_data_imported = 1
self.save()
def start_import(self):
frappe.enqueue_doc(self.doctype, self.name, "_start_import")
def _process_day_book_data(self):
def get_vouchers(collection):
vouchers = []
for voucher in collection.find_all("VOUCHER"):
if voucher.ISCANCELLED.string == "Yes":
continue
inventory_entries = voucher.find_all("INVENTORYENTRIES.LIST") + voucher.find_all("ALLINVENTORYENTRIES.LIST") + voucher.find_all("INVENTORYENTRIESIN.LIST") + voucher.find_all("INVENTORYENTRIESOUT.LIST")
if voucher.VOUCHERTYPENAME.string not in ["Journal", "Receipt", "Payment", "Contra"] and inventory_entries:
function = voucher_to_invoice
else:
function = voucher_to_journal_entry
try:
vouchers.append(function(voucher))
except:
self.log(voucher)
return vouchers
def voucher_to_journal_entry(voucher):
accounts = []
ledger_entries = voucher.find_all("ALLLEDGERENTRIES.LIST") + voucher.find_all("LEDGERENTRIES.LIST")
for entry in ledger_entries:
account = {"account": encode_company_abbr(entry.LEDGERNAME.string, self.erpnext_company), "cost_center": self.default_cost_center}
if entry.ISPARTYLEDGER.string == "Yes":
party_details = get_party(entry.LEDGERNAME.string)
if party_details:
party_type, party_account = party_details
account["party_type"] = party_type
account["account"] = party_account
account["party"] = entry.LEDGERNAME.string
amount = Decimal(entry.AMOUNT.string)
if amount > 0:
account["credit_in_account_currency"] = str(abs(amount))
else:
account["debit_in_account_currency"] = str(abs(amount))
accounts.append(account)
def log(data=None):
message = json.dumps({"data": data, "exception": traceback.format_exc()}, indent=4)
frappe.log_error(title="Tally Migration Error", message=message)
journal_entry = {
"doctype": "Journal Entry",
"tally_guid": voucher.GUID.string,
"posting_date": voucher.DATE.string,
"company": self.erpnext_company,
"accounts": accounts,
}
return journal_entry
def sanitize(string):
return re.sub("&#4;", "", string)
def voucher_to_invoice(voucher):
if voucher.VOUCHERTYPENAME.string in ["Sales", "Credit Note"]:
doctype = "Sales Invoice"
party_field = "customer"
account_field = "debit_to"
account_name = encode_company_abbr(self.tally_debtors_account, self.erpnext_company)
price_list_field = "selling_price_list"
elif voucher.VOUCHERTYPENAME.string in ["Purchase", "Debit Note"]:
doctype = "Purchase Invoice"
party_field = "supplier"
account_field = "credit_to"
account_name = encode_company_abbr(self.tally_creditors_account, self.erpnext_company)
price_list_field = "buying_price_list"
def emptify(string):
string = re.sub(r"<\w+/>", "", string)
string = re.sub(r"<([\w.]+)>\s*<\/\1>", "", string)
string = re.sub(r"\r\n", "", string)
return string
invoice = {
"doctype": doctype,
party_field: voucher.PARTYNAME.string,
"tally_guid": voucher.GUID.string,
"posting_date": voucher.DATE.string,
"due_date": voucher.DATE.string,
"items": get_voucher_items(voucher, doctype),
"taxes": get_voucher_taxes(voucher),
account_field: account_name,
price_list_field: "Tally Price List",
"set_posting_time": 1,
"disable_rounded_total": 1,
"company": self.erpnext_company,
}
return invoice
def get_voucher_items(voucher, doctype):
inventory_entries = voucher.find_all("INVENTORYENTRIES.LIST") + voucher.find_all("ALLINVENTORYENTRIES.LIST") + voucher.find_all("INVENTORYENTRIESIN.LIST") + voucher.find_all("INVENTORYENTRIESOUT.LIST")
if doctype == "Sales Invoice":
account_field = "income_account"
elif doctype == "Purchase Invoice":
account_field = "expense_account"
items = []
for entry in inventory_entries:
qty, uom = entry.ACTUALQTY.string.strip().split()
items.append({
"item_code": entry.STOCKITEMNAME.string,
"description": entry.STOCKITEMNAME.string,
"qty": qty.strip(),
"uom": uom.strip(),
"conversion_factor": 1,
"price_list_rate": entry.RATE.string.split("/")[0],
"cost_center": self.default_cost_center,
"warehouse": self.default_warehouse,
account_field: encode_company_abbr(entry.find_all("ACCOUNTINGALLOCATIONS.LIST")[0].LEDGERNAME.string, self.erpnext_company),
})
return items
def get_voucher_taxes(voucher):
ledger_entries = voucher.find_all("ALLLEDGERENTRIES.LIST") + voucher.find_all("LEDGERENTRIES.LIST")
taxes = []
for entry in ledger_entries:
if entry.ISPARTYLEDGER.string == "No":
tax_account = encode_company_abbr(entry.LEDGERNAME.string, self.erpnext_company)
taxes.append({
"charge_type": "Actual",
"account_head": tax_account,
"description": tax_account,
"tax_amount": entry.AMOUNT.string,
"cost_center": self.default_cost_center,
})
return taxes
def get_party(party):
if frappe.db.exists({"doctype": "Supplier", "supplier_name": party}):
return "Supplier", encode_company_abbr(self.tally_creditors_account, self.erpnext_company)
elif frappe.db.exists({"doctype": "Customer", "customer_name": party}):
return "Customer", encode_company_abbr(self.tally_debtors_account, self.erpnext_company)
self.publish("Process Day Book Data", _("Reading Uploaded File"), 1, 3)
collection = self.get_collection(self.day_book_data)
self.publish("Process Day Book Data", _("Processing Vouchers"), 2, 3)
vouchers = get_vouchers(collection)
self.publish("Process Day Book Data", _("Done"), 3, 3)
self.dump_processed_data({"vouchers": vouchers})
self.status = ""
self.is_day_book_data_processed = 1
self.save()
def _import_day_book_data(self):
def create_fiscal_years(vouchers):
from frappe.utils.data import add_years, getdate
earliest_date = getdate(min(voucher["posting_date"] for voucher in vouchers))
oldest_year = frappe.get_all("Fiscal Year", fields=["year_start_date", "year_end_date"], order_by="year_start_date")[0]
while earliest_date < oldest_year.year_start_date:
new_year = frappe.get_doc({"doctype": "Fiscal Year"})
new_year.year_start_date = add_years(oldest_year.year_start_date, -1)
new_year.year_end_date = add_years(oldest_year.year_end_date, -1)
if new_year.year_start_date.year == new_year.year_end_date.year:
new_year.year = new_year.year_start_date.year
else:
new_year.year = "{}-{}".format(new_year.year_start_date.year, new_year.year_end_date.year)
new_year.save()
oldest_year = new_year
def create_custom_fields(doctypes):
for doctype in doctypes:
df = {
"fieldtype": "Data",
"fieldname": "tally_guid",
"read_only": 1,
"label": "Tally GUID"
}
create_custom_field(doctype, df)
def create_price_list():
frappe.get_doc({
"doctype": "Price List",
"price_list_name": "Tally Price List",
"selling": 1,
"buying": 1,
"enabled": 1,
"currency": "INR"
}).insert()
frappe.db.set_value("Account", encode_company_abbr(self.tally_creditors_account, self.erpnext_company), "account_type", "Payable")
frappe.db.set_value("Account", encode_company_abbr(self.tally_debtors_account, self.erpnext_company), "account_type", "Receivable")
frappe.db.set_value("Company", self.erpnext_company, "round_off_account", self.round_off_account)
vouchers_file = frappe.get_doc("File", {"file_url": self.vouchers})
vouchers = json.loads(vouchers_file.get_content())
create_fiscal_years(vouchers)
create_price_list()
create_custom_fields(["Journal Entry", "Purchase Invoice", "Sales Invoice"])
total = len(vouchers)
is_last = False
for index in range(0, total, VOUCHER_CHUNK_SIZE):
if index + VOUCHER_CHUNK_SIZE >= total:
is_last = True
frappe.enqueue_doc(self.doctype, self.name, "_import_vouchers", queue="long", timeout=3600, start=index+1, total=total, is_last=is_last)
def _import_vouchers(self, start, total, is_last=False):
frappe.flags.in_migrate = True
vouchers_file = frappe.get_doc("File", {"file_url": self.vouchers})
vouchers = json.loads(vouchers_file.get_content())
chunk = vouchers[start: start + VOUCHER_CHUNK_SIZE]
for index, voucher in enumerate(chunk, start=start):
try:
doc = frappe.get_doc(voucher).insert()
doc.submit()
self.publish("Importing Vouchers", _("{} of {}").format(index, total), index, total)
except:
self.log(voucher)
if is_last:
self.status = ""
self.is_day_book_data_imported = 1
self.save()
frappe.db.set_value("Price List", "Tally Price List", "enabled", 0)
frappe.flags.in_migrate = False
def process_master_data(self):
self.status = "Processing Master Data"
self.save()
frappe.enqueue_doc(self.doctype, self.name, "_process_master_data", queue="long", timeout=3600)
def import_master_data(self):
self.status = "Importing Master Data"
self.save()
frappe.enqueue_doc(self.doctype, self.name, "_import_master_data", queue="long", timeout=3600)
def process_day_book_data(self):
self.status = "Processing Day Book Data"
self.save()
frappe.enqueue_doc(self.doctype, self.name, "_process_day_book_data", queue="long", timeout=3600)
def import_day_book_data(self):
self.status = "Importing Day Book Data"
self.save()
frappe.enqueue_doc(self.doctype, self.name, "_import_day_book_data", queue="long", timeout=3600)
def log(self, data=None):
message = "\n".join(["Data", json.dumps(data, default=str, indent=4), "Exception", traceback.format_exc()])
return frappe.log_error(title="Tally Migration Error", message=message)