From defeb737471efab0100df5682777572ed0c4e2ee Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Fri, 24 Apr 2020 04:07:14 +0530
Subject: [PATCH 01/16] feat: Add Failed error logs

fix: set defaults
---
 .../tally_migration/tally_migration.json      | 47 ++++++++++++++-----
 1 file changed, 36 insertions(+), 11 deletions(-)

diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json
index dc6f093ac9..005c8a98c6 100644
--- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json
+++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json
@@ -28,14 +28,17 @@
   "vouchers",
   "accounts_section",
   "default_warehouse",
-  "round_off_account",
+  "default_round_off_account",
   "column_break_21",
   "default_cost_center",
   "day_book_section",
   "day_book_data",
   "column_break_27",
   "is_day_book_data_processed",
-  "is_day_book_data_imported"
+  "is_day_book_data_imported",
+  "import_log_section",
+  "failed_import_log",
+  "failed_import_preview"
  ],
  "fields": [
   {
@@ -57,6 +60,7 @@
    "fieldname": "tally_creditors_account",
    "fieldtype": "Data",
    "label": "Tally Creditors Account",
+   "read_only_depends_on": "eval:doc.is_master_data_processed==1",
    "reqd": 1
   },
   {
@@ -69,6 +73,7 @@
    "fieldname": "tally_debtors_account",
    "fieldtype": "Data",
    "label": "Tally Debtors Account",
+   "read_only_depends_on": "eval:doc.is_master_data_processed==1",
    "reqd": 1
   },
   {
@@ -92,7 +97,7 @@
    "fieldname": "erpnext_company",
    "fieldtype": "Data",
    "label": "ERPNext Company",
-   "read_only_depends_on": "eval:doc.is_master_data_processed == 1"
+   "read_only_depends_on": "eval:doc.is_master_data_processed==1"
   },
   {
    "fieldname": "processed_files_section",
@@ -136,6 +141,7 @@
   },
   {
    "depends_on": "is_master_data_imported",
+   "description": "The accounts are set by the system automatically but do confirm these defaults",
    "fieldname": "accounts_section",
    "fieldtype": "Section Break",
    "label": "Accounts"
@@ -146,12 +152,6 @@
    "label": "Default Warehouse",
    "options": "Warehouse"
   },
-  {
-   "fieldname": "round_off_account",
-   "fieldtype": "Link",
-   "label": "Round Off Account",
-   "options": "Account"
-  },
   {
    "fieldname": "column_break_21",
    "fieldtype": "Column Break"
@@ -212,11 +212,36 @@
    "fieldname": "default_uom",
    "fieldtype": "Link",
    "label": "Default UOM",
-   "options": "UOM"
+   "options": "UOM",
+   "read_only_depends_on": "eval:doc.is_master_data_imported==1"
+  },
+  {
+   "default": "[]",
+   "fieldname": "failed_import_log",
+   "fieldtype": "Code",
+   "hidden": 1,
+   "label": "Failed Import Log",
+   "options": "JSON"
+  },
+  {
+   "fieldname": "failed_import_preview",
+   "fieldtype": "HTML",
+   "label": "Failed Import Log"
+  },
+  {
+   "fieldname": "import_log_section",
+   "fieldtype": "Section Break",
+   "label": "Import Log"
+  },
+  {
+   "fieldname": "default_round_off_account",
+   "fieldtype": "Link",
+   "label": "Default Round Off Account",
+   "options": "Account"
   }
  ],
  "links": [],
- "modified": "2020-04-16 13:03:28.894919",
+ "modified": "2020-04-22 16:01:13.718842",
  "modified_by": "Administrator",
  "module": "ERPNext Integrations",
  "name": "Tally Migration",

From 2de4a5a7c884080db1ac03acf40262ca887f8913 Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Mon, 27 Apr 2020 11:34:00 +0530
Subject: [PATCH 02/16] feat: Error logging via doc

fix: Handle duplicate company and COA
feat: Error logging via doc
feat: fetch defaults for accounts
---
 .../tally_migration/tally_migration.py        | 99 ++++++++++++-------
 1 file changed, 65 insertions(+), 34 deletions(-)

diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py
index 13474e19ee..32d11681fb 100644
--- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py
+++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py
@@ -6,6 +6,7 @@ from __future__ import unicode_literals
 
 import json
 import re
+import sys
 import traceback
 import zipfile
 from decimal import Decimal
@@ -15,13 +16,14 @@ from bs4 import BeautifulSoup as bs
 import frappe
 from erpnext import encode_company_abbr
 from erpnext.accounts.doctype.account.chart_of_accounts.chart_of_accounts import create_charts
+from erpnext.accounts.doctype.chart_of_accounts_importer.chart_of_accounts_importer import unset_existing_data
+
 from frappe import _
 from frappe.custom.doctype.custom_field.custom_field import create_custom_field
 from frappe.model.document import Document
 from frappe.model.naming import getseries, revert_series_if_last
 from frappe.utils.data import format_datetime
 
-
 PRIMARY_ACCOUNT = "Primary"
 VOUCHER_CHUNK_SIZE = 500
 
@@ -65,9 +67,17 @@ class TallyMigration(Document):
 				"attached_to_name": self.name,
 				"content": json.dumps(value),
 				"is_private": True
-			}).insert()
+			})
+			try:
+				f.insert()
+			except frappe.DuplicateEntryError:
+				pass
 			setattr(self, key, f.file_url)
 
+	def set_account_defaults(self):
+		self.default_cost_center, self.default_round_off_account = frappe.db.get_value("Company", self.erpnext_company, ["cost_center", "round_off_account"])
+		self.default_warehouse = frappe.db.get_value("Stock Settings", "Stock Settings", "default_warehouse")
+
 	def _process_master_data(self):
 		def get_company_name(collection):
 			return collection.find_all("REMOTECMPINFO.LIST")[0].REMOTECMPNAME.string.strip()
@@ -84,7 +94,11 @@ class TallyMigration(Document):
 			children, parents = get_children_and_parent_dict(accounts)
 			group_set =  [acc[1] for acc in accounts if acc[2]]
 			children, customers, suppliers = remove_parties(parents, children, group_set)
-			coa = traverse({}, children, roots, roots, group_set)
+
+			try:
+				coa = traverse({}, children, roots, roots, group_set)
+			except RecursionError:
+				self.log()
 
 			for account in coa:
 				coa[account]["root_type"] = root_type_map[account]
@@ -242,12 +256,18 @@ class TallyMigration(Document):
 		def create_company_and_coa(coa_file_url):
 			coa_file = frappe.get_doc("File", {"file_url": coa_file_url})
 			frappe.local.flags.ignore_chart_of_accounts = True
-			company = frappe.get_doc({
-				"doctype": "Company",
-				"company_name": self.erpnext_company,
-				"default_currency": "INR",
-				"enable_perpetual_inventory": 0,
-			}).insert()
+
+			try:
+				company = frappe.get_doc({
+					"doctype": "Company",
+					"company_name": self.erpnext_company,
+					"default_currency": "INR",
+					"enable_perpetual_inventory": 0,
+				}).insert()
+			except frappe.DuplicateEntryError:
+				company = frappe.get_doc("Company", self.erpnext_company)
+				unset_existing_data(self.erpnext_company)
+
 			frappe.local.flags.ignore_chart_of_accounts = False
 			create_charts(company.name, custom_chart=json.loads(coa_file.get_content()))
 			company.create_default_warehouses()
@@ -256,36 +276,35 @@ class TallyMigration(Document):
 			parties_file = frappe.get_doc("File", {"file_url": parties_file_url})
 			for party in json.loads(parties_file.get_content()):
 				try:
-					frappe.get_doc(party).insert()
+					party_doc = frappe.get_doc(party)
+					party_doc.insert()
 				except:
-					self.log(party)
+					self.log(party_doc)
 			addresses_file = frappe.get_doc("File", {"file_url": addresses_file_url})
 			for address in json.loads(addresses_file.get_content()):
 				try:
-					frappe.get_doc(address).insert(ignore_mandatory=True)
+					address_doc = frappe.get_doc(address)
+					address_doc.insert(ignore_mandatory=True)
 				except:
-					try:
-						gstin = address.pop("gstin", None)
-						frappe.get_doc(address).insert(ignore_mandatory=True)
-						self.log({"address": address, "message": "Invalid GSTIN: {}. Address was created without GSTIN".format(gstin)})
-					except:
-						self.log(address)
+					self.log(address_doc)
 
 		def create_items_uoms(items_file_url, uoms_file_url):
 			uoms_file = frappe.get_doc("File", {"file_url": uoms_file_url})
 			for uom in json.loads(uoms_file.get_content()):
 				if not frappe.db.exists(uom):
 					try:
-						frappe.get_doc(uom).insert()
+						uom_doc = frappe.get_doc(uom)
+						uom_doc.insert()
 					except:
-						self.log(uom)
+						self.log(uom_doc)
 
 			items_file = frappe.get_doc("File", {"file_url": items_file_url})
 			for item in json.loads(items_file.get_content()):
 				try:
-					frappe.get_doc(item).insert()
+					item_doc = frappe.get_doc(item)
+					item_doc.insert()
 				except:
-					self.log(item)
+					self.log(item_doc)
 
 		try:
 			self.publish("Import Master Data", _("Creating Company and Importing Chart of Accounts"), 1, 4)
@@ -299,6 +318,7 @@ class TallyMigration(Document):
 
 			self.publish("Import Master Data", _("Done"), 4, 4)
 
+			self.set_account_defaults()
 			self.is_master_data_imported = 1
 
 		except:
@@ -468,13 +488,13 @@ class TallyMigration(Document):
 				oldest_year = new_year
 
 		def create_custom_fields(doctypes):
+			df = {
+				"fieldtype": "Data",
+				"fieldname": "tally_guid",
+				"read_only": 1,
+				"label": "Tally GUID"
+			}
 			for doctype in doctypes:
-				df = {
-					"fieldtype": "Data",
-					"fieldname": "tally_guid",
-					"read_only": 1,
-					"label": "Tally GUID"
-				}
 				create_custom_field(doctype, df)
 
 		def create_price_list():
@@ -521,11 +541,12 @@ class TallyMigration(Document):
 
 		for index, voucher in enumerate(chunk, start=start):
 			try:
-				doc = frappe.get_doc(voucher).insert()
-				doc.submit()
+				voucher_doc = frappe.get_doc(voucher)
+				voucher_doc.insert()
+				voucher_doc.submit()
 				self.publish("Importing Vouchers", _("{} of {}").format(index, total), index, total)
 			except:
-				self.log(voucher)
+				self.log(voucher_doc)
 
 		if is_last:
 			self.status = ""
@@ -551,9 +572,19 @@ class TallyMigration(Document):
 		frappe.enqueue_doc(self.doctype, self.name, "_import_day_book_data", queue="long", timeout=3600)
 
 	def log(self, data=None):
-		data = data or self.status
-		message = "\n".join(["Data:", json.dumps(data, default=str, indent=4), "--" * 50, "\nException:", traceback.format_exc()])
-		return frappe.log_error(title="Tally Migration Error", message=message)
+		if isinstance(data, frappe.model.document.Document):
+			if sys.exc_info()[1].__class__ != frappe.DuplicateEntryError:
+				failed_import_log = json.loads(self.failed_import_log)
+				failed_import_log.append({
+					"doc": data.as_dict(),
+					"exc": traceback.format_exc()
+				})
+				self.failed_import_log = json.dumps(failed_import_log)
+				self.save()
+		else:
+			data = data or self.status
+			message = "\n".join(["Data:", json.dumps(data, default=str, indent=4), "--" * 50, "\nException:", traceback.format_exc()])
+			return frappe.log_error(title="Tally Migration Error", message=message)
 
 	def set_status(self, status=""):
 		self.status = status

From 38c677689dd54d91e0255700d94a68b0e92b3c1e Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Thu, 23 Apr 2020 14:07:14 +0530
Subject: [PATCH 03/16] fix: set default round off acount

---
 .../doctype/tally_migration/tally_migration.py                  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py
index 32d11681fb..5a5759160b 100644
--- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py
+++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py
@@ -510,7 +510,7 @@ class TallyMigration(Document):
 		try:
 			frappe.db.set_value("Account", encode_company_abbr(self.tally_creditors_account, self.erpnext_company), "account_type", "Payable")
 			frappe.db.set_value("Account", encode_company_abbr(self.tally_debtors_account, self.erpnext_company), "account_type", "Receivable")
-			frappe.db.set_value("Company", self.erpnext_company, "round_off_account", self.round_off_account)
+			frappe.db.set_value("Company", self.erpnext_company, "round_off_account", self.default_round_off_account)
 
 			vouchers_file = frappe.get_doc("File", {"file_url": self.vouchers})
 			vouchers = json.loads(vouchers_file.get_content())

From 82e7a9de2856a27ed497d7559fe775524a1c54e0 Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Mon, 27 Apr 2020 13:12:38 +0530
Subject: [PATCH 04/16] feat: error log handling

fix: toggle required accounts
---
 .../tally_migration/tally_migration.js        | 102 +++++++++++++++++-
 1 file changed, 101 insertions(+), 1 deletion(-)

diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js
index d84c8234ef..d06fe537ad 100644
--- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js
+++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js
@@ -1,7 +1,7 @@
 // Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and contributors
 // For license information, please see license.txt
 
-frappe.ui.form.on('Tally Migration', {
+frappe.ui.form.on("Tally Migration", {
 	onload: function (frm) {
 		let reload_status = true;
 		frappe.realtime.on("tally_migration_progress_update", function (data) {
@@ -36,6 +36,10 @@ frappe.ui.form.on('Tally Migration', {
 		});
 	},
 	refresh: function (frm) {
+		frm.trigger("show_import_log");
+		["default_round_off_account", "default_warehouse", "default_cost_center"].forEach(account => {
+			frm.toggle_reqd(account, frm.doc.is_master_data_imported === 1)
+		})
 		if (frm.doc.master_data && !frm.doc.is_master_data_imported) {
 			if (frm.doc.is_master_data_processed) {
 				if (frm.doc.status != "Importing Master Data") {
@@ -71,5 +75,101 @@ frappe.ui.form.on('Tally Migration', {
 				frm.reload_doc();
 			}
 		);
+	},
+	show_import_log(frm) {
+		let index = 0;
+		let import_log = JSON.parse(frm.doc.failed_import_log || "[]");
+		let logs = import_log.slice(0, 20);
+		let hidden_logs = import_log.slice(20);
+
+		frm.toggle_display("import_log_section", logs.length > 0);
+
+
+		const getError = (traceback) => {
+			let exc_error_idx = traceback.trim().lastIndexOf("\n") + 1
+			let error_line = traceback.substr(exc_error_idx)
+			let split_str_idx = (error_line.indexOf(':') > 0) ? error_line.indexOf(':') + 1 : 0;
+
+			return error_line.slice(split_str_idx).trim();
+		}
+
+		const cleanDoc = (obj) => {
+			let temp = obj;
+			$.each(temp, function(key, value){
+				if (value === "" || value === null){
+					delete obj[key];
+				} else if (Object.prototype.toString.call(value) === '[object Object]') {
+					cleanDoc(value);
+				} else if ($.isArray(value)) {
+					$.each(value, function (k,v) { cleanDoc(v); });
+				}
+			});
+			return temp;
+		};
+
+		let rows = logs
+			.map(({ doc, exc }) => {
+				let id = frappe.dom.get_unique_id();
+				let traceback = exc;
+
+				let error_message = getError(traceback);
+				index++;
+
+				let html = `
+					<button class="btn btn-default btn-xs m-3" type="button" data-toggle="collapse" data-target="#${id}-traceback" aria-expanded="false" aria-controls="${id}-traceback">
+						${__("Show Traceback")}
+					</button>
+					<div class="collapse margin-top" id="${id}-traceback">
+						<div class="well">
+							<pre style="font-size: smaller;">${traceback}</pre>
+						</div>
+					</div>`;
+
+				let show_doc = `
+					<button class='btn btn-default btn-xs m-3' type='button' data-toggle='collapse' data-target='#${id}-doc' aria-expanded='false' aria-controls='${id}-doc'>
+						${__("Show Document")}
+					</button>
+					<div class="collapse margin-top" id="${id}-doc">
+						<div class="well">
+							<pre style="font-size: smaller;">${JSON.stringify(cleanDoc(doc), null, 1)}</pre>
+						</div>
+					</div>`;
+
+				let create_button = `
+					<button class='btn btn-default btn-xs m-3' type='button' onclick='frappe.new_doc("${doc.doctype}", ${JSON.stringify(doc)})'>
+						${__("Create Document")}
+					</button>`
+
+				return `<tr>
+							<td>${index}</td>
+							<td>
+								<div>${doc.doctype}</div>
+							</td>
+							<td>
+								<div>${error_message}</div>
+								<div>${html}</div>
+								<div>${show_doc}</div>
+							</td>
+							<td>
+								<div>${create_button}</div>
+							</td>
+						</tr>`;
+				})
+			.join("");
+
+		frm.get_field("failed_import_preview").$wrapper.html(`
+			<table class="table table-bordered">
+				<tr class="text-muted">
+					<th width="5%">${__("#")}</th>
+					<th width="10%">${__("DocType")}</th>
+					<th width="75%">${__("Error Message")}</th>
+					<th width="10%">${__("Create")}</th>
+				</tr>
+				${rows}
+				<tr class="text-muted">
+					<td colspan="4">And ${hidden_logs.length} more others</td>
+				</tr>
+			</table>
+		`);
 	}
 });

From 0805307514a7c9cafa6af9d6b5e0a490d42c5800 Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Mon, 27 Apr 2020 13:42:00 +0530
Subject: [PATCH 05/16] perf: reduce failed_error_log size

---
 .../doctype/tally_migration/tally_migration.py             | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py
index 5a5759160b..46382c1e8f 100644
--- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py
+++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py
@@ -575,11 +575,14 @@ class TallyMigration(Document):
 		if isinstance(data, frappe.model.document.Document):
 			if sys.exc_info()[1].__class__ != frappe.DuplicateEntryError:
 				failed_import_log = json.loads(self.failed_import_log)
+				doc = data.as_dict()
+				doc_fields = { x.fieldname for x in frappe.get_doc("DocType", doc.doctype).fields }
+				stripped_doc = { k: v for k, v in doc.items() if k in doc_fields }
 				failed_import_log.append({
-					"doc": data.as_dict(),
+					"doc": stripped_doc,
 					"exc": traceback.format_exc()
 				})
-				self.failed_import_log = json.dumps(failed_import_log)
+				self.failed_import_log = json.dumps(failed_import_log, separators=(',', ':'))
 				self.save()
 		else:
 			data = data or self.status

From 1b5eea691cf166562efb1de019599da1d365aede Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Tue, 28 Apr 2020 15:51:40 +0530
Subject: [PATCH 06/16] feat: order failed import log by creation, handle
 rollbacks and commits

---
 .../doctype/tally_migration/tally_migration.py  | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py
index 46382c1e8f..cbc6019c6b 100644
--- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py
+++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py
@@ -29,6 +29,11 @@ VOUCHER_CHUNK_SIZE = 500
 
 
 class TallyMigration(Document):
+	def validate(self):
+		failed_import_log = json.loads(self.failed_import_log)
+		sorted_failed_import_log = sorted(failed_import_log, key=lambda row: row["doc"]["creation"])
+		self.failed_import_log = json.dumps(sorted_failed_import_log)
+
 	def autoname(self):
 		if not self.name:
 			self.name = "Tally Migration on " + format_datetime(self.creation)
@@ -320,9 +325,11 @@ class TallyMigration(Document):
 
 			self.set_account_defaults()
 			self.is_master_data_imported = 1
+			frappe.db.commit()
 
 		except:
 			self.publish("Import Master Data", _("Process Failed"), -1, 5)
+			frappe.db.rollback()
 			self.log()
 
 		finally:
@@ -343,7 +350,9 @@ class TallyMigration(Document):
 					processed_voucher = function(voucher)
 					if processed_voucher:
 						vouchers.append(processed_voucher)
+					frappe.db.commit()
 				except:
+					frappe.db.rollback()
 					self.log(voucher)
 			return vouchers
 
@@ -545,7 +554,9 @@ class TallyMigration(Document):
 				voucher_doc.insert()
 				voucher_doc.submit()
 				self.publish("Importing Vouchers", _("{} of {}").format(index, total), index, total)
+				frappe.db.commit()
 			except:
+				frappe.db.rollback()
 				self.log(voucher_doc)
 
 		if is_last:
@@ -576,14 +587,14 @@ class TallyMigration(Document):
 			if sys.exc_info()[1].__class__ != frappe.DuplicateEntryError:
 				failed_import_log = json.loads(self.failed_import_log)
 				doc = data.as_dict()
-				doc_fields = { x.fieldname for x in frappe.get_doc("DocType", doc.doctype).fields }
-				stripped_doc = { k: v for k, v in doc.items() if k in doc_fields }
 				failed_import_log.append({
-					"doc": stripped_doc,
+					"doc": doc,
 					"exc": traceback.format_exc()
 				})
 				self.failed_import_log = json.dumps(failed_import_log, separators=(',', ':'))
 				self.save()
+				frappe.db.commit()
+
 		else:
 			data = data or self.status
 			message = "\n".join(["Data:", json.dumps(data, default=str, indent=4), "--" * 50, "\nException:", traceback.format_exc()])

From 640636bbcd6f84c12c5ed0e74a3e257acb731f52 Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Tue, 28 Apr 2020 16:01:59 +0530
Subject: [PATCH 07/16] feat: maintain logs of errors successfully fixed

---
 .../tally_migration/tally_migration.json      | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json
index 005c8a98c6..417d943792 100644
--- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json
+++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.json
@@ -38,7 +38,9 @@
   "is_day_book_data_imported",
   "import_log_section",
   "failed_import_log",
-  "failed_import_preview"
+  "fixed_errors_log",
+  "failed_import_preview",
+  "fixed_error_log_preview"
  ],
  "fields": [
   {
@@ -220,7 +222,6 @@
    "fieldname": "failed_import_log",
    "fieldtype": "Code",
    "hidden": 1,
-   "label": "Failed Import Log",
    "options": "JSON"
   },
   {
@@ -238,10 +239,22 @@
    "fieldtype": "Link",
    "label": "Default Round Off Account",
    "options": "Account"
+  },
+  {
+   "default": "[]",
+   "fieldname": "fixed_errors_log",
+   "fieldtype": "Code",
+   "hidden": 1,
+   "options": "JSON"
+  },
+  {
+   "fieldname": "fixed_error_log_preview",
+   "fieldtype": "HTML",
+   "label": "Fixed Error Log"
   }
  ],
  "links": [],
- "modified": "2020-04-22 16:01:13.718842",
+ "modified": "2020-04-28 00:29:18.039826",
  "modified_by": "Administrator",
  "module": "ERPNext Integrations",
  "name": "Tally Migration",

From 2e1b531f4fce10bdaf74218fa7a0d260d67c1d7a Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Tue, 28 Apr 2020 16:02:50 +0530
Subject: [PATCH 08/16] feat: handle resolve and unresolved errors

restructured and reused components
---
 .../tally_migration/tally_migration.js        | 299 +++++++++++++-----
 1 file changed, 217 insertions(+), 82 deletions(-)

diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js
index d06fe537ad..9235bab068 100644
--- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js
+++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js
@@ -35,8 +35,10 @@ frappe.ui.form.on("Tally Migration", {
 			}
 		});
 	},
+
 	refresh: function (frm) {
-		frm.trigger("show_import_log");
+		frm.trigger("show_logs_preview");
+
 		["default_round_off_account", "default_warehouse", "default_cost_center"].forEach(account => {
 			frm.toggle_reqd(account, frm.doc.is_master_data_imported === 1)
 		})
@@ -63,6 +65,7 @@ frappe.ui.form.on("Tally Migration", {
 			}
 		}
 	},
+
 	add_button: function (frm, label, method) {
 		frm.add_custom_button(
 			label,
@@ -76,94 +79,30 @@ frappe.ui.form.on("Tally Migration", {
 			}
 		);
 	},
-	show_import_log(frm) {
-		let index = 0;
-		let import_log = JSON.parse(frm.doc.failed_import_log || "[]");
-		let logs = import_log.slice(0, 20);
-		let hidden_logs = import_log.slice(20);
 
-		frm.toggle_display("import_log_section", logs.length > 0);
-
-
-		const getError = (traceback) => {
-			let exc_error_idx = traceback.trim().lastIndexOf("\n") + 1
-			let error_line = traceback.substr(exc_error_idx)
-			let split_str_idx = (error_line.indexOf(':') > 0) ? error_line.indexOf(':') + 1 : 0;
-
-			return error_line.slice(split_str_idx).trim();
+	render_html_table(frm, shown_logs, hidden_logs, field) {
+		if (shown_logs && shown_logs.length > 0) {
+			frm.toggle_display(field, true);
+		} else {
+			frm.toggle_display(field, false);
+			return
 		}
+		let rows = get_html_rows(shown_logs, field);
+		let rows_head;
 
-		const cleanDoc = (obj) => {
-			let temp = obj;
-			$.each(temp, function(key, value){
-				if (value === "" || value === null){
-					delete obj[key];
-				} else if (Object.prototype.toString.call(value) === '[object Object]') {
-					cleanDoc(value);
-				} else if ($.isArray(value)) {
-					$.each(value, function (k,v) { cleanDoc(v); });
-				}
-			});
-			return temp;
-		};
-
-		let rows = logs
-			.map(({ doc, exc }) => {
-				let id = frappe.dom.get_unique_id();
-				let traceback = exc;
-
-				let error_message = getError(traceback);
-				index++;
-
-				let html = `
-					<button class="btn btn-default btn-xs m-3" type="button" data-toggle="collapse" data-target="#${id}-traceback" aria-expanded="false" aria-controls="${id}-traceback">
-						${__("Show Traceback")}
-					</button>
-					<div class="collapse margin-top" id="${id}-traceback">
-						<div class="well">
-							<pre style="font-size: smaller;">${traceback}</pre>
-						</div>
-					</div>`;
-
-				let show_doc = `
-					<button class='btn btn-default btn-xs m-3' type='button' data-toggle='collapse' data-target='#${id}-doc' aria-expanded='false' aria-controls='${id}-doc'>
-						${__("Show Document")}
-					</button>
-					<div class="collapse margin-top" id="${id}-doc">
-						<div class="well">
-							<pre style="font-size: smaller;">${JSON.stringify(cleanDoc(doc), null, 1)}</pre>
-						</div>
-					</div>`;
-
-				let create_button = `
-					<button class='btn btn-default btn-xs m-3' type='button' onclick='frappe.new_doc("${doc.doctype}", ${JSON.stringify(doc)})'>
-						${__("Create Document")}
-					</button>`
-
-				return `<tr>
-							<td>${index}</td>
-							<td>
-								<div>${doc.doctype}</div>
-							</td>
-							<td>
-								<div>${error_message}</div>
-								<div>${html}</div>
-								<div>${show_doc}</div>
-							</td>
-							<td>
-								<div>${create_button}</div>
-							</td>
-						</tr>`;
-				})
-			.join("");
-
-		frm.get_field("failed_import_preview").$wrapper.html(`
+		if (field === "fixed_error_log_preview") {
+			rows_head = `<th width="75%">${__("Meta Data")}</th>
+			<th width="10%">${__("Unresolve")}</th>`
+		} else {
+			rows_head = `<th width="75%">${__("Error Message")}</th>
+			<th width="10%">${__("Create")}</th>`
+		}
+		frm.get_field(field).$wrapper.html(`
 			<table class="table table-bordered">
 				<tr class="text-muted">
 					<th width="5%">${__("#")}</th>
 					<th width="10%">${__("DocType")}</th>
-					<th width="75%">${__("Error Message")}</th>
-					<th width="10%">${__("Create")}</th>
+					${rows_head}
 				</tr>
 				${rows}
 				<tr class="text-muted">
@@ -171,5 +110,201 @@ frappe.ui.form.on("Tally Migration", {
 				</tr>
 			</table>
 		`);
+	},
+
+	show_error_summary() {
+		let summary = import_log.reduce((summary, row) => {
+			if (row.doc) {
+				if (summary[row.doc.doctype]) {
+					summary[row.doc.doctype] += 1;
+				} else {
+					summary[row.doc.doctype] = 1;
+				}
+			}
+			return summary
+		}, {});
+		console.table(summary);
+	},
+
+	show_logs_preview(frm) {
+		let empty = "[]";
+		let import_log = frm.doc.failed_import_log || empty;
+		let completed_log = frm.doc.fixed_errors_log || empty;
+		let render_section = !(import_log === completed_log && import_log === empty);
+
+		frm.toggle_display("import_log_section", render_section);
+
+		if (render_section) {
+			frm.trigger("show_errored_import_log");
+			frm.trigger("show_fixed_errors_log");
+		}
+	},
+
+	show_errored_import_log(frm) {
+		let import_log = JSON.parse(frm.doc.failed_import_log || "[]");
+
+		let logs = import_log.slice(0, 20);
+		let hidden_logs = import_log.slice(20);
+
+		frm.events.render_html_table(frm, logs, hidden_logs, "failed_import_preview");
+	},
+
+	show_fixed_errors_log(frm) {
+		let completed_log = JSON.parse(frm.doc.fixed_errors_log || "[]");
+		let logs = completed_log.slice(0, 20);
+		let hidden_logs = completed_log.slice(20);
+
+		frm.events.render_html_table(frm, logs, hidden_logs, "fixed_error_log_preview");
 	}
 });
+
+const getError = (traceback) => {
+	/* Extracts the Error Message from the Python Traceback or Solved error */
+	let is_multiline = traceback.trim().indexOf("\n") != -1;
+	let message;
+
+	if (is_multiline) {
+		let exc_error_idx = traceback.trim().lastIndexOf("\n") + 1
+		let error_line = traceback.substr(exc_error_idx)
+		let split_str_idx = (error_line.indexOf(':') > 0) ? error_line.indexOf(':') + 1 : 0;
+		message = error_line.slice(split_str_idx).trim();
+	} else {
+		message = traceback;
+	}
+
+	return message
+}
+
+const cleanDoc = (obj) => {
+	/* Strips all null and empty values of your JSON object */
+	let temp = obj;
+	$.each(temp, function(key, value){
+		if (value === "" || value === null){
+			delete obj[key];
+		} else if (Object.prototype.toString.call(value) === '[object Object]') {
+			cleanDoc(value);
+		} else if ($.isArray(value)) {
+			$.each(value, function (k,v) { cleanDoc(v); });
+		}
+	});
+	return temp;
+}
+
+window.unresolve = (document) => {
+	let frm = cur_frm;
+	let failed_log = JSON.parse(frm.doc.failed_import_log);
+	let fixed_log = JSON.parse(frm.doc.fixed_errors_log);
+
+	let modified_fixed_log = fixed_log.filter(row => {
+		if (!frappe.utils.deep_equal(cleanDoc(row.doc), document)) {
+			return row
+		}
+	});
+
+	failed_log.push({ doc: document, exc: `Marked unresolved on ${Date()}` });
+
+	frm.doc.failed_import_log = JSON.stringify(failed_log);
+	frm.doc.fixed_errors_log = JSON.stringify(modified_fixed_log);
+
+	// frm.trigger('show_logs_preview')
+	frm.dirty();
+	frm.save();
+}
+
+window.create_new_doc = (doctype, document) => {
+	let frm = cur_frm;
+	let failed_log = JSON.parse(frm.doc.failed_import_log);
+	let fixed_log = JSON.parse(frm.doc.fixed_errors_log);
+
+	let modified_failed_log = failed_log.filter(row => {
+		if (!frappe.utils.deep_equal(cleanDoc(row.doc), document)) {
+			return row
+		}
+	});
+	fixed_log.push({ doc: document, exc: `Solved on ${Date()}` });
+
+	frm.doc.failed_import_log = JSON.stringify(modified_failed_log);
+	frm.doc.fixed_errors_log = JSON.stringify(fixed_log);
+
+	// frm.trigger('show_logs_preview')
+	frm.dirty();
+	frm.save();
+	frappe.new_doc(doctype, document);
+}
+
+const get_html_rows = (logs, field) => {
+	let index = 0;
+	let rows = logs
+		.map(({ doc, exc }) => {
+			let id = frappe.dom.get_unique_id();
+			let traceback = exc;
+
+			let error_message = getError(traceback);
+			index++;
+
+			let show_traceback = `
+				<button class="btn btn-default btn-xs m-3" type="button" data-toggle="collapse" data-target="#${id}-traceback" aria-expanded="false" aria-controls="${id}-traceback">
+					${__("Show Traceback")}
+				</button>
+				<div class="collapse margin-top" id="${id}-traceback">
+					<div class="well">
+						<pre style="font-size: smaller;">${traceback}</pre>
+					</div>
+				</div>`;
+
+			let show_doc = `
+				<button class='btn btn-default btn-xs m-3' type='button' data-toggle='collapse' data-target='#${id}-doc' aria-expanded='false' aria-controls='${id}-doc'>
+					${__("Show Document")}
+				</button>
+				<div class="collapse margin-top" id="${id}-doc">
+					<div class="well">
+						<pre style="font-size: smaller;">${JSON.stringify(cleanDoc(doc), null, 1)}</pre>
+					</div>
+				</div>`;
+
+			let create_button = `
+				<button class='btn btn-default btn-xs m-3' type='button' onclick='create_new_doc("${doc.doctype}", ${JSON.stringify(doc)})'>
+					${__("Create Document")}
+				</button>`
+
+			let mark_as_unresolved = `
+				<button class='btn btn-default btn-xs m-3' type='button' onclick='unresolve(${JSON.stringify(doc)})'>
+					${__("Mark as unresolved")}
+				</button>`
+
+			if (field === "fixed_error_log_preview") {
+				return `<tr>
+							<td>${index}</td>
+							<td>
+								<div>${doc.doctype}</div>
+							</td>
+							<td>
+								<div>${error_message}</div>
+								<div>${show_traceback}</div>
+								<div>${show_doc}</div>
+							</td>
+							<td>
+								<div>${mark_as_unresolved}</div>
+							</td>
+						</tr>`;
+			} else {
+				return `<tr>
+							<td>${index}</td>
+							<td>
+								<div>${doc.doctype}</div>
+							</td>
+							<td>
+								<div>${error_message}</div>
+								<div>${show_traceback}</div>
+								<div>${show_doc}</div>
+							</td>
+							<td>
+								<div>${create_button}</div>
+							</td>
+						</tr>`;
+			}
+			})
+		.join("");
+
+	return rows
+}
\ No newline at end of file

From 27b7172fa4b888feaf47cf2875405079e83196ef Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Tue, 28 Apr 2020 16:32:15 +0530
Subject: [PATCH 09/16] fix: run summary logger

style: indent fixes
---
 .../doctype/tally_migration/tally_migration.js             | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js
index 9235bab068..bf8ae8c1c6 100644
--- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js
+++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js
@@ -133,7 +133,7 @@ frappe.ui.form.on("Tally Migration", {
 		let render_section = !(import_log === completed_log && import_log === empty);
 
 		frm.toggle_display("import_log_section", render_section);
-
+		frm.trigger("show_error_summary");
 		if (render_section) {
 			frm.trigger("show_errored_import_log");
 			frm.trigger("show_fixed_errors_log");
@@ -302,9 +302,8 @@ const get_html_rows = (logs, field) => {
 								<div>${create_button}</div>
 							</td>
 						</tr>`;
-			}
-			})
-		.join("");
+		}
+	}).join("");
 
 	return rows
 }
\ No newline at end of file

From 0d61b35b0bc26ce509801b747bdc2d1d349b7d02 Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Wed, 29 Apr 2020 04:01:24 +0530
Subject: [PATCH 10/16] fix: table cleanups, updated summary block

---
 .../tally_migration/tally_migration.js        | 23 +++++++++++--------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js
index bf8ae8c1c6..9cc3c98a56 100644
--- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js
+++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js
@@ -88,32 +88,38 @@ frappe.ui.form.on("Tally Migration", {
 			return
 		}
 		let rows = get_html_rows(shown_logs, field);
-		let rows_head;
+		let rows_head, table_caption;
+
+		let table_footer = (hidden_logs && (hidden_logs.length > 0)) ? `<tr class="text-muted">
+				<td colspan="4">And ${hidden_logs.length} more others</td>
+			</tr>`: "";
 
 		if (field === "fixed_error_log_preview") {
 			rows_head = `<th width="75%">${__("Meta Data")}</th>
 			<th width="10%">${__("Unresolve")}</th>`
+			table_caption = "Resolved Issues"
 		} else {
 			rows_head = `<th width="75%">${__("Error Message")}</th>
 			<th width="10%">${__("Create")}</th>`
+			table_caption = "Error Log"
 		}
+
 		frm.get_field(field).$wrapper.html(`
 			<table class="table table-bordered">
+				<caption>${table_caption}</caption>
 				<tr class="text-muted">
 					<th width="5%">${__("#")}</th>
 					<th width="10%">${__("DocType")}</th>
 					${rows_head}
 				</tr>
 				${rows}
-				<tr class="text-muted">
-					<td colspan="4">And ${hidden_logs.length} more others</td>
-				</tr>
+				${table_footer}
 			</table>
 		`);
 	},
 
-	show_error_summary() {
-		let summary = import_log.reduce((summary, row) => {
+	show_error_summary(frm) {
+		let summary = JSON.parse(frm.doc.failed_import_log).reduce((summary, row) => {
 			if (row.doc) {
 				if (summary[row.doc.doctype]) {
 					summary[row.doc.doctype] += 1;
@@ -280,7 +286,6 @@ const get_html_rows = (logs, field) => {
 							</td>
 							<td>
 								<div>${error_message}</div>
-								<div>${show_traceback}</div>
 								<div>${show_doc}</div>
 							</td>
 							<td>
@@ -302,8 +307,8 @@ const get_html_rows = (logs, field) => {
 								<div>${create_button}</div>
 							</td>
 						</tr>`;
-		}
-	}).join("");
+			}
+		}).join("");
 
 	return rows
 }
\ No newline at end of file

From 6b850b7e71e30c2513ae8f50152eb67788604eb0 Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Wed, 29 Apr 2020 10:25:18 +0530
Subject: [PATCH 11/16] perf: reduce JS memory heap and maintain namespaces

---
 .../tally_migration/tally_migration.js        | 53 +++++++++++--------
 1 file changed, 31 insertions(+), 22 deletions(-)

diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js
index 9cc3c98a56..0cb4995ac9 100644
--- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js
+++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js
@@ -1,6 +1,8 @@
 // Copyright (c) 2019, Frappe Technologies Pvt. Ltd. and contributors
 // For license information, please see license.txt
 
+frappe.provide("erpnext.tally_migration");
+
 frappe.ui.form.on("Tally Migration", {
 	onload: function (frm) {
 		let reload_status = true;
@@ -38,10 +40,13 @@ frappe.ui.form.on("Tally Migration", {
 
 	refresh: function (frm) {
 		frm.trigger("show_logs_preview");
+		erpnext.tally_migration.failed_import_log = JSON.parse(frm.doc.failed_import_log);
+		erpnext.tally_migration.fixed_errors_log = JSON.parse(frm.doc.fixed_errors_log);
 
 		["default_round_off_account", "default_warehouse", "default_cost_center"].forEach(account => {
 			frm.toggle_reqd(account, frm.doc.is_master_data_imported === 1)
 		})
+
 		if (frm.doc.master_data && !frm.doc.is_master_data_imported) {
 			if (frm.doc.is_master_data_processed) {
 				if (frm.doc.status != "Importing Master Data") {
@@ -53,6 +58,7 @@ frappe.ui.form.on("Tally Migration", {
 				}
 			}
 		}
+
 		if (frm.doc.day_book_data && !frm.doc.is_day_book_data_imported) {
 			if (frm.doc.is_day_book_data_processed) {
 				if (frm.doc.status != "Importing Day Book Data") {
@@ -87,7 +93,7 @@ frappe.ui.form.on("Tally Migration", {
 			frm.toggle_display(field, false);
 			return
 		}
-		let rows = get_html_rows(shown_logs, field);
+		let rows = erpnext.tally_migration.get_html_rows(shown_logs, field);
 		let rows_head, table_caption;
 
 		let table_footer = (hidden_logs && (hidden_logs.length > 0)) ? `<tr class="text-muted">
@@ -119,7 +125,7 @@ frappe.ui.form.on("Tally Migration", {
 	},
 
 	show_error_summary(frm) {
-		let summary = JSON.parse(frm.doc.failed_import_log).reduce((summary, row) => {
+		let summary = erpnext.tally_migration.failed_import_log.reduce((summary, row) => {
 			if (row.doc) {
 				if (summary[row.doc.doctype]) {
 					summary[row.doc.doctype] += 1;
@@ -147,8 +153,7 @@ frappe.ui.form.on("Tally Migration", {
 	},
 
 	show_errored_import_log(frm) {
-		let import_log = JSON.parse(frm.doc.failed_import_log || "[]");
-
+		let import_log = erpnext.tally_migration.failed_import_log;
 		let logs = import_log.slice(0, 20);
 		let hidden_logs = import_log.slice(20);
 
@@ -156,7 +161,7 @@ frappe.ui.form.on("Tally Migration", {
 	},
 
 	show_fixed_errors_log(frm) {
-		let completed_log = JSON.parse(frm.doc.fixed_errors_log || "[]");
+		let completed_log = erpnext.tally_migration.fixed_errors_log;
 		let logs = completed_log.slice(0, 20);
 		let hidden_logs = completed_log.slice(20);
 
@@ -164,7 +169,7 @@ frappe.ui.form.on("Tally Migration", {
 	}
 });
 
-const getError = (traceback) => {
+erpnext.tally_migration.getError = (traceback) => {
 	/* Extracts the Error Message from the Python Traceback or Solved error */
 	let is_multiline = traceback.trim().indexOf("\n") != -1;
 	let message;
@@ -181,28 +186,28 @@ const getError = (traceback) => {
 	return message
 }
 
-const cleanDoc = (obj) => {
+erpnext.tally_migration.cleanDoc = (obj) => {
 	/* Strips all null and empty values of your JSON object */
 	let temp = obj;
 	$.each(temp, function(key, value){
 		if (value === "" || value === null){
 			delete obj[key];
 		} else if (Object.prototype.toString.call(value) === '[object Object]') {
-			cleanDoc(value);
+			erpnext.tally_migration.cleanDoc(value);
 		} else if ($.isArray(value)) {
-			$.each(value, function (k,v) { cleanDoc(v); });
+			$.each(value, function (k,v) { erpnext.tally_migration.cleanDoc(v); });
 		}
 	});
 	return temp;
 }
 
-window.unresolve = (document) => {
+erpnext.tally_migration.unresolve = (document) => {
 	let frm = cur_frm;
-	let failed_log = JSON.parse(frm.doc.failed_import_log);
-	let fixed_log = JSON.parse(frm.doc.fixed_errors_log);
+	let failed_log = erpnext.tally_migration.failed_import_log;
+	let fixed_log = erpnext.tally_migration.fixed_errors_log;
 
 	let modified_fixed_log = fixed_log.filter(row => {
-		if (!frappe.utils.deep_equal(cleanDoc(row.doc), document)) {
+		if (!frappe.utils.deep_equal(erpnext.tally_migration.cleanDoc(row.doc), document)) {
 			return row
 		}
 	});
@@ -217,13 +222,13 @@ window.unresolve = (document) => {
 	frm.save();
 }
 
-window.create_new_doc = (doctype, document) => {
+erpnext.tally_migration.resolve = (document) => {
 	let frm = cur_frm;
-	let failed_log = JSON.parse(frm.doc.failed_import_log);
-	let fixed_log = JSON.parse(frm.doc.fixed_errors_log);
+	let failed_log = erpnext.tally_migration.failed_import_log;
+	let fixed_log = erpnext.tally_migration.fixed_errors_log;
 
 	let modified_failed_log = failed_log.filter(row => {
-		if (!frappe.utils.deep_equal(cleanDoc(row.doc), document)) {
+		if (!frappe.utils.deep_equal(erpnext.tally_migration.cleanDoc(row.doc), document)) {
 			return row
 		}
 	});
@@ -235,17 +240,21 @@ window.create_new_doc = (doctype, document) => {
 	// frm.trigger('show_logs_preview')
 	frm.dirty();
 	frm.save();
+}
+
+erpnext.tally_migration.create_new_doc = (doctype, document) => {
+	erpnext.tally_migration.resolve(document);
 	frappe.new_doc(doctype, document);
 }
 
-const get_html_rows = (logs, field) => {
+erpnext.tally_migration.get_html_rows = (logs, field) => {
 	let index = 0;
 	let rows = logs
 		.map(({ doc, exc }) => {
 			let id = frappe.dom.get_unique_id();
 			let traceback = exc;
 
-			let error_message = getError(traceback);
+			let error_message = erpnext.tally_migration.getError(traceback);
 			index++;
 
 			let show_traceback = `
@@ -264,17 +273,17 @@ const get_html_rows = (logs, field) => {
 				</button>
 				<div class="collapse margin-top" id="${id}-doc">
 					<div class="well">
-						<pre style="font-size: smaller;">${JSON.stringify(cleanDoc(doc), null, 1)}</pre>
+						<pre style="font-size: smaller;">${JSON.stringify(erpnext.tally_migration.cleanDoc(doc), null, 1)}</pre>
 					</div>
 				</div>`;
 
 			let create_button = `
-				<button class='btn btn-default btn-xs m-3' type='button' onclick='create_new_doc("${doc.doctype}", ${JSON.stringify(doc)})'>
+				<button class='btn btn-default btn-xs m-3' type='button' onclick='erpnext.tally_migration.create_new_doc("${doc.doctype}", ${JSON.stringify(doc)})'>
 					${__("Create Document")}
 				</button>`
 
 			let mark_as_unresolved = `
-				<button class='btn btn-default btn-xs m-3' type='button' onclick='unresolve(${JSON.stringify(doc)})'>
+				<button class='btn btn-default btn-xs m-3' type='button' onclick='erpnext.tally_migration.unresolve(${JSON.stringify(doc)})'>
 					${__("Mark as unresolved")}
 				</button>`
 

From f4926a8205184e21641a1770b9096a8804f2cd64 Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Wed, 29 Apr 2020 11:21:16 +0530
Subject: [PATCH 12/16] fix: toggle read only for accounts post daybook data
 processing

style: code comments and updates
---
 .../doctype/tally_migration/tally_migration.js            | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js
index 0cb4995ac9..e94cca54c5 100644
--- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js
+++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js
@@ -45,6 +45,7 @@ frappe.ui.form.on("Tally Migration", {
 
 		["default_round_off_account", "default_warehouse", "default_cost_center"].forEach(account => {
 			frm.toggle_reqd(account, frm.doc.is_master_data_imported === 1)
+			frm.toggle_enable(account, frm.doc.is_day_book_data_processed != 1)
 		})
 
 		if (frm.doc.master_data && !frm.doc.is_master_data_imported) {
@@ -145,8 +146,8 @@ frappe.ui.form.on("Tally Migration", {
 		let render_section = !(import_log === completed_log && import_log === empty);
 
 		frm.toggle_display("import_log_section", render_section);
-		frm.trigger("show_error_summary");
 		if (render_section) {
+			frm.trigger("show_error_summary");
 			frm.trigger("show_errored_import_log");
 			frm.trigger("show_fixed_errors_log");
 		}
@@ -202,6 +203,7 @@ erpnext.tally_migration.cleanDoc = (obj) => {
 }
 
 erpnext.tally_migration.unresolve = (document) => {
+	/* Mark document migration as unresolved ie. move to failed error log */
 	let frm = cur_frm;
 	let failed_log = erpnext.tally_migration.failed_import_log;
 	let fixed_log = erpnext.tally_migration.fixed_errors_log;
@@ -217,12 +219,12 @@ erpnext.tally_migration.unresolve = (document) => {
 	frm.doc.failed_import_log = JSON.stringify(failed_log);
 	frm.doc.fixed_errors_log = JSON.stringify(modified_fixed_log);
 
-	// frm.trigger('show_logs_preview')
 	frm.dirty();
 	frm.save();
 }
 
 erpnext.tally_migration.resolve = (document) => {
+	/* Mark document migration as resolved ie. move to fixed error log */
 	let frm = cur_frm;
 	let failed_log = erpnext.tally_migration.failed_import_log;
 	let fixed_log = erpnext.tally_migration.fixed_errors_log;
@@ -237,12 +239,12 @@ erpnext.tally_migration.resolve = (document) => {
 	frm.doc.failed_import_log = JSON.stringify(modified_failed_log);
 	frm.doc.fixed_errors_log = JSON.stringify(fixed_log);
 
-	// frm.trigger('show_logs_preview')
 	frm.dirty();
 	frm.save();
 }
 
 erpnext.tally_migration.create_new_doc = (doctype, document) => {
+	/* Mark as resolved and create new document */
 	erpnext.tally_migration.resolve(document);
 	frappe.new_doc(doctype, document);
 }

From 51305a028be5f7a766dd099f3d93fbea0c28d93d Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Thu, 30 Apr 2020 04:07:14 +0530
Subject: [PATCH 13/16] feat: prompt that company and COA will be overwritten

---
 .../doctype/tally_migration/tally_migration.js         | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js
index e94cca54c5..d9811b58dd 100644
--- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js
+++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js
@@ -73,6 +73,16 @@ frappe.ui.form.on("Tally Migration", {
 		}
 	},
 
+	erpnext_company: function (frm) {
+		frappe.db.exists("Company", frm.doc.erpnext_company).then(exists => {
+			if (exists) {
+				frappe.msgprint(
+					__("Company {0} already exists. Continuing will overwrite the Company and Chart of Accounts", [frm.doc.erpnext_company]),
+				);
+			}
+		});
+	},
+
 	add_button: function (frm, label, method) {
 		frm.add_custom_button(
 			label,

From af612ddb6de0c59381d36f1a1212188401990f05 Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Fri, 1 May 2020 14:39:11 +0530
Subject: [PATCH 14/16] feat: maintain tally voucher numbers

feat: create seperate customer and supplier entries for same party if
case exists
---
 .../tally_migration/tally_migration.py        | 34 ++++++++++++++-----
 1 file changed, 26 insertions(+), 8 deletions(-)

diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py
index cbc6019c6b..393c5d4b02 100644
--- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py
+++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py
@@ -145,14 +145,18 @@ class TallyMigration(Document):
 		def remove_parties(parents, children, group_set):
 			customers, suppliers = set(), set()
 			for account in parents:
+				found = False
 				if self.tally_creditors_account in parents[account]:
-					children.pop(account, None)
+					found = True
 					if account not in group_set:
 						suppliers.add(account)
-				elif self.tally_debtors_account in parents[account]:
-					children.pop(account, None)
+				if self.tally_debtors_account in parents[account]:
+					found = True
 					if account not in group_set:
 						customers.add(account)
+				if found:
+					children.pop(account, None)
+
 			return children, customers, suppliers
 
 		def traverse(tree, children, accounts, roots, group_set):
@@ -170,6 +174,7 @@ class TallyMigration(Document):
 			parties, addresses = [], []
 			for account in collection.find_all("LEDGER"):
 				party_type = None
+				links = []
 				if account.NAME.string.strip() in customers:
 					party_type = "Customer"
 					parties.append({
@@ -180,7 +185,9 @@ class TallyMigration(Document):
 						"territory": "All Territories",
 						"customer_type": "Individual",
 					})
-				elif account.NAME.string.strip() in suppliers:
+					links.append({"link_doctype": party_type, "link_name": account["NAME"]})
+
+				if account.NAME.string.strip() in suppliers:
 					party_type = "Supplier"
 					parties.append({
 						"doctype": party_type,
@@ -189,6 +196,8 @@ class TallyMigration(Document):
 						"supplier_group": "All Supplier Groups",
 						"supplier_type": "Individual",
 					})
+					links.append({"link_doctype": party_type, "link_name": account["NAME"]})
+
 				if party_type:
 					address = "\n".join([a.string.strip() for a in account.find_all("ADDRESS")])
 					addresses.append({
@@ -202,7 +211,7 @@ class TallyMigration(Document):
 						"mobile": account.LEDGERPHONE.string.strip() if account.LEDGERPHONE else None,
 						"phone": account.LEDGERPHONE.string.strip() if account.LEDGERPHONE else None,
 						"gstin": account.PARTYGSTIN.string.strip() if account.PARTYGSTIN else None,
-						"links": [{"link_doctype": party_type, "link_name": account["NAME"]}],
+						"links": links
 					})
 			return parties, addresses
 
@@ -378,6 +387,7 @@ class TallyMigration(Document):
 			journal_entry = {
 				"doctype": "Journal Entry",
 				"tally_guid": voucher.GUID.string.strip(),
+				"tally_voucher_no": voucher.VOUCHERNUMBER.string.strip() if voucher.VOUCHERNUMBER else "",
 				"posting_date": voucher.DATE.string.strip(),
 				"company": self.erpnext_company,
 				"accounts": accounts,
@@ -406,6 +416,7 @@ class TallyMigration(Document):
 				"doctype": doctype,
 				party_field: voucher.PARTYNAME.string.strip(),
 				"tally_guid": voucher.GUID.string.strip(),
+				"tally_voucher_no": voucher.VOUCHERNUMBER.string.strip() if voucher.VOUCHERNUMBER else "",
 				"posting_date": voucher.DATE.string.strip(),
 				"due_date": voucher.DATE.string.strip(),
 				"items": get_voucher_items(voucher, doctype),
@@ -497,14 +508,21 @@ class TallyMigration(Document):
 				oldest_year = new_year
 
 		def create_custom_fields(doctypes):
-			df = {
+			tally_guid_df = {
 				"fieldtype": "Data",
 				"fieldname": "tally_guid",
 				"read_only": 1,
 				"label": "Tally GUID"
 			}
-			for doctype in doctypes:
-				create_custom_field(doctype, df)
+			tally_voucher_no_df = {
+				"fieldtype": "Data",
+				"fieldname": "tally_voucher_no",
+				"read_only": 1,
+				"label": "Tally Voucher Number"
+			}
+			for df in [tally_guid_df, tally_voucher_no_df]:
+				for doctype in doctypes:
+					create_custom_field(doctype, df)
 
 		def create_price_list():
 			frappe.get_doc({

From 0c353a64a44fb5567e1de315ca0fef0092c0aa68 Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Sat, 2 May 2020 18:58:33 +0530
Subject: [PATCH 15/16] fix: un-using buggy JS frappe.new_doc

---
 .../tally_migration/tally_migration.js        | 20 ++++++++++++++++---
 .../tally_migration/tally_migration.py        | 10 ++++++++++
 2 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js
index d9811b58dd..fd16d1e84a 100644
--- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js
+++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.js
@@ -253,10 +253,24 @@ erpnext.tally_migration.resolve = (document) => {
 	frm.save();
 }
 
-erpnext.tally_migration.create_new_doc = (doctype, document) => {
+erpnext.tally_migration.create_new_doc = (document) => {
 	/* Mark as resolved and create new document */
 	erpnext.tally_migration.resolve(document);
-	frappe.new_doc(doctype, document);
+	return frappe.call({
+		type: "POST",
+		method: 'erpnext.erpnext_integrations.doctype.tally_migration.tally_migration.new_doc',
+		args: {
+			document
+		},
+		freeze: true,
+		callback: function(r) {
+			if(!r.exc) {
+				frappe.model.sync(r.message);
+				frappe.get_doc(r.message.doctype, r.message.name).__run_link_triggers = true;
+				frappe.set_route("Form", r.message.doctype, r.message.name);
+			}
+		}
+	});
 }
 
 erpnext.tally_migration.get_html_rows = (logs, field) => {
@@ -290,7 +304,7 @@ erpnext.tally_migration.get_html_rows = (logs, field) => {
 				</div>`;
 
 			let create_button = `
-				<button class='btn btn-default btn-xs m-3' type='button' onclick='erpnext.tally_migration.create_new_doc("${doc.doctype}", ${JSON.stringify(doc)})'>
+				<button class='btn btn-default btn-xs m-3' type='button' onclick='erpnext.tally_migration.create_new_doc(${JSON.stringify(doc)})'>
 					${__("Create Document")}
 				</button>`
 
diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py
index 393c5d4b02..d9c5852a6e 100644
--- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py
+++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py
@@ -28,6 +28,16 @@ PRIMARY_ACCOUNT = "Primary"
 VOUCHER_CHUNK_SIZE = 500
 
 
+@frappe.whitelist()
+def new_doc(document):
+	document = json.loads(document)
+	doctype = document.pop("doctype")
+	document.pop("name", None)
+	doc = frappe.new_doc(doctype)
+	doc.update(document)
+
+	return doc
+
 class TallyMigration(Document):
 	def validate(self):
 		failed_import_log = json.loads(self.failed_import_log)

From cb1376b036edb7d3161c331cdf4f7451a2d97a3e Mon Sep 17 00:00:00 2001
From: Gavin D'souza <gavin18d@gmail.com>
Date: Sat, 9 May 2020 12:05:33 +0530
Subject: [PATCH 16/16] chore: verbose error message for coa recursion

---
 .../doctype/tally_migration/tally_migration.py                  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py
index d9c5852a6e..462685f5e7 100644
--- a/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py
+++ b/erpnext/erpnext_integrations/doctype/tally_migration/tally_migration.py
@@ -113,7 +113,7 @@ class TallyMigration(Document):
 			try:
 				coa = traverse({}, children, roots, roots, group_set)
 			except RecursionError:
-				self.log()
+				self.log(_("Error occured while parsing Chart of Accounts: Please make sure that no two accounts have the same name"))
 
 			for account in coa:
 				coa[account]["root_type"] = root_type_map[account]