Merge pull request #38496 from ruthra-kumar/bisect_accounting_reports

feat: utility to debug financial reports

erpnext/accounts/doctype/bisect_accounting_statements/bisect_accounting_statements.js

@@ -0,0 +1,100 @@
+// Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on("Bisect Accounting Statements", {
+	onload(frm) {
+		frm.trigger("render_heatmap");
+	},
+	refresh(frm) {
+		frm.add_custom_button(__('Bisect Left'), () => {
+			frm.trigger("bisect_left");
+		});
+
+		frm.add_custom_button(__('Bisect Right'), () => {
+			frm.trigger("bisect_right");
+		});
+
+		frm.add_custom_button(__('Up'), () => {
+			frm.trigger("move_up");
+		});
+		frm.add_custom_button(__('Build Tree'), () => {
+			frm.trigger("build_tree");
+		});
+	},
+	render_heatmap(frm) {
+		let bisect_heatmap = frm.get_field("bisect_heatmap").$wrapper;
+		bisect_heatmap.addClass("bisect_heatmap_location");
+
+		// milliseconds in a day
+		let msiad=24*60*60*1000;
+		let datapoints = {};
+		let fr_dt = new Date(frm.doc.from_date).getTime();
+		let to_dt = new Date(frm.doc.to_date).getTime();
+		let bisect_start = new Date(frm.doc.current_from_date).getTime();
+		let bisect_end = new Date(frm.doc.current_to_date).getTime();
+
+		for(let x=fr_dt; x <= to_dt; x+=msiad){
+			let epoch_in_seconds = x/1000;
+			if ((bisect_start <= x) && (x <= bisect_end )) {
+				datapoints[epoch_in_seconds] = 1.0;
+			} else {
+				datapoints[epoch_in_seconds] = 0.0;
+			}
+		}
+
+		new frappe.Chart(".bisect_heatmap_location", {
+			type: "heatmap",
+			data: {
+				dataPoints: datapoints,
+				start: new Date(frm.doc.from_date),
+				end: new Date(frm.doc.to_date),
+			},
+			countLabel: 'Bisecting',
+			discreteDomains: 1,
+		});
+	},
+	bisect_left(frm) {
+		frm.call({
+			doc: frm.doc,
+			method: 'bisect_left',
+			freeze: true,
+			freeze_message: __("Bisecting Left ..."),
+			callback: (r) => {
+				frm.trigger("render_heatmap");
+			}
+		});
+	},
+	bisect_right(frm) {
+		frm.call({
+			doc: frm.doc,
+			freeze: true,
+			freeze_message: __("Bisecting Right ..."),
+			method: 'bisect_right',
+			callback: (r) => {
+				frm.trigger("render_heatmap");
+			}
+		});
+	},
+	move_up(frm) {
+		frm.call({
+			doc: frm.doc,
+			freeze: true,
+			freeze_message: __("Moving up in tree ..."),
+			method: 'move_up',
+			callback: (r) => {
+				frm.trigger("render_heatmap");
+			}
+		});
+	},
+	build_tree(frm) {
+		frm.call({
+			doc: frm.doc,
+			freeze: true,
+			freeze_message: __("Rebuilding BTree for period ..."),
+			method: 'build_tree',
+			callback: (r) => {
+				frm.trigger("render_heatmap");
+			}
+		});
+	},
+});

erpnext/accounts/doctype/bisect_accounting_statements/bisect_accounting_statements.json

@@ -0,0 +1,194 @@
+{
+ "actions": [],
+ "allow_rename": 1,
+ "creation": "2023-09-15 21:28:28.054773",
+ "default_view": "List",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "section_break_cvfg",
+  "company",
+  "column_break_hcam",
+  "from_date",
+  "column_break_qxbi",
+  "to_date",
+  "column_break_iwny",
+  "algorithm",
+  "section_break_8ph9",
+  "current_node",
+  "section_break_ngid",
+  "bisect_heatmap",
+  "section_break_hmsy",
+  "bisecting_from",
+  "current_from_date",
+  "column_break_uqyd",
+  "bisecting_to",
+  "current_to_date",
+  "section_break_hbyo",
+  "heading_cppb",
+  "p_l_summary",
+  "column_break_aivo",
+  "balance_sheet_summary",
+  "b_s_summary",
+  "column_break_gvwx",
+  "difference_heading",
+  "difference"
+ ],
+ "fields": [
+  {
+   "fieldname": "column_break_qxbi",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "from_date",
+   "fieldtype": "Datetime",
+   "label": "From Date"
+  },
+  {
+   "fieldname": "to_date",
+   "fieldtype": "Datetime",
+   "label": "To Date"
+  },
+  {
+   "default": "BFS",
+   "fieldname": "algorithm",
+   "fieldtype": "Select",
+   "label": "Algorithm",
+   "options": "BFS\nDFS"
+  },
+  {
+   "fieldname": "column_break_iwny",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "current_node",
+   "fieldtype": "Link",
+   "label": "Current Node",
+   "options": "Bisect Nodes"
+  },
+  {
+   "fieldname": "section_break_hmsy",
+   "fieldtype": "Section Break"
+  },
+  {
+   "fieldname": "current_from_date",
+   "fieldtype": "Datetime",
+   "read_only": 1
+  },
+  {
+   "fieldname": "current_to_date",
+   "fieldtype": "Datetime",
+   "read_only": 1
+  },
+  {
+   "fieldname": "column_break_uqyd",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "section_break_hbyo",
+   "fieldtype": "Section Break"
+  },
+  {
+   "fieldname": "p_l_summary",
+   "fieldtype": "Float",
+   "read_only": 1
+  },
+  {
+   "fieldname": "b_s_summary",
+   "fieldtype": "Float",
+   "read_only": 1
+  },
+  {
+   "fieldname": "difference",
+   "fieldtype": "Float",
+   "read_only": 1
+  },
+  {
+   "fieldname": "column_break_aivo",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "column_break_gvwx",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "company",
+   "fieldtype": "Link",
+   "label": "Company",
+   "options": "Company"
+  },
+  {
+   "fieldname": "column_break_hcam",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "section_break_ngid",
+   "fieldtype": "Section Break"
+  },
+  {
+   "fieldname": "section_break_8ph9",
+   "fieldtype": "Section Break",
+   "hidden": 1
+  },
+  {
+   "fieldname": "bisect_heatmap",
+   "fieldtype": "HTML",
+   "label": "Heatmap"
+  },
+  {
+   "fieldname": "heading_cppb",
+   "fieldtype": "Heading",
+   "label": "Profit and Loss Summary"
+  },
+  {
+   "fieldname": "balance_sheet_summary",
+   "fieldtype": "Heading",
+   "label": "Balance Sheet Summary"
+  },
+  {
+   "fieldname": "difference_heading",
+   "fieldtype": "Heading",
+   "label": "Difference"
+  },
+  {
+   "fieldname": "bisecting_from",
+   "fieldtype": "Heading",
+   "label": "Bisecting From"
+  },
+  {
+   "fieldname": "bisecting_to",
+   "fieldtype": "Heading",
+   "label": "Bisecting To"
+  },
+  {
+   "fieldname": "section_break_cvfg",
+   "fieldtype": "Section Break"
+  }
+ ],
+ "hide_toolbar": 1,
+ "index_web_pages_for_search": 1,
+ "issingle": 1,
+ "links": [],
+ "modified": "2023-12-01 16:49:54.073890",
+ "modified_by": "Administrator",
+ "module": "Accounts",
+ "name": "Bisect Accounting Statements",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "print": 1,
+   "read": 1,
+   "role": "Administrator",
+   "share": 1,
+   "write": 1
+  }
+ ],
+ "read_only": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": []
+}

erpnext/accounts/doctype/bisect_accounting_statements/bisect_accounting_statements.py

@@ -0,0 +1,226 @@
+# Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+import datetime
+from collections import deque
+from math import floor
+
+import frappe
+from dateutil.relativedelta import relativedelta
+from frappe import _
+from frappe.model.document import Document
+from frappe.utils import getdate
+from frappe.utils.data import guess_date_format
+
+
+class BisectAccountingStatements(Document):
+	# begin: auto-generated types
+	# This code is auto-generated. Do not modify anything in this block.
+
+	from typing import TYPE_CHECKING
+
+	if TYPE_CHECKING:
+		from frappe.types import DF
+
+		algorithm: DF.Literal["BFS", "DFS"]
+		b_s_summary: DF.Float
+		company: DF.Link | None
+		current_from_date: DF.Datetime | None
+		current_node: DF.Link | None
+		current_to_date: DF.Datetime | None
+		difference: DF.Float
+		from_date: DF.Datetime | None
+		p_l_summary: DF.Float
+		to_date: DF.Datetime | None
+	# end: auto-generated types
+
+	def validate(self):
+		self.validate_dates()
+
+	def validate_dates(self):
+		if getdate(self.from_date) > getdate(self.to_date):
+			frappe.throw(
+				_("From Date: {0} cannot be greater than To date: {1}").format(
+					frappe.bold(self.from_date), frappe.bold(self.to_date)
+				)
+			)
+
+	def bfs(self, from_date: datetime, to_date: datetime):
+		# Make Root node
+		node = frappe.new_doc("Bisect Nodes")
+		node.root = None
+		node.period_from_date = from_date
+		node.period_to_date = to_date
+		node.insert()
+
+		period_queue = deque([node])
+		while period_queue:
+			cur_node = period_queue.popleft()
+			delta = cur_node.period_to_date - cur_node.period_from_date
+			if delta.days == 0:
+				continue
+			else:
+				cur_floor = floor(delta.days / 2)
+				next_to_date = cur_node.period_from_date + relativedelta(days=+cur_floor)
+				left_node = frappe.new_doc("Bisect Nodes")
+				left_node.period_from_date = cur_node.period_from_date
+				left_node.period_to_date = next_to_date
+				left_node.root = cur_node.name
+				left_node.generated = False
+				left_node.insert()
+				cur_node.left_child = left_node.name
+				period_queue.append(left_node)
+
+				next_from_date = cur_node.period_from_date + relativedelta(days=+(cur_floor + 1))
+				right_node = frappe.new_doc("Bisect Nodes")
+				right_node.period_from_date = next_from_date
+				right_node.period_to_date = cur_node.period_to_date
+				right_node.root = cur_node.name
+				right_node.generated = False
+				right_node.insert()
+				cur_node.right_child = right_node.name
+				period_queue.append(right_node)
+
+				cur_node.save()
+
+	def dfs(self, from_date: datetime, to_date: datetime):
+		# Make Root node
+		node = frappe.new_doc("Bisect Nodes")
+		node.root = None
+		node.period_from_date = from_date
+		node.period_to_date = to_date
+		node.insert()
+
+		period_stack = [node]
+		while period_stack:
+			cur_node = period_stack.pop()
+			delta = cur_node.period_to_date - cur_node.period_from_date
+			if delta.days == 0:
+				continue
+			else:
+				cur_floor = floor(delta.days / 2)
+				next_to_date = cur_node.period_from_date + relativedelta(days=+cur_floor)
+				left_node = frappe.new_doc("Bisect Nodes")
+				left_node.period_from_date = cur_node.period_from_date
+				left_node.period_to_date = next_to_date
+				left_node.root = cur_node.name
+				left_node.generated = False
+				left_node.insert()
+				cur_node.left_child = left_node.name
+				period_stack.append(left_node)
+
+				next_from_date = cur_node.period_from_date + relativedelta(days=+(cur_floor + 1))
+				right_node = frappe.new_doc("Bisect Nodes")
+				right_node.period_from_date = next_from_date
+				right_node.period_to_date = cur_node.period_to_date
+				right_node.root = cur_node.name
+				right_node.generated = False
+				right_node.insert()
+				cur_node.right_child = right_node.name
+				period_stack.append(right_node)
+
+				cur_node.save()
+
+	@frappe.whitelist()
+	def build_tree(self):
+		frappe.db.delete("Bisect Nodes")
+
+		# Convert str to datetime format
+		dt_format = guess_date_format(self.from_date)
+		from_date = datetime.datetime.strptime(self.from_date, dt_format)
+		to_date = datetime.datetime.strptime(self.to_date, dt_format)
+
+		if self.algorithm == "BFS":
+			self.bfs(from_date, to_date)
+
+		if self.algorithm == "DFS":
+			self.dfs(from_date, to_date)
+
+		# set root as current node
+		root = frappe.db.get_all("Bisect Nodes", filters={"root": ["is", "not set"]})[0]
+		self.get_report_summary()
+		self.current_node = root.name
+		self.current_from_date = self.from_date
+		self.current_to_date = self.to_date
+		self.save()
+
+	def get_report_summary(self):
+		filters = {
+			"company": self.company,
+			"filter_based_on": "Date Range",
+			"period_start_date": self.current_from_date,
+			"period_end_date": self.current_to_date,
+			"periodicity": "Yearly",
+		}
+		pl_summary = frappe.get_doc("Report", "Profit and Loss Statement")
+		self.p_l_summary = pl_summary.execute_script_report(filters=filters)[5]
+		bs_summary = frappe.get_doc("Report", "Balance Sheet")
+		self.b_s_summary = bs_summary.execute_script_report(filters=filters)[5]
+		self.difference = abs(self.p_l_summary - self.b_s_summary)
+
+	def update_node(self):
+		current_node = frappe.get_doc("Bisect Nodes", self.current_node)
+		current_node.balance_sheet_summary = self.b_s_summary
+		current_node.profit_loss_summary = self.p_l_summary
+		current_node.difference = self.difference
+		current_node.generated = True
+		current_node.save()
+
+	def current_node_has_summary_info(self):
+		"Assertion method"
+		return frappe.db.get_value("Bisect Nodes", self.current_node, "generated")
+
+	def fetch_summary_info_from_current_node(self):
+		current_node = frappe.get_doc("Bisect Nodes", self.current_node)
+		self.p_l_summary = current_node.balance_sheet_summary
+		self.b_s_summary = current_node.profit_loss_summary
+		self.difference = abs(self.p_l_summary - self.b_s_summary)
+
+	def fetch_or_calculate(self):
+		if self.current_node_has_summary_info():
+			self.fetch_summary_info_from_current_node()
+		else:
+			self.get_report_summary()
+			self.update_node()
+
+	@frappe.whitelist()
+	def bisect_left(self):
+		if self.current_node is not None:
+			cur_node = frappe.get_doc("Bisect Nodes", self.current_node)
+			if cur_node.left_child is not None:
+				lft_node = frappe.get_doc("Bisect Nodes", cur_node.left_child)
+				self.current_node = cur_node.left_child
+				self.current_from_date = lft_node.period_from_date
+				self.current_to_date = lft_node.period_to_date
+				self.fetch_or_calculate()
+				self.save()
+			else:
+				frappe.msgprint(_("No more children on Left"))
+
+	@frappe.whitelist()
+	def bisect_right(self):
+		if self.current_node is not None:
+			cur_node = frappe.get_doc("Bisect Nodes", self.current_node)
+			if cur_node.right_child is not None:
+				rgt_node = frappe.get_doc("Bisect Nodes", cur_node.right_child)
+				self.current_node = cur_node.right_child
+				self.current_from_date = rgt_node.period_from_date
+				self.current_to_date = rgt_node.period_to_date
+				self.fetch_or_calculate()
+				self.save()
+			else:
+				frappe.msgprint(_("No more children on Right"))
+
+	@frappe.whitelist()
+	def move_up(self):
+		if self.current_node is not None:
+			cur_node = frappe.get_doc("Bisect Nodes", self.current_node)
+			if cur_node.root is not None:
+				root = frappe.get_doc("Bisect Nodes", cur_node.root)
+				self.current_node = cur_node.root
+				self.current_from_date = root.period_from_date
+				self.current_to_date = root.period_to_date
+				self.fetch_or_calculate()
+				self.save()
+			else:
+				frappe.msgprint(_("Reached Root"))

erpnext/accounts/doctype/bisect_accounting_statements/test_bisect_accounting_statements.py

@@ -0,0 +1,9 @@
+# Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and Contributors
+# See license.txt
+
+# import frappe
+from frappe.tests.utils import FrappeTestCase
+
+
+class TestBisectAccountingStatements(FrappeTestCase):
+	pass

erpnext/accounts/doctype/bisect_nodes/bisect_nodes.js

@@ -0,0 +1,8 @@
+// Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and contributors
+// For license information, please see license.txt
+
+// frappe.ui.form.on("Bisect Nodes", {
+// 	refresh(frm) {
+
+// 	},
+// });

erpnext/accounts/doctype/bisect_nodes/bisect_nodes.json

@@ -0,0 +1,97 @@
+{
+ "actions": [],
+ "autoname": "autoincrement",
+ "creation": "2023-09-27 14:56:38.112462",
+ "default_view": "List",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "root",
+  "left_child",
+  "right_child",
+  "period_from_date",
+  "period_to_date",
+  "difference",
+  "balance_sheet_summary",
+  "profit_loss_summary",
+  "generated"
+ ],
+ "fields": [
+  {
+   "fieldname": "root",
+   "fieldtype": "Link",
+   "label": "Root",
+   "options": "Bisect Nodes"
+  },
+  {
+   "fieldname": "left_child",
+   "fieldtype": "Link",
+   "label": "Left Child",
+   "options": "Bisect Nodes"
+  },
+  {
+   "fieldname": "right_child",
+   "fieldtype": "Link",
+   "label": "Right Child",
+   "options": "Bisect Nodes"
+  },
+  {
+   "fieldname": "period_from_date",
+   "fieldtype": "Datetime",
+   "label": "Period_from_date"
+  },
+  {
+   "fieldname": "period_to_date",
+   "fieldtype": "Datetime",
+   "label": "Period To Date"
+  },
+  {
+   "fieldname": "difference",
+   "fieldtype": "Float",
+   "label": "Difference"
+  },
+  {
+   "fieldname": "balance_sheet_summary",
+   "fieldtype": "Float",
+   "label": "Balance Sheet Summary"
+  },
+  {
+   "fieldname": "profit_loss_summary",
+   "fieldtype": "Float",
+   "label": "Profit and Loss Summary"
+  },
+  {
+   "default": "0",
+   "fieldname": "generated",
+   "fieldtype": "Check",
+   "label": "Generated"
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "links": [],
+ "modified": "2023-12-01 17:46:12.437996",
+ "modified_by": "Administrator",
+ "module": "Accounts",
+ "name": "Bisect Nodes",
+ "naming_rule": "Autoincrement",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "Administrator",
+   "share": 1,
+   "write": 1
+  }
+ ],
+ "read_only": 1,
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": []
+}

erpnext/accounts/doctype/bisect_nodes/bisect_nodes.py

@@ -0,0 +1,29 @@
+# Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+# import frappe
+from frappe.model.document import Document
+
+
+class BisectNodes(Document):
+	# begin: auto-generated types
+	# This code is auto-generated. Do not modify anything in this block.
+
+	from typing import TYPE_CHECKING
+
+	if TYPE_CHECKING:
+		from frappe.types import DF
+
+		balance_sheet_summary: DF.Float
+		difference: DF.Float
+		generated: DF.Check
+		left_child: DF.Link | None
+		name: DF.Int | None
+		period_from_date: DF.Datetime | None
+		period_to_date: DF.Datetime | None
+		profit_loss_summary: DF.Float
+		right_child: DF.Link | None
+		root: DF.Link | None
+	# end: auto-generated types
+
+	pass

erpnext/accounts/doctype/bisect_nodes/test_bisect_nodes.py

@@ -0,0 +1,9 @@
+# Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and Contributors
+# See license.txt
+
+# import frappe
+from frappe.tests.utils import FrappeTestCase
+
+
+class TestBisectNodes(FrappeTestCase):
+	pass

erpnext/accounts/report/balance_sheet/balance_sheet.py

@@ -97,11 +97,11 @@ def execute(filters=None):
 
 	chart = get_chart_data(filters, columns, asset, liability, equity)
 
-	report_summary = get_report_summary(
+	report_summary, primitive_summary = get_report_summary(
 		period_list, asset, liability, equity, provisional_profit_loss, currency, filters
 	)
 
-	return columns, data, message, chart, report_summary
+	return columns, data, message, chart, report_summary, primitive_summary
 
 
 def get_provisional_profit_loss(
@@ -217,7 +217,7 @@ def get_report_summary(
 			"datatype": "Currency",
 			"currency": currency,
 		},
-	]
+	], (net_asset - net_liability + net_equity)
 
 
 def get_chart_data(filters, columns, asset, liability, equity):

erpnext/accounts/report/profit_and_loss_statement/profit_and_loss_statement.py

@@ -66,11 +66,11 @@ def execute(filters=None):
 	currency = filters.presentation_currency or frappe.get_cached_value(
 		"Company", filters.company, "default_currency"
 	)
-	report_summary = get_report_summary(
+	report_summary, primitive_summary = get_report_summary(
 		period_list, filters.periodicity, income, expense, net_profit_loss, currency, filters
 	)
 
-	return columns, data, None, chart, report_summary
+	return columns, data, None, chart, report_summary, primitive_summary
 
 
 def get_report_summary(
@@ -112,7 +112,7 @@ def get_report_summary(
 			"datatype": "Currency",
 			"currency": currency,
 		},
-	]
+	], net_profit
 
 
 def get_net_profit_loss(income, expense, period_list, company, currency=None, consolidated=False):