Server side script report for sales analytics (#15589)

* Sales Analytics Report

* Codacy issue fixes and  name column addition

* Minor Fixes

* Minor Changes

* Codacy Issue Fixes

* Codacy Issue Fixes

* Bug Fixes

* Code cleaning and optimization

* Deleted Duplicate code

* Indentation Issue Fixes

* Added Supplier Condition

* Cleaned code and better function naming

* Added report link for sales analytics in selling.py

* fix(patch): Patch to delete old analytics reports

* feat(refactor): Created class and refactored code using object oriented paradigm

* Column condition fix

* Minor condition fix

* Minor fix

* parent child map for purchase analytics

* Minor Fixes in get_periodic_data

* Used dots for filters instead of brackets

* Minor Bug fix in get_period_date_ranges

* Test Cases for Analytics Report

erpnext/config/selling.py

@@ -185,10 +185,10 @@ def get_data():
 			"icon": "fa fa-table",
 			"items": [
 				{
-					"type": "page",
+					"type": "report",
-					"name": "sales-analytics",
+					"is_query_report": True,
-					"label": _("Sales Analytics"),
+					"name": "Sales Analytics",
-					"icon": "fa fa-bar-chart",
+					"doctype": "Sales Order"
 				},
 				{
 					"type": "page",

erpnext/patches.txt

@@ -568,6 +568,10 @@ erpnext.patches.v11_0.remove_land_unit_icon
 erpnext.patches.v11_0.add_default_dispatch_notification_template
 erpnext.patches.v11_0.add_market_segments
 erpnext.patches.v11_0.add_sales_stages
+execute:frappe.delete_doc("Page", "Sales Analytics")
+execute:frappe.delete_doc("Page", "Purchase Analytics")
+execute:frappe.delete_doc("Page", "Stock Analytics")
+execute:frappe.delete_doc("Page", "Production Analytics")
 erpnext.patches.v11_0.ewaybill_fields_gst_india
 erpnext.patches.v11_0.drop_column_max_days_allowed
 erpnext.patches.v11_0.change_healthcare_desktop_icons

erpnext/selling/report/sales_analytics/sales_analytics.js

@@ -0,0 +1,129 @@
+// Copyright (c) 2016, Frappe Technologies Pvt. Ltd. and contributors
+// For license information, please see license.txt
+/* eslint-disable */
+
+frappe.query_reports["Sales Analytics"] = {
+	"filters": [
+		{
+			fieldname: "tree_type",
+			label: __("Tree Type"),
+			fieldtype: "Select",
+			options: ["Customer Group","Customer","Item Group","Item","Territory"],
+			default: "Customer",
+			reqd: 1
+		},
+		{
+			fieldname: "doc_type",
+			label: __("based_on"),
+			fieldtype: "Select",
+			options: ["Sales Order","Delivery Note","Sales Invoice"],
+			default: "Sales Invoice",
+			reqd: 1
+		},
+		{
+			fieldname: "value_quantity",
+			label: __("Value Or Qty"),
+			fieldtype: "Select",
+			options: [
+				{ "value": "Value", "label": __("Value") },
+				{ "value": "Quantity", "label": __("Quantity") },
+			],
+			default: "Value",
+			reqd: 1
+		},
+		{
+			fieldname: "from_date",
+			label: __("From Date"),
+			fieldtype: "Date",
+			default: frappe.defaults.get_user_default("year_start_date"),
+			reqd: 1
+		},
+		{
+			fieldname:"to_date",
+			label: __("To Date"),
+			fieldtype: "Date",
+			default: frappe.defaults.get_user_default("year_end_date"),
+			reqd: 1
+		},
+		{
+			fieldname: "company",
+			label: __("Company"),
+			fieldtype: "Link",
+			options: "Company",
+			default: frappe.defaults.get_user_default("Company"),
+			reqd: 1
+		},
+		{
+			fieldname: "range",
+			label: __("Range"),
+			fieldtype: "Select",
+			options: [
+				{ "value": "Weekly", "label": __("Weekly") },
+				{ "value": "Monthly", "label": __("Monthly") },
+				{ "value": "Quarterly", "label": __("Quarterly") },
+				{ "value": "Yearly", "label": __("Yearly") }
+			],
+			default: "Monthly",
+			reqd: 1
+		}
+	],
+	"formatter": function(value, row, column, data) {
+		if(!value){
+			value = 0
+		}
+		return value;
+	},
+	get_datatable_options(options) {
+		return Object.assign(options, {
+			checkboxColumn: true,
+			events: {
+				onCheckRow: function(data) {
+					row_name = data[2].content;
+					row_values = data.slice(5).map(function (column) {
+						return column.content;
+					})
+
+					entry  = {
+						'name':row_name,
+						'values':row_values
+					}
+
+					let raw_data = frappe.query_report.chart.data;
+					let new_datasets = raw_data.datasets;
+
+					var found = false;
+
+					for(var i=0; i < new_datasets.length;i++){
+						if(new_datasets[i].name == row_name){
+							found = true;
+							new_datasets.splice(i,1);
+							break;
+						}
+					}
+
+					if(!found){
+						new_datasets.push(entry);
+					}
+
+					let new_data = {
+						labels: raw_data.labels,
+						datasets: new_datasets
+					}
+					
+					setTimeout(() => {
+						frappe.query_report.chart.update(new_data)
+					},200)
+					
+					
+					setTimeout(() => {
+						frappe.query_report.chart.draw(true);
+					}, 800)
+
+					frappe.query_report.raw_chart_data = new_data;
+				},
+			}
+		})
+	},
+}
+
+

erpnext/selling/report/sales_analytics/sales_analytics.json

@@ -0,0 +1,32 @@
+{
+ "add_total_row": 0, 
+ "creation": "2018-09-21 12:46:29.451048", 
+ "disabled": 0, 
+ "docstatus": 0, 
+ "doctype": "Report", 
+ "idx": 0, 
+ "is_standard": "Yes", 
+ "modified": "2018-09-21 12:46:29.451048", 
+ "modified_by": "Administrator", 
+ "module": "Selling", 
+ "name": "Sales Analytics", 
+ "owner": "Administrator", 
+ "prepared_report": 0, 
+ "ref_doctype": "Sales Order", 
+ "report_name": "Sales Analytics", 
+ "report_type": "Script Report", 
+ "roles": [
+  {
+   "role": "Stock User"
+  }, 
+  {
+   "role": "Maintenance User"
+  }, 
+  {
+   "role": "Accounts User"
+  }, 
+  {
+   "role": "Sales Manager"
+  }
+ ]
+}

erpnext/selling/report/sales_analytics/sales_analytics.py

@@ -0,0 +1,286 @@
+# Copyright (c) 2013, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+from __future__ import unicode_literals
+import frappe
+from frappe import _, scrub
+from frappe.utils import getdate, flt
+from six import iteritems
+from erpnext.accounts.utils import get_fiscal_year
+
+def execute(filters=None):
+	return Analytics(filters).run()
+
+class Analytics(object):
+	def __init__(self, filters=None):
+		self.filters = frappe._dict(filters or {})
+		self.date_field = 'transaction_date' \
+			if self.filters.doc_type in ['Sales Order', 'Purchase Order'] else 'posting_date'
+		self.months = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]
+		self.get_period_date_ranges()
+
+	def run(self):
+		self.get_columns()
+		self.get_data()
+		self.get_chart_data()
+		return self.columns, self.data , None, self.chart
+
+	def get_columns(self):
+		self.columns =[{
+				"label": _(self.filters.tree_type + " ID"),
+				"options": self.filters.tree_type,
+				"fieldname": "entity",
+				"fieldtype": "Link",
+				"width": 140
+			}]
+		if self.filters.tree_type in ["Customer", "Supplier", "Item"]:
+			self.columns.append({
+				"label": _(self.filters.tree_type + " Name"),
+				"fieldname": "entity_name",
+				"fieldtype": "Data",
+				"width": 140
+			})
+		for dummy, end_date in self.periodic_daterange:
+			period = self.get_period(end_date)
+			self.columns.append({
+				"label": _(period),
+				"fieldname": scrub(period),
+				"fieldtype": "Float",
+				"width": 120
+			})
+
+		self.columns.append({
+			"label": _("Total"),
+			"fieldname": "total",
+			"fieldtype": "Float",
+			"width": 120
+		})
+
+	def get_data(self):
+		if self.filters.tree_type in ["Customer", "Supplier"]:
+			self.get_sales_transactions_based_on_customers_or_suppliers()
+			self.get_rows()
+
+		elif self.filters.tree_type == 'Item':
+			self.get_sales_transactions_based_on_items()
+			self.get_rows()
+
+		elif self.filters.tree_type in ["Customer Group", "Supplier Group", "Territory"]:
+			self.get_sales_transactions_based_on_customer_or_territory_group()
+			self.get_rows_by_group()
+
+		elif self.filters.tree_type == 'Item Group':
+			self.get_sales_transactions_based_on_item_group()
+			self.get_rows_by_group()
+
+	def get_sales_transactions_based_on_customers_or_suppliers(self):
+		if self.filters["value_quantity"] == 'Value':
+			value_field = "base_net_total as value_field"
+		else:
+			value_field = "total_qty as value_field"
+
+		if self.filters.tree_type == 'Customer':
+			entity = "customer as entity"
+			entity_name = "customer_name as entity_name"
+		else:
+			entity = "supplier as entity"
+			entity_name = "supplier_name as entity_name"
+
+		self.entries = frappe.get_all(self.filters.doc_type,
+			fields=[entity, entity_name, value_field, self.date_field],
+			filters = {
+				"docstatus": 1,
+				"company": self.filters.company,
+				self.date_field: ('between', [self.filters.from_date, self.filters.to_date])
+			}
+		)
+
+		self.entity_names = {}
+		for d in self.entries:
+			self.entity_names.setdefault(d.entity, d.entity_name)
+
+	def get_sales_transactions_based_on_items(self):
+
+		if self.filters["value_quantity"] == 'Value':
+			value_field = 'base_amount'
+		else:
+			value_field = 'qty'
+
+		self.entries = frappe.db.sql("""
+			select i.item_code as entity, i.item_name as entity_name, i.{value_field} as value_field, s.{date_field}
+			from `tab{doctype} Item` i , `tab{doctype}` s
+			where s.name = i.parent and i.docstatus = 1 and s.company = %s
+			and s.{date_field} between %s and %s
+		"""
+		.format(date_field=self.date_field, value_field = value_field, doctype=self.filters.doc_type),
+		(self.filters.company, self.filters.from_date, self.filters.to_date), as_dict=1)
+
+		self.entity_names = {}
+		for d in self.entries:
+			self.entity_names.setdefault(d.entity, d.entity_name)
+
+	def get_sales_transactions_based_on_customer_or_territory_group(self):
+		if self.filters["value_quantity"] == 'Value':
+			value_field = "base_net_total as value_field"
+		else:
+			value_field = "total_qty as value_field"
+
+		if self.filters.tree_type == 'Customer Group':
+			entity_field = 'customer_group as entity'
+		elif self.filters.tree_type == 'Supplier Group':
+			entity_field = "supplier as entity"
+			self.get_supplier_parent_child_map()
+		else:
+			entity_field = "territory as entity"
+
+		self.entries = frappe.get_all(self.filters.doc_type,
+			fields=[entity_field, value_field, self.date_field],
+			filters = {
+				"docstatus": 1,
+				"company": self.filters.company,
+				self.date_field: ('between', [self.filters.from_date, self.filters.to_date])
+			}
+		)
+		self.get_groups()
+
+	def get_sales_transactions_based_on_item_group(self):
+		if self.filters["value_quantity"] == 'Value':
+			value_field = "base_amount"
+		else:
+			value_field = "qty"
+
+		self.entries = frappe.db.sql("""
+			select i.item_group as entity, i.{value_field} as value_field, s.{date_field}
+			from `tab{doctype} Item` i , `tab{doctype}` s
+			where s.name = i.parent and i.docstatus = 1 and s.company = %s
+			and s.{date_field} between %s and %s
+		""".format(date_field=self.date_field, value_field = value_field, doctype=self.filters.doc_type),
+		(self.filters.company, self.filters.from_date, self.filters.to_date), as_dict=1)
+
+		self.get_groups()
+
+	def get_rows(self):
+		self.data=[]
+		self.get_periodic_data()
+
+		for entity, period_data in iteritems(self.entity_periodic_data):
+			row = {
+				"entity": entity,
+				"entity_name": self.entity_names.get(entity) 
+			}
+			total = 0
+			for dummy, end_date in self.periodic_daterange:
+				period = self.get_period(end_date)
+				amount = flt(period_data.get(period, 0.0))
+				row[scrub(period)] = amount
+				total += amount
+
+			row["total"] = total
+			self.data.append(row)
+	
+	def get_rows_by_group(self):
+		self.get_periodic_data()
+		out = []
+
+		for d in reversed(self.group_entries):
+			row = {
+				"entity": d.name,
+				"indent": self.depth_map.get(d.name) 
+			}
+			total = 0
+			for dummy, end_date in self.periodic_daterange:
+				period = self.get_period(end_date)
+				amount = flt(self.entity_periodic_data.get(d.name, {}).get(period, 0.0))
+				row[scrub(period)] = amount
+				if d.parent:
+					self.entity_periodic_data.setdefault(d.parent, frappe._dict()).setdefault(period, 0.0)
+					self.entity_periodic_data[d.parent][period] += amount
+				total += amount
+			row["total"] = total
+			out = [row] + out
+		self.data = out
+
+	def get_periodic_data(self):
+		self.entity_periodic_data = frappe._dict()
+
+		for d in self.entries:
+			if self.filters.tree_type == "Supplier Group":
+				d.entity = self.parent_child_map.get(d.entity)
+			period = self.get_period(d.get(self.date_field))
+			self.entity_periodic_data.setdefault(d.entity, frappe._dict()).setdefault(period, 0.0)
+			self.entity_periodic_data[d.entity][period] += flt(d.value_field)
+
+	def get_period(self, posting_date):
+		if self.filters.range == 'Weekly':
+			period = "Week " + str(posting_date.isocalendar()[1])
+		elif self.filters.range == 'Monthly':
+			period = self.months[posting_date.month - 1]
+		elif self.filters.range == 'Quarterly':
+			period = "Quarter " + str(((posting_date.month-1)//3)+1)
+		else:
+			year = get_fiscal_year(posting_date, company=self.filters.company)
+			period = str(year[2])
+
+		return period
+
+	def get_period_date_ranges(self):
+		from dateutil.relativedelta import relativedelta
+		from_date, to_date = getdate(self.filters.from_date), getdate(self.filters.to_date)
+
+		increment = {
+			"Monthly": 1,
+			"Quarterly": 3,
+			"Half-Yearly": 6,
+			"Yearly": 12
+		}.get(self.filters.range, 1)
+
+		self.periodic_daterange = []
+		for dummy in range(1, 53, increment):
+			if self.filters.range == "Weekly":
+				period_end_date = from_date + relativedelta(days=6)
+			else:
+				period_end_date = from_date + relativedelta(months=increment, days=-1)
+
+			if period_end_date > to_date:
+				period_end_date = to_date
+			self.periodic_daterange.append([from_date, period_end_date])
+
+			from_date = period_end_date + relativedelta(days=1)
+			if period_end_date == to_date:
+				break
+
+	def get_groups(self):
+		if self.filters.tree_type == "Territory":
+			parent = 'parent_territory'
+		if self.filters.tree_type == "Customer Group":
+			parent = 'parent_customer_group'
+		if self.filters.tree_type == "Item Group":
+			parent = 'parent_item_group'
+		if self.filters.tree_type == "Supplier Group":
+			parent = 'parent_supplier_group'
+
+		self.depth_map = frappe._dict()
+
+		self.group_entries = frappe.db.sql("""select name, lft, rgt , {parent} as parent
+			from `tab{tree}` order by lft"""
+			.format(tree=self.filters.tree_type, parent=parent), as_dict=1)
+
+		for d in self.group_entries:
+			if d.parent:
+				self.depth_map.setdefault(d.name, self.depth_map.get(d.parent) + 1)
+			else:
+				self.depth_map.setdefault(d.name, 0)
+
+	def get_supplier_parent_child_map(self):
+		self.parent_child_map = frappe._dict(frappe.db.sql(""" select name, supplier_group from `tabSupplier`"""))
+
+	def get_chart_data(self):
+		labels = [d.get("label") for d in self.columns[3:]]
+		self.chart = {
+			"data": {
+				'labels': labels,
+				'datasets':[
+				]
+			},
+			"type": "line"
+		}

erpnext/selling/report/sales_analytics/test_analytics.py

@@ -0,0 +1,250 @@
+# Copyright (c) 2013, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+import frappe
+import frappe.defaults
+import unittest
+from erpnext.selling.report.sales_analytics.sales_analytics import execute
+from erpnext.selling.doctype.sales_order.test_sales_order import make_sales_order
+
+class TestAnalytics(unittest.TestCase):
+
+	def tearDown(self):
+		frappe.db.sql(""" DELETE FROM `tabSales Order` """)
+
+	def test_by_entity(self):
+		create_sales_order()
+
+		filters = {
+			'doc_type': 'Sales Order',
+			'range': 'Monthly',
+			'to_date': '2018-03-31',
+			'tree_type': 'Customer',
+			'company': '_Test Company',
+			'from_date': '2017-04-01',
+			'value_quantity': 'Value'
+		}
+
+		report = execute(filters)
+
+		expected_data = [
+			{
+				"entity": "_Test Customer 1",
+				"entity_name": "_Test Customer 1",
+				"apr": 0.0,
+				"may": 0.0,
+				"jun": 0.0,
+				"jul": 0.0,
+				"aug": 0.0,
+				"sep": 0.0,
+				"oct": 0.0,
+				"nov": 0.0,
+				"dec": 0.0,
+				"jan": 0.0,
+				"feb": 2000.0,
+				"mar": 0.0,
+				"total":2000.0
+			},
+			{
+				"entity": "_Test Customer 3",
+				"entity_name": "_Test Customer 3",
+				"apr": 0.0,
+				"may": 0.0,
+				"jun": 2000.0,
+				"jul": 1000.0,
+				"aug": 0.0,
+				"sep": 0.0,
+				"oct": 0.0,
+				"nov": 0.0,
+				"dec": 0.0,
+				"jan": 0.0,
+				"feb": 0.0,
+				"mar": 0.0,
+				"total": 3000.0
+			},
+			{
+				"entity": "_Test Customer 2",
+				"entity_name": "_Test Customer 2",
+				"apr": 0.0,
+				"may": 0.0,
+				"jun": 0.0,
+				"jul": 0.0,
+				"aug": 0.0,
+				"sep": 1500.0,
+				"oct": 1000.0,
+				"nov": 0.0,
+				"dec": 0.0,
+				"jan": 0.0,
+				"feb": 0.0,
+				"mar": 0.0,
+				"total":2500.0
+			}
+		]
+		self.assertEqual(expected_data, report[1])
+
+	def test_by_group(self):
+		create_sales_order()
+
+		filters = {
+			'doc_type': 'Sales Order',
+			'range': 'Monthly',
+			'to_date': '2018-03-31',
+			'tree_type': 'Customer Group',
+			'company': '_Test Company',
+			'from_date': '2017-04-01',
+			'value_quantity': 'Value'
+		}
+
+		report = execute(filters)
+
+		expected_data = [
+			{
+				"entity": "All Customer Groups",
+				"indent": 0,
+				"apr": 0.0,
+				"may": 0.0,
+				"jun": 2000.0,
+				"jul": 1000.0,
+				"aug": 0.0,
+				"sep": 1500.0,
+				"oct": 1000.0,
+				"nov": 0.0,
+				"dec": 0.0,
+				"jan": 0.0,
+				"feb": 2000.0,
+				"mar": 0.0,
+				"total":7500.0
+			},
+			{
+				"entity": "Individual",
+				"indent": 1,
+				"apr": 0.0,
+				"may": 0.0,
+				"jun": 0.0,
+				"jul": 0.0,
+				"aug": 0.0,
+				"sep": 0.0,
+				"oct": 0.0,
+				"nov": 0.0,
+				"dec": 0.0,
+				"jan": 0.0,
+				"feb": 0.0,
+				"mar": 0.0,
+				"total": 0.0
+			},
+			{
+				"entity": "_Test Customer Group",
+				"indent": 1,
+				"apr": 0.0,
+				"may": 0.0,
+				"jun": 0.0,
+				"jul": 0.0,
+				"aug": 0.0,
+				"sep": 0.0,
+				"oct": 0.0,
+				"nov": 0.0,
+				"dec": 0.0,
+				"jan": 0.0,
+				"feb": 0.0,
+				"mar": 0.0,
+				"total":0.0
+			},
+			{
+				"entity": "_Test Customer Group 1",
+				"indent": 1,
+				"apr": 0.0,
+				"may": 0.0,
+				"jun": 0.0,
+				"jul": 0.0,
+				"aug": 0.0,
+				"sep": 0.0,
+				"oct": 0.0,
+				"nov": 0.0,
+				"dec": 0.0,
+				"jan": 0.0,
+				"feb": 0.0,
+				"mar": 0.0,
+				"total":0.0
+			}
+		]
+		self.assertEqual(expected_data, report[1])
+	
+	def test_by_quantity(self):
+		create_sales_order()
+
+		filters = {
+			'doc_type': 'Sales Order',
+			'range': 'Monthly',
+			'to_date': '2018-03-31',
+			'tree_type': 'Customer',
+			'company': '_Test Company',
+			'from_date': '2017-04-01',
+			'value_quantity': 'Quantity'
+		}
+
+		report = execute(filters)
+
+		expected_data = [
+			{
+				"entity": "_Test Customer 1",
+				"entity_name": "_Test Customer 1",
+				"apr": 0.0,
+				"may": 0.0,
+				"jun": 0.0,
+				"jul": 0.0,
+				"aug": 0.0,
+				"sep": 0.0,
+				"oct": 0.0,
+				"nov": 0.0,
+				"dec": 0.0,
+				"jan": 0.0,
+				"feb": 20.0,
+				"mar": 0.0,
+				"total":20.0
+			},
+			{
+				"entity": "_Test Customer 3",
+				"entity_name": "_Test Customer 3",
+				"apr": 0.0,
+				"may": 0.0,
+				"jun": 20.0,
+				"jul": 10.0,
+				"aug": 0.0,
+				"sep": 0.0,
+				"oct": 0.0,
+				"nov": 0.0,
+				"dec": 0.0,
+				"jan": 0.0,
+				"feb": 0.0,
+				"mar": 0.0,
+				"total": 30.0
+			},
+			{
+				"entity": "_Test Customer 2",
+				"entity_name": "_Test Customer 2",
+				"apr": 0.0,
+				"may": 0.0,
+				"jun": 0.0,
+				"jul": 0.0,
+				"aug": 0.0,
+				"sep": 15.0,
+				"oct": 10.0,
+				"nov": 0.0,
+				"dec": 0.0,
+				"jan": 0.0,
+				"feb": 0.0,
+				"mar": 0.0,
+				"total":25.0
+			}
+		]
+		self.assertEqual(expected_data, report[1])
+
+def create_sales_order():
+	frappe.set_user("Administrator")
+
+	make_sales_order(qty=10, customer = "_Test Customer 1", transaction_date='2018-02-10')
+	make_sales_order(qty=10, customer = "_Test Customer 1", transaction_date='2018-02-15')
+	make_sales_order(qty=15, customer = "_Test Customer 2", transaction_date='2017-09-23')
+	make_sales_order(qty=10, customer = "_Test Customer 2", transaction_date='2017-10-10')
+	make_sales_order(qty=20, customer = "_Test Customer 3", transaction_date='2017-06-15')
+	make_sales_order(qty=10, customer = "_Test Customer 3", transaction_date='2017-07-10')