Merge branch 'develop' into stock-reservation

erpnext/accounts/doctype/accounts_settings/accounts_settings.json

@@ -40,6 +40,8 @@
   "show_payment_schedule_in_print",
   "currency_exchange_section",
   "allow_stale",
+  "section_break_jpd0",
+  "auto_reconcile_payments",
   "stale_days",
   "invoicing_settings_tab",
   "accounts_transactions_settings_section",
@@ -59,7 +61,6 @@
   "acc_frozen_upto",
   "column_break_25",
   "frozen_accounts_modifier",
-  "report_settings_sb",
   "tab_break_dpet",
   "show_balance_in_coa"
  ],
@@ -172,11 +173,6 @@
    "fieldtype": "Int",
    "label": "Stale Days"
   },
-  {
-   "fieldname": "report_settings_sb",
-   "fieldtype": "Section Break",
-   "label": "Report Settings"
-  },
   {
    "default": "0",
    "description": "Only select this if you have set up the Cash Flow Mapper documents",
@@ -383,6 +379,17 @@
    "fieldname": "merge_similar_account_heads",
    "fieldtype": "Check",
    "label": "Merge Similar Account Heads"
+  },
+  {
+   "fieldname": "section_break_jpd0",
+   "fieldtype": "Section Break",
+   "label": "Payment Reconciliations"
+  },
+  {
+   "default": "0",
+   "fieldname": "auto_reconcile_payments",
+   "fieldtype": "Check",
+   "label": "Auto Reconcile Payments"
   }
  ],
  "icon": "icon-cog",
@@ -390,7 +397,7 @@
  "index_web_pages_for_search": 1,
  "issingle": 1,
  "links": [],
- "modified": "2023-04-17 11:45:42.049247",
+ "modified": "2023-04-21 13:11:37.130743",
  "modified_by": "Administrator",
  "module": "Accounts",
  "name": "Accounts Settings",

erpnext/accounts/doctype/payment_reconciliation/payment_reconciliation.js

@@ -82,6 +82,32 @@ erpnext.accounts.PaymentReconciliationController = class PaymentReconciliationCo
 			this.frm.change_custom_button_type('Get Unreconciled Entries', null, 'default');
 			this.frm.change_custom_button_type('Allocate', null, 'default');
 		}
+
+		// check for any running reconciliation jobs
+		if (this.frm.doc.receivable_payable_account) {
+			frappe.db.get_single_value("Accounts Settings", "auto_reconcile_payments").then((enabled) => {
+ 				if(enabled) {
+					this.frm.call({
+						'method': "erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation.is_any_doc_running",
+						"args": {
+							for_filter: {
+								company: this.frm.doc.company,
+								party_type: this.frm.doc.party_type,
+								party: this.frm.doc.party,
+								receivable_payable_account: this.frm.doc.receivable_payable_account
+							}
+						}
+					}).then(r => {
+						if (r.message) {
+							let doc_link = frappe.utils.get_form_link("Process Payment Reconciliation", r.message, true);
+							let msg = __("Payment Reconciliation Job: {0} is running for this party. Can't reconcile now.", [doc_link]);
+							this.frm.dashboard.add_comment(msg, "yellow");
+						}
+					});
+				}
+			});
+		}
+
 	}
 
 	company() {

erpnext/accounts/doctype/payment_reconciliation/payment_reconciliation.py

@@ -7,9 +7,12 @@ from frappe import _, msgprint, qb
 from frappe.model.document import Document
 from frappe.query_builder.custom import ConstantColumn
 from frappe.query_builder.functions import IfNull
-from frappe.utils import flt, getdate, nowdate, today
+from frappe.utils import flt, get_link_to_form, getdate, nowdate, today
 
 import erpnext
+from erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation import (
+	is_any_doc_running,
+)
 from erpnext.accounts.utils import (
 	QueryPaymentLedger,
 	get_outstanding_invoices,
@@ -304,9 +307,7 @@ class PaymentReconciliation(Document):
 			}
 		)
 
-	@frappe.whitelist()
+	def reconcile_allocations(self, skip_ref_details_update_for_pe=False):
-	def reconcile(self):
-		self.validate_allocation()
 		dr_or_cr = (
 			"credit_in_account_currency"
 			if erpnext.get_party_account_type(self.party_type) == "Receivable"
@@ -330,12 +331,35 @@ class PaymentReconciliation(Document):
 					self.make_difference_entry(payment_details)
 
 		if entry_list:
-			reconcile_against_document(entry_list)
+			reconcile_against_document(entry_list, skip_ref_details_update_for_pe)
 
 		if dr_or_cr_notes:
 			reconcile_dr_cr_note(dr_or_cr_notes, self.company)
 
+	@frappe.whitelist()
+	def reconcile(self):
+		if frappe.db.get_single_value("Accounts Settings", "auto_reconcile_payments"):
+			running_doc = is_any_doc_running(
+				dict(
+					company=self.company,
+					party_type=self.party_type,
+					party=self.party,
+					receivable_payable_account=self.receivable_payable_account,
+				)
+			)
+
+			if running_doc:
+				frappe.throw(
+					_("A Reconciliation Job {0} is running for the same filters. Cannot reconcile now").format(
+						get_link_to_form("Auto Reconcile", running_doc)
+					)
+				)
+				return
+
+		self.validate_allocation()
+		self.reconcile_allocations()
 		msgprint(_("Successfully Reconciled"))
+
 		self.get_unreconciled_entries()
 
 	def make_difference_entry(self, row):

erpnext/accounts/doctype/process_payment_reconciliation/process_payment_reconciliation.js

@@ -0,0 +1,130 @@
+// Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on("Process Payment Reconciliation", {
+	onload: function(frm) {
+		// set queries
+		frm.set_query("party_type", function() {
+			return {
+				"filters": {
+					"name": ["in", Object.keys(frappe.boot.party_account_types)],
+				}
+			}
+		});
+		frm.set_query('receivable_payable_account',  function(doc) {
+			return {
+				filters: {
+					"company": doc.company,
+					"is_group": 0,
+					"account_type": frappe.boot.party_account_types[doc.party_type]
+				}
+			};
+		});
+		frm.set_query('cost_center', function(doc) {
+			return {
+				filters: {
+					"company": doc.company,
+					"is_group": 0,
+				}
+			};
+		});
+		frm.set_query('bank_cash_account', function(doc) {
+			return {
+				filters:[
+					['Account', 'company', '=', doc.company],
+					['Account', 'is_group', '=', 0],
+					['Account', 'account_type', 'in', ['Bank', 'Cash']]
+				]
+			};
+		});
+
+	},
+	refresh: function(frm) {
+		if (frm.doc.docstatus==1 && ['Queued', 'Paused'].find(x => x == frm.doc.status)) {
+			let execute_btn = __("Start / Resume")
+
+			frm.add_custom_button(execute_btn, () => {
+				frm.call({
+					method: 'erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation.trigger_job_for_doc',
+					args: {
+						docname: frm.doc.name
+					}
+				}).then(r => {
+					if(!r.exc) {
+						frappe.show_alert(__("Job Started"));
+						frm.reload_doc();
+					}
+				});
+			});
+		}
+		if (frm.doc.docstatus==1 && ['Completed', 'Running', 'Paused', 'Partially Reconciled'].find(x => x == frm.doc.status)) {
+			frm.call({
+				'method': "erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation.get_reconciled_count",
+				args: {
+					"docname": frm.docname,
+				}
+			}).then(r => {
+				if (r.message) {
+					let progress = 0;
+					let description = "";
+
+					if (r.message.processed) {
+						progress = (r.message.processed/r.message.total) * 100;
+						description = r.message.processed + "/" + r.message.total +  " processed";
+					} else if (r.message.total == 0 && frm.doc.status == "Completed") {
+						progress = 100;
+					}
+
+
+					frm.dashboard.add_progress('Reconciliation Progress', progress, description);
+				}
+			})
+		}
+		if (frm.doc.docstatus==1 && frm.doc.status == 'Running') {
+			let execute_btn = __("Pause")
+
+			frm.add_custom_button(execute_btn, () => {
+				frm.call({
+					'method': "erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation.pause_job_for_doc",
+					args: {
+						"docname": frm.docname,
+					}
+				}).then(r => {
+					if (!r.exc) {
+						frappe.show_alert(__("Job Paused"));
+						frm.reload_doc()
+					}
+				});
+
+			});
+		}
+	},
+	company(frm) {
+		frm.set_value('party', '');
+		frm.set_value('receivable_payable_account', '');
+	},
+	party_type(frm) {
+		frm.set_value('party', '');
+	},
+
+	party(frm) {
+		frm.set_value('receivable_payable_account', '');
+		if (!frm.doc.receivable_payable_account && frm.doc.party_type && frm.doc.party) {
+			return frappe.call({
+				method: "erpnext.accounts.party.get_party_account",
+				args: {
+					company: frm.doc.company,
+					party_type: frm.doc.party_type,
+					party: frm.doc.party
+				},
+				callback: (r) => {
+					if (!r.exc && r.message) {
+						frm.set_value("receivable_payable_account", r.message);
+					}
+					frm.refresh();
+
+				}
+			});
+		}
+	}
+});

erpnext/accounts/doctype/process_payment_reconciliation/process_payment_reconciliation.json

@@ -0,0 +1,173 @@
+{
+ "actions": [],
+ "autoname": "format:ACC-PPR-{#####}",
+ "beta": 1,
+ "creation": "2023-03-30 21:28:39.793927",
+ "default_view": "List",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "company",
+  "party_type",
+  "column_break_io6c",
+  "party",
+  "receivable_payable_account",
+  "filter_section",
+  "from_invoice_date",
+  "to_invoice_date",
+  "column_break_kegk",
+  "from_payment_date",
+  "to_payment_date",
+  "column_break_uj04",
+  "cost_center",
+  "bank_cash_account",
+  "section_break_2n02",
+  "status",
+  "error_log",
+  "section_break_a8yx",
+  "amended_from"
+ ],
+ "fields": [
+  {
+   "allow_on_submit": 1,
+   "fieldname": "status",
+   "fieldtype": "Select",
+   "label": "Status",
+   "options": "\nQueued\nRunning\nPaused\nCompleted\nPartially Reconciled\nFailed\nCancelled",
+   "read_only": 1
+  },
+  {
+   "fieldname": "company",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "label": "Company",
+   "options": "Company",
+   "reqd": 1
+  },
+  {
+   "fieldname": "party_type",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "label": "Party Type",
+   "options": "DocType",
+   "reqd": 1
+  },
+  {
+   "fieldname": "column_break_io6c",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "party",
+   "fieldtype": "Dynamic Link",
+   "in_list_view": 1,
+   "label": "Party",
+   "options": "party_type",
+   "reqd": 1
+  },
+  {
+   "fieldname": "receivable_payable_account",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "label": "Receivable/Payable Account",
+   "options": "Account",
+   "reqd": 1
+  },
+  {
+   "fieldname": "filter_section",
+   "fieldtype": "Section Break",
+   "label": "Filters"
+  },
+  {
+   "fieldname": "from_invoice_date",
+   "fieldtype": "Date",
+   "label": "From Invoice Date"
+  },
+  {
+   "fieldname": "to_invoice_date",
+   "fieldtype": "Date",
+   "label": "To Invoice Date"
+  },
+  {
+   "fieldname": "column_break_kegk",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "from_payment_date",
+   "fieldtype": "Date",
+   "label": "From Payment Date"
+  },
+  {
+   "fieldname": "to_payment_date",
+   "fieldtype": "Date",
+   "label": "To Payment Date"
+  },
+  {
+   "fieldname": "column_break_uj04",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "cost_center",
+   "fieldtype": "Link",
+   "label": "Cost Center",
+   "options": "Cost Center"
+  },
+  {
+   "fieldname": "bank_cash_account",
+   "fieldtype": "Link",
+   "label": "Bank/Cash Account",
+   "options": "Account"
+  },
+  {
+   "fieldname": "section_break_2n02",
+   "fieldtype": "Section Break",
+   "label": "Status"
+  },
+  {
+   "depends_on": "eval:doc.error_log",
+   "fieldname": "error_log",
+   "fieldtype": "Long Text",
+   "label": "Error Log"
+  },
+  {
+   "fieldname": "amended_from",
+   "fieldtype": "Link",
+   "label": "Amended From",
+   "no_copy": 1,
+   "options": "Process Payment Reconciliation",
+   "print_hide": 1,
+   "read_only": 1
+  },
+  {
+   "fieldname": "section_break_a8yx",
+   "fieldtype": "Section Break"
+  }
+ ],
+ "index_web_pages_for_search": 1,
+ "is_submittable": 1,
+ "links": [],
+ "modified": "2023-04-21 17:19:30.912953",
+ "modified_by": "Administrator",
+ "module": "Accounts",
+ "name": "Process Payment Reconciliation",
+ "naming_rule": "Expression",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  }
+ ],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": [],
+ "title_field": "company"
+}

erpnext/accounts/doctype/process_payment_reconciliation/process_payment_reconciliation.py

@@ -0,0 +1,503 @@
+# Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+import frappe
+from frappe import _, qb
+from frappe.model.document import Document
+from frappe.utils import get_link_to_form
+from frappe.utils.scheduler import is_scheduler_inactive
+
+
+class ProcessPaymentReconciliation(Document):
+	def validate(self):
+		self.validate_receivable_payable_account()
+		self.validate_bank_cash_account()
+
+	def validate_receivable_payable_account(self):
+		if self.receivable_payable_account:
+			if self.company != frappe.db.get_value("Account", self.receivable_payable_account, "company"):
+				frappe.throw(
+					_("Receivable/Payable Account: {0} doesn't belong to company {1}").format(
+						frappe.bold(self.receivable_payable_account), frappe.bold(self.company)
+					)
+				)
+
+	def validate_bank_cash_account(self):
+		if self.bank_cash_account:
+			if self.company != frappe.db.get_value("Account", self.bank_cash_account, "company"):
+				frappe.throw(
+					_("Bank/Cash Account {0} doesn't belong to company {1}").format(
+						frappe.bold(self.bank_cash_account), frappe.bold(self.company)
+					)
+				)
+
+	def before_save(self):
+		self.status = ""
+		self.error_log = ""
+
+	def on_submit(self):
+		self.db_set("status", "Queued")
+		self.db_set("error_log", None)
+
+	def on_cancel(self):
+		self.db_set("status", "Cancelled")
+		log = frappe.db.get_value(
+			"Process Payment Reconciliation Log", filters={"process_pr": self.name}
+		)
+		if log:
+			frappe.db.set_value("Process Payment Reconciliation Log", log, "status", "Cancelled")
+
+
+@frappe.whitelist()
+def get_reconciled_count(docname: str | None = None) -> float:
+	current_status = {}
+	if docname:
+		reconcile_log = frappe.db.get_value(
+			"Process Payment Reconciliation Log", filters={"process_pr": docname}, fieldname="name"
+		)
+		if reconcile_log:
+			res = frappe.get_all(
+				"Process Payment Reconciliation Log",
+				filters={"name": reconcile_log},
+				fields=["reconciled_entries", "total_allocations"],
+				as_list=1,
+			)
+			current_status["processed"], current_status["total"] = res[0]
+
+	return current_status
+
+
+def get_pr_instance(doc: str):
+	process_payment_reconciliation = frappe.get_doc("Process Payment Reconciliation", doc)
+
+	pr = frappe.get_doc("Payment Reconciliation")
+	fields = [
+		"company",
+		"party_type",
+		"party",
+		"receivable_payable_account",
+		"from_invoice_date",
+		"to_invoice_date",
+		"from_payment_date",
+		"to_payment_date",
+	]
+	d = {}
+	for field in fields:
+		d[field] = process_payment_reconciliation.get(field)
+	pr.update(d)
+	pr.invoice_limit = 1000
+	pr.payment_limit = 1000
+	return pr
+
+
+def is_job_running(job_name: str) -> bool:
+	jobs = frappe.db.get_all("RQ Job", filters={"status": ["in", ["started", "queued"]]})
+	for x in jobs:
+		if x.job_name == job_name:
+			return True
+	return False
+
+
+@frappe.whitelist()
+def pause_job_for_doc(docname: str | None = None):
+	if docname:
+		frappe.db.set_value("Process Payment Reconciliation", docname, "status", "Paused")
+		log = frappe.db.get_value("Process Payment Reconciliation Log", filters={"process_pr": docname})
+		if log:
+			frappe.db.set_value("Process Payment Reconciliation Log", log, "status", "Paused")
+
+
+@frappe.whitelist()
+def trigger_job_for_doc(docname: str | None = None):
+	"""
+	Trigger background job
+	"""
+	if not docname:
+		return
+
+	if not frappe.db.get_single_value("Accounts Settings", "auto_reconcile_payments"):
+		frappe.throw(
+			_("Auto Reconciliation of Payments has been disabled. Enable it through {0}").format(
+				get_link_to_form("Accounts Settings", "Accounts Settings")
+			)
+		)
+
+		return
+
+	if not is_scheduler_inactive():
+		if frappe.db.get_value("Process Payment Reconciliation", docname, "status") == "Queued":
+			frappe.db.set_value("Process Payment Reconciliation", docname, "status", "Running")
+			job_name = f"start_processing_{docname}"
+			if not is_job_running(job_name):
+				job = frappe.enqueue(
+					method="erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation.reconcile_based_on_filters",
+					queue="long",
+					is_async=True,
+					job_name=job_name,
+					enqueue_after_commit=True,
+					doc=docname,
+				)
+
+		elif frappe.db.get_value("Process Payment Reconciliation", docname, "status") == "Paused":
+			frappe.db.set_value("Process Payment Reconciliation", docname, "status", "Running")
+			log = frappe.db.get_value("Process Payment Reconciliation Log", filters={"process_pr": docname})
+			if log:
+				frappe.db.set_value("Process Payment Reconciliation Log", log, "status", "Running")
+
+			# Resume tasks for running doc
+			job_name = f"start_processing_{docname}"
+			if not is_job_running(job_name):
+				job = frappe.enqueue(
+					method="erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation.reconcile_based_on_filters",
+					queue="long",
+					is_async=True,
+					job_name=job_name,
+					doc=docname,
+				)
+	else:
+		frappe.msgprint(_("Scheduler is Inactive. Can't trigger job now."))
+
+
+def trigger_reconciliation_for_queued_docs():
+	"""
+	Will be called from Cron Job
+	Fetch queued docs and start reconciliation process for each one
+	"""
+	if not frappe.db.get_single_value("Accounts Settings", "auto_reconcile_payments"):
+		frappe.throw(
+			_("Auto Reconciliation of Payments has been disabled. Enable it through {0}").format(
+				get_link_to_form("Accounts Settings", "Accounts Settings")
+			)
+		)
+
+		return
+
+	if not is_scheduler_inactive():
+		# Get all queued documents
+		all_queued = frappe.db.get_all(
+			"Process Payment Reconciliation",
+			filters={"docstatus": 1, "status": "Queued"},
+			order_by="creation desc",
+			as_list=1,
+		)
+
+		docs_to_trigger = []
+		unique_filters = set()
+		queue_size = 5
+
+		fields = ["company", "party_type", "party", "receivable_payable_account"]
+
+		def get_filters_as_tuple(fields, doc):
+			filters = ()
+			for x in fields:
+				filters += tuple(doc.get(x))
+			return filters
+
+		for x in all_queued:
+			doc = frappe.get_doc("Process Payment Reconciliation", x)
+			filters = get_filters_as_tuple(fields, doc)
+			if filters not in unique_filters:
+				unique_filters.add(filters)
+				docs_to_trigger.append(doc.name)
+			if len(docs_to_trigger) == queue_size:
+				break
+
+		# trigger reconcilation process for queue_size unique filters
+		for doc in docs_to_trigger:
+			trigger_job_for_doc(doc)
+
+	else:
+		frappe.msgprint(_("Scheduler is Inactive. Can't trigger jobs now."))
+
+
+def reconcile_based_on_filters(doc: None | str = None) -> None:
+	"""
+	Identify current state of document and execute next tasks in background
+	"""
+	if doc:
+		log = frappe.db.get_value("Process Payment Reconciliation Log", filters={"process_pr": doc})
+		if not log:
+			log = frappe.new_doc("Process Payment Reconciliation Log")
+			log.process_pr = doc
+			log.status = "Running"
+			log = log.save()
+
+			job_name = f"process_{doc}_fetch_and_allocate"
+			if not is_job_running(job_name):
+				job = frappe.enqueue(
+					method="erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation.fetch_and_allocate",
+					queue="long",
+					timeout="3600",
+					is_async=True,
+					job_name=job_name,
+					enqueue_after_commit=True,
+					doc=doc,
+				)
+		else:
+			res = frappe.get_all(
+				"Process Payment Reconciliation Log",
+				filters={"name": log},
+				fields=["allocated", "reconciled"],
+				as_list=1,
+			)
+			allocated, reconciled = res[0]
+
+			if not allocated:
+				job_name = f"process__{doc}_fetch_and_allocate"
+				if not is_job_running(job_name):
+					job = frappe.enqueue(
+						method="erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation.fetch_and_allocate",
+						queue="long",
+						timeout="3600",
+						is_async=True,
+						job_name=job_name,
+						enqueue_after_commit=True,
+						doc=doc,
+					)
+			elif not reconciled:
+				allocation = get_next_allocation(log)
+				if allocation:
+					reconcile_job_name = (
+						f"process_{doc}_reconcile_allocation_{allocation[0].idx}_{allocation[-1].idx}"
+					)
+				else:
+					reconcile_job_name = f"process_{doc}_reconcile"
+				if not is_job_running(reconcile_job_name):
+					job = frappe.enqueue(
+						method="erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation.reconcile",
+						queue="long",
+						timeout="3600",
+						is_async=True,
+						job_name=reconcile_job_name,
+						enqueue_after_commit=True,
+						doc=doc,
+					)
+			elif reconciled:
+				frappe.db.set_value("Process Payment Reconciliation", doc, "status", "Completed")
+
+
+def get_next_allocation(log: str) -> list:
+	if log:
+		allocations = []
+		next = frappe.db.get_all(
+			"Process Payment Reconciliation Log Allocations",
+			filters={"parent": log, "reconciled": 0},
+			fields=["reference_type", "reference_name"],
+			order_by="idx",
+			limit=1,
+		)
+
+		if next:
+			allocations = frappe.db.get_all(
+				"Process Payment Reconciliation Log Allocations",
+				filters={
+					"parent": log,
+					"reconciled": 0,
+					"reference_type": next[0].reference_type,
+					"reference_name": next[0].reference_name,
+				},
+				fields=["*"],
+				order_by="idx",
+			)
+
+		return allocations
+	return []
+
+
+def fetch_and_allocate(doc: str) -> None:
+	"""
+	Fetch Invoices and Payments based on filters applied. FIFO ordering is used for allocation.
+	"""
+
+	if doc:
+		log = frappe.db.get_value("Process Payment Reconciliation Log", filters={"process_pr": doc})
+		if log:
+			if not frappe.db.get_value("Process Payment Reconciliation Log", log, "allocated"):
+				reconcile_log = frappe.get_doc("Process Payment Reconciliation Log", log)
+
+				pr = get_pr_instance(doc)
+				pr.get_unreconciled_entries()
+
+				if len(pr.invoices) > 0 and len(pr.payments) > 0:
+					invoices = [x.as_dict() for x in pr.invoices]
+					payments = [x.as_dict() for x in pr.payments]
+					pr.allocate_entries(frappe._dict({"invoices": invoices, "payments": payments}))
+
+					for x in pr.get("allocation"):
+						reconcile_log.append(
+							"allocations",
+							x.as_dict().update(
+								{
+									"parenttype": "Process Payment Reconciliation Log",
+									"parent": reconcile_log.name,
+									"name": None,
+									"reconciled": False,
+								}
+							),
+						)
+				reconcile_log.allocated = True
+				reconcile_log.total_allocations = len(reconcile_log.get("allocations"))
+				reconcile_log.reconciled_entries = 0
+				reconcile_log.save()
+
+				# generate reconcile job name
+				allocation = get_next_allocation(log)
+				if allocation:
+					reconcile_job_name = (
+						f"process_{doc}_reconcile_allocation_{allocation[0].idx}_{allocation[-1].idx}"
+					)
+				else:
+					reconcile_job_name = f"process_{doc}_reconcile"
+
+				if not is_job_running(reconcile_job_name):
+					job = frappe.enqueue(
+						method="erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation.reconcile",
+						queue="long",
+						timeout="3600",
+						is_async=True,
+						job_name=reconcile_job_name,
+						enqueue_after_commit=True,
+						doc=doc,
+					)
+
+
+def reconcile(doc: None | str = None) -> None:
+	if doc:
+		log = frappe.db.get_value("Process Payment Reconciliation Log", filters={"process_pr": doc})
+		if log:
+			res = frappe.get_all(
+				"Process Payment Reconciliation Log",
+				filters={"name": log},
+				fields=["reconciled_entries", "total_allocations"],
+				as_list=1,
+				limit=1,
+			)
+
+			reconciled_entries, total_allocations = res[0]
+			if reconciled_entries != total_allocations:
+				try:
+					# Fetch next allocation
+					allocations = get_next_allocation(log)
+
+					pr = get_pr_instance(doc)
+
+					# pass allocation to PR instance
+					for x in allocations:
+						pr.append("allocation", x)
+
+					# reconcile
+					pr.reconcile_allocations(skip_ref_details_update_for_pe=True)
+
+					# If Payment Entry, update details only for newly linked references
+					# This is for performance
+					if allocations[0].reference_type == "Payment Entry":
+
+						references = [(x.invoice_type, x.invoice_number) for x in allocations]
+						pe = frappe.get_doc(allocations[0].reference_type, allocations[0].reference_name)
+						pe.flags.ignore_validate_update_after_submit = True
+						pe.set_missing_ref_details(update_ref_details_only_for=references)
+						pe.save()
+
+					# Update reconciled flag
+					allocation_names = [x.name for x in allocations]
+					ppa = qb.DocType("Process Payment Reconciliation Log Allocations")
+					qb.update(ppa).set(ppa.reconciled, True).where(ppa.name.isin(allocation_names)).run()
+
+					# Update reconciled count
+					reconciled_count = frappe.db.count(
+						"Process Payment Reconciliation Log Allocations", filters={"parent": log, "reconciled": True}
+					)
+					frappe.db.set_value(
+						"Process Payment Reconciliation Log", log, "reconciled_entries", reconciled_count
+					)
+
+				except Exception as err:
+					# Update the parent doc about the exception
+					frappe.db.rollback()
+
+					traceback = frappe.get_traceback()
+					if traceback:
+						message = "Traceback: <br>" + traceback
+						frappe.db.set_value("Process Payment Reconciliation Log", log, "error_log", message)
+						frappe.db.set_value(
+							"Process Payment Reconciliation",
+							doc,
+							"error_log",
+							message,
+						)
+					if reconciled_entries and total_allocations and reconciled_entries < total_allocations:
+						frappe.db.set_value(
+							"Process Payment Reconciliation Log", log, "status", "Partially Reconciled"
+						)
+						frappe.db.set_value(
+							"Process Payment Reconciliation",
+							doc,
+							"status",
+							"Partially Reconciled",
+						)
+					else:
+						frappe.db.set_value("Process Payment Reconciliation Log", log, "status", "Failed")
+						frappe.db.set_value(
+							"Process Payment Reconciliation",
+							doc,
+							"status",
+							"Failed",
+						)
+				finally:
+					if reconciled_entries == total_allocations:
+						frappe.db.set_value("Process Payment Reconciliation Log", log, "status", "Reconciled")
+						frappe.db.set_value("Process Payment Reconciliation Log", log, "reconciled", True)
+						frappe.db.set_value("Process Payment Reconciliation", doc, "status", "Completed")
+					else:
+
+						if not (frappe.db.get_value("Process Payment Reconciliation", doc, "status") == "Paused"):
+							# trigger next batch in job
+							# generate reconcile job name
+							allocation = get_next_allocation(log)
+							if allocation:
+								reconcile_job_name = (
+									f"process_{doc}_reconcile_allocation_{allocation[0].idx}_{allocation[-1].idx}"
+								)
+							else:
+								reconcile_job_name = f"process_{doc}_reconcile"
+
+							if not is_job_running(reconcile_job_name):
+								job = frappe.enqueue(
+									method="erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation.reconcile",
+									queue="long",
+									timeout="3600",
+									is_async=True,
+									job_name=reconcile_job_name,
+									enqueue_after_commit=True,
+									doc=doc,
+								)
+			else:
+				frappe.db.set_value("Process Payment Reconciliation Log", log, "status", "Reconciled")
+				frappe.db.set_value("Process Payment Reconciliation Log", log, "reconciled", True)
+				frappe.db.set_value("Process Payment Reconciliation", doc, "status", "Completed")
+
+
+@frappe.whitelist()
+def is_any_doc_running(for_filter: str | dict | None = None) -> str | None:
+	running_doc = None
+	if for_filter:
+		if type(for_filter) == str:
+			for_filter = frappe.json.loads(for_filter)
+
+		running_doc = frappe.db.get_value(
+			"Process Payment Reconciliation",
+			filters={
+				"docstatus": 1,
+				"status": ["in", ["Running", "Paused"]],
+				"company": for_filter.get("company"),
+				"party_type": for_filter.get("party_type"),
+				"party": for_filter.get("party"),
+				"receivable_payable_account": for_filter.get("receivable_payable_account"),
+			},
+			fieldname="name",
+		)
+	else:
+		running_doc = frappe.db.get_value(
+			"Process Payment Reconciliation", filters={"docstatus": 1, "status": "Running"}
+		)
+	return running_doc

erpnext/accounts/doctype/process_payment_reconciliation/process_payment_reconciliation_dashboard.py

@@ -0,0 +1,15 @@
+from frappe import _
+
+
+def get_data():
+	return {
+		"fieldname": "process_pr",
+		"transactions": [
+			{
+				"label": _("Reconciliation Logs"),
+				"items": [
+					"Process Payment Reconciliation Log",
+				],
+			},
+		],
+	}

erpnext/accounts/doctype/process_payment_reconciliation/process_payment_reconciliation_list.js

@@ -0,0 +1,15 @@
+frappe.listview_settings['Process Payment Reconciliation'] = {
+	add_fields: ["status"],
+	get_indicator: function(doc) {
+		let colors = {
+			'Queued': 'orange',
+			'Paused': 'orange',
+			'Completed': 'green',
+			'Partially Reconciled': 'orange',
+			'Running': 'blue',
+			'Failed': 'red',
+		};
+		let status = doc.status;
+		return [__(status), colors[status], 'status,=,'+status];
+	},
+};

erpnext/accounts/doctype/process_payment_reconciliation/test_process_payment_reconciliation.py

@@ -0,0 +1,9 @@
+# Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and Contributors
+# See license.txt
+
+# import frappe
+from frappe.tests.utils import FrappeTestCase
+
+
+class TestProcessPaymentReconciliation(FrappeTestCase):
+	pass

erpnext/accounts/doctype/process_payment_reconciliation_log/process_payment_reconciliation_log.js

@@ -0,0 +1,17 @@
+// Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and contributors
+// For license information, please see license.txt
+
+frappe.ui.form.on("Process Payment Reconciliation Log", {
+	refresh(frm) {
+		if (['Completed', 'Running', 'Paused', 'Partially Reconciled'].find(x => x == frm.doc.status)) {
+			let progress = 0;
+			if (frm.doc.reconciled_entries != 0) {
+				progress = frm.doc.reconciled_entries / frm.doc.total_allocations * 100;
+			} else if(frm.doc.total_allocations == 0 && frm.doc.status == "Completed"){
+				progress = 100;
+			}
+			frm.dashboard.add_progress(__('Reconciliation Progress'), progress);
+		}
+
+	},
+});

erpnext/accounts/doctype/process_payment_reconciliation_log/process_payment_reconciliation_log.json

@@ -0,0 +1,137 @@
+{
+ "actions": [],
+ "autoname": "format:PPR-LOG-{##}",
+ "beta": 1,
+ "creation": "2023-03-13 15:00:09.149681",
+ "default_view": "List",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "process_pr",
+  "section_break_fvdw",
+  "status",
+  "tasks_section",
+  "allocated",
+  "reconciled",
+  "column_break_yhin",
+  "total_allocations",
+  "reconciled_entries",
+  "section_break_4ywv",
+  "error_log",
+  "allocations_section",
+  "allocations"
+ ],
+ "fields": [
+  {
+   "fieldname": "allocations",
+   "fieldtype": "Table",
+   "label": "Allocations",
+   "options": "Process Payment Reconciliation Log Allocations",
+   "read_only": 1
+  },
+  {
+   "default": "0",
+   "description": "All allocations have been successfully reconciled",
+   "fieldname": "reconciled",
+   "fieldtype": "Check",
+   "label": "Reconciled",
+   "read_only": 1
+  },
+  {
+   "fieldname": "total_allocations",
+   "fieldtype": "Int",
+   "in_list_view": 1,
+   "label": "Total Allocations",
+   "read_only": 1
+  },
+  {
+   "default": "0",
+   "description": "Invoices and Payments have been Fetched and Allocated",
+   "fieldname": "allocated",
+   "fieldtype": "Check",
+   "label": "Allocated",
+   "read_only": 1
+  },
+  {
+   "fieldname": "reconciled_entries",
+   "fieldtype": "Int",
+   "in_list_view": 1,
+   "label": "Reconciled Entries",
+   "read_only": 1
+  },
+  {
+   "fieldname": "tasks_section",
+   "fieldtype": "Section Break",
+   "label": "Tasks"
+  },
+  {
+   "fieldname": "allocations_section",
+   "fieldtype": "Section Break",
+   "label": "Allocations"
+  },
+  {
+   "fieldname": "column_break_yhin",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "section_break_4ywv",
+   "fieldtype": "Section Break"
+  },
+  {
+   "depends_on": "eval:doc.error_log",
+   "fieldname": "error_log",
+   "fieldtype": "Long Text",
+   "label": "Reconciliation Error Log",
+   "read_only": 1
+  },
+  {
+   "fieldname": "process_pr",
+   "fieldtype": "Link",
+   "in_list_view": 1,
+   "label": "Parent Document",
+   "options": "Process Payment Reconciliation",
+   "read_only": 1,
+   "reqd": 1
+  },
+  {
+   "fieldname": "section_break_fvdw",
+   "fieldtype": "Section Break",
+   "label": "Status"
+  },
+  {
+   "fieldname": "status",
+   "fieldtype": "Select",
+   "label": "Status",
+   "options": "Running\nPaused\nReconciled\nPartially Reconciled\nFailed\nCancelled",
+   "read_only": 1
+  }
+ ],
+ "in_create": 1,
+ "index_web_pages_for_search": 1,
+ "links": [],
+ "modified": "2023-04-21 17:36:26.642617",
+ "modified_by": "Administrator",
+ "module": "Accounts",
+ "name": "Process Payment Reconciliation Log",
+ "naming_rule": "Expression",
+ "owner": "Administrator",
+ "permissions": [
+  {
+   "create": 1,
+   "delete": 1,
+   "email": 1,
+   "export": 1,
+   "print": 1,
+   "read": 1,
+   "report": 1,
+   "role": "System Manager",
+   "share": 1,
+   "write": 1
+  }
+ ],
+ "search_fields": "allocated, reconciled, total_allocations, reconciled_entries",
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": []
+}

erpnext/accounts/doctype/process_payment_reconciliation_log/process_payment_reconciliation_log.py

@@ -0,0 +1,9 @@
+# Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+# import frappe
+from frappe.model.document import Document
+
+
+class ProcessPaymentReconciliationLog(Document):
+	pass

erpnext/accounts/doctype/process_payment_reconciliation_log/process_payment_reconciliation_log_list.js

@@ -0,0 +1,15 @@
+frappe.listview_settings['Process Payment Reconciliation Log'] = {
+	add_fields: ["status"],
+	get_indicator: function(doc) {
+		var colors = {
+			'Partially Reconciled': 'orange',
+			'Paused': 'orange',
+			'Reconciled': 'green',
+			'Failed': 'red',
+			'Cancelled': 'red',
+			'Running': 'blue',
+		};
+		let status = doc.status;
+		return [__(status), colors[status], "status,=,"+status];
+	},
+};

erpnext/accounts/doctype/process_payment_reconciliation_log/test_process_payment_reconciliation_log.py

@@ -0,0 +1,9 @@
+# Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and Contributors
+# See license.txt
+
+# import frappe
+from frappe.tests.utils import FrappeTestCase
+
+
+class TestProcessPaymentReconciliationLog(FrappeTestCase):
+	pass

erpnext/accounts/doctype/process_payment_reconciliation_log_allocations/process_payment_reconciliation_log_allocations.json

@@ -0,0 +1,170 @@
+{
+ "actions": [],
+ "creation": "2023-03-13 13:51:27.351463",
+ "default_view": "List",
+ "doctype": "DocType",
+ "editable_grid": 1,
+ "engine": "InnoDB",
+ "field_order": [
+  "reference_type",
+  "reference_name",
+  "reference_row",
+  "column_break_3",
+  "invoice_type",
+  "invoice_number",
+  "section_break_6",
+  "allocated_amount",
+  "unreconciled_amount",
+  "column_break_8",
+  "amount",
+  "is_advance",
+  "section_break_5",
+  "difference_amount",
+  "column_break_7",
+  "difference_account",
+  "exchange_rate",
+  "currency",
+  "reconciled"
+ ],
+ "fields": [
+  {
+   "fieldname": "reference_type",
+   "fieldtype": "Link",
+   "label": "Reference Type",
+   "options": "DocType",
+   "read_only": 1,
+   "reqd": 1
+  },
+  {
+   "fieldname": "reference_name",
+   "fieldtype": "Dynamic Link",
+   "in_list_view": 1,
+   "label": "Reference Name",
+   "options": "reference_type",
+   "read_only": 1,
+   "reqd": 1
+  },
+  {
+   "fieldname": "reference_row",
+   "fieldtype": "Data",
+   "hidden": 1,
+   "label": "Reference Row",
+   "read_only": 1
+  },
+  {
+   "fieldname": "column_break_3",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "invoice_type",
+   "fieldtype": "Link",
+   "label": "Invoice Type",
+   "options": "DocType",
+   "read_only": 1,
+   "reqd": 1
+  },
+  {
+   "fieldname": "invoice_number",
+   "fieldtype": "Dynamic Link",
+   "in_list_view": 1,
+   "label": "Invoice Number",
+   "options": "invoice_type",
+   "read_only": 1,
+   "reqd": 1
+  },
+  {
+   "fieldname": "section_break_6",
+   "fieldtype": "Section Break"
+  },
+  {
+   "fieldname": "allocated_amount",
+   "fieldtype": "Currency",
+   "in_list_view": 1,
+   "label": "Allocated Amount",
+   "options": "currency",
+   "reqd": 1
+  },
+  {
+   "fieldname": "unreconciled_amount",
+   "fieldtype": "Currency",
+   "hidden": 1,
+   "label": "Unreconciled Amount",
+   "options": "currency",
+   "read_only": 1
+  },
+  {
+   "fieldname": "column_break_8",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "amount",
+   "fieldtype": "Currency",
+   "hidden": 1,
+   "label": "Amount",
+   "options": "currency",
+   "read_only": 1
+  },
+  {
+   "fieldname": "is_advance",
+   "fieldtype": "Data",
+   "hidden": 1,
+   "label": "Is Advance",
+   "read_only": 1
+  },
+  {
+   "fieldname": "section_break_5",
+   "fieldtype": "Section Break"
+  },
+  {
+   "fieldname": "difference_amount",
+   "fieldtype": "Currency",
+   "in_list_view": 1,
+   "label": "Difference Amount",
+   "options": "Currency",
+   "read_only": 1
+  },
+  {
+   "fieldname": "column_break_7",
+   "fieldtype": "Column Break"
+  },
+  {
+   "fieldname": "difference_account",
+   "fieldtype": "Link",
+   "label": "Difference Account",
+   "options": "Account",
+   "read_only": 1
+  },
+  {
+   "fieldname": "exchange_rate",
+   "fieldtype": "Float",
+   "label": "Exchange Rate",
+   "read_only": 1
+  },
+  {
+   "fieldname": "currency",
+   "fieldtype": "Link",
+   "hidden": 1,
+   "label": "Currency",
+   "options": "Currency"
+  },
+  {
+   "default": "0",
+   "fieldname": "reconciled",
+   "fieldtype": "Check",
+   "in_list_view": 1,
+   "label": "Reconciled"
+  }
+ ],
+ "istable": 1,
+ "links": [],
+ "modified": "2023-03-20 21:05:43.121945",
+ "modified_by": "Administrator",
+ "module": "Accounts",
+ "name": "Process Payment Reconciliation Log Allocations",
+ "owner": "Administrator",
+ "permissions": [],
+ "sort_field": "modified",
+ "sort_order": "DESC",
+ "states": [],
+ "track_changes": 1
+}

erpnext/accounts/doctype/process_payment_reconciliation_log_allocations/process_payment_reconciliation_log_allocations.py

@@ -0,0 +1,9 @@
+# Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and contributors
+# For license information, please see license.txt
+
+# import frappe
+from frappe.model.document import Document
+
+
+class ProcessPaymentReconciliationLogAllocations(Document):
+	pass

erpnext/accounts/utils.py

@@ -436,7 +436,7 @@ def add_cc(args=None):
 	return cc.name
 
 
-def reconcile_against_document(args):  # nosemgrep
+def reconcile_against_document(args, skip_ref_details_update_for_pe=False):  # nosemgrep
 	"""
 	Cancel PE or JV, Update against document, split if required and resubmit
 	"""
@@ -465,7 +465,9 @@ def reconcile_against_document(args):  # nosemgrep
 			if voucher_type == "Journal Entry":
 				update_reference_in_journal_entry(entry, doc, do_not_save=True)
 			else:
-				update_reference_in_payment_entry(entry, doc, do_not_save=True)
+				update_reference_in_payment_entry(
+					entry, doc, do_not_save=True, skip_ref_details_update_for_pe=skip_ref_details_update_for_pe
+				)
 
 		doc.save(ignore_permissions=True)
 		# re-submit advance entry
@@ -602,7 +604,9 @@ def update_reference_in_journal_entry(d, journal_entry, do_not_save=False):
 		journal_entry.save(ignore_permissions=True)
 
 
-def update_reference_in_payment_entry(d, payment_entry, do_not_save=False):
+def update_reference_in_payment_entry(
+	d, payment_entry, do_not_save=False, skip_ref_details_update_for_pe=False
+):
 	reference_details = {
 		"reference_doctype": d.against_voucher_type,
 		"reference_name": d.against_voucher,
@@ -646,6 +650,7 @@ def update_reference_in_payment_entry(d, payment_entry, do_not_save=False):
 	payment_entry.flags.ignore_validate_update_after_submit = True
 	payment_entry.setup_party_account_field()
 	payment_entry.set_missing_values()
+	if not skip_ref_details_update_for_pe:
 		payment_entry.set_missing_ref_details()
 	payment_entry.set_amounts()
 

erpnext/hooks.py

@@ -362,6 +362,7 @@ scheduler_events = {
 	"cron": {
 		"0/15 * * * *": [
 			"erpnext.manufacturing.doctype.bom_update_log.bom_update_log.resume_bom_cost_update_jobs",
+			"erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation.trigger_reconciliation_for_queued_docs",
 		],
 		"0/30 * * * *": [
 			"erpnext.utilities.doctype.video.video.update_youtube_data",

erpnext/manufacturing/doctype/bom/bom.py

@@ -1317,7 +1317,7 @@ def item_query(doctype, txt, searchfield, start, page_len, filters):
 		if not field in searchfields
 	]
 
-	query_filters = {"disabled": 0, "end_of_life": (">", today())}
+	query_filters = {"disabled": 0, "ifnull(end_of_life, '3099-12-31')": (">", today())}
 
 	or_cond_filters = {}
 	if txt:

erpnext/stock/stock_ledger.py

@@ -1398,6 +1398,10 @@ def update_qty_in_future_sle(args, allow_negative_stock=False):
 def regenerate_sle_for_batch_stock_reco(detail):
 	doc = frappe.get_cached_doc("Stock Reconciliation", detail.voucher_no)
 	doc.recalculate_current_qty(detail.item_code, detail.batch_no)
+
+	if not frappe.db.exists(
+		"Repost Item Valuation", {"voucher_no": doc.name, "status": "Queued", "docstatus": "1"}
+	):
 		doc.repost_future_sle_and_gle()
 
 
@@ -1451,6 +1455,7 @@ def get_next_stock_reco(kwargs):
 				(
 					CombineDatetime(sle.posting_date, sle.posting_time)
 					> CombineDatetime(kwargs.get("posting_date"), kwargs.get("posting_time"))
+				)
 				| (
 					(
 						CombineDatetime(sle.posting_date, sle.posting_time)
@@ -1460,7 +1465,6 @@ def get_next_stock_reco(kwargs):
 				)
 			)
 		)
-		)
 		.orderby(CombineDatetime(sle.posting_date, sle.posting_time))
 		.orderby(sle.creation)
 		.limit(1)