Shiloh/brotherton-erpnext
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Reconcile Payments in background (#34596)

Browse Source 
* feat: auto reconcile in background

* chore: Option to enable auto reconciliation in settings

* refactor: validate if feature is enabled in settings

* refactor: check for running job while using reconciliation tool

* chore: using doc to get filter values

* chore: use frappe.db.get_value in validations

* chore: cleanup commented out code

* chore: replace get_list with get_all

* chore: use block scope variable

* chore: type information for functions

* refactor: flag to ignore job validation check

* refactor: update parent doc status if all reconciled

* chore: create test_records file

* test: create a bunch of vouchers for testing auto reconcile

* chore: renamed auto_reconcile to process_payment_reconciliation

* chore: another child doctype to hold payments

* chore: remove duplicate field

* chore: add fetched payments to log

* chore: Popup comment message update

* chore: replace get_all with get_value

* chore: replace label in settings page

* chore: remove unit test and records

* refactor: status in reconciliation log

* refactor: set status in log as well

* chore: fix field name

* chore: change triggered job name

* chore: use status field in list view of log

* chore: status while there are no allocations

* refactor: split trigger function into two

* chore: adding cancelled status

* refactor: function trigger queued docs

* chore: cron job scheduled

* chore: fixing accouts settings json file

* chore: typos and variable scope

* chore: use 'pluck' in db call

* chore: remove redundant whitelist decorator

* chore: use single DB call to fetch values

* chore: replace get_all with get_value

* refactor: use raw db calls to fetch reconciliation log records

Using get_doc on `Process Payment Reconciliation Log` is costly when
handling large volumes of invoices.

Use raw frappe.db.get_all to selectively pull status and reconciled count

* chore: update status on successful batch operation

* chore: make payment table readonly

* chore: ability to pause the background job

* chore: remove isolate_each_allocation

* chore: more description in progress bar

* refactor: partially working state

* refactor: update reconcile flag and setting hard limits for fetching

* chore: make allocation editable -- NEED TO REVERT

* chore: pause button

* refactor: skip setter function in Payment Entry for better performan

* refactor: split reconcile function and skip a setter function

1. Split reconcile function into 2
2. While reconciling against payment entry, skip a
set_missing_ref_details setter method

* chore: increase payment limit

* refactor: replace frappe.db.get_all with frappe.db.get_value

* chore: remove unwanted doctypes

* refactor: make allocation table readonly

* perf: update ref_details only for newly linked invoices

* chore: rename skip flag

* refactor(UI): receivable_payable field should auto populate

* refactor: no control statements in finally block

* chore: cleanup section and rename checkbox

* chore: update new fieldname in code

* chore: update error msg

* refactor: start and pause integrated into status

pause checkbox has been removed

* refactor: added cancelled status to the log doctype

1. Moved the status section to the bottom in parent doc
2. Using alerts to indicate Job trigger status
This commit is contained in:
ruthra kumar 2023-04-22 17:24:35 +05:30 committed by GitHub
parent 58a5f816db
commit ed14d1ce44
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 1290 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
erpnext/accounts/doctype/accounts_settings/accounts_settings.json
View File

@ -40,6 +40,8 @@
"show_payment_schedule_in_print",
"currency_exchange_section",
"allow_stale",
"section_break_jpd0",
"auto_reconcile_payments",
"stale_days",
"invoicing_settings_tab",
"accounts_transactions_settings_section",
@ -59,7 +61,6 @@
"acc_frozen_upto",
"column_break_25",
"frozen_accounts_modifier",
"report_settings_sb",
"tab_break_dpet",
"show_balance_in_coa"
],
@ -172,11 +173,6 @@
"fieldtype": "Int",
"label": "Stale Days"
},
{
"fieldname": "report_settings_sb",
"fieldtype": "Section Break",
"label": "Report Settings"
},
{
"default": "0",
"description": "Only select this if you have set up the Cash Flow Mapper documents",
@ -383,6 +379,17 @@
"fieldname": "merge_similar_account_heads",
"fieldtype": "Check",
"label": "Merge Similar Account Heads"
},
{
"fieldname": "section_break_jpd0",
"fieldtype": "Section Break",
"label": "Payment Reconciliations"
},
{
"default": "0",
"fieldname": "auto_reconcile_payments",
"fieldtype": "Check",
"label": "Auto Reconcile Payments"
}
],
"icon": "icon-cog",
@ -390,7 +397,7 @@
"index_web_pages_for_search": 1,
"issingle": 1,
"links": [],
"modified": "2023-04-17 11:45:42.049247",
"modified": "2023-04-21 13:11:37.130743",
"modified_by": "Administrator",
"module": "Accounts",
"name": "Accounts Settings",

26
erpnext/accounts/doctype/payment_reconciliation/payment_reconciliation.js
View File

@ -82,6 +82,32 @@ erpnext.accounts.PaymentReconciliationController = class PaymentReconciliationCo
this.frm.change_custom_button_type('Get Unreconciled Entries', null, 'default');
this.frm.change_custom_button_type('Allocate', null, 'default');
}
// check for any running reconciliation jobs
if (this.frm.doc.receivable_payable_account) {
frappe.db.get_single_value("Accounts Settings", "auto_reconcile_payments").then((enabled) => {
if(enabled) {
this.frm.call({
'method': "erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation.is_any_doc_running",
"args": {
for_filter: {
company: this.frm.doc.company,
party_type: this.frm.doc.party_type,
party: this.frm.doc.party,
receivable_payable_account: this.frm.doc.receivable_payable_account
}
}
}).then(r => {
if (r.message) {
let doc_link = frappe.utils.get_form_link("Process Payment Reconciliation", r.message, true);
let msg = __("Payment Reconciliation Job: {0} is running for this party. Can't reconcile now.", [doc_link]);
this.frm.dashboard.add_comment(msg, "yellow");
}
});
}
});
}
}
company() {

34
erpnext/accounts/doctype/payment_reconciliation/payment_reconciliation.py
View File

@ -7,9 +7,12 @@ from frappe import _, msgprint, qb
from frappe.model.document import Document
from frappe.query_builder.custom import ConstantColumn
from frappe.query_builder.functions import IfNull
from frappe.utils import flt, getdate, nowdate, today
from frappe.utils import flt, get_link_to_form, getdate, nowdate, today
import erpnext
from erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation import (
is_any_doc_running,
)
from erpnext.accounts.utils import (
QueryPaymentLedger,
get_outstanding_invoices,
@ -304,9 +307,7 @@ class PaymentReconciliation(Document):
}
)
@frappe.whitelist()
def reconcile(self):
self.validate_allocation()
def reconcile_allocations(self, skip_ref_details_update_for_pe=False):
dr_or_cr = (
"credit_in_account_currency"
if erpnext.get_party_account_type(self.party_type) == "Receivable"
@ -330,12 +331,35 @@ class PaymentReconciliation(Document):
self.make_difference_entry(payment_details)
if entry_list:
reconcile_against_document(entry_list)
reconcile_against_document(entry_list, skip_ref_details_update_for_pe)
if dr_or_cr_notes:
reconcile_dr_cr_note(dr_or_cr_notes, self.company)
@frappe.whitelist()
def reconcile(self):
if frappe.db.get_single_value("Accounts Settings", "auto_reconcile_payments"):
running_doc = is_any_doc_running(
dict(
company=self.company,
party_type=self.party_type,
party=self.party,
receivable_payable_account=self.receivable_payable_account,
)
)
if running_doc:
frappe.throw(
_("A Reconciliation Job {0} is running for the same filters. Cannot reconcile now").format(
get_link_to_form("Auto Reconcile", running_doc)
)
)
return
self.validate_allocation()
self.reconcile_allocations()
msgprint(_("Successfully Reconciled"))
self.get_unreconciled_entries()
def make_difference_entry(self, row):

0
erpnext/accounts/doctype/process_payment_reconciliation/__init__.py Normal file
View File

130
erpnext/accounts/doctype/process_payment_reconciliation/process_payment_reconciliation.js Normal file
View File

@ -0,0 +1,130 @@
// Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and contributors
// For license information, please see license.txt
frappe.ui.form.on("Process Payment Reconciliation", {
onload: function(frm) {
// set queries
frm.set_query("party_type", function() {
return {
"filters": {
"name": ["in", Object.keys(frappe.boot.party_account_types)],
}
}
});
frm.set_query('receivable_payable_account', function(doc) {
return {
filters: {
"company": doc.company,
"is_group": 0,
"account_type": frappe.boot.party_account_types[doc.party_type]
}
};
});
frm.set_query('cost_center', function(doc) {
return {
filters: {
"company": doc.company,
"is_group": 0,
}
};
});
frm.set_query('bank_cash_account', function(doc) {
return {
filters:[
['Account', 'company', '=', doc.company],
['Account', 'is_group', '=', 0],
['Account', 'account_type', 'in', ['Bank', 'Cash']]
]
};
});
},
refresh: function(frm) {
if (frm.doc.docstatus==1 && ['Queued', 'Paused'].find(x => x == frm.doc.status)) {
let execute_btn = __("Start / Resume")
frm.add_custom_button(execute_btn, () => {
frm.call({
method: 'erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation.trigger_job_for_doc',
args: {
docname: frm.doc.name
}
}).then(r => {
if(!r.exc) {
frappe.show_alert(__("Job Started"));
frm.reload_doc();
}
});
});
}
if (frm.doc.docstatus==1 && ['Completed', 'Running', 'Paused', 'Partially Reconciled'].find(x => x == frm.doc.status)) {
frm.call({
'method': "erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation.get_reconciled_count",
args: {
"docname": frm.docname,
}
}).then(r => {
if (r.message) {
let progress = 0;
let description = "";
if (r.message.processed) {
progress = (r.message.processed/r.message.total) * 100;
description = r.message.processed + "/" + r.message.total + " processed";
} else if (r.message.total == 0 && frm.doc.status == "Completed") {
progress = 100;
}
frm.dashboard.add_progress('Reconciliation Progress', progress, description);
}
})
}
if (frm.doc.docstatus==1 && frm.doc.status == 'Running') {
let execute_btn = __("Pause")
frm.add_custom_button(execute_btn, () => {
frm.call({
'method': "erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation.pause_job_for_doc",
args: {
"docname": frm.docname,
}
}).then(r => {
if (!r.exc) {
frappe.show_alert(__("Job Paused"));
frm.reload_doc()
}
});
});
}
},
company(frm) {
frm.set_value('party', '');
frm.set_value('receivable_payable_account', '');
},
party_type(frm) {
frm.set_value('party', '');
},
party(frm) {
frm.set_value('receivable_payable_account', '');
if (!frm.doc.receivable_payable_account && frm.doc.party_type && frm.doc.party) {
return frappe.call({
method: "erpnext.accounts.party.get_party_account",
args: {
company: frm.doc.company,
party_type: frm.doc.party_type,
party: frm.doc.party
},
callback: (r) => {
if (!r.exc && r.message) {
frm.set_value("receivable_payable_account", r.message);
}
frm.refresh();
}
});
}
}
});

173
erpnext/accounts/doctype/process_payment_reconciliation/process_payment_reconciliation.json Normal file
View File

@ -0,0 +1,173 @@
{
"actions": [],
"autoname": "format:ACC-PPR-{#####}",
"beta": 1,
"creation": "2023-03-30 21:28:39.793927",
"default_view": "List",
"doctype": "DocType",
"editable_grid": 1,
"engine": "InnoDB",
"field_order": [
"company",
"party_type",
"column_break_io6c",
"party",
"receivable_payable_account",
"filter_section",
"from_invoice_date",
"to_invoice_date",
"column_break_kegk",
"from_payment_date",
"to_payment_date",
"column_break_uj04",
"cost_center",
"bank_cash_account",
"section_break_2n02",
"status",
"error_log",
"section_break_a8yx",
"amended_from"
],
"fields": [
{
"allow_on_submit": 1,
"fieldname": "status",
"fieldtype": "Select",
"label": "Status",
"options": "\nQueued\nRunning\nPaused\nCompleted\nPartially Reconciled\nFailed\nCancelled",
"read_only": 1
},
{
"fieldname": "company",
"fieldtype": "Link",
"in_list_view": 1,
"label": "Company",
"options": "Company",
"reqd": 1
},
{
"fieldname": "party_type",
"fieldtype": "Link",
"in_list_view": 1,
"label": "Party Type",
"options": "DocType",
"reqd": 1
},
{
"fieldname": "column_break_io6c",
"fieldtype": "Column Break"
},
{
"fieldname": "party",
"fieldtype": "Dynamic Link",
"in_list_view": 1,
"label": "Party",
"options": "party_type",
"reqd": 1
},
{
"fieldname": "receivable_payable_account",
"fieldtype": "Link",
"in_list_view": 1,
"label": "Receivable/Payable Account",
"options": "Account",
"reqd": 1
},
{
"fieldname": "filter_section",
"fieldtype": "Section Break",
"label": "Filters"
},
{
"fieldname": "from_invoice_date",
"fieldtype": "Date",
"label": "From Invoice Date"
},
{
"fieldname": "to_invoice_date",
"fieldtype": "Date",
"label": "To Invoice Date"
},
{
"fieldname": "column_break_kegk",
"fieldtype": "Column Break"
},
{
"fieldname": "from_payment_date",
"fieldtype": "Date",
"label": "From Payment Date"
},
{
"fieldname": "to_payment_date",
"fieldtype": "Date",
"label": "To Payment Date"
},
{
"fieldname": "column_break_uj04",
"fieldtype": "Column Break"
},
{
"fieldname": "cost_center",
"fieldtype": "Link",
"label": "Cost Center",
"options": "Cost Center"
},
{
"fieldname": "bank_cash_account",
"fieldtype": "Link",
"label": "Bank/Cash Account",
"options": "Account"
},
{
"fieldname": "section_break_2n02",
"fieldtype": "Section Break",
"label": "Status"
},
{
"depends_on": "eval:doc.error_log",
"fieldname": "error_log",
"fieldtype": "Long Text",
"label": "Error Log"
},
{
"fieldname": "amended_from",
"fieldtype": "Link",
"label": "Amended From",
"no_copy": 1,
"options": "Process Payment Reconciliation",
"print_hide": 1,
"read_only": 1
},
{
"fieldname": "section_break_a8yx",
"fieldtype": "Section Break"
}
],
"index_web_pages_for_search": 1,
"is_submittable": 1,
"links": [],
"modified": "2023-04-21 17:19:30.912953",
"modified_by": "Administrator",
"module": "Accounts",
"name": "Process Payment Reconciliation",
"naming_rule": "Expression",
"owner": "Administrator",
"permissions": [
{
"create": 1,
"delete": 1,
"email": 1,
"export": 1,
"print": 1,
"read": 1,
"report": 1,
"role": "System Manager",
"share": 1,
"write": 1
}
],
"sort_field": "modified",
"sort_order": "DESC",
"states": [],
"title_field": "company"
}

503
erpnext/accounts/doctype/process_payment_reconciliation/process_payment_reconciliation.py Normal file
View File

@ -0,0 +1,503 @@
# Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and contributors
# For license information, please see license.txt
import frappe
from frappe import _, qb
from frappe.model.document import Document
from frappe.utils import get_link_to_form
from frappe.utils.scheduler import is_scheduler_inactive
class ProcessPaymentReconciliation(Document):
def validate(self):
self.validate_receivable_payable_account()
self.validate_bank_cash_account()
def validate_receivable_payable_account(self):
if self.receivable_payable_account:
if self.company != frappe.db.get_value("Account", self.receivable_payable_account, "company"):
frappe.throw(
_("Receivable/Payable Account: {0} doesn't belong to company {1}").format(
frappe.bold(self.receivable_payable_account), frappe.bold(self.company)
)
)
def validate_bank_cash_account(self):
if self.bank_cash_account:
if self.company != frappe.db.get_value("Account", self.bank_cash_account, "company"):
frappe.throw(
_("Bank/Cash Account {0} doesn't belong to company {1}").format(
frappe.bold(self.bank_cash_account), frappe.bold(self.company)
)
)
def before_save(self):
self.status = ""
self.error_log = ""
def on_submit(self):
self.db_set("status", "Queued")
self.db_set("error_log", None)
def on_cancel(self):
self.db_set("status", "Cancelled")
log = frappe.db.get_value(
"Process Payment Reconciliation Log", filters={"process_pr": self.name}
)
if log:
frappe.db.set_value("Process Payment Reconciliation Log", log, "status", "Cancelled")
@frappe.whitelist()
def get_reconciled_count(docname: str | None = None) -> float:
current_status = {}
if docname:
reconcile_log = frappe.db.get_value(
"Process Payment Reconciliation Log", filters={"process_pr": docname}, fieldname="name"
)
if reconcile_log:
res = frappe.get_all(
"Process Payment Reconciliation Log",
filters={"name": reconcile_log},
fields=["reconciled_entries", "total_allocations"],
as_list=1,
)
current_status["processed"], current_status["total"] = res[0]
return current_status
def get_pr_instance(doc: str):
process_payment_reconciliation = frappe.get_doc("Process Payment Reconciliation", doc)
pr = frappe.get_doc("Payment Reconciliation")
fields = [
"company",
"party_type",
"party",
"receivable_payable_account",
"from_invoice_date",
"to_invoice_date",
"from_payment_date",
"to_payment_date",
]
d = {}
for field in fields:
d[field] = process_payment_reconciliation.get(field)
pr.update(d)
pr.invoice_limit = 1000
pr.payment_limit = 1000
return pr
def is_job_running(job_name: str) -> bool:
jobs = frappe.db.get_all("RQ Job", filters={"status": ["in", ["started", "queued"]]})
for x in jobs:
if x.job_name == job_name:
return True
return False
@frappe.whitelist()
def pause_job_for_doc(docname: str | None = None):
if docname:
frappe.db.set_value("Process Payment Reconciliation", docname, "status", "Paused")
log = frappe.db.get_value("Process Payment Reconciliation Log", filters={"process_pr": docname})
if log:
frappe.db.set_value("Process Payment Reconciliation Log", log, "status", "Paused")
@frappe.whitelist()
def trigger_job_for_doc(docname: str | None = None):
"""
Trigger background job
"""
if not docname:
return
if not frappe.db.get_single_value("Accounts Settings", "auto_reconcile_payments"):
frappe.throw(
_("Auto Reconciliation of Payments has been disabled. Enable it through {0}").format(
get_link_to_form("Accounts Settings", "Accounts Settings")
)
)
return
if not is_scheduler_inactive():
if frappe.db.get_value("Process Payment Reconciliation", docname, "status") == "Queued":
frappe.db.set_value("Process Payment Reconciliation", docname, "status", "Running")
job_name = f"start_processing_{docname}"
if not is_job_running(job_name):
job = frappe.enqueue(
method="erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation.reconcile_based_on_filters",
queue="long",
is_async=True,
job_name=job_name,
enqueue_after_commit=True,
doc=docname,
)
elif frappe.db.get_value("Process Payment Reconciliation", docname, "status") == "Paused":
frappe.db.set_value("Process Payment Reconciliation", docname, "status", "Running")
log = frappe.db.get_value("Process Payment Reconciliation Log", filters={"process_pr": docname})
if log:
frappe.db.set_value("Process Payment Reconciliation Log", log, "status", "Running")
# Resume tasks for running doc
job_name = f"start_processing_{docname}"
if not is_job_running(job_name):
job = frappe.enqueue(
method="erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation.reconcile_based_on_filters",
queue="long",
is_async=True,
job_name=job_name,
doc=docname,
)
else:
frappe.msgprint(_("Scheduler is Inactive. Can't trigger job now."))
def trigger_reconciliation_for_queued_docs():
"""
Will be called from Cron Job
Fetch queued docs and start reconciliation process for each one
"""
if not frappe.db.get_single_value("Accounts Settings", "auto_reconcile_payments"):
frappe.throw(
_("Auto Reconciliation of Payments has been disabled. Enable it through {0}").format(
get_link_to_form("Accounts Settings", "Accounts Settings")
)
)
return
if not is_scheduler_inactive():
# Get all queued documents
all_queued = frappe.db.get_all(
"Process Payment Reconciliation",
filters={"docstatus": 1, "status": "Queued"},
order_by="creation desc",
as_list=1,
)
docs_to_trigger = []
unique_filters = set()
queue_size = 5
fields = ["company", "party_type", "party", "receivable_payable_account"]
def get_filters_as_tuple(fields, doc):
filters = ()
for x in fields:
filters += tuple(doc.get(x))
return filters
for x in all_queued:
doc = frappe.get_doc("Process Payment Reconciliation", x)
filters = get_filters_as_tuple(fields, doc)
if filters not in unique_filters:
unique_filters.add(filters)
docs_to_trigger.append(doc.name)
if len(docs_to_trigger) == queue_size:
break
# trigger reconcilation process for queue_size unique filters
for doc in docs_to_trigger:
trigger_job_for_doc(doc)
else:
frappe.msgprint(_("Scheduler is Inactive. Can't trigger jobs now."))
def reconcile_based_on_filters(doc: None | str = None) -> None:
"""
Identify current state of document and execute next tasks in background
"""
if doc:
log = frappe.db.get_value("Process Payment Reconciliation Log", filters={"process_pr": doc})
if not log:
log = frappe.new_doc("Process Payment Reconciliation Log")
log.process_pr = doc
log.status = "Running"
log = log.save()
job_name = f"process_{doc}_fetch_and_allocate"
if not is_job_running(job_name):
job = frappe.enqueue(
method="erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation.fetch_and_allocate",
queue="long",
timeout="3600",
is_async=True,
job_name=job_name,
enqueue_after_commit=True,
doc=doc,
)
else:
res = frappe.get_all(
"Process Payment Reconciliation Log",
filters={"name": log},
fields=["allocated", "reconciled"],
as_list=1,
)
allocated, reconciled = res[0]
if not allocated:
job_name = f"process__{doc}_fetch_and_allocate"
if not is_job_running(job_name):
job = frappe.enqueue(
method="erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation.fetch_and_allocate",
queue="long",
timeout="3600",
is_async=True,
job_name=job_name,
enqueue_after_commit=True,
doc=doc,
)
elif not reconciled:
allocation = get_next_allocation(log)
if allocation:
reconcile_job_name = (
f"process_{doc}_reconcile_allocation_{allocation[0].idx}_{allocation[-1].idx}"
)
else:
reconcile_job_name = f"process_{doc}_reconcile"
if not is_job_running(reconcile_job_name):
job = frappe.enqueue(
method="erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation.reconcile",
queue="long",
timeout="3600",
is_async=True,
job_name=reconcile_job_name,
enqueue_after_commit=True,
doc=doc,
)
elif reconciled:
frappe.db.set_value("Process Payment Reconciliation", doc, "status", "Completed")
def get_next_allocation(log: str) -> list:
if log:
allocations = []
next = frappe.db.get_all(
"Process Payment Reconciliation Log Allocations",
filters={"parent": log, "reconciled": 0},
fields=["reference_type", "reference_name"],
order_by="idx",
limit=1,
)
if next:
allocations = frappe.db.get_all(
"Process Payment Reconciliation Log Allocations",
filters={
"parent": log,
"reconciled": 0,
"reference_type": next[0].reference_type,
"reference_name": next[0].reference_name,
},
fields=["*"],
order_by="idx",
)
return allocations
return []
def fetch_and_allocate(doc: str) -> None:
"""
Fetch Invoices and Payments based on filters applied. FIFO ordering is used for allocation.
"""
if doc:
log = frappe.db.get_value("Process Payment Reconciliation Log", filters={"process_pr": doc})
if log:
if not frappe.db.get_value("Process Payment Reconciliation Log", log, "allocated"):
reconcile_log = frappe.get_doc("Process Payment Reconciliation Log", log)
pr = get_pr_instance(doc)
pr.get_unreconciled_entries()
if len(pr.invoices) > 0 and len(pr.payments) > 0:
invoices = [x.as_dict() for x in pr.invoices]
payments = [x.as_dict() for x in pr.payments]
pr.allocate_entries(frappe._dict({"invoices": invoices, "payments": payments}))
for x in pr.get("allocation"):
reconcile_log.append(
"allocations",
x.as_dict().update(
{
"parenttype": "Process Payment Reconciliation Log",
"parent": reconcile_log.name,
"name": None,
"reconciled": False,
}
),
)
reconcile_log.allocated = True
reconcile_log.total_allocations = len(reconcile_log.get("allocations"))
reconcile_log.reconciled_entries = 0
reconcile_log.save()
# generate reconcile job name
allocation = get_next_allocation(log)
if allocation:
reconcile_job_name = (
f"process_{doc}_reconcile_allocation_{allocation[0].idx}_{allocation[-1].idx}"
)
else:
reconcile_job_name = f"process_{doc}_reconcile"
if not is_job_running(reconcile_job_name):
job = frappe.enqueue(
method="erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation.reconcile",
queue="long",
timeout="3600",
is_async=True,
job_name=reconcile_job_name,
enqueue_after_commit=True,
doc=doc,
)
def reconcile(doc: None | str = None) -> None:
if doc:
log = frappe.db.get_value("Process Payment Reconciliation Log", filters={"process_pr": doc})
if log:
res = frappe.get_all(
"Process Payment Reconciliation Log",
filters={"name": log},
fields=["reconciled_entries", "total_allocations"],
as_list=1,
limit=1,
)
reconciled_entries, total_allocations = res[0]
if reconciled_entries != total_allocations:
try:
# Fetch next allocation
allocations = get_next_allocation(log)
pr = get_pr_instance(doc)
# pass allocation to PR instance
for x in allocations:
pr.append("allocation", x)
# reconcile
pr.reconcile_allocations(skip_ref_details_update_for_pe=True)
# If Payment Entry, update details only for newly linked references
# This is for performance
if allocations[0].reference_type == "Payment Entry":
references = [(x.invoice_type, x.invoice_number) for x in allocations]
pe = frappe.get_doc(allocations[0].reference_type, allocations[0].reference_name)
pe.flags.ignore_validate_update_after_submit = True
pe.set_missing_ref_details(update_ref_details_only_for=references)
pe.save()
# Update reconciled flag
allocation_names = [x.name for x in allocations]
ppa = qb.DocType("Process Payment Reconciliation Log Allocations")
qb.update(ppa).set(ppa.reconciled, True).where(ppa.name.isin(allocation_names)).run()
# Update reconciled count
reconciled_count = frappe.db.count(
"Process Payment Reconciliation Log Allocations", filters={"parent": log, "reconciled": True}
)
frappe.db.set_value(
"Process Payment Reconciliation Log", log, "reconciled_entries", reconciled_count
)
except Exception as err:
# Update the parent doc about the exception
frappe.db.rollback()
traceback = frappe.get_traceback()
if traceback:
message = "Traceback: <br>" + traceback
frappe.db.set_value("Process Payment Reconciliation Log", log, "error_log", message)
frappe.db.set_value(
"Process Payment Reconciliation",
doc,
"error_log",
message,
)
if reconciled_entries and total_allocations and reconciled_entries < total_allocations:
frappe.db.set_value(
"Process Payment Reconciliation Log", log, "status", "Partially Reconciled"
)
frappe.db.set_value(
"Process Payment Reconciliation",
doc,
"status",
"Partially Reconciled",
)
else:
frappe.db.set_value("Process Payment Reconciliation Log", log, "status", "Failed")
frappe.db.set_value(
"Process Payment Reconciliation",
doc,
"status",
"Failed",
)
finally:
if reconciled_entries == total_allocations:
frappe.db.set_value("Process Payment Reconciliation Log", log, "status", "Reconciled")
frappe.db.set_value("Process Payment Reconciliation Log", log, "reconciled", True)
frappe.db.set_value("Process Payment Reconciliation", doc, "status", "Completed")
else:
if not (frappe.db.get_value("Process Payment Reconciliation", doc, "status") == "Paused"):
# trigger next batch in job
# generate reconcile job name
allocation = get_next_allocation(log)
if allocation:
reconcile_job_name = (
f"process_{doc}_reconcile_allocation_{allocation[0].idx}_{allocation[-1].idx}"
)
else:
reconcile_job_name = f"process_{doc}_reconcile"
if not is_job_running(reconcile_job_name):
job = frappe.enqueue(
method="erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation.reconcile",
queue="long",
timeout="3600",
is_async=True,
job_name=reconcile_job_name,
enqueue_after_commit=True,
doc=doc,
)
else:
frappe.db.set_value("Process Payment Reconciliation Log", log, "status", "Reconciled")
frappe.db.set_value("Process Payment Reconciliation Log", log, "reconciled", True)
frappe.db.set_value("Process Payment Reconciliation", doc, "status", "Completed")
@frappe.whitelist()
def is_any_doc_running(for_filter: str | dict | None = None) -> str | None:
running_doc = None
if for_filter:
if type(for_filter) == str:
for_filter = frappe.json.loads(for_filter)
running_doc = frappe.db.get_value(
"Process Payment Reconciliation",
filters={
"docstatus": 1,
"status": ["in", ["Running", "Paused"]],
"company": for_filter.get("company"),
"party_type": for_filter.get("party_type"),
"party": for_filter.get("party"),
"receivable_payable_account": for_filter.get("receivable_payable_account"),
},
fieldname="name",
)
else:
running_doc = frappe.db.get_value(
"Process Payment Reconciliation", filters={"docstatus": 1, "status": "Running"}
)
return running_doc

15
erpnext/accounts/doctype/process_payment_reconciliation/process_payment_reconciliation_dashboard.py Normal file
View File

@ -0,0 +1,15 @@
from frappe import _
def get_data():
return {
"fieldname": "process_pr",
"transactions": [
{
"label": _("Reconciliation Logs"),
"items": [
"Process Payment Reconciliation Log",
],
},
],
}

15
erpnext/accounts/doctype/process_payment_reconciliation/process_payment_reconciliation_list.js Normal file
View File

@ -0,0 +1,15 @@
frappe.listview_settings['Process Payment Reconciliation'] = {
add_fields: ["status"],
get_indicator: function(doc) {
let colors = {
'Queued': 'orange',
'Paused': 'orange',
'Completed': 'green',
'Partially Reconciled': 'orange',
'Running': 'blue',
'Failed': 'red',
};
let status = doc.status;
return [__(status), colors[status], 'status,=,'+status];
},
};

9
erpnext/accounts/doctype/process_payment_reconciliation/test_process_payment_reconciliation.py Normal file
View File

@ -0,0 +1,9 @@
# Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and Contributors
# See license.txt
# import frappe
from frappe.tests.utils import FrappeTestCase
class TestProcessPaymentReconciliation(FrappeTestCase):
pass

0
erpnext/accounts/doctype/process_payment_reconciliation_log/__init__.py Normal file
View File

17
erpnext/accounts/doctype/process_payment_reconciliation_log/process_payment_reconciliation_log.js Normal file
View File

@ -0,0 +1,17 @@
// Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and contributors
// For license information, please see license.txt
frappe.ui.form.on("Process Payment Reconciliation Log", {
refresh(frm) {
if (['Completed', 'Running', 'Paused', 'Partially Reconciled'].find(x => x == frm.doc.status)) {
let progress = 0;
if (frm.doc.reconciled_entries != 0) {
progress = frm.doc.reconciled_entries / frm.doc.total_allocations * 100;
} else if(frm.doc.total_allocations == 0 && frm.doc.status == "Completed"){
progress = 100;
}
frm.dashboard.add_progress(__('Reconciliation Progress'), progress);
}
},
});

137
erpnext/accounts/doctype/process_payment_reconciliation_log/process_payment_reconciliation_log.json Normal file
View File

@ -0,0 +1,137 @@
{
"actions": [],
"autoname": "format:PPR-LOG-{##}",
"beta": 1,
"creation": "2023-03-13 15:00:09.149681",
"default_view": "List",
"doctype": "DocType",
"editable_grid": 1,
"engine": "InnoDB",
"field_order": [
"process_pr",
"section_break_fvdw",
"status",
"tasks_section",
"allocated",
"reconciled",
"column_break_yhin",
"total_allocations",
"reconciled_entries",
"section_break_4ywv",
"error_log",
"allocations_section",
"allocations"
],
"fields": [
{
"fieldname": "allocations",
"fieldtype": "Table",
"label": "Allocations",
"options": "Process Payment Reconciliation Log Allocations",
"read_only": 1
},
{
"default": "0",
"description": "All allocations have been successfully reconciled",
"fieldname": "reconciled",
"fieldtype": "Check",
"label": "Reconciled",
"read_only": 1
},
{
"fieldname": "total_allocations",
"fieldtype": "Int",
"in_list_view": 1,
"label": "Total Allocations",
"read_only": 1
},
{
"default": "0",
"description": "Invoices and Payments have been Fetched and Allocated",
"fieldname": "allocated",
"fieldtype": "Check",
"label": "Allocated",
"read_only": 1
},
{
"fieldname": "reconciled_entries",
"fieldtype": "Int",
"in_list_view": 1,
"label": "Reconciled Entries",
"read_only": 1
},
{
"fieldname": "tasks_section",
"fieldtype": "Section Break",
"label": "Tasks"
},
{
"fieldname": "allocations_section",
"fieldtype": "Section Break",
"label": "Allocations"
},
{
"fieldname": "column_break_yhin",
"fieldtype": "Column Break"
},
{
"fieldname": "section_break_4ywv",
"fieldtype": "Section Break"
},
{
"depends_on": "eval:doc.error_log",
"fieldname": "error_log",
"fieldtype": "Long Text",
"label": "Reconciliation Error Log",
"read_only": 1
},
{
"fieldname": "process_pr",
"fieldtype": "Link",
"in_list_view": 1,
"label": "Parent Document",
"options": "Process Payment Reconciliation",
"read_only": 1,
"reqd": 1
},
{
"fieldname": "section_break_fvdw",
"fieldtype": "Section Break",
"label": "Status"
},
{
"fieldname": "status",
"fieldtype": "Select",
"label": "Status",
"options": "Running\nPaused\nReconciled\nPartially Reconciled\nFailed\nCancelled",
"read_only": 1
}
],
"in_create": 1,
"index_web_pages_for_search": 1,
"links": [],
"modified": "2023-04-21 17:36:26.642617",
"modified_by": "Administrator",
"module": "Accounts",
"name": "Process Payment Reconciliation Log",
"naming_rule": "Expression",
"owner": "Administrator",
"permissions": [
{
"create": 1,
"delete": 1,
"email": 1,
"export": 1,
"print": 1,
"read": 1,
"report": 1,
"role": "System Manager",
"share": 1,
"write": 1
}
],
"search_fields": "allocated, reconciled, total_allocations, reconciled_entries",
"sort_field": "modified",
"sort_order": "DESC",
"states": []
}

9
erpnext/accounts/doctype/process_payment_reconciliation_log/process_payment_reconciliation_log.py Normal file
View File

@ -0,0 +1,9 @@
# Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and contributors
# For license information, please see license.txt
# import frappe
from frappe.model.document import Document
class ProcessPaymentReconciliationLog(Document):
pass

15
erpnext/accounts/doctype/process_payment_reconciliation_log/process_payment_reconciliation_log_list.js Normal file
View File

@ -0,0 +1,15 @@
frappe.listview_settings['Process Payment Reconciliation Log'] = {
add_fields: ["status"],
get_indicator: function(doc) {
var colors = {
'Partially Reconciled': 'orange',
'Paused': 'orange',
'Reconciled': 'green',
'Failed': 'red',
'Cancelled': 'red',
'Running': 'blue',
};
let status = doc.status;
return [__(status), colors[status], "status,=,"+status];
},
};

9
erpnext/accounts/doctype/process_payment_reconciliation_log/test_process_payment_reconciliation_log.py Normal file
View File

@ -0,0 +1,9 @@
# Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and Contributors
# See license.txt
# import frappe
from frappe.tests.utils import FrappeTestCase
class TestProcessPaymentReconciliationLog(FrappeTestCase):
pass

0
erpnext/accounts/doctype/process_payment_reconciliation_log_allocations/__init__.py Normal file
View File

170
erpnext/accounts/doctype/process_payment_reconciliation_log_allocations/process_payment_reconciliation_log_allocations.json Normal file
View File

@ -0,0 +1,170 @@
{
"actions": [],
"creation": "2023-03-13 13:51:27.351463",
"default_view": "List",
"doctype": "DocType",
"editable_grid": 1,
"engine": "InnoDB",
"field_order": [
"reference_type",
"reference_name",
"reference_row",
"column_break_3",
"invoice_type",
"invoice_number",
"section_break_6",
"allocated_amount",
"unreconciled_amount",
"column_break_8",
"amount",
"is_advance",
"section_break_5",
"difference_amount",
"column_break_7",
"difference_account",
"exchange_rate",
"currency",
"reconciled"
],
"fields": [
{
"fieldname": "reference_type",
"fieldtype": "Link",
"label": "Reference Type",
"options": "DocType",
"read_only": 1,
"reqd": 1
},
{
"fieldname": "reference_name",
"fieldtype": "Dynamic Link",
"in_list_view": 1,
"label": "Reference Name",
"options": "reference_type",
"read_only": 1,
"reqd": 1
},
{
"fieldname": "reference_row",
"fieldtype": "Data",
"hidden": 1,
"label": "Reference Row",
"read_only": 1
},
{
"fieldname": "column_break_3",
"fieldtype": "Column Break"
},
{
"fieldname": "invoice_type",
"fieldtype": "Link",
"label": "Invoice Type",
"options": "DocType",
"read_only": 1,
"reqd": 1
},
{
"fieldname": "invoice_number",
"fieldtype": "Dynamic Link",
"in_list_view": 1,
"label": "Invoice Number",
"options": "invoice_type",
"read_only": 1,
"reqd": 1
},
{
"fieldname": "section_break_6",
"fieldtype": "Section Break"
},
{
"fieldname": "allocated_amount",
"fieldtype": "Currency",
"in_list_view": 1,
"label": "Allocated Amount",
"options": "currency",
"reqd": 1
},
{
"fieldname": "unreconciled_amount",
"fieldtype": "Currency",
"hidden": 1,
"label": "Unreconciled Amount",
"options": "currency",
"read_only": 1
},
{
"fieldname": "column_break_8",
"fieldtype": "Column Break"
},
{
"fieldname": "amount",
"fieldtype": "Currency",
"hidden": 1,
"label": "Amount",
"options": "currency",
"read_only": 1
},
{
"fieldname": "is_advance",
"fieldtype": "Data",
"hidden": 1,
"label": "Is Advance",
"read_only": 1
},
{
"fieldname": "section_break_5",
"fieldtype": "Section Break"
},
{
"fieldname": "difference_amount",
"fieldtype": "Currency",
"in_list_view": 1,
"label": "Difference Amount",
"options": "Currency",
"read_only": 1
},
{
"fieldname": "column_break_7",
"fieldtype": "Column Break"
},
{
"fieldname": "difference_account",
"fieldtype": "Link",
"label": "Difference Account",
"options": "Account",
"read_only": 1
},
{
"fieldname": "exchange_rate",
"fieldtype": "Float",
"label": "Exchange Rate",
"read_only": 1
},
{
"fieldname": "currency",
"fieldtype": "Link",
"hidden": 1,
"label": "Currency",
"options": "Currency"
},
{
"default": "0",
"fieldname": "reconciled",
"fieldtype": "Check",
"in_list_view": 1,
"label": "Reconciled"
}
],
"istable": 1,
"links": [],
"modified": "2023-03-20 21:05:43.121945",
"modified_by": "Administrator",
"module": "Accounts",
"name": "Process Payment Reconciliation Log Allocations",
"owner": "Administrator",
"permissions": [],
"sort_field": "modified",
"sort_order": "DESC",
"states": [],
"track_changes": 1
}

9
erpnext/accounts/doctype/process_payment_reconciliation_log_allocations/process_payment_reconciliation_log_allocations.py Normal file
View File

@ -0,0 +1,9 @@
# Copyright (c) 2023, Frappe Technologies Pvt. Ltd. and contributors
# For license information, please see license.txt
# import frappe
from frappe.model.document import Document
class ProcessPaymentReconciliationLogAllocations(Document):
pass

13
erpnext/accounts/utils.py
View File

@ -436,7 +436,7 @@ def add_cc(args=None):
return cc.name
def reconcile_against_document(args): # nosemgrep
def reconcile_against_document(args, skip_ref_details_update_for_pe=False): # nosemgrep
"""
Cancel PE or JV, Update against document, split if required and resubmit
"""
@ -465,7 +465,9 @@ def reconcile_against_document(args): # nosemgrep
if voucher_type == "Journal Entry":
update_reference_in_journal_entry(entry, doc, do_not_save=True)
else:
update_reference_in_payment_entry(entry, doc, do_not_save=True)
update_reference_in_payment_entry(
entry, doc, do_not_save=True, skip_ref_details_update_for_pe=skip_ref_details_update_for_pe
)
doc.save(ignore_permissions=True)
# re-submit advance entry
@ -602,7 +604,9 @@ def update_reference_in_journal_entry(d, journal_entry, do_not_save=False):
journal_entry.save(ignore_permissions=True)
def update_reference_in_payment_entry(d, payment_entry, do_not_save=False):
def update_reference_in_payment_entry(
d, payment_entry, do_not_save=False, skip_ref_details_update_for_pe=False
):
reference_details = {
"reference_doctype": d.against_voucher_type,
"reference_name": d.against_voucher,
@ -646,7 +650,8 @@ def update_reference_in_payment_entry(d, payment_entry, do_not_save=False):
payment_entry.flags.ignore_validate_update_after_submit = True
payment_entry.setup_party_account_field()
payment_entry.set_missing_values()
payment_entry.set_missing_ref_details()
if not skip_ref_details_update_for_pe:
payment_entry.set_missing_ref_details()
payment_entry.set_amounts()
if not do_not_save:

1
erpnext/hooks.py
View File

@ -362,6 +362,7 @@ scheduler_events = {
"cron": {
"0/15 * * * *": [
"erpnext.manufacturing.doctype.bom_update_log.bom_update_log.resume_bom_cost_update_jobs",
"erpnext.accounts.doctype.process_payment_reconciliation.process_payment_reconciliation.trigger_reconciliation_for_queued_docs",
],
"0/30 * * * *": [
"erpnext.utilities.doctype.video.video.update_youtube_data",