From d7b738ff61e0ebab62ff2c15247f90b79401e9a4 Mon Sep 17 00:00:00 2001 From: Nabin Hait Date: Tue, 27 Feb 2024 14:40:15 +0530 Subject: [PATCH 01/14] perf: Optimization for providional gl entries --- .../purchase_invoice/purchase_invoice.py | 55 ++++++++++--------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/erpnext/accounts/doctype/purchase_invoice/purchase_invoice.py b/erpnext/accounts/doctype/purchase_invoice/purchase_invoice.py index cfaaf7677869..35652b71c4de 100644 --- a/erpnext/accounts/doctype/purchase_invoice/purchase_invoice.py +++ b/erpnext/accounts/doctype/purchase_invoice/purchase_invoice.py @@ -720,7 +720,7 @@ def make_item_gl_entries(self, gl_entries): "Company", self.company, "enable_provisional_accounting_for_non_stock_items" ) ) - + provisional_enpenses_booked_in_pr = False purchase_receipt_doc_map = {} for item in self.get("items"): @@ -859,34 +859,37 @@ def make_item_gl_entries(self, gl_entries): if provisional_accounting_for_non_stock_items: if item.purchase_receipt: - provisional_account, pr_qty, pr_base_rate = frappe.get_cached_value( - "Purchase Receipt Item", - item.pr_detail, - ["provisional_expense_account", "qty", "base_rate"], - ) - provisional_account = provisional_account or self.get_company_default( - "default_provisional_account" - ) - purchase_receipt_doc = purchase_receipt_doc_map.get(item.purchase_receipt) + if not provisional_enpenses_booked_in_pr: + provisional_account, pr_qty, pr_base_rate = frappe.get_cached_value( + "Purchase Receipt Item", + item.pr_detail, + ["provisional_expense_account", "qty", "base_rate"], + ) + provisional_account = provisional_account or self.get_company_default( + "default_provisional_account" + ) + # Post reverse entry for Stock-Received-But-Not-Billed if it is booked in Purchase Receipt + provision_gle_against_pr = frappe.db.get_value( + "GL Entry", + { + "is_cancelled": 0, + "voucher_type": "Purchase Receipt", + "voucher_no": item.purchase_receipt, + "voucher_detail_no": item.pr_detail, + "account": provisional_account, + }, + ["name"], + ) + if provision_gle_against_pr: + provisional_enpenses_booked_in_pr = True - if not purchase_receipt_doc: - purchase_receipt_doc = frappe.get_doc("Purchase Receipt", item.purchase_receipt) - purchase_receipt_doc_map[item.purchase_receipt] = purchase_receipt_doc + if provisional_enpenses_booked_in_pr: + purchase_receipt_doc = purchase_receipt_doc_map.get(item.purchase_receipt) - # Post reverse entry for Stock-Received-But-Not-Billed if it is booked in Purchase Receipt - expense_booked_in_pr = frappe.db.get_value( - "GL Entry", - { - "is_cancelled": 0, - "voucher_type": "Purchase Receipt", - "voucher_no": item.purchase_receipt, - "voucher_detail_no": item.pr_detail, - "account": provisional_account, - }, - "name", - ) + if not purchase_receipt_doc: + purchase_receipt_doc = frappe.get_doc("Purchase Receipt", item.purchase_receipt) + purchase_receipt_doc_map[item.purchase_receipt] = purchase_receipt_doc - if expense_booked_in_pr: # Intentionally passing purchase invoice item to handle partial billing purchase_receipt_doc.add_provisional_gl_entry( item, From f204d810bba5a6fdb3137b780489907c30b650fc Mon Sep 17 00:00:00 2001 From: Nabin Hait Date: Mon, 4 Mar 2024 15:46:15 +0530 Subject: [PATCH 02/14] perf: Performance optimization for validating budget --- erpnext/accounts/doctype/budget/budget.py | 6 +- .../budget_account/budget_account.json | 124 +++++------------- 2 files changed, 40 insertions(+), 90 deletions(-) diff --git a/erpnext/accounts/doctype/budget/budget.py b/erpnext/accounts/doctype/budget/budget.py index 6cfd15d3ec82..2f3b9a4784e3 100644 --- a/erpnext/accounts/doctype/budget/budget.py +++ b/erpnext/accounts/doctype/budget/budget.py @@ -109,6 +109,8 @@ def before_naming(self): def validate_expense_against_budget(args, expense_amount=0): args = frappe._dict(args) + if not frappe.get_all("Budget", limit=1): + return if args.get("company") and not args.fiscal_year: args.fiscal_year = get_fiscal_year(args.get("posting_date"), company=args.get("company"))[0] @@ -142,13 +144,13 @@ def validate_expense_against_budget(args, expense_amount=0): if ( args.get(budget_against) and args.account - and frappe.db.get_value("Account", {"name": args.account, "root_type": "Expense"}) + and (frappe.get_cached_value("Account", args.account, "root_type") == "Expense") ): doctype = dimension.get("document_type") if frappe.get_cached_value("DocType", doctype, "is_tree"): - lft, rgt = frappe.db.get_value(doctype, args.get(budget_against), ["lft", "rgt"]) + lft, rgt = frappe.get_cached_value(doctype, args.get(budget_against), ["lft", "rgt"]) condition = """and exists(select name from `tab%s` where lft<=%s and rgt>=%s and name=b.%s)""" % ( doctype, diff --git a/erpnext/accounts/doctype/budget_account/budget_account.json b/erpnext/accounts/doctype/budget_account/budget_account.json index ead07614a7f6..c7d872647f15 100644 --- a/erpnext/accounts/doctype/budget_account/budget_account.json +++ b/erpnext/accounts/doctype/budget_account/budget_account.json @@ -1,94 +1,42 @@ { - "allow_copy": 0, - "allow_import": 0, - "allow_rename": 0, - "beta": 0, - "creation": "2016-05-16 11:54:09.286135", - "custom": 0, - "docstatus": 0, - "doctype": "DocType", - "document_type": "", - "editable_grid": 1, - "engine": "InnoDB", + "actions": [], + "creation": "2016-05-16 11:54:09.286135", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "account", + "budget_amount" + ], "fields": [ { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "account", - "fieldtype": "Link", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 1, - "label": "Account", - "length": 0, - "no_copy": 0, - "options": "Account", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 - }, + "fieldname": "account", + "fieldtype": "Link", + "in_list_view": 1, + "label": "Account", + "options": "Account", + "reqd": 1, + "search_index": 1 + }, { - "allow_on_submit": 0, - "bold": 0, - "collapsible": 0, - "columns": 0, - "fieldname": "budget_amount", - "fieldtype": "Currency", - "hidden": 0, - "ignore_user_permissions": 0, - "ignore_xss_filter": 0, - "in_filter": 0, - "in_list_view": 1, - "label": "Budget Amount", - "length": 0, - "no_copy": 0, - "options": "Company:company:default_currency", - "permlevel": 0, - "precision": "", - "print_hide": 0, - "print_hide_if_no_value": 0, - "read_only": 0, - "remember_last_selected_value": 0, - "report_hide": 0, - "reqd": 1, - "search_index": 0, - "set_only_once": 0, - "unique": 0 + "fieldname": "budget_amount", + "fieldtype": "Currency", + "in_list_view": 1, + "label": "Budget Amount", + "options": "Company:company:default_currency", + "reqd": 1 } - ], - "hide_heading": 0, - "hide_toolbar": 0, - "idx": 0, - "image_view": 0, - "in_create": 0, - - "is_submittable": 0, - "issingle": 0, - "istable": 1, - "max_attachments": 0, - "modified": "2017-01-02 17:02:53.339420", - "modified_by": "Administrator", - "module": "Accounts", - "name": "Budget Account", - "name_case": "", - "owner": "Administrator", - "permissions": [], - "quick_entry": 1, - "read_only": 0, - "read_only_onload": 0, - "sort_field": "modified", - "sort_order": "DESC", - "track_seen": 0 + ], + "istable": 1, + "links": [], + "modified": "2024-03-04 15:43:27.016947", + "modified_by": "Administrator", + "module": "Accounts", + "name": "Budget Account", + "owner": "Administrator", + "permissions": [], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "states": [] } \ No newline at end of file From 8cd8b8f885419fac8297b171e90be7657357a0e3 Mon Sep 17 00:00:00 2001 From: Nabin Hait Date: Mon, 4 Mar 2024 15:47:09 +0530 Subject: [PATCH 03/14] perf: Cached accounting dimensions details --- .../accounting_dimension/accounting_dimension.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/erpnext/accounts/doctype/accounting_dimension/accounting_dimension.py b/erpnext/accounts/doctype/accounting_dimension/accounting_dimension.py index 8afd313322ec..52dda51708fd 100644 --- a/erpnext/accounts/doctype/accounting_dimension/accounting_dimension.py +++ b/erpnext/accounts/doctype/accounting_dimension/accounting_dimension.py @@ -236,14 +236,15 @@ def get_accounting_dimensions(as_list=True, filters=None): def get_checks_for_pl_and_bs_accounts(): - dimensions = frappe.db.sql( - """SELECT p.label, p.disabled, p.fieldname, c.default_dimension, c.company, c.mandatory_for_pl, c.mandatory_for_bs - FROM `tabAccounting Dimension`p ,`tabAccounting Dimension Detail` c - WHERE p.name = c.parent""", - as_dict=1, - ) + if frappe.flags.accounting_dimensions_details is None: + frappe.flags.accounting_dimensions_details = frappe.db.sql( + """SELECT p.label, p.disabled, p.fieldname, c.default_dimension, c.company, c.mandatory_for_pl, c.mandatory_for_bs + FROM `tabAccounting Dimension`p ,`tabAccounting Dimension Detail` c + WHERE p.name = c.parent""", + as_dict=1, + ) - return dimensions + return frappe.flags.accounting_dimensions_details def get_dimension_with_children(doctype, dimensions): From aa75a6014264853668acd15e996e8fd0af8e7ebe Mon Sep 17 00:00:00 2001 From: Nabin Hait Date: Mon, 4 Mar 2024 15:47:56 +0530 Subject: [PATCH 04/14] perf: Optimzed code for merging similar gl entries --- erpnext/accounts/doctype/gl_entry/gl_entry.py | 4 +- erpnext/accounts/general_ledger.py | 41 +++++++++---------- 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/erpnext/accounts/doctype/gl_entry/gl_entry.py b/erpnext/accounts/doctype/gl_entry/gl_entry.py index 6e07b0ec430c..c0f6502910e3 100644 --- a/erpnext/accounts/doctype/gl_entry/gl_entry.py +++ b/erpnext/accounts/doctype/gl_entry/gl_entry.py @@ -127,7 +127,7 @@ def pl_must_have_cost_center(self): frappe.throw(msg, title=_("Missing Cost Center")) def validate_dimensions_for_pl_and_bs(self): - account_type = frappe.db.get_value("Account", self.account, "report_type") + account_type = frappe.get_cached_value("Account", self.account, "report_type") for dimension in get_checks_for_pl_and_bs_accounts(): if ( @@ -252,7 +252,7 @@ def on_cancel(self): def validate_balance_type(account, adv_adj=False): if not adv_adj and account: - balance_must_be = frappe.db.get_value("Account", account, "balance_must_be") + balance_must_be = frappe.get_cached_value("Account", account, "balance_must_be") if balance_must_be: balance = frappe.db.sql( """select sum(debit) - sum(credit) diff --git a/erpnext/accounts/general_ledger.py b/erpnext/accounts/general_ledger.py index 38fcd976ad4a..b1223b271f57 100644 --- a/erpnext/accounts/general_ledger.py +++ b/erpnext/accounts/general_ledger.py @@ -234,11 +234,13 @@ def get_cost_center_allocation_data(company, posting_date): def merge_similar_entries(gl_map, precision=None): merged_gl_map = [] accounting_dimensions = get_accounting_dimensions() + merge_properties = get_merge_properties(accounting_dimensions) for entry in gl_map: + entry.merge_key = get_merge_key(entry, merge_properties) # if there is already an entry in this account then just add it # to that entry - same_head = check_if_in_list(entry, merged_gl_map, accounting_dimensions) + same_head = check_if_in_list(entry, merged_gl_map) if same_head: same_head.debit = flt(same_head.debit) + flt(entry.debit) same_head.debit_in_account_currency = flt(same_head.debit_in_account_currency) + flt( @@ -272,37 +274,34 @@ def merge_similar_entries(gl_map, precision=None): return merged_gl_map - -def check_if_in_list(gle, gl_map, dimensions=None): - account_head_fieldnames = [ - "voucher_detail_no", +def get_merge_properties(dimensions=None): + merge_properties = [ + "account", + "cost_center", "party", + "party_type", + "voucher_detail_no", "against_voucher", - "cost_center", "against_voucher_type", - "party_type", "project", "finance_book", ] - if dimensions: - account_head_fieldnames = account_head_fieldnames + dimensions - - for e in gl_map: - same_head = True - if e.account != gle.account: - same_head = False - continue + merge_properties.extend(dimensions) + return merge_properties - for fieldname in account_head_fieldnames: - if cstr(e.get(fieldname)) != cstr(gle.get(fieldname)): - same_head = False - break +def get_merge_key(entry, merge_properties): + merge_key = [] + for fieldname in merge_properties: + merge_key.append(entry.get(fieldname, '')) + + return tuple(merge_key) - if same_head: +def check_if_in_list(gle, gl_map): + for e in gl_map: + if e.merge_key == gle.merge_key: return e - def toggle_debit_credit_if_negative(gl_map): for entry in gl_map: # toggle debit, credit if negative entry From acc0b2faf82c2b352cc2caf14ef26e6e55f34230 Mon Sep 17 00:00:00 2001 From: Nabin Hait Date: Wed, 6 Mar 2024 17:43:27 +0530 Subject: [PATCH 05/14] fix: linter issues --- erpnext/accounts/general_ledger.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/erpnext/accounts/general_ledger.py b/erpnext/accounts/general_ledger.py index b1223b271f57..432abc250b12 100644 --- a/erpnext/accounts/general_ledger.py +++ b/erpnext/accounts/general_ledger.py @@ -7,7 +7,7 @@ import frappe from frappe import _ from frappe.model.meta import get_field_precision -from frappe.utils import cint, cstr, flt, formatdate, getdate, now +from frappe.utils import cint, flt, formatdate, getdate, now import erpnext from erpnext.accounts.doctype.accounting_dimension.accounting_dimension import ( @@ -274,6 +274,7 @@ def merge_similar_entries(gl_map, precision=None): return merged_gl_map + def get_merge_properties(dimensions=None): merge_properties = [ "account", @@ -290,18 +291,21 @@ def get_merge_properties(dimensions=None): merge_properties.extend(dimensions) return merge_properties + def get_merge_key(entry, merge_properties): merge_key = [] for fieldname in merge_properties: - merge_key.append(entry.get(fieldname, '')) - + merge_key.append(entry.get(fieldname, "")) + return tuple(merge_key) + def check_if_in_list(gle, gl_map): for e in gl_map: if e.merge_key == gle.merge_key: return e + def toggle_debit_credit_if_negative(gl_map): for entry in gl_map: # toggle debit, credit if negative entry From e4bd1738752896e9a50231561c8ca5f3a2ee15c5 Mon Sep 17 00:00:00 2001 From: Nabin Hait Date: Wed, 6 Mar 2024 20:32:47 +0530 Subject: [PATCH 06/14] perf: Cache accounting dimension filter map --- .../accounting_dimension_filter.py | 57 ++++++++++--------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/erpnext/accounts/doctype/accounting_dimension_filter/accounting_dimension_filter.py b/erpnext/accounts/doctype/accounting_dimension_filter/accounting_dimension_filter.py index 80f736fa5bbe..56f2eb71393f 100644 --- a/erpnext/accounts/doctype/accounting_dimension_filter/accounting_dimension_filter.py +++ b/erpnext/accounts/doctype/accounting_dimension_filter/accounting_dimension_filter.py @@ -38,37 +38,40 @@ def validate_applicable_accounts(self): def get_dimension_filter_map(): - filters = frappe.db.sql( - """ - SELECT - a.applicable_on_account, d.dimension_value, p.accounting_dimension, - p.allow_or_restrict, a.is_mandatory - FROM - `tabApplicable On Account` a, `tabAllowed Dimension` d, - `tabAccounting Dimension Filter` p - WHERE - p.name = a.parent - AND p.disabled = 0 - AND p.name = d.parent - """, - as_dict=1, - ) + if not frappe.flags.get("dimension_filter_map"): + filters = frappe.db.sql( + """ + SELECT + a.applicable_on_account, d.dimension_value, p.accounting_dimension, + p.allow_or_restrict, a.is_mandatory + FROM + `tabApplicable On Account` a, `tabAllowed Dimension` d, + `tabAccounting Dimension Filter` p + WHERE + p.name = a.parent + AND p.disabled = 0 + AND p.name = d.parent + """, + as_dict=1, + ) - dimension_filter_map = {} + dimension_filter_map = {} - for f in filters: - f.fieldname = scrub(f.accounting_dimension) + for f in filters: + f.fieldname = scrub(f.accounting_dimension) - build_map( - dimension_filter_map, - f.fieldname, - f.applicable_on_account, - f.dimension_value, - f.allow_or_restrict, - f.is_mandatory, - ) + build_map( + dimension_filter_map, + f.fieldname, + f.applicable_on_account, + f.dimension_value, + f.allow_or_restrict, + f.is_mandatory, + ) + + frappe.flags.dimension_filter_map = dimension_filter_map - return dimension_filter_map + return frappe.flags.dimension_filter_map def build_map(map_object, dimension, account, filter_value, allow_or_restrict, is_mandatory): From 5cd9bf3bda1c63fc04b6b4db8ddfdb2f9c7b456e Mon Sep 17 00:00:00 2001 From: Nabin Hait Date: Wed, 6 Mar 2024 21:04:07 +0530 Subject: [PATCH 07/14] fix: minor fixes --- .../doctype/accounting_dimension/accounting_dimension.py | 1 + .../doctype/accounting_dimension/test_accounting_dimension.py | 1 + .../accounting_dimension_filter/accounting_dimension_filter.py | 1 + 3 files changed, 3 insertions(+) diff --git a/erpnext/accounts/doctype/accounting_dimension/accounting_dimension.py b/erpnext/accounts/doctype/accounting_dimension/accounting_dimension.py index 52dda51708fd..c6c5f207a6a1 100644 --- a/erpnext/accounts/doctype/accounting_dimension/accounting_dimension.py +++ b/erpnext/accounts/doctype/accounting_dimension/accounting_dimension.py @@ -237,6 +237,7 @@ def get_accounting_dimensions(as_list=True, filters=None): def get_checks_for_pl_and_bs_accounts(): if frappe.flags.accounting_dimensions_details is None: + # nosemgrep frappe.flags.accounting_dimensions_details = frappe.db.sql( """SELECT p.label, p.disabled, p.fieldname, c.default_dimension, c.company, c.mandatory_for_pl, c.mandatory_for_bs FROM `tabAccounting Dimension`p ,`tabAccounting Dimension Detail` c diff --git a/erpnext/accounts/doctype/accounting_dimension/test_accounting_dimension.py b/erpnext/accounts/doctype/accounting_dimension/test_accounting_dimension.py index 25ef2ea5c2c2..6de3215cc596 100644 --- a/erpnext/accounts/doctype/accounting_dimension/test_accounting_dimension.py +++ b/erpnext/accounts/doctype/accounting_dimension/test_accounting_dimension.py @@ -78,6 +78,7 @@ def test_mandatory(self): def tearDown(self): disable_dimension() + frappe.flags.accounting_dimensions_details = None def create_dimension(): diff --git a/erpnext/accounts/doctype/accounting_dimension_filter/accounting_dimension_filter.py b/erpnext/accounts/doctype/accounting_dimension_filter/accounting_dimension_filter.py index 56f2eb71393f..652627642949 100644 --- a/erpnext/accounts/doctype/accounting_dimension_filter/accounting_dimension_filter.py +++ b/erpnext/accounts/doctype/accounting_dimension_filter/accounting_dimension_filter.py @@ -39,6 +39,7 @@ def validate_applicable_accounts(self): def get_dimension_filter_map(): if not frappe.flags.get("dimension_filter_map"): + # nosemgrep filters = frappe.db.sql( """ SELECT From 05385e4acb0f3af7e9cd19c752df8d7287f73401 Mon Sep 17 00:00:00 2001 From: Nabin Hait Date: Wed, 6 Mar 2024 22:33:36 +0530 Subject: [PATCH 08/14] perf: skip unnecessary validation while transaction cancellation --- .../accounting_dimension/test_accounting_dimension.py | 1 + .../test_accounting_dimension_filter.py | 2 ++ .../doctype/payment_ledger_entry/payment_ledger_entry.py | 9 +++++---- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/erpnext/accounts/doctype/accounting_dimension/test_accounting_dimension.py b/erpnext/accounts/doctype/accounting_dimension/test_accounting_dimension.py index 6de3215cc596..2ceaffc4e252 100644 --- a/erpnext/accounts/doctype/accounting_dimension/test_accounting_dimension.py +++ b/erpnext/accounts/doctype/accounting_dimension/test_accounting_dimension.py @@ -79,6 +79,7 @@ def test_mandatory(self): def tearDown(self): disable_dimension() frappe.flags.accounting_dimensions_details = None + frappe.flags.dimension_filter_map = None def create_dimension(): diff --git a/erpnext/accounts/doctype/accounting_dimension_filter/test_accounting_dimension_filter.py b/erpnext/accounts/doctype/accounting_dimension_filter/test_accounting_dimension_filter.py index f13f2f9f2791..3dc87bb4e386 100644 --- a/erpnext/accounts/doctype/accounting_dimension_filter/test_accounting_dimension_filter.py +++ b/erpnext/accounts/doctype/accounting_dimension_filter/test_accounting_dimension_filter.py @@ -47,6 +47,8 @@ def test_mandatory_dimension_validation(self): def tearDown(self): disable_dimension_filter() disable_dimension() + frappe.flags.accounting_dimensions_details = None + frappe.flags.dimension_filter_map = None for si in self.invoice_list: si.load_from_db() diff --git a/erpnext/accounts/doctype/payment_ledger_entry/payment_ledger_entry.py b/erpnext/accounts/doctype/payment_ledger_entry/payment_ledger_entry.py index bcbcb670faf2..bd438ee76063 100644 --- a/erpnext/accounts/doctype/payment_ledger_entry/payment_ledger_entry.py +++ b/erpnext/accounts/doctype/payment_ledger_entry/payment_ledger_entry.py @@ -132,11 +132,12 @@ def validate(self): def on_update(self): adv_adj = self.flags.adv_adj if not self.flags.from_repost: - self.validate_account_details() - self.validate_dimensions_for_pl_and_bs() - self.validate_allowed_dimensions() - validate_balance_type(self.account, adv_adj) validate_frozen_account(self.account, adv_adj) + if not self.delinked: + self.validate_account_details() + self.validate_dimensions_for_pl_and_bs() + self.validate_allowed_dimensions() + validate_balance_type(self.account, adv_adj) # update outstanding amount if ( From 49c74369a58c313f114a8a9d99bb3b316febe4cf Mon Sep 17 00:00:00 2001 From: Nabin Hait Date: Wed, 13 Mar 2024 12:54:13 +0530 Subject: [PATCH 09/14] perf: refactored handling provisional gl entries for non-stock items --- .../purchase_invoice/purchase_invoice.py | 168 ++++++++++-------- 1 file changed, 97 insertions(+), 71 deletions(-) diff --git a/erpnext/accounts/doctype/purchase_invoice/purchase_invoice.py b/erpnext/accounts/doctype/purchase_invoice/purchase_invoice.py index 35652b71c4de..df64e829dfee 100644 --- a/erpnext/accounts/doctype/purchase_invoice/purchase_invoice.py +++ b/erpnext/accounts/doctype/purchase_invoice/purchase_invoice.py @@ -3,7 +3,7 @@ import frappe -from frappe import _, throw +from frappe import _, qb, throw from frappe.model.mapper import get_mapped_doc from frappe.query_builder.functions import Sum from frappe.utils import cint, cstr, flt, formatdate, get_link_to_form, getdate, nowdate @@ -575,13 +575,12 @@ def on_update_after_submit(self): self.db_set("repost_required", self.needs_repost) def make_gl_entries(self, gl_entries=None, from_repost=False): - if not gl_entries: - gl_entries = self.get_gl_entries() + update_outstanding = "No" if (cint(self.is_paid) or self.write_off_account) else "Yes" + if self.docstatus == 1: + if not gl_entries: + gl_entries = self.get_gl_entries() - if gl_entries: - update_outstanding = "No" if (cint(self.is_paid) or self.write_off_account) else "Yes" - - if self.docstatus == 1: + if gl_entries: make_gl_entries( gl_entries, update_outstanding=update_outstanding, @@ -589,29 +588,43 @@ def make_gl_entries(self, gl_entries=None, from_repost=False): from_repost=from_repost, ) self.make_exchange_gain_loss_journal() - elif self.docstatus == 2: - provisional_entries = [a for a in gl_entries if a.voucher_type == "Purchase Receipt"] - make_reverse_gl_entries(voucher_type=self.doctype, voucher_no=self.name) - if provisional_entries: - for entry in provisional_entries: - frappe.db.set_value( - "GL Entry", - {"voucher_type": "Purchase Receipt", "voucher_detail_no": entry.voucher_detail_no}, - "is_cancelled", - 1, - ) + elif self.docstatus == 2: + make_reverse_gl_entries(voucher_type=self.doctype, voucher_no=self.name) + self.cancel_provisional_entries() - if update_outstanding == "No": - update_outstanding_amt( - self.credit_to, - "Supplier", - self.supplier, - self.doctype, - self.return_against if cint(self.is_return) and self.return_against else self.name, - ) + self.update_supplier_outstanding(update_outstanding) - elif self.docstatus == 2 and cint(self.update_stock) and self.auto_accounting_for_stock: - make_reverse_gl_entries(voucher_type=self.doctype, voucher_no=self.name) + def cancel_provisional_entries(self): + rows = set() + purchase_receipts = set() + for d in self.items: + if d.purchase_receipt: + purchase_receipts.add(d.purchase_receipt) + rows.add(d.name) + + if rows: + # cancel gl entries + gle = qb.DocType("GL Entry") + gle_update_query = ( + qb.update(gle) + .set(gle.is_cancelled, 1) + .where( + (gle.voucher_type == "Purchase Receipt") + & (gle.voucher_no.isin(purchase_receipts)) + & (gle.voucher_detail_no.isin(rows)) + ) + ) + gle_update_query.run() + + def update_supplier_outstanding(self, update_outstanding): + if update_outstanding == "No": + update_outstanding_amt( + self.credit_to, + "Supplier", + self.supplier, + self.doctype, + self.return_against if cint(self.is_return) and self.return_against else self.name, + ) def get_gl_entries(self, warehouse_account=None): self.auto_accounting_for_stock = erpnext.is_perpetual_inventory_enabled(self.company) @@ -720,8 +733,9 @@ def make_item_gl_entries(self, gl_entries): "Company", self.company, "enable_provisional_accounting_for_non_stock_items" ) ) - provisional_enpenses_booked_in_pr = False - purchase_receipt_doc_map = {} + self.provisional_enpenses_booked_in_pr = False + if provisional_accounting_for_non_stock_items: + self.get_provisional_accounts() for item in self.get("items"): if flt(item.base_net_amount): @@ -858,47 +872,7 @@ def make_item_gl_entries(self, gl_entries): dummy, amount = self.get_amount_and_base_amount(item, None) if provisional_accounting_for_non_stock_items: - if item.purchase_receipt: - if not provisional_enpenses_booked_in_pr: - provisional_account, pr_qty, pr_base_rate = frappe.get_cached_value( - "Purchase Receipt Item", - item.pr_detail, - ["provisional_expense_account", "qty", "base_rate"], - ) - provisional_account = provisional_account or self.get_company_default( - "default_provisional_account" - ) - # Post reverse entry for Stock-Received-But-Not-Billed if it is booked in Purchase Receipt - provision_gle_against_pr = frappe.db.get_value( - "GL Entry", - { - "is_cancelled": 0, - "voucher_type": "Purchase Receipt", - "voucher_no": item.purchase_receipt, - "voucher_detail_no": item.pr_detail, - "account": provisional_account, - }, - ["name"], - ) - if provision_gle_against_pr: - provisional_enpenses_booked_in_pr = True - - if provisional_enpenses_booked_in_pr: - purchase_receipt_doc = purchase_receipt_doc_map.get(item.purchase_receipt) - - if not purchase_receipt_doc: - purchase_receipt_doc = frappe.get_doc("Purchase Receipt", item.purchase_receipt) - purchase_receipt_doc_map[item.purchase_receipt] = purchase_receipt_doc - - # Intentionally passing purchase invoice item to handle partial billing - purchase_receipt_doc.add_provisional_gl_entry( - item, - gl_entries, - self.posting_date, - provisional_account, - reverse=1, - item_amount=(min(item.qty, pr_qty) * pr_base_rate), - ) + self.make_provisional_gl_entry(gl_entries, item) if not self.is_internal_transfer(): gl_entries.append( @@ -995,6 +969,58 @@ def make_item_gl_entries(self, gl_entries): if item.is_fixed_asset and item.landed_cost_voucher_amount: self.update_gross_purchase_amount_for_linked_assets(item) + def get_provisional_accounts(self): + self.provisional_accounts = frappe._dict() + linked_purchase_receipts = set([d.purchase_receipt for d in self.items if d.purchase_receipt]) + pr_items = frappe.get_all( + "Purchase Receipt Item", + filters={"parent": ("in", linked_purchase_receipts)}, + fields=["name", "provisional_expense_account", "qty", "base_rate"], + ) + default_provisional_account = self.get_company_default("default_provisional_account") + for item in pr_items: + self.provisional_accounts[item.name] = { + "provisional_account": item.provisional_expense_account or default_provisional_account, + "qty": item.qty, + "base_rate": item.base_rate, + } + + def make_provisional_gl_entry(self, gl_entries, item): + if item.purchase_receipt: + if not self.provisional_enpenses_booked_in_pr: + pr_item = self.provisional_accounts.get(item.pr_detail, {}) + provisional_account = pr_item.get("provisional_account") + pr_qty = pr_item.get("qty") + pr_base_rate = pr_item.get("base_rate") + + # Post reverse entry for Stock-Received-But-Not-Billed if it is booked in Purchase Receipt + provision_gle_against_pr = frappe.db.get_value( + "GL Entry", + { + "is_cancelled": 0, + "voucher_type": "Purchase Receipt", + "voucher_no": item.purchase_receipt, + "voucher_detail_no": item.pr_detail, + "account": provisional_account, + }, + ["name"], + ) + if provision_gle_against_pr: + self.provisional_enpenses_booked_in_pr = True + + if self.provisional_enpenses_booked_in_pr: + purchase_receipt_doc = frappe.get_cached_doc("Purchase Receipt", item.purchase_receipt) + + # Intentionally passing purchase invoice item to handle partial billing + purchase_receipt_doc.add_provisional_gl_entry( + item, + gl_entries, + self.posting_date, + provisional_account, + reverse=1, + item_amount=(min(item.qty, pr_qty) * pr_base_rate), + ) + def update_gross_purchase_amount_for_linked_assets(self, item): assets = frappe.db.get_all( "Asset", From c15b2d5490975a2eb69ee0819611bc46fcd0dd3e Mon Sep 17 00:00:00 2001 From: Nabin Hait Date: Wed, 13 Mar 2024 12:56:28 +0530 Subject: [PATCH 10/14] perf: validate expense against budget only if budget exists --- erpnext/accounts/doctype/budget/budget.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/erpnext/accounts/doctype/budget/budget.py b/erpnext/accounts/doctype/budget/budget.py index 2f3b9a4784e3..42b6b44690b1 100644 --- a/erpnext/accounts/doctype/budget/budget.py +++ b/erpnext/accounts/doctype/budget/budget.py @@ -118,6 +118,11 @@ def validate_expense_against_budget(args, expense_amount=0): "Company", args.get("company"), "exception_budget_approver_role" ) + if not frappe.get_cached_value( + "Budget", {"fiscal_year": args.fiscal_year, "company": args.company} + ): # nosec + return + if not args.account: args.account = args.get("expense_account") From 6ff9e6ee848e68c1d84bc17ad682b376214c5d59 Mon Sep 17 00:00:00 2001 From: Nabin Hait Date: Wed, 13 Mar 2024 12:57:33 +0530 Subject: [PATCH 11/14] perf: Get bin details only for stock items --- erpnext/stock/get_item_details.py | 36 +++++++++++++++++-------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/erpnext/stock/get_item_details.py b/erpnext/stock/get_item_details.py index 37a6a0d6892d..831fcac93cea 100644 --- a/erpnext/stock/get_item_details.py +++ b/erpnext/stock/get_item_details.py @@ -104,22 +104,8 @@ def get_item_details(args, doc=None, for_validate=False, overwrite_warehouse=Tru if args.customer and cint(args.is_pos): out.update(get_pos_profile_item_details(args.company, args, update_data=True)) - if ( - args.get("doctype") == "Material Request" - and args.get("material_request_type") == "Material Transfer" - ): - out.update(get_bin_details(args.item_code, args.get("from_warehouse"))) - - elif out.get("warehouse"): - if doc and doc.get("doctype") == "Purchase Order": - # calculate company_total_stock only for po - bin_details = get_bin_details( - args.item_code, out.warehouse, args.company, include_child_warehouses=True - ) - else: - bin_details = get_bin_details(args.item_code, out.warehouse, include_child_warehouses=True) - - out.update(bin_details) + if item.is_stock_item: + update_bin_details(args, out, doc) # update args with out, if key or value not exists for key, value in out.items(): @@ -202,6 +188,24 @@ def set_valuation_rate(out, args): out.update(get_valuation_rate(args.item_code, args.company, out.get("warehouse"))) +def update_bin_details(args, out, doc): + if ( + args.get("doctype") == "Material Request" + and args.get("material_request_type") == "Material Transfer" + ): + out.update(get_bin_details(args.item_code, args.get("from_warehouse"))) + + elif out.get("warehouse"): + company = args.company if (doc and doc.get("doctype") == "Purchase Order") else None + + # calculate company_total_stock only for po + bin_details = get_bin_details( + args.item_code, out.warehouse, company, include_child_warehouses=True + ) + + out.update(bin_details) + + def process_args(args): if isinstance(args, str): args = json.loads(args) From d279e23623f13f93c6f7c870fef0c2335e6d2b15 Mon Sep 17 00:00:00 2001 From: Nabin Hait Date: Wed, 13 Mar 2024 12:58:13 +0530 Subject: [PATCH 12/14] fix: added index for price_list column in Item Price --- erpnext/stock/doctype/item_price/item_price.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/erpnext/stock/doctype/item_price/item_price.json b/erpnext/stock/doctype/item_price/item_price.json index f4d9bb0742d6..e83730fa23d4 100644 --- a/erpnext/stock/doctype/item_price/item_price.json +++ b/erpnext/stock/doctype/item_price/item_price.json @@ -104,7 +104,8 @@ "in_standard_filter": 1, "label": "Price List", "options": "Price List", - "reqd": 1 + "reqd": 1, + "search_index": 1 }, { "bold": 1, @@ -220,7 +221,7 @@ "idx": 1, "index_web_pages_for_search": 1, "links": [], - "modified": "2022-11-15 08:26:04.041861", + "modified": "2024-03-13 12:23:39.630290", "modified_by": "Administrator", "module": "Stock", "name": "Item Price", From 8d682fa8840ef69a1d8a78e4bb418668f251585b Mon Sep 17 00:00:00 2001 From: Nabin Hait Date: Wed, 13 Mar 2024 13:00:10 +0530 Subject: [PATCH 13/14] perf: Caching in checking allowance for qty and amount --- erpnext/controllers/accounts_controller.py | 4 ++-- erpnext/controllers/status_updater.py | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/erpnext/controllers/accounts_controller.py b/erpnext/controllers/accounts_controller.py index 0795ab0f7473..cb342c4ead6f 100644 --- a/erpnext/controllers/accounts_controller.py +++ b/erpnext/controllers/accounts_controller.py @@ -1605,8 +1605,8 @@ def validate_multiple_billing(self, ref_dt, item_ref_dn, based_on): item_allowance = {} global_qty_allowance, global_amount_allowance = None, None - role_allowed_to_over_bill = frappe.db.get_single_value( - "Accounts Settings", "role_allowed_to_over_bill" + role_allowed_to_over_bill = frappe.get_cached_value( + "Accounts Settings", None, "role_allowed_to_over_bill" ) user_roles = frappe.get_roles() diff --git a/erpnext/controllers/status_updater.py b/erpnext/controllers/status_updater.py index d09001c8fc19..04d0dd36775b 100644 --- a/erpnext/controllers/status_updater.py +++ b/erpnext/controllers/status_updater.py @@ -573,6 +573,7 @@ def update_billing_status(self, zero_amount_refdoc, ref_dt, ref_fieldname): ref_doc.set_status(update=True) +@frappe.request_cache def get_allowance_for( item_code, item_allowance=None, @@ -602,20 +603,20 @@ def get_allowance_for( global_amount_allowance, ) - qty_allowance, over_billing_allowance = frappe.db.get_value( + qty_allowance, over_billing_allowance = frappe.get_cached_value( "Item", item_code, ["over_delivery_receipt_allowance", "over_billing_allowance"] ) if qty_or_amount == "qty" and not qty_allowance: if global_qty_allowance == None: global_qty_allowance = flt( - frappe.db.get_single_value("Stock Settings", "over_delivery_receipt_allowance") + frappe.get_cached_value("Stock Settings", None, "over_delivery_receipt_allowance") ) qty_allowance = global_qty_allowance elif qty_or_amount == "amount" and not over_billing_allowance: if global_amount_allowance == None: global_amount_allowance = flt( - frappe.db.get_single_value("Accounts Settings", "over_billing_allowance") + frappe.get_cached_value("Accounts Settings", None, "over_billing_allowance") ) over_billing_allowance = global_amount_allowance From b07769d8d7890580a9c47598bd166347ca8c1aec Mon Sep 17 00:00:00 2001 From: Nabin Hait Date: Wed, 13 Mar 2024 13:02:51 +0530 Subject: [PATCH 14/14] perf: Caching in gl entry --- erpnext/accounts/doctype/gl_entry/gl_entry.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/erpnext/accounts/doctype/gl_entry/gl_entry.py b/erpnext/accounts/doctype/gl_entry/gl_entry.py index c0f6502910e3..939d84109008 100644 --- a/erpnext/accounts/doctype/gl_entry/gl_entry.py +++ b/erpnext/accounts/doctype/gl_entry/gl_entry.py @@ -159,7 +159,7 @@ def validate_dimensions_for_pl_and_bs(self): def check_pl_account(self): if ( self.is_opening == "Yes" - and frappe.db.get_value("Account", self.account, "report_type") == "Profit and Loss" + and frappe.get_cached_value("Account", self.account, "report_type") == "Profit and Loss" and not self.is_cancelled ): frappe.throw( @@ -279,7 +279,7 @@ def update_outstanding_amt( party_condition = "" if against_voucher_type == "Sales Invoice": - party_account = frappe.db.get_value(against_voucher_type, against_voucher, "debit_to") + party_account = frappe.get_cached_value(against_voucher_type, against_voucher, "debit_to") account_condition = "and account in ({0}, {1})".format( frappe.db.escape(account), frappe.db.escape(party_account) ) @@ -347,7 +347,7 @@ def update_outstanding_amt( def validate_frozen_account(account, adv_adj=None): frozen_account = frappe.get_cached_value("Account", account, "freeze_account") if frozen_account == "Yes" and not adv_adj: - frozen_accounts_modifier = frappe.db.get_value( + frozen_accounts_modifier = frappe.get_cached_value( "Accounts Settings", None, "frozen_accounts_modifier" )