feat: Sync old shopify orders (#23841)

* feat: Sync old shopify orders

* fix: Old orders syncing by date

* fix: Remove unnecessary code

* fix: Remove unintentional changes

Co-authored-by: Saurabh <saurabh@erpnext.com>

erpnext/erpnext_integrations/connectors/shopify_connection.py

@@ -2,12 +2,13 @@ from __future__ import unicode_literals
 import frappe
 from frappe import _
 import json
-from frappe.utils import cstr, cint, nowdate, flt
+from frappe.utils import cstr, cint, nowdate, getdate, flt, get_request_session, get_datetime
 from erpnext.erpnext_integrations.utils import validate_webhooks_request
 from erpnext.selling.doctype.sales_order.sales_order import make_delivery_note, make_sales_invoice
 from erpnext.erpnext_integrations.doctype.shopify_settings.sync_product import sync_item_from_shopify
 from erpnext.erpnext_integrations.doctype.shopify_settings.sync_customer import create_customer
 from erpnext.erpnext_integrations.doctype.shopify_log.shopify_log import make_shopify_log, dump_request_data
+from erpnext.erpnext_integrations.doctype.shopify_settings.shopify_settings import get_shopify_url, get_header
 
 @frappe.whitelist(allow_guest=True)
 @validate_webhooks_request("Shopify Settings", 'X-Shopify-Hmac-Sha256', secret_key='shared_secret')
@@ -18,7 +19,7 @@ def store_request_data(order=None, event=None):
 
 	dump_request_data(order, event)
 
-def sync_sales_order(order, request_id=None):
+def sync_sales_order(order, request_id=None, old_order_sync=False):
 	frappe.set_user('Administrator')
 	shopify_settings = frappe.get_doc("Shopify Settings")
 	frappe.flags.request_id = request_id
@@ -27,7 +28,7 @@ def sync_sales_order(order, request_id=None):
 		try:
 			validate_customer(order, shopify_settings)
 			validate_item(order, shopify_settings)
-			create_order(order, shopify_settings)
+			create_order(order, shopify_settings, old_order_sync=old_order_sync)
 		except Exception as e:
 			make_shopify_log(status="Error", exception=e)
 
@@ -77,13 +78,13 @@ def validate_item(order, shopify_settings):
 		if item.get("product_id") and not frappe.db.get_value("Item", {"shopify_product_id": item.get("product_id")}, "name"):
 			sync_item_from_shopify(shopify_settings, item)
 
-def create_order(order, shopify_settings, company=None):
+def create_order(order, shopify_settings, old_order_sync=False, company=None):
 	so = create_sales_order(order, shopify_settings, company)
 	if so:
 		if order.get("financial_status") == "paid":
-			create_sales_invoice(order, shopify_settings, so)
+			create_sales_invoice(order, shopify_settings, so, old_order_sync=old_order_sync)
 
-		if order.get("fulfillments"):
+		if order.get("fulfillments") and not old_order_sync:
 			create_delivery_note(order, shopify_settings, so)
 
 def create_sales_order(shopify_order, shopify_settings, company=None):
@@ -92,7 +93,7 @@ def create_sales_order(shopify_order, shopify_settings, company=None):
 	so = frappe.db.get_value("Sales Order", {"shopify_order_id": shopify_order.get("id")}, "name")
 
 	if not so:
-		items = get_order_items(shopify_order.get("line_items"), shopify_settings)
+		items = get_order_items(shopify_order.get("line_items"), shopify_settings, getdate(shopify_order.get('created_at')))
 
 		if not items:
 			message = 'Following items exists in the shopify order but relevant records were not found in the shopify Product master'
@@ -106,8 +107,10 @@ def create_sales_order(shopify_order, shopify_settings, company=None):
 			"doctype": "Sales Order",
 			"naming_series": shopify_settings.sales_order_series or "SO-Shopify-",
 			"shopify_order_id": shopify_order.get("id"),
+			"shopify_order_number": shopify_order.get("name"),
 			"customer": customer or shopify_settings.default_customer,
-			"delivery_date": nowdate(),
+			"transaction_date": getdate(shopify_order.get("created_at")) or nowdate(),
+			"delivery_date": getdate(shopify_order.get("created_at")) or nowdate(),
 			"company": shopify_settings.company,
 			"selling_price_list": shopify_settings.price_list,
 			"ignore_pricing_rule": 1,
@@ -132,12 +135,20 @@ def create_sales_order(shopify_order, shopify_settings, company=None):
 	frappe.db.commit()
 	return so
 
-def create_sales_invoice(shopify_order, shopify_settings, so):
+def create_sales_invoice(shopify_order, shopify_settings, so, old_order_sync=False):
 	if not frappe.db.get_value("Sales Invoice", {"shopify_order_id": shopify_order.get("id")}, "name")\
 		and so.docstatus==1 and not so.per_billed and cint(shopify_settings.sync_sales_invoice):
 
+		if old_order_sync:
+			posting_date = getdate(shopify_order.get('created_at'))
+		else:
+			posting_date = nowdate()
+
 		si = make_sales_invoice(so.name, ignore_permissions=True)
 		si.shopify_order_id = shopify_order.get("id")
+		si.shopify_order_number = shopify_order.get("name")
+		si.set_posting_time = 1
+		si.posting_date = posting_date
 		si.naming_series = shopify_settings.sales_invoice_series or "SI-Shopify-"
 		si.flags.ignore_mandatory = True
 		set_cost_center(si.items, shopify_settings.cost_center)
@@ -169,6 +180,9 @@ def create_delivery_note(shopify_order, shopify_settings, so):
 
 			dn = make_delivery_note(so.name)
 			dn.shopify_order_id = fulfillment.get("order_id")
+			dn.shopify_order_number = shopify_order.get("name")
+			dn.set_posting_time = 1
+			dn.posting_date = getdate(fulfillment.get("created_at"))
 			dn.shopify_fulfillment_id = fulfillment.get("id")
 			dn.naming_series = shopify_settings.delivery_note_series or "DN-Shopify-"
 			dn.items = get_fulfillment_items(dn.items, fulfillment.get("line_items"), shopify_settings)
@@ -187,7 +201,7 @@ def get_discounted_amount(order):
 		discounted_amount += flt(discount.get("amount"))
 	return discounted_amount
 
-def get_order_items(order_items, shopify_settings):
+def get_order_items(order_items, shopify_settings, delivery_date):
 	items = []
 	all_product_exists = True
 	product_not_exists = []
@@ -205,7 +219,7 @@ def get_order_items(order_items, shopify_settings):
 				"item_code": item_code,
 				"item_name": shopify_item.get("name"),
 				"rate": shopify_item.get("price"),
-				"delivery_date": nowdate(),
+				"delivery_date": delivery_date,
 				"qty": shopify_item.get("quantity"),
 				"stock_uom": shopify_item.get("uom") or _("Nos"),
 				"warehouse": shopify_settings.warehouse
@@ -265,3 +279,64 @@ def get_tax_account_head(tax):
 		frappe.throw(_("Tax Account not specified for Shopify Tax {0}").format(tax.get("title")))
 
 	return tax_account
+
+@frappe.whitelist(allow_guest=True)
+def sync_old_orders():
+	frappe.set_user('Administrator')
+	shopify_settings = frappe.get_doc('Shopify Settings')
+
+	if not shopify_settings.sync_missing_orders:
+		return
+
+	url = get_url(shopify_settings)
+	session = get_request_session()
+
+	try:
+		res = session.get(url, headers=get_header(shopify_settings))
+		res.raise_for_status()
+		orders = res.json()["orders"]
+
+		for order in orders:
+			if is_sync_complete(shopify_settings, order):
+				stop_sync(shopify_settings)
+				return
+
+			sync_sales_order(order=order, old_order_sync=True)
+			last_order_id = order.get('id')
+
+		if last_order_id:
+			shopify_settings.load_from_db()
+			shopify_settings.last_order_id = last_order_id
+			shopify_settings.save()
+			frappe.db.commit()
+
+	except Exception as e:
+		raise e
+
+def stop_sync(shopify_settings):
+	shopify_settings.sync_missing_orders = 0
+	shopify_settings.last_order_id = ''
+	shopify_settings.save()
+	frappe.db.commit()
+
+def get_url(shopify_settings):
+	last_order_id = shopify_settings.last_order_id
+
+	if not last_order_id:
+		if shopify_settings.sync_based_on == 'Date':
+			url = get_shopify_url("admin/api/2020-10/orders.json?limit=250&created_at_min={0}&since_id=0".format(
+				get_datetime(shopify_settings.from_date)), shopify_settings)
+		else:
+			url = get_shopify_url("admin/api/2020-10/orders.json?limit=250&since_id={0}".format(
+				shopify_settings.from_order_id), shopify_settings)
+	else:
+		url = get_shopify_url("admin/api/2020-10/orders.json?limit=250&since_id={0}".format(last_order_id), shopify_settings)
+
+	return url
+
+def is_sync_complete(shopify_settings, order):
+	if shopify_settings.sync_based_on == 'Date':
+		return getdate(shopify_settings.to_date) < getdate(order.get('created_at'))
+	else:
+		return cstr(order.get('id')) == cstr(shopify_settings.to_order_id)
+

erpnext/erpnext_integrations/doctype/shopify_settings/shopify_settings.json

@@ -1,7 +1,9 @@
 {
+ "actions": [],
  "creation": "2015-05-18 05:21:07.270859",
  "doctype": "DocType",
  "document_type": "System",
+ "engine": "InnoDB",
  "field_order": [
   "status_html",
   "enable_shopify",
@@ -40,7 +42,16 @@
   "sales_invoice_series",
   "section_break_22",
   "html_16",
-  "taxes"
+  "taxes",
+  "syncing_details_section",
+  "sync_missing_orders",
+  "sync_based_on",
+  "column_break_41",
+  "from_date",
+  "to_date",
+  "from_order_id",
+  "to_order_id",
+  "last_order_id"
  ],
  "fields": [
   {
@@ -255,10 +266,71 @@
    "fieldtype": "Table",
    "label": "Shopify Tax Account",
    "options": "Shopify Tax Account"
+  },
+  {
+   "collapsible": 1,
+   "fieldname": "syncing_details_section",
+   "fieldtype": "Section Break",
+   "label": "Syncing Missing Orders"
+  },
+  {
+   "depends_on": "eval:doc.sync_missing_orders",
+   "fieldname": "last_order_id",
+   "fieldtype": "Data",
+   "label": "Last Order Id",
+   "read_only": 1
+  },
+  {
+   "fieldname": "column_break_41",
+   "fieldtype": "Column Break"
+  },
+  {
+   "default": "0",
+   "description": "On checking this Order from the ",
+   "fieldname": "sync_missing_orders",
+   "fieldtype": "Check",
+   "label": "Sync Missing Old Shopify Orders"
+  },
+  {
+   "depends_on": "eval:doc.sync_missing_orders",
+   "fieldname": "sync_based_on",
+   "fieldtype": "Select",
+   "label": "Sync Based On",
+   "mandatory_depends_on": "eval:doc.sync_missing_orders",
+   "options": "\nDate\nShopify Order Id"
+  },
+  {
+   "depends_on": "eval:doc.sync_based_on == 'Date' && doc.sync_missing_orders",
+   "fieldname": "from_date",
+   "fieldtype": "Date",
+   "label": "From Date",
+   "mandatory_depends_on": "eval:doc.sync_based_on == 'Date' && doc.sync_missing_orders"
+  },
+  {
+   "depends_on": "eval:doc.sync_based_on == 'Date' && doc.sync_missing_orders",
+   "fieldname": "to_date",
+   "fieldtype": "Date",
+   "label": "To Date",
+   "mandatory_depends_on": "eval:doc.sync_based_on == 'Date' && doc.sync_missing_orders"
+  },
+  {
+   "depends_on": "eval:doc.sync_based_on == 'Shopify Order Id' && doc.sync_missing_orders",
+   "fieldname": "from_order_id",
+   "fieldtype": "Data",
+   "label": "From Order Id",
+   "mandatory_depends_on": "eval:doc.sync_based_on == 'Shopify Order Id' && doc.sync_missing_orders"
+  },
+  {
+   "depends_on": "eval:doc.sync_based_on == 'Shopify Order Id' && doc.sync_missing_orders",
+   "fieldname": "to_order_id",
+   "fieldtype": "Data",
+   "label": "To Order Id",
+   "mandatory_depends_on": "eval:doc.sync_based_on == 'Shopify Order Id' && doc.sync_missing_orders"
   }
  ],
  "issingle": 1,
- "modified": "2020-09-18 17:26:09.703215",
+ "links": [],
+ "modified": "2020-11-05 20:44:03.664891",
  "modified_by": "Administrator",
  "module": "ERPNext Integrations",
  "name": "Shopify Settings",

erpnext/erpnext_integrations/doctype/shopify_settings/shopify_settings.py

@@ -87,7 +87,7 @@ def get_shopify_url(path, settings):
 def get_header(settings):
 	header = {'Content-Type': 'application/json'}
 
-	return header;
+	return header
 
 @frappe.whitelist()
 def get_series():
@@ -121,17 +121,23 @@ def setup_custom_fields():
 		],
 		"Sales Order": [
 			dict(fieldname='shopify_order_id', label='Shopify Order Id',
-				fieldtype='Data', insert_after='title', read_only=1, print_hide=1)
+				fieldtype='Data', insert_after='title', read_only=1, print_hide=1),
+			dict(fieldname='shopify_order_number', label='Shopify Order Number',
+				fieldtype='Data', insert_after='shopify_order_id', read_only=1, print_hide=1)
 		],
 		"Delivery Note":[
 			dict(fieldname='shopify_order_id', label='Shopify Order Id',
 				fieldtype='Data', insert_after='title', read_only=1, print_hide=1),
+			dict(fieldname='shopify_order_number', label='Shopify Order Number',
+				fieldtype='Data', insert_after='shopify_order_id', read_only=1, print_hide=1),
 			dict(fieldname='shopify_fulfillment_id', label='Shopify Fulfillment Id',
 				fieldtype='Data', insert_after='title', read_only=1, print_hide=1)
 		],
 		"Sales Invoice": [
 			dict(fieldname='shopify_order_id', label='Shopify Order Id',
-				fieldtype='Data', insert_after='title', read_only=1, print_hide=1)
+				fieldtype='Data', insert_after='title', read_only=1, print_hide=1),
+			dict(fieldname='shopify_order_number', label='Shopify Order Number',
+				fieldtype='Data', insert_after='shopify_order_id', read_only=1, print_hide=1)
 		]
 	}
 

erpnext/erpnext_integrations/doctype/shopify_settings/test_shopify_settings.py

@@ -75,7 +75,7 @@ class ShopifySettings(unittest.TestCase):
 		with open (os.path.join(os.path.dirname(__file__), "test_data", "shopify_order.json")) as shopify_order:
 			shopify_order = json.load(shopify_order)
 
-		create_order(shopify_order.get("order"), self.shopify_settings, "_Test Company")
+		create_order(shopify_order.get("order"), self.shopify_settings, False, company="_Test Company")
 
 		sales_order = frappe.get_doc("Sales Order", {"shopify_order_id": cstr(shopify_order.get("order").get("id"))})
 

erpnext/hooks.py

@@ -307,6 +307,7 @@ scheduler_events = {
 		"erpnext.projects.doctype.project.project.collect_project_status",
 		"erpnext.hr.doctype.shift_type.shift_type.process_auto_attendance_for_all_shifts",
 		"erpnext.support.doctype.issue.issue.set_service_level_agreement_variance",
+		"erpnext.erpnext_integrations.connectors.shopify_connection.sync_old_orders",
 	],
 	"daily": [
 		"erpnext.stock.reorder_item.reorder_item",

erpnext/patches.txt

@@ -733,4 +733,5 @@ erpnext.patches.v13_0.print_uom_after_quantity_patch
 erpnext.patches.v13_0.set_payment_channel_in_payment_gateway_account
 erpnext.patches.v13_0.create_healthcare_custom_fields_in_stock_entry_detail
 erpnext.patches.v13_0.update_reason_for_resignation_in_employee
+erpnext.patches.v13_0.update_custom_fields_for_shopify
 execute:frappe.delete_doc("Report", "Quoted Item Comparison")

erpnext/patches/v13_0/update_custom_fields_for_shopify.py

@@ -0,0 +1,10 @@
+# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors
+# MIT License. See license.txt
+
+from __future__ import unicode_literals
+import frappe
+from erpnext.erpnext_integrations.doctype.shopify_settings.shopify_settings import setup_custom_fields
+
+def execute():
+	if frappe.db.get_single_value('Shopify Settings', 'enable_shopify'):
+		setup_custom_fields()