feat: Added Scheduler Job to auto update statistics

- Added frequency in Video Settings
- Cron job to check half hourly job
- Hourly job to run job based on frequency
- Patch to set youtube id in video for ease in computing

erpnext/hooks.py

@@ -282,6 +282,11 @@ auto_cancel_exempted_doctypes= [
 ]
 
 scheduler_events = {
+	"cron": {
+		"0/30 * * * *": [
+			"erpnext.utilities.doctype.video.video.update_youtube_data_half_hourly",
+		]
+	},
 	"all": [
 		"erpnext.projects.doctype.project.project.project_status_update_reminder",
 		"erpnext.healthcare.doctype.patient_appointment.patient_appointment.send_appointment_reminder",
@@ -297,6 +302,7 @@ scheduler_events = {
 		"erpnext.projects.doctype.project.project.collect_project_status",
 		"erpnext.hr.doctype.shift_type.shift_type.process_auto_attendance_for_all_shifts",
 		"erpnext.support.doctype.issue.issue.set_service_level_agreement_variance",
+		"erpnext.utilities.doctype.video.video.update_youtube_data"
 	],
 	"daily": [
 		"erpnext.stock.reorder_item.reorder_item",

erpnext/patches.txt

@@ -718,3 +718,4 @@ erpnext.patches.v13_0.delete_report_requested_items_to_order
 erpnext.patches.v12_0.update_item_tax_template_company
 erpnext.patches.v13_0.move_branch_code_to_bank_account
 erpnext.patches.v13_0.healthcare_lab_module_rename_doctypes
+erpnext.patches.v13_0.set_youtube_video_id

erpnext/patches/v13_0/set_youtube_video_id.py

@@ -0,0 +1,8 @@
+from __future__ import unicode_literals
+import frappe
+from erpnext.utilities.doctype.video.video import get_id_from_url
+
+def execute():
+	for video in frappe.get_all("Video", fields=["name", "url", "youtube_video_id"]):
+		if video.url and not video.youtube_video_id:
+			frappe.db.set_value("Video", video.name, "youtube_video_id", get_id_from_url(video.url))

erpnext/utilities/doctype/video/video.json

@@ -11,6 +11,7 @@
   "title",
   "provider",
   "url",
+  "youtube_video_id",
   "column_break_4",
   "publish_date",
   "duration",
@@ -118,11 +119,18 @@
    "fieldname": "youtube_tracking_section",
    "fieldtype": "Section Break",
    "label": "Youtube Statistics"
+  },
+  {
+   "fieldname": "youtube_video_id",
+   "fieldtype": "Data",
+   "hidden": 1,
+   "label": "Youtube ID",
+   "read_only": 1
   }
  ],
  "image_field": "image",
  "links": [],
- "modified": "2020-09-04 12:59:28.283622",
+ "modified": "2020-09-07 17:02:20.185794",
  "modified_by": "Administrator",
  "module": "Utilities",
  "name": "Video",

erpnext/utilities/doctype/video/video.py

@@ -4,8 +4,8 @@
 
 from __future__ import unicode_literals
 import frappe
-import json
 import re
+import pytz
 from frappe.model.document import Document
 from frappe import _
 from six import string_types
@@ -13,21 +13,33 @@ from pyyoutube import Api
 
 class Video(Document):
 	def validate(self):
+		self.set_video_id()
 		self.set_youtube_statistics()
 
-	def set_youtube_statistics(self):
+	def set_video_id(self):
-		tracking_enabled = frappe.db.get_single_value("Video Settings", "enable_youtube_tracking")
+		if self.provider == "YouTube" and self.url and not self.get("youtube_video_id"):
-		if self.provider == "YouTube" and not tracking_enabled:
+			self.youtube_video_id = get_id_from_url(self.url)
+
+	@classmethod
+	def set_youtube_statistics(self, video_ids=None, update=True):
+		if self.provider == "YouTube" and not is_tracking_enabled():
 			return
 
 		api_key = frappe.db.get_single_value("Video Settings", "api_key")
-		youtube_id = get_id_from_url(self.url)
 		api = Api(api_key=api_key)
 
 		try:
-			video = api.get_video_by_id(video_id=youtube_id)
+			video_id = video_ids or self.youtube_video_id
+			video = api.get_video_by_id(video_id=video_id)
+
+			if video_ids:
+				video_stats = video.items
+			else:
 				video_stats = video.items[0].to_dict().get('statistics')
 
+			if not update:
+				return video_stats
+
 			self.like_count = video_stats.get('likeCount')
 			self.view_count = video_stats.get('viewCount')
 			self.dislike_count = video_stats.get('dislikeCount')
@@ -37,16 +49,106 @@ class Video(Document):
 			title = "Failed to Update YouTube Statistics for Video: {0}".format(self.name)
 			frappe.log_error(title + "\n\n" +  frappe.get_traceback(), title=title)
 
+def is_tracking_enabled():
+	return frappe.db.get_single_value("Video Settings", "enable_youtube_tracking")
+
+def get_frequency(value):
+	if not value:
+		return None
+
+	# Return frequency in hours
+	if value != "Daily":
+		return frappe.utils.cint(value[:2].strip())
+	else:
+		# 24 hours for Daily
+		return 24
+
+
+def update_youtube_data_half_hourly():
+	# Called every 30 mins via hooks
+	frequency = get_frequency(frappe.db.get_single_value("Video Settings", "frequency"))
+	if not is_tracking_enabled() or not frequency:
+		return
+
+	if frequency == 30:
+		batch_update_data()
+
+
+def update_youtube_data():
+	# Called every hour via hooks
+	frequency = get_frequency(frappe.db.get_single_value("Video Settings", "frequency"))
+
+	# if frequency is 30 mins dont proceed, as its handled in another method
+	if not is_tracking_enabled() or not frequency or frequency == 30:
+		return
+
+	time = datetime.now()
+	timezone = pytz.timezone(frappe.utils.get_time_zone())
+	site_time = time.astimezone(timezone)
+
+	if site_time.hour % frequency == 0:
+		batch_update_youtube_data()
+
+def get_formatted_ids(video_list):
+	# format ids to comma separated string for bulk request
+	ids = []
+	for video in video_list:
+		ids.append(video.youtube_video_id)
+
+	return ','.join(ids)
+
 @frappe.whitelist()
 def get_id_from_url(url):
-	'''
+	"""
 		Returns video id from url
-
 		:param youtube url: String URL
-	'''
+	"""
 	if not isinstance(url, string_types):
 		frappe.throw(_("URL can only be a string"), title=_("Invalid URL"))
 
 	pattern = re.compile(r'[a-z\:\//\.]+(youtube|youtu)\.(com|be)/(watch\?v=|embed/|.+\?v=)?([^"&?\s]{11})?')
 	id = pattern.match(url)
 	return id.groups()[-1]
+
+
+@frappe.whitelist()
+def batch_update_youtube_data():
+	def prepare_and_set_data(video_list):
+		video_ids = get_formatted_ids(video_list)
+		Video.provider = "YouTube"
+		stats = Video.set_youtube_statistics(video_ids=video_ids, update=False)
+		set_youtube_data(stats)
+
+	def set_youtube_data(entries):
+		for entry in entries:
+			video_stats = entry.to_dict().get('statistics')
+			video_id = entry.to_dict().get('id')
+			stats = {
+				'like_count' : video_stats.get('likeCount'),
+				'view_count' : video_stats.get('viewCount'),
+				'dislike_count' : video_stats.get('dislikeCount'),
+				'comment_count' : video_stats.get('commentCount')
+			}
+
+			frappe.db.sql("""
+				UPDATE `tabVideo`
+				SET
+					like_count  = %(like_count)s,
+					view_count = %(view_count)s,
+					dislike_count = %(dislike_count)s,
+					comment_count = %(comment_count)s
+				WHERE youtube_video_id = '{0}'""".format(video_id), stats)
+
+			frappe.log_error("yooooooooo")
+
+	video_list = frappe.get_all("Video", fields=["youtube_video_id"])
+	if len(video_list) > 50:
+		# Update in batches of 50
+		start, end = 0, 50
+		while start < len(video_list):
+			batch = video_list[start:end]
+			prepare_and_set_data(batch)
+			start += 50
+			end += 50
+	else:
+		prepare_and_set_data(video_list)

erpnext/utilities/doctype/video_settings/video_settings.json

@@ -6,7 +6,8 @@
  "engine": "InnoDB",
  "field_order": [
   "enable_youtube_tracking",
-  "api_key"
+  "api_key",
+  "frequency"
  ],
  "fields": [
   {
@@ -21,11 +22,21 @@
    "fieldtype": "Data",
    "label": "API Key",
    "mandatory_depends_on": "eval:doc.enable_youtube_tracking"
+  },
+  {
+   "default": "1 hr",
+   "depends_on": "eval:doc.enable_youtube_tracking",
+   "fieldname": "frequency",
+   "fieldtype": "Select",
+   "label": "Frequency",
+   "mandatory_depends_on": "eval:doc.enable_youtube_tracking",
+   "options": "30 mins\n1 hr\n6 hrs\nDaily"
   }
  ],
+ "index_web_pages_for_search": 1,
  "issingle": 1,
  "links": [],
- "modified": "2020-08-02 03:56:49.673870",
+ "modified": "2020-09-07 16:09:00.360668",
  "modified_by": "Administrator",
  "module": "Utilities",
  "name": "Video Settings",