From 06ee0ea00bab3793ef7fcb48348ba4a3ef501c0b Mon Sep 17 00:00:00 2001 From: marination Date: Mon, 7 Sep 2020 18:49:06 +0530 Subject: [PATCH] feat: Added Scheduler Job to auto update statistics - Added frequency in Video Settings - Cron job to check half hourly job - Hourly job to run job based on frequency - Patch to set youtube id in video for ease in computing --- erpnext/hooks.py | 6 + erpnext/patches.txt | 1 + erpnext/patches/v13_0/set_youtube_video_id.py | 8 ++ erpnext/utilities/doctype/video/video.json | 10 +- erpnext/utilities/doctype/video/video.py | 124 ++++++++++++++++-- .../video_settings/video_settings.json | 15 ++- 6 files changed, 150 insertions(+), 14 deletions(-) create mode 100644 erpnext/patches/v13_0/set_youtube_video_id.py diff --git a/erpnext/hooks.py b/erpnext/hooks.py index 95a836fe65..db1fd2f3fc 100644 --- a/erpnext/hooks.py +++ b/erpnext/hooks.py @@ -282,6 +282,11 @@ auto_cancel_exempted_doctypes= [ ] scheduler_events = { + "cron": { + "0/30 * * * *": [ + "erpnext.utilities.doctype.video.video.update_youtube_data_half_hourly", + ] + }, "all": [ "erpnext.projects.doctype.project.project.project_status_update_reminder", "erpnext.healthcare.doctype.patient_appointment.patient_appointment.send_appointment_reminder", @@ -297,6 +302,7 @@ scheduler_events = { "erpnext.projects.doctype.project.project.collect_project_status", "erpnext.hr.doctype.shift_type.shift_type.process_auto_attendance_for_all_shifts", "erpnext.support.doctype.issue.issue.set_service_level_agreement_variance", + "erpnext.utilities.doctype.video.video.update_youtube_data" ], "daily": [ "erpnext.stock.reorder_item.reorder_item", diff --git a/erpnext/patches.txt b/erpnext/patches.txt index 3bd416952f..2a52ff67e5 100644 --- a/erpnext/patches.txt +++ b/erpnext/patches.txt @@ -718,3 +718,4 @@ erpnext.patches.v13_0.delete_report_requested_items_to_order erpnext.patches.v12_0.update_item_tax_template_company erpnext.patches.v13_0.move_branch_code_to_bank_account erpnext.patches.v13_0.healthcare_lab_module_rename_doctypes +erpnext.patches.v13_0.set_youtube_video_id diff --git a/erpnext/patches/v13_0/set_youtube_video_id.py b/erpnext/patches/v13_0/set_youtube_video_id.py new file mode 100644 index 0000000000..8e5dd306a9 --- /dev/null +++ b/erpnext/patches/v13_0/set_youtube_video_id.py @@ -0,0 +1,8 @@ +from __future__ import unicode_literals +import frappe +from erpnext.utilities.doctype.video.video import get_id_from_url + +def execute(): + for video in frappe.get_all("Video", fields=["name", "url", "youtube_video_id"]): + if video.url and not video.youtube_video_id: + frappe.db.set_value("Video", video.name, "youtube_video_id", get_id_from_url(video.url)) \ No newline at end of file diff --git a/erpnext/utilities/doctype/video/video.json b/erpnext/utilities/doctype/video/video.json index 11df56c77d..2a82db2514 100644 --- a/erpnext/utilities/doctype/video/video.json +++ b/erpnext/utilities/doctype/video/video.json @@ -11,6 +11,7 @@ "title", "provider", "url", + "youtube_video_id", "column_break_4", "publish_date", "duration", @@ -118,11 +119,18 @@ "fieldname": "youtube_tracking_section", "fieldtype": "Section Break", "label": "Youtube Statistics" + }, + { + "fieldname": "youtube_video_id", + "fieldtype": "Data", + "hidden": 1, + "label": "Youtube ID", + "read_only": 1 } ], "image_field": "image", "links": [], - "modified": "2020-09-04 12:59:28.283622", + "modified": "2020-09-07 17:02:20.185794", "modified_by": "Administrator", "module": "Utilities", "name": "Video", diff --git a/erpnext/utilities/doctype/video/video.py b/erpnext/utilities/doctype/video/video.py index 2299f95f76..d8653f6446 100644 --- a/erpnext/utilities/doctype/video/video.py +++ b/erpnext/utilities/doctype/video/video.py @@ -4,8 +4,8 @@ from __future__ import unicode_literals import frappe -import json import re +import pytz from frappe.model.document import Document from frappe import _ from six import string_types @@ -13,20 +13,32 @@ from pyyoutube import Api class Video(Document): def validate(self): + self.set_video_id() self.set_youtube_statistics() - def set_youtube_statistics(self): - tracking_enabled = frappe.db.get_single_value("Video Settings", "enable_youtube_tracking") - if self.provider == "YouTube" and not tracking_enabled: + def set_video_id(self): + if self.provider == "YouTube" and self.url and not self.get("youtube_video_id"): + self.youtube_video_id = get_id_from_url(self.url) + + @classmethod + def set_youtube_statistics(self, video_ids=None, update=True): + if self.provider == "YouTube" and not is_tracking_enabled(): return api_key = frappe.db.get_single_value("Video Settings", "api_key") - youtube_id = get_id_from_url(self.url) api = Api(api_key=api_key) try: - video = api.get_video_by_id(video_id=youtube_id) - video_stats = video.items[0].to_dict().get('statistics') + video_id = video_ids or self.youtube_video_id + video = api.get_video_by_id(video_id=video_id) + + if video_ids: + video_stats = video.items + else: + video_stats = video.items[0].to_dict().get('statistics') + + if not update: + return video_stats self.like_count = video_stats.get('likeCount') self.view_count = video_stats.get('viewCount') @@ -37,16 +49,106 @@ class Video(Document): title = "Failed to Update YouTube Statistics for Video: {0}".format(self.name) frappe.log_error(title + "\n\n" + frappe.get_traceback(), title=title) +def is_tracking_enabled(): + return frappe.db.get_single_value("Video Settings", "enable_youtube_tracking") + +def get_frequency(value): + if not value: + return None + + # Return frequency in hours + if value != "Daily": + return frappe.utils.cint(value[:2].strip()) + else: + # 24 hours for Daily + return 24 + + +def update_youtube_data_half_hourly(): + # Called every 30 mins via hooks + frequency = get_frequency(frappe.db.get_single_value("Video Settings", "frequency")) + if not is_tracking_enabled() or not frequency: + return + + if frequency == 30: + batch_update_data() + + +def update_youtube_data(): + # Called every hour via hooks + frequency = get_frequency(frappe.db.get_single_value("Video Settings", "frequency")) + + # if frequency is 30 mins dont proceed, as its handled in another method + if not is_tracking_enabled() or not frequency or frequency == 30: + return + + time = datetime.now() + timezone = pytz.timezone(frappe.utils.get_time_zone()) + site_time = time.astimezone(timezone) + + if site_time.hour % frequency == 0: + batch_update_youtube_data() + +def get_formatted_ids(video_list): + # format ids to comma separated string for bulk request + ids = [] + for video in video_list: + ids.append(video.youtube_video_id) + + return ','.join(ids) + @frappe.whitelist() def get_id_from_url(url): - ''' + """ Returns video id from url - :param youtube url: String URL - ''' + """ if not isinstance(url, string_types): frappe.throw(_("URL can only be a string"), title=_("Invalid URL")) pattern = re.compile(r'[a-z\:\//\.]+(youtube|youtu)\.(com|be)/(watch\?v=|embed/|.+\?v=)?([^"&?\s]{11})?') id = pattern.match(url) - return id.groups()[-1] \ No newline at end of file + return id.groups()[-1] + + +@frappe.whitelist() +def batch_update_youtube_data(): + def prepare_and_set_data(video_list): + video_ids = get_formatted_ids(video_list) + Video.provider = "YouTube" + stats = Video.set_youtube_statistics(video_ids=video_ids, update=False) + set_youtube_data(stats) + + def set_youtube_data(entries): + for entry in entries: + video_stats = entry.to_dict().get('statistics') + video_id = entry.to_dict().get('id') + stats = { + 'like_count' : video_stats.get('likeCount'), + 'view_count' : video_stats.get('viewCount'), + 'dislike_count' : video_stats.get('dislikeCount'), + 'comment_count' : video_stats.get('commentCount') + } + + frappe.db.sql(""" + UPDATE `tabVideo` + SET + like_count = %(like_count)s, + view_count = %(view_count)s, + dislike_count = %(dislike_count)s, + comment_count = %(comment_count)s + WHERE youtube_video_id = '{0}'""".format(video_id), stats) + + frappe.log_error("yooooooooo") + + video_list = frappe.get_all("Video", fields=["youtube_video_id"]) + if len(video_list) > 50: + # Update in batches of 50 + start, end = 0, 50 + while start < len(video_list): + batch = video_list[start:end] + prepare_and_set_data(batch) + start += 50 + end += 50 + else: + prepare_and_set_data(video_list) \ No newline at end of file diff --git a/erpnext/utilities/doctype/video_settings/video_settings.json b/erpnext/utilities/doctype/video_settings/video_settings.json index 0a0efd9a53..fb3274decd 100644 --- a/erpnext/utilities/doctype/video_settings/video_settings.json +++ b/erpnext/utilities/doctype/video_settings/video_settings.json @@ -6,7 +6,8 @@ "engine": "InnoDB", "field_order": [ "enable_youtube_tracking", - "api_key" + "api_key", + "frequency" ], "fields": [ { @@ -21,11 +22,21 @@ "fieldtype": "Data", "label": "API Key", "mandatory_depends_on": "eval:doc.enable_youtube_tracking" + }, + { + "default": "1 hr", + "depends_on": "eval:doc.enable_youtube_tracking", + "fieldname": "frequency", + "fieldtype": "Select", + "label": "Frequency", + "mandatory_depends_on": "eval:doc.enable_youtube_tracking", + "options": "30 mins\n1 hr\n6 hrs\nDaily" } ], + "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2020-08-02 03:56:49.673870", + "modified": "2020-09-07 16:09:00.360668", "modified_by": "Administrator", "module": "Utilities", "name": "Video Settings",