From d4817c8685f3b31e72c8d5acca297cbbb6d6877b Mon Sep 17 00:00:00 2001 From: marination Date: Sun, 2 Aug 2020 05:35:45 +0530 Subject: [PATCH 01/13] feat: Track Youtube interactions via Video DocType --- erpnext/utilities/doctype/video/video.js | 34 ++++++++- erpnext/utilities/doctype/video/video.json | 64 +++++++++++++++-- erpnext/utilities/doctype/video/video.py | 20 +++++- .../doctype/video_settings/__init__.py | 0 .../video_settings/test_video_settings.py | 10 +++ .../doctype/video_settings/video_settings.js | 8 +++ .../video_settings/video_settings.json | 49 +++++++++++++ .../doctype/video_settings/video_settings.py | 10 +++ erpnext/utilities/report/__init__.py | 0 .../report/youtube_interactions/__init__.py | 0 .../youtube_interactions.js | 9 +++ .../youtube_interactions.json | 27 +++++++ .../youtube_interactions.py | 72 +++++++++++++++++++ requirements.txt | 1 + 14 files changed, 296 insertions(+), 8 deletions(-) create mode 100644 erpnext/utilities/doctype/video_settings/__init__.py create mode 100644 erpnext/utilities/doctype/video_settings/test_video_settings.py create mode 100644 erpnext/utilities/doctype/video_settings/video_settings.js create mode 100644 erpnext/utilities/doctype/video_settings/video_settings.json create mode 100644 erpnext/utilities/doctype/video_settings/video_settings.py create mode 100644 erpnext/utilities/report/__init__.py create mode 100644 erpnext/utilities/report/youtube_interactions/__init__.py create mode 100644 erpnext/utilities/report/youtube_interactions/youtube_interactions.js create mode 100644 erpnext/utilities/report/youtube_interactions/youtube_interactions.json create mode 100644 erpnext/utilities/report/youtube_interactions/youtube_interactions.py diff --git a/erpnext/utilities/doctype/video/video.js b/erpnext/utilities/doctype/video/video.js index 056bd3ccd6..4dd4e67a7f 100644 --- a/erpnext/utilities/doctype/video/video.js +++ b/erpnext/utilities/doctype/video/video.js @@ -2,7 +2,37 @@ // For license information, please see license.txt frappe.ui.form.on('Video', { - // refresh: function(frm) { + refresh: function (frm) { + if (frm.doc.provider === "YouTube") { + frappe.db.get_single_value("Video Settings", "enable_youtube_tracking").then(value => { + if (value) { + frm.events.get_video_stats(frm); + } else { + frm.set_df_property('youtube_tracking_section', 'hidden', true); + } + }); + } - // } + frm.add_custom_button("Watch Video", () => frappe.help.show_video(frm.doc.url, frm.doc.title)); + }, + + get_video_stats: (frm) => { + const expression = '(?:youtube.com/(?:[^/]+/.+/|(?:v|e(?:mbed)?)/|.*[?&]v=)|youtu.be/)([^"&?\\s]{11})'; + var youtube_id = frm.doc.url.match(expression)[1]; + + frappe.call({ + method: "erpnext.utilities.doctype.video.video.update_video_stats", + args: { + youtube_id: youtube_id + }, + callback: (r) => { + var result = r.message; + var fields = ['like_count', 'view_count', 'dislike_count', 'comment_count']; + fields.forEach((field) => { + frm.doc[field] = result[field]; + }) + frm.refresh_fields(); + } + }); + } }); diff --git a/erpnext/utilities/doctype/video/video.json b/erpnext/utilities/doctype/video/video.json index 5d2cc13348..a6c0f3f82a 100644 --- a/erpnext/utilities/doctype/video/video.json +++ b/erpnext/utilities/doctype/video/video.json @@ -15,7 +15,14 @@ "publish_date", "duration", "section_break_7", - "description" + "description", + "image", + "youtube_tracking_section", + "like_count", + "view_count", + "col_break", + "dislike_count", + "comment_count" ], "fields": [ { @@ -37,7 +44,6 @@ { "fieldname": "url", "fieldtype": "Data", - "in_list_view": 1, "label": "URL", "reqd": 1 }, @@ -48,11 +54,12 @@ { "fieldname": "publish_date", "fieldtype": "Date", + "in_list_view": 1, "label": "Publish Date" }, { "fieldname": "duration", - "fieldtype": "Data", + "fieldtype": "Duration", "label": "Duration" }, { @@ -62,13 +69,60 @@ { "fieldname": "description", "fieldtype": "Text Editor", - "in_list_view": 1, "label": "Description", "reqd": 1 + }, + { + "fieldname": "like_count", + "fieldtype": "Float", + "in_list_view": 1, + "label": "Likes", + "no_copy": 1, + "read_only": 1 + }, + { + "fieldname": "view_count", + "fieldtype": "Float", + "in_list_view": 1, + "label": "Views", + "no_copy": 1, + "read_only": 1 + }, + { + "fieldname": "col_break", + "fieldtype": "Column Break" + }, + { + "fieldname": "dislike_count", + "fieldtype": "Float", + "label": "Dislikes", + "no_copy": 1, + "read_only": 1 + }, + { + "fieldname": "comment_count", + "fieldtype": "Float", + "label": "Comments", + "no_copy": 1, + "read_only": 1 + }, + { + "fieldname": "image", + "fieldtype": "Attach Image", + "hidden": 1, + "label": "Image", + "no_copy": 1 + }, + { + "depends_on": "eval:doc.provider==\"YouTube\"", + "fieldname": "youtube_tracking_section", + "fieldtype": "Section Break", + "label": "Youtube Statistics" } ], + "image_field": "image", "links": [], - "modified": "2020-07-21 19:29:46.603734", + "modified": "2020-08-02 04:26:16.345569", "modified_by": "Administrator", "module": "Utilities", "name": "Video", diff --git a/erpnext/utilities/doctype/video/video.py b/erpnext/utilities/doctype/video/video.py index 3c17b560f3..263884a93d 100644 --- a/erpnext/utilities/doctype/video/video.py +++ b/erpnext/utilities/doctype/video/video.py @@ -3,8 +3,26 @@ # For license information, please see license.txt from __future__ import unicode_literals -# import frappe +import frappe from frappe.model.document import Document +from pyyoutube import Api class Video(Document): pass + +@frappe.whitelist() +def update_video_stats(youtube_id): + ''' + :param youtube_id: Unique ID from URL + ''' + api_key = frappe.db.get_single_value("Video Settings", "api_key") + api = Api(api_key=api_key) + + video = api.get_video_by_id(video_id=youtube_id) + video_stats = video.items[0].to_dict().get('statistics') + return { + 'like_count' : video_stats.get('likeCount'), + 'view_count' : video_stats.get('viewCount'), + 'dislike_count' : video_stats.get('dislikeCount'), + 'comment_count' : video_stats.get('commentCount') + } \ No newline at end of file diff --git a/erpnext/utilities/doctype/video_settings/__init__.py b/erpnext/utilities/doctype/video_settings/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/erpnext/utilities/doctype/video_settings/test_video_settings.py b/erpnext/utilities/doctype/video_settings/test_video_settings.py new file mode 100644 index 0000000000..b217afe3d8 --- /dev/null +++ b/erpnext/utilities/doctype/video_settings/test_video_settings.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors +# See license.txt +from __future__ import unicode_literals + +# import frappe +import unittest + +class TestVideoSettings(unittest.TestCase): + pass diff --git a/erpnext/utilities/doctype/video_settings/video_settings.js b/erpnext/utilities/doctype/video_settings/video_settings.js new file mode 100644 index 0000000000..9ac8b9ec16 --- /dev/null +++ b/erpnext/utilities/doctype/video_settings/video_settings.js @@ -0,0 +1,8 @@ +// Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and contributors +// For license information, please see license.txt + +frappe.ui.form.on('Video Settings', { + // refresh: function(frm) { + + // } +}); diff --git a/erpnext/utilities/doctype/video_settings/video_settings.json b/erpnext/utilities/doctype/video_settings/video_settings.json new file mode 100644 index 0000000000..0a0efd9a53 --- /dev/null +++ b/erpnext/utilities/doctype/video_settings/video_settings.json @@ -0,0 +1,49 @@ +{ + "actions": [], + "creation": "2020-08-02 03:50:21.339609", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "enable_youtube_tracking", + "api_key" + ], + "fields": [ + { + "default": "0", + "fieldname": "enable_youtube_tracking", + "fieldtype": "Check", + "label": "Enable YouTube Tracking" + }, + { + "depends_on": "eval:doc.enable_youtube_tracking", + "fieldname": "api_key", + "fieldtype": "Data", + "label": "API Key", + "mandatory_depends_on": "eval:doc.enable_youtube_tracking" + } + ], + "issingle": 1, + "links": [], + "modified": "2020-08-02 03:56:49.673870", + "modified_by": "Administrator", + "module": "Utilities", + "name": "Video Settings", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "print": 1, + "read": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "track_changes": 1 +} \ No newline at end of file diff --git a/erpnext/utilities/doctype/video_settings/video_settings.py b/erpnext/utilities/doctype/video_settings/video_settings.py new file mode 100644 index 0000000000..7008066909 --- /dev/null +++ b/erpnext/utilities/doctype/video_settings/video_settings.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and contributors +# For license information, please see license.txt + +from __future__ import unicode_literals +# import frappe +from frappe.model.document import Document + +class VideoSettings(Document): + pass diff --git a/erpnext/utilities/report/__init__.py b/erpnext/utilities/report/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/erpnext/utilities/report/youtube_interactions/__init__.py b/erpnext/utilities/report/youtube_interactions/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/erpnext/utilities/report/youtube_interactions/youtube_interactions.js b/erpnext/utilities/report/youtube_interactions/youtube_interactions.js new file mode 100644 index 0000000000..f194cca834 --- /dev/null +++ b/erpnext/utilities/report/youtube_interactions/youtube_interactions.js @@ -0,0 +1,9 @@ +// Copyright (c) 2016, Frappe Technologies Pvt. Ltd. and contributors +// For license information, please see license.txt +/* eslint-disable */ + +frappe.query_reports["YouTube Interactions"] = { + "filters": [ + + ] +}; diff --git a/erpnext/utilities/report/youtube_interactions/youtube_interactions.json b/erpnext/utilities/report/youtube_interactions/youtube_interactions.json new file mode 100644 index 0000000000..a40247b6df --- /dev/null +++ b/erpnext/utilities/report/youtube_interactions/youtube_interactions.json @@ -0,0 +1,27 @@ +{ + "add_total_row": 0, + "creation": "2020-08-02 05:05:00.457093", + "disable_prepared_report": 0, + "disabled": 0, + "docstatus": 0, + "doctype": "Report", + "idx": 0, + "is_standard": "Yes", + "modified": "2020-08-02 05:05:00.457093", + "modified_by": "Administrator", + "module": "Utilities", + "name": "YouTube Interactions", + "owner": "Administrator", + "prepared_report": 0, + "ref_doctype": "Video", + "report_name": "YouTube Interactions", + "report_type": "Script Report", + "roles": [ + { + "role": "All" + }, + { + "role": "System Manager" + } + ] +} \ No newline at end of file diff --git a/erpnext/utilities/report/youtube_interactions/youtube_interactions.py b/erpnext/utilities/report/youtube_interactions/youtube_interactions.py new file mode 100644 index 0000000000..169d0716b0 --- /dev/null +++ b/erpnext/utilities/report/youtube_interactions/youtube_interactions.py @@ -0,0 +1,72 @@ +# Copyright (c) 2013, Frappe Technologies Pvt. Ltd. and contributors +# For license information, please see license.txt + +from __future__ import unicode_literals +import frappe +from frappe import _ + +def execute(filters=None): + columns = get_columns() + data = get_data() + return columns, data + +def get_columns(): + return [ + { + "label": _("Published Date"), + "fieldname": "publish_date", + "fieldtype": "Date", + "width": 100 + }, + { + "label": _("Title"), + "fieldname": "title", + "fieldtype": "Data", + "width": 100 + }, + { + "label": _("Provider"), + "fieldname": "provider", + "fieldtype": "Data", + "width": 100 + }, + { + "label": _("Views"), + "fieldname": "view_count", + "fieldtype": "Float", + "width": 100 + }, + { + "label": _("Likes"), + "fieldname": "like_count", + "fieldtype": "Float", + "width": 100 + }, + { + "label": _("Dislikes"), + "fieldname": "dislike_count", + "fieldtype": "Float", + "width": 100 + }, + { + "label": _("Views"), + "fieldname": "view_count", + "fieldtype": "Float", + "width": 100 + }, + { + "label": _("Like:Dislike Ratio"), + "fieldname": "ratio", + "fieldtype": "Data", + "width": 100 + } + ] + +def get_data(): + return frappe.db.sql(""" + SELECT + publish_date, title, provider, + view_count, like_count, dislike_count, comment_count + FROM `tabVideo` + WHERE view_count is not null + ORDER BY view_count desc""") \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 912d61f7a6..872d78caa3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,7 @@ plaid-python==3.4.0 pycountry==19.8.18 PyGithub==1.44.1 python-stdnum==1.12 +python-youtube==0.6.0 taxjar==1.9.0 tweepy==3.8.0 Unidecode==1.1.1 From ccf4ab9f852f7af8cf669abd1bd664d9e15dfc58 Mon Sep 17 00:00:00 2001 From: marination Date: Sun, 2 Aug 2020 16:26:36 +0530 Subject: [PATCH 02/13] chore: Added Interactions Report and behaviour fixes - Youtube Interactions Report with Chart and Summary - Statistics change in doc on refresh and get updated in db as well --- erpnext/utilities/doctype/video/video.js | 11 +-- erpnext/utilities/doctype/video/video.py | 27 ++++++- .../youtube_interactions.js | 13 ++- .../youtube_interactions.py | 79 ++++++++++++++----- 4 files changed, 97 insertions(+), 33 deletions(-) diff --git a/erpnext/utilities/doctype/video/video.js b/erpnext/utilities/doctype/video/video.js index 4dd4e67a7f..c2994ecc96 100644 --- a/erpnext/utilities/doctype/video/video.js +++ b/erpnext/utilities/doctype/video/video.js @@ -21,17 +21,10 @@ frappe.ui.form.on('Video', { var youtube_id = frm.doc.url.match(expression)[1]; frappe.call({ - method: "erpnext.utilities.doctype.video.video.update_video_stats", + method: "erpnext.utilities.doctype.video.video.get_video_stats", args: { + docname: frm.doc.name, youtube_id: youtube_id - }, - callback: (r) => { - var result = r.message; - var fields = ['like_count', 'view_count', 'dislike_count', 'comment_count']; - fields.forEach((field) => { - frm.doc[field] = result[field]; - }) - frm.refresh_fields(); } }); } diff --git a/erpnext/utilities/doctype/video/video.py b/erpnext/utilities/doctype/video/video.py index 263884a93d..bea7904609 100644 --- a/erpnext/utilities/doctype/video/video.py +++ b/erpnext/utilities/doctype/video/video.py @@ -5,24 +5,43 @@ from __future__ import unicode_literals import frappe from frappe.model.document import Document +from six import string_types from pyyoutube import Api class Video(Document): pass @frappe.whitelist() -def update_video_stats(youtube_id): - ''' +def get_video_stats(docname, youtube_id, update=True): + '''Returns/Sets video statistics + :param docname: Name of Video :param youtube_id: Unique ID from URL + :param update: Updates db stats value if True, else returns statistics ''' + if isinstance(update, string_types): + update = json.loads(update) + api_key = frappe.db.get_single_value("Video Settings", "api_key") api = Api(api_key=api_key) video = api.get_video_by_id(video_id=youtube_id) video_stats = video.items[0].to_dict().get('statistics') - return { + stats = { 'like_count' : video_stats.get('likeCount'), 'view_count' : video_stats.get('viewCount'), 'dislike_count' : video_stats.get('dislikeCount'), 'comment_count' : video_stats.get('commentCount') - } \ No newline at end of file + } + + if not update: + return stats + + frappe.db.sql(""" + UPDATE `tabVideo` + SET + like_count = %(like_count)s, + view_count = %(view_count)s, + dislike_count = %(dislike_count)s, + comment_count = %(comment_count)s + WHERE name = {0}""".format(frappe.db.escape(docname)), stats) + frappe.db.commit() \ No newline at end of file diff --git a/erpnext/utilities/report/youtube_interactions/youtube_interactions.js b/erpnext/utilities/report/youtube_interactions/youtube_interactions.js index f194cca834..6e3e4e6980 100644 --- a/erpnext/utilities/report/youtube_interactions/youtube_interactions.js +++ b/erpnext/utilities/report/youtube_interactions/youtube_interactions.js @@ -4,6 +4,17 @@ frappe.query_reports["YouTube Interactions"] = { "filters": [ - + { + fieldname: "from_date", + label: __("From Date"), + fieldtype: "Date", + default: frappe.datetime.add_months(frappe.datetime.now_date(), -12), + }, + { + fieldname:"to_date", + label: __("To Date"), + fieldtype: "Date", + default: frappe.datetime.now_date(), + } ] }; diff --git a/erpnext/utilities/report/youtube_interactions/youtube_interactions.py b/erpnext/utilities/report/youtube_interactions/youtube_interactions.py index 169d0716b0..3516a35097 100644 --- a/erpnext/utilities/report/youtube_interactions/youtube_interactions.py +++ b/erpnext/utilities/report/youtube_interactions/youtube_interactions.py @@ -4,11 +4,16 @@ from __future__ import unicode_literals import frappe from frappe import _ +from frappe.utils import flt def execute(filters=None): + if not frappe.db.get_single_value("Video Settings", "enable_youtube_tracking") or not filters: + return [], [] + columns = get_columns() - data = get_data() - return columns, data + data = get_data(filters) + chart_data, summary = get_chart_summary_data(data) + return columns, data, None, chart_data, summary def get_columns(): return [ @@ -22,25 +27,25 @@ def get_columns(): "label": _("Title"), "fieldname": "title", "fieldtype": "Data", - "width": 100 + "width": 200 }, { - "label": _("Provider"), - "fieldname": "provider", - "fieldtype": "Data", + "label": _("Duration"), + "fieldname": "duration", + "fieldtype": "Duration", "width": 100 }, { "label": _("Views"), "fieldname": "view_count", "fieldtype": "Float", - "width": 100 + "width": 200 }, { "label": _("Likes"), "fieldname": "like_count", "fieldtype": "Float", - "width": 100 + "width": 200 }, { "label": _("Dislikes"), @@ -49,24 +54,60 @@ def get_columns(): "width": 100 }, { - "label": _("Views"), - "fieldname": "view_count", + "label": _("Comments"), + "fieldname": "comment_count", "fieldtype": "Float", "width": 100 - }, - { - "label": _("Like:Dislike Ratio"), - "fieldname": "ratio", - "fieldtype": "Data", - "width": 100 } ] -def get_data(): +def get_data(filters): return frappe.db.sql(""" SELECT - publish_date, title, provider, + publish_date, title, provider, duration, view_count, like_count, dislike_count, comment_count FROM `tabVideo` WHERE view_count is not null - ORDER BY view_count desc""") \ No newline at end of file + and publish_date between %(from_date)s and %(to_date)s + ORDER BY view_count desc""", filters, as_dict=1) + +def get_chart_summary_data(data): + labels, likes, views = [], [], [] + total_views = 0 + + for row in data: + labels.append(row.get('title')) + likes.append(row.get('like_count')) + views.append(row.get('view_count')) + total_views += flt(row.get('view_count')) + + + chart_data = { + "data" : { + "labels" : labels, + "datasets" : [ + { + "name" : "Likes", + "values" : likes + }, + { + "name" : "Views", + "values" : views + } + ] + }, + "type": "bar", + "barOptions": { + "stacked": 1 + }, + } + + summary = [ + { + "value": total_views, + "indicator": "Blue", + "label": "Total Views", + "datatype": "Float", + } + ] + return chart_data, summary \ No newline at end of file From c16ace6732ba27061da55d9f20171fe4693972d7 Mon Sep 17 00:00:00 2001 From: marination Date: Sun, 2 Aug 2020 16:43:15 +0530 Subject: [PATCH 03/13] fix: Codacy --- erpnext/utilities/doctype/video/video.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/erpnext/utilities/doctype/video/video.py b/erpnext/utilities/doctype/video/video.py index bea7904609..002ee681ed 100644 --- a/erpnext/utilities/doctype/video/video.py +++ b/erpnext/utilities/doctype/video/video.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals import frappe +import json from frappe.model.document import Document from six import string_types from pyyoutube import Api @@ -14,6 +15,7 @@ class Video(Document): @frappe.whitelist() def get_video_stats(docname, youtube_id, update=True): '''Returns/Sets video statistics + :param docname: Name of Video :param youtube_id: Unique ID from URL :param update: Updates db stats value if True, else returns statistics @@ -43,5 +45,5 @@ def get_video_stats(docname, youtube_id, update=True): view_count = %(view_count)s, dislike_count = %(dislike_count)s, comment_count = %(comment_count)s - WHERE name = {0}""".format(frappe.db.escape(docname)), stats) + WHERE name = {0}""".format(frappe.db.escape(docname)), stats) #nosec frappe.db.commit() \ No newline at end of file From e3495116dd959bccbfaa39157e62b151c688c47e Mon Sep 17 00:00:00 2001 From: marination Date: Sun, 2 Aug 2020 17:13:34 +0530 Subject: [PATCH 04/13] chore: Error Logging and exception hnadling on connection failure --- erpnext/utilities/doctype/video/video.py | 42 +++++++++++++----------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/erpnext/utilities/doctype/video/video.py b/erpnext/utilities/doctype/video/video.py index 002ee681ed..a2a4a7b745 100644 --- a/erpnext/utilities/doctype/video/video.py +++ b/erpnext/utilities/doctype/video/video.py @@ -26,24 +26,28 @@ def get_video_stats(docname, youtube_id, update=True): api_key = frappe.db.get_single_value("Video Settings", "api_key") api = Api(api_key=api_key) - video = api.get_video_by_id(video_id=youtube_id) - video_stats = video.items[0].to_dict().get('statistics') - stats = { - 'like_count' : video_stats.get('likeCount'), - 'view_count' : video_stats.get('viewCount'), - 'dislike_count' : video_stats.get('dislikeCount'), - 'comment_count' : video_stats.get('commentCount') - } + try: + video = api.get_video_by_id(video_id=youtube_id) + video_stats = video.items[0].to_dict().get('statistics') + stats = { + 'like_count' : video_stats.get('likeCount'), + 'view_count' : video_stats.get('viewCount'), + 'dislike_count' : video_stats.get('dislikeCount'), + 'comment_count' : video_stats.get('commentCount') + } - if not update: - return stats + if not update: + return stats - frappe.db.sql(""" - UPDATE `tabVideo` - SET - like_count = %(like_count)s, - view_count = %(view_count)s, - dislike_count = %(dislike_count)s, - comment_count = %(comment_count)s - WHERE name = {0}""".format(frappe.db.escape(docname)), stats) #nosec - frappe.db.commit() \ No newline at end of file + frappe.db.sql(""" + UPDATE `tabVideo` + SET + like_count = %(like_count)s, + view_count = %(view_count)s, + dislike_count = %(dislike_count)s, + comment_count = %(comment_count)s + WHERE name = {0}""".format(frappe.db.escape(docname)), stats) #nosec + frappe.db.commit() + except: + message = "Please make sure you are connected to the Internet" + frappe.log_error(message + "\n\n" + frappe.get_traceback(), "Failed to Update YouTube Statistics for Video: {0}".format(docname)) \ No newline at end of file From 9ce38de439ebd2a240539542a8de40f8ae2d2417 Mon Sep 17 00:00:00 2001 From: marination Date: Mon, 7 Sep 2020 13:27:17 +0530 Subject: [PATCH 05/13] chore: Key Validation and restructure - Added API Key validation in Video Settings - Moved Statistics section above description in Video - Moved id retrieving from URL to py side - Removed js side calls, added section toggling in js - Restructured Code --- erpnext/utilities/doctype/video/video.js | 28 ++------ erpnext/utilities/doctype/video/video.json | 10 +-- erpnext/utilities/doctype/video/video.py | 71 +++++++++---------- .../doctype/video_settings/video_settings.py | 16 ++++- 4 files changed, 61 insertions(+), 64 deletions(-) diff --git a/erpnext/utilities/doctype/video/video.js b/erpnext/utilities/doctype/video/video.js index c2994ecc96..9cb5a155ad 100644 --- a/erpnext/utilities/doctype/video/video.js +++ b/erpnext/utilities/doctype/video/video.js @@ -3,29 +3,15 @@ frappe.ui.form.on('Video', { refresh: function (frm) { - if (frm.doc.provider === "YouTube") { - frappe.db.get_single_value("Video Settings", "enable_youtube_tracking").then(value => { - if (value) { - frm.events.get_video_stats(frm); - } else { - frm.set_df_property('youtube_tracking_section', 'hidden', true); - } - }); - } - + frm.events.toggle_youtube_statistics_section(frm); frm.add_custom_button("Watch Video", () => frappe.help.show_video(frm.doc.url, frm.doc.title)); }, - get_video_stats: (frm) => { - const expression = '(?:youtube.com/(?:[^/]+/.+/|(?:v|e(?:mbed)?)/|.*[?&]v=)|youtu.be/)([^"&?\\s]{11})'; - var youtube_id = frm.doc.url.match(expression)[1]; - - frappe.call({ - method: "erpnext.utilities.doctype.video.video.get_video_stats", - args: { - docname: frm.doc.name, - youtube_id: youtube_id - } - }); + toggle_youtube_statistics_section: (frm) => { + if (frm.doc.provider === "YouTube") { + frappe.db.get_single_value("Video Settings", "enable_youtube_tracking").then( val => { + frm.toggle_display("youtube_tracking_section", val); + }); + } } }); diff --git a/erpnext/utilities/doctype/video/video.json b/erpnext/utilities/doctype/video/video.json index a6c0f3f82a..11df56c77d 100644 --- a/erpnext/utilities/doctype/video/video.json +++ b/erpnext/utilities/doctype/video/video.json @@ -14,15 +14,15 @@ "column_break_4", "publish_date", "duration", - "section_break_7", - "description", - "image", "youtube_tracking_section", "like_count", "view_count", "col_break", "dislike_count", - "comment_count" + "comment_count", + "section_break_7", + "description", + "image" ], "fields": [ { @@ -122,7 +122,7 @@ ], "image_field": "image", "links": [], - "modified": "2020-08-02 04:26:16.345569", + "modified": "2020-09-04 12:59:28.283622", "modified_by": "Administrator", "module": "Utilities", "name": "Video", diff --git a/erpnext/utilities/doctype/video/video.py b/erpnext/utilities/doctype/video/video.py index a2a4a7b745..2299f95f76 100644 --- a/erpnext/utilities/doctype/video/video.py +++ b/erpnext/utilities/doctype/video/video.py @@ -5,49 +5,48 @@ from __future__ import unicode_literals import frappe import json +import re from frappe.model.document import Document +from frappe import _ from six import string_types from pyyoutube import Api class Video(Document): - pass + def validate(self): + self.set_youtube_statistics() + + def set_youtube_statistics(self): + tracking_enabled = frappe.db.get_single_value("Video Settings", "enable_youtube_tracking") + if self.provider == "YouTube" and not tracking_enabled: + return + + api_key = frappe.db.get_single_value("Video Settings", "api_key") + youtube_id = get_id_from_url(self.url) + api = Api(api_key=api_key) + + try: + video = api.get_video_by_id(video_id=youtube_id) + video_stats = video.items[0].to_dict().get('statistics') + + self.like_count = video_stats.get('likeCount') + self.view_count = video_stats.get('viewCount') + self.dislike_count = video_stats.get('dislikeCount') + self.comment_count = video_stats.get('commentCount') + + except Exception: + title = "Failed to Update YouTube Statistics for Video: {0}".format(self.name) + frappe.log_error(title + "\n\n" + frappe.get_traceback(), title=title) @frappe.whitelist() -def get_video_stats(docname, youtube_id, update=True): - '''Returns/Sets video statistics - - :param docname: Name of Video - :param youtube_id: Unique ID from URL - :param update: Updates db stats value if True, else returns statistics +def get_id_from_url(url): ''' - if isinstance(update, string_types): - update = json.loads(update) + Returns video id from url - api_key = frappe.db.get_single_value("Video Settings", "api_key") - api = Api(api_key=api_key) + :param youtube url: String URL + ''' + if not isinstance(url, string_types): + frappe.throw(_("URL can only be a string"), title=_("Invalid URL")) - try: - video = api.get_video_by_id(video_id=youtube_id) - video_stats = video.items[0].to_dict().get('statistics') - stats = { - 'like_count' : video_stats.get('likeCount'), - 'view_count' : video_stats.get('viewCount'), - 'dislike_count' : video_stats.get('dislikeCount'), - 'comment_count' : video_stats.get('commentCount') - } - - if not update: - return stats - - frappe.db.sql(""" - UPDATE `tabVideo` - SET - like_count = %(like_count)s, - view_count = %(view_count)s, - dislike_count = %(dislike_count)s, - comment_count = %(comment_count)s - WHERE name = {0}""".format(frappe.db.escape(docname)), stats) #nosec - frappe.db.commit() - except: - message = "Please make sure you are connected to the Internet" - frappe.log_error(message + "\n\n" + frappe.get_traceback(), "Failed to Update YouTube Statistics for Video: {0}".format(docname)) \ No newline at end of file + pattern = re.compile(r'[a-z\:\//\.]+(youtube|youtu)\.(com|be)/(watch\?v=|embed/|.+\?v=)?([^"&?\s]{11})?') + id = pattern.match(url) + return id.groups()[-1] \ No newline at end of file diff --git a/erpnext/utilities/doctype/video_settings/video_settings.py b/erpnext/utilities/doctype/video_settings/video_settings.py index 7008066909..36fb54f015 100644 --- a/erpnext/utilities/doctype/video_settings/video_settings.py +++ b/erpnext/utilities/doctype/video_settings/video_settings.py @@ -3,8 +3,20 @@ # For license information, please see license.txt from __future__ import unicode_literals -# import frappe +import frappe +from frappe import _ from frappe.model.document import Document +from apiclient.discovery import build class VideoSettings(Document): - pass + def validate(self): + self.validate_youtube_api_key() + + def validate_youtube_api_key(self): + if self.enable_youtube_tracking and self.api_key: + try: + build("youtube", "v3", developerKey=self.api_key) + except Exception: + title = _("Failed to Authenticate the API key.") + frappe.log_error(title + "\n\n" + frappe.get_traceback(), title=title) + frappe.throw(title + " Please check the error logs.", title=_("Invalid Credentials")) \ No newline at end of file From 06ee0ea00bab3793ef7fcb48348ba4a3ef501c0b Mon Sep 17 00:00:00 2001 From: marination Date: Mon, 7 Sep 2020 18:49:06 +0530 Subject: [PATCH 06/13] feat: Added Scheduler Job to auto update statistics - Added frequency in Video Settings - Cron job to check half hourly job - Hourly job to run job based on frequency - Patch to set youtube id in video for ease in computing --- erpnext/hooks.py | 6 + erpnext/patches.txt | 1 + erpnext/patches/v13_0/set_youtube_video_id.py | 8 ++ erpnext/utilities/doctype/video/video.json | 10 +- erpnext/utilities/doctype/video/video.py | 124 ++++++++++++++++-- .../video_settings/video_settings.json | 15 ++- 6 files changed, 150 insertions(+), 14 deletions(-) create mode 100644 erpnext/patches/v13_0/set_youtube_video_id.py diff --git a/erpnext/hooks.py b/erpnext/hooks.py index 95a836fe65..db1fd2f3fc 100644 --- a/erpnext/hooks.py +++ b/erpnext/hooks.py @@ -282,6 +282,11 @@ auto_cancel_exempted_doctypes= [ ] scheduler_events = { + "cron": { + "0/30 * * * *": [ + "erpnext.utilities.doctype.video.video.update_youtube_data_half_hourly", + ] + }, "all": [ "erpnext.projects.doctype.project.project.project_status_update_reminder", "erpnext.healthcare.doctype.patient_appointment.patient_appointment.send_appointment_reminder", @@ -297,6 +302,7 @@ scheduler_events = { "erpnext.projects.doctype.project.project.collect_project_status", "erpnext.hr.doctype.shift_type.shift_type.process_auto_attendance_for_all_shifts", "erpnext.support.doctype.issue.issue.set_service_level_agreement_variance", + "erpnext.utilities.doctype.video.video.update_youtube_data" ], "daily": [ "erpnext.stock.reorder_item.reorder_item", diff --git a/erpnext/patches.txt b/erpnext/patches.txt index 3bd416952f..2a52ff67e5 100644 --- a/erpnext/patches.txt +++ b/erpnext/patches.txt @@ -718,3 +718,4 @@ erpnext.patches.v13_0.delete_report_requested_items_to_order erpnext.patches.v12_0.update_item_tax_template_company erpnext.patches.v13_0.move_branch_code_to_bank_account erpnext.patches.v13_0.healthcare_lab_module_rename_doctypes +erpnext.patches.v13_0.set_youtube_video_id diff --git a/erpnext/patches/v13_0/set_youtube_video_id.py b/erpnext/patches/v13_0/set_youtube_video_id.py new file mode 100644 index 0000000000..8e5dd306a9 --- /dev/null +++ b/erpnext/patches/v13_0/set_youtube_video_id.py @@ -0,0 +1,8 @@ +from __future__ import unicode_literals +import frappe +from erpnext.utilities.doctype.video.video import get_id_from_url + +def execute(): + for video in frappe.get_all("Video", fields=["name", "url", "youtube_video_id"]): + if video.url and not video.youtube_video_id: + frappe.db.set_value("Video", video.name, "youtube_video_id", get_id_from_url(video.url)) \ No newline at end of file diff --git a/erpnext/utilities/doctype/video/video.json b/erpnext/utilities/doctype/video/video.json index 11df56c77d..2a82db2514 100644 --- a/erpnext/utilities/doctype/video/video.json +++ b/erpnext/utilities/doctype/video/video.json @@ -11,6 +11,7 @@ "title", "provider", "url", + "youtube_video_id", "column_break_4", "publish_date", "duration", @@ -118,11 +119,18 @@ "fieldname": "youtube_tracking_section", "fieldtype": "Section Break", "label": "Youtube Statistics" + }, + { + "fieldname": "youtube_video_id", + "fieldtype": "Data", + "hidden": 1, + "label": "Youtube ID", + "read_only": 1 } ], "image_field": "image", "links": [], - "modified": "2020-09-04 12:59:28.283622", + "modified": "2020-09-07 17:02:20.185794", "modified_by": "Administrator", "module": "Utilities", "name": "Video", diff --git a/erpnext/utilities/doctype/video/video.py b/erpnext/utilities/doctype/video/video.py index 2299f95f76..d8653f6446 100644 --- a/erpnext/utilities/doctype/video/video.py +++ b/erpnext/utilities/doctype/video/video.py @@ -4,8 +4,8 @@ from __future__ import unicode_literals import frappe -import json import re +import pytz from frappe.model.document import Document from frappe import _ from six import string_types @@ -13,20 +13,32 @@ from pyyoutube import Api class Video(Document): def validate(self): + self.set_video_id() self.set_youtube_statistics() - def set_youtube_statistics(self): - tracking_enabled = frappe.db.get_single_value("Video Settings", "enable_youtube_tracking") - if self.provider == "YouTube" and not tracking_enabled: + def set_video_id(self): + if self.provider == "YouTube" and self.url and not self.get("youtube_video_id"): + self.youtube_video_id = get_id_from_url(self.url) + + @classmethod + def set_youtube_statistics(self, video_ids=None, update=True): + if self.provider == "YouTube" and not is_tracking_enabled(): return api_key = frappe.db.get_single_value("Video Settings", "api_key") - youtube_id = get_id_from_url(self.url) api = Api(api_key=api_key) try: - video = api.get_video_by_id(video_id=youtube_id) - video_stats = video.items[0].to_dict().get('statistics') + video_id = video_ids or self.youtube_video_id + video = api.get_video_by_id(video_id=video_id) + + if video_ids: + video_stats = video.items + else: + video_stats = video.items[0].to_dict().get('statistics') + + if not update: + return video_stats self.like_count = video_stats.get('likeCount') self.view_count = video_stats.get('viewCount') @@ -37,16 +49,106 @@ class Video(Document): title = "Failed to Update YouTube Statistics for Video: {0}".format(self.name) frappe.log_error(title + "\n\n" + frappe.get_traceback(), title=title) +def is_tracking_enabled(): + return frappe.db.get_single_value("Video Settings", "enable_youtube_tracking") + +def get_frequency(value): + if not value: + return None + + # Return frequency in hours + if value != "Daily": + return frappe.utils.cint(value[:2].strip()) + else: + # 24 hours for Daily + return 24 + + +def update_youtube_data_half_hourly(): + # Called every 30 mins via hooks + frequency = get_frequency(frappe.db.get_single_value("Video Settings", "frequency")) + if not is_tracking_enabled() or not frequency: + return + + if frequency == 30: + batch_update_data() + + +def update_youtube_data(): + # Called every hour via hooks + frequency = get_frequency(frappe.db.get_single_value("Video Settings", "frequency")) + + # if frequency is 30 mins dont proceed, as its handled in another method + if not is_tracking_enabled() or not frequency or frequency == 30: + return + + time = datetime.now() + timezone = pytz.timezone(frappe.utils.get_time_zone()) + site_time = time.astimezone(timezone) + + if site_time.hour % frequency == 0: + batch_update_youtube_data() + +def get_formatted_ids(video_list): + # format ids to comma separated string for bulk request + ids = [] + for video in video_list: + ids.append(video.youtube_video_id) + + return ','.join(ids) + @frappe.whitelist() def get_id_from_url(url): - ''' + """ Returns video id from url - :param youtube url: String URL - ''' + """ if not isinstance(url, string_types): frappe.throw(_("URL can only be a string"), title=_("Invalid URL")) pattern = re.compile(r'[a-z\:\//\.]+(youtube|youtu)\.(com|be)/(watch\?v=|embed/|.+\?v=)?([^"&?\s]{11})?') id = pattern.match(url) - return id.groups()[-1] \ No newline at end of file + return id.groups()[-1] + + +@frappe.whitelist() +def batch_update_youtube_data(): + def prepare_and_set_data(video_list): + video_ids = get_formatted_ids(video_list) + Video.provider = "YouTube" + stats = Video.set_youtube_statistics(video_ids=video_ids, update=False) + set_youtube_data(stats) + + def set_youtube_data(entries): + for entry in entries: + video_stats = entry.to_dict().get('statistics') + video_id = entry.to_dict().get('id') + stats = { + 'like_count' : video_stats.get('likeCount'), + 'view_count' : video_stats.get('viewCount'), + 'dislike_count' : video_stats.get('dislikeCount'), + 'comment_count' : video_stats.get('commentCount') + } + + frappe.db.sql(""" + UPDATE `tabVideo` + SET + like_count = %(like_count)s, + view_count = %(view_count)s, + dislike_count = %(dislike_count)s, + comment_count = %(comment_count)s + WHERE youtube_video_id = '{0}'""".format(video_id), stats) + + frappe.log_error("yooooooooo") + + video_list = frappe.get_all("Video", fields=["youtube_video_id"]) + if len(video_list) > 50: + # Update in batches of 50 + start, end = 0, 50 + while start < len(video_list): + batch = video_list[start:end] + prepare_and_set_data(batch) + start += 50 + end += 50 + else: + prepare_and_set_data(video_list) \ No newline at end of file diff --git a/erpnext/utilities/doctype/video_settings/video_settings.json b/erpnext/utilities/doctype/video_settings/video_settings.json index 0a0efd9a53..fb3274decd 100644 --- a/erpnext/utilities/doctype/video_settings/video_settings.json +++ b/erpnext/utilities/doctype/video_settings/video_settings.json @@ -6,7 +6,8 @@ "engine": "InnoDB", "field_order": [ "enable_youtube_tracking", - "api_key" + "api_key", + "frequency" ], "fields": [ { @@ -21,11 +22,21 @@ "fieldtype": "Data", "label": "API Key", "mandatory_depends_on": "eval:doc.enable_youtube_tracking" + }, + { + "default": "1 hr", + "depends_on": "eval:doc.enable_youtube_tracking", + "fieldname": "frequency", + "fieldtype": "Select", + "label": "Frequency", + "mandatory_depends_on": "eval:doc.enable_youtube_tracking", + "options": "30 mins\n1 hr\n6 hrs\nDaily" } ], + "index_web_pages_for_search": 1, "issingle": 1, "links": [], - "modified": "2020-08-02 03:56:49.673870", + "modified": "2020-09-07 16:09:00.360668", "modified_by": "Administrator", "module": "Utilities", "name": "Video Settings", From f18dd05c00c92033cd0f1e1db49f6e6b60d22e5f Mon Sep 17 00:00:00 2001 From: marination Date: Mon, 7 Sep 2020 18:59:57 +0530 Subject: [PATCH 07/13] fix: Missing import and typo --- erpnext/utilities/doctype/video/video.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/erpnext/utilities/doctype/video/video.py b/erpnext/utilities/doctype/video/video.py index d8653f6446..3b6bd7e0df 100644 --- a/erpnext/utilities/doctype/video/video.py +++ b/erpnext/utilities/doctype/video/video.py @@ -8,6 +8,7 @@ import re import pytz from frappe.model.document import Document from frappe import _ +from datetime import datetime from six import string_types from pyyoutube import Api @@ -71,7 +72,7 @@ def update_youtube_data_half_hourly(): return if frequency == 30: - batch_update_data() + batch_update_youtube_data() def update_youtube_data(): From 53cd60c217fc2442dc8d1db23c2595446efc08bb Mon Sep 17 00:00:00 2001 From: marination Date: Mon, 7 Sep 2020 20:45:50 +0530 Subject: [PATCH 08/13] fix: Patch reload doctype --- erpnext/patches/v13_0/set_youtube_video_id.py | 2 ++ erpnext/utilities/doctype/video/video.py | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/erpnext/patches/v13_0/set_youtube_video_id.py b/erpnext/patches/v13_0/set_youtube_video_id.py index 8e5dd306a9..c3b49eb4fe 100644 --- a/erpnext/patches/v13_0/set_youtube_video_id.py +++ b/erpnext/patches/v13_0/set_youtube_video_id.py @@ -3,6 +3,8 @@ import frappe from erpnext.utilities.doctype.video.video import get_id_from_url def execute(): + frappe.reload_doc("utilities", "doctype","video") + for video in frappe.get_all("Video", fields=["name", "url", "youtube_video_id"]): if video.url and not video.youtube_video_id: frappe.db.set_value("Video", video.name, "youtube_video_id", get_id_from_url(video.url)) \ No newline at end of file diff --git a/erpnext/utilities/doctype/video/video.py b/erpnext/utilities/doctype/video/video.py index 3b6bd7e0df..6971beb5d7 100644 --- a/erpnext/utilities/doctype/video/video.py +++ b/erpnext/utilities/doctype/video/video.py @@ -140,8 +140,6 @@ def batch_update_youtube_data(): comment_count = %(comment_count)s WHERE youtube_video_id = '{0}'""".format(video_id), stats) - frappe.log_error("yooooooooo") - video_list = frappe.get_all("Video", fields=["youtube_video_id"]) if len(video_list) > 50: # Update in batches of 50 From 0a0e258d8b43f88dc51fc1ed1aac48b55d4ca22a Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 10 Sep 2020 13:12:35 +0530 Subject: [PATCH 09/13] feat: Add Desk Page for Utilities for Video group --- .../desk_page/utilities/utilities.json | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 erpnext/utilities/desk_page/utilities/utilities.json diff --git a/erpnext/utilities/desk_page/utilities/utilities.json b/erpnext/utilities/desk_page/utilities/utilities.json new file mode 100644 index 0000000000..591eab5ed4 --- /dev/null +++ b/erpnext/utilities/desk_page/utilities/utilities.json @@ -0,0 +1,29 @@ +{ + "cards": [ + { + "hidden": 0, + "label": "Video", + "links": "[\n {\n \"description\": \"Video\",\n \"label\": \"Video\",\n \"name\": \"Video\",\n \"type\": \"doctype\"\n },\n {\n \"description\": \"Video settings\",\n \"label\": \"Video Settings\",\n \"name\": \"Video Settings\",\n \"type\": \"doctype\"\n }\n]" + } + ], + "category": "Modules", + "charts": [], + "creation": "2020-09-10 12:21:22.335307", + "developer_mode_only": 0, + "disable_user_customization": 0, + "docstatus": 0, + "doctype": "Desk Page", + "extends_another_page": 0, + "hide_custom": 0, + "idx": 0, + "is_standard": 1, + "label": "Utilities", + "modified": "2020-09-10 12:33:30.089853", + "modified_by": "user@erpnext.com", + "module": "Utilities", + "name": "Utilities", + "owner": "user@erpnext.com", + "pin_to_bottom": 1, + "pin_to_top": 0, + "shortcuts": [] +} \ No newline at end of file From 71ac39936977e6408d815a5ecb100fd51f8dadb8 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Thu, 10 Sep 2020 13:57:44 +0530 Subject: [PATCH 10/13] fix: Commonified function and updated scheduler events Co-authored-by: marination --- erpnext/hooks.py | 3 +- erpnext/utilities/doctype/video/video.py | 42 ++++++++++-------------- 2 files changed, 19 insertions(+), 26 deletions(-) diff --git a/erpnext/hooks.py b/erpnext/hooks.py index 3beae3b6c8..f8b6be70ca 100644 --- a/erpnext/hooks.py +++ b/erpnext/hooks.py @@ -284,7 +284,7 @@ auto_cancel_exempted_doctypes= [ scheduler_events = { "cron": { "0/30 * * * *": [ - "erpnext.utilities.doctype.video.video.update_youtube_data_half_hourly", + "erpnext.utilities.doctype.video.video.update_youtube_data", ] }, "all": [ @@ -302,7 +302,6 @@ scheduler_events = { "erpnext.projects.doctype.project.project.collect_project_status", "erpnext.hr.doctype.shift_type.shift_type.process_auto_attendance_for_all_shifts", "erpnext.support.doctype.issue.issue.set_service_level_agreement_variance", - "erpnext.utilities.doctype.video.video.update_youtube_data" ], "daily": [ "erpnext.stock.reorder_item.reorder_item", diff --git a/erpnext/utilities/doctype/video/video.py b/erpnext/utilities/doctype/video/video.py index 6971beb5d7..7918b828aa 100644 --- a/erpnext/utilities/doctype/video/video.py +++ b/erpnext/utilities/doctype/video/video.py @@ -50,45 +50,37 @@ class Video(Document): title = "Failed to Update YouTube Statistics for Video: {0}".format(self.name) frappe.log_error(title + "\n\n" + frappe.get_traceback(), title=title) + def is_tracking_enabled(): return frappe.db.get_single_value("Video Settings", "enable_youtube_tracking") -def get_frequency(value): - if not value: - return None - # Return frequency in hours +def get_frequency(value): + # Return numeric value from frequency field, return 1 as fallback default value: 1 hour if value != "Daily": return frappe.utils.cint(value[:2].strip()) - else: - # 24 hours for Daily + elif value: return 24 - - -def update_youtube_data_half_hourly(): - # Called every 30 mins via hooks - frequency = get_frequency(frappe.db.get_single_value("Video Settings", "frequency")) - if not is_tracking_enabled() or not frequency: - return - - if frequency == 30: - batch_update_youtube_data() + return 1 def update_youtube_data(): - # Called every hour via hooks - frequency = get_frequency(frappe.db.get_single_value("Video Settings", "frequency")) + # Called every 30 minutes via hooks + enable_youtube_tracking, frequency = frappe.db.get_value("Video Settings", "Video Settings", ["enable_youtube_tracking", "frequency"]) - # if frequency is 30 mins dont proceed, as its handled in another method - if not is_tracking_enabled() or not frequency or frequency == 30: + if not enable_youtube_tracking: return + frequency = get_frequency(frequency) time = datetime.now() timezone = pytz.timezone(frappe.utils.get_time_zone()) site_time = time.astimezone(timezone) - if site_time.hour % frequency == 0: + if frequency == 30: batch_update_youtube_data() + elif site_time.hour % frequency == 0: + batch_update_youtube_data() + def get_formatted_ids(video_list): # format ids to comma separated string for bulk request @@ -98,6 +90,7 @@ def get_formatted_ids(video_list): return ','.join(ids) + @frappe.whitelist() def get_id_from_url(url): """ @@ -128,7 +121,8 @@ def batch_update_youtube_data(): 'like_count' : video_stats.get('likeCount'), 'view_count' : video_stats.get('viewCount'), 'dislike_count' : video_stats.get('dislikeCount'), - 'comment_count' : video_stats.get('commentCount') + 'comment_count' : video_stats.get('commentCount'), + 'video_id': video_id } frappe.db.sql(""" @@ -138,7 +132,7 @@ def batch_update_youtube_data(): view_count = %(view_count)s, dislike_count = %(dislike_count)s, comment_count = %(comment_count)s - WHERE youtube_video_id = '{0}'""".format(video_id), stats) + WHERE youtube_video_id = %(video_id)s""", stats) video_list = frappe.get_all("Video", fields=["youtube_video_id"]) if len(video_list) > 50: @@ -150,4 +144,4 @@ def batch_update_youtube_data(): start += 50 end += 50 else: - prepare_and_set_data(video_list) \ No newline at end of file + prepare_and_set_data(video_list) From 799d663398781f55af8e3e6b2f239b23f0e3eecf Mon Sep 17 00:00:00 2001 From: marination Date: Thu, 10 Sep 2020 16:42:20 +0530 Subject: [PATCH 11/13] fix: Make set_youtube_statistics reusable and job logic fix - Make sure job runs only once every hour - set_youtube_statistics should be usable in and outside class --- erpnext/utilities/doctype/video/video.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/erpnext/utilities/doctype/video/video.py b/erpnext/utilities/doctype/video/video.py index 7918b828aa..f519146e6b 100644 --- a/erpnext/utilities/doctype/video/video.py +++ b/erpnext/utilities/doctype/video/video.py @@ -21,7 +21,6 @@ class Video(Document): if self.provider == "YouTube" and self.url and not self.get("youtube_video_id"): self.youtube_video_id = get_id_from_url(self.url) - @classmethod def set_youtube_statistics(self, video_ids=None, update=True): if self.provider == "YouTube" and not is_tracking_enabled(): return @@ -78,7 +77,8 @@ def update_youtube_data(): if frequency == 30: batch_update_youtube_data() - elif site_time.hour % frequency == 0: + elif site_time.hour % frequency == 0 and site_time.minute < 15: + # make sure it runs within the first 15 mins of the hour batch_update_youtube_data() @@ -109,8 +109,8 @@ def get_id_from_url(url): def batch_update_youtube_data(): def prepare_and_set_data(video_list): video_ids = get_formatted_ids(video_list) - Video.provider = "YouTube" - stats = Video.set_youtube_statistics(video_ids=video_ids, update=False) + video_doc = frappe.new_doc("Video") + stats = video_doc.set_youtube_statistics(video_ids=video_ids, update=False) set_youtube_data(stats) def set_youtube_data(entries): From 37b99d202d6f4d7c5ffdffb1739d91aa74965757 Mon Sep 17 00:00:00 2001 From: marination Date: Thu, 10 Sep 2020 18:27:20 +0530 Subject: [PATCH 12/13] fix: Separate function to get bulk and single youtube stats --- erpnext/utilities/doctype/video/video.py | 39 ++++++++++++------------ 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/erpnext/utilities/doctype/video/video.py b/erpnext/utilities/doctype/video/video.py index f519146e6b..c2e414eef8 100644 --- a/erpnext/utilities/doctype/video/video.py +++ b/erpnext/utilities/doctype/video/video.py @@ -14,31 +14,21 @@ from pyyoutube import Api class Video(Document): def validate(self): - self.set_video_id() - self.set_youtube_statistics() + if self.provider == "YouTube" and is_tracking_enabled(): + self.set_video_id() + self.set_youtube_statistics() def set_video_id(self): - if self.provider == "YouTube" and self.url and not self.get("youtube_video_id"): + if self.url and not self.get("youtube_video_id"): self.youtube_video_id = get_id_from_url(self.url) - def set_youtube_statistics(self, video_ids=None, update=True): - if self.provider == "YouTube" and not is_tracking_enabled(): - return - + def set_youtube_statistics(self): api_key = frappe.db.get_single_value("Video Settings", "api_key") api = Api(api_key=api_key) try: - video_id = video_ids or self.youtube_video_id - video = api.get_video_by_id(video_id=video_id) - - if video_ids: - video_stats = video.items - else: - video_stats = video.items[0].to_dict().get('statistics') - - if not update: - return video_stats + video = api.get_video_by_id(video_id=self.youtube_video_id) + video_stats = video.items[0].to_dict().get('statistics') self.like_count = video_stats.get('likeCount') self.view_count = video_stats.get('viewCount') @@ -49,7 +39,6 @@ class Video(Document): title = "Failed to Update YouTube Statistics for Video: {0}".format(self.name) frappe.log_error(title + "\n\n" + frappe.get_traceback(), title=title) - def is_tracking_enabled(): return frappe.db.get_single_value("Video Settings", "enable_youtube_tracking") @@ -107,10 +96,20 @@ def get_id_from_url(url): @frappe.whitelist() def batch_update_youtube_data(): + def get_youtube_statistics(video_ids): + api_key = frappe.db.get_single_value("Video Settings", "api_key") + api = Api(api_key=api_key) + try: + video = api.get_video_by_id(video_id=video_ids) + video_stats = video.items + return video_stats + except Exception: + title = "Failed to Update YouTube Statistics" + frappe.log_error(title + "\n\n" + frappe.get_traceback(), title=title) + def prepare_and_set_data(video_list): video_ids = get_formatted_ids(video_list) - video_doc = frappe.new_doc("Video") - stats = video_doc.set_youtube_statistics(video_ids=video_ids, update=False) + stats = get_youtube_statistics(video_ids) set_youtube_data(stats) def set_youtube_data(entries): From 6087fcaa9baeb4431b3ad22b073b4b6c351aaef1 Mon Sep 17 00:00:00 2001 From: marination Date: Tue, 22 Sep 2020 17:48:04 +0530 Subject: [PATCH 13/13] chore: Added video settings to List View Menu --- erpnext/utilities/doctype/video/video_list.js | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 erpnext/utilities/doctype/video/video_list.js diff --git a/erpnext/utilities/doctype/video/video_list.js b/erpnext/utilities/doctype/video/video_list.js new file mode 100644 index 0000000000..8273a4a781 --- /dev/null +++ b/erpnext/utilities/doctype/video/video_list.js @@ -0,0 +1,7 @@ +frappe.listview_settings["Video"] = { + onload: (listview) => { + listview.page.add_menu_item(__("Video Settings"), function() { + frappe.set_route("Form","Video Settings", "Video Settings"); + }); + } +} \ No newline at end of file