diff --git a/erpnext/hooks.py b/erpnext/hooks.py index 463ad6c94b..f8b6be70ca 100644 --- a/erpnext/hooks.py +++ b/erpnext/hooks.py @@ -282,6 +282,11 @@ auto_cancel_exempted_doctypes= [ ] scheduler_events = { + "cron": { + "0/30 * * * *": [ + "erpnext.utilities.doctype.video.video.update_youtube_data", + ] + }, "all": [ "erpnext.projects.doctype.project.project.project_status_update_reminder", "erpnext.healthcare.doctype.patient_appointment.patient_appointment.send_appointment_reminder", diff --git a/erpnext/patches.txt b/erpnext/patches.txt index a919b670cb..c1aa4a632e 100644 --- a/erpnext/patches.txt +++ b/erpnext/patches.txt @@ -727,3 +727,4 @@ erpnext.patches.v13_0.drop_razorpay_payload_column erpnext.patches.v13_0.update_start_end_date_for_old_shift_assignment erpnext.patches.v13_0.setting_custom_roles_for_some_regional_reports erpnext.patches.v13_0.change_default_pos_print_format +erpnext.patches.v13_0.set_youtube_video_id diff --git a/erpnext/patches/v13_0/set_youtube_video_id.py b/erpnext/patches/v13_0/set_youtube_video_id.py new file mode 100644 index 0000000000..c3b49eb4fe --- /dev/null +++ b/erpnext/patches/v13_0/set_youtube_video_id.py @@ -0,0 +1,10 @@ +from __future__ import unicode_literals +import frappe +from erpnext.utilities.doctype.video.video import get_id_from_url + +def execute(): + frappe.reload_doc("utilities", "doctype","video") + + for video in frappe.get_all("Video", fields=["name", "url", "youtube_video_id"]): + if video.url and not video.youtube_video_id: + frappe.db.set_value("Video", video.name, "youtube_video_id", get_id_from_url(video.url)) \ No newline at end of file diff --git a/erpnext/utilities/desk_page/utilities/utilities.json b/erpnext/utilities/desk_page/utilities/utilities.json new file mode 100644 index 0000000000..591eab5ed4 --- /dev/null +++ b/erpnext/utilities/desk_page/utilities/utilities.json @@ -0,0 +1,29 @@ +{ + "cards": [ + { + "hidden": 0, + "label": "Video", + "links": "[\n {\n \"description\": \"Video\",\n \"label\": \"Video\",\n \"name\": \"Video\",\n \"type\": \"doctype\"\n },\n {\n \"description\": \"Video settings\",\n \"label\": \"Video Settings\",\n \"name\": \"Video Settings\",\n \"type\": \"doctype\"\n }\n]" + } + ], + "category": "Modules", + "charts": [], + "creation": "2020-09-10 12:21:22.335307", + "developer_mode_only": 0, + "disable_user_customization": 0, + "docstatus": 0, + "doctype": "Desk Page", + "extends_another_page": 0, + "hide_custom": 0, + "idx": 0, + "is_standard": 1, + "label": "Utilities", + "modified": "2020-09-10 12:33:30.089853", + "modified_by": "user@erpnext.com", + "module": "Utilities", + "name": "Utilities", + "owner": "user@erpnext.com", + "pin_to_bottom": 1, + "pin_to_top": 0, + "shortcuts": [] +} \ No newline at end of file diff --git a/erpnext/utilities/doctype/video/video.js b/erpnext/utilities/doctype/video/video.js index 056bd3ccd6..9cb5a155ad 100644 --- a/erpnext/utilities/doctype/video/video.js +++ b/erpnext/utilities/doctype/video/video.js @@ -2,7 +2,16 @@ // For license information, please see license.txt frappe.ui.form.on('Video', { - // refresh: function(frm) { + refresh: function (frm) { + frm.events.toggle_youtube_statistics_section(frm); + frm.add_custom_button("Watch Video", () => frappe.help.show_video(frm.doc.url, frm.doc.title)); + }, - // } + toggle_youtube_statistics_section: (frm) => { + if (frm.doc.provider === "YouTube") { + frappe.db.get_single_value("Video Settings", "enable_youtube_tracking").then( val => { + frm.toggle_display("youtube_tracking_section", val); + }); + } + } }); diff --git a/erpnext/utilities/doctype/video/video.json b/erpnext/utilities/doctype/video/video.json index 5d2cc13348..2a82db2514 100644 --- a/erpnext/utilities/doctype/video/video.json +++ b/erpnext/utilities/doctype/video/video.json @@ -11,11 +11,19 @@ "title", "provider", "url", + "youtube_video_id", "column_break_4", "publish_date", "duration", + "youtube_tracking_section", + "like_count", + "view_count", + "col_break", + "dislike_count", + "comment_count", "section_break_7", - "description" + "description", + "image" ], "fields": [ { @@ -37,7 +45,6 @@ { "fieldname": "url", "fieldtype": "Data", - "in_list_view": 1, "label": "URL", "reqd": 1 }, @@ -48,11 +55,12 @@ { "fieldname": "publish_date", "fieldtype": "Date", + "in_list_view": 1, "label": "Publish Date" }, { "fieldname": "duration", - "fieldtype": "Data", + "fieldtype": "Duration", "label": "Duration" }, { @@ -62,13 +70,67 @@ { "fieldname": "description", "fieldtype": "Text Editor", - "in_list_view": 1, "label": "Description", "reqd": 1 + }, + { + "fieldname": "like_count", + "fieldtype": "Float", + "in_list_view": 1, + "label": "Likes", + "no_copy": 1, + "read_only": 1 + }, + { + "fieldname": "view_count", + "fieldtype": "Float", + "in_list_view": 1, + "label": "Views", + "no_copy": 1, + "read_only": 1 + }, + { + "fieldname": "col_break", + "fieldtype": "Column Break" + }, + { + "fieldname": "dislike_count", + "fieldtype": "Float", + "label": "Dislikes", + "no_copy": 1, + "read_only": 1 + }, + { + "fieldname": "comment_count", + "fieldtype": "Float", + "label": "Comments", + "no_copy": 1, + "read_only": 1 + }, + { + "fieldname": "image", + "fieldtype": "Attach Image", + "hidden": 1, + "label": "Image", + "no_copy": 1 + }, + { + "depends_on": "eval:doc.provider==\"YouTube\"", + "fieldname": "youtube_tracking_section", + "fieldtype": "Section Break", + "label": "Youtube Statistics" + }, + { + "fieldname": "youtube_video_id", + "fieldtype": "Data", + "hidden": 1, + "label": "Youtube ID", + "read_only": 1 } ], + "image_field": "image", "links": [], - "modified": "2020-07-21 19:29:46.603734", + "modified": "2020-09-07 17:02:20.185794", "modified_by": "Administrator", "module": "Utilities", "name": "Video", diff --git a/erpnext/utilities/doctype/video/video.py b/erpnext/utilities/doctype/video/video.py index 3c17b560f3..c2e414eef8 100644 --- a/erpnext/utilities/doctype/video/video.py +++ b/erpnext/utilities/doctype/video/video.py @@ -3,8 +3,144 @@ # For license information, please see license.txt from __future__ import unicode_literals -# import frappe +import frappe +import re +import pytz from frappe.model.document import Document +from frappe import _ +from datetime import datetime +from six import string_types +from pyyoutube import Api class Video(Document): - pass + def validate(self): + if self.provider == "YouTube" and is_tracking_enabled(): + self.set_video_id() + self.set_youtube_statistics() + + def set_video_id(self): + if self.url and not self.get("youtube_video_id"): + self.youtube_video_id = get_id_from_url(self.url) + + def set_youtube_statistics(self): + api_key = frappe.db.get_single_value("Video Settings", "api_key") + api = Api(api_key=api_key) + + try: + video = api.get_video_by_id(video_id=self.youtube_video_id) + video_stats = video.items[0].to_dict().get('statistics') + + self.like_count = video_stats.get('likeCount') + self.view_count = video_stats.get('viewCount') + self.dislike_count = video_stats.get('dislikeCount') + self.comment_count = video_stats.get('commentCount') + + except Exception: + title = "Failed to Update YouTube Statistics for Video: {0}".format(self.name) + frappe.log_error(title + "\n\n" + frappe.get_traceback(), title=title) + +def is_tracking_enabled(): + return frappe.db.get_single_value("Video Settings", "enable_youtube_tracking") + + +def get_frequency(value): + # Return numeric value from frequency field, return 1 as fallback default value: 1 hour + if value != "Daily": + return frappe.utils.cint(value[:2].strip()) + elif value: + return 24 + return 1 + + +def update_youtube_data(): + # Called every 30 minutes via hooks + enable_youtube_tracking, frequency = frappe.db.get_value("Video Settings", "Video Settings", ["enable_youtube_tracking", "frequency"]) + + if not enable_youtube_tracking: + return + + frequency = get_frequency(frequency) + time = datetime.now() + timezone = pytz.timezone(frappe.utils.get_time_zone()) + site_time = time.astimezone(timezone) + + if frequency == 30: + batch_update_youtube_data() + elif site_time.hour % frequency == 0 and site_time.minute < 15: + # make sure it runs within the first 15 mins of the hour + batch_update_youtube_data() + + +def get_formatted_ids(video_list): + # format ids to comma separated string for bulk request + ids = [] + for video in video_list: + ids.append(video.youtube_video_id) + + return ','.join(ids) + + +@frappe.whitelist() +def get_id_from_url(url): + """ + Returns video id from url + :param youtube url: String URL + """ + if not isinstance(url, string_types): + frappe.throw(_("URL can only be a string"), title=_("Invalid URL")) + + pattern = re.compile(r'[a-z\:\//\.]+(youtube|youtu)\.(com|be)/(watch\?v=|embed/|.+\?v=)?([^"&?\s]{11})?') + id = pattern.match(url) + return id.groups()[-1] + + +@frappe.whitelist() +def batch_update_youtube_data(): + def get_youtube_statistics(video_ids): + api_key = frappe.db.get_single_value("Video Settings", "api_key") + api = Api(api_key=api_key) + try: + video = api.get_video_by_id(video_id=video_ids) + video_stats = video.items + return video_stats + except Exception: + title = "Failed to Update YouTube Statistics" + frappe.log_error(title + "\n\n" + frappe.get_traceback(), title=title) + + def prepare_and_set_data(video_list): + video_ids = get_formatted_ids(video_list) + stats = get_youtube_statistics(video_ids) + set_youtube_data(stats) + + def set_youtube_data(entries): + for entry in entries: + video_stats = entry.to_dict().get('statistics') + video_id = entry.to_dict().get('id') + stats = { + 'like_count' : video_stats.get('likeCount'), + 'view_count' : video_stats.get('viewCount'), + 'dislike_count' : video_stats.get('dislikeCount'), + 'comment_count' : video_stats.get('commentCount'), + 'video_id': video_id + } + + frappe.db.sql(""" + UPDATE `tabVideo` + SET + like_count = %(like_count)s, + view_count = %(view_count)s, + dislike_count = %(dislike_count)s, + comment_count = %(comment_count)s + WHERE youtube_video_id = %(video_id)s""", stats) + + video_list = frappe.get_all("Video", fields=["youtube_video_id"]) + if len(video_list) > 50: + # Update in batches of 50 + start, end = 0, 50 + while start < len(video_list): + batch = video_list[start:end] + prepare_and_set_data(batch) + start += 50 + end += 50 + else: + prepare_and_set_data(video_list) diff --git a/erpnext/utilities/doctype/video/video_list.js b/erpnext/utilities/doctype/video/video_list.js new file mode 100644 index 0000000000..8273a4a781 --- /dev/null +++ b/erpnext/utilities/doctype/video/video_list.js @@ -0,0 +1,7 @@ +frappe.listview_settings["Video"] = { + onload: (listview) => { + listview.page.add_menu_item(__("Video Settings"), function() { + frappe.set_route("Form","Video Settings", "Video Settings"); + }); + } +} \ No newline at end of file diff --git a/erpnext/utilities/doctype/video_settings/__init__.py b/erpnext/utilities/doctype/video_settings/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/erpnext/utilities/doctype/video_settings/test_video_settings.py b/erpnext/utilities/doctype/video_settings/test_video_settings.py new file mode 100644 index 0000000000..b217afe3d8 --- /dev/null +++ b/erpnext/utilities/doctype/video_settings/test_video_settings.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and Contributors +# See license.txt +from __future__ import unicode_literals + +# import frappe +import unittest + +class TestVideoSettings(unittest.TestCase): + pass diff --git a/erpnext/utilities/doctype/video_settings/video_settings.js b/erpnext/utilities/doctype/video_settings/video_settings.js new file mode 100644 index 0000000000..9ac8b9ec16 --- /dev/null +++ b/erpnext/utilities/doctype/video_settings/video_settings.js @@ -0,0 +1,8 @@ +// Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and contributors +// For license information, please see license.txt + +frappe.ui.form.on('Video Settings', { + // refresh: function(frm) { + + // } +}); diff --git a/erpnext/utilities/doctype/video_settings/video_settings.json b/erpnext/utilities/doctype/video_settings/video_settings.json new file mode 100644 index 0000000000..fb3274decd --- /dev/null +++ b/erpnext/utilities/doctype/video_settings/video_settings.json @@ -0,0 +1,60 @@ +{ + "actions": [], + "creation": "2020-08-02 03:50:21.339609", + "doctype": "DocType", + "editable_grid": 1, + "engine": "InnoDB", + "field_order": [ + "enable_youtube_tracking", + "api_key", + "frequency" + ], + "fields": [ + { + "default": "0", + "fieldname": "enable_youtube_tracking", + "fieldtype": "Check", + "label": "Enable YouTube Tracking" + }, + { + "depends_on": "eval:doc.enable_youtube_tracking", + "fieldname": "api_key", + "fieldtype": "Data", + "label": "API Key", + "mandatory_depends_on": "eval:doc.enable_youtube_tracking" + }, + { + "default": "1 hr", + "depends_on": "eval:doc.enable_youtube_tracking", + "fieldname": "frequency", + "fieldtype": "Select", + "label": "Frequency", + "mandatory_depends_on": "eval:doc.enable_youtube_tracking", + "options": "30 mins\n1 hr\n6 hrs\nDaily" + } + ], + "index_web_pages_for_search": 1, + "issingle": 1, + "links": [], + "modified": "2020-09-07 16:09:00.360668", + "modified_by": "Administrator", + "module": "Utilities", + "name": "Video Settings", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "print": 1, + "read": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "quick_entry": 1, + "sort_field": "modified", + "sort_order": "DESC", + "track_changes": 1 +} \ No newline at end of file diff --git a/erpnext/utilities/doctype/video_settings/video_settings.py b/erpnext/utilities/doctype/video_settings/video_settings.py new file mode 100644 index 0000000000..36fb54f015 --- /dev/null +++ b/erpnext/utilities/doctype/video_settings/video_settings.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2020, Frappe Technologies Pvt. Ltd. and contributors +# For license information, please see license.txt + +from __future__ import unicode_literals +import frappe +from frappe import _ +from frappe.model.document import Document +from apiclient.discovery import build + +class VideoSettings(Document): + def validate(self): + self.validate_youtube_api_key() + + def validate_youtube_api_key(self): + if self.enable_youtube_tracking and self.api_key: + try: + build("youtube", "v3", developerKey=self.api_key) + except Exception: + title = _("Failed to Authenticate the API key.") + frappe.log_error(title + "\n\n" + frappe.get_traceback(), title=title) + frappe.throw(title + " Please check the error logs.", title=_("Invalid Credentials")) \ No newline at end of file diff --git a/erpnext/utilities/report/__init__.py b/erpnext/utilities/report/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/erpnext/utilities/report/youtube_interactions/__init__.py b/erpnext/utilities/report/youtube_interactions/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/erpnext/utilities/report/youtube_interactions/youtube_interactions.js b/erpnext/utilities/report/youtube_interactions/youtube_interactions.js new file mode 100644 index 0000000000..6e3e4e6980 --- /dev/null +++ b/erpnext/utilities/report/youtube_interactions/youtube_interactions.js @@ -0,0 +1,20 @@ +// Copyright (c) 2016, Frappe Technologies Pvt. Ltd. and contributors +// For license information, please see license.txt +/* eslint-disable */ + +frappe.query_reports["YouTube Interactions"] = { + "filters": [ + { + fieldname: "from_date", + label: __("From Date"), + fieldtype: "Date", + default: frappe.datetime.add_months(frappe.datetime.now_date(), -12), + }, + { + fieldname:"to_date", + label: __("To Date"), + fieldtype: "Date", + default: frappe.datetime.now_date(), + } + ] +}; diff --git a/erpnext/utilities/report/youtube_interactions/youtube_interactions.json b/erpnext/utilities/report/youtube_interactions/youtube_interactions.json new file mode 100644 index 0000000000..a40247b6df --- /dev/null +++ b/erpnext/utilities/report/youtube_interactions/youtube_interactions.json @@ -0,0 +1,27 @@ +{ + "add_total_row": 0, + "creation": "2020-08-02 05:05:00.457093", + "disable_prepared_report": 0, + "disabled": 0, + "docstatus": 0, + "doctype": "Report", + "idx": 0, + "is_standard": "Yes", + "modified": "2020-08-02 05:05:00.457093", + "modified_by": "Administrator", + "module": "Utilities", + "name": "YouTube Interactions", + "owner": "Administrator", + "prepared_report": 0, + "ref_doctype": "Video", + "report_name": "YouTube Interactions", + "report_type": "Script Report", + "roles": [ + { + "role": "All" + }, + { + "role": "System Manager" + } + ] +} \ No newline at end of file diff --git a/erpnext/utilities/report/youtube_interactions/youtube_interactions.py b/erpnext/utilities/report/youtube_interactions/youtube_interactions.py new file mode 100644 index 0000000000..3516a35097 --- /dev/null +++ b/erpnext/utilities/report/youtube_interactions/youtube_interactions.py @@ -0,0 +1,113 @@ +# Copyright (c) 2013, Frappe Technologies Pvt. Ltd. and contributors +# For license information, please see license.txt + +from __future__ import unicode_literals +import frappe +from frappe import _ +from frappe.utils import flt + +def execute(filters=None): + if not frappe.db.get_single_value("Video Settings", "enable_youtube_tracking") or not filters: + return [], [] + + columns = get_columns() + data = get_data(filters) + chart_data, summary = get_chart_summary_data(data) + return columns, data, None, chart_data, summary + +def get_columns(): + return [ + { + "label": _("Published Date"), + "fieldname": "publish_date", + "fieldtype": "Date", + "width": 100 + }, + { + "label": _("Title"), + "fieldname": "title", + "fieldtype": "Data", + "width": 200 + }, + { + "label": _("Duration"), + "fieldname": "duration", + "fieldtype": "Duration", + "width": 100 + }, + { + "label": _("Views"), + "fieldname": "view_count", + "fieldtype": "Float", + "width": 200 + }, + { + "label": _("Likes"), + "fieldname": "like_count", + "fieldtype": "Float", + "width": 200 + }, + { + "label": _("Dislikes"), + "fieldname": "dislike_count", + "fieldtype": "Float", + "width": 100 + }, + { + "label": _("Comments"), + "fieldname": "comment_count", + "fieldtype": "Float", + "width": 100 + } + ] + +def get_data(filters): + return frappe.db.sql(""" + SELECT + publish_date, title, provider, duration, + view_count, like_count, dislike_count, comment_count + FROM `tabVideo` + WHERE view_count is not null + and publish_date between %(from_date)s and %(to_date)s + ORDER BY view_count desc""", filters, as_dict=1) + +def get_chart_summary_data(data): + labels, likes, views = [], [], [] + total_views = 0 + + for row in data: + labels.append(row.get('title')) + likes.append(row.get('like_count')) + views.append(row.get('view_count')) + total_views += flt(row.get('view_count')) + + + chart_data = { + "data" : { + "labels" : labels, + "datasets" : [ + { + "name" : "Likes", + "values" : likes + }, + { + "name" : "Views", + "values" : views + } + ] + }, + "type": "bar", + "barOptions": { + "stacked": 1 + }, + } + + summary = [ + { + "value": total_views, + "indicator": "Blue", + "label": "Total Views", + "datatype": "Float", + } + ] + return chart_data, summary \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index b7ba412893..c4f9171fca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,7 @@ plaid-python==6.0.0 pycountry==19.8.18 PyGithub==1.44.1 python-stdnum==1.12 +python-youtube==0.6.0 taxjar==1.9.0 tweepy==3.8.0 Unidecode==1.1.1