From 4acb38f9701527e070750b0892c8d5649ce10087 Mon Sep 17 00:00:00 2001 From: Shaun Date: Sun, 11 Nov 2018 15:08:07 +1100 Subject: [PATCH] add caching using cPickle --- addon.xml | 2 +- resources/lib/datamanager.py | 136 +++++++++++++++++++++++++++++++- resources/lib/functions.py | 119 +++++++++++++++++++--------- resources/lib/item_functions.py | 49 +----------- resources/lib/play_utils.py | 51 +++++++++++- 5 files changed, 263 insertions(+), 94 deletions(-) diff --git a/addon.xml b/addon.xml index 91f885e..4398fc2 100644 --- a/addon.xml +++ b/addon.xml @@ -1,7 +1,7 @@ diff --git a/resources/lib/datamanager.py b/resources/lib/datamanager.py index feda9f0..edb5f1f 100644 --- a/resources/lib/datamanager.py +++ b/resources/lib/datamanager.py @@ -2,17 +2,23 @@ import json from collections import defaultdict +import threading +import hashlib +import os +import cPickle from downloadutils import DownloadUtils from simple_logging import SimpleLogging +from item_functions import extract_item_info + +import xbmc +import xbmcaddon log = SimpleLogging(__name__) class DataManager(): - cacheDataResult = None - dataUrl = None - cacheDataPath = None - canRefreshNow = False + + addon_dir = xbmc.translatePath(xbmcaddon.Addon().getAddonInfo('profile')) def __init__(self, *args): log.debug("DataManager __init__") @@ -25,4 +31,126 @@ class DataManager(): result = self.loadJasonData(jsonData) return result + def get_items(self, url, gui_options): + m = hashlib.md5() + m.update(url) + url_hash = m.hexdigest() + cache_file = os.path.join(self.addon_dir, "cache_" + url_hash + ".pickle") + + item_list = [] + baseline_name = None + cache_thread = CacheManagerThread() + cache_thread.cache_file = cache_file + cache_thread.cache_url = url + cache_thread.gui_options = gui_options + + if os.path.isfile(cache_file): + log.debug("Loading url data from pickle data") + + with open(cache_file, 'rb') as handle: + item_list = cPickle.load(handle) + + cache_thread.cached_data = item_list + + else: + log.debug("Loading url data from server") + + results = self.GetContent(url) + + if results is None: + results = [] + + if isinstance(results, dict) and results.get("Items") is not None: + baseline_name = results.get("BaselineItemName") + results = results.get("Items", []) + elif isinstance(results, list) and len(results) > 0 and results[0].get("Items") is not None: + baseline_name = results[0].get("BaselineItemName") + results = results[0].get("Items") + + for item in results: + item_data = extract_item_info(item, gui_options) + item_list.append(item_data) + + cache_thread.fresh_data = item_list + + cache_thread.start() + + return baseline_name, item_list + + +class CacheManagerThread(threading.Thread): + cached_data = None + fresh_data = None + cache_file = None + cache_url = None + gui_options = None + + def __init__(self, *args): + threading.Thread.__init__(self, *args) + + @staticmethod + def get_data_hash(items): + + m = hashlib.md5() + for item in items: + item_string = "%s_%s_%s_%s_%s" % ( + item.name, + item.play_count, + item.favorite, + item.resume_time, + item.recursive_unplayed_items_count + ) + item_string = item_string.encode("UTF-8") + m.update(item_string) + + return m.hexdigest() + + def run(self): + + log.debug("CacheManagerThread : Started") + + if self.cached_data is None and self.fresh_data is not None: + log.debug("CacheManagerThread : Saving New Data") + with open(self.cache_file, 'wb') as handle: + cPickle.dump(self.fresh_data, handle, protocol=cPickle.HIGHEST_PROTOCOL) + return + + cached_hash = self.get_data_hash(self.cached_data) + log.debug("CacheManagerThread : Cache Hash : {0}", cached_hash) + + data_manager = DataManager() + results = data_manager.GetContent(self.cache_url) + if results is None: + results = [] + + if isinstance(results, dict) and results.get("Items") is not None: + results = results.get("Items", []) + elif isinstance(results, list) and len(results) > 0 and results[0].get("Items") is not None: + results = results[0].get("Items") + + loaded_items = [] + for item in results: + item_data = extract_item_info(item, self.gui_options) + loaded_items.append(item_data) + + loaded_hash = self.get_data_hash(loaded_items) + log.debug("CacheManagerThread : Loaded Hash : {0}", loaded_hash) + + # if they dont match then save the data and trigger a content reload + if cached_hash != loaded_hash: + log.debug("CacheManagerThread : Saving new cache data and reloading container") + with open(self.cache_file, 'wb') as handle: + cPickle.dump(loaded_items, handle, protocol=cPickle.HIGHEST_PROTOCOL) + + # we need to refresh but will wait until the main function has finished + loops = 0 + #while (self.dataManager.canRefreshNow == False and loops < 200 and not xbmc.Monitor().abortRequested()): + # log.debug("Cache_Data_Manager: Not finished yet") + # xbmc.sleep(100) + # loops = loops + 1 + + log.debug("CacheManagerThread : Sending container refresh (" + str(loops) + ")") + xbmc.executebuiltin("Container.Refresh") + + log.debug("CacheManagerThread : Exited") diff --git a/resources/lib/functions.py b/resources/lib/functions.py index 5e64261..5d3616d 100644 --- a/resources/lib/functions.py +++ b/resources/lib/functions.py @@ -11,6 +11,8 @@ import StringIO import encodings import binascii import re +import hashlib +import cPickle import xbmcplugin import xbmcgui @@ -389,16 +391,17 @@ def getContent(url, params): log.debug("ADDING NEXT URL: {0}", url_next) # use the data manager to get the data - result = dataManager.GetContent(url) + #result = dataManager.GetContent(url) - total_records = 0 - if result is not None and isinstance(result, dict): - total_records = result.get("TotalRecordCount", 0) + #total_records = 0 + #if result is not None and isinstance(result, dict): + # total_records = result.get("TotalRecordCount", 0) - dir_items, detected_type = processDirectory(result, progress, params) + dir_items, detected_type = processDirectory(url, progress, params) if dir_items is None: return + total_records = len(dir_items) # add paging items if page_limit > 0 and media_type.startswith("movie"): if url_prev: @@ -450,7 +453,7 @@ def getContent(url, params): return -def processDirectory(results, progress, params): +def processDirectory(url, progress, params): log.debug("== ENTER: processDirectory ==") settings = xbmcaddon.Addon() @@ -468,24 +471,61 @@ def processDirectory(results, progress, params): name_format_type = None name_format = None - dirItems = [] - if results is None: - results = [] + gui_options = {} + gui_options["server"] = server + gui_options["name_format"] = name_format + gui_options["name_format_type"] = name_format_type + + # get me some items + ############################## + + baseline_name, item_list = dataManager.get_items(url, gui_options) + + ''' + m = hashlib.md5() + m.update(url) + url_hash = m.hexdigest() + cache_file = os.path.join(__addondir__, "cache_" + url_hash + ".pickle") baseline_name = None - if isinstance(results, dict) and results.get("Items") is not None: - baseline_name = results.get("BaselineItemName") - results = results.get("Items", []) - elif isinstance(results, list) and len(results) > 0 and results[0].get("Items") is not None: - baseline_name = results[0].get("BaselineItemName") - results = results[0].get("Items") + + if os.path.isfile(cache_file): + log.debug("Loading url data from pickle data") + + with open(cache_file, 'rb') as handle: + item_list = cPickle.load(handle) + + else: + log.debug("Loading url data from server") + + results = dataManager.GetContent(url) + + if results is None: + results = [] + + if isinstance(results, dict) and results.get("Items") is not None: + baseline_name = results.get("BaselineItemName") + results = results.get("Items", []) + elif isinstance(results, list) and len(results) > 0 and results[0].get("Items") is not None: + baseline_name = results[0].get("BaselineItemName") + results = results[0].get("Items") + + item_list = [] + for item in results: + item_data = extract_item_info(item, gui_options) + item_list.append(item_data) + + with open(cache_file, 'wb') as handle: + cPickle.dump(item_list, handle, protocol=cPickle.HIGHEST_PROTOCOL) + ''' + ############################################## # flatten single season # if there is only one result and it is a season and you have flatten signle season turned on then # build a new url, set the content media type and call get content again flatten_single_season = settings.getSetting("flatten_single_season") == "true" - if flatten_single_season and len(results) == 1 and results[0].get("Type", "") == "Season": - season_id = results[0].get("Id") + if flatten_single_season and len(item_list) == 1 and item_list[0].item_type == "Season": + season_id = item_list[0].id season_url = ('{server}/emby/Users/{userid}/items' + '?ParentId=' + season_id + '&IsVirtualUnAired=false' + @@ -507,21 +547,17 @@ def processDirectory(results, progress, params): show_empty_folders = settings.getSetting("show_empty_folders") == 'true' - item_count = len(results) + item_count = len(item_list) current_item = 1 first_season_item = None total_unwatched = 0 total_episodes = 0 total_watched = 0 - gui_options = {} - gui_options["server"] = server - - gui_options["name_format"] = name_format - gui_options["name_format_type"] = name_format_type detected_type = None + dir_items = [] - for item in results: + for item_details in item_list: if progress is not None: percent_done = (float(current_item) / float(item_count)) * 100 @@ -529,7 +565,6 @@ def processDirectory(results, progress, params): current_item = current_item + 1 # get the infofrom the item - item_details = extract_item_info(item, gui_options) item_details.baseline_itemname = baseline_name if detected_type is not None: @@ -539,7 +574,8 @@ def processDirectory(results, progress, params): detected_type = item_details.item_type if item_details.item_type == "Season" and first_season_item is None: - first_season_item = item + log.debug("Setting First Season to : {0}", item_details) + first_season_item = item_details total_unwatched += item_details.unwatched_episodes total_episodes += item_details.total_episodes @@ -551,7 +587,7 @@ def processDirectory(results, progress, params): item_details.art["poster"] = item_details.art["tvshow.poster"] item_details.art["thumb"] = item_details.art["tvshow.poster"] - if item["IsFolder"] is True: + if item_details.is_folder is True: if item_details.item_type == "Series": u = ('{server}/emby/Shows/' + item_details.id + '/Seasons' @@ -567,10 +603,10 @@ def processDirectory(results, progress, params): '&Fields={field_filters}' + '&format=json') - if show_empty_folders or item["RecursiveItemCount"] != 0: + if show_empty_folders or item_details.recursive_item_count != 0: gui_item = add_gui_item(u, item_details, display_options) if gui_item: - dirItems.append(gui_item) + dir_items.append(gui_item) elif item_details.item_type == "MusicArtist": u = ('{server}/emby/Users/{userid}/items' + @@ -581,22 +617,26 @@ def processDirectory(results, progress, params): '&format=json') gui_item = add_gui_item(u, item_details, display_options) if gui_item: - dirItems.append(gui_item) + dir_items.append(gui_item) else: u = item_details.id gui_item = add_gui_item(u, item_details, display_options, folder=False) if gui_item: - dirItems.append(gui_item) + dir_items.append(gui_item) # add the all episodes item show_all_episodes = settings.getSetting('show_all_episodes') == 'true' + + if first_season_item is not None: + log.debug("All Seasons Entry : {0} {1}", len(dir_items), first_season_item.__dict__) + if (show_all_episodes and first_season_item is not None - and len(dirItems) > 1 - and first_season_item.get("SeriesId") is not None): + and len(dir_items) > 1 + and first_season_item.series_id is not None): series_url = ('{server}/emby/Users/{userid}/items' + - '?ParentId=' + first_season_item.get("SeriesId") + + '?ParentId=' + first_season_item.series_id + '&IsVirtualUnAired=false' + '&IsMissing=false' + '&Fields={field_filters}' + @@ -611,13 +651,13 @@ def processDirectory(results, progress, params): item_details = ItemDetails() - item_details.id = first_season_item.get("Id") + item_details.id = first_season_item.id item_details.name = string_load(30290) - item_details.art = getArt(first_season_item, server) + item_details.art = first_season_item.art item_details.play_count = played item_details.overlay = overlay item_details.name_format = "Episode|episode_name_format" - item_details.series_name = first_season_item.get("SeriesName") + item_details.series_name = first_season_item.series_name item_details.item_type = "Season" item_details.unwatched_episodes = total_unwatched item_details.total_episodes = total_episodes @@ -625,10 +665,11 @@ def processDirectory(results, progress, params): item_details.mode = "GET_CONTENT" gui_item = add_gui_item(series_url, item_details, display_options, folder=True) + log.debug("All Seasons GUI Item Entry : {0}", gui_item) if gui_item: - dirItems.append(gui_item) + dir_items.append(gui_item) - return dirItems, detected_type + return dir_items, detected_type def show_menu(params): diff --git a/resources/lib/item_functions.py b/resources/lib/item_functions.py index a501c96..95515b0 100644 --- a/resources/lib/item_functions.py +++ b/resources/lib/item_functions.py @@ -11,7 +11,6 @@ import xbmcgui from utils import getArt from simple_logging import SimpleLogging from downloadutils import DownloadUtils -from datamanager import DataManager from kodi_utils import HomeWindow log = SimpleLogging(__name__) @@ -35,7 +34,7 @@ class ItemDetails(): episode_number = 0 season_number = 0 track_number = 0 - + series_id = None art = None mpaa = None @@ -105,6 +104,7 @@ def extract_item_info(item, gui_options): item_details.season_number = item["ParentIndexNumber"] elif item_details.item_type == "Season": item_details.season_number = item["IndexNumber"] + item_details.series_id = item["SeriesId"] if item_details.season_number is None: item_details.season_number = 0 @@ -494,48 +494,3 @@ def add_gui_item(url, item_details, display_options, folder=True): return (u, list_item, folder) -def get_next_episode(item): - - if item.get("Type", "na") != "Episode": - log.debug("Not an episode, can not get next") - return None - - parendId = item.get("ParentId", "na") - item_index = item.get("IndexNumber", -1) - - if parendId == "na": - log.debug("No parent id, can not get next") - return None - - if item_index == -1: - log.debug("No episode number, can not get next") - return None - - url = ( '{server}/emby/Users/{userid}/Items?' + - '?Recursive=true' + - '&ParentId=' + parendId + - '&IsVirtualUnaired=false' + - '&IsMissing=False' + - '&IncludeItemTypes=Episode' + - '&ImageTypeLimit=1' + - '&format=json') - - data_manager = DataManager() - items_result = data_manager.GetContent(url) - log.debug("get_next_episode, sibling list: {0}", items_result) - - if items_result is None: - log.debug("get_next_episode no results") - return None - - item_list = items_result.get("Items", []) - - for item in item_list: - index = item.get("IndexNumber", -1) - # find the very next episode in the season - if index == item_index + 1: - log.debug("get_next_episode, found next episode: {0}", item) - return item - - return None - diff --git a/resources/lib/play_utils.py b/resources/lib/play_utils.py index 9fdb9e5..c9c3bbc 100644 --- a/resources/lib/play_utils.py +++ b/resources/lib/play_utils.py @@ -19,7 +19,7 @@ from .utils import PlayUtils, getArt, id_generator, send_event_notification from .kodi_utils import HomeWindow from .translation import string_load from .datamanager import DataManager -from .item_functions import get_next_episode, extract_item_info, add_gui_item +from .item_functions import extract_item_info, add_gui_item from .clientinfo import ClientInformation from .functions import delete from .cache_images import CacheArtwork @@ -353,6 +353,50 @@ def playFile(play_info, monitor): current_position = xbmc.Player().getTime() log.debug("Playback_Start_Seek target:{0} current:{1}", target_seek, current_position) +def get_next_episode(item): + + if item.get("Type", "na") != "Episode": + log.debug("Not an episode, can not get next") + return None + + parendId = item.get("ParentId", "na") + item_index = item.get("IndexNumber", -1) + + if parendId == "na": + log.debug("No parent id, can not get next") + return None + + if item_index == -1: + log.debug("No episode number, can not get next") + return None + + url = ( '{server}/emby/Users/{userid}/Items?' + + '?Recursive=true' + + '&ParentId=' + parendId + + '&IsVirtualUnaired=false' + + '&IsMissing=False' + + '&IncludeItemTypes=Episode' + + '&ImageTypeLimit=1' + + '&format=json') + + data_manager = DataManager() + items_result = data_manager.GetContent(url) + log.debug("get_next_episode, sibling list: {0}", items_result) + + if items_result is None: + log.debug("get_next_episode no results") + return None + + item_list = items_result.get("Items", []) + + for item in item_list: + index = item.get("IndexNumber", -1) + # find the very next episode in the season + if index == item_index + 1: + log.debug("get_next_episode, found next episode: {0}", item) + return item + + return None def send_next_episode_details(item): @@ -943,8 +987,9 @@ class PlaybackService(xbmc.Monitor): log.debug("Screen Saver Activated") # stop playback when switching users - if xbmc.Player().isPlaying(): - xbmc.Player().stop() + player = xbmc.Player() + if player.isPlaying(): + player.stop() #xbmc.executebuiltin("Dialog.Close(selectdialog, true)")