diff --git a/resources/lib/emby.py b/resources/lib/emby.py new file mode 100644 index 00000000..c7fc6ee1 --- /dev/null +++ b/resources/lib/emby.py @@ -0,0 +1,204 @@ +# -*- coding: utf-8 -*- + + +''' The goal is to reduce memory usage. + Generators to prevent having to hold all the info in memory + while downloading from emby servers. + + Working with json, so we can resume where we left off. +''' +################################################################################################# + +import json +import logging +import hashlib +import threading +import Queue + +import xbmc + +import downloadutils +import database +from utils import window, settings +from contextlib import closing + +################################################################################################# + +log = logging.getLogger("EMBY."+__name__) +limit = min(int(settings('limitIndex')), 50) +do = downloadutils.DownloadUtils() + +################################################################################################# + +def get_embyserver_url(handler): + return "{server}/emby/%s" % handler + +def basic_info(): + return "Etag" + +def complete_info(): + return ( + "Path,Genres,SortName,Studios,Writer,ProductionYear,Taglines," + "CommunityRating,OfficialRating,CumulativeRunTimeTicks," + "Metascore,AirTime,DateCreated,MediaStreams,People,Overview," + "CriticRating,CriticRatingSummary,Etag,ShortOverview,ProductionLocations," + "Tags,ProviderIds,ParentId,RemoteTrailers,SpecialEpisodeNumbers," + "MediaSources,VoteCount,ItemCounts" + ) + +def _http(action, url, request={}): + #request.update({'type': action, 'url': url}) + #return HTTP.request_url(request) + + return do.downloadUrl(url, action_type=action, parameters=request['params']) + +def _get(handler, params=None): + return _http("GET", get_embyserver_url(handler), {'params': params}) + +def _post(handler, json=None, params=None): + return _http("POST", get_embyserver_url(handler), {'params': params, 'json': json}) + +def _delete(handler, params=None): + return _http("DELETE", get_embyserver_url(handler), {'params': params}) + + +def emby_session(handler="", params=None, action="GET", json=None): + + if action == "POST": + return _post("Sessions%s" % handler, json, params) + elif action == "DELETE": + return _delete("Sessions%s" % handler, params) + else: + return _get("Sessions%s" % handler, params) + +def user(handler="", params=None, action="GET", json=None): + + if action == "POST": + return _post("Users/{UserId}%s" % handler, json, params) + elif action == "DELETE": + return _delete(session, "Users/{UserId}%s" % handler, params) + else: + return _get(session, "Users/{UserId}%s" % handler, params) + +def item(handler="", params=None): + return user("/Items%s" % handler, params) + +def show(handler, params): + return _get("Shows%s" % handler, params) + +################################################################################################# + +# Single item functions + +################################################################################################# + +def get_item(item_id, fields=None): + return item(params={ + 'Ids': item_id, + 'EnableTotalRecordCount': False, + 'Fields': fields + }) + +def get_seasons(self, show_id): + return show("/%s/Seasons?UserId={UserId}" % show_id, { + 'IsVirtualUnaired': False, + 'Fields': "Etag" + }) + +################################################################################################# + +# Get multiple items (Generator) + +''' This should help with memory issues. + for items in generator(...): + #do something + + If all items are required at once: + a = (items['Items'] for items in generator(...)) +''' + +################################################################################################# + +def get_items(parent_id, item_type, basic=False): + + query = { + 'url': "Users/{UserId}/Items", + 'params': { + 'ParentId': parent_id, + 'IncludeItemTypes': item_type, + 'SortBy': "SortName", + 'Fields': basic_info() if basic else complete_info() + } + } + for items in _get_items(query): + yield items + +def get_item_list(item_list, basic=False): + + for item_ids in _split_list(item_list, limit): + query = { + 'url': "Users/{UserId}/Items", + 'params': { + "Ids": ",".join(item_ids), + 'Fields': basic_info() if basic else complete_info() + } + } + for items in _get_items(query): + yield items + +def _split_list(item_list, size): + # Split up list in pieces of size. Will generate a list of lists + return [item_list[i:i + size] for i in range(0, len(item_list), size)] + +def _get_items(query): + + ''' query = { + 'url': string, + 'params': dict -- opt, include StartIndex to resume + } + ''' + items = { + 'Items': [], + 'TotalRecordCount': 0, + 'RestorePoint': {} + } + + url = query['url'] + params = query.get('params', {}) + params.update({ + 'CollapseBoxSetItems': False, + 'IsVirtualUnaired': False, + 'EnableTotalRecordCount': False, + 'LocationTypes': "FileSystem,Remote,Offline", + 'IsMissing': False, + 'Recursive': True, + 'SortOrder': "Ascending" + }) + + try: + test_params = dict(params) + test_params['Limit'] = 1 + test_params['EnableTotalRecordCount'] = True + + items['TotalRecordCount'] = _get(url, test_params)['TotalRecordCount'] + + except Exception as error: + log.error("Failed to retrieve the server response %s: %s params:%s", url, error, params) + + else: + index = params.get('StartIndex', 0) + total = items['TotalRecordCount'] + + while index < total: + + params['StartIndex'] = index + params['Limit'] = limit + result = _get(url, params) + + items['Items'].extend(result['Items']) + items['RestorePoint'] = query + yield items + + del items['Items'][:] + index += limit + diff --git a/resources/lib/librarysync.py b/resources/lib/librarysync.py index f5290c12..010b8f5a 100644 --- a/resources/lib/librarysync.py +++ b/resources/lib/librarysync.py @@ -17,6 +17,7 @@ import clientinfo import database import downloadutils import itemtypes +import emby as mb import embydb_functions as embydb import read_embyserver as embyserver import userclient @@ -59,7 +60,6 @@ class LibrarySync(threading.Thread): self.doUtils = downloadutils.DownloadUtils().downloadUrl self.user = userclient.UserClient() self.emby = embyserver.Read_EmbyServer() - self.kodi_version = int(xbmc.getInfoLabel('System.BuildVersion')[:2]) threading.Thread.__init__(self) @@ -402,8 +402,8 @@ class LibrarySync(threading.Thread): heading=lang(29999), message="%s %s..." % (lang(33017), view_name)) - all_movies = self.emby.getMovies(view['id'], dialog=pdialog) - movies.add_all("Movie", all_movies, view) + for all_movies in mb.get_items(view['id'], "Movie"): + movies.add_all("Movie", all_movies['Items'], view) log.debug("Movies finished.") return True @@ -415,8 +415,8 @@ class LibrarySync(threading.Thread): if pdialog: pdialog.update(heading=lang(29999), message=lang(33018)) - boxsets = self.emby.getBoxset(dialog=pdialog) - movies.add_all("BoxSet", boxsets) + for boxsets in mb.get_items(None, "BoxSet"): + movies.add_all("BoxSet", boxsets) log.debug("Boxsets finished.") return True @@ -434,7 +434,6 @@ class LibrarySync(threading.Thread): log.info("Processing: %s", view) # Get items per view - viewId = view['id'] viewName = view['name'] if pdialog: @@ -443,8 +442,8 @@ class LibrarySync(threading.Thread): message="%s %s..." % (lang(33019), viewName)) # Initial or repair sync - all_mvideos = self.emby.getMusicVideos(viewId, dialog=pdialog) - mvideos.add_all("MusicVideo", all_mvideos, view) + for all_mvideos in mb.get_items(view['id'], "MusicVideo"): + mvideos.add_all("MusicVideo", all_mvideos['Items'], view) else: log.debug("MusicVideos finished.") @@ -469,10 +468,8 @@ class LibrarySync(threading.Thread): heading=lang(29999), message="%s %s..." % (lang(33020), view['name'])) - all_tvshows = self.emby.getShows(view['id'], dialog=pdialog) - #log.info([item['Id'] for item in all_tvshows['Items']]) - #for all_tvshows in self.emby.get_parent_child(view['id'], "Series"): - tvshows.add_all("Series", all_tvshows, view) + for all_tvshows in mb.get_items(view['id'], "Series"): + tvshows.add_all("Series", all_tvshows['Items'], view) else: log.debug("TVShows finished.") diff --git a/resources/lib/objects/tvshows.py b/resources/lib/objects/tvshows.py index 9c614229..c4b794ec 100644 --- a/resources/lib/objects/tvshows.py +++ b/resources/lib/objects/tvshows.py @@ -63,6 +63,8 @@ class TVShows(Items): def compare_all(self): # Pull the list of movies and boxsets in Kodi + import emby as mb + pdialog = self.pdialog views = self.emby_db.getView_byType('tvshows') views += self.emby_db.getView_byType('mixed') @@ -116,7 +118,7 @@ class TVShows(Items): updatelist.append(itemid) log.info("TVShows to update for %s: %s", viewName, updatelist) - embytvshows = self.emby.getFullItems(updatelist) + embytvshows = (items['Items'] for items in mb.get_item_list(updatelist, True)) self.total = len(updatelist) del updatelist[:]