Generators

resources/lib/emby.py

@@ -0,0 +1,204 @@
+# -*- coding: utf-8 -*-
+
+
+''' The goal is to reduce memory usage.
+    Generators to prevent having to hold all the info in memory
+    while downloading from emby servers.
+
+    Working with json, so we can resume where we left off.
+'''
+#################################################################################################
+
+import json
+import logging
+import hashlib
+import threading
+import Queue
+
+import xbmc
+
+import downloadutils
+import database
+from utils import window, settings
+from contextlib import closing
+
+#################################################################################################
+
+log = logging.getLogger("EMBY."+__name__)
+limit = min(int(settings('limitIndex')), 50)
+do = downloadutils.DownloadUtils()
+
+#################################################################################################
+
+def get_embyserver_url(handler):
+    return  "{server}/emby/%s" % handler
+
+def basic_info():
+    return  "Etag"
+
+def complete_info():
+    return  (
+        "Path,Genres,SortName,Studios,Writer,ProductionYear,Taglines,"
+        "CommunityRating,OfficialRating,CumulativeRunTimeTicks,"
+        "Metascore,AirTime,DateCreated,MediaStreams,People,Overview,"
+        "CriticRating,CriticRatingSummary,Etag,ShortOverview,ProductionLocations,"
+        "Tags,ProviderIds,ParentId,RemoteTrailers,SpecialEpisodeNumbers,"
+        "MediaSources,VoteCount,ItemCounts"
+    )
+
+def _http(action, url, request={}):
+    #request.update({'type': action, 'url': url})
+    #return  HTTP.request_url(request)
+
+    return do.downloadUrl(url, action_type=action, parameters=request['params'])
+
+def _get(handler, params=None):
+    return  _http("GET", get_embyserver_url(handler), {'params': params})
+
+def _post(handler, json=None, params=None):
+    return  _http("POST", get_embyserver_url(handler), {'params': params, 'json': json})
+
+def _delete(handler, params=None):
+    return  _http("DELETE", get_embyserver_url(handler), {'params': params})
+
+
+def emby_session(handler="", params=None, action="GET", json=None):
+
+    if action == "POST":
+        return _post("Sessions%s" % handler, json, params)
+    elif action == "DELETE":
+        return _delete("Sessions%s" % handler, params)
+    else:
+        return _get("Sessions%s" % handler, params)
+
+def user(handler="", params=None, action="GET", json=None):
+
+    if action == "POST":
+        return _post("Users/{UserId}%s" % handler, json, params)
+    elif action == "DELETE":
+        return  _delete(session, "Users/{UserId}%s" % handler, params)
+    else:
+        return _get(session, "Users/{UserId}%s" % handler, params)
+
+def item(handler="", params=None):
+    return  user("/Items%s" % handler, params)
+
+def show(handler, params):
+    return  _get("Shows%s" % handler, params)
+
+#################################################################################################
+
+# Single item functions
+
+#################################################################################################
+
+def get_item(item_id, fields=None):
+    return  item(params={
+                'Ids': item_id,
+                'EnableTotalRecordCount': False,
+                'Fields': fields
+            })
+
+def get_seasons(self, show_id):
+    return  show("/%s/Seasons?UserId={UserId}" % show_id, {
+                'IsVirtualUnaired': False,
+                'Fields': "Etag"
+            })
+
+#################################################################################################
+
+# Get multiple items (Generator)
+
+''' This should help with memory issues.
+    for items in generator(...):
+        #do something
+
+    If all items are required at once:
+    a = (items['Items'] for items in generator(...))
+'''
+
+#################################################################################################
+
+def get_items(parent_id, item_type, basic=False):
+
+    query = {
+        'url': "Users/{UserId}/Items",
+        'params': {
+            'ParentId': parent_id,
+            'IncludeItemTypes': item_type,
+            'SortBy': "SortName",
+            'Fields': basic_info() if basic else complete_info()
+        }
+    }
+    for items in _get_items(query):
+        yield items
+
+def get_item_list(item_list, basic=False):
+
+    for item_ids in _split_list(item_list, limit):
+        query = {
+            'url': "Users/{UserId}/Items",
+            'params': {
+                "Ids": ",".join(item_ids),
+                'Fields': basic_info() if basic else complete_info()
+            }
+        }
+        for items in _get_items(query):
+            yield items
+
+def _split_list(item_list, size):
+    # Split up list in pieces of size. Will generate a list of lists
+    return [item_list[i:i + size] for i in range(0, len(item_list), size)]
+
+def _get_items(query):
+
+    ''' query = {
+            'url': string,
+            'params': dict -- opt, include StartIndex to resume
+        }
+    '''
+    items = {
+        'Items': [],
+        'TotalRecordCount': 0,
+        'RestorePoint': {}
+    }
+
+    url = query['url']
+    params = query.get('params', {})
+    params.update({
+        'CollapseBoxSetItems': False,
+        'IsVirtualUnaired': False,
+        'EnableTotalRecordCount': False,
+        'LocationTypes': "FileSystem,Remote,Offline",
+        'IsMissing': False,
+        'Recursive': True,
+        'SortOrder': "Ascending"
+    })
+
+    try:
+        test_params = dict(params)
+        test_params['Limit'] = 1
+        test_params['EnableTotalRecordCount'] = True
+
+        items['TotalRecordCount'] = _get(url, test_params)['TotalRecordCount']
+
+    except Exception as error:
+        log.error("Failed to retrieve the server response %s: %s params:%s", url, error, params)
+
+    else:
+        index = params.get('StartIndex', 0)
+        total = items['TotalRecordCount']
+
+        while index < total:
+
+            params['StartIndex'] = index
+            params['Limit'] = limit
+            result = _get(url, params)
+
+            items['Items'].extend(result['Items'])
+            items['RestorePoint'] = query
+            yield items
+
+            del items['Items'][:]
+            index += limit
+

resources/lib/librarysync.py

@@ -17,6 +17,7 @@ import clientinfo
 import database
 import downloadutils
 import itemtypes
+import emby as mb
 import embydb_functions as embydb
 import read_embyserver as embyserver
 import userclient
@@ -59,7 +60,6 @@ class LibrarySync(threading.Thread):
         self.doUtils = downloadutils.DownloadUtils().downloadUrl
         self.user = userclient.UserClient()
         self.emby = embyserver.Read_EmbyServer()
-
         self.kodi_version = int(xbmc.getInfoLabel('System.BuildVersion')[:2])
 
         threading.Thread.__init__(self)
@@ -402,8 +402,8 @@ class LibrarySync(threading.Thread):
                         heading=lang(29999),
                         message="%s %s..." % (lang(33017), view_name))
 
-            all_movies = self.emby.getMovies(view['id'], dialog=pdialog)
-            movies.add_all("Movie", all_movies, view)
+            for all_movies in mb.get_items(view['id'], "Movie"):
+                movies.add_all("Movie", all_movies['Items'], view)
 
         log.debug("Movies finished.")
         return True
@@ -415,7 +415,7 @@ class LibrarySync(threading.Thread):
         if pdialog:
             pdialog.update(heading=lang(29999), message=lang(33018))
 
-        boxsets = self.emby.getBoxset(dialog=pdialog)
+        for boxsets in mb.get_items(None, "BoxSet"):
             movies.add_all("BoxSet", boxsets)
 
         log.debug("Boxsets finished.")
@@ -434,7 +434,6 @@ class LibrarySync(threading.Thread):
             log.info("Processing: %s", view)
 
             # Get items per view
-            viewId = view['id']
             viewName = view['name']
 
             if pdialog:
@@ -443,8 +442,8 @@ class LibrarySync(threading.Thread):
                         message="%s %s..." % (lang(33019), viewName))
 
             # Initial or repair sync
-            all_mvideos = self.emby.getMusicVideos(viewId, dialog=pdialog)
-            mvideos.add_all("MusicVideo", all_mvideos, view)
+            for all_mvideos in mb.get_items(view['id'], "MusicVideo"):
+                mvideos.add_all("MusicVideo", all_mvideos['Items'], view)
 
         else:
             log.debug("MusicVideos finished.")
@@ -469,10 +468,8 @@ class LibrarySync(threading.Thread):
                         heading=lang(29999),
                         message="%s %s..." % (lang(33020), view['name']))
 
-            all_tvshows = self.emby.getShows(view['id'], dialog=pdialog)
-            #log.info([item['Id'] for item in all_tvshows['Items']])
-            #for all_tvshows in self.emby.get_parent_child(view['id'], "Series"):
-            tvshows.add_all("Series", all_tvshows, view)
+            for all_tvshows in mb.get_items(view['id'], "Series"):
+                tvshows.add_all("Series", all_tvshows['Items'], view)
 
         else:
             log.debug("TVShows finished.")

resources/lib/objects/tvshows.py

@@ -63,6 +63,8 @@ class TVShows(Items):
 
     def compare_all(self):
         # Pull the list of movies and boxsets in Kodi
+        import emby as mb
+
         pdialog = self.pdialog
         views = self.emby_db.getView_byType('tvshows')
         views += self.emby_db.getView_byType('mixed')
@@ -116,7 +118,7 @@ class TVShows(Items):
                     updatelist.append(itemid)
 
             log.info("TVShows to update for %s: %s", viewName, updatelist)
-            embytvshows = self.emby.getFullItems(updatelist)
+            embytvshows = (items['Items'] for items in mb.get_item_list(updatelist, True))
             self.total = len(updatelist)
             del updatelist[:]