Optimize music library lookups for larger libraries

jellyfin_kodi/downloader.py

@@ -164,6 +164,27 @@ def get_episode_by_season(show_id, season_id):
         yield items
 
 
+def get_item_count(parent_id, item_type=None, params=None):
+
+    url = "Users/{UserId}/Items"
+
+    query_params = {
+        'ParentId': parent_id,
+        'IncludeItemTypes': item_type,
+        'EnableTotalRecordCount': True,
+        'LocationTypes': "FileSystem,Remote,Offline",
+        'Recursive': True,
+        'Limit': 1
+    }
+    if params:
+        query_params['params'].update(params)
+
+    result = _get(url, query_params)
+
+    total = result.get('TotalRecordCount')
+
+    return total
+
 def get_items(parent_id, item_type=None, basic=False, params=None):
 
     query = {
@@ -191,23 +212,27 @@ def get_items(parent_id, item_type=None, basic=False, params=None):
 
 def get_artists(parent_id=None):
 
-    url = "Artists"
+    #url = "Artists"
 
-    params = {
+    query = {
-        'UserId': "{UserId}",
+        'url': 'Artists',
-        'ParentId': parent_id,
+        'params': {
-        'SortBy': "SortName",
+            'UserId': "{UserId}",
-        'SortOrder': "Ascending",
+            'ParentId': parent_id,
-        'Fields': api.music_info(),
+            'SortBy': "SortName",
-        'CollapseBoxSetItems': False,
+            'SortOrder': "Ascending",
-        'IsVirtualUnaired': False,
+            'Fields': api.music_info(),
-        'EnableTotalRecordCount': False,
+            'CollapseBoxSetItems': False,
-        'LocationTypes': "FileSystem,Remote,Offline",
+            'IsVirtualUnaired': False,
-        'IsMissing': False,
+            'EnableTotalRecordCount': False,
-        'Recursive': True
+            'LocationTypes': "FileSystem,Remote,Offline",
+            'IsMissing': False,
+            'Recursive': True
+        }
     }
 
-    return _get(url, params)
+    for items in _get_items(query):
+        yield items
 
 
 def get_library_items(library_id, item_type):

jellyfin_kodi/full_sync.py

@@ -417,37 +417,47 @@ class FullSync(object):
 
                     library_id = library['Id']
 
-                    # Get all items in the library to process locally
+                    total_items = server.get_item_count(library_id, 'MusicArtist,MusicAlbum,Audio')
-                    artists_data = server.get_artists(library_id)
+                    count = 0
-                    artists = artists_data['Items']
-                    num_artists = artists_data['TotalRecordCount']
-                    albums = server.get_library_items(library_id, 'MusicAlbum')['Items']
-                    songs = server.get_library_items(library_id, 'Audio')['Items']
 
-                    for index, artist in enumerate(artists):
+                    '''
-                        artist_name = artist.get('Name')
+                    Music database syncing.  Artists must be in the database
+                    before albums, albums before songs.  Pulls batches of items
+                    in sizes of setting "Paging - Max items".  'artists',
+                    'albums', and 'songs' are generators containing a dict of
+                    api responses
+                    '''
+                    artists = server.get_artists(library_id)
+                    for batch in artists:
+                        for item in batch['Items']:
+                            LOG.debug('Artist: {}'.format(item.get('Name')))
+                            percent = int((float(count) / float(total_items)) * 100)
+                            dialog.update(percent, message='Artist: {}'.format(item.get('Name')))
+                            obj.artist(item)
+                            count += 1
+                            # Delete item once it's been processed for memory management
+                            del item
 
-                        # Update percentage dialog
+                    albums = server.get_items(library_id, item_type='MusicAlbum', params={'SortBy': 'AlbumArtist'})
-                        percent = int((float(index) / float(num_artists)) * 100)
+                    for batch in albums:
-                        dialog.update(percent, heading="%s: %s" % (translate('addon_name'), library['Name']), message=artist_name)
+                        for item in batch['Items']:
+                            LOG.debug('Album: {}'.format(item.get('Name')))
+                            percent = int((float(count) / float(total_items)) * 100)
+                            dialog.update(percent, message='Album: {} - {}'.format(item.get('AlbumArtist', ''), item.get('Name')))
+                            obj.album(item)
+                            count += 1
+                            del item
 
-                        # Add artist to database
+                    songs = server.get_items(library_id, item_type='Audio', params={'SortBy': 'AlbumArtist'})
-                        obj.artist(artist)
+                    for batch in songs:
+                        for item in batch['Items']:
+                            LOG.debug('Song: {}'.format(item.get('Name')))
+                            percent = int((float(count) / float(total_items)) * 100)
+                            dialog.update(percent, message='Track: {} - {}'.format(item.get('AlbumArtist', ''), item.get('Name')))
+                            obj.song(item)
+                            count += 1
+                            del item
 
-                        # Get all albums for each artist
-                        artist_albums = [album for album in albums if artist_name in album.get('Artists')]
-                        for album in artist_albums:
-                            # Add album to database
-                            obj.album(album)
-                            album_id = album.get('Id')
-                            # Get all songs in each album
-                            album_songs = [song for song in songs if album_id == song.get('AlbumId')]
-                            for song in album_songs:
-                                dialog.update(percent,
-                                              message="%s/%s/%s" % (artist_name, album['Name'][:7], song['Name'][:7]))
-                                # Add song to database
-                                obj.song(song)
-#
                     if self.update_library:
                         self.music_compare(library, obj, jellyfindb)