From db6cad8e15d3999b06b771a1385ad97a605a8dd1 Mon Sep 17 00:00:00 2001 From: Matt Date: Mon, 7 Sep 2020 22:26:36 -0400 Subject: [PATCH] Optimize music library lookups for larger libraries --- jellyfin_kodi/downloader.py | 53 ++++++++++++++++++++++-------- jellyfin_kodi/full_sync.py | 64 +++++++++++++++++++++---------------- 2 files changed, 76 insertions(+), 41 deletions(-) diff --git a/jellyfin_kodi/downloader.py b/jellyfin_kodi/downloader.py index b2db586b..77965986 100644 --- a/jellyfin_kodi/downloader.py +++ b/jellyfin_kodi/downloader.py @@ -164,6 +164,27 @@ def get_episode_by_season(show_id, season_id): yield items +def get_item_count(parent_id, item_type=None, params=None): + + url = "Users/{UserId}/Items" + + query_params = { + 'ParentId': parent_id, + 'IncludeItemTypes': item_type, + 'EnableTotalRecordCount': True, + 'LocationTypes': "FileSystem,Remote,Offline", + 'Recursive': True, + 'Limit': 1 + } + if params: + query_params['params'].update(params) + + result = _get(url, query_params) + + total = result.get('TotalRecordCount') + + return total + def get_items(parent_id, item_type=None, basic=False, params=None): query = { @@ -191,23 +212,27 @@ def get_items(parent_id, item_type=None, basic=False, params=None): def get_artists(parent_id=None): - url = "Artists" + #url = "Artists" - params = { - 'UserId': "{UserId}", - 'ParentId': parent_id, - 'SortBy': "SortName", - 'SortOrder': "Ascending", - 'Fields': api.music_info(), - 'CollapseBoxSetItems': False, - 'IsVirtualUnaired': False, - 'EnableTotalRecordCount': False, - 'LocationTypes': "FileSystem,Remote,Offline", - 'IsMissing': False, - 'Recursive': True + query = { + 'url': 'Artists', + 'params': { + 'UserId': "{UserId}", + 'ParentId': parent_id, + 'SortBy': "SortName", + 'SortOrder': "Ascending", + 'Fields': api.music_info(), + 'CollapseBoxSetItems': False, + 'IsVirtualUnaired': False, + 'EnableTotalRecordCount': False, + 'LocationTypes': "FileSystem,Remote,Offline", + 'IsMissing': False, + 'Recursive': True + } } - return _get(url, params) + for items in _get_items(query): + yield items def get_library_items(library_id, item_type): diff --git a/jellyfin_kodi/full_sync.py b/jellyfin_kodi/full_sync.py index 7a3ca779..a64e0305 100644 --- a/jellyfin_kodi/full_sync.py +++ b/jellyfin_kodi/full_sync.py @@ -417,37 +417,47 @@ class FullSync(object): library_id = library['Id'] - # Get all items in the library to process locally - artists_data = server.get_artists(library_id) - artists = artists_data['Items'] - num_artists = artists_data['TotalRecordCount'] - albums = server.get_library_items(library_id, 'MusicAlbum')['Items'] - songs = server.get_library_items(library_id, 'Audio')['Items'] + total_items = server.get_item_count(library_id, 'MusicArtist,MusicAlbum,Audio') + count = 0 - for index, artist in enumerate(artists): - artist_name = artist.get('Name') + ''' + Music database syncing. Artists must be in the database + before albums, albums before songs. Pulls batches of items + in sizes of setting "Paging - Max items". 'artists', + 'albums', and 'songs' are generators containing a dict of + api responses + ''' + artists = server.get_artists(library_id) + for batch in artists: + for item in batch['Items']: + LOG.debug('Artist: {}'.format(item.get('Name'))) + percent = int((float(count) / float(total_items)) * 100) + dialog.update(percent, message='Artist: {}'.format(item.get('Name'))) + obj.artist(item) + count += 1 + # Delete item once it's been processed for memory management + del item - # Update percentage dialog - percent = int((float(index) / float(num_artists)) * 100) - dialog.update(percent, heading="%s: %s" % (translate('addon_name'), library['Name']), message=artist_name) + albums = server.get_items(library_id, item_type='MusicAlbum', params={'SortBy': 'AlbumArtist'}) + for batch in albums: + for item in batch['Items']: + LOG.debug('Album: {}'.format(item.get('Name'))) + percent = int((float(count) / float(total_items)) * 100) + dialog.update(percent, message='Album: {} - {}'.format(item.get('AlbumArtist', ''), item.get('Name'))) + obj.album(item) + count += 1 + del item - # Add artist to database - obj.artist(artist) + songs = server.get_items(library_id, item_type='Audio', params={'SortBy': 'AlbumArtist'}) + for batch in songs: + for item in batch['Items']: + LOG.debug('Song: {}'.format(item.get('Name'))) + percent = int((float(count) / float(total_items)) * 100) + dialog.update(percent, message='Track: {} - {}'.format(item.get('AlbumArtist', ''), item.get('Name'))) + obj.song(item) + count += 1 + del item - # Get all albums for each artist - artist_albums = [album for album in albums if artist_name in album.get('Artists')] - for album in artist_albums: - # Add album to database - obj.album(album) - album_id = album.get('Id') - # Get all songs in each album - album_songs = [song for song in songs if album_id == song.get('AlbumId')] - for song in album_songs: - dialog.update(percent, - message="%s/%s/%s" % (artist_name, album['Name'][:7], song['Name'][:7])) - # Add song to database - obj.song(song) -# if self.update_library: self.music_compare(library, obj, jellyfindb)