From db6cad8e15d3999b06b771a1385ad97a605a8dd1 Mon Sep 17 00:00:00 2001 From: Matt Date: Mon, 7 Sep 2020 22:26:36 -0400 Subject: [PATCH 1/4] Optimize music library lookups for larger libraries --- jellyfin_kodi/downloader.py | 53 ++++++++++++++++++++++-------- jellyfin_kodi/full_sync.py | 64 +++++++++++++++++++++---------------- 2 files changed, 76 insertions(+), 41 deletions(-) diff --git a/jellyfin_kodi/downloader.py b/jellyfin_kodi/downloader.py index b2db586b..77965986 100644 --- a/jellyfin_kodi/downloader.py +++ b/jellyfin_kodi/downloader.py @@ -164,6 +164,27 @@ def get_episode_by_season(show_id, season_id): yield items +def get_item_count(parent_id, item_type=None, params=None): + + url = "Users/{UserId}/Items" + + query_params = { + 'ParentId': parent_id, + 'IncludeItemTypes': item_type, + 'EnableTotalRecordCount': True, + 'LocationTypes': "FileSystem,Remote,Offline", + 'Recursive': True, + 'Limit': 1 + } + if params: + query_params['params'].update(params) + + result = _get(url, query_params) + + total = result.get('TotalRecordCount') + + return total + def get_items(parent_id, item_type=None, basic=False, params=None): query = { @@ -191,23 +212,27 @@ def get_items(parent_id, item_type=None, basic=False, params=None): def get_artists(parent_id=None): - url = "Artists" + #url = "Artists" - params = { - 'UserId': "{UserId}", - 'ParentId': parent_id, - 'SortBy': "SortName", - 'SortOrder': "Ascending", - 'Fields': api.music_info(), - 'CollapseBoxSetItems': False, - 'IsVirtualUnaired': False, - 'EnableTotalRecordCount': False, - 'LocationTypes': "FileSystem,Remote,Offline", - 'IsMissing': False, - 'Recursive': True + query = { + 'url': 'Artists', + 'params': { + 'UserId': "{UserId}", + 'ParentId': parent_id, + 'SortBy': "SortName", + 'SortOrder': "Ascending", + 'Fields': api.music_info(), + 'CollapseBoxSetItems': False, + 'IsVirtualUnaired': False, + 'EnableTotalRecordCount': False, + 'LocationTypes': "FileSystem,Remote,Offline", + 'IsMissing': False, + 'Recursive': True + } } - return _get(url, params) + for items in _get_items(query): + yield items def get_library_items(library_id, item_type): diff --git a/jellyfin_kodi/full_sync.py b/jellyfin_kodi/full_sync.py index 7a3ca779..a64e0305 100644 --- a/jellyfin_kodi/full_sync.py +++ b/jellyfin_kodi/full_sync.py @@ -417,37 +417,47 @@ class FullSync(object): library_id = library['Id'] - # Get all items in the library to process locally - artists_data = server.get_artists(library_id) - artists = artists_data['Items'] - num_artists = artists_data['TotalRecordCount'] - albums = server.get_library_items(library_id, 'MusicAlbum')['Items'] - songs = server.get_library_items(library_id, 'Audio')['Items'] + total_items = server.get_item_count(library_id, 'MusicArtist,MusicAlbum,Audio') + count = 0 - for index, artist in enumerate(artists): - artist_name = artist.get('Name') + ''' + Music database syncing. Artists must be in the database + before albums, albums before songs. Pulls batches of items + in sizes of setting "Paging - Max items". 'artists', + 'albums', and 'songs' are generators containing a dict of + api responses + ''' + artists = server.get_artists(library_id) + for batch in artists: + for item in batch['Items']: + LOG.debug('Artist: {}'.format(item.get('Name'))) + percent = int((float(count) / float(total_items)) * 100) + dialog.update(percent, message='Artist: {}'.format(item.get('Name'))) + obj.artist(item) + count += 1 + # Delete item once it's been processed for memory management + del item - # Update percentage dialog - percent = int((float(index) / float(num_artists)) * 100) - dialog.update(percent, heading="%s: %s" % (translate('addon_name'), library['Name']), message=artist_name) + albums = server.get_items(library_id, item_type='MusicAlbum', params={'SortBy': 'AlbumArtist'}) + for batch in albums: + for item in batch['Items']: + LOG.debug('Album: {}'.format(item.get('Name'))) + percent = int((float(count) / float(total_items)) * 100) + dialog.update(percent, message='Album: {} - {}'.format(item.get('AlbumArtist', ''), item.get('Name'))) + obj.album(item) + count += 1 + del item - # Add artist to database - obj.artist(artist) + songs = server.get_items(library_id, item_type='Audio', params={'SortBy': 'AlbumArtist'}) + for batch in songs: + for item in batch['Items']: + LOG.debug('Song: {}'.format(item.get('Name'))) + percent = int((float(count) / float(total_items)) * 100) + dialog.update(percent, message='Track: {} - {}'.format(item.get('AlbumArtist', ''), item.get('Name'))) + obj.song(item) + count += 1 + del item - # Get all albums for each artist - artist_albums = [album for album in albums if artist_name in album.get('Artists')] - for album in artist_albums: - # Add album to database - obj.album(album) - album_id = album.get('Id') - # Get all songs in each album - album_songs = [song for song in songs if album_id == song.get('AlbumId')] - for song in album_songs: - dialog.update(percent, - message="%s/%s/%s" % (artist_name, album['Name'][:7], song['Name'][:7])) - # Add song to database - obj.song(song) -# if self.update_library: self.music_compare(library, obj, jellyfindb) From 983c208415fbd5365f539a55b28b22a174af521e Mon Sep 17 00:00:00 2001 From: Matt Date: Mon, 7 Sep 2020 22:33:21 -0400 Subject: [PATCH 2/4] Remove unused functions --- jellyfin_kodi/downloader.py | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/jellyfin_kodi/downloader.py b/jellyfin_kodi/downloader.py index 77965986..ff4fb65b 100644 --- a/jellyfin_kodi/downloader.py +++ b/jellyfin_kodi/downloader.py @@ -235,41 +235,6 @@ def get_artists(parent_id=None): yield items -def get_library_items(library_id, item_type): - url = "Users/{UserId}/Items" - - params = { - 'ParentId': library_id, - 'IncludeItemTypes': item_type, - 'SortBy': "SortName", - 'SortOrder': "Ascending", - 'Fields': api.info(), - 'Recursive': True, - } - - return _get(url, params) - - -def get_albums_by_artist(artist_id, basic=False): - - params = { - 'SortBy': "DateCreated", - 'ArtistIds': artist_id - } - for items in get_items(None, "MusicAlbum", basic, params): - yield items - - -def get_songs_by_artist(artist_id, basic=False): - - params = { - 'SortBy': "DateCreated", - 'ArtistIds': artist_id - } - for items in get_items(None, "Audio", basic, params): - yield items - - @stop def _get_items(query, server_id=None): From c44a0795724995d23f3c9bb29950c74fdeae358a Mon Sep 17 00:00:00 2001 From: Matt Date: Mon, 7 Sep 2020 22:47:10 -0400 Subject: [PATCH 3/4] remove commented variable --- jellyfin_kodi/downloader.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/jellyfin_kodi/downloader.py b/jellyfin_kodi/downloader.py index ff4fb65b..c87746f0 100644 --- a/jellyfin_kodi/downloader.py +++ b/jellyfin_kodi/downloader.py @@ -212,8 +212,6 @@ def get_items(parent_id, item_type=None, basic=False, params=None): def get_artists(parent_id=None): - #url = "Artists" - query = { 'url': 'Artists', 'params': { From 813ec68e382965b5a60f0a526a9d89edfed047a2 Mon Sep 17 00:00:00 2001 From: Matt Date: Wed, 9 Sep 2020 18:33:47 -0400 Subject: [PATCH 4/4] Review suggestions --- jellyfin_kodi/downloader.py | 4 +--- jellyfin_kodi/full_sync.py | 4 ---- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/jellyfin_kodi/downloader.py b/jellyfin_kodi/downloader.py index c87746f0..bf2a07d6 100644 --- a/jellyfin_kodi/downloader.py +++ b/jellyfin_kodi/downloader.py @@ -181,9 +181,7 @@ def get_item_count(parent_id, item_type=None, params=None): result = _get(url, query_params) - total = result.get('TotalRecordCount') - - return total + return result.get('TotalRecordCount', 1) def get_items(parent_id, item_type=None, basic=False, params=None): diff --git a/jellyfin_kodi/full_sync.py b/jellyfin_kodi/full_sync.py index a64e0305..5f93d5ab 100644 --- a/jellyfin_kodi/full_sync.py +++ b/jellyfin_kodi/full_sync.py @@ -435,8 +435,6 @@ class FullSync(object): dialog.update(percent, message='Artist: {}'.format(item.get('Name'))) obj.artist(item) count += 1 - # Delete item once it's been processed for memory management - del item albums = server.get_items(library_id, item_type='MusicAlbum', params={'SortBy': 'AlbumArtist'}) for batch in albums: @@ -446,7 +444,6 @@ class FullSync(object): dialog.update(percent, message='Album: {} - {}'.format(item.get('AlbumArtist', ''), item.get('Name'))) obj.album(item) count += 1 - del item songs = server.get_items(library_id, item_type='Audio', params={'SortBy': 'AlbumArtist'}) for batch in songs: @@ -456,7 +453,6 @@ class FullSync(object): dialog.update(percent, message='Track: {} - {}'.format(item.get('AlbumArtist', ''), item.get('Name'))) obj.song(item) count += 1 - del item if self.update_library: self.music_compare(library, obj, jellyfindb)