jellyfin-kodi/jellyfin_kodi/full_sync.py

587 lines
22 KiB
Python
Raw Normal View History

2018-09-06 08:36:32 +00:00
# -*- coding: utf-8 -*-
from __future__ import division, absolute_import, print_function, unicode_literals
2018-09-06 08:36:32 +00:00
##################################################################################################
from contextlib import contextmanager
2018-09-06 08:36:32 +00:00
import datetime
from kodi_six import xbmc
2018-09-06 08:36:32 +00:00
import downloader as server
import helper.xmls as xmls
2020-03-29 03:25:13 +00:00
from objects import Movies, TVShows, MusicVideos, Music
from database import Database, get_sync, save_sync, jellyfin_db
2020-07-31 21:45:54 +00:00
from helper import translate, settings, window, progress, dialog
from helper.utils import get_screensaver, set_screensaver
2020-04-19 01:05:59 +00:00
from helper import LazyLogger
2020-07-31 21:45:54 +00:00
from helper.exceptions import LibraryException, PathValidationException
2018-09-06 08:36:32 +00:00
##################################################################################################
2020-04-19 01:05:59 +00:00
LOG = LazyLogger(__name__)
2018-09-06 08:36:32 +00:00
##################################################################################################
class FullSync(object):
2019-01-25 15:30:30 +00:00
''' This should be called like a context.
i.e. with FullSync('jellyfin') as sync:
2019-01-25 15:30:30 +00:00
sync.libraries()
'''
# Borg - multiple instances, shared state
_shared_state = {}
2018-09-06 08:36:32 +00:00
sync = None
running = False
2019-01-25 11:32:29 +00:00
screensaver = None
2018-09-06 08:36:32 +00:00
2019-01-25 15:30:30 +00:00
def __init__(self, library, server):
2019-07-09 20:05:28 +00:00
''' You can call all big syncing methods here.
2019-01-25 15:30:30 +00:00
Initial, update, repair, remove.
'''
self.__dict__ = self._shared_state
2019-01-25 15:30:30 +00:00
if self.running:
dialog("ok", "{jellyfin}", translate(33197))
2019-01-25 15:30:30 +00:00
raise Exception("Sync is already running.")
self.library = library
self.server = server
def __enter__(self):
''' Do everything we need before the sync
'''
LOG.info("-->[ fullsync ]")
2019-01-25 11:32:29 +00:00
if not settings('dbSyncScreensaver.bool'):
xbmc.executebuiltin('InhibitIdleShutdown(true)')
self.screensaver = get_screensaver()
set_screensaver(value="")
2018-09-06 08:36:32 +00:00
2019-01-25 15:30:30 +00:00
self.running = True
window('jellyfin_sync.bool', True)
2019-01-25 15:30:30 +00:00
return self
2019-01-25 15:30:30 +00:00
def libraries(self, library_id=None, update=False):
2019-01-25 15:30:30 +00:00
''' Map the syncing process and start the sync. Ensure only one sync is running.
'''
self.direct_path = settings('useDirectPaths') == "1"
self.update_library = update
self.sync = get_sync()
2019-01-25 15:30:30 +00:00
if library_id:
libraries = library_id.split(',')
2018-09-06 08:36:32 +00:00
2019-01-25 15:30:30 +00:00
for selected in libraries:
2019-01-25 15:30:30 +00:00
if selected not in [x.replace('Mixed:', "") for x in self.sync['Libraries']]:
library = self.get_libraries(selected)
2019-01-25 15:30:30 +00:00
if library:
2019-01-25 15:30:30 +00:00
self.sync['Libraries'].append("Mixed:%s" % selected if library[1] == 'mixed' else selected)
if library[1] in ('mixed', 'movies'):
self.sync['Libraries'].append('Boxsets:%s' % selected)
else:
self.sync['Libraries'].append(selected)
else:
2019-01-25 15:30:30 +00:00
self.mapping()
2019-01-25 15:30:30 +00:00
xmls.sources()
if not xmls.advanced_settings() and self.sync['Libraries']:
self.start()
2018-09-06 08:36:32 +00:00
def get_libraries(self, library_id=None):
with Database('jellyfin') as jellyfindb:
if library_id is None:
return jellyfin_db.JellyfinDatabase(jellyfindb.cursor).get_views()
else:
return jellyfin_db.JellyfinDatabase(jellyfindb.cursor).get_view(library_id)
2018-09-06 08:36:32 +00:00
def mapping(self):
''' Load the mapping of the full sync.
This allows us to restore a previous sync.
'''
if self.sync['Libraries']:
if not dialog("yesno", "{jellyfin}", translate(33102)):
2018-09-06 08:36:32 +00:00
if not dialog("yesno", "{jellyfin}", translate(33173)):
dialog("ok", "{jellyfin}", translate(33122))
2018-09-06 08:36:32 +00:00
raise LibraryException("ProgressStopped")
else:
self.sync['Libraries'] = []
self.sync['RestorePoint'] = {}
2018-09-06 08:36:32 +00:00
else:
LOG.info("generate full sync")
libraries = []
for library in self.get_libraries():
2018-09-06 08:36:32 +00:00
if library[2] in ('movies', 'tvshows', 'musicvideos', 'music', 'mixed'):
libraries.append({'Id': library[0], 'Name': library[1], 'Media': library[2]})
2018-09-06 08:36:32 +00:00
libraries = self.select_libraries(libraries)
if [x['Media'] for x in libraries if x['Media'] in ('movies', 'mixed')]:
self.sync['Libraries'].append("Boxsets:")
save_sync(self.sync)
2018-09-06 08:36:32 +00:00
def select_libraries(self, libraries):
''' Select all or certain libraries to be whitelisted.
2018-09-06 08:36:32 +00:00
'''
choices = [x['Name'] for x in libraries]
choices.insert(0, translate(33121))
selection = dialog("multi", translate(33120), choices)
2018-09-06 08:36:32 +00:00
if selection is None:
raise LibraryException('LibrarySelection')
elif not selection:
LOG.info("Nothing was selected.")
2018-09-23 08:03:15 +00:00
2018-09-06 08:36:32 +00:00
raise LibraryException('SyncLibraryLater')
if 0 in selection:
selection = list(range(1, len(libraries) + 1))
selected_libraries = []
for x in selection:
library = libraries[x - 1]
if library['Media'] != 'mixed':
selected_libraries.append(library['Id'])
else:
selected_libraries.append("Mixed:%s" % library['Id'])
self.sync['Libraries'] = selected_libraries
return [libraries[x - 1] for x in selection]
def start(self):
2019-07-09 20:05:28 +00:00
2018-09-06 08:36:32 +00:00
''' Main sync process.
'''
LOG.info("starting sync with %s", self.sync['Libraries'])
2018-09-23 08:03:15 +00:00
save_sync(self.sync)
2018-09-06 08:36:32 +00:00
start_time = datetime.datetime.now()
2019-01-25 11:32:29 +00:00
for library in list(self.sync['Libraries']):
2019-01-25 11:32:29 +00:00
self.process_library(library)
2019-01-25 11:32:29 +00:00
if not library.startswith('Boxsets:') and library not in self.sync['Whitelist']:
self.sync['Whitelist'].append(library)
2018-09-06 08:36:32 +00:00
2019-01-25 11:32:29 +00:00
self.sync['Libraries'].pop(self.sync['Libraries'].index(library))
self.sync['RestorePoint'] = {}
2018-09-06 08:36:32 +00:00
elapsed = datetime.datetime.now() - start_time
settings('SyncInstallRunDone.bool', True)
self.library.save_last_sync()
save_sync(self.sync)
2018-09-06 08:36:32 +00:00
xbmc.executebuiltin('UpdateLibrary(video)')
dialog("notification", heading="{jellyfin}", message="%s %s" % (translate(33025), str(elapsed).split('.')[0]),
icon="{jellyfin}", sound=False)
2018-09-06 08:36:32 +00:00
LOG.info("Full sync completed in: %s", str(elapsed).split('.')[0])
def process_library(self, library_id):
''' Add a library by it's id. Create a node and a playlist whenever appropriate.
'''
media = {
'movies': self.movies,
'musicvideos': self.musicvideos,
'tvshows': self.tvshows,
'music': self.music
}
try:
if library_id.startswith('Boxsets:'):
boxset_library = {}
2018-09-06 08:36:32 +00:00
2020-08-02 22:50:57 +00:00
libraries = self.get_libraries(library_id.split('Boxsets:')[1] if len(library_id) > len('Boxsets:') else None)
for entry in libraries:
if entry[2] == 'boxsets':
boxset_library = {'Id': entry[0], 'Name': entry[1]}
break
if boxset_library:
if library_id.endswith('Refresh'):
self.refresh_boxsets(boxset_library)
else:
self.boxsets(boxset_library)
2018-09-06 08:36:32 +00:00
return
2019-09-04 02:58:31 +00:00
library = self.server.jellyfin.get_item(library_id.replace('Mixed:', ""))
2018-09-06 08:36:32 +00:00
if library_id.startswith('Mixed:'):
for mixed in ('movies', 'tvshows'):
media[mixed](library)
self.sync['RestorePoint'] = {}
else:
if library['CollectionType']:
settings('enableMusic.bool', True)
media[library['CollectionType']](library)
except LibraryException as error:
if error.status == 'StopCalled':
save_sync(self.sync)
raise
2020-07-31 21:45:54 +00:00
except PathValidationException:
raise
2018-09-06 08:36:32 +00:00
except Exception as error:
2020-07-31 21:45:54 +00:00
dialog("ok", "{jellyfin}", translate(33119))
2018-09-06 08:36:32 +00:00
2020-07-31 21:45:54 +00:00
LOG.error("full sync exited unexpectedly")
LOG.exception(error)
2020-07-31 21:45:54 +00:00
save_sync(self.sync)
2018-09-06 08:36:32 +00:00
raise
@contextmanager
def video_database_locks(self):
with self.library.database_lock:
with Database() as videodb:
with Database('jellyfin') as jellyfindb:
yield videodb, jellyfindb
2018-09-06 08:36:32 +00:00
@progress()
def movies(self, library, dialog):
''' Process movies from a single library.
'''
processed_ids = []
2019-01-25 15:30:30 +00:00
for items in server.get_items(library['Id'], "Movie", False, self.sync['RestorePoint'].get('params')):
with self.video_database_locks() as (videodb, jellyfindb):
2020-08-02 21:00:34 +00:00
obj = Movies(self.server, jellyfindb, videodb, self.direct_path, library)
self.sync['RestorePoint'] = items['RestorePoint']
start_index = items['RestorePoint']['params']['StartIndex']
2019-07-09 20:05:28 +00:00
for index, movie in enumerate(items['Items']):
2018-09-06 08:36:32 +00:00
dialog.update(int((float(start_index + index) / float(items['TotalRecordCount'])) * 100),
heading="%s: %s" % (translate('addon_name'), library['Name']),
message=movie['Name'])
2020-08-02 21:00:34 +00:00
obj.movie(movie)
processed_ids.append(movie['Id'])
2018-09-06 08:36:32 +00:00
with self.video_database_locks() as (videodb, jellyfindb):
2020-08-02 21:00:34 +00:00
obj = Movies(self.server, jellyfindb, videodb, self.direct_path, library)
obj.item_ids = processed_ids
2018-09-06 08:36:32 +00:00
if self.update_library:
self.movies_compare(library, obj, jellyfindb)
def movies_compare(self, library, obj, jellyfinydb):
''' Compare entries from library to what's in the jellyfindb. Remove surplus
'''
db = jellyfin_db.JellyfinDatabase(jellyfinydb.cursor)
items = db.get_item_by_media_folder(library['Id'])
current = obj.item_ids
for x in items:
2019-01-20 10:49:15 +00:00
if x[0] not in current and x[1] == 'Movie':
obj.remove(x[0])
2018-09-06 08:36:32 +00:00
@progress()
def tvshows(self, library, dialog):
''' Process tvshows and episodes from a single library.
'''
processed_ids = []
2019-01-25 15:30:30 +00:00
for items in server.get_items(library['Id'], "Series", False, self.sync['RestorePoint'].get('params')):
with self.video_database_locks() as (videodb, jellyfindb):
2020-08-02 21:00:34 +00:00
obj = TVShows(self.server, jellyfindb, videodb, self.direct_path, library, True)
self.sync['RestorePoint'] = items['RestorePoint']
start_index = items['RestorePoint']['params']['StartIndex']
2018-09-06 08:36:32 +00:00
for index, show in enumerate(items['Items']):
2018-09-06 08:36:32 +00:00
percent = int((float(start_index + index) / float(items['TotalRecordCount'])) * 100)
message = show['Name']
dialog.update(percent, heading="%s: %s" % (translate('addon_name'), library['Name']), message=message)
2018-09-06 08:36:32 +00:00
2020-08-02 21:00:34 +00:00
if obj.tvshow(show) is not False:
2018-09-06 08:36:32 +00:00
for episodes in server.get_episode_by_show(show['Id']):
for episode in episodes['Items']:
2018-09-06 08:36:32 +00:00
dialog.update(percent, message="%s/%s" % (message, episode['Name'][:10]))
obj.episode(episode)
processed_ids.append(show['Id'])
with self.video_database_locks() as (videodb, jellyfindb):
2020-08-02 21:00:34 +00:00
obj = TVShows(self.server, jellyfindb, videodb, self.direct_path, library, True)
obj.item_ids = processed_ids
if self.update_library:
self.tvshows_compare(library, obj, jellyfindb)
def tvshows_compare(self, library, obj, jellyfindb):
''' Compare entries from library to what's in the jellyfindb. Remove surplus
'''
db = jellyfin_db.JellyfinDatabase(jellyfindb.cursor)
2018-09-06 08:36:32 +00:00
items = db.get_item_by_media_folder(library['Id'])
for x in list(items):
items.extend(obj.get_child(x[0]))
current = obj.item_ids
2018-09-06 08:36:32 +00:00
for x in items:
2019-01-20 10:49:15 +00:00
if x[0] not in current and x[1] == 'Series':
obj.remove(x[0])
2018-09-06 08:36:32 +00:00
@progress()
def musicvideos(self, library, dialog):
''' Process musicvideos from a single library.
'''
2020-02-27 02:05:38 +00:00
processed_ids = []
2019-01-25 15:30:30 +00:00
for items in server.get_items(library['Id'], "MusicVideo", False, self.sync['RestorePoint'].get('params')):
2018-09-06 08:36:32 +00:00
with self.video_database_locks() as (videodb, jellyfindb):
2020-08-02 21:00:34 +00:00
obj = MusicVideos(self.server, jellyfindb, videodb, self.direct_path, library)
2018-09-06 08:36:32 +00:00
self.sync['RestorePoint'] = items['RestorePoint']
start_index = items['RestorePoint']['params']['StartIndex']
for index, mvideo in enumerate(items['Items']):
dialog.update(int((float(start_index + index) / float(items['TotalRecordCount'])) * 100),
heading="%s: %s" % (translate('addon_name'), library['Name']),
message=mvideo['Name'])
2020-08-02 21:00:34 +00:00
obj.musicvideo(mvideo)
2020-02-27 02:05:38 +00:00
processed_ids.append(mvideo['Id'])
with self.video_database_locks() as (videodb, jellyfindb):
2020-08-02 21:00:34 +00:00
obj = MusicVideos(self.server, jellyfindb, videodb, self.direct_path, library)
2020-02-27 02:05:38 +00:00
obj.item_ids = processed_ids
if self.update_library:
self.musicvideos_compare(library, obj, jellyfindb)
def musicvideos_compare(self, library, obj, jellyfindb):
''' Compare entries from library to what's in the jellyfindb. Remove surplus
'''
db = jellyfin_db.JellyfinDatabase(jellyfindb.cursor)
2018-09-06 08:36:32 +00:00
items = db.get_item_by_media_folder(library['Id'])
current = obj.item_ids
2018-09-06 08:36:32 +00:00
for x in items:
2019-01-20 10:49:15 +00:00
if x[0] not in current and x[1] == 'MusicVideo':
obj.remove(x[0])
2018-09-06 08:36:32 +00:00
@progress()
def music(self, library, dialog):
''' Process artists, album, songs from a single library.
'''
with self.library.music_database_lock:
with Database('music') as musicdb:
with Database('jellyfin') as jellyfindb:
2020-08-02 21:00:34 +00:00
obj = Music(self.server, jellyfindb, musicdb, self.direct_path, library)
2018-09-06 08:36:32 +00:00
library_id = library['Id']
2020-08-02 21:00:34 +00:00
total_items = server.get_item_count(library_id, 'MusicArtist,MusicAlbum,Audio')
count = 0
'''
Music database syncing. Artists must be in the database
before albums, albums before songs. Pulls batches of items
in sizes of setting "Paging - Max items". 'artists',
'albums', and 'songs' are generators containing a dict of
api responses
'''
artists = server.get_artists(library_id)
for batch in artists:
for item in batch['Items']:
LOG.debug('Artist: {}'.format(item.get('Name')))
percent = int((float(count) / float(total_items)) * 100)
dialog.update(percent, message='Artist: {}'.format(item.get('Name')))
obj.artist(item)
count += 1
albums = server.get_items(library_id, item_type='MusicAlbum', params={'SortBy': 'AlbumArtist'})
for batch in albums:
for item in batch['Items']:
LOG.debug('Album: {}'.format(item.get('Name')))
percent = int((float(count) / float(total_items)) * 100)
dialog.update(percent, message='Album: {} - {}'.format(item.get('AlbumArtist', ''), item.get('Name')))
obj.album(item)
count += 1
songs = server.get_items(library_id, item_type='Audio', params={'SortBy': 'AlbumArtist'})
for batch in songs:
for item in batch['Items']:
LOG.debug('Song: {}'.format(item.get('Name')))
percent = int((float(count) / float(total_items)) * 100)
dialog.update(percent, message='Track: {} - {}'.format(item.get('AlbumArtist', ''), item.get('Name')))
obj.song(item)
count += 1
if self.update_library:
self.music_compare(library, obj, jellyfindb)
def music_compare(self, library, obj, jellyfindb):
''' Compare entries from library to what's in the jellyfindb. Remove surplus
'''
db = jellyfin_db.JellyfinDatabase(jellyfindb.cursor)
items = db.get_item_by_media_folder(library['Id'])
for x in list(items):
items.extend(obj.get_child(x[0]))
current = obj.item_ids
for x in items:
2019-01-20 10:49:15 +00:00
if x[0] not in current and x[1] == 'MusicArtist':
obj.remove(x[0])
@progress(translate(33018))
2020-08-02 22:50:57 +00:00
def boxsets(self, library, dialog=None):
2018-09-06 08:36:32 +00:00
''' Process all boxsets.
'''
2020-08-02 22:50:57 +00:00
for items in server.get_items(library['Id'], "BoxSet", False, self.sync['RestorePoint'].get('params')):
2018-09-06 08:36:32 +00:00
with self.video_database_locks() as (videodb, jellyfindb):
2020-08-02 21:00:34 +00:00
obj = Movies(self.server, jellyfindb, videodb, self.direct_path, library)
2018-09-06 08:36:32 +00:00
self.sync['RestorePoint'] = items['RestorePoint']
start_index = items['RestorePoint']['params']['StartIndex']
2018-09-06 08:36:32 +00:00
for index, boxset in enumerate(items['Items']):
2018-09-06 08:36:32 +00:00
dialog.update(int((float(start_index + index) / float(items['TotalRecordCount'])) * 100),
heading="%s: %s" % (translate('addon_name'), translate('boxsets')),
message=boxset['Name'])
obj.boxset(boxset)
2018-09-06 08:36:32 +00:00
2020-08-02 22:50:57 +00:00
def refresh_boxsets(self, library):
2018-09-06 08:36:32 +00:00
''' Delete all exisitng boxsets and re-add.
'''
with self.video_database_locks() as (videodb, jellyfindb):
2020-08-02 21:00:34 +00:00
obj = Movies(self.server, jellyfindb, videodb, self.direct_path, library)
obj.boxsets_reset()
2018-09-06 08:36:32 +00:00
2018-10-10 21:38:49 +00:00
self.boxsets(None)
2019-01-25 15:30:30 +00:00
@progress(translate(33144))
2019-01-25 15:30:30 +00:00
def remove_library(self, library_id, dialog):
''' Remove library by their id from the Kodi database.
'''
direct_path = self.library.direct_path
with Database('jellyfin') as jellyfindb:
2019-01-25 15:30:30 +00:00
db = jellyfin_db.JellyfinDatabase(jellyfindb.cursor)
2019-01-25 15:30:30 +00:00
library = db.get_view(library_id.replace('Mixed:', ""))
items = db.get_item_by_media_folder(library_id.replace('Mixed:', ""))
media = 'music' if library[1] == 'music' else 'video'
if media == 'music':
settings('MusicRescan.bool', False)
if items:
count = 0
with self.library.music_database_lock if media == 'music' else self.library.database_lock:
with Database(media) as kodidb:
if library[1] == 'mixed':
movies = [x for x in items if x[1] == 'Movie']
tvshows = [x for x in items if x[1] == 'Series']
2020-08-02 21:00:34 +00:00
obj = Movies(self.server, jellyfindb, kodidb, direct_path, library).remove
2019-01-25 15:30:30 +00:00
for item in movies:
obj(item[0])
dialog.update(int((float(count) / float(len(items)) * 100)), heading="%s: %s" % (translate('addon_name'), library[0]))
2019-01-25 15:30:30 +00:00
count += 1
2020-08-02 21:00:34 +00:00
obj = TVShows(self.server, jellyfindb, kodidb, direct_path, library).remove
2019-01-25 15:30:30 +00:00
for item in tvshows:
obj(item[0])
dialog.update(int((float(count) / float(len(items)) * 100)), heading="%s: %s" % (translate('addon_name'), library[0]))
2019-01-25 15:30:30 +00:00
count += 1
else:
2020-03-29 03:25:13 +00:00
default_args = (self.server, jellyfindb, kodidb, direct_path)
2019-01-25 15:30:30 +00:00
for item in items:
2020-03-29 03:25:13 +00:00
if item[1] in ('Series', 'Season', 'Episode'):
2020-03-29 03:37:07 +00:00
TVShows(*default_args).remove(item[0])
2020-03-29 03:25:13 +00:00
elif item[1] in ('Movie', 'BoxSet'):
2020-03-29 03:37:07 +00:00
Movies(*default_args).remove(item[0])
2020-03-29 03:25:13 +00:00
elif item[1] in ('MusicAlbum', 'MusicArtist', 'AlbumArtist', 'Audio'):
2020-03-29 03:37:07 +00:00
Music(*default_args).remove(item[0])
2020-03-29 03:25:13 +00:00
elif item[1] == 'MusicVideo':
2020-03-29 03:37:07 +00:00
MusicVideos(*default_args).remove(item[0])
2019-01-25 15:30:30 +00:00
dialog.update(int((float(count) / float(len(items)) * 100)), heading="%s: %s" % (translate('addon_name'), library[0]))
2019-01-25 15:30:30 +00:00
count += 1
self.sync = get_sync()
if library_id in self.sync['Whitelist']:
self.sync['Whitelist'].remove(library_id)
2019-07-09 20:05:28 +00:00
2019-01-25 15:30:30 +00:00
elif 'Mixed:%s' % library_id in self.sync['Whitelist']:
self.sync['Whitelist'].remove('Mixed:%s' % library_id)
save_sync(self.sync)
def __exit__(self, exc_type, exc_val, exc_tb):
''' Exiting sync
'''
self.running = False
window('jellyfin_sync', clear=True)
2019-01-25 15:30:30 +00:00
if not settings('dbSyncScreensaver.bool') and self.screensaver is not None:
xbmc.executebuiltin('InhibitIdleShutdown(false)')
set_screensaver(value=self.screensaver)
LOG.info("--<[ fullsync ]")