# -*- coding: utf-8 -*-
##################################################################################################
import logging
import sqlite3
import threading
from datetime import datetime, timedelta, time
import xbmc
import xbmcgui
import xbmcvfs
import api
import utils
import clientinfo
import downloadutils
import itemtypes
import embydb_functions as embydb
import read_embyserver as embyserver
import userclient
import views
from objects import Movies, MusicVideos, TVShows, Music
from utils import window, settings, language as lang, should_stop
from ga_client import GoogleAnalytics
import database
from contextlib import closing
##################################################################################################
log = logging.getLogger("EMBY."+__name__)
##################################################################################################
class LibrarySync(threading.Thread):
_shared_state = {}
isFastSync = False
stop_thread = False
suspend_thread = False
# Track websocketclient updates
addedItems = []
updateItems = []
userdataItems = []
removeItems = []
forceLibraryUpdate = False
refresh_views = False
def __init__(self):
self.__dict__ = self._shared_state
self.monitor = xbmc.Monitor()
self.clientInfo = clientinfo.ClientInfo()
self.doUtils = downloadutils.DownloadUtils().downloadUrl
self.user = userclient.UserClient()
self.emby = embyserver.Read_EmbyServer()
self.kodi_version = int(xbmc.getInfoLabel('System.BuildVersion')[:2])
threading.Thread.__init__(self)
def progressDialog(self, title):
dialog = None
dialog = xbmcgui.DialogProgressBG()
dialog.create("Emby for Kodi", title)
log.debug("Show progress dialog: %s" % title)
return dialog
def startSync(self):
ga = GoogleAnalytics()
# Run at start up - optional to use the server plugin
if settings('SyncInstallRunDone') == "true":
# Validate views
self.refreshViews()
completed = False
# Verify if server plugin is installed.
if settings('serverSync') == "true":
# Try to use fast start up
url = "{server}/emby/Plugins?format=json"
result = self.doUtils(url)
for plugin in result:
if plugin['Name'] == "Emby.Kodi Sync Queue":
log.debug("Found server plugin.")
self.isFastSync = True
ga.sendEventData("SyncAction", "FastSync")
completed = self.fastSync()
break
if not completed:
# Fast sync failed or server plugin is not found
ga.sendEventData("SyncAction", "Sync")
completed = ManualSync().sync()
else:
# Install sync is not completed
ga.sendEventData("SyncAction", "FullSync")
completed = self.fullSync()
return completed
def fastSync(self):
lastSync = settings('LastIncrementalSync')
if not lastSync:
lastSync = "2010-01-01T00:00:00Z"
lastSyncTime = utils.convertDate(lastSync)
log.info("Last sync run: %s" % lastSyncTime)
# get server RetentionDateTime
result = self.doUtils("{server}/emby/Emby.Kodi.SyncQueue/GetServerDateTime?format=json")
try:
retention_time = result['RetentionDateTime']
except (TypeError, KeyError):
retention_time = "2010-01-01T00:00:00Z"
retention_time = utils.convertDate(retention_time)
log.info("RetentionDateTime: %s" % retention_time)
# if last sync before retention time do a full sync
if retention_time > lastSyncTime:
log.info("Fast sync server retention insufficient, fall back to full sync")
return False
params = {'LastUpdateDT': lastSync}
if settings('enableMusic') != "true":
params['filter'] = "music"
url = "{server}/emby/Emby.Kodi.SyncQueue/{UserId}/GetItems?format=json"
result = self.doUtils(url, parameters=params)
try:
processlist = {
'added': result['ItemsAdded'],
'update': result['ItemsUpdated'],
'userdata': result['UserDataChanged'],
'remove': result['ItemsRemoved']
}
except (KeyError, TypeError):
log.error("Failed to retrieve latest updates using fast sync.")
return False
else:
log.info("Fast sync changes: %s" % result)
for action in processlist:
self.triage_items(action, processlist[action])
return True
def saveLastSync(self):
# Save last sync time
overlap = 2
try: # datetime fails when used more than once, TypeError
if self.isFastSync:
result = self.doUtils("{server}/emby/Emby.Kodi.SyncQueue/GetServerDateTime?format=json")
server_time = result['ServerDateTime']
server_time = utils.convertDate(server_time)
else:
raise Exception("Fast sync server plugin is not enabled.")
except Exception as e:
# If the server plugin is not installed or an error happened.
log.debug("An exception occurred: %s" % e)
time_now = datetime.utcnow()-timedelta(minutes=overlap)
lastSync = time_now.strftime('%Y-%m-%dT%H:%M:%SZ')
log.info("New sync time: client time -%s min: %s" % (overlap, lastSync))
else:
lastSync = (server_time - timedelta(minutes=overlap)).strftime('%Y-%m-%dT%H:%M:%SZ')
log.info("New sync time: server time -%s min: %s" % (overlap, lastSync))
finally:
settings('LastIncrementalSync', value=lastSync)
def dbCommit(self, connection):
# Central commit, verifies if Kodi database update is running
kodidb_scan = window('emby_kodiScan') == "true"
count = 0
while kodidb_scan:
log.info("Kodi scan is running. Waiting...")
kodidb_scan = window('emby_kodiScan') == "true"
if count == 10:
log.info("Flag still active, but will try to commit")
window('emby_kodiScan', clear=True)
if should_stop():
log.info("Commit unsuccessful. Sync terminated.")
break
if self.monitor.waitForAbort(1):
# Abort was requested while waiting. We should exit
log.info("Commit unsuccessful.")
break
count += 1
try:
connection.commit()
log.info("Commit successful.")
except sqlite3.OperationalError as error:
log.error(error)
if "database is locked" in error:
log.info("retrying...")
window('emby_kodiScan', value="true")
self.dbCommit(connection)
def fullSync(self, manualrun=False, repair=False):
# Only run once when first setting up. Can be run manually.
music_enabled = settings('enableMusic') == "true"
xbmc.executebuiltin('InhibitIdleShutdown(true)')
screensaver = utils.getScreensaver()
utils.setScreensaver(value="")
window('emby_dbScan', value="true")
# Add sources
utils.sourcesXML()
# use emby and video DBs
with database.DatabaseConn('emby') as conn_emby, database.DatabaseConn('video') as conn_video:
with closing(conn_emby.cursor()) as cursor_emby, closing(conn_video.cursor()) as cursor_video:
# content sync: movies, tvshows, musicvideos, music
if manualrun:
message = "Manual sync"
elif repair:
message = "Repair sync"
repair_list = []
choices = ['all', 'movies', 'musicvideos', 'tvshows']
if music_enabled:
choices.append('music')
if self.kodi_version > 15:
# Jarvis or higher
types = xbmcgui.Dialog().multiselect(lang(33094), choices)
if types is None:
pass
elif 0 in types: # all
choices.pop(0)
repair_list.extend(choices)
else:
for index in types:
repair_list.append(choices[index])
else:
resp = xbmcgui.Dialog().select(lang(33094), choices)
if resp == 0: # all
choices.pop(resp)
repair_list.extend(choices)
else:
repair_list.append(choices[resp])
log.info("Repair queued for: %s", repair_list)
else:
message = "Initial sync"
window('emby_initialScan', value="true")
pDialog = self.progressDialog("%s" % message)
starttotal = datetime.now()
# Set views
views.Views(cursor_emby, cursor_video).maintain()
conn_emby.commit()
#self.maintainViews(cursor_emby, cursor_video)
# Sync video library
process = {
'movies': self.movies,
'musicvideos': self.musicvideos,
'tvshows': self.tvshows
}
for itemtype in process:
if repair and itemtype not in repair_list:
continue
startTime = datetime.now()
completed = process[itemtype](cursor_emby, cursor_video, pDialog)
if not completed:
xbmc.executebuiltin('InhibitIdleShutdown(false)')
utils.setScreensaver(value=screensaver)
window('emby_dbScan', clear=True)
if pDialog:
pDialog.close()
return False
else:
elapsedTime = datetime.now() - startTime
log.info("SyncDatabase (finished %s in: %s)"
% (itemtype, str(elapsedTime).split('.')[0]))
# sync music
# use emby and music
if music_enabled:
if repair and 'music' not in repair_list:
pass
else:
with database.DatabaseConn('emby') as conn_emby, database.DatabaseConn('music') as conn_music:
with closing(conn_emby.cursor()) as cursor_emby, closing(conn_music.cursor()) as cursor_music:
startTime = datetime.now()
completed = self.music(cursor_emby, cursor_music, pDialog)
if not completed:
xbmc.executebuiltin('InhibitIdleShutdown(false)')
utils.setScreensaver(value=screensaver)
window('emby_dbScan', clear=True)
if pDialog:
pDialog.close()
return False
else:
elapsedTime = datetime.now() - startTime
log.info("SyncDatabase (finished music in: %s)"
% (str(elapsedTime).split('.')[0]))
if pDialog:
pDialog.close()
with database.DatabaseConn('emby') as conn_emby:
with closing(conn_emby.cursor()) as cursor_emby:
emby_db = embydb.Embydb_Functions(cursor_emby)
current_version = emby_db.get_version(self.clientInfo.get_version())
window('emby_version', current_version)
settings('SyncInstallRunDone', value="true")
self.saveLastSync()
xbmc.executebuiltin('UpdateLibrary(video)')
elapsedtotal = datetime.now() - starttotal
xbmc.executebuiltin('InhibitIdleShutdown(false)')
utils.setScreensaver(value=screensaver)
window('emby_dbScan', clear=True)
window('emby_initialScan', clear=True)
xbmcgui.Dialog().notification(
heading=lang(29999),
message="%s %s %s" %
(message, lang(33025), str(elapsedtotal).split('.')[0]),
icon="special://home/addons/plugin.video.emby/icon.png",
sound=False)
return True
def refreshViews(self):
with database.DatabaseConn('emby') as conn_emby, database.DatabaseConn() as conn_video:
with closing(conn_emby.cursor()) as cursor_emby, closing(conn_video.cursor()) as cursor_video:
# Compare views, assign correct tags to items
views.Views(cursor_emby, cursor_video).maintain()
def movies(self, embycursor, kodicursor, pdialog):
# Get movies from emby
emby_db = embydb.Embydb_Functions(embycursor)
movies = Movies(embycursor, kodicursor, pdialog)
views = emby_db.getView_byType('movies')
views += emby_db.getView_byType('mixed')
log.info("Media folders: %s" % views)
##### PROCESS MOVIES #####
for view in views:
log.info("Processing: %s", view)
view_name = view['name']
# Get items per view
if pdialog:
pdialog.update(
heading=lang(29999),
message="%s %s..." % (lang(33017), view_name))
all_movies = self.emby.getMovies(view['id'], dialog=pdialog)
movies.add_all("Movie", all_movies, view)
log.debug("Movies finished.")
##### PROCESS BOXSETS #####
if pdialog:
pdialog.update(heading=lang(29999), message=lang(33018))
boxsets = self.emby.getBoxset(dialog=pdialog)
movies.add_all("BoxSet", boxsets)
log.debug("Boxsets finished.")
return True
def musicvideos(self, embycursor, kodicursor, pdialog):
# Get musicvideos from emby
emby_db = embydb.Embydb_Functions(embycursor)
mvideos = MusicVideos(embycursor, kodicursor, pdialog)
views = emby_db.getView_byType('musicvideos')
log.info("Media folders: %s" % views)
for view in views:
log.info("Processing: %s", view)
# Get items per view
viewId = view['id']
viewName = view['name']
if pdialog:
pdialog.update(
heading=lang(29999),
message="%s %s..." % (lang(33019), viewName))
# Initial or repair sync
all_mvideos = self.emby.getMusicVideos(viewId, dialog=pdialog)
mvideos.add_all("MusicVideo", all_mvideos, view)
else:
log.debug("MusicVideos finished.")
return True
def tvshows(self, embycursor, kodicursor, pdialog):
# Get shows from emby
emby_db = embydb.Embydb_Functions(embycursor)
tvshows = TVShows(embycursor, kodicursor, pdialog)
views = emby_db.getView_byType('tvshows')
views += emby_db.getView_byType('mixed')
log.info("Media folders: %s" % views)
for view in views:
# Get items per view
if pdialog:
pdialog.update(
heading=lang(29999),
message="%s %s..." % (lang(33020), view['name']))
all_tvshows = self.emby.getShows(view['id'], dialog=pdialog)
tvshows.add_all("Series", all_tvshows, view)
else:
log.debug("TVShows finished.")
return True
def music(self, embycursor, kodicursor, pdialog):
# Get music from emby
emby_db = embydb.Embydb_Functions(embycursor)
music = Music(embycursor, kodicursor, pdialog)
views = emby_db.getView_byType('music')
log.info("Media folders: %s", views)
# Add music artists and everything will fall into place
if pdialog:
pdialog.update(heading=lang(29999),
message="%s Music..." % lang(33021))
for view in views:
all_artists = self.emby.getArtists(view['id'], dialog=pdialog)
music.add_all("MusicArtist", all_artists)
log.debug("Finished syncing music")
return True
# Reserved for websocket_client.py and fast start
def triage_items(self, process, items):
processlist = {
'added': self.addedItems,
'update': self.updateItems,
'userdata': self.userdataItems,
'remove': self.removeItems
}
if items:
if process == "userdata":
itemids = []
for item in items:
itemids.append(item['ItemId'])
items = itemids
log.info("Queue %s: %s" % (process, items))
processlist[process].extend(items)
def incrementalSync(self):
update_embydb = False
pDialog = None
# do a view update if needed
if self.refresh_views:
self.refreshViews()
self.refresh_views = False
self.forceLibraryUpdate = True
# do a lib update if any items in list
totalUpdates = len(self.addedItems) + len(self.updateItems) + len(self.userdataItems) + len(self.removeItems)
if totalUpdates > 0:
with database.DatabaseConn('emby') as conn_emby, database.DatabaseConn('video') as conn_video:
with closing(conn_emby.cursor()) as cursor_emby, closing(conn_video.cursor()) as cursor_video:
emby_db = embydb.Embydb_Functions(cursor_emby)
incSyncIndicator = int(settings('incSyncIndicator') or 10)
if incSyncIndicator != -1 and totalUpdates > incSyncIndicator:
# Only present dialog if we are going to process items
pDialog = self.progressDialog('Incremental sync')
log.info("incSyncIndicator=" + str(incSyncIndicator) + " totalUpdates=" + str(totalUpdates))
process = {
'added': self.addedItems,
'update': self.updateItems,
'userdata': self.userdataItems,
'remove': self.removeItems
}
for process_type in ['added', 'update', 'userdata', 'remove']:
if process[process_type] and window('emby_kodiScan') != "true":
listItems = list(process[process_type])
del process[process_type][:] # Reset class list
items_process = itemtypes.Items(cursor_emby, cursor_video)
update = False
# Prepare items according to process process_type
if process_type == "added":
items = self.emby.sortby_mediatype(listItems)
elif process_type in ("userdata", "remove"):
items = emby_db.sortby_mediaType(listItems, unsorted=False)
else:
items = emby_db.sortby_mediaType(listItems)
if items.get('Unsorted'):
sorted_items = self.emby.sortby_mediatype(items['Unsorted'])
doupdate = items_process.itemsbyId(sorted_items, "added", pDialog)
if doupdate:
embyupdate, kodiupdate_video = doupdate
if embyupdate:
update_embydb = True
if kodiupdate_video:
self.forceLibraryUpdate = True
del items['Unsorted']
doupdate = items_process.itemsbyId(items, process_type, pDialog)
if doupdate:
embyupdate, kodiupdate_video = doupdate
if embyupdate:
update_embydb = True
if kodiupdate_video:
self.forceLibraryUpdate = True
# if stuff happened then do some stuff
if update_embydb:
update_embydb = False
log.info("Updating emby database.")
self.saveLastSync()
if self.forceLibraryUpdate:
# Force update the Kodi library
self.forceLibraryUpdate = False
log.info("Updating video library.")
window('emby_kodiScan', value="true")
xbmc.executebuiltin('UpdateLibrary(video)')
if pDialog:
pDialog.close()
def compareDBVersion(self, current, minimum):
# It returns True is database is up to date. False otherwise.
log.info("current: %s minimum: %s" % (current, minimum))
try:
currMajor, currMinor, currPatch = current.split(".")
minMajor, minMinor, minPatch = minimum.split(".")
except ValueError as error:
raise ValueError("Unable to compare versions: %s, %s" % (current, minimum))
if currMajor > minMajor:
return True
elif currMajor == minMajor and (currMinor > minMinor or
(currMinor == minMinor and currPatch >= minPatch)):
return True
else:
# Database out of date.
return False
def _verify_emby_database(self):
# Create the tables for the emby database
with database.DatabaseConn('emby') as conn:
with closing(conn.cursor()) as cursor:
# emby, view, version
cursor.execute(
"""CREATE TABLE IF NOT EXISTS emby(
emby_id TEXT UNIQUE, media_folder TEXT, emby_type TEXT, media_type TEXT,
kodi_id INTEGER, kodi_fileid INTEGER, kodi_pathid INTEGER, parent_id INTEGER,
checksum INTEGER)""")
cursor.execute(
"""CREATE TABLE IF NOT EXISTS view(
view_id TEXT UNIQUE, view_name TEXT, media_type TEXT, kodi_tagid INTEGER)""")
cursor.execute("CREATE TABLE IF NOT EXISTS version(idVersion TEXT)")
def run(self):
try:
self.run_internal()
except Warning as e:
if "restricted" in e:
pass
elif "401" in e:
pass
except Exception as e:
ga = GoogleAnalytics()
errStrings = ga.formatException()
ga.sendEventData("Exception", errStrings[0], errStrings[1])
window('emby_dbScan', clear=True)
log.exception(e)
xbmcgui.Dialog().ok(
heading=lang(29999),
line1=(
"Library sync thread has exited! "
"You should restart Kodi now. "
"Please report this on the forum."),
line2=(errStrings[0] + " (" + errStrings[1] + ")"))
def run_internal(self):
dialog = xbmcgui.Dialog()
startupComplete = False
log.warn("---===### Starting LibrarySync ###===---")
# Verify database structure, otherwise create it.
self._verify_emby_database()
while not self.monitor.abortRequested():
# In the event the server goes offline
while self.suspend_thread:
# Set in service.py
if self.monitor.waitForAbort(5):
# Abort was requested while waiting. We should exit
break
if (window('emby_dbCheck') != "true" and settings('SyncInstallRunDone') == "true"):
# Verify the validity of the database
log.info("Doing DB Version Check")
with database.DatabaseConn('emby') as conn:
with closing(conn.cursor()) as cursor:
emby_db = embydb.Embydb_Functions(cursor)
currentVersion = emby_db.get_version()
###$ Begin migration $###
if not currentVersion:
currentVersion = emby_db.get_version(settings('dbCreatedWithVersion') or self.clientInfo.get_version())
log.info("Migration of database version completed")
###$ End migration $###
window('emby_version', value=currentVersion)
minVersion = window('emby_minDBVersion')
uptoDate = self.compareDBVersion(currentVersion, minVersion)
if not uptoDate:
log.warn("Database version out of date: %s minimum version required: %s"
% (currentVersion, minVersion))
resp = dialog.yesno(lang(29999), lang(33022))
if not resp:
log.warn("Database version is out of date! USER IGNORED!")
dialog.ok(lang(29999), lang(33023))
else:
database.db_reset()
break
window('emby_dbCheck', value="true")
if not startupComplete:
# Verify the video database can be found
videoDb = database.video_database()
if not xbmcvfs.exists(videoDb):
# Database does not exists
log.error(
"The current Kodi version is incompatible "
"with the Emby for Kodi add-on. Please visit "
"https://github.com/MediaBrowser/Emby.Kodi/wiki "
"to know which Kodi versions are supported.")
dialog.ok(
heading=lang(29999),
line1=lang(33024))
break
# Run start up sync
log.warn("Database version: %s", window('emby_version'))
log.info("SyncDatabase (started)")
startTime = datetime.now()
librarySync = self.startSync()
elapsedTime = datetime.now() - startTime
log.info("SyncDatabase (finished in: %s) %s"
% (str(elapsedTime).split('.')[0], librarySync))
# Add other servers at this point
# TODO: re-add once plugin listing is created
# self.user.load_connect_servers()
# Only try the initial sync once per kodi session regardless
# This will prevent an infinite loop in case something goes wrong.
startupComplete = True
# Process updates
if window('emby_dbScan') != "true" and window('emby_shouldStop') != "true":
self.incrementalSync()
if window('emby_onWake') == "true" and window('emby_online') == "true":
# Kodi is waking up
# Set in kodimonitor.py
window('emby_onWake', clear=True)
if window('emby_syncRunning') != "true":
log.info("SyncDatabase onWake (started)")
librarySync = self.startSync()
log.info("SyncDatabase onWake (finished) %s" % librarySync)
if self.stop_thread:
# Set in service.py
log.debug("Service terminated thread.")
break
if self.monitor.waitForAbort(1):
# Abort was requested while waiting. We should exit
break
log.warn("###===--- LibrarySync Stopped ---===###")
def stopThread(self):
self.stop_thread = True
log.debug("Ending thread...")
def suspendThread(self):
self.suspend_thread = True
log.debug("Pausing thread...")
def resumeThread(self):
self.suspend_thread = False
log.debug("Resuming thread...")
class ManualSync(LibrarySync):
def __init__(self):
LibrarySync.__init__(self)
def sync(self):
return self.fullSync(manualrun=True)
def movies(self, embycursor, kodicursor, pdialog):
return Movies(embycursor, kodicursor, pdialog).compare_all()
def musicvideos(self, embycursor, kodicursor, pdialog):
return MusicVideos(embycursor, kodicursor, pdialog).compare_all()
def tvshows(self, embycursor, kodicursor, pdialog):
return TVShows(embycursor, kodicursor, pdialog).compare_all()
def music(self, embycursor, kodicursor, pdialog):
return Music(embycursor, kodicursor).compare_all()