From dc13c6996d767aad56eb795326d24b48bb9d3aaf Mon Sep 17 00:00:00 2001 From: Marcel van der Veldt Date: Wed, 18 Mar 2015 16:47:55 +0100 Subject: [PATCH] added sync settings improved performance by getting all kodi movies at once instead of 1 by 1 movies section = finished, TV section work in progress --- resources/lib/CreateFiles.py | 4 +- resources/lib/LibrarySync.py | 292 ++++++++++++++++------------------- resources/lib/ReadEmbyDB.py | 13 +- resources/lib/ReadKodiDB.py | 45 ++++++ resources/settings.xml | 29 +++- 5 files changed, 216 insertions(+), 167 deletions(-) diff --git a/resources/lib/CreateFiles.py b/resources/lib/CreateFiles.py index 2afe8ba4..11ac55eb 100644 --- a/resources/lib/CreateFiles.py +++ b/resources/lib/CreateFiles.py @@ -101,6 +101,8 @@ class CreateFiles(): SubElement(root, "thumb").text = API().getArtwork(item, "Primary") SubElement(root, "fanart").text = API().getArtwork(item, "Backdrop") + + SubElement(root, "title").text = utils.convertEncoding(item["Name"]) SubElement(root, "originaltitle").text = utils.convertEncoding(item["Name"]) SubElement(root, "sorttitle").text = utils.convertEncoding(item["SortName"]) @@ -168,7 +170,7 @@ class CreateFiles(): if studios != None: for studio in studios: - SubElement(root, "studio").text = utils.convertEncoding(studio) + SubElement(root, "studio").text = utils.convertEncoding(studio).replace("/", "&") if item.get("ProductionLocations") != None: for country in item.get("ProductionLocations"): diff --git a/resources/lib/LibrarySync.py b/resources/lib/LibrarySync.py index b6fb9246..84ecbd15 100644 --- a/resources/lib/LibrarySync.py +++ b/resources/lib/LibrarySync.py @@ -29,11 +29,11 @@ dataPath = os.path.join(addondir,"library") movieLibrary = os.path.join(dataPath,'movies') tvLibrary = os.path.join(dataPath,'tvshows') -sleepVal = 10 +sleepVal = 20 showProgress = True processMovies = True -processTvShows = True +processTvShows = False class LibrarySync(): @@ -43,153 +43,125 @@ class LibrarySync(): WINDOW = xbmcgui.Window( 10000 ) WINDOW.setProperty("librarysync", "busy") pDialog = None + + #set some variable to check if this is the first run + startupDone = False + startupStr = WINDOW.getProperty("startup") + if startupStr == "done": + startupDone = True + + #are we running startup sync or background sync ? + if not startupDone: + syncOption = addon.getSetting("syncSettingStartup") + else: + syncOption = addon.getSetting("syncSettingBackground") + + #what sync method to perform ? + if syncOption == "Full Sync": + self.MoviesSync(True) + if syncOption == "Incremental Sync": + self.MoviesSync(False) + + WINDOW.setProperty("startup", "done") + + + + def MoviesSync(self, fullsync=True): + + WINDOW = xbmcgui.Window( 10000 ) + WINDOW.setProperty("librarysync", "busy") + pDialog = None try: - if(showProgress): + if(addon.getSetting("enableProgressFullSync")): pDialog = xbmcgui.DialogProgressBG() if(pDialog != None): pDialog.create('Sync DB', 'Sync DB') - updateNeeded = False + allEmbyMovieIds = list() + + views = ReadEmbyDB().getCollections("movies") + + for view in views: + + updateNeeded = False + + #process new movies + allMB3Movies = ReadEmbyDB().getMovies(view.get('id'), True, fullsync) + allKodiIds = set(ReadKodiDB().getKodiMoviesIds(True)) - #process full movies sync - if processMovies: - allMovies = list() - - views = ReadEmbyDB().getCollections("movies") - for view in views: - - movieData = ReadEmbyDB().getMovies(view.get('id'), True) - - if(self.ShouldStop()): - return True - - if(movieData == None): - return False - - if(pDialog != None): - pDialog.update(0, "Sync DB : Processing " + view.get('title')) - total = len(movieData) + 1 - count = 1 - - for item in movieData: - xbmc.sleep(sleepVal) - if not item.get('IsFolder'): - kodiItem = ReadKodiDB().getKodiMovie(item["Id"]) - allMovies.append(item["Id"]) - progMessage = "Processing" - item['Tag'] = [] - item['Tag'].append(view.get('title')) - if kodiItem == None: - WriteKodiDB().addMovieToKodiLibrary(item) - updateNeeded = True - progMessage = "Adding" - else: - WriteKodiDB().updateMovieToKodiLibrary(item, kodiItem) - progMessage = "Updating" - - if(self.ShouldStop()): - return True - - # update progress bar - if(pDialog != None): - percentage = int(((float(count) / float(total)) * 100)) - pDialog.update(percentage, message=progMessage + " Movie: " + str(count)) - count += 1 - - #process full tv shows sync - if processTvShows: - allTVShows = list() - allEpisodes = list() - tvShowData = ReadEmbyDB().getTVShows(True) - if(self.ShouldStop()): return True - - if (tvShowData == None): - return - + + if(allMB3Movies == None): + return False + if(pDialog != None): - pDialog.update(0, "Sync DB : Processing TV Shows") - total = len(tvShowData) + 1 - count = 0 + pDialog.update(0, "Sync DB : Processing " + view.get('title')) + total = len(allMB3Movies) + 1 + count = 1 + + for item in allMB3Movies: - for item in tvShowData: - xbmc.sleep(sleepVal) - if item.get('IsFolder'): - kodiItem = ReadKodiDB().getKodiTVShow(item["Id"]) - allTVShows.append(item["Id"]) - progMessage = "Processing" - if kodiItem == None: - WriteKodiDB().addTVShowToKodiLibrary(item) + if not item.get('IsFolder'): + allEmbyMovieIds.append(item["Id"]) + progMessage = "Updating movies" + item['Tag'] = [] + item['Tag'].append(view.get('title')) + + if item["Id"] not in allKodiIds: + xbmc.sleep(sleepVal) + WriteKodiDB().addMovieToKodiLibrary(item) updateNeeded = True progMessage = "Adding" - else: - WriteKodiDB().updateTVShowToKodiLibrary(item, kodiItem) - progMessage = "Updating" - + if(self.ShouldStop()): return True - - # update progress bar - if(pDialog != None): - percentage = int(((float(count) / float(total)) * 100)) - pDialog.update(percentage, message=progMessage + " Tv Show: " + str(count)) - count += 1 - - - #process episodes (will only be possible when tv show is scanned to library) - #TODO --> maybe pull full info only when needed ? - allEpisodes = list() - - for tvshow in allTVShows: - - episodeData = ReadEmbyDB().getEpisodes(tvshow,True) - kodiEpisodes = ReadKodiDB().getKodiEpisodes(tvshow) - - if(self.ShouldStop()): - return True - - if(pDialog != None): - pDialog.update(0, "Sync DB : Processing Episodes") - total = len(episodeData) + 1 - count = 0 - - #we have to compare the lists somehow - for item in episodeData: - xbmc.sleep(sleepVal) - comparestring1 = str(item.get("ParentIndexNumber")) + "-" + str(item.get("IndexNumber")) - matchFound = False - progMessage = "Processing" - if kodiEpisodes != None: - for KodiItem in kodiEpisodes: - - allEpisodes.append(KodiItem["episodeid"]) - comparestring2 = str(KodiItem["season"]) + "-" + str(KodiItem["episode"]) - if comparestring1 == comparestring2: - #match found - update episode - WriteKodiDB().updateEpisodeToKodiLibrary(item,KodiItem,tvshow) - matchFound = True - progMessage = "Updating" - - if not matchFound: - #no match so we have to create it - print "episode not found...creating it: " - WriteKodiDB().addEpisodeToKodiLibrary(item,tvshow) - updateNeeded = True - progMessage = "Adding" - + if(self.ShouldStop()): - return True - + return True + # update progress bar if(pDialog != None): percentage = int(((float(count) / float(total)) * 100)) - pDialog.update(percentage, message=progMessage + " Episode: " + str(count)) - count += 1 + pDialog.update(percentage, message=progMessage + " Movie: " + str(count)) + count += 1 + + #initiate library update and wait for finish before processing any updates + if updateNeeded: + self.doKodiLibraryUpdate() + + if(self.ShouldStop()): + return True + + #process updates + allKodiMovies = ReadKodiDB().getKodiMovies(True) + for item in allMB3Movies: - + if not item.get('IsFolder'): + progMessage = "Updating movies" + item['Tag'] = [] + item['Tag'].append(view.get('title')) + + progMessage = "Updating" + + for kodimovie in allKodiMovies: + if item["Id"] in kodimovie["file"]: + WriteKodiDB().updateMovieToKodiLibrary(item,kodimovie) + break + + if(self.ShouldStop()): + return True + + if(self.ShouldStop()): + return True + + # update progress bar + if(pDialog != None): + percentage = int(((float(count) / float(total)) * 100)) + pDialog.update(percentage, message=progMessage + " Movie: " + str(count)) + count += 1 if(pDialog != None): pDialog.update(0, message="Removing Deleted Items") @@ -199,36 +171,26 @@ class LibrarySync(): cleanNeeded = False - # process deletes for movies - if processMovies: - allLocaldirs, filesMovies = xbmcvfs.listdir(movieLibrary) - allMB3Movies = set(allMovies) - for dir in allLocaldirs: - if not dir in allMB3Movies: + # process any deletes only at fullsync + if fullsync: + allKodiIds = ReadKodiDB().getKodiMoviesIds(True) + allEmbyMovieIds = set(allEmbyMovieIds) + for kodiId in allKodiIds: + if not kodiId in allEmbyMovieIds: + xbmc.sleep(sleepVal) + print "delete needed for: " + kodiId WriteKodiDB().deleteMovieFromKodiLibrary(dir) - cleanneeded = True - - if(self.ShouldStop()): - return True + cleanNeeded = True - # process deletes for episodes - if processTvShows: - # TODO --> process deletes for episodes !!! - allLocaldirs, filesTVShows = xbmcvfs.listdir(tvLibrary) - allMB3TVShows = set(allTVShows) - for dir in allLocaldirs: - if not dir in allMB3TVShows: - WriteKodiDB().deleteTVShowFromKodiLibrary(dir) - cleanneeded = True + if(self.ShouldStop()): + return True + + #initiate library clean and wait for finish before processing any updates + if cleanNeeded: + doKodiLibraryUpdate(True) if(self.ShouldStop()): return True - - if cleanNeeded: - WINDOW.setProperty("cleanNeeded", "true") - - if updateNeeded: - WINDOW.setProperty("updateNeeded", "true") finally: WINDOW.clearProperty("librarysync") @@ -236,7 +198,19 @@ class LibrarySync(): pDialog.close() return True - + + def doKodiLibraryUpdate(self,clean=False): + #initiate library update and wait for finish before processing any updates + if clean: + xbmc.executebuiltin("CleanLibrary(video)") + else: + xbmc.executebuiltin("UpdateLibrary(video)") + xbmc.sleep(1000) + while (xbmc.getCondVisibility("Library.IsScanningVideo")): + if(self.ShouldStop()): + return True + xbmc.sleep(250) + def updatePlayCounts(self): #update all playcounts from MB3 to Kodi library @@ -245,7 +219,7 @@ class LibrarySync(): pDialog = None try: - if(showProgress): + if(addon.getSetting("enableProgressPlayCountSync")): pDialog = xbmcgui.DialogProgressBG() if(pDialog != None): pDialog.create('Sync PlayCounts', 'Sync PlayCounts') @@ -254,20 +228,20 @@ class LibrarySync(): if processMovies: views = ReadEmbyDB().getCollections("movies") for view in views: - movieData = ReadEmbyDB().getMovies(view.get('id'),False) + allMB3Movies = ReadEmbyDB().getMovies(view.get('id'),False) if(self.ShouldStop()): return True - if(movieData == None): + if(allMB3Movies == None): return False if(pDialog != None): pDialog.update(0, "Sync PlayCounts: Processing Movies") - totalCount = len(movieData) + 1 + totalCount = len(allMB3Movies) + 1 count = 1 - for item in movieData: + for item in allMB3Movies: xbmc.sleep(sleepVal) if not item.get('IsFolder'): kodiItem = ReadKodiDB().getKodiMovie(item["Id"]) diff --git a/resources/lib/ReadEmbyDB.py b/resources/lib/ReadEmbyDB.py index d6b46555..6a4eb6cf 100644 --- a/resources/lib/ReadEmbyDB.py +++ b/resources/lib/ReadEmbyDB.py @@ -12,7 +12,7 @@ from DownloadUtils import DownloadUtils addon = xbmcaddon.Addon(id='plugin.video.mb3sync') class ReadEmbyDB(): - def getMovies(self, id, fullinfo = False): + def getMovies(self, id, fullinfo = False, fullSync = True): result = None addon = xbmcaddon.Addon(id='plugin.video.mb3sync') @@ -21,12 +21,17 @@ class ReadEmbyDB(): server = host + ":" + port downloadUtils = DownloadUtils() - userid = downloadUtils.getUserId() + userid = downloadUtils.getUserId() + + if not fullSync: + sortstring = "?Limit=20&SortBy=DateCreated" + else: + sortstring = "&SortBy=SortName" if fullinfo: - url = server + '/mediabrowser/Users/' + userid + '/items?ParentId=' + id + '&SortBy=SortName&Fields=Path,Genres,SortName,Studios,Writer,ProductionYear,Taglines,CommunityRating,OfficialRating,CumulativeRunTimeTicks,Metascore,AirTime,DateCreated,MediaStreams,People,Overview&Recursive=true&SortOrder=Ascending&IncludeItemTypes=Movie&CollapseBoxSetItems=false&format=json&ImageTypeLimit=1' + url = server + '/mediabrowser/Users/' + userid + '/items?ParentId=' + id + sortstring + '&Fields=Path,Genres,SortName,Studios,Writer,ProductionYear,Taglines,CommunityRating,OfficialRating,CumulativeRunTimeTicks,Metascore,AirTime,DateCreated,MediaStreams,People,Overview&Recursive=true&SortOrder=Descending&IncludeItemTypes=Movie&CollapseBoxSetItems=false&format=json&ImageTypeLimit=1' else: - url = server + '/mediabrowser/Users/' + userid + '/items?ParentId=' + id + '&SortBy=SortName&Fields=CumulativeRunTimeTicks&Recursive=true&SortOrder=Ascending&IncludeItemTypes=Movie&CollapseBoxSetItems=false&format=json&ImageTypeLimit=1' + url = server + '/mediabrowser/Users/' + userid + '/items?ParentId=' + id + sortstring + '&Fields=CumulativeRunTimeTicks&Recursive=true&SortOrder=Descending&IncludeItemTypes=Movie&CollapseBoxSetItems=false&format=json&ImageTypeLimit=1' jsonData = downloadUtils.downloadUrl(url, suppress=True, popup=0) if jsonData != None and jsonData != "": diff --git a/resources/lib/ReadKodiDB.py b/resources/lib/ReadKodiDB.py index f717b0a7..f46720c0 100644 --- a/resources/lib/ReadKodiDB.py +++ b/resources/lib/ReadKodiDB.py @@ -7,9 +7,17 @@ import xbmc import xbmcgui import xbmcaddon import json +import os + +addon = xbmcaddon.Addon(id='plugin.video.mb3sync') +addondir = xbmc.translatePath(addon.getAddonInfo('profile')) +dataPath = os.path.join(addondir,"library") +movieLibrary = os.path.join(dataPath,'movies') +tvLibrary = os.path.join(dataPath,'tvshows') class ReadKodiDB(): def getKodiMovie(self, id): + #returns a single movie from Kodi db selected on MB item ID json_response = xbmc.executeJSONRPC('{"jsonrpc": "2.0", "method": "VideoLibrary.GetMovies", "params": { "filter": {"operator": "contains", "field": "path", "value": "' + id + '"}, "properties" : ["art", "rating", "thumbnail", "resume", "runtime", "year", "genre", "cast", "trailer", "country", "studio", "set", "imdbnumber", "mpaa", "tagline", "plotoutline","plot", "sorttitle", "director", "writer", "playcount", "tag", "file"], "sort": { "order": "ascending", "method": "label", "ignorearticle": true } }, "id": "libMovies"}') jsonobject = json.loads(json_response.decode('utf-8','replace')) movie = None @@ -22,6 +30,43 @@ class ReadKodiDB(): return movie + def getKodiMovies(self,fullInfo = False): + #returns all movies in Kodi db inserted by MB + if fullInfo: + json_response = xbmc.executeJSONRPC('{"jsonrpc": "2.0", "method": "VideoLibrary.GetMovies", "params": { "filter": {"operator": "contains", "field": "path", "value": "plugin.video.mb3sync"}, "properties" : ["art", "rating", "thumbnail", "resume", "runtime", "year", "genre", "cast", "trailer", "country", "studio", "set", "imdbnumber", "mpaa", "tagline", "plotoutline","plot", "sorttitle", "director", "writer", "playcount", "tag", "file"] }, "id": "libMovies"}') + else: + json_response = xbmc.executeJSONRPC('{"jsonrpc": "2.0", "method": "VideoLibrary.GetMovies", "params": { "filter": {"operator": "contains", "field": "path", "value": "plugin.video.mb3sync"}, "properties" : ["resume", "playcount", "file"] }, "id": "libMovies"}') + jsonobject = json.loads(json_response.decode('utf-8','replace')) + movies = None + + if(jsonobject.has_key('result')): + result = jsonobject['result'] + if(result.has_key('movies')): + movies = result['movies'] + + return movies + + def getKodiMoviesIds(self,returnMB3Ids = False): + # returns a list of movieIds or MB3 Id's from all movies currently in the Kodi library + allKodiMovies = self.getKodiMovies(False) + allKodiMovieIds = list() + + if allKodiMovies != None: + for kodimovie in allKodiMovies: + if returnMB3Ids: + filepath = kodimovie["file"] + filepath = filepath.replace(movieLibrary + os.sep, "") + filepath = filepath.replace(".strm", "") + filepath = filepath.split(os.sep)[0] + id = filepath + else: + id = str(kodimovie["movieid"]) + allKodiMovieIds.append(id) + + return allKodiMovieIds + + + def getKodiTVShow(self, id): json_response = xbmc.executeJSONRPC('{"jsonrpc": "2.0", "method": "VideoLibrary.GetTVShows", "params": { "filter": {"operator": "contains", "field": "path", "value": "' + id + '"}, "properties": ["art", "genre", "plot", "mpaa", "cast", "studio", "sorttitle", "title", "originaltitle", "imdbnumber", "year", "premiered", "rating", "thumbnail", "playcount", "file", "fanart"], "sort": { "order": "ascending", "method": "label", "ignorearticle": true } }, "id": "libTvShows"}') jsonobject = json.loads(json_response.decode('utf-8','replace')) diff --git a/resources/settings.xml b/resources/settings.xml index 9423ff3b..8eafd65c 100644 --- a/resources/settings.xml +++ b/resources/settings.xml @@ -1,5 +1,24 @@ + + + + + + + + + + + + + + + + + + + @@ -8,10 +27,14 @@ + + - - - + + + + + \ No newline at end of file