move LIMIT/DTHREADS into function _get_items to allow for on the fly changes without a restart or reboot

This commit is contained in:
macearl 2020-10-19 19:52:06 +02:00
parent 6ca09d542c
commit 5ce1b7d3e6

View file

@ -20,8 +20,6 @@ from helper import LazyLogger
#################################################################################################
LOG = LazyLogger(__name__)
LIMIT = min(int(settings('limitIndex') or 50), 50)
DTHREADS = int(settings('limitThreads') or 3)
#################################################################################################
@ -246,6 +244,9 @@ def _get_items(query, server_id=None):
'RestorePoint': {}
}
limit = min(int(settings('limitIndex') or 50), 50)
dthreads = int(settings('limitThreads') or 3)
url = query['url']
query.setdefault('params', {})
params = query['params']
@ -270,16 +271,16 @@ def _get_items(query, server_id=None):
return params_copy
query_params = [
get_query_params(params, offset, LIMIT)
get_query_params(params, offset, limit)
for offset
in range(params['StartIndex'], items['TotalRecordCount'], LIMIT)
in range(params['StartIndex'], items['TotalRecordCount'], limit)
]
# multiprocessing.dummy.Pool completes all requests in multiple threads but has to
# complete all tasks before allowing any results to be processed. ThreadPoolExecutor
# allows for completed tasks to be processed while other tasks are completed on other
# threads. Dont be a dummy.Pool, be a ThreadPoolExecutor
p = concurrent.futures.ThreadPoolExecutor(DTHREADS)
p = concurrent.futures.ThreadPoolExecutor(dthreads)
results = p.map(lambda params: _get(url, params, server_id=server_id), query_params)