Replace task_initargs with use of partial()

This commit is contained in:
James R. Barlow
2020-04-26 03:49:20 -07:00
parent 86145a8c76
commit 8c381a0227
3 changed files with 9 additions and 14 deletions
+7 -11
View File
@@ -53,7 +53,7 @@ def log_listener(queue):
traceback.print_exc(file=sys.stderr)
def process_init(queue, userfn, *userargs):
def process_init(queue, user_init):
"""Initialize a process pool worker"""
# Ignore SIGINT (our parent process will kill us gracefully)
@@ -65,13 +65,13 @@ def process_init(queue, userfn, *userargs):
root.handlers = []
root.addHandler(h)
if userfn:
userfn(*userargs)
if user_init:
user_init()
def thread_init(_queue, userfn, *userargs):
if userfn:
userfn(*userargs)
def thread_init(_queue, user_init):
if user_init:
user_init()
def exec_progress_pool(
@@ -80,7 +80,6 @@ def exec_progress_pool(
max_workers: int,
tqdm_kwargs: dict,
task_initializer: Optional[Callable] = None,
task_initargs: Optional[tuple] = None,
task: Optional[Callable] = None,
task_arguments: Optional[Iterable] = None,
task_finished: Optional[Callable] = None,
@@ -88,9 +87,6 @@ def exec_progress_pool(
log_queue = multiprocessing.Queue(-1)
listener = threading.Thread(target=log_listener, args=(log_queue,))
if not task_initargs:
task_initargs = tuple()
if use_threads:
pool_class = ThreadPool
initializer = thread_init
@@ -103,7 +99,7 @@ def exec_progress_pool(
pool = pool_class(
processes=max_workers,
initializer=initializer,
initargs=(log_queue, task_initializer, *task_initargs),
initargs=(log_queue, task_initializer),
)
try:
results = pool.imap_unordered(task, task_arguments)
+2 -2
View File
@@ -23,6 +23,7 @@ import signal
import sys
import threading
from collections import namedtuple
from functools import partial
from pathlib import Path
from tempfile import mkdtemp
@@ -252,8 +253,7 @@ def exec_concurrent(context):
unit_scale=0.5,
disable=not context.options.progress_bar,
),
task_initializer=worker_init,
task_initargs=(PIL.Image.MAX_IMAGE_PIXELS,),
task_initializer=partial(worker_init, PIL.Image.MAX_IMAGE_PIXELS),
task=exec_page_sync,
task_arguments=context.get_page_contexts(),
task_finished=update_page,
-1
View File
@@ -657,7 +657,6 @@ def _pdf_pageinfo_concurrent(pdf, infile, pages_xml, detailed_analysis, progbar)
total=len(pdf.pages), desc="Scan", unit='page', disable=not progbar
),
task_initializer=None,
task_initargs=None,
task=_pdf_pageinfo_sync,
task_arguments=contexts,
task_finished=update_pageinfo,