Replace task_initargs with use of partial()
This commit is contained in:
@@ -53,7 +53,7 @@ def log_listener(queue):
|
||||
traceback.print_exc(file=sys.stderr)
|
||||
|
||||
|
||||
def process_init(queue, userfn, *userargs):
|
||||
def process_init(queue, user_init):
|
||||
"""Initialize a process pool worker"""
|
||||
|
||||
# Ignore SIGINT (our parent process will kill us gracefully)
|
||||
@@ -65,13 +65,13 @@ def process_init(queue, userfn, *userargs):
|
||||
root.handlers = []
|
||||
root.addHandler(h)
|
||||
|
||||
if userfn:
|
||||
userfn(*userargs)
|
||||
if user_init:
|
||||
user_init()
|
||||
|
||||
|
||||
def thread_init(_queue, userfn, *userargs):
|
||||
if userfn:
|
||||
userfn(*userargs)
|
||||
def thread_init(_queue, user_init):
|
||||
if user_init:
|
||||
user_init()
|
||||
|
||||
|
||||
def exec_progress_pool(
|
||||
@@ -80,7 +80,6 @@ def exec_progress_pool(
|
||||
max_workers: int,
|
||||
tqdm_kwargs: dict,
|
||||
task_initializer: Optional[Callable] = None,
|
||||
task_initargs: Optional[tuple] = None,
|
||||
task: Optional[Callable] = None,
|
||||
task_arguments: Optional[Iterable] = None,
|
||||
task_finished: Optional[Callable] = None,
|
||||
@@ -88,9 +87,6 @@ def exec_progress_pool(
|
||||
log_queue = multiprocessing.Queue(-1)
|
||||
listener = threading.Thread(target=log_listener, args=(log_queue,))
|
||||
|
||||
if not task_initargs:
|
||||
task_initargs = tuple()
|
||||
|
||||
if use_threads:
|
||||
pool_class = ThreadPool
|
||||
initializer = thread_init
|
||||
@@ -103,7 +99,7 @@ def exec_progress_pool(
|
||||
pool = pool_class(
|
||||
processes=max_workers,
|
||||
initializer=initializer,
|
||||
initargs=(log_queue, task_initializer, *task_initargs),
|
||||
initargs=(log_queue, task_initializer),
|
||||
)
|
||||
try:
|
||||
results = pool.imap_unordered(task, task_arguments)
|
||||
|
||||
@@ -23,6 +23,7 @@ import signal
|
||||
import sys
|
||||
import threading
|
||||
from collections import namedtuple
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
from tempfile import mkdtemp
|
||||
|
||||
@@ -252,8 +253,7 @@ def exec_concurrent(context):
|
||||
unit_scale=0.5,
|
||||
disable=not context.options.progress_bar,
|
||||
),
|
||||
task_initializer=worker_init,
|
||||
task_initargs=(PIL.Image.MAX_IMAGE_PIXELS,),
|
||||
task_initializer=partial(worker_init, PIL.Image.MAX_IMAGE_PIXELS),
|
||||
task=exec_page_sync,
|
||||
task_arguments=context.get_page_contexts(),
|
||||
task_finished=update_page,
|
||||
|
||||
@@ -657,7 +657,6 @@ def _pdf_pageinfo_concurrent(pdf, infile, pages_xml, detailed_analysis, progbar)
|
||||
total=len(pdf.pages), desc="Scan", unit='page', disable=not progbar
|
||||
),
|
||||
task_initializer=None,
|
||||
task_initargs=None,
|
||||
task=_pdf_pageinfo_sync,
|
||||
task_arguments=contexts,
|
||||
task_finished=update_pageinfo,
|
||||
|
||||
Reference in New Issue
Block a user