tqdm_kwargs to progress_kwargs

This commit is contained in:
James R. Barlow
2023-10-24 00:54:31 -07:00
parent a4059762e6
commit 6127f7abd6
8 changed files with 19 additions and 19 deletions
+6 -6
View File
@@ -46,7 +46,7 @@ class Executor(ABC):
*,
use_threads: bool,
max_workers: int,
tqdm_kwargs: dict,
progress_kwargs: dict,
worker_initializer: Callable | None = None,
task: Callable | None = None,
task_arguments: Iterable | None = None,
@@ -60,7 +60,7 @@ class Executor(ABC):
heavily, and parallelizing it with threads is not expected to be
performant).
max_workers: The maximum number of workers that should be run.
tqdm_kwargs: Arguments to set up the progress bar.
progress_kwargs: Arguments to set up the progress bar.
worker_initializer: Called when a worker is initialized, in the worker's
execution context. If the child workers are processes, it must be
possible to marshall/pickle the worker initializer.
@@ -86,7 +86,7 @@ class Executor(ABC):
self._execute(
use_threads=use_threads,
max_workers=max_workers,
tqdm_kwargs=tqdm_kwargs,
progress_kwargs=progress_kwargs,
worker_initializer=worker_initializer,
task=task,
task_arguments=task_arguments,
@@ -99,7 +99,7 @@ class Executor(ABC):
*,
use_threads: bool,
max_workers: int,
tqdm_kwargs: dict,
progress_kwargs: dict,
worker_initializer: Callable,
task: Callable,
task_arguments: Iterable,
@@ -125,13 +125,13 @@ class SerialExecutor(Executor):
*,
use_threads: bool,
max_workers: int,
tqdm_kwargs: dict,
progress_kwargs: dict,
worker_initializer: Callable,
task: Callable,
task_arguments: Iterable,
task_finished: Callable,
): # pylint: disable=unused-argument
with self.pbar_class(**tqdm_kwargs) as pbar:
with self.pbar_class(**progress_kwargs) as pbar:
for args in task_arguments:
result = task(args)
task_finished(result, pbar)
+1 -1
View File
@@ -78,7 +78,7 @@ def exec_hocr_to_ocr_pdf(context: PdfContext, executor: Executor) -> Sequence[st
executor(
use_threads=options.use_threads,
max_workers=max_workers,
tqdm_kwargs=dict(
progress_kwargs=dict(
total=(2 * len(context.pdfinfo)),
desc='Grafting hOCR to PDF',
unit='page',
+1 -1
View File
@@ -119,7 +119,7 @@ def exec_concurrent(context: PdfContext, executor: Executor) -> Sequence[str]:
executor(
use_threads=options.use_threads,
max_workers=max_workers,
tqdm_kwargs=dict(
progress_kwargs=dict(
total=(2 * len(context.pdfinfo)),
desc='OCR' if options.tesseract_timeout > 0 else 'Image processing',
unit='page',
+1 -1
View File
@@ -73,7 +73,7 @@ def exec_pdf_to_hocr(context: PdfContext, executor: Executor) -> None:
executor(
use_threads=options.use_threads,
max_workers=max_workers,
tqdm_kwargs=dict(
progress_kwargs=dict(
total=(2 * len(context.pdfinfo)),
desc='hOCR',
unit='page',
+2 -2
View File
@@ -101,7 +101,7 @@ class StandardExecutor(Executor):
*,
use_threads: bool,
max_workers: int,
tqdm_kwargs: dict,
progress_kwargs: dict,
worker_initializer: Callable,
task: Callable,
task_arguments: Iterable,
@@ -127,7 +127,7 @@ class StandardExecutor(Executor):
listener = threading.Thread(target=log_listener, args=(log_queue,))
listener.start()
with self.pbar_class(**tqdm_kwargs) as pbar, executor_class(
with self.pbar_class(**progress_kwargs) as pbar, executor_class(
max_workers=max_workers,
initializer=initializer,
initargs=(log_queue, worker_initializer, logging.getLogger("").level),
+3 -3
View File
@@ -114,14 +114,14 @@ class LambdaExecutor(Executor):
*,
use_threads: bool,
max_workers: int,
tqdm_kwargs: dict,
progress_kwargs: dict,
worker_initializer: Callable,
task: Callable,
task_arguments: Iterable,
task_finished: Callable,
):
if use_threads and max_workers == 1:
with self.pbar_class(**tqdm_kwargs) as pbar:
with self.pbar_class(**progress_kwargs) as pbar:
for args in task_arguments:
result = task(args)
task_finished(result, pbar)
@@ -157,7 +157,7 @@ class LambdaExecutor(Executor):
for process in processes:
process.start()
with self.pbar_class(**tqdm_kwargs) as pbar:
with self.pbar_class(**progress_kwargs) as pbar:
while connections:
for result in wait(connections):
if not isinstance(result, Connection):
+4 -4
View File
@@ -394,7 +394,7 @@ def _produce_jbig2_images(
executor(
use_threads=True,
max_workers=options.jobs,
tqdm_kwargs=dict(
progress_kwargs=dict(
total=len(jbig2_groups),
desc="JBIG2",
unit='item',
@@ -485,7 +485,7 @@ def transcode_jpegs(
executor(
use_threads=True, # Processes are significantly slower at this task
max_workers=options.jobs,
tqdm_kwargs=dict(
progress_kwargs=dict(
desc="Recompressing JPEGs",
total=len(jpegs),
unit='image',
@@ -558,7 +558,7 @@ def deflate_jpegs(pdf: Pdf, root: Path, options, executor: Executor) -> None:
executor(
use_threads=True, # We're sharing the pdf directly, must use threads
max_workers=options.jobs,
tqdm_kwargs=dict(
progress_kwargs=dict(
desc="Deflating JPEGs",
total=len(jpegs),
unit='image',
@@ -641,7 +641,7 @@ def transcode_pngs(
executor(
use_threads=True,
max_workers=options.jobs,
tqdm_kwargs=dict(
progress_kwargs=dict(
desc="PNGs",
total=len(images),
unit='image',
+1 -1
View File
@@ -755,7 +755,7 @@ def _pdf_pageinfo_concurrent(
executor(
use_threads=use_threads,
max_workers=n_workers,
tqdm_kwargs=dict(
progress_kwargs=dict(
total=total, desc="Scanning contents", unit='page', disable=not progbar
),
worker_initializer=partial(