Refactor concurrency so that it is pluggable

However, this may not be the best idea because it involves global
state that could be overridden by a parallel call to ocrmypdf.ocr.
This commit is contained in:
James R. Barlow
2021-01-24 23:46:29 -08:00
parent 34e564cd7d
commit 26b4d9bb4b
6 changed files with 242 additions and 135 deletions
+42 -1
View File
@@ -9,7 +9,7 @@ from abc import ABC, abstractmethod, abstractstaticmethod
from argparse import ArgumentParser, Namespace
from collections import namedtuple
from pathlib import Path
from typing import TYPE_CHECKING, AbstractSet, List, Optional
from typing import TYPE_CHECKING, AbstractSet, Callable, Iterable, List, Optional
import pluggy
@@ -62,6 +62,47 @@ def check_options(options: Namespace) -> None:
"""
class ParallelExecutor(ABC):
@abstractstaticmethod
def __call__(
use_threads: bool,
max_workers: int,
tqdm_kwargs: dict,
worker_initializer: Callable,
task: Callable,
task_finished: Callable,
task_arguments: Optional[Iterable] = None,
):
"""
Args:
use_threads: If False, the workload is the sort that will benefit from
running in a multiprocessing context (for example, it uses Python
heavily, and parallelizing it with threads is not expected to be
performant).
max_workers: The maximum number of workers that should be run.
tdqm_kwargs: Arguments to set up the progress bar.
worker_initializer: Called when the worker is initialized, in the worker's
execution context. Must be possible to marshall to the worker.
task: Called when the worker starts a new task, in the worker's execution
context. Must be possible to marshallable to the worker.
task_finished: Called when a worker finishes a task, in the parent's
context.
task_arguments: An iterable that generates a group of parameters for each
task. This runs in the parent's context, but the parameters must be
marshallable to the worker.
"""
@hookspec(firstresult=True)
def get_parallel_executor() -> Callable:
"""Called to perform parallel execution
This may be used to replace OCRmyPDF's default parallel execution system
with a third party alternative. For example, you could make OCRmyPDF run in a
distributed environment.
"""
@hookspec
def validate(pdfinfo: 'PdfInfo', options: Namespace) -> None:
"""Called to give a plugin an opportunity to review *options* and *pdfinfo*.