From 5ee45411c92536f8aab544d9bd0fa53b740bbd0e Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Thu, 13 Jun 2019 01:02:07 -0700 Subject: [PATCH] Decide on OMP_THREAD_LIMIT more intelligently --- src/ocrmypdf/_sync.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/ocrmypdf/_sync.py b/src/ocrmypdf/_sync.py index a9f22294..c61391e7 100644 --- a/src/ocrmypdf/_sync.py +++ b/src/ocrmypdf/_sync.py @@ -224,7 +224,21 @@ def exec_concurrent(context): # Run exec_page_sync on every page context max_workers = min(len(context.pdfinfo), context.options.jobs) if max_workers > 1: - context.log.info("Start processing %d pages concurrent" % max_workers) + context.log.info("Start processing %d pages concurrent", max_workers) + + # Tesseract 4.0 is multithreaded, and we also run multiple workers. We want to + # avoid the situation where we end up trying to run NxN jobs on N CPU cores, + # as that gives poor performance. Performance testing shows we're better off + # parallelizing ocrmypdf and forcing Tesseract to be single threaded, which we + # get by setting the envvar OMP_THREAD_LIMIT to 1. But if the page count of the + # input file is small, then we allow Tesseract to use threads, subject to the + # constraint: (ocrmypdf workers) * (tesseract threads) <= max_workers + tess_threads = min(1, context.options.jobs // max_workers) + if context.options.tesseract_env is None: + context.options.tesseract_env = os.environ.copy() + context.options.tesseract_env.setdefault('OMP_THREAD_LIMIT', str(tess_threads)) + if tess_threads > 1: + context.log.info("Using Tesseract OpenMP thread limit %d", tess_threads) if context.options.use_threads: from multiprocessing.dummy import Pool @@ -300,12 +314,6 @@ def run_pipeline(options, api=False): if not options.jobs: options.jobs = available_cpu_count() - # Performance is improved by setting Tesseract to single threaded. In tests - # this gives better throughput than letting a smaller number of Tesseract - # jobs run multithreaded. Same story for pngquant. Tess <4 ignores this - # variable, but harmless to set if ignored. - os.environ.setdefault('OMP_THREAD_LIMIT', '1') - work_folder = mkdtemp(prefix="com.github.ocrmypdf.") try: check_requested_output_file(options)