Temporarily disable multiprocessing while scanning

Seems to have some sporadic consistency problems.

docs/api.rst

@@ -51,6 +51,14 @@ Forking a child process to call ``ocrmypdf.ocr()`` is suggested. That
 way your application will survive and remain interactive even if
 OCRmyPDF does not.
 
+.. warning::
+
+    On Windows, the script that calls ``ocrmypdf.ocr()`` must be protected
+    by an "ifmain" guard (``if __name__ == '__main__'``) or you must use
+    ``ocrmypdf.ocr(...use_threads=True)``. If you do not take at least one
+    of these steps, Windows fork semantics will prevent OCRmyPDF from working
+    correct.
+
 Logging
 -------
 

docs/batch.rst

@@ -202,6 +202,13 @@ Alternatives
 -  `Watchman <https://facebook.github.io/watchman/>`__ is a more
    powerful alternative to ``watchmedo``.
 
+AWS Lambda is not viable
+------------------------
+
+AWS Lambda and its equivalents have low limits on execution time and payload
+size, relative to OCRmyPDF's needs. As of this writing, the request/response
+payload for AWS Lambda was 6 MB, which means many PDFs will not fit.
+
 macOS Automator
 ===============
 

docs/installation.rst

@@ -137,11 +137,12 @@ Installing the latest version on Ubuntu 18.04 LTS
 -------------------------------------------------
 
 Ubuntu 18.04 includes ocrmypdf 6.1.2 - you can install that with ``apt``, but
-it is quite old now. To install a more recent version, first install several
+it is quite old now. To install a more recent version, uninstall the old version
-system dependencies:
+of ocrmypdf, and install the following dependencies:
 
 .. code-block:: bash
 
+    sudo apt-get -y remove ocrmypdf
     sudo apt-get -y update
     sudo apt-get -y install \
         ghostscript \

misc/watcher.py

@@ -25,6 +25,7 @@ from pathlib import Path
 import pikepdf
 from watchdog.events import PatternMatchingEventHandler
 from watchdog.observers import Observer
+from watchdog.observers.polling import PollingObserver
 
 import ocrmypdf
 
@@ -37,7 +38,8 @@ ON_SUCCESS_DELETE = bool(os.getenv('OCR_ON_SUCCESS_DELETE', False))
 DESKEW = bool(os.getenv('OCR_DESKEW', False))
 OCR_JSON_SETTINGS = json.loads(os.getenv('OCR_JSON_SETTINGS', '{}'))
 POLL_NEW_FILE_SECONDS = os.getenv('OCR_POLL_NEW_FILE_SECONDS', 1)
-LOGLEVEL = os.environ.get('OCR_LOGLEVEL', 'INFO').upper()
+USE_POLLING = bool(os.getenv('OCR_USE_POLLING', False))
+LOGLEVEL = os.getenv('OCR_LOGLEVEL', 'INFO').upper()
 PATTERNS = ['*.pdf']
 
 log = logging.getLogger('ocrmypdf-watcher')
@@ -112,6 +114,7 @@ def main():
     ocrmypdf.configure_logging(
         verbosity=ocrmypdf.Verbosity.default, manage_root_logger=True
     )
+    log.setLevel(LOGLEVEL)
     log.info(
         f"Starting OCRmyPDF watcher with config:\n"
         f"Input Directory: {INPUT_DIRECTORY}\n"
@@ -126,6 +129,7 @@ def main():
         f"DESKEW: {DESKEW}\n"
         f"ARGS: {OCR_JSON_SETTINGS}\n"
         f"POLL_NEW_FILE_SECONDS: {POLL_NEW_FILE_SECONDS}\n"
+        f"USE_POLLING: {USE_POLLING}\n"
         f"LOGLEVEL: {LOGLEVEL}\n"
     )
 
@@ -134,6 +138,9 @@ def main():
         sys.exit(1)
 
     handler = HandleObserverEvent(patterns=PATTERNS)
+    if USE_POLLING:
+        observer = PollingObserver()
+    else:
         observer = Observer()
     observer.schedule(handler, INPUT_DIRECTORY, recursive=True)
     observer.start()

src/ocrmypdf/_concurrent.py

@@ -0,0 +1,136 @@
+# © 2020 James R. Barlow: github.com/jbarlow83
+#
+# This file is part of OCRmyPDF.
+#
+# OCRmyPDF is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# OCRmyPDF is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+import logging.handlers
+import multiprocessing
+import os
+import signal
+import sys
+import threading
+from multiprocessing import Pool as ProcessPool
+from multiprocessing.dummy import Pool as ThreadPool
+from typing import Callable, Iterable, Optional
+
+from tqdm import tqdm
+
+
+def log_listener(queue):
+    """Listen to the worker processes and forward the messages to logging
+
+    For simplicity this is a thread rather than a process. Only one process
+    should actually write to sys.stderr or whatever we're using, so if this is
+    made into a process the main application needs to be directed to it.
+
+    See https://docs.python.org/3/howto/logging-cookbook.html#logging-to-a-single-file-from-multiple-processes
+    """
+
+    while True:
+        try:
+            record = queue.get()
+            if record is None:
+                break
+            logger = logging.getLogger(record.name)
+            logger.handle(record)
+        except Exception:
+            import traceback
+
+            print("Logging problem", file=sys.stderr)
+            traceback.print_exc(file=sys.stderr)
+
+
+def process_init(queue, userfn, *userargs):
+    """Initialize a process pool worker"""
+
+    # Ignore SIGINT (our parent process will kill us gracefully)
+    signal.signal(signal.SIGINT, signal.SIG_IGN)
+
+    # Reconfigure the root logger for this process to send all messages to a queue
+    h = logging.handlers.QueueHandler(queue)
+    root = logging.getLogger()
+    root.handlers = []
+    root.addHandler(h)
+
+    if userfn:
+        userfn(*userargs)
+
+
+def thread_init(_queue, userfn, *userargs):
+    if userfn:
+        userfn(*userargs)
+
+
+def exec_progress_pool(
+    *,
+    use_threads: bool,
+    max_workers: int,
+    tqdm_kwargs: dict,
+    task_initializer: Optional[Callable] = None,
+    task_initargs: Optional[tuple] = None,
+    task: Optional[Callable] = None,
+    task_arguments: Optional[Iterable] = None,
+    task_finished: Optional[Callable] = None,
+):
+    log_queue = multiprocessing.Queue(-1)
+    listener = threading.Thread(target=log_listener, args=(log_queue,))
+
+    if not task_initargs:
+        task_initargs = tuple()
+
+    if use_threads:
+        pool_class = ThreadPool
+        initializer = thread_init
+    else:
+        pool_class = ProcessPool
+        initializer = process_init
+    listener.start()
+
+    with tqdm(**tqdm_kwargs) as pbar:
+        pool = pool_class(
+            processes=max_workers,
+            initializer=initializer,
+            initargs=(log_queue, task_initializer, *task_initargs),
+        )
+        try:
+            results = pool.imap_unordered(task, task_arguments)
+            while True:
+                try:
+                    result = results.next()
+                    task_finished(result, pbar)
+                except StopIteration:
+                    break
+        except KeyboardInterrupt:
+            # Terminate pool so we exit instantly
+            pool.terminate()
+            # Don't try listener.join() here, will deadlock
+            raise
+        except Exception:
+            if not os.environ.get("PYTEST_CURRENT_TEST", ""):
+                # Unless inside pytest, exit immediately because no one wants
+                # to wait for child processes to finalize results that will be
+                # thrown away. Inside pytest, we want child processes to exit
+                # cleanly so that they output an error messages or coverage data
+                # we need from them.
+                pool.terminate()
+            raise
+        finally:
+            # Terminate log listener
+            log_queue.put_nowait(None)
+            pool.close()
+            pool.join()
+
+    listener.join()

src/ocrmypdf/_pipeline.py

@@ -144,11 +144,9 @@ def triage(original_filename, input_file, output_file, options, log):
     return output_file
 
 
-def get_pdfinfo(input_file, detailed_page_analysis=False, progbar=False):
+def get_pdfinfo(input_file, progbar=False):
     try:
-        return PdfInfo(
+        return PdfInfo(input_file, progbar=progbar)
-            input_file, detailed_page_analysis=detailed_page_analysis, progbar=progbar
-        )
     except pikepdf.PasswordError:
         raise EncryptedPdfError()
     except pikepdf.PdfError:

src/ocrmypdf/_sync.py

@@ -21,14 +21,13 @@ import multiprocessing
 import os
 import signal
 import sys
-import threading
 from collections import namedtuple
 from pathlib import Path
 from tempfile import mkdtemp
 
 import PIL
-from tqdm import tqdm
 
+from ._concurrent import exec_progress_pool
 from ._graft import OcrGrafter
 from ._jobcontext import PDFContext, cleanup_working_files, make_logger
 from ._pipeline import (
@@ -178,53 +177,13 @@ def post_process(pdf_file, context):
     return optimize_pdf(pdf_out, context)
 
 
-def worker_init(queue, max_pixels):
+def worker_init(max_pixels):
-    """Initialize a process pool worker"""
-
-    # Ignore SIGINT (our parent process will kill us gracefully)
-    signal.signal(signal.SIGINT, signal.SIG_IGN)
-
-    # Reconfigure the root logger for this process to send all messages to a queue
-    h = logging.handlers.QueueHandler(queue)
-    root = logging.getLogger()
-    root.handlers = []
-    root.addHandler(h)
-
     # In Windows, child process will not inherit our change to this value in
-    # the parent process, so ensure workers get it set
+    # the parent process, so ensure workers get it set. Not needed when running
+    # threaded, but harmless to set again.
     PIL.Image.MAX_IMAGE_PIXELS = max_pixels
 
 
-def worker_thread_init(_queue, max_pixels):
-    # This is probably not needed since threads should all see the same memory,
-    # but done for consistency.
-    PIL.Image.MAX_IMAGE_PIXELS = max_pixels
-
-
-def log_listener(queue):
-    """Listen to the worker processes and forward the messages to logging
-
-    For simplicity this is a thread rather than a process. Only one process
-    should actually write to sys.stderr or whatever we're using, so if this is
-    made into a process the main application needs to be directed to it.
-
-    See https://docs.python.org/3/howto/logging-cookbook.html#logging-to-a-single-file-from-multiple-processes
-    """
-
-    while True:
-        try:
-            record = queue.get()
-            if record is None:
-                break
-            logger = logging.getLogger(record.name)
-            logger.handle(record)
-        except Exception:
-            import traceback
-
-            print("Logging problem", file=sys.stderr)
-            traceback.print_exc(file=sys.stderr)
-
-
 def exec_concurrent(context):
     """Execute the pipeline concurrently"""
 
@@ -252,64 +211,31 @@ def exec_concurrent(context):
     if tess_threads > 1:
         context.log.info("Using Tesseract OpenMP thread limit %d", tess_threads)
 
-    if context.options.use_threads:
-        from multiprocessing.dummy import Pool
-
-        initializer = worker_thread_init
-    else:
-        Pool = multiprocessing.Pool
-        initializer = worker_init
-
     sidecars = [None] * len(context.pdfinfo)
     ocrgraft = OcrGrafter(context)
 
-    log_queue = multiprocessing.Queue(-1)
+    def update_page(result, pbar):
-    listener = threading.Thread(target=log_listener, args=(log_queue,))
+        sidecars[result.pageno] = result.text
-    listener.start()
+        pbar.update()
-    with tqdm(
+        ocrgraft.graft_page(result)
+        pbar.update()
+
+    exec_progress_pool(
+        use_threads=context.options.use_threads,
+        max_workers=max_workers,
+        tqdm_kwargs=dict(
             total=(2 * len(context.pdfinfo)),
             desc='OCR',
             unit='page',
             unit_scale=0.5,
             disable=not context.options.progress_bar,
-    ) as pbar:
+        ),
-        pool = Pool(
+        task_initializer=worker_init,
-            processes=max_workers,
+        task_initargs=(PIL.Image.MAX_IMAGE_PIXELS,),
-            initializer=initializer,
+        task=exec_page_sync,
-            initargs=(log_queue, PIL.Image.MAX_IMAGE_PIXELS),
+        task_arguments=context.get_page_contexts(),
+        task_finished=update_page,
     )
-        try:
-            results = pool.imap_unordered(exec_page_sync, context.get_page_contexts())
-            while True:
-                try:
-                    page_result = results.next()
-                    sidecars[page_result.pageno] = page_result.text
-                    pbar.update()
-                    ocrgraft.graft_page(page_result)
-                    pbar.update()
-                except StopIteration:
-                    break
-        except KeyboardInterrupt:
-            # Terminate pool so we exit instantly
-            pool.terminate()
-            # Don't try listener.join() here, will deadlock
-            raise
-        except Exception:
-            if not os.environ.get("PYTEST_CURRENT_TEST", ""):
-                # Unless inside pytest, exit immediately because no one wants
-                # to wait for child processes to finalize results that will be
-                # thrown away. Inside pytest, we want child processes to exit
-                # cleanly so that they output an error messages or coverage data
-                # we need from them.
-                pool.terminate()
-            raise
-        finally:
-            # Terminate log listener
-            log_queue.put_nowait(None)
-            pool.close()
-            pool.join()
-
-    listener.join()
 
     # Output sidecar text
     if context.options.sidecar:
@@ -381,11 +307,7 @@ def run_pipeline(options, api=False):
         )
 
         # Gather pdfinfo and create context
-        pdfinfo = get_pdfinfo(
+        pdfinfo = get_pdfinfo(origin_pdf, progbar=options.progress_bar)
-            origin_pdf,
-            detailed_page_analysis=options.redo_ocr,
-            progbar=options.progress_bar,
-        )
 
         context = PDFContext(options, work_folder, origin_pdf, pdfinfo)
 

src/ocrmypdf/api.py

@@ -67,7 +67,11 @@ class Verbosity(IntEnum):
     debug_all = 2  #: More detailed debugging from ocrmypdf and dependent modules
 
 
-def configure_logging(verbosity, progress_bar_friendly=True, manage_root_logger=False):
+def configure_logging(
+    verbosity: Verbosity,
+    progress_bar_friendly: bool = True,
+    manage_root_logger: bool = False,
+):
     """Set up logging.
 
     Library users may wish to use this function if they want their log output to be
@@ -128,7 +132,7 @@ def configure_logging(verbosity, progress_bar_friendly=True, manage_root_logger=
     return log
 
 
-def create_options(*, input_file, output_file, **kwargs):
+def create_options(*, input_file: os.PathLike, output_file: os.PathLike, **kwargs):
     cmdline = []
     deferred = []
 

src/ocrmypdf/exec/ghostscript.py

@@ -83,55 +83,6 @@ def _gs_error_reported(stream):
     return re.search(r'error', stream, flags=re.IGNORECASE)
 
 
-def extract_text(input_file, pageno=1):
-    """Use the txtwrite device to get text layout information out
-
-    For details on options of -dTextFormat see
-    https://www.ghostscript.com/doc/current/VectorDevices.htm#TXT
-
-    Format is like
-    <page>
-    <line>
-    <span bbox="left top right bottom" font="..." size="...">
-    <char bbox="...." c="X"/>
-
-    :param pageno: number of page to extract, or all pages if None
-    :return: XML-ish text representation in bytes
-    """
-
-    if pageno is not None:
-        pages = ['-dFirstPage=%i' % pageno, '-dLastPage=%i' % pageno]
-    else:
-        pages = []
-
-    # Note due to bug https://bugs.ghostscript.com/show_bug.cgi?id=701971
-    # Ghostscript <= 9.50 will truncate output unless we write to stdout, so
-    # don't write to a file.
-    args_gs = (
-        [
-            GS,
-            '-dQUIET',
-            '-dSAFER',
-            '-dBATCH',
-            '-dNOPAUSE',
-            '-sDEVICE=txtwrite',
-            '-dTextFormat=0',
-        ]
-        + pages
-        + ['-o', '-', fspath(input_file), "-sstdout=%stderr"]
-    )
-
-    try:
-        p = run(args_gs, stdout=PIPE, stderr=PIPE, check=True)
-    except CalledProcessError as e:
-        raise SubprocessOutputError(
-            'Ghostscript text extraction failed\n%s\n%s'
-            % (input_file, e.stderr.decode(errors='replace'))
-        )
-
-    return p.stdout
-
-
 def rasterize_pdf(
     input_file,
     output_file,

src/ocrmypdf/helpers.py

@@ -28,7 +28,7 @@ from pathlib import Path
 log = logging.getLogger(__name__)
 
 
-def safe_symlink(input_file, soft_link_name, *args, **kwargs):
+def safe_symlink(input_file: os.PathLike, soft_link_name: os.PathLike, *args, **kwargs):
     """
     Helper function: relinks soft symbolic link if necessary
     """
@@ -76,12 +76,12 @@ def is_iterable_notstr(thing):
     return isinstance(thing, Iterable) and not isinstance(thing, str)
 
 
-def monotonic(L):
+def monotonic(L: Iterable):
     """Does list increase monotonically?"""
     return all(b > a for a, b in zip(L, L[1:]))
 
 
-def page_number(input_file):
+def page_number(input_file: os.PathLike):
     """Get one-based page number implied by filename (000002.pdf -> 2)"""
     return int(os.path.basename(os.fspath(input_file))[0:6])
 
@@ -97,7 +97,7 @@ def available_cpu_count():
     return 1
 
 
-def is_file_writable(test_file):
+def is_file_writable(test_file: os.PathLike):
     """Intentionally racy test if target is writable.
 
     We intend to write to the output file if and only if we succeed and

src/ocrmypdf/pdfinfo/ghosttext.py

@@ -1,102 +0,0 @@
-# © 2018 James R. Barlow: github.com/jbarlow83
-#
-# This file is part of OCRmyPDF.
-#
-# OCRmyPDF is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# OCRmyPDF is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
-
-import logging
-import re
-import xml.etree.ElementTree as ET
-
-from ..exec import ghostscript
-
-gslog = logging.getLogger()
-
-# Forgive me for I have sinned
-# I am using regular expressions to parse XML. However the XML in this case,
-# generated by Ghostscript, is self-consistent enough to be parseable.
-regex_remove_char_tags = re.compile(
-    br"""
-    <char\b
-    (?:   [^>]   # anything single character but >
-        | \">\"  # special case: trap ">"
-    )*
-    />           # terminate with '/>'
-""",
-    re.VERBOSE,
-)
-
-
-def page_get_textblocks(infile, pageno, xmltext, height):
-    """Get text boxes out of Ghostscript txtwrite xml"""
-
-    root = xmltext
-    if not hasattr(xmltext, 'findall'):
-        return []
-
-    def blocks():
-        for span in root.findall('.//span'):
-            bbox_str = span.attrib['bbox']
-            font_size = span.attrib['size']
-            pts = [int(pt) for pt in bbox_str.split()]
-            pts[1] = pts[1] - int(float(font_size) + 0.5)
-            bbox_topdown = tuple(pts)
-            bb = bbox_topdown
-            bbox_bottomup = (bb[0], height - bb[3], bb[2], height - bb[1])
-            yield bbox_bottomup
-
-    def joined_blocks():
-        prev = None
-        for bbox in blocks():
-            if prev is None:
-                prev = bbox
-            if bbox[1] == prev[1] and bbox[3] == prev[3]:
-                gap = prev[2] - bbox[0]
-                height = abs(bbox[3] - bbox[1])
-                if gap < height:
-                    # Join boxes
-                    prev = (prev[0], prev[1], bbox[2], bbox[3])
-                    continue
-            # yield previously joined bboxes and start anew
-            yield prev
-            prev = bbox
-        if prev is not None:
-            yield prev
-
-    return [block for block in joined_blocks()]
-
-
-def extract_text_xml(infile, pdf, pageno=None, log=gslog):
-    existing_text = ghostscript.extract_text(infile, pageno=None)
-    existing_text = regex_remove_char_tags.sub(b' ', existing_text)
-
-    try:
-        root = ET.fromstringlist([b'<document>\n', existing_text, b'</document>\n'])
-        page_xml = root.findall('page')
-    except ET.ParseError as e:
-        log.error(
-            "An error occurred while attempting to retrieve existing text in "
-            "the input file. Will attempt to continue assuming that there is "
-            "no existing text in the file. The error was:"
-        )
-        log.error(e)
-        page_xml = [None] * len(pdf.pages)
-
-    page_count_difference = len(pdf.pages) - len(page_xml)
-    if page_count_difference != 0:
-        log.error("The number of pages in the input file is inconsistent.")
-        log.error(f"Expected {len(pdf.pages)}, txtwrite says {len(page_xml)}")
-        if page_count_difference > 0:
-            page_xml.extend([None] * page_count_difference)
-    return page_xml

src/ocrmypdf/pdfinfo/info.py

@@ -17,22 +17,21 @@
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
 import logging
+import os
 import re
 from collections import defaultdict, namedtuple
 from decimal import Decimal
 from enum import Enum
 from math import hypot, isclose
-from os import fspath
+from os import PathLike, fspath
 from pathlib import Path
 from warnings import warn
 
 import pikepdf
 from pikepdf import PdfMatrix
-from tqdm import tqdm
 
+from ocrmypdf._concurrent import exec_progress_pool
 from ocrmypdf.exceptions import EncryptedPdfError
-from ocrmypdf.exec import ghostscript
-from ocrmypdf.pdfinfo import ghosttext
 from ocrmypdf.pdfinfo.layout import get_page_analysis, get_text_boxes
 
 logger = logging.getLogger()
@@ -40,7 +39,7 @@ logger = logging.getLogger()
 Colorspace = Enum('Colorspace', 'gray rgb cmyk lab icc index sep devn pattern jpeg2000')
 
 Encoding = Enum(
-    'Encoding', 'ccitt jpeg jpeg2000 jbig2 asciihex ascii85 lzw flate ' + 'runlength'
+    'Encoding', 'ccitt jpeg jpeg2000 jbig2 asciihex ascii85 lzw flate runlength'
 )
 
 FRIENDLY_COLORSPACE = {
@@ -558,7 +557,7 @@ def simplify_textboxes(miner, textbox_getter):
         yield TextboxInfo(box.bbox, visible, corrupt)
 
 
-def _pdf_get_pageinfo(pdf, pageno: int, infile, xmltext):
+def _pdf_get_pageinfo(pdf, pageno: int, infile: PathLike):
     pageinfo = {}
     pageinfo['pageno'] = pageno
     pageinfo['images'] = []
@@ -568,12 +567,6 @@ def _pdf_get_pageinfo(pdf, pageno: int, infile, xmltext):
     width_pt = mediabox[2] - mediabox[0]
     height_pt = mediabox[3] - mediabox[1]
 
-    if xmltext is not None:
-        bboxes = ghosttext.page_get_textblocks(
-            fspath(infile), pageno, xmltext=xmltext, height=height_pt
-        )
-        pageinfo['bboxes'] = bboxes
-    else:
     pscript5_mode = str(pdf.docinfo.get('/Creator')).startswith('PScript5')
     miner = get_page_analysis(infile, pageno, pscript5_mode)
     pageinfo['textboxes'] = list(simplify_textboxes(miner, get_text_boxes))
@@ -616,27 +609,60 @@ def _pdf_get_pageinfo(pdf, pageno: int, infile, xmltext):
     return pageinfo
 
 
-def _pdf_get_all_pageinfo(infile, detailed_analysis=False, log=None, progbar=False):
+worker_pdf = None
+
+
+def _pdf_pageinfo_sync(args):
+    pageno, infile = args
+    page = PageInfo(worker_pdf, pageno, infile)
+    return page
+
+
+def _pdf_pageinfo_concurrent(pdf, infile, progbar):
+    pages = [None] * len(pdf.pages)
+
+    def update_pageinfo(result, pbar):
+        page = result
+        pages[page.pageno] = page
+        pbar.update()
+
+    contexts = ((n, infile) for n in range(len(pdf.pages)))
+    global worker_pdf
+    worker_pdf = pdf
+
+    if os.name == 'nt' or True:
+        # We can't parallelize on Windows, because Windows cannot fork.
+        # We are trying to fork, then take advantage of the preloaded pikepdf.Pdf
+        # object in memory to save time reloading it, hence the silly global
+        # variable. Hey, it works. Threads are not helpful here because they
+        # will all just fight over the lock. So on Windows just run sequentially.
+        use_threads = True
+        max_workers = 1
+    else:
+        use_threads = False
+        max_workers = min(len(pages), 16)
+
+    exec_progress_pool(
+        use_threads=use_threads,
+        max_workers=max_workers,
+        tqdm_kwargs=dict(
+            total=len(pdf.pages), desc="Scan", unit='page', disable=not progbar
+        ),
+        task_initializer=None,
+        task_initargs=None,
+        task=_pdf_pageinfo_sync,
+        task_arguments=contexts,
+        task_finished=update_pageinfo,
+    )
+    return pages
+
+
+def _pdf_get_all_pageinfo(infile, log=None, progbar=False):
     pdf = pikepdf.open(infile)  # Do not close in this function
     try:
         if pdf.is_encrypted:
             raise EncryptedPdfError()  # Triggered by encryption with empty passwd
-        if detailed_analysis:
+        pages = _pdf_pageinfo_concurrent(pdf, infile, progbar)
-            pages_xml = None
-        else:
-            pages_xml = ghosttext.extract_text_xml(infile, pdf, pageno=None, log=log)
-
-        pages = []
-        for n, _ in tqdm(
-            enumerate(pdf.pages),
-            total=len(pdf.pages),
-            desc="Scan",
-            unit='page',
-            disable=not progbar,
-        ):
-            page_xml = pages_xml[n] if pages_xml else None
-            page = PageInfo(pdf, n, infile, page_xml, detailed_analysis)
-            pages.append(page)
     except Exception:
         pdf.close()
         raise
@@ -645,11 +671,10 @@ def _pdf_get_all_pageinfo(infile, detailed_analysis=False, log=None, progbar=Fal
 
 
 class PageInfo:
-    def __init__(self, pdf, pageno, infile, xmltext, detailed_analysis=False):
+    def __init__(self, pdf, pageno, infile):
         self._pageno = pageno
         self._infile = infile
-        self._pageinfo = _pdf_get_pageinfo(pdf, pageno, infile, xmltext)
+        self._pageinfo = _pdf_get_pageinfo(pdf, pageno, infile)
-        self._detailed_analysis = detailed_analysis
 
     @property
     def pageno(self):
@@ -661,8 +686,6 @@ class PageInfo:
 
     @property
     def has_corrupt_text(self):
-        if not self._detailed_analysis:
-            raise NotImplementedError('Did not do detailed analysis')
         return any(tbox.is_corrupt for tbox in self._pageinfo['textboxes'])
 
     @property
@@ -713,7 +736,7 @@ class PageInfo:
 
         if 'textboxes' not in self._pageinfo:
             if visible is not None and corrupt is not None:
-                raise NotImplementedError('Ghostscript textboxes cannot be classified')
+                raise NotImplementedError('Incomplete information on textboxes')
             return self._pageinfo['bboxes']
 
         return (
@@ -758,13 +781,9 @@ class PageInfo:
 class PdfInfo:
     """Get summary information about a PDF"""
 
-    def __init__(self, infile, detailed_page_analysis=False, log=logger, progbar=False):
+    def __init__(self, infile, log=logger, progbar=False):
         self._infile = infile
-        if ghostscript.version() in ('9.52',):
+        self._pages, pdf = _pdf_get_all_pageinfo(infile, log=log, progbar=progbar)
-            detailed_page_analysis = True  # txtwrite doesn't work in these versions
-        self._pages, pdf = _pdf_get_all_pageinfo(
-            infile, detailed_page_analysis, log=log, progbar=progbar
-        )
         self._needs_rendering = pdf.root.get('/NeedsRendering', False)
         self._has_acroform = False
         if '/AcroForm' in pdf.root:

tests/cache/manifest.jsonl

@@ -69,3 +69,4 @@
 {"tesseract_version": "tesseract 4.1.0  leptonica-1.78.0   libgif 5.1.4 : libjpeg 9c : libpng 1.6.37 : libtiff 4.1.0 : zlib 1.2.11 : libwebp 1.0.3 : libopenjp2 2.3.1  Found AVX2  Found AVX  Found SSE ", "platform": "Darwin-18.7.0-x86_64-i386-64bit", "python": "3.7.5", "argv_slug": "__-l__eng__000002_ocr.png__000002_ocr_tess__pdf__txt", "sourcefile": "resources/3small.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000002_ocr.png", "$TMPDIR/000002_ocr_tess", "pdf", "txt"]}
 {"tesseract_version": "tesseract 4.1.0  leptonica-1.78.0   libgif 5.1.4 : libjpeg 9c : libpng 1.6.37 : libtiff 4.1.0 : zlib 1.2.11 : libwebp 1.0.3 : libopenjp2 2.3.1  Found AVX2  Found AVX  Found SSE ", "platform": "Darwin-18.7.0-x86_64-i386-64bit", "python": "3.7.5", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/3small.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
 {"tesseract_version": "tesseract 4.1.0  leptonica-1.78.0   libgif 5.1.4 : libjpeg 9c : libpng 1.6.37 : libtiff 4.1.0 : zlib 1.2.11 : libwebp 1.0.3 : libopenjp2 2.3.1  Found AVX2  Found AVX  Found SSE ", "platform": "Darwin-18.7.0-x86_64-i386-64bit", "python": "3.7.5", "argv_slug": "__-l__eng__000003_ocr.png__000003_ocr_tess__pdf__txt", "sourcefile": "resources/3small.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000003_ocr.png", "$TMPDIR/000003_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "tesseract 4.1.1  leptonica-1.79.0   libgif 5.2.1 : libjpeg 9d : libpng 1.6.37 : libtiff 4.1.0 : zlib 1.2.11 : libwebp 1.1.0 : libopenjp2 2.3.1  Found AVX2  Found AVX  Found FMA  Found SSE ", "platform": "Darwin-18.7.0-x86_64-i386-64bit", "python": "3.7.7", "argv_slug": "__-l__eng__000002_ocr.png__000002_ocr_hocr__hocr__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "$TMPDIR/000002_ocr.png", "$TMPDIR/000002_ocr_hocr", "hocr", "txt"]}

tests/cache/multipage/__-l__eng__000002_ocr.png__000002_ocr_hocr__hocr__txt/hocr.bin

@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+  <title></title>
+  <meta http-equiv="Content-Type" content="text/html;charset=utf-8"/>
+  <meta name='ocr-system' content='tesseract 4.1.1' />
+  <meta name='ocr-capabilities' content='ocr_page ocr_carea ocr_par ocr_line ocrx_word ocrp_wconf'/>
+ </head>
+ <body>
+  <div class='ocr_page' id='page_1' title='image "/var/folders/2s/7t022mgj0h5cprbq0dtb1ksm0000gn/T/com.github.ocrmypdf.626pjb52/000002_ocr.png"; bbox 0 0 12000 12000; ppageno 0'>
+   <div class='ocr_carea' id='block_1_1' title="bbox 1055 986 5888 1952">
+    <p class='ocr_par' id='par_1_1' lang='eng' title="bbox 1055 986 5888 1952">
+     <span class='ocr_line' id='line_1_1' title="bbox 1056 986 5888 1347; baseline -0.001 -70; x_size 367.5; x_descenders 78.5; x_ascenders 70.5">
+      <span class='ocrx_word' id='word_1_1' title='bbox 1056 991 2984 1277; x_wconf 11'>YOOOxXYOO0O</span>
+      <span class='ocrx_word' id='word_1_2' title='bbox 3132 986 4023 1347; x_wconf 91'>pixels</span>
+      <span class='ocrx_word' id='word_1_3' title='bbox 4160 1001 4447 1277; x_wconf 96'>at</span>
+      <span class='ocrx_word' id='word_1_4' title='bbox 4576 991 5219 1277; x_wconf 93'>GOO</span>
+      <span class='ocrx_word' id='word_1_5' title='bbox 5365 986 5888 1273; x_wconf 92'>DPI</span>
+     </span>
+     <span class='ocr_line' id='line_1_2' title="bbox 1055 1587 3452 1952; baseline 0 -73; x_size 366; x_descenders 78; x_ascenders 70">
+      <span class='ocrx_word' id='word_1_6' title='bbox 1055 1592 1394 1879; x_wconf 37'>oO]</span>
+      <span class='ocrx_word' id='word_1_7' title='bbox 1617 1587 3452 1952; x_wconf 62'>megapixels</span>
+     </span>
+    </p>
+   </div>
+  </div>
+ </body>
+</html>

tests/cache/multipage/__-l__eng__000002_ocr.png__000002_ocr_hocr__hocr__txt/stderr.bin

@@ -0,0 +1 @@
+Tesseract Open Source OCR Engine v4.1.1 with Leptonica

tests/cache/multipage/__-l__eng__000002_ocr.png__000002_ocr_hocr__hocr__txt/txt.bin

@@ -0,0 +1,3 @@
+YOOOxXYOO0O pixels at GOO DPI
+oO] megapixels
+

tests/test_main.py

@@ -101,10 +101,10 @@ def test_skip_ocr(spoof_tesseract_cache, resources, outpdf):
 
 def test_redo_ocr(resources, outpdf):
     in_ = resources / 'graph_ocred.pdf'
-    before = PdfInfo(in_, detailed_page_analysis=True)
+    before = PdfInfo(in_)
     out = outpdf
     out = check_ocrmypdf(in_, out, '--redo-ocr')
-    after = PdfInfo(out, detailed_page_analysis=True)
+    after = PdfInfo(out)
     assert before[0].has_text and after[0].has_text
     assert (
         before[0].get_textareas() != after[0].get_textareas()

tests/test_pdfinfo.py

@@ -151,22 +151,6 @@ def test_pickle(resources):
     pickle.dumps(pdf)
 
 
-def test_regex():
-    rx = pdfinfo.ghosttext.regex_remove_char_tags
-
-    must_match = [
-        b'<char bbox="0 108 0 108" c="/"/>',
-        b'<char bbox="0 108 0 108" c=">"/>',
-        b'<char bbox="0 108 0 108" c="X"/>',
-    ]
-    must_not_match = [b'<span stuff="c">', b'<span>', b'</span>', b'</page>']
-
-    for s in must_match:
-        assert rx.match(s)
-    for s in must_not_match:
-        assert not rx.match(s)
-
-
 def test_vector(resources):
     filename = resources / 'vector.pdf'
     pdf = pdfinfo.PdfInfo(filename)
@@ -184,16 +168,9 @@ def test_ocr_detection(resources):
 @pytest.mark.parametrize(
     'testfile', ('truetype_font_nomapping.pdf', 'type3_font_nomapping.pdf')
 )
-@pytest.mark.xfail(
-    ghostscript.version() in ('9.52',), reason="gs 9.52 txtwrite doesn't work"
-)
 def test_corrupt_font_detection(resources, testfile):
     filename = resources / testfile
-    with pytest.raises(NotImplementedError):
     pdf = pdfinfo.PdfInfo(filename)
-        pdf[0].has_corrupt_text
-
-    pdf = pdfinfo.PdfInfo(filename, detailed_page_analysis=True)
     assert pdf[0].has_corrupt_text
 
 

tests/test_validation.py

@@ -150,7 +150,7 @@ def test_false_action_store_true():
 @pytest.mark.parametrize('progress_bar', [True, False])
 def test_no_progress_bar(progress_bar, resources):
     opts = make_opts(progress_bar=progress_bar, input_file=(resources / 'trivial.pdf'))
-    with patch('ocrmypdf.pdfinfo.info.tqdm', autospec=True) as tqdmpatch:
+    with patch('ocrmypdf._concurrent.tqdm', autospec=True) as tqdmpatch:
         vd.check_options(opts)
         pdfinfo = PdfInfo(opts.input_file, progbar=opts.progress_bar)
         assert pdfinfo is not None