v11.4.0 release notes

Fixes #692

docs/advanced.rst

@@ -125,8 +125,7 @@ include:
 .. envvar:: OMP_THREAD_LIMIT
 
    Controls the number of threads Tesseract will use. OCRmyPDF will
-   manage this environment if it is not already set. (Currently, it will
-   set it to 1 because this gives the best results in testing.)
+   manage this environment variable if it is not already set.
 
 For example, if you have a development build of Tesseract don't wish to
 use the system installation, you can launch OCRmyPDF as follows:

docs/api.rst

@@ -20,7 +20,7 @@ and largely have the same functions.
 
     import ocrmypdf
 
-    if __name__ == '__main__':  # To ensure correct behavior on Windows
+    if __name__ == '__main__':  # To ensure correct behavior on Windows and macOS
         ocrmypdf.ocr('input.pdf', 'output.pdf', deskew=True)
 
 With a few exceptions, all of the command line arguments are available
@@ -42,8 +42,7 @@ execution. To do this, it will:
 - execute other subprocesses (forking and executing other programs)
 
 The Python process that calls ``ocrmypdf.ocr()`` must be sufficiently
-privileged to perform these actions. If it is not, ``ocrmypdf()`` will
-fail.
+privileged to perform these actions.
 
 There is no currently no option to manage how jobs are scheduled other
 than the argument ``jobs=`` which will limit the number of worker

docs/installation.rst

@@ -495,10 +495,6 @@ Installing on Windows
 Native Windows
 --------------
 
-.. note::
-
-    It is easier to install OCRmyPDF on Windows Subsystem for Linux.
-
 .. note::
 
     Administrator privileges will be required for some of these steps.
@@ -509,30 +505,33 @@ You must install the following for Windows:
 * Tesseract 4.0 or later
 * Ghostscript 9.50 or later
 
-You can install these with the Chocolatey package manager:
+Using the `Chocolatey <https://chocolatey.org/>`_ package manager, install the
+following when running in an Administrator command prompt:
 
 * ``choco install python3``
 * ``choco install --pre tesseract``
 * ``choco install ghostscript``
+* ``choco install pngquant`` (optional)
 
-Also consider adding:
+The commands above will install Python 3.x (latest version), Tesseract, Ghostscript
+and pngquant. Chocolatey may also need to install the Windows Visual C++ Runtime
+DLLs or other Windows patches, and may require a reboot.
 
-* ``choco install pngquant``
+You may then use ``pip`` to install ocrmypdf. (This can performed by a user or
+Administrator.):
 
-Windows 10 64-bit and 64-bit versions of applications are recommended. Earlier
-versions of Windows and 32-bit versions of these programs are not tested, and not
-supported at this time.
+* ``pip install ocrmypdf
 
-OCRmyPDF will check for Tesseract-OCR and Ghostscript in your Program Files folder.
-If they are in some other location, you may need to modify the ``PATH``
-environment variable so Tesseract, Ghostscript, and other any optional executables can
-be found. You can enter it in the command line or
-`follow these directions <https://www.computerhope.com/issues/ch000549.htm#dospath>`_
-to make the change persistent and system-wide.
+Chocolatey automatically selects appropriate versions of these applications. If you
+are installing them manually, please install 64-bit versions of all applications for
+64-bit Windows, or 32-bit versions of all applications for 32-bit Windows. Mixing
+the "bitness" of these programs will lead to errors.
 
-You may then use pip to install ocrmypdf:
-
-* ``pip install ocrmypdf``
+OCRmyPDF will check the Windows Registry and standard locations in your Program Files
+for third party software it needs (specifically, Tesseract and Ghostscript). To
+override the versions OCRmyPDF selects, you can modify the ``PATH`` environment
+variable. `Follow these directions <https://www.computerhope.com/issues/ch000549.htm#dospath>`_
+to change the PATH.
 
 Windows Subsystem for Linux
 ---------------------------

docs/release_notes.rst

@@ -12,6 +12,25 @@ may be unreliable. Use the API to depend on precise behavior.
 The public API may be useful in scripts that launch OCRmyPDF processes or that
 wish to use some of its features for working with PDFs.
 
+v11.4.0
+=======
+
+-  When looking for Tesseract and Ghostscript, we now check the Windows Registry to
+   see if their installers registered the location of their executables. This should
+   help Windows users who have installed these programs to non-standard
+   locations.
+-  We now report on the progress of PDF/A conversion, since this operation is
+   sometimes slow.
+-  Improved command line completions.
+-  The prefix of the temporary folder OCRmyPDF creates has been changed from
+   ``com.github.ocrmypdf`` to ``ocrmypdf.io``. Scripts that chose to depend on this
+   prefix may need to be adjusted. (This has always been an implementation detail so is
+   not considered part of the semantic versioning "contract".)
+-  Fixed issue #692, where a particular file with malformed fonts would flood an
+   internal message cue by generating so many debug messages.
+-  Fixed an exception on processing hOCR files with no page record. Tesseract
+   is not known to generate such files.
+
 v11.3.4
 =======
 

misc/completion/ocrmypdf.fish

@@ -59,7 +59,8 @@ complete -c ocrmypdf -x -l output-type -a '(__fish_ocrmypdf_output_type)' -d "se
 
 function __fish_ocrmypdf_pdf_renderer
     echo -e "auto\t"(_ "auto select PDF renderer")
-    echo -e "hocr\t"(_ "use hocr renderer")
+    echo -e "hocr\t"(_ "use hOCR renderer")
+    echo -e "hocrdebug\t"(_ "uses hOCR renderer in debug mode, showing recognized text")
     echo -e "sandwich\t"(_ "use sandwich renderer")
 end
 complete -c ocrmypdf -x -l pdf-renderer -a '(__fish_ocrmypdf_pdf_renderer)' -d "select PDF renderer options"
@@ -135,4 +136,4 @@ complete -c ocrmypdf -r -l user-words -d "specify location of user words file"
 complete -c ocrmypdf -r -l user-patterns -d "specify location of user patterns file"
 complete -c ocrmypdf -x -l fast-web-view -d "if file size if above this amount in MB, linearize PDF"
 
-complete -c ocrmypdf -x -a "(__fish_complete_suffix .pdf)"
+complete -c ocrmypdf -x -a "(__fish_complete_suffix .pdf; __fish_complete_suffix .PDF; __fish_complete_suffix .jpg; __fish_complete_suffix .png)"

src/ocrmypdf/_exec/ghostscript.py

@@ -21,7 +21,7 @@ from PIL import Image
 
 from ocrmypdf.exceptions import MissingDependencyError, SubprocessOutputError
 from ocrmypdf.helpers import Resolution
-from ocrmypdf.subprocess import get_version, run
+from ocrmypdf.subprocess import get_version, run, run_polling_stderr
 
 log = logging.getLogger(__name__)
 
@@ -139,12 +139,39 @@ def rasterize_pdf(
         im.save(fspath(output_file), dpi=page_dpi)
 
 
+class GhostscriptFollower:
+    re_process = re.compile(r"Processing pages \d+ through (\d+).")
+    re_page = re.compile(r"Page (\d+)")
+
+    def __init__(self, progressbar_class):
+        self.count = 0
+        self.progressbar_class = progressbar_class
+        self.progressbar = None
+
+    def __call__(self, line):
+        if not self.progressbar_class:
+            return
+        if not self.progressbar:
+            m = self.re_process.match(line.strip())
+            if m:
+                self.count = int(m.group(1))
+                self.progressbar = self.progressbar_class(
+                    total=self.count, desc="PDF/A conversion", unit='page'
+                )
+                return
+        else:
+            m = self.re_page.match(line.strip())
+            if m:
+                self.progressbar.update()
+
+
 def generate_pdfa(
     pdf_pages,
     output_file: os.PathLike,
     compression: str,
     pdf_version: str = '1.5',
     pdfa_part: str = '2',
+    progressbar_class=None,
 ):
     # Ghostscript's compression is all or nothing. We can either force all images
     # to JPEG, force all to Flate/PNG, or let it decide how to encode the images.
@@ -188,7 +215,6 @@ def generate_pdfa(
     args_gs = (
         [
             GS,
-            "-dQUIET",
             "-dBATCH",
             "-dNOPAUSE",
             "-dSAFER",
@@ -208,16 +234,26 @@ def generate_pdfa(
         ]
     )
     args_gs.extend(fspath(s) for s in pdf_pages)  # Stringify Path objs
+
     try:
         with Path(output_file).open('wb') as output:
-            p = run(args_gs, stdout=output, stderr=PIPE, check=True)
+            p = run_polling_stderr(
+                args_gs,
+                stdout=output,
+                stderr=PIPE,
+                check=True,
+                text=True,
+                encoding='utf-8',
+                errors='replace',
+                callback=GhostscriptFollower(progressbar_class),
+            )
     except CalledProcessError as e:
         # Ghostscript does not change return code when it fails to create
         # PDF/A - check PDF/A status elsewhere
-        log.error(e.stderr.decode(errors='replace'))
-        raise SubprocessOutputError('Ghostscript PDF/A rendering failed')
+        log.error(e.stderr)
+        raise SubprocessOutputError('Ghostscript PDF/A rendering failed') from e
     else:
-        stderr = p.stderr.decode('utf-8', errors='replace')
+        stderr = p.stderr
         if _gs_error_reported(stderr):
             last_part = None
             repcount = 0

src/ocrmypdf/_exec/tesseract.py

@@ -99,7 +99,14 @@ def get_languages():
 
     args_tess = ['tesseract', '--list-langs']
     try:
-        proc = run(args_tess, text=True, stdout=PIPE, stderr=STDOUT, check=True)
+        proc = run(
+            args_tess,
+            text=True,
+            stdout=PIPE,
+            stderr=STDOUT,
+            logs_errors_to_stdout=True,
+            check=True,
+        )
         output = proc.stdout
     except CalledProcessError as e:
         raise MissingDependencyError(lang_error(e.output)) from e

src/ocrmypdf/_pipeline.py

@@ -19,6 +19,7 @@ import img2pdf
 import pikepdf
 from pikepdf.models.metadata import encode_pdf_date
 from PIL import Image, ImageColor, ImageDraw
+from tqdm import tqdm
 
 from ocrmypdf import leptonica
 from ocrmypdf._exec import unpaper
@@ -601,14 +602,17 @@ def create_pdf_page_from_image(image: Path, page_context: PageContext):
 
 
 def render_hocr_page(hocr: Path, page_context: PageContext):
+    options = page_context.options
     output_file = page_context.get_path('ocr_hocr.pdf')
-    dpi = get_page_square_dpi(page_context.pageinfo, page_context.options)
+    dpi = get_page_square_dpi(page_context.pageinfo, options)
+    debug_mode = options.pdf_renderer == 'hocrdebug'
+
     hocrtransform = HocrTransform(hocr, dpi.x)  # square
     hocrtransform.to_pdf(
         output_file,
         image_filename=None,
-        show_bounding_boxes=False,
-        invisible_text=True,
+        show_bounding_boxes=False if not debug_mode else True,
+        invisible_text=True if not debug_mode else False,
         interword_spaces=True,
     )
     return output_file
@@ -709,6 +713,7 @@ def convert_to_pdfa(input_pdf: Path, input_ps_stub: Path, context: PdfContext):
         output_file=output_file,
         compression=options.pdfa_image_compression,
         pdfa_part=options.output_type[-1],  # is pdfa-1, pdfa-2, or pdfa-3
+        progressbar_class=tqdm if options.progress_bar else None,
     )
 
     return output_file

src/ocrmypdf/_sync.py

@@ -207,7 +207,7 @@ def exec_page_sync(page_context: PageContext):
             visible_image_out, page_context
         )
 
-    if options.pdf_renderer == 'hocr':
+    if options.pdf_renderer.startswith('hocr'):
         (hocr_out, text_out) = ocr_engine_hocr(ocr_image_out, page_context)
         ocr_out = render_hocr_page(hocr_out, page_context)
     elif options.pdf_renderer == 'sandwich':
@@ -330,7 +330,7 @@ def run_pipeline(options, *, plugin_manager, api=False):
     if not plugin_manager:
         plugin_manager = get_plugin_manager(options.plugins)
 
-    work_folder = Path(mkdtemp(prefix="com.github.ocrmypdf."))
+    work_folder = Path(mkdtemp(prefix="ocrmypdf.io."))
     debug_log_handler = None
     if (
         (options.keep_temporary_files or options.verbose >= 1)

src/ocrmypdf/_validation.py

@@ -78,7 +78,7 @@ def check_options_languages(options, ocr_engine_languages):
 def check_options_output(options):
     is_latin = options.languages.issubset(HOCR_OK_LANGS)
 
-    if options.pdf_renderer == 'hocr' and not is_latin:
+    if options.pdf_renderer.startswith('hocr') and not is_latin:
         msg = (
             "The 'hocr' PDF renderer is known to cause problems with one "
             "or more of the languages in your document.  Use "

src/ocrmypdf/api.py

@@ -9,6 +9,7 @@ import logging
 import os
 import sys
 from enum import IntEnum
+from io import IOBase
 from pathlib import Path
 from typing import AnyStr, BinaryIO, Iterable, Optional, Union
 from warnings import warn
@@ -174,14 +175,14 @@ def create_options(
         else:
             raise TypeError(f"{arg}: {val} ({type(val)})")
 
-    try:
-        cmdline.append(os.fspath(input_file))
-    except TypeError:
+    if isinstance(input_file, (BinaryIO, IOBase)):
         cmdline.append('stream://input_file')
-    try:
-        cmdline.append(os.fspath(output_file))
-    except TypeError:
+    else:
+        cmdline.append(os.fspath(input_file))
+    if isinstance(output_file, (BinaryIO, IOBase)):
         cmdline.append('stream://output_file')
+    else:
+        cmdline.append(os.fspath(output_file))
 
     parser._api_mode = True
     options = parser.parse_args(cmdline)

src/ocrmypdf/builtin_plugins/ghostscript.py

@@ -79,12 +79,21 @@ def rasterize_pdf_page(
 
 
 @hookimpl
-def generate_pdfa(pdf_pages, pdfmark, output_file, compression, pdf_version, pdfa_part):
+def generate_pdfa(
+    pdf_pages,
+    pdfmark,
+    output_file,
+    compression,
+    pdf_version,
+    pdfa_part,
+    progressbar_class,
+):
     ghostscript.generate_pdfa(
         pdf_pages=[*pdf_pages, pdfmark],
         output_file=output_file,
         compression=compression,
         pdf_version=pdf_version,
         pdfa_part=pdfa_part,
+        progressbar_class=progressbar_class,
     )
     return output_file

src/ocrmypdf/cli.py

@@ -6,12 +6,15 @@
 
 
 import argparse
+from typing import Optional, Type, TypeVar
 
 from ocrmypdf._version import PROGRAM_NAME as _PROGRAM_NAME
 from ocrmypdf._version import __version__ as _VERSION
 
+T = TypeVar('T')
 
-def numeric(basetype, min_=None, max_=None):
+
+def numeric(basetype: Type[T], min_: Optional[T] = None, max_: Optional[T] = None):
     """Validator for numeric params"""
     min_ = basetype(min_) if min_ is not None else None
     max_ = basetype(max_) if max_ is not None else None
@@ -407,7 +410,7 @@ Online documentation is located at:
     )
     advanced.add_argument(
         '--pdf-renderer',
-        choices=['auto', 'hocr', 'sandwich'],
+        choices=['auto', 'hocr', 'sandwich', 'hocrdebug'],
         default='auto',
         help="Choose OCR PDF renderer - the default option is to let OCRmyPDF "
         "choose.  See documentation for discussion.",

src/ocrmypdf/helpers.py

@@ -216,10 +216,7 @@ def check_pdf(input_file: Path) -> bool:
             pdf.close()
 
 
-T = TypeVar('T')
-
-
-def clamp(n: T, smallest: T, largest: T) -> T:
+def clamp(n, smallest, largest):  # mypy doesn't understand types for this
     """Clamps the value of n to between smallest and largest."""
     return max(smallest, min(n, largest))
 
@@ -232,6 +229,7 @@ def pikepdf_enable_mmap():
     #     log.debug("pikepdf mmap not available")
     # We found a race condition probably related to pybind issue #2252 that can
     # cause a crash. For now, disable pikepdf mmap to be on the safe side.
+    # Fix is not in pybind11 2.6.0
     log.debug("pikepdf mmap disabled")
     return
 

src/ocrmypdf/hocrtransform.py

@@ -42,6 +42,8 @@ from reportlab.lib.colors import black, cyan, magenta, red
 from reportlab.lib.units import inch
 from reportlab.pdfgen.canvas import Canvas
 
+Element = ElementTree.Element
+
 Rect = namedtuple('Rect', ['x1', 'y1', 'x2', 'y2'])
 
 
@@ -105,7 +107,7 @@ class HocrTransform:
         else:
             return ''
 
-    def _get_element_text(self, element):
+    def _get_element_text(self, element: Element):
         """
         Return the textual content of the element and its children
         """
@@ -119,7 +121,7 @@ class HocrTransform:
         return text
 
     @classmethod
-    def element_coordinates(cls, element) -> Rect:
+    def element_coordinates(cls, element: Element) -> Rect:
         """
         Returns a tuple containing the coordinates of the bounding box around
         an element
@@ -133,7 +135,7 @@ class HocrTransform:
         return out
 
     @classmethod
-    def baseline(cls, element) -> Tuple[float, float]:
+    def baseline(cls, element: Element) -> Tuple[float, float]:
         """
         Returns a tuple containing the baseline slope and intercept.
         """
@@ -149,7 +151,7 @@ class HocrTransform:
         """
         return Rect._make((c / self.dpi * inch) for c in pxl)
 
-    def _child_xpath(self, html_tag, html_class=None):
+    def _child_xpath(self, html_tag: str, html_class: Optional[str] = None) -> str:
         xpath = f".//{self.xmlns}{html_tag}"
         if html_class:
             xpath += f"[@class='{html_class}']"
@@ -280,13 +282,15 @@ class HocrTransform:
     def _do_line(
         self,
         pdf: Canvas,
-        line,
+        line: Optional[Element],
         elemclass: str,
         fontname: str,
         invisible_text: bool,
         interword_spaces: bool,
         show_bounding_boxes: bool,
     ):
+        if not line:
+            return
         pxl_line_coords = self.element_coordinates(line)
         line_box = self.pt_from_pixel(pxl_line_coords)
         line_height = line_box.y2 - line_box.y1

src/ocrmypdf/leptonica.py

@@ -27,15 +27,16 @@ from tempfile import TemporaryFile
 
 from ocrmypdf.exceptions import MissingDependencyError
 from ocrmypdf.lib._leptonica import ffi
-from ocrmypdf.subprocess import shim_paths_with_program_files
 
 # pylint: disable=protected-access
 
 logger = logging.getLogger(__name__)
 
 if os.name == 'nt':
+    from ocrmypdf.subprocess._windows import shim_env_path
+
     libname = 'liblept-5'
-    os.environ['PATH'] = shim_paths_with_program_files()
+    os.environ['PATH'] = shim_env_path()
 else:
     libname = 'lept'
 _libpath = find_library(libname)
@@ -58,9 +59,9 @@ if not _libpath:
         ---------------------------------------------------------------------
         """
     )
-if os.name == 'nt':  
-    # On Windows, recent versions of libpng require zlib. We have to make sure 
-    # the zlib version being loaded is the same one that libpng was built with. 
+if os.name == 'nt':
+    # On Windows, recent versions of libpng require zlib. We have to make sure
+    # the zlib version being loaded is the same one that libpng was built with.
     # This tries to import zlib from Tesseract's installation folder, falling back
     # to find_library() if liblept is being loaded from somewhere else.
     # Loading zlib from other places could cause a version mismatch

src/ocrmypdf/optimize.py

@@ -25,7 +25,6 @@ from typing import (
     Optional,
     Sequence,
     Tuple,
-    Union,
 )
 
 import img2pdf
@@ -294,10 +293,6 @@ def extract_images_jbig2(pike: Pdf, root: Path, options) -> Dict[int, List[XrefE
         group = pageno // options.jbig2_page_group_size
         jbig2_groups[group].append(xref_ext)
 
-    # Elide empty groups
-    jbig2_groups = {
-        group: xrefs for group, xrefs in jbig2_groups.items() if len(xrefs) > 0
-    }
     log.debug("Optimizable images: JBIG2 groups: %s", (len(jbig2_groups),))
     return jbig2_groups
 

src/ocrmypdf/pdfa.py

@@ -33,6 +33,7 @@ def _postscript_objdef(
     objtype = '/stream' if stream_name else '/dict'
 
     if stream_name:
+        assert stream_data is not None
         a85_data = base64.a85encode(stream_data, adobe=True).decode('ascii')
         yield f'{stream_name} ' + a85_data
         yield 'def'

src/ocrmypdf/pdfinfo/info.py

@@ -627,9 +627,12 @@ def _pdf_get_pageinfo(
 worker_pdf = None
 
 
-def _pdf_pageinfo_sync_init(infile):
+def _pdf_pageinfo_sync_init(infile: Path, pdfminer_loglevel):
     global worker_pdf  # pylint: disable=global-statement
     pikepdf_enable_mmap()
+
+    logging.getLogger('pdfminer').setLevel(pdfminer_loglevel)
+
     # If this function is called as a thread initializer, we need a messy hack
     # to close worker_pdf. If called as a process, it will be released when the
     # process is terminated.
@@ -674,7 +677,9 @@ def _pdf_pageinfo_concurrent(
             tqdm_kwargs=dict(
                 total=total, desc="Scanning contents", unit='page', disable=not progbar
             ),
-            task_initializer=partial(_pdf_pageinfo_sync_init, infile),
+            task_initializer=partial(
+                _pdf_pageinfo_sync_init, infile, logging.getLogger('pdfminer').level
+            ),
             task=_pdf_pageinfo_sync,
             task_arguments=contexts,
             task_finished=update_pageinfo,

src/ocrmypdf/pluginspec.py

@@ -59,7 +59,7 @@ def check_options(options: Namespace) -> None:
     Note:
         This hook will be called from the main process, and may modify global state
         before child worker processes are forked.
-	"""
+    """
 
 
 @hookspec
@@ -280,6 +280,7 @@ def generate_pdfa(
     compression: str,
     pdf_version: str,
     pdfa_part: str,
+    progressbar_class,
 ) -> Path:
     """Generate a PDF/A.
 
@@ -302,10 +303,21 @@ def generate_pdfa(
             At its own discretion, the PDF/A generator may raise the version,
             but should not lower it.
         pdfa_part: The desired PDF/A compliance level, such as ``'2B'``.
+        progressbar_class: The class of a progress bar with a tqdm-like API. An
+            instance of this class will be initialized when PDF/A conversion
+            begins, using
+            ``instance = progressbar_class(total: int, desc: str, unit:str)``,
+            defining the number of work units, a user-visible description,
+            and the name of the work units ("page"). Then ``instance.update()``
+            will be called when a work unit is completed. If ``None``, no
+            progress information is reported.
 
     Returns:
         Path: If successful, the hook should return ``output_file``.
 
     Note:
         This is a :ref:`firstresult hook<firstresult>`.
+
+    See also:
+        https://github.com/tqdm/tqdm
     """

src/ocrmypdf/py.typed

@@ -0,0 +1 @@
+# ocrmypdf is typed

src/ocrmypdf/subprocess/__init__.py

@@ -10,14 +10,13 @@
 import logging
 import os
 import re
-import shutil
 import sys
 from collections.abc import Mapping
 from contextlib import suppress
 from distutils.version import LooseVersion
 from functools import lru_cache
 from pathlib import Path
-from subprocess import PIPE, STDOUT, CalledProcessError
+from subprocess import PIPE, STDOUT, CalledProcessError, CompletedProcess, Popen
 from subprocess import run as subprocess_run
 
 from ocrmypdf.exceptions import MissingDependencyError
@@ -41,26 +40,7 @@ def run(args, *, env=None, logs_errors_to_stdout=False, **kwargs):
             if there is an error. If False, stderr is logged. Could be used with
             stderr=STDOUT, stdout=PIPE for example.
     """
-    if not env:
-        env = os.environ
-
-    # Search in spoof path if necessary
-    program = args[0]
-
-    if os.name == 'nt':
-        args = _fix_windows_args(program, args, env)
-
-    log.debug("Running: %s", args)
-    process_log = log.getChild(os.path.basename(program))
-    if sys.version_info < (3, 7):
-        if os.name == 'nt':
-            # Can't use close_fds=True on Windows with Python 3.6 or older
-            # https://bugs.python.org/issue19575, etc.
-            kwargs['close_fds'] = False
-        if 'text' in kwargs:
-            # Convert run(...text=) to run(...universal_newlines=) for Python 3.6
-            kwargs['universal_newlines'] = kwargs['text']
-            del kwargs['text']
+    args, env, process_log, _text = _fix_process_args(args, env, kwargs)
 
     stderr = None
     stderr_name = 'stderr' if not logs_errors_to_stdout else 'stdout'
@@ -82,28 +62,64 @@ def run(args, *, env=None, logs_errors_to_stdout=False, **kwargs):
     return proc
 
 
-def _fix_windows_args(program, args, env):
-    """Adjust our desired program and command line arguments for use on Windows"""
+def run_polling_stderr(args, *, callback, check=False, env=None, **kwargs):
+    """Run a process like ``ocrmypdf.subprocess.run``, and poll stderr.
 
-    if sys.version_info < (3, 8):
-        # bpo-33617 - Windows needs manual Path -> str conversion
-        args = [os.fspath(arg) for arg in args]
-        program = os.fspath(program)
+    Every line of produced by stderr will be forwarded to the callback function.
+    The intended use is monitoring progress of subprocesses that output their
+    own progress indicators. In addition, each line will be logged if debug
+    logging is enabled.
 
-    # If we are running a .py on Windows, ensure we call it with this Python
-    # (to support test suite shims)
-    if program.lower().endswith('.py'):
-        args = [sys.executable] + args
+    Requires stderr to be opened in text mode for ease of handling errors. In
+    addition the expected encoding= and errors= arguments should be set. Note
+    that if stdout is already set up, it need not be binary.
+    """
+    args, env, process_log, text = _fix_process_args(args, env, kwargs)
+    assert text, "Must use text=True"
 
-    paths = os.pathsep.join(os.get_exec_path(env))
-    if not shutil.which(args[0], path=paths):
-        # If the program we want is not on the PATH, add some interesting
-        # locations in %PROGRAMFILES% to the PATH and try again
-        shimmed_path = shim_paths_with_program_files(env)
-        new_args0 = shutil.which(args[0], path=shimmed_path)
-        if new_args0:
-            args[0] = new_args0
-    return args
+    proc = Popen(args, env=env, **kwargs)
+
+    lines = []
+    while proc.poll() is None:
+        for msg in iter(proc.stderr.readline, ''):
+            if process_log.isEnabledFor(logging.DEBUG):
+                process_log.debug(msg.strip())
+            callback(msg)
+            lines.append(msg)
+    stderr = ''.join(lines)
+
+    if check and proc.returncode != 0:
+        raise CalledProcessError(proc.returncode, args, output=None, stderr=stderr)
+    return CompletedProcess(args, proc.returncode, None, stderr=stderr)
+
+
+def _fix_process_args(args, env, kwargs):
+    assert 'universal_newlines' not in kwargs, "Use text= instead of universal_newlines"
+
+    if not env:
+        env = os.environ
+
+    # Search in spoof path if necessary
+    program = args[0]
+
+    if os.name == 'nt':
+        from ocrmypdf.subprocess._windows import fix_windows_args
+
+        args = fix_windows_args(program, args, env)
+
+    log.debug("Running: %s", args)
+    process_log = log.getChild(os.path.basename(program))
+    text = kwargs.get('text', False)
+    if sys.version_info < (3, 7):
+        if os.name == 'nt':
+            # Can't use close_fds=True on Windows with Python 3.6 or older
+            # https://bugs.python.org/issue19575, etc.
+            kwargs['close_fds'] = False
+        if 'text' in kwargs:
+            # Convert run(...text=) to run(...universal_newlines=) for Python 3.6
+            kwargs['universal_newlines'] = kwargs['text']
+            del kwargs['text']
+    return args, env, process_log, text
 
 
 @lru_cache(maxsize=None)
@@ -143,44 +159,18 @@ def get_version(
         raise MissingDependencyError(
             f"Could not find program '{program}' on the PATH"
         ) from e
-    try:
-        version = re.match(regex, output.strip()).group(1)
-    except AttributeError as e:
+
+    match = re.match(regex, output.strip())
+    if not match:
         raise MissingDependencyError(
             f"The program '{program}' did not report its version. "
             f"Message was:\n{output}"
         )
+    version = match.group(1)
 
     return version
 
 
-def shim_paths_with_program_files(env=None):
-    if not env:
-        env = os.environ
-    program_files = env.get('PROGRAMFILES', '')
-    if not program_files:
-        return env.get('PATH', '')
-
-    def path_walker():
-        for path in Path(program_files).iterdir():
-            if not path.is_dir():
-                continue
-            if path.name.lower() == 'tesseract-ocr':
-                yield path
-            elif path.name.lower() == 'gs':
-                yield from (p for p in path.glob('**/bin') if p.is_dir())
-
-    paths = sorted(
-        (p for p in path_walker()), key=lambda p: (p.name, p.parent.name), reverse=True
-    )
-    paths.extend(
-        Path(str_path)
-        for str_path in os.get_exec_path(env)
-        if Path(str_path) not in set(paths)
-    )
-    return os.pathsep.join(str(p) for p in paths)
-
-
 missing_program = '''
 The program '{program}' could not be executed or was not found on your
 system PATH.

src/ocrmypdf/subprocess/_windows.py

@@ -0,0 +1,162 @@
+# © 2020 James R. Barlow: github.com/jbarlow83
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import logging
+import os
+import shutil
+import sys
+from distutils.version import LooseVersion
+from itertools import chain, filterfalse
+from pathlib import Path
+from typing import Any, Callable, Iterator, Optional, Tuple, TypeVar, cast
+
+try:
+    import winreg
+except ModuleNotFoundError as e:
+    raise ModuleNotFoundError("This module is for Windows only") from e
+
+log = logging.getLogger(__name__)
+
+T = TypeVar('T')
+
+
+def registry_enum(
+    key: winreg.HKEYType, enum_fn: Callable[[winreg.HKEYType, int], T]
+) -> Iterator[T]:
+    LIMIT = 999
+    n = 0
+    while n < LIMIT:
+        try:
+            yield enum_fn(key, n)
+            n += 1
+        except OSError:
+            break
+    if n == LIMIT:
+        raise ValueError(f"Too many registry keys under {key}")
+
+
+def registry_subkeys(key: winreg.HKEYType) -> Iterator[str]:
+    return registry_enum(key, winreg.EnumKey)
+
+
+def registry_values(key: winreg.HKEYType) -> Iterator[Tuple[str, Any, int]]:
+    return registry_enum(key, winreg.EnumValue)
+
+
+def registry_path_ghostscript(env=None) -> Iterator[Path]:
+    try:
+        with winreg.OpenKey(
+            winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\Artifex\GPL Ghostscript"
+        ) as k:
+            latest_gs = max(registry_subkeys(k), key=LooseVersion)
+        with winreg.OpenKey(
+            winreg.HKEY_LOCAL_MACHINE, fr"SOFTWARE\Artifex\GPL Ghostscript\{latest_gs}"
+        ) as k:
+            _, gs_path, _ = next(registry_values(k))
+            yield Path(gs_path) / 'bin'
+    except OSError as e:
+        log.warning(e)
+
+
+def registry_path_tesseract(env=None) -> Iterator[Path]:
+    try:
+        with winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\Tesseract-OCR") as k:
+            for subkey, val, _valtype in registry_values(k):
+                if subkey == 'InstallDir':
+                    tesseract_path = Path(val)
+                    yield tesseract_path
+    except OSError as e:
+        log.warning(e)
+
+
+def program_files_paths(env=None) -> Iterator[Path]:
+    if not env:
+        env = os.environ
+    program_files = env.get('PROGRAMFILES', '')
+
+    def path_walker() -> Iterator[Path]:
+        for path in Path(program_files).iterdir():
+            if not path.is_dir():
+                continue
+            if path.name.lower() == 'tesseract-ocr':
+                yield path
+            elif path.name.lower() == 'gs':
+                yield from (p for p in path.glob('**/bin') if p.is_dir())
+
+    return iter(
+        sorted(
+            (p for p in path_walker()),
+            key=lambda p: (p.name, p.parent.name),
+            reverse=True,
+        )
+    )
+
+
+def paths_from_env(env=None) -> Iterator[Path]:
+    return (Path(p) for p in os.get_exec_path(env) if p)
+
+
+def shim_path(new_paths: Callable[[Any], Iterator[Path]], env=None) -> str:
+    if not env:
+        env = os.environ
+    return os.pathsep.join(str(p) for p in new_paths(env) if p)
+
+
+SHIMS = [
+    paths_from_env,
+    registry_path_ghostscript,
+    registry_path_tesseract,
+    program_files_paths,
+]
+
+
+def fix_windows_args(program, args, env):
+    """Adjust our desired program and command line arguments for use on Windows"""
+
+    if sys.version_info < (3, 8):
+        # bpo-33617 - Windows needs manual Path -> str conversion
+        args = [os.fspath(arg) for arg in args]
+        program = os.fspath(program)
+
+    # If we are running a .py on Windows, ensure we call it with this Python
+    # (to support test suite shims)
+    if program.lower().endswith('.py'):
+        args = [sys.executable] + args
+
+    # If the program we want is not on the PATH, check elsewhere
+    for shim in SHIMS:
+        shimmed_path = shim_path(shim, env)
+        new_args0 = shutil.which(args[0], path=shimmed_path)
+        if new_args0:
+            args[0] = new_args0
+            break
+
+    return args
+
+
+def unique_everseen(iterable, key=None):
+    "List unique elements, preserving order. Remember all elements ever seen."
+    # unique_everseen('AAAABBBCCDAABBB') --> A B C D
+    # unique_everseen('ABBCcAD', str.lower) --> A B C D
+    seen = set()
+    seen_add = seen.add
+    if key is None:
+        key = lambda x: x
+    for element in iterable:
+        k = key(element)
+        if k not in seen:
+            seen_add(k)
+            yield element
+
+
+def shim_env_path(env=None):
+    if env is None:
+        env = os.environ
+
+    shim_paths = chain.from_iterable(shim(env) for shim in SHIMS)
+    return os.pathsep.join(
+        str(p) for p in unique_everseen(shim_paths, key=lambda p: str.casefold(str(p)))
+    )

tests/plugins/gs_feature_elision.py

@@ -45,5 +45,6 @@ def generate_pdfa(pdf_pages, pdfmark, output_file, compression, pdf_version, pdf
             compression=compression,
             pdf_version=pdf_version,
             pdfa_part=pdfa_part,
+            progressbar_class=None,
         )
         return output_file

tests/plugins/gs_pdfa_failure.py

@@ -23,7 +23,7 @@ from unittest.mock import patch
 
 from ocrmypdf import hookimpl
 from ocrmypdf.builtin_plugins import ghostscript
-from ocrmypdf.subprocess import run
+from ocrmypdf.subprocess import run_polling_stderr
 
 
 def run_rig_args(args, **kwargs):
@@ -33,13 +33,13 @@ def run_rig_args(args, **kwargs):
     new_args = [
         arg for arg in args if not arg.startswith('-dPDFA') and not arg.endswith('.ps')
     ]
-    proc = run(new_args, **kwargs)
+    proc = run_polling_stderr(new_args, **kwargs)
     return proc
 
 
 @hookimpl
 def generate_pdfa(pdf_pages, pdfmark, output_file, compression, pdf_version, pdfa_part):
-    with patch('ocrmypdf._exec.ghostscript.run', new=run_rig_args):
+    with patch('ocrmypdf._exec.ghostscript.run_polling_stderr', new=run_rig_args):
         ghostscript.generate_pdfa(
             pdf_pages=pdf_pages,
             pdfmark=pdfmark,
@@ -47,5 +47,6 @@ def generate_pdfa(pdf_pages, pdfmark, output_file, compression, pdf_version, pdf
             compression=compression,
             pdf_version=pdf_version,
             pdfa_part=pdfa_part,
+            progressbar_class=None,
         )
         return output_file

tests/plugins/gs_render_failure.py

@@ -34,7 +34,7 @@ def raise_gs_fail(*args, **kwargs):
 
 @hookimpl
 def generate_pdfa(pdf_pages, pdfmark, output_file, compression, pdf_version, pdfa_part):
-    with patch('ocrmypdf._exec.ghostscript.run', new=raise_gs_fail):
+    with patch('ocrmypdf._exec.ghostscript.run_polling_stderr', new=raise_gs_fail):
         ghostscript.generate_pdfa(
             pdf_pages=pdf_pages,
             pdfmark=pdfmark,
@@ -42,5 +42,6 @@ def generate_pdfa(pdf_pages, pdfmark, output_file, compression, pdf_version, pdf
             compression=compression,
             pdf_version=pdf_version,
             pdfa_part=pdfa_part,
+            progressbar_class=None,
         )
         return output_file

tests/plugins/tesseract_cache.py

@@ -165,7 +165,7 @@ def cached_run(options, run_args, **run_kwargs):
 
     def clean_sys_argv():
         for arg in run_args[1:]:
-            yield re.sub(r'.*/com.github.ocrmypdf[^/]+[/](.*)', r'$TMPDIR/\1', arg)
+            yield re.sub(r'.*/ocrmypdf[.]io[.][^/]+[/](.*)', r'$TMPDIR/\1', arg)
 
     manifest['args'] = list(clean_sys_argv())
     with (Path(CACHE_ROOT) / 'manifest.jsonl').open('a') as f:

tests/test_helpers.py

@@ -13,7 +13,6 @@ from unittest.mock import MagicMock
 import pytest
 
 from ocrmypdf import helpers as helpers
-from ocrmypdf.subprocess import shim_paths_with_program_files
 
 
 class TestSafeSymlink:
@@ -94,7 +93,10 @@ class TestFileIsWritable:
         assert not helpers.is_file_writable(pathmock)
 
 
+@pytest.mark.skipif(os.name != 'nt', reason="Windows test")
 def test_shim_paths(tmp_path):
+    from ocrmypdf.subprocess._windows import shim_env_path
+
     progfiles = tmp_path / 'Program Files'
     progfiles.mkdir()
     (progfiles / 'tesseract-ocr').mkdir()
@@ -103,9 +105,9 @@ def test_shim_paths(tmp_path):
     syspath = tmp_path / 'bin'
     env = {'PROGRAMFILES': str(progfiles), 'PATH': str(syspath)}
 
-    result_str = shim_paths_with_program_files(env=env)
+    result_str = shim_env_path(env=env)
     results = result_str.split(os.pathsep)
-    assert results[0].endswith('tesseract-ocr')
-    assert results[1].endswith(os.path.join('gs', '9.52', 'bin'))
-    assert results[2].endswith(os.path.join('gs', '9.51', 'bin'))
-    assert results[3] == str(syspath)
+    assert results[0] == str(syspath), results
+    assert results[-3].endswith('tesseract-ocr'), results
+    assert results[-2].endswith(os.path.join('gs', '9.52', 'bin')), results
+    assert results[-1].endswith(os.path.join('gs', '9.51', 'bin')), results