v13.4.4 release notes

Thanks @QuLogic

.docker/Dockerfile

@@ -1,7 +1,7 @@
 # OCRmyPDF
 #
 
-FROM ubuntu:21.04 as base
+FROM ubuntu:22.04 as base
 
 ENV LANG=C.UTF-8
 ENV TZ=UTC

.github/workflows/build.yml

@@ -21,17 +21,19 @@ jobs:
       matrix:
         include:
           - os: ubuntu-18.04
-            python: 3.7
+            python: "3.7"
           - os: ubuntu-20.04
-            python: 3.8
+            python: "3.8"
           - os: ubuntu-20.04
-            python: 3.9
+            python: "3.9"
+          - os: ubuntu-20.04
+            python: "3.10"
           - os: ubuntu-latest
-            python: 3.9
+            python: "3.9"
           - os: ubuntu-latest
             python: "pypy-3.7"
           - os: ubuntu-latest
-            python: 3.9
+            python: "3.9"
             tesseract5: true
 
     env:
@@ -120,7 +122,7 @@ jobs:
     strategy:
       matrix:
         os: [macos-latest]
-        python: ["3.9"]
+        python: ["3.9", "3.10"]
 
     env:
       OS: ${{ matrix.os }}
@@ -175,7 +177,7 @@ jobs:
     strategy:
       matrix:
         os: [windows-latest]
-        python: ["3.9"]
+        python: ["3.9", "3.10"]
 
     env:
       OS: ${{ matrix.os }}

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.0.1
+    rev: v4.1.0
     hooks:
       - id: check-case-conflict
       - id: check-merge-conflict
@@ -8,26 +8,26 @@ repos:
       - id: check-yaml
       - id: debug-statements
   - repo: https://github.com/pycqa/isort
-    rev: 5.9.3
+    rev: 5.10.1
     hooks:
       - id: isort
         args: ["--profile", "black"]
   - repo: https://github.com/psf/black
-    rev: 21.9b0
+    rev: 22.3.0
     hooks:
       - id: black
         language_version: python
   - repo: https://github.com/asottile/setup-cfg-fmt
-    rev: v1.19.0
+    rev: v1.20.1
     hooks:
       - id: setup-cfg-fmt
   - repo: https://github.com/asottile/pyupgrade
-    rev: v2.29.0
+    rev: v2.31.1
     hooks:
       - id: pyupgrade
         args: ["--py37-plus"]
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.910-1
+    rev: v0.942
     hooks:
       - id: mypy
         additional_dependencies:

.readthedocs.yaml

@@ -14,7 +14,7 @@ formats:
 
 # Optionally set the version of Python and requirements required to build your docs
 python:
-  version: 3.7
+  version: "3.7"
   install:
     - method: pip
       path: .

docs/advanced.rst

@@ -236,12 +236,6 @@ PDF.js viewer.
 
 This works in all versions of Tesseract.
 
-The ``tesseract`` renderer
---------------------------
-
-The ``tesseract`` renderer was removed. OCRmyPDF's new approach to text
-layer grafting makes it functionally equivalent to ``sandwich``.
-
 Return code policy
 ==================
 

docs/api.rst

@@ -68,6 +68,14 @@ OCRmyPDF, use processes.
     not take at least one of these steps, process semantics will prevent
     OCRmyPDF from working correctly.
 
+.. warning::
+
+    On macOS with Python 3.7, you must call
+    :func:`multiprocessing.set_start_method("spawn")`. Without this, multiprocessing
+    will be unstable. From the command line, OCRmyPDF does this automatically,
+    but as an API user you must do this. See Python bpo-33725 for details.
+    Python 3.8+ also resolve this automatically.
+
 Logging
 -------
 

docs/cookbook.rst

@@ -104,6 +104,9 @@ This produces a file named "output.pdf" and a companion text file named
     because of options like ``--skip-big`` or ``--tesseract-timeout``, those pages
     will not be in the sidecar.
 
+    If you don't want to generate the output PDF, use ``--output-type=none`` to
+    avoid generating one. Set the output filename to ``-`` (i.e. redirect to stdout).
+
     To extract all text from a PDF, whether generated from OCR or otherwise,
     use a program like Poppler's ``pdftotext`` or ``pdfgrep``.
 

docs/docker.rst

@@ -59,7 +59,7 @@ Using the Docker image on the command line
 ==========================================
 
 **Unlike typical Docker containers**, in this section the OCRmyPDF Docker
-container is emphemeral – it runs for one OCR job and terminates, just like a
+container is ephemeral – it runs for one OCR job and terminates, just like a
 command line program. We are using Docker to deliver an application (as opposed
 to the more conventional case, where a Docker container runs as a server).
 

docs/installation.rst

@@ -211,69 +211,6 @@ user's ``PATH`` to check for the user's Python packages.
 
 To add JBIG2 encoding, see :ref:`jbig2`.
 
-Ubuntu 16.04 LTS
-----------------
-
-No package is available for Ubuntu 16.04. OCRmyPDF 8.0 and newer require
-Python 3.7. Ubuntu 16.04 ships Python 3.5, but you can install Python
-3.7 on it. Or, you can skip Python 3.7 and install OCRmyPDF 7.x or older
-- for that procedure, please see the installation documentation for the
-version of OCRmyPDF you plan to use.
-
-**Install system packages for OCRmyPDF**
-
-.. code-block:: bash
-
-    sudo apt-get update
-    sudo apt-get install -y software-properties-common python-software-properties
-    sudo add-apt-repository -y \
-        ppa:jonathonf/python-3.7 \
-        ppa:alex-p/tesseract-ocr
-    sudo apt-get update
-    sudo apt-get install -y \
-        ghostscript \
-        libexempi3 \
-        libffi6 \
-        pngquant \
-        python3.7 \
-        qpdf \
-        tesseract-ocr \
-        unpaper
-
-This will install a Python 3.7 binary at ``/usr/bin/python3.7``
-alongside the system's Python 3.5. Do not remove the system Python. This
-will also install Tesseract 4.0 from a PPA, since the version available
-in Ubuntu 16.04 is too old for OCRmyPDF.
-
-Now install pip for Python 3.7. This will install the Python 3.7 version
-of ``pip`` at ``/usr/local/bin/pip``.
-
-.. code-block:: bash
-
-    curl https://bootstrap.pypa.io/get-pip.py | sudo python3.7
-
-**Install OCRmyPDF**
-
-OCRmyPDF requires the locale to be set for UTF-8. **On some minimal
-Ubuntu installations**, such as the Ubuntu 16.04 Docker images it may be
-necessary to set the locale.
-
-.. code-block:: bash
-
-    # Optional: Only need to set these if they are not already set
-    export LC_ALL=C.UTF-8
-    export LANG=C.UTF-8
-
-Now install OCRmyPDF for the current user, and ensure that the ``PATH``
-environment variable contains ``$HOME/.local/bin``.
-
-.. code-block:: bash
-
-    export PATH=$HOME/.local/bin:$PATH
-    pip3.7 install --user ocrmypdf
-
-To add JBIG2 encoding, see :ref:`jbig2`.
-
 Arch Linux (AUR)
 ----------------
 

docs/jbig2.rst

@@ -32,6 +32,9 @@ For all other Linux, you must build a JBIG2 encoder from source:
 
 .. _jbig2-lossy:
 
+Dependencies include libtoolize and libleptonica, which on Ubuntu systems 
+are packaged as libtool and libleptonica-dev.
+
 Lossy mode JBIG2
 ================
 

docs/languages.rst

@@ -54,6 +54,33 @@ to what languages it should search for. Multiple languages can be
 requested using either ``-l eng+fra`` (English and French) or
 ``-l eng -l fra``.
 
+Gentoo users
+============
+
+On Gentoo the package ``app-text/tessdata_fast``, which ``app-text/tesseract`` depends on, handles Tesseract languages.
+It accepts USE flags to select what languages should be installed, these can be set in ``/etc/portage/package.use``.
+Alternatively one can globally set the `L10N use extension <https://wiki.gentoo.org/wiki/Localization/Guide#L10N>`__ in ``/etc/portage/make.conf``.
+This enables these languages for all packages (e.g. including aspell).
+
+.. code-block:: bash
+
+   # Display a list of all Tesseract language packs
+   equery uses app-text/tessdata_fast
+   
+   # Add English and German language support for Tesseract only
+   echo 'app-text/tessdata_fast l10n_de l10n_en' >> /etc/portage/package.use
+   
+   # Add global English and German language support (the `l10n_` from equery has to be omited)
+   echo L10N="de en" >> /etc/portage/make.conf
+   
+   # update system to reflect changed USE flags
+   emerge --update --deep --newuse @world
+
+You can then pass the ``-l LANG`` argument to OCRmyPDF to give a hint as
+to what languages it should search for. Multiple languages can be
+requested using either ``-l eng+fra`` (English and French) or
+``-l eng -l fra``.
+
 macOS users
 ===========
 

docs/plugins.rst

@@ -152,6 +152,16 @@ hooks. As such, you cannot "chain" a series of plugin filters together in this
 way. Instead, a single hook implementation should be responsible for any such
 chaining operations.
 
+Examples
+========
+
+* OCRmyPDF's test suite contains several plugins that are used to simulate certain
+  test conditions.
+* `ocrmypdf-papermerge <https://github.com/papermerge/OCRmyPDF_papermerge>`_ is
+  a production plugin that integrates OCRmyPDF and the Papermerge document
+  management system.
+
+
 Custom command line arguments
 -----------------------------
 

docs/release_notes.rst

@@ -12,11 +12,105 @@ may be unreliable. Use the API to depend on precise behavior.
 The public API may be useful in scripts that launch OCRmyPDF processes or that
 wish to use some of its features for working with PDFs.
 
-.. note::
+The most recent release of OCRmyPDF is |OCRmyPDF PyPI|. Any newer versions
+referred to in these notes may exist the main branch but have not been
+tagged yet.
 
-   Python 3.6 reaches end of life on December 23, 2021. We have already ended support 
-   for Python 3.6 but might release fixes for critical issues if necessary before that
-   date.
+.. |OCRmyPDF PyPI| image:: https://img.shields.io/pypi/v/ocrmypdf.svg
+
+v13.4.4
+=======
+
+-  Updated pdfminer.six version.
+-  Docker image changed to Ubuntu 22.04 now that it is released and provides the
+   dependencies we need. This seems more consistent than our recent change to
+   Debian.
+
+v13.4.3
+=======
+
+-  Fix error on pytest.skip() with older versions of pytest.
+-  Documentation updates.
+
+v13.4.2
+=======
+
+-  Worked around a
+   `major regression in Ghostscript 9.56.0 <https://bugs.ghostscript.com/show_bug.cgi?id=705187>`__
+   where **all OCR text is stripped out of the PDF**. It simply removes all text,
+   even generated by software other than OCRmyPDF. Fortunately, we can ask
+   Ghostscript 9.56.0 to use its old behavior that worked correctly for our purposes.
+   Users must avoid the combination (Ghostscript 9.56.0, ocrmypdf <13.4.2) since
+   older versions of OCRmyPDF have no way of detecting that this particular
+   version of Ghostscript removes all OCR text.
+-  Marked pdfminer 20220319 as supported.
+-  Fixed some deprecation warnings from recent versions of Pillow and pytest.
+-  Test suite now covers Python 3.10 (Python 3.10 worked fine before, but was not
+   being tested).
+-  Docker image now uses debian:bookworm-slim as the base image to fix the Docker
+   image build.
+
+v13.4.1
+=======
+
+-  Temporarily make threads rather than processes the default executor worker, due
+   to a persistent deadlock issue when processes are used. Add a new command line
+   argument ``--no-use-threads`` to disable this.
+
+v13.4.0
+=======
+
+-  Fixed test failures when using pikepdf 5.0.0.
+-  Various improvements to the optimizer. In particular, we now recognize PDF images
+   that are encoded with both deflate (PNG) and DCT (JPEG), and also produce PDF
+   with images compressed with deflate and DCT, since this often yields file size
+   improvements compared to plain DCT.
+
+v13.3.0
+=======
+
+-  Made a harmless but "scary" exception after failing to optimize an image less scary.
+-  Added a warning if a page image is too large for unpaper to clean. The image is
+   passed through without cleaning. This is due to a hard-coded limitation in a
+   C library used by unpaper so it cannot be rectified easily.
+-  We now use better default settings when calling img2pdf.
+-  We no longer try to optimize images that we failed to save in certain situations.
+-  We now account for some differences in text output from Tesseract 5 compared to
+   Tesseract 4.
+-  Better handling of Ghostscript producing empty images when attempting to rasterize
+   page images.
+
+v13.2.0
+=======
+
+-  Removed all runtime uses of distutils since it is deprecated in standard library. We
+   previous used ``distutils.version`` to examine version numbers of dependencies
+   at run time, and now use ``packaging.version`` for this. This is a new
+   dependency.
+-  Fixed an error message advising the user that Ghostscript was not installed being
+   suppressed when this condition actually happens.
+-  Fixed an issue with incorrect page number and totals being displayed in the progress
+   bar. This was purely a display/presentation issue. :issue:`876`.
+
+v13.1.1
+=======
+
+-  Fixed issue with attempting to deskew a blank page on Tesseract 5. :issue:`868`.
+
+v13.1.0
+=======
+
+-  Changed to using Python concurrent.futures-based parallel execution instead of
+   pools, since futures have now exceed pools in features.
+-  If a child worker is terminated (perhaps by the operating system or the user
+   killing it in a task manager), the parallel task will fail an error message.
+   Previously, the main ocrmypdf process would "hang" indefinitely, waiting for the
+   child to report.
+-  Added new argument ``--tesseract-thresholding`` to provide control over Tesseract 5's
+   threshold parameter.
+-  Documentation updates and changes. Better documentation for ``--output-type none``,
+   added a few releases ago. Removed some obsolete documentation.
+-  Improved bash completions - thanks to @FPille.
 
 v13.0.0
 =======
@@ -24,7 +118,7 @@ v13.0.0
 **Breaking changes**
 
 -  The deprecated module ``ocrmypdf.leptonica`` has been removed.
--  We no longer depend on Leptonica (``liblept``) or CFFI (``libffi``, 
+-  We no longer depend on Leptonica (``liblept``) or CFFI (``libffi``,
    ``python3-cffi``). (Note that Tesseract still requires Leptonica; OCRmyPDF no longer
    directly uses this library.)
 -  The argument ``--remove-background`` is temporarily disabled while we search for an
@@ -33,7 +127,7 @@ v13.0.0
    Tesseract 5.x has implemented improvements to thresholding, so this feature will be
    redundant anyway.
 -  ``--deskew`` was previous calculated by a Leptonica algorithm. We now use a feature
-   of Tesseract to term the appropriate the angle to deskew a page. The deskew angle
+   of Tesseract to find the appropriate the angle to deskew a page. The deskew angle
    according to Tesseract may differ from Leptonica's algorithm. At least in theory,
    Tesseract's deskew angle is informed by a more complex analysis than Leptonica,
    so this should improve results in general. We also use Pillow to perform the
@@ -48,7 +142,7 @@ v13.0.0
 
 **Fixes**
 
--  Fixed an issue where OCRmyPDF failed to find Ghostscript on Windows even when 
+-  Fixed an issue where OCRmyPDF failed to find Ghostscript on Windows even when
    installed, and would exit with an error.
 -  By removing Leptonica, we fixed all issues related to Leptonica on Apple
    Silicon or Leptonica failing to import on Windows.
@@ -56,7 +150,7 @@ v13.0.0
 v12.7.2
 =======
 
--  Fixed "invalid version number" error for Tesseract packaging with nonstandard 
+-  Fixed "invalid version number" error for Tesseract packaging with nonstandard
    version "5.0.0-rc1.20211030".
 -  Fixed use of deprecated ``importlib.resources.read_binary``.
 -  Replace some uses of string paths with ``pathlib.Path``.

misc/completion/ocrmypdf.bash

@@ -1,6 +1,6 @@
 # ocrmypdf completion                                     -*- shell-script -*-
 
-# Copyright 2019 Frank Pille
+# Copyright 2019, 2021 Frank Pille
 # Copyright 2020 Alex Willner
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -23,95 +23,285 @@
 
 set -o errexit
 
-_ocrmypdf()
+__ocrmypdf_arguments()
 {
-    local cur prev cword words split
+    local arguments="--help                   (show help message)
+--language               (language(s) of the file to be OCRed)
+--image-dpi              (assume this DPI if input image DPI is unknown)
+--output-type            (select PDF output options)
+--sidecar                (write OCR to text file)
+--version                (print program version and exit)
+--jobs                   (how many worker processes to use)
+--quiet                  (suppress INFO messages)
+--verbose                (set verbosity level)
+--title                  (set metadata)
+--author                 (set metadata)
+--subject                (set metadata)
+--keywords               (set metadata)
+--rotate-pages           (rotate pages to correct orientation)
+--remove-background      (attempt to remove background from pages)
+--deskew                 (fix small horizontal alignment skew)
+--clean                  (clean document images before OCR)
+--clean-final            (clean document images and keep result)
+--unpaper-args           (a quoted string of arguments to pass to unpaper)
+--oversample             (oversample images to this DPI)
+--remove-vectors         (don\'t send vector objects to OCR)
+--threshold              (threshold images before OCR)
+--force-ocr              (OCR documents that already have printable text)
+--skip-text              (skip OCR on any pages that already contain text)
+--redo-ocr               (redo OCR on any pages that seem to have OCR already)
+--skip-big               (skip OCR on pages larger than this many MPixels)
+--optimize               (select optimization level)
+--jpeg-quality           (JPEG quality [0..100])
+--png-quality            (PNG quality [0..100])
+--jbig2-lossy            (enable lossy JBIG2 (see docs))
+--pages                  (apply OCR to only the specified pages)
+--max-image-mpixels      (image decompression bomb threshold)
+--pdf-renderer           (select PDF renderer options)
+--rotate-pages-threshold (page rotation confidence)
+--pdfa-image-compression (set PDF/A image compression options)
+--fast-web-view          (if file size if above this amount in MB linearize PDF)
+--plugin                 (name of plugin to import)
+--keep-temporary-files   (keep temporary files (debug)
+--tesseract-config       (set custom tesseract config file)
+--tesseract-pagesegmode  (set tesseract --psm)
+--tesseract-oem          (set tesseract --oem)
+--tesseract-thresholding (set tesseract image thresholding)
+--tesseract-timeout      (maximum number of seconds to wait for OCR)
+--user-words             (specify location of user words file)
+--user-patterns          (specify location of user patterns file)
+--no-progress-bar        (disable the progress bar)
+"
 
-    # Homebrew on Macs have version 1.3 of bash-completion which doesn't include - see #502
-    if declare -F _init_completions >/dev/null 2>&1; then
-        _init_completion -s || return
-    else
-        COMPREPLY=()
-        _get_comp_words_by_ref cur prev words cword
+    COMPREPLY=( $( compgen -W "$arguments" -- "$cur") )
+
+    # Remove description if only one completion exists
+    if [[ ${#COMPREPLY[*]} -eq 1 ]]; then
+        COMPREPLY=( ${COMPREPLY[0]%% *} )
     fi
+}
 
-    if [[ $cur == -* ]]; then
-        COMPREPLY=( $( compgen -W '--language --image-dpi --output-type
-            --sidecar --version --jobs --quiet --verbose --title --author
-            --subject --keywords --rotate-pages --deskew
-            --clean --clean-final --unpaper-args --oversample --remove-vectors
-            --force-ocr --skip-text --redo-ocr
-            --skip-big --jpeg-quality --png-quality --jbig2-lossy
-            --max-image-mpixels --tesseract-config --tesseract-pagesegmode
-            --help --tesseract-oem --pdf-renderer --tesseract-timeout
-            --rotate-pages-threshold --pdfa-image-compression --user-words
-            --user-patterns --keep-temporary-files --output-type
-            --no-progress-bar --pages --fast-web-view' \
-            --  "$cur" ) )
-        return
-    else
-        _filedir
-        return
+__ocrmypdf_output-type()
+{
+    local choices="pdfa   (output a PDF/A (default))
+pdf    (output a standard PDF)
+pdfa-1 (output a PDF/A-1b)
+pdfa-2 (output a PDF/A-2b)
+pdfa-3 (output a PDF/A-3b)
+none   (do not produce an output PDF (for example, if you only care about --sidecar))"
+
+    COMPREPLY=( $( compgen -W "$choices" -- "$cur") )
+
+    # Remove description if only one completion exists
+    if [[ ${#COMPREPLY[*]} -eq 1 ]]; then
+        COMPREPLY=( ${COMPREPLY[0]%% *} )
     fi
+}
 
+__ocrmypdf_verbose()
+{
+    local choices="0  (standard output messages)
+1  (troubleshooting output messages)
+2  (debugging output messages)"
+
+    COMPREPLY=( $( compgen -W "$choices" -- "$cur") )
+
+    # Remove description if only one completion exists
+    if [[ ${#COMPREPLY[*]} -eq 1 ]]; then
+        COMPREPLY=( ${COMPREPLY[0]%% *} )
+    fi
+}
+
+__ocrmypdf_optimize()
+{
+    local choices="0  (do not optimize)
+1  (do safe, lossless optimizations (default))
+2  (do some lossy optimizations)
+3  (do aggressive lossy optimizations (including lossy JBIG2))"
+
+    COMPREPLY=( $( compgen -W "$choices" -- "$cur") )
+
+    # Remove description if only one completion exists
+    if [[ ${#COMPREPLY[*]} -eq 1 ]]; then
+        COMPREPLY=( ${COMPREPLY[0]%% *} )
+    fi
+}
+
+__ocrmypdf_pdf-renderer()
+{
+    local choices="auto      (auto select PDF renderer)
+hocr      (use hOCR renderer)
+hocrdebug (uses hOCR renderer in debug mode, showing recognized text)
+sandwich  (use sandwich renderer)"
+
+    COMPREPLY=( $( compgen -W "$choices" -- "$cur") )
+
+    # Remove description if only one completion exists
+    if [[ ${#COMPREPLY[*]} -eq 1 ]]; then
+        COMPREPLY=( ${COMPREPLY[0]%% *} )
+    fi
+}
+
+__ocrmypdf_pdfa-image-compression()
+{
+    local choices="auto     (let Ghostscript decide how to compress images)
+jpeg     (convert color and grayscale images to JPEG)
+lossless (convert color and grayscale images to lossless (PNG))"
+
+    COMPREPLY=( $( compgen -W "$choices" -- "$cur") )
+
+    # Remove description if only one completion exists
+    if [[ ${#COMPREPLY[*]} -eq 1 ]]; then
+        COMPREPLY=( ${COMPREPLY[0]%% *} )
+    fi
+}
+
+__ocrmypdf_tesseract-pagesegmode()
+{
+    local choices="0  (orientation and script detection (OSD) only)
+1  (automatic page segmentation with OSD)
+2  (automatic page segmentation, but no OSD, or OCR)
+3  (fully automatic page segmentation, but no OSD (default))
+4  (assume a single column of text of variable sizes)
+5  (assume a single uniform block of vertically aligned text)
+6  (assume a single uniform block of text)
+7  (treat the image as a single text line)
+8  (treat the image as a single word)
+9  (treat the image as a single word in a circle)
+10 (treat the image as a single character)
+11 (sparse text - find as much text as possible in no particular order)
+12 (sparse text with OSD)
+13 (raw line - treat the image as a single text line)"
+
+    COMPREPLY=( $( compgen -W "$choices" -- "$cur") )
+
+    # Remove description if only one completion exists
+    if [[ ${#COMPREPLY[*]} -eq 1 ]]; then
+        COMPREPLY=( ${COMPREPLY[0]%% *} )
+    fi
+}
+
+__ocrmypdf_tesseract-oem()
+{
+    local choices="0 (legacy engine only)
+1 (neural nets LSTM engine only)
+2 (legacy + LSTM engines)
+3 (default, based on what is available)"
+
+    COMPREPLY=( $( compgen -W "$choices" -- "$cur") )
+
+    # Remove description if only one completion exists
+    if [[ ${#COMPREPLY[*]} -eq 1 ]]; then
+        COMPREPLY=( ${COMPREPLY[0]%% *} )
+    fi
+}
+
+__ocrmypdf_tesseract-thresholding()
+{
+    local choices="auto          (let OCRmyPDF pick thresholding - current always uses otsu)
+otsu          (use hOCR renderer)
+adaptive-otsu (use adaptive Otsu thresholding)
+sauvola       (use Sauvola thresholding)"
+
+    COMPREPLY=( $( compgen -W "$choices" -- "$cur") )
+    # Remove description if only one completion exists
+    if [[ ${#COMPREPLY[*]} -eq 1 ]]; then
+        COMPREPLY=( ${COMPREPLY[0]%% *} )
+    fi
+}
+
+
+__ocrmypdf_check_previous()
+{
     case $prev in
-        --version|-h|--help)
-            return
-            ;;
-        --user-words|--user-patterns|--tesseract-config)
-            _filedir
-            return
-            ;;
-        --output-type)
-            COMPREPLY=( $( compgen -W 'pdfa pdf pdfa-1 pdfa-2 pdfa-3' -- \
-                "$cur" ) )
-            return
-            ;;
-        --pdf-renderer)
-            COMPREPLY=( $( compgen -W 'auto hocr sandwich' -- "$cur" ) )
-            return
-            ;;
-        --pdfa-image-compression)
-            COMPREPLY=( $( compgen -W 'auto jpeg lossless' -- "$cur" ) )
-            return
-            ;;
-        -O|--optimize|--tesseract-oem)
-            COMPREPLY=( $( compgen -W '{0..3}' -- "$cur" ) )
-            return
-            ;;
-        --jpeg-quality|--png-quality)
-            COMPREPLY=( $( compgen -W '{0..100}' -- "$cur" ) )
-            return
+        -h|--help|--version)
+            return 0
             ;;
         -l|--language)
             COMPREPLY=$( command tesseract --list-langs 2>/dev/null )
             COMPREPLY=( $( compgen -W '${COMPREPLY[@]##*:}' -- "$cur" ) )
-            return
+            return 0
             ;;
-        --image-dpi|--oversample|--skip-big|--max-image-mpixels|\
-        --tesseract-timeout|--rotate-pages-threshold)
-            COMPREPLY=( $( compgen -P "$cur" -W '{0..9}' ) )
-            return
+        --output-type)
+            __ocrmypdf_output-type
+            return 0
             ;;
         -j|--jobs)
             COMPREPLY=( $( compgen -W '{1..'$( _ncpus )'}' -- "$cur" ) )
-            return
+            return 0
             ;;
         -v|--verbose)
-            COMPREPLY=( $( compgen -W '{0..2}' -- "$cur" ) ) # max level ?
-            return
+            __ocrmypdf_verbose
+            return 0
+            ;;
+        -O|--optimize)
+            __ocrmypdf_optimize
+            return 0
+            ;;
+        --pdf-renderer)
+            __ocrmypdf_pdf-renderer
+            return 0
+            ;;
+        --pdfa-image-compression)
+            __ocrmypdf_pdfa-image-compression
+            return 0
             ;;
         --tesseract-pagesegmode)
-            COMPREPLY=( $( compgen -W '{1..13}' -- "$cur" ) )
-            return
+            __ocrmypdf_tesseract-pagesegmode
+            return 0
             ;;
-        --sidecar|--title|--author|--subject|--keywords|--unpaper-args|--pages|--fast-web-view)
+        --tesseract-oem)
+            __ocrmypdf_tesseract-oem
+            return 0
+            ;;
+        --tesseract-thresholding)
+            __ocrmypdf_tesseract-thresholding
+            return 0
+            ;;
+
+        --title|--author|--subject|--keywords|--unpaper-args|--pages|--plugin|\
+        --jpeg-quality|--png-quality|--image-dpi|--oversample|--skip-big|--max-image-mpixels|\
+        --tesseract-timeout|--rotate-pages-threshold|--fast-web-view)
             # argument required but no completions available
-            return
+            return 0
+            ;;
+        --tesseract-config|--user-words|--user-patterns|--sidecar)
+            _filedir
+            return 0
             ;;
     esac
 
-    $split && return
+    return 1
+}
+
+_ocrmypdf()
+{
+    local OLDIFS="$IFS"
+    local IFS=$'\n'
+
+    local cur prev
+
+    # Homebrew on Macs have version 1.3 of bash-completion which doesn't include - see #502
+    if declare -F _init_completion >/dev/null 2>&1; then
+      _init_completion  || return
+    else
+        COMPREPLY=()
+        _get_comp_words_by_ref cur prev
+    fi
+
+    if __ocrmypdf_check_previous -ne 0; then
+        return
+    fi
+
+    if [[ "$cur" == -* ]]; then
+        __ocrmypdf_arguments
+    else
+        _filedir
+    fi
+
+    IFS="$OLDIFS"
+
+    return
 } &&
 complete -F _ocrmypdf ocrmypdf
 

misc/completion/ocrmypdf.fish

@@ -105,20 +105,20 @@ complete -c ocrmypdf -x -l pages -d "apply OCR to only the specified pages"
 complete -c ocrmypdf -x -l tesseract-config -d "set custom tesseract config file"
 
 function __fish_ocrmypdf_tesseract_pagesegmode
-  echo -e "0\t"(_ "orientation and script detection (OSD) only")
-  echo -e "1\t"(_ "automatic page segmentation with OSD")
-  echo -e "2\t"(_ "automatic page segmentation, but no OSD, or OCR")
-  echo -e "3\t"(_ "fully automatic page segmentation, but no OSD (default)")
-  echo -e "4\t"(_ "assume a single column of text of variable sizes")
-  echo -e "5\t"(_ "assume a single uniform block of vertically aligned text")
-  echo -e "6\t"(_ "assume a single uniform block of text")
-  echo -e "7\t"(_ "treat the image as a single text line")
-  echo -e "8\t"(_ "treat the image as a single word")
-  echo -e "9\t"(_ "treat the image as a single word in a circle")
-  echo -e "10\t"(_ "treat the image as a single character")
-  echo -e "11\t"(_ "sparse text - find as much text as possible in no particular order")
-  echo -e "12\t"(_ "sparse text with OSD")
-  echo -e "13\t"(_ "raw line - treat the image as a single text line")
+    echo -e "0\t"(_ "orientation and script detection (OSD) only")
+    echo -e "1\t"(_ "automatic page segmentation with OSD")
+    echo -e "2\t"(_ "automatic page segmentation, but no OSD, or OCR")
+    echo -e "3\t"(_ "fully automatic page segmentation, but no OSD (default)")
+    echo -e "4\t"(_ "assume a single column of text of variable sizes")
+    echo -e "5\t"(_ "assume a single uniform block of vertically aligned text")
+    echo -e "6\t"(_ "assume a single uniform block of text")
+    echo -e "7\t"(_ "treat the image as a single text line")
+    echo -e "8\t"(_ "treat the image as a single word")
+    echo -e "9\t"(_ "treat the image as a single word in a circle")
+    echo -e "10\t"(_ "treat the image as a single character")
+    echo -e "11\t"(_ "sparse text - find as much text as possible in no particular order")
+    echo -e "12\t"(_ "sparse text with OSD")
+    echo -e "13\t"(_ "raw line - treat the image as a single text line")
 end
 complete -c ocrmypdf -x -l tesseract-pagesegmode -a '(__fish_ocrmypdf_tesseract_pagesegmode)' -d "set tesseract --psm"
 
@@ -129,6 +129,15 @@ function __fish_ocrmypdf_tesseract_oem
     echo -e "3\t"(_ "default, based on what is available")
 end
 complete -c ocrmypdf -x -l tesseract-oem -a '(__fish_ocrmypdf_tesseract_oem)' -d "set tesseract --oem"
+
+function __fish_ocrmypdf_tesseract_thresholding
+    echo -e "auto\t"(_ "let OCRmyPDF pick thresholding (current always uses otsu)")
+    echo -e "otsu\t"(_ "legacy Otsu thresholding")
+    echo -e "adaptive-otsu\t"(_ "use adaptive Otsu thresholding")
+    echo -e "sauvola\t"(_ "use Sauvola thresholding")
+end
+complete -c ocrmypdf -x -l tesseract-thresholding -a '(__fish_ocrmypdf_tesseract_thresholding)' -d "set tesseract thresholding method (needs Tesseract 5.x)"
+
 complete -c ocrmypdf -x -l tesseract-timeout -d "maximum number of seconds to wait for OCR"
 complete -c ocrmypdf -x -l rotate-pages-threshold -d "page rotation confidence"
 

misc/webservice.py

@@ -59,7 +59,7 @@ def do_ocrmypdf(file):
         return Response("--sidecar not supported", 501, mimetype='text/plain')
 
     ocrmypdf_args = ["ocrmypdf", *cmd_args, up_file, down_file]
-    proc = run(ocrmypdf_args, stdout=PIPE, stderr=PIPE, encoding="utf-8")
+    proc = run(ocrmypdf_args, capture_output=True, encoding="utf-8")
     if proc.returncode != 0:
         stderr = proc.stderr
         return Response(stderr, 400, mimetype='text/plain')

pyproject.toml

@@ -44,24 +44,38 @@ source = ["src/ocrmypdf"]
 [tool.coverage.report]
 # Regexes for lines to exclude from consideration
 exclude_lines = [
-    # Have to re-enable the standard pragma
-    "pragma: no cover",
-
-    # Don't complain if tests don't hit defensive assertion code:
-    "raise AssertionError",
-    "raise NotImplementedError",
-
-    # Don't complain if non-runnable code isn't run:
-    "if 0:",
-	  "if False:",
-    "if __name__ == .__main__.:",
-    "if TYPE_CHECKING:"
+  # Have to re-enable the standard pragma
+  "pragma: no cover",
+  # Don't complain if tests don't hit defensive assertion code:
+  "raise AssertionError",
+  "raise NotImplementedError",
+  # Don't complain if non-runnable code isn't run:
+  "if 0:",
+  "if False:",
+  "if __name__ == .__main__.:",
+  "if TYPE_CHECKING:"
 ]
 
 [tool.isort]
 profile = "black"
 known_first_party = "ocrmypdf"
-known_third_party = ["PIL", "flask", "img2pdf", "ocrmypdf", "pdfminer", "pikepdf", "pkg_resources", "pluggy", "pytest", "reportlab", "setuptools", "sphinx_rtd_theme", "tqdm", "watchdog", "werkzeug"]
+known_third_party = [
+  "PIL",
+  "flask",
+  "img2pdf",
+  "ocrmypdf",
+  "pdfminer",
+  "pikepdf",
+  "pkg_resources",
+  "pluggy",
+  "pytest",
+  "reportlab",
+  "setuptools",
+  "sphinx_rtd_theme",
+  "tqdm",
+  "watchdog",
+  "werkzeug"
+]
 
 [tool.pytest.ini_options]
 minversion = "6.0"
@@ -75,6 +89,13 @@ filterwarnings = ["ignore:.*XMLParser.*:DeprecationWarning"]
 
 [[tool.mypy.overrides]]
 module = [
-  'pluggy', 'tqdm', 'coloredlogs', 'img2pdf', 'pdfminer.*', 'reportlab.*'
+  'pluggy',
+  'tqdm',
+  'coloredlogs',
+  'img2pdf',
+  'pdfminer.*',
+  'reportlab.*',
+  'fitz',
+  'libxmp.utils'
 ]
 ignore_missing_imports = true

setup.cfg

@@ -48,8 +48,9 @@ install_requires =
     Pillow>=8.2.0
     coloredlogs>=14.0  # strictly optional
     img2pdf>=0.3.0,<0.5  # pure Python
-    pdfminer.six!=20200720,>=20191110,<=20211012
-    pikepdf>=4.0.0
+    packaging>=20
+    pdfminer.six!=20200720,>=20191110,<=20220506
+    pikepdf!=5.0.0,>=4.0.0
     pluggy>=0.13.0,<2
     reportlab>=3.5.66
     tqdm>=4

src/ocrmypdf/_exec/ghostscript.py

@@ -18,12 +18,19 @@ from shutil import which
 from subprocess import PIPE, CalledProcessError
 from typing import Optional
 
-from PIL import Image
+from PIL import Image, UnidentifiedImageError
 
 from ocrmypdf.exceptions import MissingDependencyError, SubprocessOutputError
 from ocrmypdf.helpers import Resolution
 from ocrmypdf.subprocess import get_version, run, run_polling_stderr
 
+# Remove this workaround when we require Pillow >= 10
+try:
+    Transpose = Image.Transpose  # type: ignore
+except AttributeError:
+    # Pillow 9 shim
+    Transpose = Image  # type: ignore
+
 log = logging.getLogger(__name__)
 
 missing_gs_error = """
@@ -71,7 +78,8 @@ def jpeg_passthrough_available() -> bool:
 
 
 def _gs_error_reported(stream) -> bool:
-    return True if re.search(r'error', stream, flags=re.IGNORECASE) else False
+    match = re.search(r'error', stream, flags=re.IGNORECASE)
+    return bool(match)
 
 
 def rasterize_pdf(
@@ -124,20 +132,27 @@ def rasterize_pdf(
         if _gs_error_reported(stderr):
             log.error(stderr)
 
-    with Image.open(BytesIO(p.stdout)) as im:
-        if rotation is not None:
-            log.debug("Rotating output by %i", rotation)
-            # rotation is a clockwise angle and Image.ROTATE_* is
-            # counterclockwise so this cancels out the rotation
-            if rotation == 90:
-                im = im.transpose(Image.ROTATE_90)
-            elif rotation == 180:
-                im = im.transpose(Image.ROTATE_180)
-            elif rotation == 270:
-                im = im.transpose(Image.ROTATE_270)
-            if rotation % 180 == 90:
-                page_dpi = page_dpi.flip_axis()
-        im.save(fspath(output_file), dpi=page_dpi)
+    try:
+        with Image.open(BytesIO(p.stdout)) as im:
+            if rotation is not None:
+                log.debug("Rotating output by %i", rotation)
+                # rotation is a clockwise angle and Image.ROTATE_* is
+                # counterclockwise so this cancels out the rotation
+                if rotation == 90:
+                    im = im.transpose(Transpose.ROTATE_90)
+                elif rotation == 180:
+                    im = im.transpose(Transpose.ROTATE_180)
+                elif rotation == 270:
+                    im = im.transpose(Transpose.ROTATE_270)
+                if rotation % 180 == 90:
+                    page_dpi = page_dpi.flip_axis()
+            im.save(fspath(output_file), dpi=page_dpi)
+    except UnidentifiedImageError:
+        log.error(
+            f"Ghostscript (using {raster_device} at {raster_dpi} dpi) produced "
+            "an invalid page image file."
+        )
+        raise
 
 
 class GhostscriptFollower:
@@ -161,8 +176,7 @@ class GhostscriptFollower:
                 )
                 return
         else:
-            m = self.re_page.match(line.strip())
-            if m:
+            if self.re_page.match(line.strip()):
                 self.progressbar.update()
 
 
@@ -203,14 +217,18 @@ def generate_pdfa(
     # Older versions of Ghostscript expect a leading slash in
     # sColorConversionStrategy, newer ones should not have it. See Ghostscript
     # git commit fe1c025d.
-    strategy = ('/' + strategy) if version() < '9.19' else strategy
+    gs_version = version()
+    strategy = ('/' + strategy) if gs_version < '9.19' else strategy
 
-    if version() == '9.23':
+    if gs_version == '9.23':
         # 9.23: added JPEG passthrough as a new feature, but with a bug that
         # incorrectly formats some images. Fixed as of 9.24. So we disable this
         # feature for 9.23.
         # https://bugs.ghostscript.com/show_bug.cgi?id=699216
         compression_args.append('-dPassThroughJPEGImages=false')
+    elif gs_version == '9.56.0':
+        # 9.56.0 breaks our OCR...?
+        compression_args.append('-dNEWPDF=false')
 
     # nb no need to specify ProcessColorModel when ColorConversionStrategy
     # is set; see:

src/ocrmypdf/_exec/tesseract.py

@@ -9,13 +9,13 @@
 
 import logging
 import re
-from distutils.version import StrictVersion
 from math import pi
 from os import fspath
 from pathlib import Path
 from subprocess import PIPE, STDOUT, CalledProcessError, TimeoutExpired
 from typing import Dict, Iterator, List, Optional
 
+from packaging.version import Version
 from PIL import Image
 
 from ocrmypdf.exceptions import (
@@ -46,6 +46,13 @@ HOCR_TEMPLATE = """<?xml version="1.0" encoding="UTF-8"?>
 </html>
 """
 
+TESSERACT_THRESHOLDING_METHODS: Dict[str, int] = {
+    'auto': 0,
+    'otsu': 0,
+    'adaptive-otsu': 1,
+    'sauvola': 2,
+}
+
 
 class TesseractLoggerAdapter(logging.LoggerAdapter):
     def process(self, msg, kwargs):
@@ -53,25 +60,54 @@ class TesseractLoggerAdapter(logging.LoggerAdapter):
         return '[tesseract] %s' % (msg), kwargs
 
 
-class TesseractVersion(StrictVersion):
-
-    version_re = re.compile(
-        r'''
-        ^(\d+) \. (\d+) (\. (\d+))?  # groups: 1/major, 2/minor, 3/[skip], 4/patch
-        [-]?  # optional hyphen separator
-        (?: ((?:alpha|beta|rc|dev)\d*)? [.\-\ ]? (\d+)? )?  # 5/prerelease, 6/prerelease_num
-        (?:(?:-\d+)?-g[0-9a-f]+)?  # untagged git version
-        $
-        ''',
-        re.VERBOSE | re.ASCII,
+TESSERACT_VERSION_PATTERN = r"""
+    v?
+    (?:
+        (?:(?P<epoch>[0-9]+)!)?                           # epoch
+        (?P<release>[0-9]+(?:\.[0-9]+)*)                  # release segment
+        (?P<pre>                                          # pre-release
+            [-_\.]?
+            (?P<pre_l>(a|b|c|rc|alpha|beta|pre|preview))
+            [-_\.]?
+            (?P<pre_n>[0-9]+)?
+        )?
+        (?P<post>                                         # post release
+            (?:-(?P<post_n1>[0-9]+))
+            |
+            (?:
+                [-_\.]?
+                (?P<post_l>post|rev|r)
+                [-_\.]?
+                (?P<post_n2>[0-9]+)?
+            )
+        )?
+        (?P<dev>                                          # dev release
+            [-_\.]?
+            (?P<dev_l>dev)
+            [-_\.]?
+            (?P<dev_n>[0-9]+)?
+        )?
+        (?P<date>
+            [-_\.]
+            (?:20[0-9][0-9] [0-1][0-9] [0-3][0-9])       # yyyy mm dd
+        )?
+        (?P<gitcount>
+            [-_\.]?
+            [0-9]+
+        )?
+        (?P<gitcommit>
+            [-_\.]?
+            g[0-9a-f]{2,10}
+        )?
     )
+    (?:\+(?P<local>[a-z0-9]+(?:[-_\.][a-z0-9]+)*))?       # local version
+"""
 
-    def parse(self, vstring):
-        try:
-            super().parse(vstring)
-        except TypeError as e:
-            if 'int() argument must be a string' in str(e):
-                super().parse(vstring + '-0')
+
+class TesseractVersion(Version):
+    _regex = re.compile(
+        r"^\s*" + TESSERACT_VERSION_PATTERN + r"\s*$", re.VERBOSE | re.IGNORECASE
+    )
 
 
 def version() -> str:
@@ -87,6 +123,11 @@ def has_user_words():
     return version() >= '4.1'
 
 
+def has_thresholding():
+    """Does Tesseract have -c thresholding method capability?"""
+    return version() >= '5.0'
+
+
 def get_languages():
     def lang_error(output):
         msg = (
@@ -188,7 +229,9 @@ def get_deskew(
     except CalledProcessError as e:
         tesseract_log_output(e.stdout)
         tesseract_log_output(e.stderr)
-        if b'Empty page!!' in e.output:  # Not enough info for a skew angle
+        if b'Empty page!!' in e.output or (
+            e.output == b'' and e.returncode == 1
+        ):  # Not enough info for a skew angle - Tess 4 and 5 return different errors
             return 0.0
 
         raise SubprocessOutputError() from e
@@ -265,9 +308,11 @@ def generate_hocr(
     tessconfig: List[str],
     timeout: float,
     pagesegmode: int,
+    thresholding: int,
     user_words,
     user_patterns,
 ):
+    """Generate a hOCR file, which must be converted to PDF."""
     prefix = output_hocr.with_suffix('')
 
     args_tesseract = tess_base_args(languages, engine_mode)
@@ -275,6 +320,9 @@ def generate_hocr(
     if pagesegmode is not None:
         args_tesseract.extend(['--psm', str(pagesegmode)])
 
+    if thresholding != 0 and has_thresholding():
+        args_tesseract.extend(['-c', f'thresholding_method={thresholding}'])
+
     if user_words:
         args_tesseract.extend(['--user-words', user_words])
 
@@ -296,7 +344,7 @@ def generate_hocr(
         _generate_null_hocr(output_hocr, output_text, input_file)
     except CalledProcessError as e:
         tesseract_log_output(e.output)
-        if b'Image too large' in e.output:
+        if b'Image too large' in e.output or b'Empty page!!' in e.output:
             _generate_null_hocr(output_hocr, output_text, input_file)
             return
 
@@ -326,20 +374,15 @@ def generate_pdf(
     tessconfig: List[str],
     timeout: float,
     pagesegmode: int,
+    thresholding: int,
     user_words,
     user_patterns,
 ):
-    """Use Tesseract to render a PDF.
+    """Generate a PDF using Tesseract's internal PDF generator.
 
-    input_file -- image to analyze
-    output_pdf -- file to generate
-    output_text -- OCR text file
-    languages -- list of languages to consider
-    engine_mode -- engine mode argument for tess v4
-    tessconfig -- tesseract configuration
-    timeout -- timeout (seconds)
+    We specifically a text-only PDF which is more suitable for combining with
+    the input page.
     """
-
     args_tesseract = tess_base_args(languages, engine_mode)
 
     if pagesegmode is not None:
@@ -347,6 +390,9 @@ def generate_pdf(
 
     args_tesseract.extend(['-c', 'textonly_pdf=1'])
 
+    if thresholding != 0 and has_thresholding():
+        args_tesseract.extend(['-c', f'thresholding_method={thresholding}'])
+
     if user_words:
         args_tesseract.extend(['--user-words', user_words])
 
@@ -370,7 +416,7 @@ def generate_pdf(
         use_skip_page(output_pdf, output_text)
     except CalledProcessError as e:
         tesseract_log_output(e.output)
-        if b'Image too large' in e.output:
+        if b'Image too large' in e.output or b'Empty page!!' in e.output:
             use_skip_page(output_pdf, output_text)
             return
         raise SubprocessOutputError() from e

src/ocrmypdf/_exec/unpaper.py

@@ -13,69 +13,94 @@
 import logging
 import os
 import shlex
+from contextlib import contextmanager
 from decimal import Decimal
 from pathlib import Path
 from subprocess import PIPE, STDOUT
 from tempfile import TemporaryDirectory
-from typing import List, Optional, Tuple, Union
+from typing import Iterator, List, Optional, Tuple, Union
 
 from PIL import Image
 
 from ocrmypdf.exceptions import MissingDependencyError, SubprocessOutputError
-from ocrmypdf.subprocess import get_version
-from ocrmypdf.subprocess import run as external_run
+from ocrmypdf.subprocess import get_version, run
+
+UNPAPER_IMAGE_PIXEL_LIMIT = 256 * 1024 * 1024
 
 DecFloat = Union[Decimal, float]
 
 log = logging.getLogger(__name__)
 
 
+class UnpaperImageTooLargeError(Exception):
+    def __init__(
+        self,
+        w,
+        h,
+        message="Image with size {}x{} is too large for cleaning with 'unpaper'.",
+    ):
+        self.w = w
+        self.h = h
+        self.message = message.format(w, h)
+        super().__init__(self.message)
+
+
 def version() -> str:
     return get_version('unpaper')
 
 
-def _setup_unpaper_io(tmpdir: Path, input_file: Path) -> Tuple[Path, Path]:
+def _convert_image(im: Image.Image) -> Tuple[Image.Image, bool, str]:
     SUFFIXES = {'1': '.pbm', 'L': '.pgm', 'RGB': '.ppm'}
-    with Image.open(input_file) as im:
-        im_modified = False
-        if im.mode not in SUFFIXES:
-            log.info("Converting image to other colorspace")
-            try:
-                if im.mode == 'P' and len(im.getcolors()) == 2:
-                    im = im.convert(mode='1')
-                else:
-                    im = im.convert(mode='RGB')
-            except OSError as e:
-                raise MissingDependencyError(
-                    "Could not convert image with type " + im.mode
-                ) from e
-            else:
-                im_modified = True
+    im_modified = False
+
+    if im.mode not in SUFFIXES:
+        log.info("Converting image to other colorspace")
         try:
-            suffix = SUFFIXES[im.mode]
-        except KeyError:
+            if im.mode == 'P' and len(im.getcolors()) == 2:
+                im = im.convert(mode='1')
+            else:
+                im = im.convert(mode='RGB')
+        except OSError as e:
             raise MissingDependencyError(
-                "Failed to convert image to a supported format."
-            ) from None
-
-        if im_modified or input_file.suffix != '.pnm':
-            input_pnm = tmpdir / 'input.pnm'
-            im.save(input_pnm, format='PPM')
+                "Could not convert image with type " + im.mode
+            ) from e
         else:
-            # No changes, PNG input, just use the file we already have
-            input_pnm = input_file
-        output_pnm = tmpdir / f'output{suffix}'
-    return input_pnm, output_pnm
+            im_modified = True
+    try:
+        suffix = SUFFIXES[im.mode]
+    except KeyError:
+        raise MissingDependencyError(
+            "Failed to convert image to a supported format."
+        ) from None
+    return im, im_modified, suffix
 
 
-def run(
+@contextmanager
+def _setup_unpaper_io(input_file: Path) -> Iterator[Tuple[Path, Path, Path]]:
+    with Image.open(input_file) as im:
+        if im.width * im.height >= UNPAPER_IMAGE_PIXEL_LIMIT:
+            raise UnpaperImageTooLargeError(w=im.width, h=im.height)
+        im, im_modified, suffix = _convert_image(im)
+
+        with TemporaryDirectory() as tmpdir:
+            tmppath = Path(tmpdir)
+            if im_modified or input_file.suffix != '.pnm':
+                input_pnm = tmppath / 'input.pnm'
+                im.save(input_pnm, format='PPM')
+            else:
+                # No changes, PNG input, just use the file we already have
+                input_pnm = input_file
+
+            output_pnm = tmppath / f'output{suffix}'
+            yield input_pnm, output_pnm, tmppath
+
+
+def run_unpaper(
     input_file: Path, output_file: Path, *, dpi: DecFloat, mode_args: List[str]
 ) -> None:
     args_unpaper = ['unpaper', '-v', '--dpi', str(round(dpi, 6))] + mode_args
 
-    with TemporaryDirectory() as tmpdir:
-        input_pnm, output_pnm = _setup_unpaper_io(Path(tmpdir), input_file)
-
+    with _setup_unpaper_io(input_file) as (input_pnm, output_pnm, tmpdir):
         # To prevent any shenanigans from accepting arbitrary parameters in
         # --unpaper-args, we:
         # 1) run with cwd set to a tmpdir with only unpaper's files
@@ -84,7 +109,7 @@ def run(
         # This should ensure that a user cannot clobber some other file with
         # their unpaper arguments (whether intentionally or otherwise)
         args_unpaper.extend([os.fspath(input_pnm), os.fspath(output_pnm)])
-        external_run(
+        run(
             args_unpaper,
             close_fds=True,
             check=True,
@@ -117,7 +142,7 @@ def clean(
     *,
     dpi: DecFloat,
     unpaper_args: Optional[List[str]] = None,
-):
+) -> Path:
     default_args = [
         '--layout',
         'none',
@@ -131,4 +156,9 @@ def clean(
     ]
     if not unpaper_args:
         unpaper_args = default_args
-    run(input_file, output_file, dpi=dpi, mode_args=unpaper_args)
+    try:
+        run_unpaper(input_file, output_file, dpi=dpi, mode_args=unpaper_args)
+        return output_file
+    except UnpaperImageTooLargeError as e:
+        log.warning(str(e))
+        return input_file

src/ocrmypdf/_pipeline.py

@@ -32,12 +32,19 @@ from ocrmypdf.exceptions import (
     PriorOcrFoundError,
     UnsupportedImageFormatError,
 )
-from ocrmypdf.helpers import Resolution, safe_symlink
+from ocrmypdf.helpers import IMG2PDF_KWARGS, Resolution, safe_symlink
 from ocrmypdf.hocrtransform import HocrTransform
 from ocrmypdf.optimize import optimize
 from ocrmypdf.pdfa import generate_pdfa_ps
 from ocrmypdf.pdfinfo import Colorspace, Encoding, PdfInfo
 
+# Remove this workaround when we require Pillow >= 10
+try:
+    BICUBIC = Image.Resampling.BICUBIC  # type: ignore
+except AttributeError:
+    # Pillow 9 shim
+    BICUBIC = Image.BICUBIC  # type: ignore
+
 log = logging.getLogger(__name__)
 
 VECTOR_PAGE_DPI = 400
@@ -98,8 +105,8 @@ def triage_image_file(input_file, output_file, options):
             img2pdf.convert(
                 os.fspath(input_file),
                 layout_fun=layout_fun,
-                with_pdfrw=False,
                 outputstream=outf,
+                **IMG2PDF_KWARGS,
             )
         log.info("Successfully converted to PDF, processing...")
     except img2pdf.ImageOpenError as e:
@@ -484,7 +491,9 @@ def preprocess_deskew(input_file: Path, page_context: PageContext):
         # According to Pillow docs, .rotate() will automatically use Image.NEAREST
         # resampling if image is mode '1' or 'P'
         deskewed = im.rotate(
-            deskew_angle_degrees, resample=Image.BICUBIC, fillcolor='white'
+            deskew_angle_degrees,
+            resample=BICUBIC,
+            fillcolor=ImageColor.getcolor('white', mode=im.mode),
         )
         deskewed.save(output_file, dpi=dpi)
 
@@ -494,13 +503,12 @@ def preprocess_deskew(input_file: Path, page_context: PageContext):
 def preprocess_clean(input_file: Path, page_context: PageContext):
     output_file = page_context.get_path('pp_clean.png')
     dpi = get_page_square_dpi(page_context.pageinfo, page_context.options)
-    unpaper.clean(
+    return unpaper.clean(
         input_file,
         output_file,
         dpi=dpi.x,
         unpaper_args=page_context.options.unpaper_args,
     )
-    return output_file
 
 
 def create_ocr_image(image: Path, page_context: PageContext):
@@ -613,7 +621,7 @@ def create_pdf_page_from_image(
 
         layout_fun = img2pdf.get_layout_fun(pagesize)
         img2pdf.convert(
-            imfile, with_pdfrw=False, layout_fun=layout_fun, outputstream=pdf
+            imfile, layout_fun=layout_fun, outputstream=pdf, **IMG2PDF_KWARGS
         )
         log.debug('convert done')
 

src/ocrmypdf/_plugin_manager.py

@@ -11,7 +11,7 @@ import importlib.util
 import pkgutil
 import sys
 from pathlib import Path
-from typing import List, Tuple, Union
+from typing import List, Sequence, Tuple, Union
 
 import pluggy
 
@@ -110,7 +110,7 @@ def get_plugin_manager(plugins: List[Union[str, Path]], builtins=True):
 
 
 def get_parser_options_plugins(
-    args,
+    args: Sequence[str],
 ) -> Tuple[argparse.ArgumentParser, argparse.Namespace, pluggy.PluginManager]:
     pre_options, _unused = plugins_only_parser.parse_known_args(args=args)
     plugin_manager = get_plugin_manager(pre_options.plugins)

src/ocrmypdf/_sync.py

@@ -5,15 +5,18 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 
+import argparse
 import logging
 import logging.handlers
 import os
 import sys
 import threading
+from concurrent.futures.process import BrokenProcessPool
+from concurrent.futures.thread import BrokenThreadPool
 from functools import partial
 from pathlib import Path
 from tempfile import mkdtemp
-from typing import List, NamedTuple, Optional, Tuple
+from typing import List, NamedTuple, Optional, Tuple, cast
 
 import PIL
 
@@ -46,7 +49,7 @@ from ocrmypdf._pipeline import (
     triage,
     validate_pdfinfo_options,
 )
-from ocrmypdf._plugin_manager import get_plugin_manager
+from ocrmypdf._plugin_manager import OcrmypdfPluginManager, get_plugin_manager
 from ocrmypdf._validation import (
     check_requested_output_file,
     create_input_file,
@@ -165,7 +168,7 @@ def make_intermediate_images(
     return ocr_image, preprocess_out
 
 
-def exec_page_sync(page_context: PageContext):
+def exec_page_sync(page_context: PageContext) -> PageResult:
     options = page_context.options
     tls.pageno = page_context.pageno + 1
 
@@ -223,7 +226,7 @@ def exec_page_sync(page_context: PageContext):
     )
 
 
-def post_process(pdf_file, context: PdfContext, executor: Executor):
+def post_process(pdf_file: Path, context: PdfContext, executor: Executor) -> Path:
     pdf_out = pdf_file
     if context.options.output_type.startswith('pdfa'):
         ps_stub_out = generate_postscript_stub(context)
@@ -233,7 +236,7 @@ def post_process(pdf_file, context: PdfContext, executor: Executor):
     return optimize_pdf(pdf_out, context, executor)
 
 
-def worker_init(max_pixels: int):
+def worker_init(max_pixels: int) -> None:
     # In Windows, child process will not inherit our change to this value in
     # the parent process, so ensure workers get it set. Not needed when running
     # threaded, but harmless to set again.
@@ -241,7 +244,7 @@ def worker_init(max_pixels: int):
     pikepdf_enable_mmap()
 
 
-def exec_concurrent(context: PdfContext, executor: Executor):
+def exec_concurrent(context: PdfContext, executor: Executor) -> None:
     """Execute the pipeline concurrently"""
 
     # Run exec_page_sync on every page context
@@ -302,7 +305,9 @@ def exec_concurrent(context: PdfContext, executor: Executor):
         copy_final(pdf, options.output_file, context)
 
 
-def configure_debug_logging(log_filename: Path, prefix: str = ''):
+def configure_debug_logging(
+    log_filename: Path, prefix: str = ''
+) -> logging.FileHandler:
     """
     Create a debug log file at a specified location.
 
@@ -321,7 +326,12 @@ def configure_debug_logging(log_filename: Path, prefix: str = ''):
     return log_file_handler
 
 
-def run_pipeline(options, *, plugin_manager, api=False):
+def run_pipeline(
+    options: argparse.Namespace,
+    *,
+    plugin_manager: Optional[OcrmypdfPluginManager],
+    api: bool = False,
+) -> ExitCode:
     # Any changes to options will not take effect for options that are already
     # bound to function parameters in the pipeline. (For example
     # options.input_file, options.pdf_renderer are already bound.)
@@ -380,7 +390,7 @@ def run_pipeline(options, *, plugin_manager, api=False):
             hasattr(options.output_file, 'writable') and options.output_file.writable()
         ):
             log.info("Output written to stream")
-        elif samefile(options.output_file, os.devnull):
+        elif samefile(options.output_file, Path(os.devnull)):
             pass  # Say nothing when sending to dev null
         else:
             if options.output_type.startswith('pdfa'):
@@ -407,6 +417,7 @@ def run_pipeline(options, *, plugin_manager, api=False):
             log.error("KeyboardInterrupt")
         return ExitCode.ctrl_c
     except (ExitCodeException if not api else NeverRaise) as e:
+        e = cast(ExitCodeException, e)
         if options.verbose >= 1:
             log.exception("ExitCodeException")
         elif str(e):
@@ -421,6 +432,17 @@ def run_pipeline(options, *, plugin_manager, api=False):
             "image pixel limit."
         )
         return ExitCode.other_error
+    except (
+        BrokenProcessPool if not api else NeverRaise,
+        BrokenThreadPool if not api else NeverRaise,
+    ) as e:
+        log.exception(
+            "A worker process was terminated unexpectedly. This is known to occur if "
+            "processing your file takes all available swap space and RAM. It may "
+            "help to try again with a smaller number of jobs, using the --jobs "
+            "argument."
+        )
+        return ExitCode.child_process_error
     except (Exception if not api else NeverRaise):  # pylint: disable=broad-except
         log.exception("An exception occurred while executing the pipeline")
         return ExitCode.other_error

src/ocrmypdf/_validation.py

@@ -41,7 +41,7 @@ log = logging.getLogger(__name__)
 
 
 def check_platform():
-    if os.name == 'nt' and sys.maxsize <= 2 ** 32:  # pragma: no cover
+    if os.name == 'nt' and sys.maxsize <= 2**32:  # pragma: no cover
         # 32-bit interpreter on Windows
         log.error(
             "You are running OCRmyPDF in a 32-bit (x86) Python interpreter."
@@ -78,11 +78,11 @@ def check_options_output(options):
             "`--pdf-renderer auto` (the default) to avoid this issue."
         )
 
-    if options.output_type == 'none' and options.output_file != os.devnull:
+    if options.output_type == 'none' and options.output_file not in (os.devnull, '-'):
         raise BadArgsError(
-            "Since you specified `--pdf-renderer none`, the output file "
+            "Since you specified `--output-type none`, the output file "
             f"{options.output_file} cannot be produced. Set the output file to "
-            f"{os.devnull} to suppress this message."
+            f"`-` to suppress this message."
         )
 
     lossless_reconstruction = False

src/ocrmypdf/api.py

@@ -242,6 +242,7 @@ def ocr(  # pylint: disable=unused-argument
     tesseract_config: Iterable[str] = None,
     tesseract_pagesegmode: int = None,
     tesseract_oem: int = None,
+    tesseract_thresholding: int = None,
     pdf_renderer=None,
     tesseract_timeout: float = None,
     rotate_pages_threshold: float = None,

src/ocrmypdf/builtin_plugins/concurrency.py

@@ -19,6 +19,7 @@ import queue
 import signal
 import sys
 import threading
+from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed
 from contextlib import suppress
 from multiprocessing.pool import Pool, ThreadPool
 from typing import Callable, Iterable, Type, Union
@@ -30,7 +31,7 @@ from ocrmypdf._logging import TqdmConsole
 from ocrmypdf.exceptions import InputFileError
 from ocrmypdf.helpers import remove_all_log_handlers
 
-ProcessPool = Pool
+FuturesExecutorClass = Union[Type[ThreadPoolExecutor], Type[ProcessPoolExecutor]]
 Queue = Union[multiprocessing.Queue, queue.Queue]
 UserInit = Callable[[], None]
 WorkerInit = Callable[[Queue, UserInit, int], None]
@@ -110,11 +111,11 @@ class StandardExecutor(Executor):
     ):
         if use_threads:
             log_queue: Queue = queue.Queue(-1)
-            pool_class: Type[Pool] = ThreadPool
+            executor_class: FuturesExecutorClass = ThreadPoolExecutor
             initializer: WorkerInit = thread_init
         else:
             log_queue = multiprocessing.Queue(-1)
-            pool_class = ProcessPool
+            executor_class = ProcessPoolExecutor
             initializer = process_init
 
         # Regardless of whether we use_threads for worker processes, the log_listener
@@ -123,39 +124,35 @@ class StandardExecutor(Executor):
         listener = threading.Thread(target=log_listener, args=(log_queue,))
         listener.start()
 
-        with self.pbar_class(**tqdm_kwargs) as pbar:
-            pool = pool_class(
-                processes=max_workers,
-                initializer=initializer,
-                initargs=(log_queue, worker_initializer, logging.getLogger("").level),
-            )
+        with self.pbar_class(**tqdm_kwargs) as pbar, executor_class(
+            max_workers=max_workers,
+            initializer=initializer,
+            initargs=(log_queue, worker_initializer, logging.getLogger("").level),
+        ) as executor:
+            futures = [executor.submit(task, args) for args in task_arguments]
             try:
-                results = pool.imap_unordered(task, task_arguments)
-                for result in results:
-                    if task_finished:
-                        task_finished(result, pbar)
-                    else:
-                        pbar.update()
+                for future in as_completed(futures):
+                    result = future.result()
+                    task_finished(result, pbar)
             except KeyboardInterrupt:
                 # Terminate pool so we exit instantly
-                pool.terminate()
-                # Don't try listener.join() here, will deadlock
+                executor.shutdown(wait=False, cancel_futures=True)
                 raise
             except Exception:
                 if not os.environ.get("PYTEST_CURRENT_TEST", ""):
-                    # Unless inside pytest, exit immediately because no one wants
-                    # to wait for child processes to finalize results that will be
-                    # thrown away. Inside pytest, we want child processes to exit
-                    # cleanly so that they output an error messages or coverage data
-                    # we need from them.
-                    pool.terminate()
+                    # Normally we shutdown without waiting for other child workers
+                    # on error, because there is no point in waiting for them. Their
+                    # results will be discard. But if the condition above is True,
+                    # then we are running in pytest, and we want everything to exit
+                    # as cleanly as possible so that we get good error messages.
+                    executor.shutdown(wait=False, cancel_futures=True)
                 raise
             finally:
                 # Terminate log listener
                 log_queue.put_nowait(None)
-                pool.close()
-                pool.join()
 
+        # When the above succeeds, wait for the listener thread to exit. (If
+        # an exception occurs, we don't try to join, in case it deadlocks.)
         listener.join()
 
 

src/ocrmypdf/builtin_plugins/ghostscript.py

@@ -18,13 +18,13 @@ log = logging.getLogger(__name__)
 
 @hookimpl
 def check_options(options):
-    gs_version = ghostscript.version()
     check_external_program(
         program='gs',
         package='ghostscript',
-        version_checker=gs_version,
+        version_checker=ghostscript.version,
         need_version='9.15',  # limited by Travis CI / Ubuntu 14.04 backports
     )
+    gs_version = ghostscript.version()
     if gs_version in ('9.24', '9.51'):
         raise MissingDependencyError(
             f"Ghostscript {gs_version} contains serious regressions and is not "

src/ocrmypdf/builtin_plugins/tesseract_ocr.py

@@ -10,7 +10,7 @@ import os
 
 from ocrmypdf import hookimpl
 from ocrmypdf._exec import tesseract
-from ocrmypdf.cli import numeric
+from ocrmypdf.cli import numeric, str_to_int
 from ocrmypdf.helpers import clamp
 from ocrmypdf.pluginspec import OcrEngine
 from ocrmypdf.subprocess import check_external_program
@@ -43,13 +43,27 @@ def add_options(parser):
         metavar='MODE',
         choices=range(0, 4),
         help=(
-            "Set Tesseract 4.0 OCR engine mode: "
+            "Set Tesseract 4.0+ OCR engine mode: "
             "0 - original Tesseract only; "
             "1 - neural nets LSTM only; "
             "2 - Tesseract + LSTM; "
             "3 - default."
         ),
     )
+    tess.add_argument(
+        '--tesseract-thresholding',
+        action='store',
+        type=str_to_int(tesseract.TESSERACT_THRESHOLDING_METHODS),
+        default='auto',
+        metavar='METHOD',
+        help=(
+            "Set Tesseract 5.0+ input image thresholding mode. This may improve OCR "
+            "results on low quality images or those that contain high constrast color. "
+            "legacy-otsu is the Tesseract default; adaptive-otsu is an improved Otsu "
+            "algorithm with improved sort for background color changes; sauvola is "
+            "based on local standard deviation."
+        ),
+    )
     tess.add_argument(
         '--tesseract-timeout',
         default=180.0,
@@ -89,8 +103,14 @@ def check_options(options):
 
     if not tesseract.has_user_words() and (options.user_words or options.user_patterns):
         log.warning(
-            "Tesseract 4.0 ignores --user-words and --user-patterns, so these "
-            "arguments have no effect."
+            "Tesseract 4.0 (which you have installed) ignores --user-words and "
+            "--user-patterns, so these arguments have no effect."
+        )
+    if not tesseract.has_thresholding() and options.tesseract_thresholding != 0:
+        log.warning(
+            "The installed version of Tesseract does not support changes to its "
+            "thresholding method. The --tesseract-threshold argument will be "
+            "ignored."
         )
     if options.tesseract_pagesegmode in (0, 2):
         log.warning(
@@ -162,6 +182,7 @@ class TesseractOcrEngine(OcrEngine):
             tessconfig=options.tesseract_config,
             timeout=options.tesseract_timeout,
             pagesegmode=options.tesseract_pagesegmode,
+            thresholding=options.tesseract_thresholding,
             user_words=options.user_words,
             user_patterns=options.user_patterns,
         )
@@ -177,6 +198,7 @@ class TesseractOcrEngine(OcrEngine):
             tessconfig=options.tesseract_config,
             timeout=options.tesseract_timeout,
             pagesegmode=options.tesseract_pagesegmode,
+            thresholding=options.tesseract_thresholding,
             user_words=options.user_words,
             user_patterns=options.user_patterns,
         )

src/ocrmypdf/cli.py

@@ -6,12 +6,12 @@
 
 
 import argparse
-from typing import Any, Callable, Optional, TypeVar
+from typing import Any, Callable, Mapping, Optional, TypeVar
 
 from ocrmypdf._version import PROGRAM_NAME as _PROGRAM_NAME
 from ocrmypdf._version import __version__ as _VERSION
 
-T = TypeVar('T')
+T = TypeVar('T', int, float)
 
 
 def numeric(
@@ -21,17 +21,32 @@ def numeric(
     min_ = basetype(min_) if min_ is not None else None
     max_ = basetype(max_) if max_ is not None else None
 
-    def _numeric(string):
-        value = basetype(string)
+    def _numeric(s: str) -> T:
+        value = basetype(s)
         if (min_ is not None and value < min_) or (max_ is not None and value > max_):
-            msg = f"{string!r} not in valid range {(min_, max_)!r}"
-            raise argparse.ArgumentTypeError(msg)
+            raise argparse.ArgumentTypeError(
+                f"{s!r} not in valid range {(min_, max_)!r}"
+            )
         return value
 
     _numeric.__name__ = basetype.__name__
     return _numeric
 
 
+def str_to_int(mapping: Mapping[str, int]):
+    """Accept text on command line and convert to integer."""
+
+    def _str_to_int(s: str) -> int:
+        try:
+            return mapping[s]
+        except KeyError:
+            raise argparse.ArgumentTypeError(
+                f"{s!r} must be one of: {', '.join(mapping.keys())}"
+            )
+
+    return _str_to_int
+
+
 class ArgumentParser(argparse.ArgumentParser):
     """Override parser's default behavior of calling sys.exit()
 
@@ -127,7 +142,7 @@ Online documentation is located at:
         'output_file',
         metavar="output_pdf",
         help="Output searchable PDF file (or '-' to write to standard output). "
-        "Existing files will be ovewritten. If same as input file, the "
+        "Existing files will be overwritten. If same as input file, the "
         "input file will be updated only if processing is successful.",
     )
     parser.add_argument(
@@ -213,7 +228,13 @@ Online documentation is located at:
         help=argparse.SUPPRESS,
     )
     jobcontrol.add_argument(
-        '--use-threads', action='store_true', help=argparse.SUPPRESS
+        '--use-threads', action='store_true', default=True, help=argparse.SUPPRESS
+    )
+    jobcontrol.add_argument(
+        '--no-use-threads',
+        action='store_false',
+        dest='use_threads',
+        help=argparse.SUPPRESS,
     )
 
     metadata = parser.add_argument_group(

src/ocrmypdf/helpers.py

@@ -19,10 +19,21 @@ from math import isclose, isfinite
 from pathlib import Path
 from typing import Any, Sequence
 
+import img2pdf
 import pikepdf
+from packaging.version import Version
 
 log = logging.getLogger(__name__)
 
+if Version(img2pdf.__version__) < Version('0.4.0'):
+    IMG2PDF_KWARGS = dict(without_pdfw=True)
+elif Version(img2pdf.__version__) < Version('0.4.3'):
+    IMG2PDF_KWARGS = dict(engine=img2pdf.Engine.pikepdf)
+else:
+    IMG2PDF_KWARGS = dict(
+        engine=img2pdf.Engine.pikepdf, rotation=img2pdf.Rotation.ifvalid
+    )
+
 
 class Resolution(namedtuple('Resolution', ('x', 'y'))):
     """The number of pixels per inch in each 2D direction.
@@ -209,11 +220,19 @@ def check_pdf(input_file: Path) -> bool:
     else:
         with pdf:
             messages = pdf.check()
+            success = True
             for msg in messages:
                 if 'error' in msg.lower():
                     log.error(msg)
+                    success = False
+                elif (
+                    "/DecodeParms: operation for dictionary attempted on object "
+                    "of type null" in msg
+                ):
+                    pass  # Ignore/spurious warning
                 else:
                     log.warning(msg)
+                    success = False
 
             sio = StringIO()
             linearize_msgs = ''
@@ -228,7 +247,7 @@ def check_pdf(input_file: Path) -> bool:
                 if linearize_msgs:
                     log.warning(linearize_msgs)
 
-            if not messages and not linearize_msgs:
+            if success and not linearize_msgs:
                 return True
             return False
 

src/ocrmypdf/optimize.py

@@ -8,6 +8,7 @@
 import logging
 import sys
 import tempfile
+import threading
 from collections import defaultdict
 from os import fspath
 from pathlib import Path
@@ -23,6 +24,7 @@ from typing import (
     Sequence,
     Tuple,
 )
+from zlib import compress
 
 import img2pdf
 from pikepdf import (
@@ -31,6 +33,7 @@ from pikepdf import (
     Object,
     ObjectStreamMode,
     Pdf,
+    PdfError,
     PdfImage,
     Stream,
     UnsupportedImageTypeError,
@@ -41,7 +44,7 @@ from ocrmypdf._concurrent import Executor, SerialExecutor
 from ocrmypdf._exec import jbig2enc, pngquant
 from ocrmypdf._jobcontext import PdfContext
 from ocrmypdf.exceptions import OutputFileAccessError
-from ocrmypdf.helpers import safe_symlink
+from ocrmypdf.helpers import IMG2PDF_KWARGS, safe_symlink
 
 log = logging.getLogger(__name__)
 
@@ -78,33 +81,49 @@ def extract_image_filter(
     if image.Subtype != Name.Image:
         return None
     if image.Length < 100:
-        log.debug(f"Skipping small image, xref {xref}")
+        log.debug(f"xref {xref}: skipping image with small stream size")
         return None
     if image.Width < 8 or image.Height < 8:  # Issue 732
-        log.debug(f"Skipping oddly sized image, xref {xref}")
+        log.debug(f"xref {xref}: skipping image with unusually small dimensions")
         return None
 
     pim = PdfImage(image)
 
     if len(pim.filter_decodeparms) > 1:
-        log.debug(f"Skipping multiply filtered image, xref {xref}")
-        return None
-    filtdp = pim.filter_decodeparms[0]
+        first_filtdp = pim.filter_decodeparms[0]
+        second_filtdp = pim.filter_decodeparms[1]
+        if (
+            len(pim.filter_decodeparms) == 2
+            and first_filtdp[0] == Name.FlateDecode
+            and first_filtdp[1].get(Name.Predictor, 1) == 1
+            and second_filtdp[0] == Name.DCTDecode
+            and not second_filtdp[1]
+        ):
+            log.debug(
+                f"xref {xref}: found image compressed as /FlateDecode /DCTDecode, "
+                "marked for JPEG optimization"
+            )
+            filtdp = pim.filter_decodeparms[1]
+        else:
+            log.debug(f"xref {xref}: skipping image with multiple compression filters")
+            return None
+    else:
+        filtdp = pim.filter_decodeparms[0]
 
     if pim.bits_per_component > 8:
-        log.debug(f"Skipping wide gamut image, xref {xref}")
+        log.debug(f"xref {xref}: skipping wide gamut image")
         return None  # Don't mess with wide gamut images
 
     if filtdp[0] == Name.JPXDecode:
-        log.debug(f"Skipping JPEG2000 image, xref {xref}")
+        log.debug(f"xref {xref}: skipping JPEG2000 image")
         return None  # Don't do JPEG2000
 
     if filtdp[0] == Name.CCITTFaxDecode and filtdp[1].get('/K', 0) >= 0:
-        log.debug(f"Skipping CCITT Group 3 image, xref {xref}")
+        log.debug(f"xref {xref}: skipping CCITT Group 3 image")
         return None  # pikepdf doesn't support Group 3 yet
 
     if Name.Decode in image:
-        log.debug(f"Skipping image with Decode table, xref {xref}")
+        log.debug(f"xref {xref}: skipping image with Decode table")
         return None  # Don't mess with custom Decode tables
 
     return pim, filtdp
@@ -172,14 +191,6 @@ def extract_image_generic(
         # jpeg_quality_estimate = 117.0 * (bytes_per_pixel ** 0.213)
         # if jpeg_quality_estimate < 65:
         #     return None
-
-        # We could get the ICC profile here, but there's no need to look at it
-        # for quality transcoding
-        # if icc:
-        #     stream = BytesIO(raw_jpeg.read_raw_bytes())
-        #     iccbytes = icc.read_bytes()
-        #     with Image.open(stream) as im:
-        #         im.save(jpg_name(root, xref), icc_profile=iccbytes)
         try:
             imgname = root / f'{xref:08d}'
             with imgname.open('wb') as f:
@@ -200,7 +211,11 @@ def extract_image_generic(
     elif not pim.indexed and pim.colorspace in pim.SIMPLE_COLORSPACES:
         # An optimization opportunity here, not currently taken, is directly
         # generating a PNG from compressed data
-        pim.as_pil_image().save(png_name(root, xref))
+        try:
+            pim.as_pil_image().save(png_name(root, xref))
+        except NotImplementedError:
+            log.warning("PDF contains an atypical image that cannot be optimized.")
+            return None
         return XrefExt(xref, '.png')
     elif (
         not pim.indexed
@@ -255,9 +270,9 @@ def extract_images(
                 # Ignore soft masks
                 smask_xref = Xref(image.SMask.objgen[0])
                 exclude_xrefs.add(smask_xref)
-                log.debug(f"Skipping image {smask_xref} because it is an SMask")
+                log.debug(f"xref {smask_xref}: skipping image because it is an SMask")
             include_xrefs.add(xref)
-            log.debug(f"Treating {xref} as an optimization candidate")
+            log.debug(f"xref {xref}: treating as an optimization candidate")
             if xref not in pageno_for_xref:
                 pageno_for_xref[xref] = pageno
 
@@ -269,7 +284,9 @@ def extract_images(
                 pike=pike, root=root, image=image, xref=xref, options=options
             )
         except Exception:  # pylint: disable=broad-except
-            log.exception(f"While extracting image xref {xref}, an error occurred")
+            log.exception(
+                f"xref {xref}: While extracting this image, an error occurred"
+            )
             errors += 1
         else:
             if result:
@@ -290,7 +307,7 @@ def extract_images_generic(
             pngs.append(xref_ext.xref)
         elif xref_ext.ext == '.jpg':
             jpegs.append(xref_ext.xref)
-    log.debug("Optimizable images: JPEGs: %s PNGs: %s", len(jpegs), len(pngs))
+    log.debug(f"Optimizable images: JPEGs: {len(jpegs)} PNGs: {len(pngs)}")
     return jpegs, pngs
 
 
@@ -302,7 +319,7 @@ def extract_images_jbig2(pike: Pdf, root: Path, options) -> Dict[int, List[XrefE
         group = pageno // options.jbig2_page_group_size
         jbig2_groups[group].append(xref_ext)
 
-    log.debug("Optimizable images: JBIG2 groups: %s", (len(jbig2_groups),))
+    log.debug(f"Optimizable images: JBIG2 groups: {len(jbig2_groups)}")
     return jbig2_groups
 
 
@@ -401,15 +418,11 @@ def convert_to_jbig2(
 def _optimize_jpeg(args: Tuple[Xref, Path, Path, int]) -> Tuple[Xref, Optional[Path]]:
     xref, in_jpg, opt_jpg, jpeg_quality = args
 
-    # This may produce a debug warning from PIL
-    # DEBUG:PIL.Image:Error closing: 'NoneType' object has no attribute
-    # 'close'.  Seems to be mostly harmless
-    # https://github.com/python-pillow/Pillow/issues/1144
     with Image.open(in_jpg) as im:
         im.save(opt_jpg, optimize=True, quality=jpeg_quality)
 
     if opt_jpg.stat().st_size > in_jpg.stat().st_size:
-        log.debug("xref %s, jpeg, made larger - skip", xref)
+        log.debug(f"xref {xref}, jpeg, made larger - skip")
         opt_jpg.unlink()
         return xref, None
     return xref, opt_jpg
@@ -436,7 +449,7 @@ def transcode_jpegs(
         use_threads=True,  # Processes are significantly slower at this task
         max_workers=options.jobs,
         tqdm_kwargs=dict(
-            desc="JPEGs",
+            desc="Recompressing JPEGs",
             total=len(jpegs),
             unit='image',
             disable=not options.progress_bar,
@@ -447,10 +460,77 @@ def transcode_jpegs(
     )
 
 
+def _find_deflatable_jpeg(
+    *, pike: Pdf, root: Path, image: Stream, xref: Xref, options
+) -> Optional[XrefExt]:
+    result = extract_image_filter(pike, root, image, xref)
+    if result is None:
+        return None
+    pim, filtdp = result
+
+    if filtdp[0] == Name.DCTDecode and not filtdp[1] and options.optimize >= 1:
+        return XrefExt(xref, '.memory')
+
+    return None
+
+
+def _deflate_jpeg(args: Tuple[Pdf, threading.Lock, Xref, int]) -> Tuple[Xref, bytes]:
+    pike, lock, xref, complevel = args
+    with lock:
+        xobj = pike.get_object(xref, 0)
+        try:
+            data = xobj.read_raw_bytes()
+        except PdfError:
+            return xref, b''
+    compdata = compress(data, complevel)
+    if len(compdata) >= len(data):
+        return xref, b''
+    return xref, compdata
+
+
+def deflate_jpegs(pike: Pdf, root: Path, options, executor: Executor) -> None:
+    jpegs = []
+    for _pageno, xref_ext in extract_images(pike, root, options, _find_deflatable_jpeg):
+        xref = xref_ext.xref
+        log.debug(f'xref {xref}: marking this JPEG as deflatable')
+        jpegs.append(xref)
+
+    complevel = 9 if options.optimize == 3 else 6
+
+    # Our calls to xobj.write() in finish() need coordination
+    lock = threading.Lock()
+
+    def deflate_args() -> Iterator:
+        for xref in jpegs:
+            yield pike, lock, xref, complevel
+
+    def finish(result, pbar):
+        xref, compdata = result
+        if len(compdata) > 0:
+            with lock:
+                xobj = pike.get_object(xref, 0)
+                xobj.write(compdata, filter=[Name.FlateDecode, Name.DCTDecode])
+        pbar.update()
+
+    executor(
+        use_threads=True,  # We're sharing the pdf directly, must use threads
+        max_workers=options.jobs,
+        tqdm_kwargs=dict(
+            desc="Deflating JPEGs",
+            total=len(jpegs),
+            unit='image',
+            disable=not options.progress_bar,
+        ),
+        task=_deflate_jpeg,
+        task_arguments=deflate_args(),
+        task_finished=finish,
+    )
+
+
 def _transcode_png(pike: Pdf, filename: Path, xref: Xref) -> bool:
     output = filename.with_suffix('.png.pdf')
     with output.open('wb') as f:
-        img2pdf.convert(fspath(filename), outputstream=f)
+        img2pdf.convert(fspath(filename), outputstream=f, **IMG2PDF_KWARGS)
 
     with Pdf.open(output) as pdf_image:
         foreign_image = next(iter(pdf_image.pages[0].images.values()))
@@ -560,6 +640,7 @@ def optimize(
 
         jpegs, pngs = extract_images_generic(pike, root, options)
         transcode_jpegs(pike, jpegs, root, options, executor)
+        deflate_jpegs(pike, root, options, executor)
         # if options.optimize >= 2:
         # Try pngifying the jpegs
         #    transcode_pngs(pike, jpegs, jpg_name, root, options)

src/ocrmypdf/pdfinfo/info.py

@@ -25,6 +25,7 @@ from typing import (
     Mapping,
     NamedTuple,
     Optional,
+    Sequence,
     Tuple,
     Union,
 )
@@ -646,8 +647,8 @@ def _pdf_pageinfo_concurrent(
     max_workers,
     check_pages,
     detailed_analysis=False,
-):
-    pages = [None] * len(pdf.pages)
+) -> Sequence[Optional['PageInfo']]:
+    pages: Sequence[Optional['PageInfo']] = [None] * len(pdf.pages)
 
     def update_pageinfo(result, pbar):
         page = result
@@ -918,17 +919,17 @@ class PdfInfo:
                     self._has_acroform = True
 
     @property
-    def pages(self):
+    def pages(self) -> Sequence[Optional[PageInfo]]:
         return self._pages
 
     @property
     def min_version(self) -> str:
         # The minimum PDF is the maximum version that any particular page needs
-        return max(page.min_version for page in self.pages)
+        return max(page.min_version for page in self.pages if page)
 
     @property
     def has_userunit(self) -> bool:
-        return any(page.userunit != 1.0 for page in self.pages)
+        return any(page.userunit != 1.0 for page in self.pages if page)
 
     @property
     def has_acroform(self) -> bool:

src/ocrmypdf/pluginspec.py

@@ -132,6 +132,7 @@ def get_progressbar_class():
     Here is how OCRmyPDF will use the progress bar:
 
     Example:
+
         pbar_class = pm.hook.get_progressbar_class()
         with pbar_class(**tqdm_kwargs) as pbar:
             ...
@@ -235,9 +236,9 @@ def filter_page_image(page: 'PageContext', image_filename: Path) -> Path:
     ``image_filename``. The hook may overwrite ``image_filename`` with a new file.
 
     The output image should preserve the same physical unit dimensions, that is
-    (width * dpi_x, height * dpi_y). That is, if the image is resized, the DPI
+    ``(width * dpi_x, height * dpi_y)``. That is, if the image is resized, the DPI
     must be adjusted by the reciprocal. If this is not preserved, the PDF page
-    will be resized and the OCR layer misaligned. OCRmyPDF does not nothing
+    will be resized and the OCR layer misaligned. OCRmyPDF does nothing
     to enforce these constraints; it is up to the plugin to do sensible things.
 
     OCRmyPDF will create the PDF page based on the image format used (unless the
@@ -399,8 +400,7 @@ def get_ocr_engine() -> OcrEngine:
     """Returns an OcrEngine to use for processing this file.
 
     The OcrEngine may be instantiated multiple times, by both the main process
-    and child process. As such, it must be obtain store any state in ``options``
-    or some common location.
+    and child process.
 
     Note:
         This is a :ref:`firstresult hook<firstresult>`.

src/ocrmypdf/subprocess/__init__.py

@@ -13,12 +13,13 @@ import re
 import sys
 from collections.abc import Mapping
 from contextlib import suppress
-from distutils.version import LooseVersion, Version
 from functools import lru_cache
 from subprocess import PIPE, STDOUT, CalledProcessError, CompletedProcess, Popen
 from subprocess import run as subprocess_run
 from typing import Callable, Optional, Type, Union
 
+from packaging.version import Version
+
 from ocrmypdf.exceptions import MissingDependencyError
 
 # pylint: disable=logging-format-interpolation
@@ -266,11 +267,11 @@ def check_external_program(
     *,
     program: str,
     package: str,
-    version_checker: Union[str, Callable],
+    version_checker: Callable,
     need_version: str,
     required_for: Optional[str] = None,
     recommended=False,
-    version_parser: Type[Version] = LooseVersion,
+    version_parser: Type[Version] = Version,
 ):
     """Check for required version of external program and raise exception if not.
 
@@ -291,7 +292,7 @@ def check_external_program(
     try:
         if callable(version_checker):
             found_version = version_checker()
-        else:
+        else:  # deprecated
             found_version = version_checker
     except (CalledProcessError, FileNotFoundError, MissingDependencyError):
         _error_missing_program(program, package, required_for, recommended)

src/ocrmypdf/subprocess/_windows.py

@@ -4,11 +4,13 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+# type: ignore
+# Non-Windows mypy now breaks when trying to typecheck winreg
+
 import logging
 import os
 import shutil
 import sys
-from distutils.version import LooseVersion
 from itertools import chain
 from pathlib import Path
 from typing import Any, Callable, Iterable, Iterator, Set, Tuple, TypeVar
@@ -18,11 +20,23 @@ try:
 except ModuleNotFoundError as e:
     raise ModuleNotFoundError("This module is for Windows only") from e
 
+
 log = logging.getLogger(__name__)
 
 T = TypeVar('T')
 
 
+def ghostscript_version_key(s: str) -> Tuple[int, int, int]:
+    """Compare Ghostscript version numbers."""
+    try:
+        release = [int(elem) for elem in s.split('.', maxsplit=3)]
+        while len(release) < 3:
+            release.append(0)
+        return (release[0], release[1], release[2])
+    except ValueError:
+        return (0, 0, 0)
+
+
 def registry_enum(
     key: winreg.HKEYType, enum_fn: Callable[[winreg.HKEYType, int], T]
 ) -> Iterator[T]:
@@ -51,7 +65,9 @@ def registry_path_ghostscript(env=None) -> Iterator[Path]:
         with winreg.OpenKey(
             winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\Artifex\GPL Ghostscript"
         ) as k:
-            latest_gs = max(registry_subkeys(k), key=LooseVersion, default='0')
+            latest_gs = max(
+                registry_subkeys(k), key=ghostscript_version_key, default=(0, 0, 0)
+            )
         with winreg.OpenKey(
             winreg.HKEY_LOCAL_MACHINE, fr"SOFTWARE\Artifex\GPL Ghostscript\{latest_gs}"
         ) as k:

tests/cache/manifest.jsonl

@@ -1,77 +1,84 @@
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000003_pp_rm_bg.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000003_pp_rm_bg.png", "stdout"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000005_pp_rm_bg.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000005_pp_rm_bg.png", "stdout"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000004_pp_rm_bg.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000004_pp_rm_bg.png", "stdout"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000005_pp_rm_bg.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000005_pp_rm_bg.png", "stdout"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000001_pp_rm_bg.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000001_pp_rm_bg.png", "stdout"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000004_pp_rm_bg.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000004_pp_rm_bg.png", "stdout"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000001_pp_rm_bg.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000001_pp_rm_bg.png", "stdout"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000006_pp_rm_bg.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000006_pp_rm_bg.png", "stdout"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000006_pp_rm_bg.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000006_pp_rm_bg.png", "stdout"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000003_ocr.png__000003_ocr_hocr__hocr__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "$TMPDIR/000003_ocr.png", "$TMPDIR/000003_ocr_hocr", "hocr", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/graph_ocred.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000003_ocr.png__000003_ocr_hocr__hocr__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "$TMPDIR/000003_ocr.png", "$TMPDIR/000003_ocr_hocr", "hocr", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000004_ocr.png__000004_ocr_hocr__hocr__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "$TMPDIR/000004_ocr.png", "$TMPDIR/000004_ocr_hocr", "hocr", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000004_ocr.png__000004_ocr_hocr__hocr__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "$TMPDIR/000004_ocr.png", "$TMPDIR/000004_ocr_hocr", "hocr", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000004_ocr.png__000004_ocr_tess__pdf__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000004_ocr.png", "$TMPDIR/000004_ocr_tess", "pdf", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000003_ocr.png__000003_ocr_tess__pdf__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000003_ocr.png", "$TMPDIR/000003_ocr_tess", "pdf", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000005_ocr.png__000005_ocr_hocr__hocr__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "$TMPDIR/000005_ocr.png", "$TMPDIR/000005_ocr_hocr", "hocr", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/skew.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000005_ocr.png__000005_ocr_hocr__hocr__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "$TMPDIR/000005_ocr.png", "$TMPDIR/000005_ocr_hocr", "hocr", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/ccitt.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000006_ocr.png__000006_ocr_hocr__hocr__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "$TMPDIR/000006_ocr.png", "$TMPDIR/000006_ocr_hocr", "hocr", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000005_ocr.png__000005_ocr_tess__pdf__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000005_ocr.png", "$TMPDIR/000005_ocr_tess", "pdf", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000006_ocr.png__000006_ocr_tess__pdf__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000006_ocr.png", "$TMPDIR/000006_ocr_tess", "pdf", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000006_ocr.png__000006_ocr_hocr__hocr__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "$TMPDIR/000006_ocr.png", "$TMPDIR/000006_ocr_hocr", "hocr", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/ccitt.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/skew.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__7__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/skew.pdf", "args": ["-l", "eng", "--psm", "7", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__7__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/skew.pdf", "args": ["-l", "eng", "--psm", "7", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000001_rasterize.png__stdout", "sourcefile": "resources/lichtenstein.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000001_rasterize.png", "stdout"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/lichtenstein.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000002_ocr.png__000002_ocr_tess__pdf__txt", "sourcefile": "resources/3small.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000002_ocr.png", "$TMPDIR/000002_ocr_tess", "pdf", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000001_rasterize.png__stdout", "sourcefile": "resources/palette.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000001_rasterize.png", "stdout"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/3small.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/lichtenstein.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000003_ocr.png__000003_ocr_tess__pdf__txt", "sourcefile": "resources/3small.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000003_ocr.png", "$TMPDIR/000003_ocr_tess", "pdf", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000001_rasterize.png__stdout", "sourcefile": "resources/jbig2.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000001_rasterize.png", "stdout"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/palette.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000001_rasterize.png__stdout", "sourcefile": "resources/ccitt.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000001_rasterize.png", "stdout"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/palette.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/jbig2.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/aspect.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/aspect.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/ccitt.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/ccitt.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__osd__--psm__0__000002_rasterize_preview.jpg__stdout", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "osd", "--psm", "0", "$TMPDIR/000002_rasterize_preview.jpg", "stdout"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__osd__--psm__0__000004_rasterize_preview.jpg__stdout", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "osd", "--psm", "0", "$TMPDIR/000004_rasterize_preview.jpg", "stdout"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__osd__--psm__0__000001_rasterize_preview.jpg__stdout", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "osd", "--psm", "0", "$TMPDIR/000001_rasterize_preview.jpg", "stdout"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__osd__--psm__0__000003_rasterize_preview.jpg__stdout", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "osd", "--psm", "0", "$TMPDIR/000003_rasterize_preview.jpg", "stdout"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__osd__--psm__0__000001_rasterize_preview.jpg__stdout", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "osd", "--psm", "0", "$TMPDIR/000001_rasterize_preview.jpg", "stdout"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__osd__--psm__0__000003_rasterize_preview.jpg__stdout", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "osd", "--psm", "0", "$TMPDIR/000003_rasterize_preview.jpg", "stdout"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__osd__--psm__0__000004_rasterize_preview.jpg__stdout", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "osd", "--psm", "0", "$TMPDIR/000004_rasterize_preview.jpg", "stdout"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__osd__--psm__0__000002_rasterize_preview.jpg__stdout", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "osd", "--psm", "0", "$TMPDIR/000002_rasterize_preview.jpg", "stdout"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/jbig2.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/jbig2.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000002_ocr.png__000002_ocr_tess__pdf__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000002_ocr.png", "$TMPDIR/000002_ocr_tess", "pdf", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000004_ocr.png__000004_ocr_hocr__hocr__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "$TMPDIR/000004_ocr.png", "$TMPDIR/000004_ocr_hocr", "hocr", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000003_ocr.png__000003_ocr_tess__pdf__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000003_ocr.png", "$TMPDIR/000003_ocr_tess", "pdf", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000004_ocr.png__000004_ocr_tess__pdf__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000004_ocr.png", "$TMPDIR/000004_ocr_tess", "pdf", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000003_ocr.png__000003_ocr_hocr__hocr__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "$TMPDIR/000003_ocr.png", "$TMPDIR/000003_ocr_hocr", "hocr", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000002_ocr.png__000002_ocr_hocr__hocr__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "$TMPDIR/000002_ocr.png", "$TMPDIR/000002_ocr_hocr", "hocr", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/2400dpi.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/poster.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__osd__--psm__0__000001_rasterize_preview.jpg__stdout", "sourcefile": "resources/poster.pdf", "args": ["-l", "osd", "--psm", "0", "$TMPDIR/000001_rasterize_preview.jpg", "stdout"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000003_rasterize.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000003_rasterize.png", "stdout"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000005_rasterize.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000005_rasterize.png", "stdout"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000004_rasterize.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000004_rasterize.png", "stdout"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000005_rasterize.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000005_rasterize.png", "stdout"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000004_rasterize.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000004_rasterize.png", "stdout"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000001_rasterize.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000001_rasterize.png", "stdout"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000001_rasterize.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000001_rasterize.png", "stdout"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000006_rasterize.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000006_rasterize.png", "stdout"]}
-{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000006_rasterize.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000006_rasterize.png", "stdout"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000003_pp_rm_bg.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000003_pp_rm_bg.png", "stdout"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000005_pp_rm_bg.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000005_pp_rm_bg.png", "stdout"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000004_pp_rm_bg.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000004_pp_rm_bg.png", "stdout"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000005_pp_rm_bg.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000005_pp_rm_bg.png", "stdout"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000001_pp_rm_bg.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000001_pp_rm_bg.png", "stdout"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000004_pp_rm_bg.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000004_pp_rm_bg.png", "stdout"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000001_pp_rm_bg.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000001_pp_rm_bg.png", "stdout"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000006_pp_rm_bg.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000006_pp_rm_bg.png", "stdout"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000006_pp_rm_bg.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000006_pp_rm_bg.png", "stdout"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000003_ocr.png__000003_ocr_hocr__hocr__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "$TMPDIR/000003_ocr.png", "$TMPDIR/000003_ocr_hocr", "hocr", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/graph_ocred.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000003_ocr.png__000003_ocr_hocr__hocr__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "$TMPDIR/000003_ocr.png", "$TMPDIR/000003_ocr_hocr", "hocr", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000004_ocr.png__000004_ocr_hocr__hocr__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "$TMPDIR/000004_ocr.png", "$TMPDIR/000004_ocr_hocr", "hocr", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000004_ocr.png__000004_ocr_hocr__hocr__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "$TMPDIR/000004_ocr.png", "$TMPDIR/000004_ocr_hocr", "hocr", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000004_ocr.png__000004_ocr_tess__pdf__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000004_ocr.png", "$TMPDIR/000004_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000003_ocr.png__000003_ocr_tess__pdf__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000003_ocr.png", "$TMPDIR/000003_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000005_ocr.png__000005_ocr_hocr__hocr__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "$TMPDIR/000005_ocr.png", "$TMPDIR/000005_ocr_hocr", "hocr", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/skew.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000005_ocr.png__000005_ocr_hocr__hocr__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "$TMPDIR/000005_ocr.png", "$TMPDIR/000005_ocr_hocr", "hocr", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/ccitt.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000006_ocr.png__000006_ocr_hocr__hocr__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "$TMPDIR/000006_ocr.png", "$TMPDIR/000006_ocr_hocr", "hocr", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000005_ocr.png__000005_ocr_tess__pdf__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000005_ocr.png", "$TMPDIR/000005_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000006_ocr.png__000006_ocr_tess__pdf__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000006_ocr.png", "$TMPDIR/000006_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000006_ocr.png__000006_ocr_hocr__hocr__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "$TMPDIR/000006_ocr.png", "$TMPDIR/000006_ocr_hocr", "hocr", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/ccitt.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/skew.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__7__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/skew.pdf", "args": ["-l", "eng", "--psm", "7", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__7__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/skew.pdf", "args": ["-l", "eng", "--psm", "7", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000001_rasterize.png__stdout", "sourcefile": "resources/lichtenstein.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000001_rasterize.png", "stdout"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/lichtenstein.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000002_ocr.png__000002_ocr_tess__pdf__txt", "sourcefile": "resources/3small.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000002_ocr.png", "$TMPDIR/000002_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000001_rasterize.png__stdout", "sourcefile": "resources/palette.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000001_rasterize.png", "stdout"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/3small.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/lichtenstein.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000003_ocr.png__000003_ocr_tess__pdf__txt", "sourcefile": "resources/3small.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000003_ocr.png", "$TMPDIR/000003_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000001_rasterize.png__stdout", "sourcefile": "resources/jbig2.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000001_rasterize.png", "stdout"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/palette.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000001_rasterize.png__stdout", "sourcefile": "resources/ccitt.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000001_rasterize.png", "stdout"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/palette.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/jbig2.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/aspect.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/aspect.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/ccitt.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/ccitt.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__osd__--psm__0__000002_rasterize_preview.jpg__stdout", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "osd", "--psm", "0", "$TMPDIR/000002_rasterize_preview.jpg", "stdout"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__osd__--psm__0__000004_rasterize_preview.jpg__stdout", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "osd", "--psm", "0", "$TMPDIR/000004_rasterize_preview.jpg", "stdout"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__osd__--psm__0__000001_rasterize_preview.jpg__stdout", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "osd", "--psm", "0", "$TMPDIR/000001_rasterize_preview.jpg", "stdout"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__osd__--psm__0__000003_rasterize_preview.jpg__stdout", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "osd", "--psm", "0", "$TMPDIR/000003_rasterize_preview.jpg", "stdout"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__osd__--psm__0__000001_rasterize_preview.jpg__stdout", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "osd", "--psm", "0", "$TMPDIR/000001_rasterize_preview.jpg", "stdout"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__osd__--psm__0__000003_rasterize_preview.jpg__stdout", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "osd", "--psm", "0", "$TMPDIR/000003_rasterize_preview.jpg", "stdout"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__osd__--psm__0__000004_rasterize_preview.jpg__stdout", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "osd", "--psm", "0", "$TMPDIR/000004_rasterize_preview.jpg", "stdout"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__osd__--psm__0__000002_rasterize_preview.jpg__stdout", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "osd", "--psm", "0", "$TMPDIR/000002_rasterize_preview.jpg", "stdout"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/jbig2.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/jbig2.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000002_ocr.png__000002_ocr_tess__pdf__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000002_ocr.png", "$TMPDIR/000002_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_hocr", "hocr", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000004_ocr.png__000004_ocr_hocr__hocr__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "$TMPDIR/000004_ocr.png", "$TMPDIR/000004_ocr_hocr", "hocr", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000003_ocr.png__000003_ocr_tess__pdf__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000003_ocr.png", "$TMPDIR/000003_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000004_ocr.png__000004_ocr_tess__pdf__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000004_ocr.png", "$TMPDIR/000004_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000003_ocr.png__000003_ocr_hocr__hocr__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "$TMPDIR/000003_ocr.png", "$TMPDIR/000003_ocr_hocr", "hocr", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000002_ocr.png__000002_ocr_hocr__hocr__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "$TMPDIR/000002_ocr.png", "$TMPDIR/000002_ocr_hocr", "hocr", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/2400dpi.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/poster.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__osd__--psm__0__000001_rasterize_preview.jpg__stdout", "sourcefile": "resources/poster.pdf", "args": ["-l", "osd", "--psm", "0", "$TMPDIR/000001_rasterize_preview.jpg", "stdout"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000003_rasterize.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000003_rasterize.png", "stdout"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000005_rasterize.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000005_rasterize.png", "stdout"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000004_rasterize.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000004_rasterize.png", "stdout"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000005_rasterize.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000005_rasterize.png", "stdout"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000004_rasterize.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000004_rasterize.png", "stdout"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000001_rasterize.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000001_rasterize.png", "stdout"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000001_rasterize.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000001_rasterize.png", "stdout"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000006_rasterize.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000006_rasterize.png", "stdout"]}
+{"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "system": "Linux", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000006_rasterize.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000006_rasterize.png", "stdout"]}
+{"tesseract_version": "4.1.1", "system": "Linux", "python": "3.9.5", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/trivial.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "4.1.1", "system": "Linux", "python": "3.9.5", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/trivial.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "4.1.1", "system": "Linux", "python": "3.9.5", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/trivial.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "4.1.1", "system": "Linux", "python": "3.9.5", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/trivial.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "4.1.1", "system": "Linux", "python": "3.9.5", "argv_slug": "__-l__eng__--oem__1__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/trivial.pdf", "args": ["-l", "eng", "--oem", "1", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "5.0.0", "system": "Linux", "python": "3.9.5", "argv_slug": "__-l__eng__thresholding_method=1__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/trivial.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "-c", "thresholding_method=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "5.0.0", "system": "Linux", "python": "3.9.5", "argv_slug": "__-l__eng__thresholding_method=2__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/trivial.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "-c", "thresholding_method=2", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}

tests/cache/trivial/__-l__eng__--oem__1__000001_ocr.png__000001_ocr_tess__pdf__txt/stderr.bin

@@ -0,0 +1 @@
+Tesseract Open Source OCR Engine v4.1.1 with Leptonica

tests/cache/trivial/__-l__eng__--oem__1__000001_ocr.png__000001_ocr_tess__pdf__txt/txt.bin

@@ -0,0 +1 @@
+

tests/cache/trivial/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/stderr.bin

@@ -0,0 +1 @@
+Tesseract Open Source OCR Engine v4.1.1 with Leptonica

tests/cache/trivial/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/txt.bin

@@ -0,0 +1 @@
+

tests/conftest.py

@@ -9,17 +9,15 @@ import os
 import platform
 import sys
 from pathlib import Path
-from subprocess import PIPE, run
+from subprocess import PIPE, CompletedProcess, run
+from typing import List
 
 import pytest
 
 from ocrmypdf import api, pdfinfo
 from ocrmypdf._exec import unpaper
 from ocrmypdf._plugin_manager import get_parser_options_plugins
-
-if sys.version_info < (3, 5):
-    print("Requires Python 3.5+")
-    sys.exit(1)
+from ocrmypdf.exceptions import ExitCode
 
 
 def is_linux():
@@ -46,36 +44,35 @@ def have_unpaper():
 
 TESTS_ROOT = Path(__file__).parent.resolve()
 PROJECT_ROOT = TESTS_ROOT
-OCRMYPDF = [sys.executable, '-m', 'ocrmypdf']
 
 
 @pytest.fixture
-def resources():
+def resources() -> Path:
     return Path(TESTS_ROOT) / 'resources'
 
 
 @pytest.fixture
-def ocrmypdf_exec():
-    return OCRMYPDF
+def ocrmypdf_exec() -> List[str]:
+    return [sys.executable, '-m', 'ocrmypdf']
 
 
 @pytest.fixture(scope="function")
-def outdir(tmp_path):
+def outdir(tmp_path) -> Path:
     return tmp_path
 
 
 @pytest.fixture(scope="function")
-def outpdf(tmp_path):
+def outpdf(tmp_path) -> Path:
     return tmp_path / 'out.pdf'
 
 
 @pytest.fixture(scope="function")
-def outtxt(tmp_path):
+def outtxt(tmp_path) -> Path:
     return tmp_path / 'out.txt'
 
 
 @pytest.fixture(scope="function")
-def no_outpdf(tmp_path):
+def no_outpdf(tmp_path) -> Path:
     """This just documents the fact that a test is not expected to produce
     output. Unfortunately an assertion failure inside a test fixture produces
     an error rather than a test failure, so no testing is done. It's up to
@@ -83,13 +80,13 @@ def no_outpdf(tmp_path):
     return tmp_path / 'no_output.pdf'
 
 
-def check_ocrmypdf(input_file, output_file, *args):
-    """Run ocrmypdf and confirmed that a valid file was created"""
-    args = [str(input_file), str(output_file)] + [
+def check_ocrmypdf(input_file: Path, output_file: Path, *args) -> Path:
+    """Run ocrmypdf and confirm that a valid plausible PDF was created."""
+    api_args = [str(input_file), str(output_file)] + [
         str(arg) for arg in args if arg is not None
     ]
 
-    _parser, options, plugin_manager = get_parser_options_plugins(args=args)
+    _parser, options, plugin_manager = get_parser_options_plugins(args=api_args)
     api.check_options(options, plugin_manager)
     result = api.run_pipeline(options, plugin_manager=plugin_manager, api=True)
 
@@ -100,44 +97,52 @@ def check_ocrmypdf(input_file, output_file, *args):
     return output_file
 
 
-def run_ocrmypdf_api(input_file, output_file, *args):
-    """Run ocrmypdf via API and let caller deal with results
+def run_ocrmypdf_api(input_file: Path, output_file: Path, *args) -> ExitCode:
+    """Run ocrmypdf via its API in-process, and let test deal with results.
 
-    Does not currently have a way to manipulate the PATH except for Tesseract.
+    This simulates calling the command line interface in a subprocess, but
+    is easier for debuggers and code coverage to follow.
+
+    Any exception raised will be trapped and converted to an exit code.
+    The return code must always be checked or the test may declare a failure
+    to be pass.
     """
 
-    args = [str(input_file), str(output_file)] + [
+    api_args = [str(input_file), str(output_file)] + [
         str(arg) for arg in args if arg is not None
     ]
-    _parser, options, plugin_manager = get_parser_options_plugins(args=args)
+    _parser, options, plugin_manager = get_parser_options_plugins(args=api_args)
 
     api.check_options(options, plugin_manager)
     return api.run_pipeline(options, plugin_manager=None, api=False)
 
 
-def run_ocrmypdf(input_file, output_file, *args, text=True):
-    "Run ocrmypdf and let caller deal with results"
+def run_ocrmypdf(
+    input_file: Path, output_file: Path, *args, text: bool = True
+) -> CompletedProcess:
+    """Run ocrmypdf in a subprocess and let test deal with results.
+
+    If an exception is thrown this fact will be returned as part of the result
+    text and return code rather than exception objects.
+    """
 
     p_args = (
-        OCRMYPDF
+        [sys.executable, '-m', 'ocrmypdf']
         + [str(arg) for arg in args if arg is not None]
         + [str(input_file), str(output_file)]
     )
 
-    env = os.environ.copy()
     p = run(
         p_args,
-        stdout=PIPE,
-        stderr=PIPE,
+        capture_output=True,
         text=text,
-        env=env,
         check=False,
     )
     # print(p.stderr)
-    return p, p.stdout, p.stderr
+    return p
 
 
-def first_page_dimensions(pdf):
+def first_page_dimensions(pdf: Path):
     info = pdfinfo.PdfInfo(pdf)
     page0 = info[0]
     return (page0.width_inches, page0.height_inches)

tests/plugins/tesseract_cache.py

@@ -50,6 +50,7 @@ import logging
 import platform
 import re
 import shutil
+import threading
 from functools import partial
 from pathlib import Path
 from subprocess import PIPE, CalledProcessError, CompletedProcess
@@ -158,7 +159,7 @@ def cached_run(options, run_args, **run_kwargs):
 
     manifest = {}
     manifest['tesseract_version'] = TesseractOcrEngine.version().replace('\n', ' ')
-    manifest['platform'] = platform.platform()
+    manifest['system'] = platform.system()
     manifest['python'] = platform.python_version()
     manifest['argv_slug'] = cache_folder.name
     manifest['sourcefile'] = str(Path(source_file).relative_to(TESTS_ROOT))
@@ -176,26 +177,40 @@ def cached_run(options, run_args, **run_kwargs):
 
 
 class CacheOcrEngine(TesseractOcrEngine):
+    # Concurrent threads (with --use-threads) might try to use different parts
+    # of the OcrEngine, so we need a lock to protect the state of patched
+    # module whenever it's patched. Should refactor ocrmypdf._exec.tesseract so that
+    # it does not to be patched at all for testing.
+    lock = threading.Lock()
+
     @staticmethod
     def get_orientation(input_file, options):
-        with patch('ocrmypdf._exec.tesseract.run', new=partial(cached_run, options)):
+        with CacheOcrEngine.lock, patch(
+            'ocrmypdf._exec.tesseract.run', new=partial(cached_run, options)
+        ):
             return TesseractOcrEngine.get_orientation(input_file, options)
 
     @staticmethod
     def get_deskew(input_file, options) -> float:
-        with patch('ocrmypdf._exec.tesseract.run', new=partial(cached_run, options)):
+        with CacheOcrEngine.lock, patch(
+            'ocrmypdf._exec.tesseract.run', new=partial(cached_run, options)
+        ):
             return TesseractOcrEngine.get_deskew(input_file, options)
 
     @staticmethod
     def generate_hocr(input_file, output_hocr, output_text, options):
-        with patch('ocrmypdf._exec.tesseract.run', new=partial(cached_run, options)):
+        with CacheOcrEngine.lock, patch(
+            'ocrmypdf._exec.tesseract.run', new=partial(cached_run, options)
+        ):
             TesseractOcrEngine.generate_hocr(
                 input_file, output_hocr, output_text, options
             )
 
     @staticmethod
     def generate_pdf(input_file, output_pdf, output_text, options):
-        with patch('ocrmypdf._exec.tesseract.run', new=partial(cached_run, options)):
+        with CacheOcrEngine.lock, patch(
+            'ocrmypdf._exec.tesseract.run', new=partial(cached_run, options)
+        ):
             TesseractOcrEngine.generate_pdf(
                 input_file, output_pdf, output_text, options
             )

tests/plugins/tesseract_simulate_oom_killer.py

@@ -0,0 +1,83 @@
+# © 2021 James R. Barlow: github.com/jbarlow83
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+# type: ignore
+
+"""Tesseract no-op plugin that simulates the OOM killer on page 4.
+
+OCRmyPDF can use a lot of memory, even that it might trigger the
+OOM killer on Linux or similar features on other platforms. We want to
+ensure we fail with an error rather than deadlock in such cases.
+
+Page 4 was chosen because of this number's association with bad luck
+in many East Asian cultures.
+"""
+
+import os
+import signal
+import sys
+from pathlib import Path
+
+from ocrmypdf import hookimpl
+
+# Ugly hack that let us use the NoopOcrEngine without setting up packaging for our
+# tests.
+# This hack also requires us to set type: ignore
+parent_file = Path(__file__).with_name('tesseract_noop.py')
+parent = compile(parent_file.read_text(), parent_file, mode='exec')
+exec(parent)
+NoopOcrEngine = locals()['NoopOcrEngine']
+
+
+class Page4Engine(NoopOcrEngine):
+    def __str__(self):
+        return f"NO-OP Page 4 {NoopOcrEngine.version()}"
+
+    @staticmethod
+    def generate_hocr(input_file: Path, output_hocr, output_text, options):
+        if input_file.stem.startswith('000004'):
+            # Suicide
+            os.kill(os.getpid(), signal.SIGKILL)
+        else:
+            return NoopOcrEngine.generate_hocr(
+                input_file, output_hocr, output_text, options
+            )
+
+    @staticmethod
+    def generate_pdf(input_file, output_pdf, output_text, options):
+        if input_file.stem.startswith('000004'):
+            # Suicide
+            os.kill(os.getpid(), signal.SIGKILL)
+        else:
+            return NoopOcrEngine.generate_pdf(
+                input_file, output_pdf, output_text, options
+            )
+
+
+@hookimpl
+def check_options(options):
+    if options.use_threads:
+        raise ValueError("I'm not compatible with use_threads")
+
+
+@hookimpl
+def get_ocr_engine():
+    return Page4Engine()

tests/test_completion.py

@@ -24,8 +24,7 @@ def test_fish():
             ['fish', '-n', 'misc/completion/ocrmypdf.fish'],
             check=True,
             encoding='utf-8',
-            stdout=PIPE,
-            stderr=PIPE,
+            capture_output=True,
         )
         assert proc.stderr == '', proc.stderr
     except FileNotFoundError:
@@ -41,8 +40,7 @@ def test_bash():
             ['bash', '-n', 'misc/completion/ocrmypdf.bash'],
             check=True,
             encoding='utf-8',
-            stdout=PIPE,
-            stderr=PIPE,
+            capture_output=True,
         )
         assert proc.stderr == '', proc.stderr
     except FileNotFoundError:

tests/test_concurrency.py

@@ -0,0 +1,26 @@
+# © 2021 James R. Barlow: github.com/jbarlow83
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+
+import pytest
+
+from ocrmypdf import ExitCode
+
+from .conftest import run_ocrmypdf_api
+
+
+@pytest.mark.skipif(True, reason="--use-threads is currently default")
+@pytest.mark.skipif(os.name == 'nt', reason="Windows doesn't have SIGKILL")
+def test_simulate_oom_killer(resources, no_outpdf):
+    exitcode = run_ocrmypdf_api(
+        resources / 'multipage.pdf',
+        no_outpdf,
+        '--force-ocr',
+        '--plugin',
+        'tests/plugins/tesseract_simulate_oom_killer.py',
+    )
+    assert exitcode == ExitCode.child_process_error

tests/test_ghostscript.py

@@ -6,11 +6,13 @@
 
 
 import logging
+import subprocess
 from decimal import Decimal
+from unittest.mock import patch
 
 import pikepdf
 import pytest
-from PIL import Image
+from PIL import Image, UnidentifiedImageError
 
 from ocrmypdf._exec.ghostscript import rasterize_pdf
 from ocrmypdf.exceptions import ExitCode
@@ -76,7 +78,7 @@ def test_rasterize_rotated(francais, outdir, caplog):
 
 
 def test_gs_render_failure(resources, outpdf):
-    p, _out, err = run_ocrmypdf(
+    p = run_ocrmypdf(
         resources / 'blank.pdf',
         outpdf,
         '--plugin',
@@ -84,12 +86,12 @@ def test_gs_render_failure(resources, outpdf):
         '--plugin',
         'tests/plugins/gs_render_failure.py',
     )
-    assert 'Casper is not a friendly ghost' in err
+    assert 'Casper is not a friendly ghost' in p.stderr
     assert p.returncode == ExitCode.child_process_error
 
 
 def test_gs_raster_failure(resources, outpdf):
-    p, _out, err = run_ocrmypdf(
+    p = run_ocrmypdf(
         resources / 'francais.pdf',
         outpdf,
         '--plugin',
@@ -97,12 +99,12 @@ def test_gs_raster_failure(resources, outpdf):
         '--plugin',
         'tests/plugins/gs_raster_failure.py',
     )
-    assert 'Ghost story archive not found' in err
+    assert 'Ghost story archive not found' in p.stderr
     assert p.returncode == ExitCode.child_process_error
 
 
 def test_ghostscript_pdfa_failure(resources, outpdf):
-    p, _out, _err = run_ocrmypdf(
+    p = run_ocrmypdf(
         resources / 'francais.pdf',
         outpdf,
         '--plugin',
@@ -124,3 +126,20 @@ def test_ghostscript_feature_elision(resources, outpdf):
         '--plugin',
         'tests/plugins/gs_feature_elision.py',
     )
+
+
+def test_rasterize_pdf_errors(resources, no_outpdf, caplog):
+    with patch('ocrmypdf._exec.ghostscript.run') as mock:
+        # ghostscript can produce
+        mock.return_value = subprocess.CompletedProcess(
+            ['fakegs'], returncode=0, stdout=b'', stderr=b'error this is an error'
+        )
+        with pytest.raises(UnidentifiedImageError):
+            rasterize_pdf(
+                resources / 'francais.pdf',
+                no_outpdf,
+                raster_device='pngmono',
+                raster_dpi=Resolution(100, 100),
+            )
+        assert "this is an error" in caplog.text
+        assert "invalid page image file" in caplog.text

tests/test_main.py

@@ -203,7 +203,7 @@ def test_force_ocr_on_pdf_with_no_images(resources, no_outpdf):
     # As a correctness test, make sure that --force-ocr on a PDF with no
     # content still triggers tesseract. If tesseract crashes, then it was
     # called.
-    p, _, _ = run_ocrmypdf(
+    p = run_ocrmypdf(
         resources / 'blank.pdf',
         no_outpdf,
         '--force-ocr',
@@ -241,7 +241,7 @@ def test_german(resources, outdir):
 
 
 def test_klingon(resources, outpdf):
-    p, _, _ = run_ocrmypdf(resources / 'francais.pdf', outpdf, '-l', 'klz')
+    p = run_ocrmypdf(resources / 'francais.pdf', outpdf, '-l', 'klz')
     assert p.returncode == ExitCode.missing_dependency
 
 
@@ -316,9 +316,45 @@ def test_pagesegmode(renderer, resources, outpdf):
     )
 
 
+def test_tesseract_oem(resources, outpdf):
+    check_ocrmypdf(
+        resources / 'trivial.pdf',
+        outpdf,
+        '--tesseract-oem',
+        '1',
+        '--plugin',
+        'tests/plugins/tesseract_cache.py',
+    )
+
+
+@pytest.mark.parametrize('value', ['auto', 'otsu', 'adaptive-otsu', 'sauvola'])
+def test_tesseract_thresholding(value, resources, outpdf):
+    check_ocrmypdf(
+        resources / 'trivial.pdf',
+        outpdf,
+        '--tesseract-thresholding',
+        value,
+        '--plugin',
+        'tests/plugins/tesseract_cache.py',
+    )
+
+
+@pytest.mark.parametrize('value', ['abcxyz'])
+def test_tesseract_thresholding_invalid(value, resources, no_outpdf):
+    with pytest.raises(SystemExit, match='2'):
+        run_ocrmypdf_api(
+            resources / 'trivial.pdf',
+            no_outpdf,
+            '--tesseract-thresholding',
+            value,
+            '--plugin',
+            'tests/plugins/tesseract_cache.py',
+        )
+
+
 @pytest.mark.parametrize('renderer', RENDERERS)
 def test_tesseract_crash(renderer, resources, no_outpdf):
-    p, _, err = run_ocrmypdf(
+    p = run_ocrmypdf(
         resources / 'ccitt.pdf',
         no_outpdf,
         '-v',
@@ -330,11 +366,11 @@ def test_tesseract_crash(renderer, resources, no_outpdf):
     )
     assert p.returncode == ExitCode.child_process_error
     assert not no_outpdf.exists()
-    assert "SubprocessOutputError" in err
+    assert "SubprocessOutputError" in p.stderr
 
 
 def test_tesseract_crash_autorotate(resources, no_outpdf):
-    p, out, err = run_ocrmypdf(
+    p = run_ocrmypdf(
         resources / 'ccitt.pdf',
         no_outpdf,
         '-r',
@@ -343,9 +379,9 @@ def test_tesseract_crash_autorotate(resources, no_outpdf):
     )
     assert p.returncode == ExitCode.child_process_error
     assert not no_outpdf.exists()
-    assert "uncaught exception" in err
-    print(out)
-    print(err)
+    assert "uncaught exception" in p.stderr
+    print(p.stdout)
+    print(p.stderr)
 
 
 @pytest.mark.parametrize('renderer', RENDERERS)
@@ -365,7 +401,7 @@ def test_tesseract_image_too_big(renderer, resources, outpdf):
 
 
 def test_algo4(resources, outpdf):
-    p, _, _ = run_ocrmypdf(
+    p = run_ocrmypdf(
         resources / 'encrypted_algo4.pdf',
         outpdf,
         '--plugin',
@@ -435,7 +471,7 @@ def test_destination_not_writable(resources, outdir):
     protected_file = outdir / 'protected.pdf'
     protected_file.touch()
     protected_file.chmod(0o400)  # Read-only
-    p, _out, _err = run_ocrmypdf(
+    p = run_ocrmypdf(
         resources / 'jbig2.pdf',
         protected_file,
         '--plugin',
@@ -476,7 +512,7 @@ THIS FILE IS INVALID
 '''
         )
 
-    p, _out, err = run_ocrmypdf(
+    p = run_ocrmypdf(
         resources / 'ccitt.pdf',
         outdir / 'out.pdf',
         '--pdf-renderer',
@@ -485,8 +521,8 @@ THIS FILE IS INVALID
         cfg_file,
     )
     assert (
-        "parameter not found" in err.lower()
-        or "error occurred while parsing" in err.lower()
+        "parameter not found" in p.stderr.lower()
+        or "error occurred while parsing" in p.stderr.lower()
     ), "No error message"
     assert p.returncode == ExitCode.invalid_config
 
@@ -594,8 +630,7 @@ def test_compression_preserved(ocrmypdf_exec, resources, image, outpdf):
         ]
         p = run(
             p_args,
-            stdout=PIPE,
-            stderr=PIPE,
+            capture_output=True,
             stdin=input_stream,
             text=True,
             check=False,
@@ -655,8 +690,7 @@ def test_compression_changed(ocrmypdf_exec, resources, image, compression, outpd
         ]
         p = run(
             p_args,
-            stdout=PIPE,
-            stderr=PIPE,
+            capture_output=True,
             stdin=input_stream,
             text=True,
             check=False,
@@ -745,15 +779,13 @@ def test_pdfa_n(pdfa_level, resources, outpdf):
 
 
 def test_decompression_bomb_error(resources, outpdf):
-    p, _out, err = run_ocrmypdf(resources / 'hugemono.pdf', outpdf)
-    assert 'decompression bomb' in err and '--max-image-mpixels' in err
+    p = run_ocrmypdf(resources / 'hugemono.pdf', outpdf)
+    assert 'decompression bomb' in p.stderr and '--max-image-mpixels' in p.stderr
 
 
 @pytest.mark.slow
 def test_decompression_bomb_succeeds(resources, outpdf):
-    p, _out, err = run_ocrmypdf(
-        resources / 'hugemono.pdf', outpdf, '--max-image-mpixels', '2000'
-    )
+    p = run_ocrmypdf(resources / 'hugemono.pdf', outpdf, '--max-image-mpixels', '2000')
     assert p.returncode == 0
 
 
@@ -782,7 +814,7 @@ def test_text_curves(resources, outpdf):
 
 
 def test_output_is_dir(resources, outdir):
-    p, _out, err = run_ocrmypdf(
+    p = run_ocrmypdf(
         resources / 'trivial.pdf',
         outdir,
         '--force-ocr',
@@ -790,28 +822,28 @@ def test_output_is_dir(resources, outdir):
         'tests/plugins/tesseract_noop.py',
     )
     assert p.returncode == ExitCode.file_access_error
-    assert 'is not a writable file' in err
+    assert 'is not a writable file' in p.stderr
 
 
 @pytest.mark.skipif(os.name == 'nt', reason="symlink needs admin permissions")
 def test_output_is_symlink(resources, outdir):
     sym = Path(outdir / 'this_is_a_symlink')
     sym.symlink_to(outdir / 'out.pdf')
-    p, _out, err = run_ocrmypdf(
+    p = run_ocrmypdf(
         resources / 'trivial.pdf',
         sym,
         '--force-ocr',
         '--plugin',
         'tests/plugins/tesseract_noop.py',
     )
-    assert p.returncode == ExitCode.ok, err
+    assert p.returncode == ExitCode.ok, p.stderr
     assert (outdir / 'out.pdf').stat().st_size > 0, 'target file not created'
 
 
 def test_livecycle(resources, no_outpdf):
-    p, _, err = run_ocrmypdf(resources / 'livecycle.pdf', no_outpdf)
+    p = run_ocrmypdf(resources / 'livecycle.pdf', no_outpdf)
 
-    assert p.returncode == ExitCode.input_file, err
+    assert p.returncode == ExitCode.input_file, p.stderr
 
 
 def test_version_check():
@@ -868,7 +900,7 @@ def test_image_dpi_not_image(caplog, resources, outpdf):
 
 
 def test_outputtype_none_bad_setup(resources, outpdf):
-    p, _out, err = run_ocrmypdf(
+    p = run_ocrmypdf(
         resources / 'trivial.pdf',
         outpdf,
         '--output-type=none',
@@ -876,13 +908,13 @@ def test_outputtype_none_bad_setup(resources, outpdf):
         'tests/plugins/tesseract_noop.py',
     )
     assert p.returncode == ExitCode.bad_args
-    assert 'Set the output file to' in err
+    assert 'Set the output file to' in p.stderr
 
 
 def test_outputtype_none(resources, outtxt):
-    p, _out, err = run_ocrmypdf(
+    p = run_ocrmypdf(
         resources / 'trivial.pdf',
-        os.devnull,
+        '-',
         '--output-type=none',
         '--sidecar',
         outtxt,
@@ -890,4 +922,3 @@ def test_outputtype_none(resources, outtxt):
         'tests/plugins/tesseract_noop.py',
     )
     assert p.returncode == ExitCode.ok
-    assert outtxt.exists()

tests/test_metadata.py

@@ -62,7 +62,7 @@ def test_override_metadata(output_type, resources, outpdf):
     german = 'Du siehst den Wald vor lauter Bäumen nicht.'
     chinese = '孔子'
 
-    p, _out, err = run_ocrmypdf(
+    p = run_ocrmypdf(
         input_file,
         outpdf,
         '--title',
@@ -75,7 +75,7 @@ def test_override_metadata(output_type, resources, outpdf):
         'tests/plugins/tesseract_noop.py',
     )
 
-    assert p.returncode == ExitCode.ok, err
+    assert p.returncode == ExitCode.ok, p.stderr
 
     before = pikepdf.open(input_file)
     after = pikepdf.open(outpdf)
@@ -99,7 +99,7 @@ def test_high_unicode(resources, no_outpdf):
     input_file = resources / 'c02-22.pdf'
     high_unicode = 'U+1030C is: 𐌌'
 
-    p, _out, err = run_ocrmypdf(
+    p = run_ocrmypdf(
         input_file,
         no_outpdf,
         '--subject',
@@ -110,7 +110,7 @@ def test_high_unicode(resources, no_outpdf):
         'tests/plugins/tesseract_noop.py',
     )
 
-    assert p.returncode == ExitCode.bad_args, err
+    assert p.returncode == ExitCode.bad_args, p.stderr
 
 
 @pytest.mark.skipif(not fitz, reason="test uses fitz")

tests/test_optimize.py

@@ -17,7 +17,7 @@ from PIL import Image, ImageDraw
 from ocrmypdf import optimize as opt
 from ocrmypdf._exec import jbig2enc, pngquant
 from ocrmypdf._exec.ghostscript import rasterize_pdf
-from ocrmypdf.helpers import Resolution
+from ocrmypdf.helpers import IMG2PDF_KWARGS, Resolution
 
 from .conftest import check_ocrmypdf
 
@@ -81,7 +81,7 @@ def test_jbig2_lossy(lossy, resources, outpdf):
         '--image-dpi',
         '200',
         '--optimize',
-        3,
+        '3',
         '--jpg-quality',
         '50',
         '--png-quality',
@@ -139,8 +139,8 @@ def test_multiple_pngs(resources, outdir):
         img2pdf.convert(
             fspath(resources / 'baiona_colormapped.png'),
             fspath(resources / 'baiona_gray.png'),
-            with_pdfrw=False,
             outputstream=inpdf,
+            **IMG2PDF_KWARGS,
         )
 
     def mockquant(input_file, output_file, *_args):

tests/test_pdfinfo.py

@@ -17,7 +17,7 @@ from reportlab.pdfgen.canvas import Canvas
 
 from ocrmypdf import pdfinfo
 from ocrmypdf.exceptions import InputFileError
-from ocrmypdf.helpers import Resolution
+from ocrmypdf.helpers import IMG2PDF_KWARGS, Resolution
 from ocrmypdf.pdfinfo import Colorspace, Encoding
 from ocrmypdf.pdfinfo.layout import PDFPage
 
@@ -67,9 +67,9 @@ def test_single_page_image(eight_by_eight, outpdf):
         img2pdf.convert(
             bio,
             producer="img2pdf",
-            with_pdfrw=False,
             layout_fun=layout_fun,
             outputstream=f,
+            **IMG2PDF_KWARGS,
         )
     info = pdfinfo.PdfInfo(outpdf)
 
@@ -173,17 +173,15 @@ def test_stack_abuse():
     p = pikepdf.Pdf.new()
 
     stream = pikepdf.Stream(p, b'q ' * 35)
-    with pytest.warns(None) as record:
+    with pytest.warns(UserWarning, match="overflowed"):
         pdfinfo.info._interpret_contents(stream)
-    assert 'overflowed' in str(record[0].message)
 
     stream = pikepdf.Stream(p, b'q Q Q Q Q')
-    with pytest.warns(None) as record:
+    with pytest.warns(UserWarning, match="underflowed") as record:
         pdfinfo.info._interpret_contents(stream)
-    assert 'underflowed' in str(record[0].message)
 
     stream = pikepdf.Stream(p, b'q ' * 135)
-    with pytest.warns(None):
+    with pytest.warns(UserWarning):
         with pytest.raises(RuntimeError):
             pdfinfo.info._interpret_contents(stream)
 

tests/test_preprocessing.py

@@ -40,6 +40,11 @@ def test_deskew(resources, outdir):
     assert -0.5 < skew_angle < 0.5, "Deskewing failed"
 
 
+def test_deskew_blank_page(resources, outpdf):
+    # Tesseract doesn't like blank pages - make sure we can get through
+    check_ocrmypdf(resources / 'blank.pdf', outpdf, '--deskew')
+
+
 @pytest.mark.xfail(reason="remove background disabled")
 def test_remove_background(resources, outdir):
     # Ensure the input image does not contain pure white/black

tests/test_rotation.py

@@ -18,13 +18,19 @@ from reportlab.pdfgen.canvas import Canvas
 
 from ocrmypdf._exec import ghostscript
 from ocrmypdf._plugin_manager import get_plugin_manager
-from ocrmypdf.helpers import Resolution
+from ocrmypdf.helpers import IMG2PDF_KWARGS, Resolution
 from ocrmypdf.pdfinfo import PdfInfo
 
 from .conftest import check_ocrmypdf, run_ocrmypdf
 
 # pylintx: disable=unused-variable
 
+# Remove this workaround when we require Pillow >= 10
+try:
+    Transpose = Image.Transpose  # type: ignore
+except AttributeError:
+    # Pillow 9 shim
+    Transpose = Image  # type: ignore
 
 RENDERERS = ['hocr', 'sandwich']
 
@@ -222,7 +228,7 @@ def test_rotate_page_level(image_angle, page_angle, resources, outdir):
         with Image.open(fspath(resources / 'typewriter.png')) as im:
             if image_angle != 0:
                 ccw_angle = -image_angle % 360
-                im = im.transpose(getattr(Image, f'ROTATE_{ccw_angle}'))
+                im = im.transpose(getattr(Transpose, f'ROTATE_{ccw_angle}'))
             im.save(memimg, format='PNG')
         memimg.seek(0)
         mempdf = BytesIO()
@@ -230,6 +236,7 @@ def test_rotate_page_level(image_angle, page_angle, resources, outdir):
             memimg.read(),
             layout_fun=img2pdf.get_fixed_dpi_layout_fun((200, 200)),
             outputstream=mempdf,
+            **IMG2PDF_KWARGS,
         )
         mempdf.seek(0)
         pike = pikepdf.open(mempdf)
@@ -242,7 +249,7 @@ def test_rotate_page_level(image_angle, page_angle, resources, outdir):
     test = make_rotate_test('test', image_angle, page_angle)
     out = test.with_suffix('.out.pdf')
 
-    p, _, err = run_ocrmypdf(
+    p = run_ocrmypdf(
         test,
         out,
         '-O0',
@@ -251,7 +258,7 @@ def test_rotate_page_level(image_angle, page_angle, resources, outdir):
         '0.001',
         text=False,
     )
-    err = err.decode('utf-8', errors='replace')
+    err = p.stderr.decode('utf-8', errors='replace')
     assert p.returncode == 0, err
 
     assert compare_images_monochrome(outdir, reference, 1, out, 1) > 0.2

tests/test_stdio.py

@@ -30,12 +30,12 @@ def test_stdin(ocrmypdf_exec, resources, outpdf):
             '--plugin',
             'tests/plugins/tesseract_noop.py',
         ]
-        run(p_args, stdout=PIPE, stderr=PIPE, stdin=input_stream, check=True)
+        run(p_args, capture_output=True, stdin=input_stream, check=True)
 
 
 def test_stdout(ocrmypdf_exec, resources, outpdf):
     if 'COV_CORE_DATAFILE' in os.environ:
-        pytest.skip(msg="Coverage uses stdout")
+        pytest.skip("Coverage uses stdout")
 
     input_file = str(resources / 'francais.pdf')
     output_file = str(outpdf)
@@ -60,10 +60,10 @@ def test_stdout(ocrmypdf_exec, resources, outpdf):
 @pytest.mark.skipif(os.name == 'nt', reason="invalid test on Windows")
 def test_bad_locale(monkeypatch):
     monkeypatch.setenv('LC_ALL', 'C')
-    p, out, err = run_ocrmypdf('a', 'b')
-    assert out == '', "stdout not clean"
+    p = run_ocrmypdf('a', 'b')
+    assert p.stdout == '', "stdout not clean"
     assert p.returncode != 0
-    assert 'configured to use ASCII as encoding' in err, "should whine"
+    assert 'configured to use ASCII as encoding' in p.stderr, "should whine"
 
 
 @pytest.mark.xfail(
@@ -72,9 +72,9 @@ def test_bad_locale(monkeypatch):
 )
 def test_dev_null(resources):
     if 'COV_CORE_DATAFILE' in os.environ:
-        pytest.skip(msg="Coverage uses stdout")
+        pytest.skip("Coverage uses stdout")
 
-    p, out, _err = run_ocrmypdf(
+    p = run_ocrmypdf(
         resources / 'trivial.pdf',
         os.devnull,
         '--force-ocr',
@@ -82,4 +82,4 @@ def test_dev_null(resources):
         'tests/plugins/tesseract_noop.py',
     )
     assert p.returncode == 0, "could not send output to /dev/null"
-    assert len(out) == 0, "wrote to stdout"
+    assert len(p.stdout) == 0, "wrote to stdout"

tests/test_tesseract.py

@@ -82,6 +82,7 @@ def test_image_too_large_hocr(monkeypatch, resources, outdir):
         tessconfig=[],
         timeout=180.0,
         pagesegmode=None,
+        thresholding=0,
         user_words=None,
         user_patterns=None,
     )
@@ -102,6 +103,7 @@ def test_image_too_large_pdf(monkeypatch, resources, outdir):
         tessconfig=[],
         timeout=180.0,
         pagesegmode=None,
+        thresholding=0,
         user_words=None,
         user_patterns=None,
     )

tests/test_unpaper.py

@@ -5,19 +5,24 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 
+import logging
 from os import fspath
 from unittest.mock import patch
 
 import pytest
+from PIL import Image
 
+from ocrmypdf._exec import unpaper
 from ocrmypdf._plugin_manager import get_parser_options_plugins
 from ocrmypdf._validation import check_options
 from ocrmypdf.exceptions import ExitCode, MissingDependencyError
 
-from .conftest import check_ocrmypdf, have_unpaper, run_ocrmypdf
+from .conftest import check_ocrmypdf, have_unpaper, ocrmypdf_exec, run_ocrmypdf
 
 # pylint: disable=redefined-outer-name
 
+needs_unpaper = pytest.mark.skipif(not have_unpaper(), reason="requires unpaper")
+
 
 def test_no_unpaper(resources, no_outpdf):
     input_ = fspath(resources / "c02-22.pdf")
@@ -45,7 +50,7 @@ def test_old_unpaper(resources, no_outpdf):
         mock.assert_called()
 
 
-@pytest.mark.skipif(not have_unpaper(), reason="requires unpaper")
+@needs_unpaper
 def test_clean(resources, outpdf):
     check_ocrmypdf(
         resources / "skew.pdf",
@@ -56,7 +61,7 @@ def test_clean(resources, outpdf):
     )
 
 
-@pytest.mark.skipif(not have_unpaper(), reason="requires unpaper")
+@needs_unpaper
 def test_unpaper_args_valid(resources, outpdf):
     check_ocrmypdf(
         resources / "skew.pdf",
@@ -69,9 +74,9 @@ def test_unpaper_args_valid(resources, outpdf):
     )
 
 
-@pytest.mark.skipif(not have_unpaper(), reason="requires unpaper")
+@needs_unpaper
 def test_unpaper_args_invalid_filename(resources, outpdf):
-    p, _out, err = run_ocrmypdf(
+    p = run_ocrmypdf(
         resources / "skew.pdf",
         outpdf,
         "-c",
@@ -80,13 +85,13 @@ def test_unpaper_args_invalid_filename(resources, outpdf):
         '--plugin',
         'tests/plugins/tesseract_noop.py',
     )
-    assert "No filenames allowed" in err
+    assert "No filenames allowed" in p.stderr
     assert p.returncode == ExitCode.bad_args
 
 
-@pytest.mark.skipif(not have_unpaper(), reason="requires unpaper")
+@needs_unpaper
 def test_unpaper_args_invalid(resources, outpdf):
-    p, _out, _err = run_ocrmypdf(
+    p = run_ocrmypdf(
         resources / "skew.pdf",
         outpdf,
         "-c",
@@ -98,3 +103,16 @@ def test_unpaper_args_invalid(resources, outpdf):
     # Can't tell difference between unpaper choking on bad arguments or some
     # other unpaper failure
     assert p.returncode == ExitCode.child_process_error
+
+
+@needs_unpaper
+def test_unpaper_image_too_big(resources, outdir, caplog):
+    with patch('ocrmypdf._exec.unpaper.UNPAPER_IMAGE_PIXEL_LIMIT', 42):
+        infile = resources / 'crom.png'
+        unpaper.clean(infile, outdir / 'out.png', dpi=300) == infile
+
+        assert any(
+            'too large for cleaning' in rec.message
+            for rec in caplog.get_records('call')
+            if rec.levelno == logging.WARNING
+        )

tests/test_validation.py

@@ -103,13 +103,19 @@ def test_user_words(caplog):
         opts = make_opts(user_words='foo')
         plugin_manager = get_plugin_manager(opts.plugins)
         vd._check_options(opts, plugin_manager, set())
-        assert '4.0 ignores --user-words' in caplog.text
+        assert (
+            'Tesseract 4.0 (which you have installed) ignores --user-words'
+            in caplog.text
+        )
     caplog.clear()
     with patch('ocrmypdf._exec.tesseract.has_user_words', return_value=True):
         opts = make_opts(user_patterns='foo')
         plugin_manager = get_plugin_manager(opts.plugins)
         vd._check_options(opts, plugin_manager, set())
-        assert '4.0 ignores --user-words' not in caplog.text
+        assert (
+            'Tesseract 4.0 (which you have installed) ignores --user-words'
+            not in caplog.text
+        )
 
 
 def test_pillow_options():