Add new argument --tesseract-thresholding to control tesseract thresholding where available

Also add missing test for --tesseract-oem

misc/completion/ocrmypdf.bash

@@ -44,6 +44,7 @@ _ocrmypdf()
             --skip-big --jpeg-quality --png-quality --jbig2-lossy
             --max-image-mpixels --tesseract-config --tesseract-pagesegmode
             --help --tesseract-oem --pdf-renderer --tesseract-timeout
+            --tesseract-thresholding
             --rotate-pages-threshold --pdfa-image-compression --user-words
             --user-patterns --keep-temporary-files --output-type
             --no-progress-bar --pages --fast-web-view' \
@@ -105,6 +106,10 @@ _ocrmypdf()
             COMPREPLY=( $( compgen -W '{1..13}' -- "$cur" ) )
             return
             ;;
+        --tesseract-thresholding)
+            COMPREPLY=( $( compgen -W 'auto otsu adaptive-otsu sauvola' -- "$cur" ) )
+            return
+            ;;
         --sidecar|--title|--author|--subject|--keywords|--unpaper-args|--pages|--fast-web-view)
             # argument required but no completions available
             return

misc/completion/ocrmypdf.fish

@@ -129,6 +129,15 @@ function __fish_ocrmypdf_tesseract_oem
     echo -e "3\t"(_ "default, based on what is available")
 end
 complete -c ocrmypdf -x -l tesseract-oem -a '(__fish_ocrmypdf_tesseract_oem)' -d "set tesseract --oem"
+
+function __fish_ocrmypdf_tesseract_thresholding
+    echo -e "auto\t"(_ "let OCRmyPDF pick thresholding (current always uses otsu)")
+    echo -e "otsu\t"(_ "legacy Otsu thresholding")
+    echo -e "adaptive-otsu\t"(_ "use adaptive Otsu thresholding")
+    echo -e "sauvola\t"(_ "use Sauvola thresholding")
+end
+complete -c ocrmypdf -x -l tesseract-thresholding -a '(__fish_ocrmypdf_tesseract_thresholding)' -d "set tesseract thresholding method (needs Tesseract 5.x)"
+
 complete -c ocrmypdf -x -l tesseract-timeout -d "maximum number of seconds to wait for OCR"
 complete -c ocrmypdf -x -l rotate-pages-threshold -d "page rotation confidence"
 

src/ocrmypdf/_exec/tesseract.py

@@ -46,6 +46,13 @@ HOCR_TEMPLATE = """<?xml version="1.0" encoding="UTF-8"?>
 </html>
 """
 
+TESSERACT_THRESHOLDING_METHODS: Dict[str, int] = {
+    'auto': 0,
+    'otsu': 0,
+    'adaptive-otsu': 1,
+    'sauvola': 2,
+}
+
 
 class TesseractLoggerAdapter(logging.LoggerAdapter):
     def process(self, msg, kwargs):
@@ -87,6 +94,11 @@ def has_user_words():
     return version() >= '4.1'
 
 
+def has_thresholding():
+    """Does Tesseract have -c thresholding method capability?"""
+    return version() >= '5.0'
+
+
 def get_languages():
     def lang_error(output):
         msg = (
@@ -265,9 +277,11 @@ def generate_hocr(
     tessconfig: List[str],
     timeout: float,
     pagesegmode: int,
+    thresholding: int,
     user_words,
     user_patterns,
 ):
+    """Generate a hOCR file, which must be converted to PDF."""
     prefix = output_hocr.with_suffix('')
 
     args_tesseract = tess_base_args(languages, engine_mode)
@@ -275,6 +289,9 @@ def generate_hocr(
     if pagesegmode is not None:
         args_tesseract.extend(['--psm', str(pagesegmode)])
 
+    if thresholding != 0 and has_thresholding():
+        args_tesseract.extend(['-c', f'thresholding_method={thresholding}'])
+
     if user_words:
         args_tesseract.extend(['--user-words', user_words])
 
@@ -326,20 +343,15 @@ def generate_pdf(
     tessconfig: List[str],
     timeout: float,
     pagesegmode: int,
+    thresholding: int,
     user_words,
     user_patterns,
 ):
-    """Use Tesseract to render a PDF.
+    """Generate a PDF using Tesseract's internal PDF generator.
 
-    input_file -- image to analyze
-    output_pdf -- file to generate
-    output_text -- OCR text file
-    languages -- list of languages to consider
-    engine_mode -- engine mode argument for tess v4
-    tessconfig -- tesseract configuration
-    timeout -- timeout (seconds)
+    We specifically a text-only PDF which is more suitable for combining with
+    the input page.
     """
-
     args_tesseract = tess_base_args(languages, engine_mode)
 
     if pagesegmode is not None:
@@ -347,6 +359,9 @@ def generate_pdf(
 
     args_tesseract.extend(['-c', 'textonly_pdf=1'])
 
+    if thresholding != 0 and has_thresholding():
+        args_tesseract.extend(['-c', f'thresholding_method={thresholding}'])
+
     if user_words:
         args_tesseract.extend(['--user-words', user_words])
 

src/ocrmypdf/api.py

@@ -242,6 +242,7 @@ def ocr(  # pylint: disable=unused-argument
     tesseract_config: Iterable[str] = None,
     tesseract_pagesegmode: int = None,
     tesseract_oem: int = None,
+    tesseract_thresholding: int = None,
     pdf_renderer=None,
     tesseract_timeout: float = None,
     rotate_pages_threshold: float = None,

src/ocrmypdf/builtin_plugins/tesseract_ocr.py

@@ -10,7 +10,7 @@ import os
 
 from ocrmypdf import hookimpl
 from ocrmypdf._exec import tesseract
-from ocrmypdf.cli import numeric
+from ocrmypdf.cli import numeric, str_to_int
 from ocrmypdf.helpers import clamp
 from ocrmypdf.pluginspec import OcrEngine
 from ocrmypdf.subprocess import check_external_program
@@ -43,13 +43,27 @@ def add_options(parser):
         metavar='MODE',
         choices=range(0, 4),
         help=(
-            "Set Tesseract 4.0 OCR engine mode: "
+            "Set Tesseract 4.0+ OCR engine mode: "
             "0 - original Tesseract only; "
             "1 - neural nets LSTM only; "
             "2 - Tesseract + LSTM; "
             "3 - default."
         ),
     )
+    tess.add_argument(
+        '--tesseract-thresholding',
+        action='store',
+        type=str_to_int(tesseract.TESSERACT_THRESHOLDING_METHODS),
+        default='auto',
+        metavar='METHOD',
+        help=(
+            "Set Tesseract 5.0+ input image thresholding mode. This may improve OCR "
+            "results on low quality images or those that contain high constrast color. "
+            "legacy-otsu is the Tesseract default; adaptive-otsu is an improved Otsu "
+            "algorithm with improved sort for background color changes; sauvola is "
+            "based on local standard deviation."
+        ),
+    )
     tess.add_argument(
         '--tesseract-timeout',
         default=180.0,
@@ -89,8 +103,14 @@ def check_options(options):
 
     if not tesseract.has_user_words() and (options.user_words or options.user_patterns):
         log.warning(
-            "Tesseract 4.0 ignores --user-words and --user-patterns, so these "
-            "arguments have no effect."
+            "Tesseract 4.0 (which you have installed) ignores --user-words and "
+            "--user-patterns, so these arguments have no effect."
+        )
+    if not tesseract.has_thresholding() and options.tesseract_thresholding != 0:
+        log.warning(
+            "The installed version of Tesseract does not support changes to its "
+            "thresholding method. The --tesseract-threshold argument will be "
+            "ignored."
         )
     if options.tesseract_pagesegmode in (0, 2):
         log.warning(
@@ -162,6 +182,7 @@ class TesseractOcrEngine(OcrEngine):
             tessconfig=options.tesseract_config,
             timeout=options.tesseract_timeout,
             pagesegmode=options.tesseract_pagesegmode,
+            thresholding=options.tesseract_thresholding,
             user_words=options.user_words,
             user_patterns=options.user_patterns,
         )
@@ -177,6 +198,7 @@ class TesseractOcrEngine(OcrEngine):
             tessconfig=options.tesseract_config,
             timeout=options.tesseract_timeout,
             pagesegmode=options.tesseract_pagesegmode,
+            thresholding=options.tesseract_thresholding,
             user_words=options.user_words,
             user_patterns=options.user_patterns,
         )

src/ocrmypdf/cli.py

@@ -6,12 +6,12 @@
 
 
 import argparse
-from typing import Any, Callable, Optional, TypeVar
+from typing import Any, Callable, Mapping, Optional, TypeVar
 
 from ocrmypdf._version import PROGRAM_NAME as _PROGRAM_NAME
 from ocrmypdf._version import __version__ as _VERSION
 
-T = TypeVar('T')
+T = TypeVar('T', int, float)
 
 
 def numeric(
@@ -21,17 +21,32 @@ def numeric(
     min_ = basetype(min_) if min_ is not None else None
     max_ = basetype(max_) if max_ is not None else None
 
-    def _numeric(string):
-        value = basetype(string)
+    def _numeric(s: str) -> T:
+        value = basetype(s)
         if (min_ is not None and value < min_) or (max_ is not None and value > max_):
-            msg = f"{string!r} not in valid range {(min_, max_)!r}"
-            raise argparse.ArgumentTypeError(msg)
+            raise argparse.ArgumentTypeError(
+                f"{s!r} not in valid range {(min_, max_)!r}"
+            )
         return value
 
     _numeric.__name__ = basetype.__name__
     return _numeric
 
 
+def str_to_int(mapping: Mapping[str, int]):
+    """Accept text on command line and convert to integer."""
+
+    def _str_to_int(s: str) -> int:
+        try:
+            return mapping[s]
+        except KeyError:
+            raise argparse.ArgumentTypeError(
+                f"{s!r} must be one of: {', '.join(mapping.keys())}"
+            )
+
+    return _str_to_int
+
+
 class ArgumentParser(argparse.ArgumentParser):
     """Override parser's default behavior of calling sys.exit()
 

tests/cache/manifest.jsonl

@@ -75,3 +75,8 @@
 {"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000001_rasterize.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000001_rasterize.png", "stdout"]}
 {"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000006_rasterize.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000006_rasterize.png", "stdout"]}
 {"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000006_rasterize.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000006_rasterize.png", "stdout"]}
+{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-40-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/trivial.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-40-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/trivial.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-40-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/trivial.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-40-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/trivial.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-40-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__--oem__1__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/trivial.pdf", "args": ["-l", "eng", "--oem", "1", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}

tests/cache/trivial/__-l__eng__--oem__1__000001_ocr.png__000001_ocr_tess__pdf__txt/stderr.bin

@@ -0,0 +1 @@
+Tesseract Open Source OCR Engine v4.1.1 with Leptonica

tests/cache/trivial/__-l__eng__--oem__1__000001_ocr.png__000001_ocr_tess__pdf__txt/txt.bin

@@ -0,0 +1 @@
+

tests/cache/trivial/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/stderr.bin

@@ -0,0 +1 @@
+Tesseract Open Source OCR Engine v4.1.1 with Leptonica

tests/cache/trivial/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/txt.bin

@@ -0,0 +1 @@
+

tests/test_main.py

@@ -316,6 +316,42 @@ def test_pagesegmode(renderer, resources, outpdf):
     )
 
 
+def test_tesseract_oem(resources, outpdf):
+    check_ocrmypdf(
+        resources / 'trivial.pdf',
+        outpdf,
+        '--tesseract-oem',
+        '1',
+        '--plugin',
+        'tests/plugins/tesseract_cache.py',
+    )
+
+
+@pytest.mark.parametrize('value', ['auto', 'otsu', 'adaptive-otsu', 'sauvola'])
+def test_tesseract_thresholding(value, resources, outpdf):
+    check_ocrmypdf(
+        resources / 'trivial.pdf',
+        outpdf,
+        '--tesseract-thresholding',
+        value,
+        '--plugin',
+        'tests/plugins/tesseract_cache.py',
+    )
+
+
+@pytest.mark.parametrize('value', ['abcxyz'])
+def test_tesseract_thresholding_invalid(value, resources, no_outpdf):
+    with pytest.raises(SystemExit, match='2'):
+        run_ocrmypdf_api(
+            resources / 'trivial.pdf',
+            no_outpdf,
+            '--tesseract-thresholding',
+            value,
+            '--plugin',
+            'tests/plugins/tesseract_cache.py',
+        )
+
+
 @pytest.mark.parametrize('renderer', RENDERERS)
 def test_tesseract_crash(renderer, resources, no_outpdf):
     p, _, err = run_ocrmypdf(

tests/test_tesseract.py

@@ -82,6 +82,7 @@ def test_image_too_large_hocr(monkeypatch, resources, outdir):
         tessconfig=[],
         timeout=180.0,
         pagesegmode=None,
+        thresholding=0,
         user_words=None,
         user_patterns=None,
     )
@@ -102,6 +103,7 @@ def test_image_too_large_pdf(monkeypatch, resources, outdir):
         tessconfig=[],
         timeout=180.0,
         pagesegmode=None,
+        thresholding=0,
         user_words=None,
         user_patterns=None,
     )

tests/test_validation.py

@@ -103,13 +103,19 @@ def test_user_words(caplog):
         opts = make_opts(user_words='foo')
         plugin_manager = get_plugin_manager(opts.plugins)
         vd._check_options(opts, plugin_manager, set())
-        assert '4.0 ignores --user-words' in caplog.text
+        assert (
+            'Tesseract 4.0 (which you have installed) ignores --user-words'
+            in caplog.text
+        )
     caplog.clear()
     with patch('ocrmypdf._exec.tesseract.has_user_words', return_value=True):
         opts = make_opts(user_patterns='foo')
         plugin_manager = get_plugin_manager(opts.plugins)
         vd._check_options(opts, plugin_manager, set())
-        assert '4.0 ignores --user-words' not in caplog.text
+        assert (
+            'Tesseract 4.0 (which you have installed) ignores --user-words'
+            not in caplog.text
+        )
 
 
 def test_pillow_options():