diff --git a/misc/completion/ocrmypdf.bash b/misc/completion/ocrmypdf.bash index e3bd25a9..65e032ae 100644 --- a/misc/completion/ocrmypdf.bash +++ b/misc/completion/ocrmypdf.bash @@ -44,6 +44,7 @@ _ocrmypdf() --skip-big --jpeg-quality --png-quality --jbig2-lossy --max-image-mpixels --tesseract-config --tesseract-pagesegmode --help --tesseract-oem --pdf-renderer --tesseract-timeout + --tesseract-thresholding --rotate-pages-threshold --pdfa-image-compression --user-words --user-patterns --keep-temporary-files --output-type --no-progress-bar --pages --fast-web-view' \ @@ -105,6 +106,10 @@ _ocrmypdf() COMPREPLY=( $( compgen -W '{1..13}' -- "$cur" ) ) return ;; + --tesseract-thresholding) + COMPREPLY=( $( compgen -W 'auto otsu adaptive-otsu sauvola' -- "$cur" ) ) + return + ;; --sidecar|--title|--author|--subject|--keywords|--unpaper-args|--pages|--fast-web-view) # argument required but no completions available return diff --git a/misc/completion/ocrmypdf.fish b/misc/completion/ocrmypdf.fish index fd1e54a0..ea9afcd0 100644 --- a/misc/completion/ocrmypdf.fish +++ b/misc/completion/ocrmypdf.fish @@ -129,6 +129,15 @@ function __fish_ocrmypdf_tesseract_oem echo -e "3\t"(_ "default, based on what is available") end complete -c ocrmypdf -x -l tesseract-oem -a '(__fish_ocrmypdf_tesseract_oem)' -d "set tesseract --oem" + +function __fish_ocrmypdf_tesseract_thresholding + echo -e "auto\t"(_ "let OCRmyPDF pick thresholding (current always uses otsu)") + echo -e "otsu\t"(_ "legacy Otsu thresholding") + echo -e "adaptive-otsu\t"(_ "use adaptive Otsu thresholding") + echo -e "sauvola\t"(_ "use Sauvola thresholding") +end +complete -c ocrmypdf -x -l tesseract-thresholding -a '(__fish_ocrmypdf_tesseract_thresholding)' -d "set tesseract thresholding method (needs Tesseract 5.x)" + complete -c ocrmypdf -x -l tesseract-timeout -d "maximum number of seconds to wait for OCR" complete -c ocrmypdf -x -l rotate-pages-threshold -d "page rotation confidence" diff --git a/src/ocrmypdf/_exec/tesseract.py b/src/ocrmypdf/_exec/tesseract.py index a3688f65..9e08c99c 100644 --- a/src/ocrmypdf/_exec/tesseract.py +++ b/src/ocrmypdf/_exec/tesseract.py @@ -46,6 +46,13 @@ HOCR_TEMPLATE = """ """ +TESSERACT_THRESHOLDING_METHODS: Dict[str, int] = { + 'auto': 0, + 'otsu': 0, + 'adaptive-otsu': 1, + 'sauvola': 2, +} + class TesseractLoggerAdapter(logging.LoggerAdapter): def process(self, msg, kwargs): @@ -87,6 +94,11 @@ def has_user_words(): return version() >= '4.1' +def has_thresholding(): + """Does Tesseract have -c thresholding method capability?""" + return version() >= '5.0' + + def get_languages(): def lang_error(output): msg = ( @@ -265,9 +277,11 @@ def generate_hocr( tessconfig: List[str], timeout: float, pagesegmode: int, + thresholding: int, user_words, user_patterns, ): + """Generate a hOCR file, which must be converted to PDF.""" prefix = output_hocr.with_suffix('') args_tesseract = tess_base_args(languages, engine_mode) @@ -275,6 +289,9 @@ def generate_hocr( if pagesegmode is not None: args_tesseract.extend(['--psm', str(pagesegmode)]) + if thresholding != 0 and has_thresholding(): + args_tesseract.extend(['-c', f'thresholding_method={thresholding}']) + if user_words: args_tesseract.extend(['--user-words', user_words]) @@ -326,20 +343,15 @@ def generate_pdf( tessconfig: List[str], timeout: float, pagesegmode: int, + thresholding: int, user_words, user_patterns, ): - """Use Tesseract to render a PDF. + """Generate a PDF using Tesseract's internal PDF generator. - input_file -- image to analyze - output_pdf -- file to generate - output_text -- OCR text file - languages -- list of languages to consider - engine_mode -- engine mode argument for tess v4 - tessconfig -- tesseract configuration - timeout -- timeout (seconds) + We specifically a text-only PDF which is more suitable for combining with + the input page. """ - args_tesseract = tess_base_args(languages, engine_mode) if pagesegmode is not None: @@ -347,6 +359,9 @@ def generate_pdf( args_tesseract.extend(['-c', 'textonly_pdf=1']) + if thresholding != 0 and has_thresholding(): + args_tesseract.extend(['-c', f'thresholding_method={thresholding}']) + if user_words: args_tesseract.extend(['--user-words', user_words]) diff --git a/src/ocrmypdf/api.py b/src/ocrmypdf/api.py index ac21ebb7..5c23151c 100644 --- a/src/ocrmypdf/api.py +++ b/src/ocrmypdf/api.py @@ -242,6 +242,7 @@ def ocr( # pylint: disable=unused-argument tesseract_config: Iterable[str] = None, tesseract_pagesegmode: int = None, tesseract_oem: int = None, + tesseract_thresholding: int = None, pdf_renderer=None, tesseract_timeout: float = None, rotate_pages_threshold: float = None, diff --git a/src/ocrmypdf/builtin_plugins/tesseract_ocr.py b/src/ocrmypdf/builtin_plugins/tesseract_ocr.py index 8d070ad4..a0de9841 100644 --- a/src/ocrmypdf/builtin_plugins/tesseract_ocr.py +++ b/src/ocrmypdf/builtin_plugins/tesseract_ocr.py @@ -10,7 +10,7 @@ import os from ocrmypdf import hookimpl from ocrmypdf._exec import tesseract -from ocrmypdf.cli import numeric +from ocrmypdf.cli import numeric, str_to_int from ocrmypdf.helpers import clamp from ocrmypdf.pluginspec import OcrEngine from ocrmypdf.subprocess import check_external_program @@ -43,13 +43,27 @@ def add_options(parser): metavar='MODE', choices=range(0, 4), help=( - "Set Tesseract 4.0 OCR engine mode: " + "Set Tesseract 4.0+ OCR engine mode: " "0 - original Tesseract only; " "1 - neural nets LSTM only; " "2 - Tesseract + LSTM; " "3 - default." ), ) + tess.add_argument( + '--tesseract-thresholding', + action='store', + type=str_to_int(tesseract.TESSERACT_THRESHOLDING_METHODS), + default='auto', + metavar='METHOD', + help=( + "Set Tesseract 5.0+ input image thresholding mode. This may improve OCR " + "results on low quality images or those that contain high constrast color. " + "legacy-otsu is the Tesseract default; adaptive-otsu is an improved Otsu " + "algorithm with improved sort for background color changes; sauvola is " + "based on local standard deviation." + ), + ) tess.add_argument( '--tesseract-timeout', default=180.0, @@ -89,8 +103,14 @@ def check_options(options): if not tesseract.has_user_words() and (options.user_words or options.user_patterns): log.warning( - "Tesseract 4.0 ignores --user-words and --user-patterns, so these " - "arguments have no effect." + "Tesseract 4.0 (which you have installed) ignores --user-words and " + "--user-patterns, so these arguments have no effect." + ) + if not tesseract.has_thresholding() and options.tesseract_thresholding != 0: + log.warning( + "The installed version of Tesseract does not support changes to its " + "thresholding method. The --tesseract-threshold argument will be " + "ignored." ) if options.tesseract_pagesegmode in (0, 2): log.warning( @@ -162,6 +182,7 @@ class TesseractOcrEngine(OcrEngine): tessconfig=options.tesseract_config, timeout=options.tesseract_timeout, pagesegmode=options.tesseract_pagesegmode, + thresholding=options.tesseract_thresholding, user_words=options.user_words, user_patterns=options.user_patterns, ) @@ -177,6 +198,7 @@ class TesseractOcrEngine(OcrEngine): tessconfig=options.tesseract_config, timeout=options.tesseract_timeout, pagesegmode=options.tesseract_pagesegmode, + thresholding=options.tesseract_thresholding, user_words=options.user_words, user_patterns=options.user_patterns, ) diff --git a/src/ocrmypdf/cli.py b/src/ocrmypdf/cli.py index ff722dd6..b8d58b58 100644 --- a/src/ocrmypdf/cli.py +++ b/src/ocrmypdf/cli.py @@ -6,12 +6,12 @@ import argparse -from typing import Any, Callable, Optional, TypeVar +from typing import Any, Callable, Mapping, Optional, TypeVar from ocrmypdf._version import PROGRAM_NAME as _PROGRAM_NAME from ocrmypdf._version import __version__ as _VERSION -T = TypeVar('T') +T = TypeVar('T', int, float) def numeric( @@ -21,17 +21,32 @@ def numeric( min_ = basetype(min_) if min_ is not None else None max_ = basetype(max_) if max_ is not None else None - def _numeric(string): - value = basetype(string) + def _numeric(s: str) -> T: + value = basetype(s) if (min_ is not None and value < min_) or (max_ is not None and value > max_): - msg = f"{string!r} not in valid range {(min_, max_)!r}" - raise argparse.ArgumentTypeError(msg) + raise argparse.ArgumentTypeError( + f"{s!r} not in valid range {(min_, max_)!r}" + ) return value _numeric.__name__ = basetype.__name__ return _numeric +def str_to_int(mapping: Mapping[str, int]): + """Accept text on command line and convert to integer.""" + + def _str_to_int(s: str) -> int: + try: + return mapping[s] + except KeyError: + raise argparse.ArgumentTypeError( + f"{s!r} must be one of: {', '.join(mapping.keys())}" + ) + + return _str_to_int + + class ArgumentParser(argparse.ArgumentParser): """Override parser's default behavior of calling sys.exit() diff --git a/tests/cache/manifest.jsonl b/tests/cache/manifest.jsonl index 80662e47..7656b2a2 100644 --- a/tests/cache/manifest.jsonl +++ b/tests/cache/manifest.jsonl @@ -75,3 +75,8 @@ {"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000001_rasterize.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000001_rasterize.png", "stdout"]} {"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000006_rasterize.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000006_rasterize.png", "stdout"]} {"tesseract_version": "5.0.0-beta-20210916-12-g19cc9", "platform": "Linux-5.10.16.3-microsoft-standard-WSL2-x86_64-with-glibc2.29", "python": "3.8.10", "argv_slug": "__-l__eng__--psm__2__000006_rasterize.png__stdout", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "--psm", "2", "$TMPDIR/000006_rasterize.png", "stdout"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-40-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/trivial.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-40-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/trivial.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-40-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/trivial.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-40-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/trivial.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]} +{"tesseract_version": "4.1.1", "platform": "Linux-5.11.0-40-generic-x86_64-with-glibc2.33", "python": "3.9.5", "argv_slug": "__-l__eng__--oem__1__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/trivial.pdf", "args": ["-l", "eng", "--oem", "1", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]} diff --git a/tests/cache/trivial/__-l__eng__--oem__1__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin b/tests/cache/trivial/__-l__eng__--oem__1__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin new file mode 100644 index 00000000..55189be2 Binary files /dev/null and b/tests/cache/trivial/__-l__eng__--oem__1__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin differ diff --git a/tests/cache/trivial/__-l__eng__--oem__1__000001_ocr.png__000001_ocr_tess__pdf__txt/stderr.bin b/tests/cache/trivial/__-l__eng__--oem__1__000001_ocr.png__000001_ocr_tess__pdf__txt/stderr.bin new file mode 100644 index 00000000..16b617e5 --- /dev/null +++ b/tests/cache/trivial/__-l__eng__--oem__1__000001_ocr.png__000001_ocr_tess__pdf__txt/stderr.bin @@ -0,0 +1 @@ +Tesseract Open Source OCR Engine v4.1.1 with Leptonica diff --git a/tests/cache/trivial/__-l__eng__--oem__1__000001_ocr.png__000001_ocr_tess__pdf__txt/stdout.bin b/tests/cache/trivial/__-l__eng__--oem__1__000001_ocr.png__000001_ocr_tess__pdf__txt/stdout.bin new file mode 100644 index 00000000..e69de29b diff --git a/tests/cache/trivial/__-l__eng__--oem__1__000001_ocr.png__000001_ocr_tess__pdf__txt/txt.bin b/tests/cache/trivial/__-l__eng__--oem__1__000001_ocr.png__000001_ocr_tess__pdf__txt/txt.bin new file mode 100644 index 00000000..8214d0ee --- /dev/null +++ b/tests/cache/trivial/__-l__eng__--oem__1__000001_ocr.png__000001_ocr_tess__pdf__txt/txt.bin @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/tests/cache/trivial/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin b/tests/cache/trivial/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin new file mode 100644 index 00000000..84d54f98 Binary files /dev/null and b/tests/cache/trivial/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin differ diff --git a/tests/cache/trivial/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/stderr.bin b/tests/cache/trivial/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/stderr.bin new file mode 100644 index 00000000..16b617e5 --- /dev/null +++ b/tests/cache/trivial/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/stderr.bin @@ -0,0 +1 @@ +Tesseract Open Source OCR Engine v4.1.1 with Leptonica diff --git a/tests/cache/trivial/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/stdout.bin b/tests/cache/trivial/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/stdout.bin new file mode 100644 index 00000000..e69de29b diff --git a/tests/cache/trivial/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/txt.bin b/tests/cache/trivial/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/txt.bin new file mode 100644 index 00000000..8214d0ee --- /dev/null +++ b/tests/cache/trivial/__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt/txt.bin @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/tests/test_main.py b/tests/test_main.py index 26c93050..f308b1f3 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -316,6 +316,42 @@ def test_pagesegmode(renderer, resources, outpdf): ) +def test_tesseract_oem(resources, outpdf): + check_ocrmypdf( + resources / 'trivial.pdf', + outpdf, + '--tesseract-oem', + '1', + '--plugin', + 'tests/plugins/tesseract_cache.py', + ) + + +@pytest.mark.parametrize('value', ['auto', 'otsu', 'adaptive-otsu', 'sauvola']) +def test_tesseract_thresholding(value, resources, outpdf): + check_ocrmypdf( + resources / 'trivial.pdf', + outpdf, + '--tesseract-thresholding', + value, + '--plugin', + 'tests/plugins/tesseract_cache.py', + ) + + +@pytest.mark.parametrize('value', ['abcxyz']) +def test_tesseract_thresholding_invalid(value, resources, no_outpdf): + with pytest.raises(SystemExit, match='2'): + run_ocrmypdf_api( + resources / 'trivial.pdf', + no_outpdf, + '--tesseract-thresholding', + value, + '--plugin', + 'tests/plugins/tesseract_cache.py', + ) + + @pytest.mark.parametrize('renderer', RENDERERS) def test_tesseract_crash(renderer, resources, no_outpdf): p, _, err = run_ocrmypdf( diff --git a/tests/test_tesseract.py b/tests/test_tesseract.py index 17940949..c4670722 100644 --- a/tests/test_tesseract.py +++ b/tests/test_tesseract.py @@ -82,6 +82,7 @@ def test_image_too_large_hocr(monkeypatch, resources, outdir): tessconfig=[], timeout=180.0, pagesegmode=None, + thresholding=0, user_words=None, user_patterns=None, ) @@ -102,6 +103,7 @@ def test_image_too_large_pdf(monkeypatch, resources, outdir): tessconfig=[], timeout=180.0, pagesegmode=None, + thresholding=0, user_words=None, user_patterns=None, ) diff --git a/tests/test_validation.py b/tests/test_validation.py index 9b6f35c0..280e25f8 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -103,13 +103,19 @@ def test_user_words(caplog): opts = make_opts(user_words='foo') plugin_manager = get_plugin_manager(opts.plugins) vd._check_options(opts, plugin_manager, set()) - assert '4.0 ignores --user-words' in caplog.text + assert ( + 'Tesseract 4.0 (which you have installed) ignores --user-words' + in caplog.text + ) caplog.clear() with patch('ocrmypdf._exec.tesseract.has_user_words', return_value=True): opts = make_opts(user_patterns='foo') plugin_manager = get_plugin_manager(opts.plugins) vd._check_options(opts, plugin_manager, set()) - assert '4.0 ignores --user-words' not in caplog.text + assert ( + 'Tesseract 4.0 (which you have installed) ignores --user-words' + not in caplog.text + ) def test_pillow_options():