Change most tests to use ocrmypdf API instead of subprocess

The main benefit of this is code coverage gains can actually follow it.
Also removes most ugly os.environ hacks.

src/ocrmypdf/__main__.py

@@ -35,8 +35,6 @@ def run(args=None):
     if not check_closed_streams(options):
         return ExitCode.bad_args
 
-    if os.environ.get('PYTEST_CURRENT_TEST'):
-        os.environ['_OCRMYPDF_TEST_INFILE'] = options.input_file
     if hasattr(os, 'nice'):
         os.nice(5)
 

src/ocrmypdf/_pipeline.py

@@ -348,6 +348,7 @@ def get_orientation_correction(preview, page_context):
         engine_mode=page_context.options.tesseract_oem,
         timeout=page_context.options.tesseract_timeout,
         log=page_context.log,
+        tesseract_env=page_context.options.tesseract_env,
     )
 
     direction = {0: '⇧', 90: '⇨', 180: '⇩', 270: '⇦'}
@@ -548,6 +549,7 @@ def ocr_tesseract_hocr(input_file, page_context):
         pagesegmode=options.tesseract_pagesegmode,
         user_words=options.user_words,
         user_patterns=options.user_patterns,
+        tesseract_env=options.tesseract_env,
         log=page_context.log,
     )
     return (hocr_out, hocr_text_out)
@@ -627,6 +629,7 @@ def ocr_tesseract_textonly_pdf(input_image, page_context):
         pagesegmode=options.tesseract_pagesegmode,
         user_words=options.user_words,
         user_patterns=options.user_patterns,
+        tesseract_env=options.tesseract_env,
         log=page_context.log,
     )
     return (output_pdf, output_text)

src/ocrmypdf/api.py

@@ -16,6 +16,7 @@
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
 import logging
+import os
 import sys
 from enum import IntEnum
 from pathlib import Path
@@ -113,13 +114,16 @@ def configure_logging(verbosity, progress_bar_friendly=True, manage_root_logger=
 
 def create_options(*, input_file, output_file, **kwargs):
     cmdline = []
-    filters = []
+    deferred = []
 
     for arg, val in kwargs.items():
         if val is None:
             continue
         if arg.startswith('filter') and (callable(val) or isinstance(val, str)):
-            filters.append((arg, val))
+            deferred.append((arg, val))
+            continue
+        elif arg == 'tesseract_env':
+            deferred.append((arg, val))
             continue
         cmd_style_arg = arg.replace('_', '-')
         cmdline.append(f"--{cmd_style_arg}")
@@ -132,15 +136,20 @@ def create_options(*, input_file, output_file, **kwargs):
         elif isinstance(val, Path):
             cmdline.append(str(val))
         else:
-            raise TypeError(f"{val} ({type(val)})")
+            raise TypeError(f"{arg}: {val} ({type(val)})")
 
     cmdline.append(str(input_file))
     cmdline.append(str(output_file))
 
     parser.api_mode = True
     options = parser.parse_args(cmdline)
-    for keyword, function in filters:
-        setattr(options, keyword, function)
+    for keyword, val in deferred:
+        setattr(options, keyword, val)
+
+    # If we are running a Tesseract spoof, ensure it knows what the input file is
+    if os.environ.get('PYTEST_CURRENT_TEST') and options.tesseract_env:
+        options.tesseract_env['_OCRMYPDF_TEST_INFILE'] = input_file
+
     return options
 
 
@@ -190,9 +199,18 @@ def ocrmypdf(  # pylint: disable=unused-argument
     keep_temporary_files=None,
     progress_bar=None,
     filter_ocr_image=None,
+    tesseract_env=None,
 ):
     """Run OCRmyPDF on one PDF or image.
 
+    For most arguments, see documentation for the equivalent command line parameter.
+    A few specific arguments are discussed here:
+
+    Args:
+        use_threads (bool): Use worker threads instead of processes. This reduces
+            performance but may make debugging easier since it is easier to set
+            breakpoints.
+        tesseract_env (dict): Override environment variables for Tesseract
     Raises:
         ocrmypdf.PdfMergeFailedError: If the input PDF is malformed, preventing merging
             with the OCR layer.

src/ocrmypdf/cli.py

@@ -482,3 +482,4 @@ debugging.add_argument(
     action='store_true',
     help="Keep temporary files (helpful for debugging)",
 )
+debugging.add_argument('--tesseract-env', type=str, help=argparse.SUPPRESS)

src/ocrmypdf/exec/__init__.py

@@ -28,7 +28,7 @@ from collections.abc import Mapping
 log = logging.Logger(__name__)
 
 
-def get_version(program, *, version_arg='--version', regex=r'(\d+(\.\d+)*)'):
+def get_version(program, *, version_arg='--version', regex=r'(\d+(\.\d+)*)', env=None):
     "Get the version of the specified program"
     args_prog = [program, version_arg]
     try:
@@ -39,6 +39,7 @@ def get_version(program, *, version_arg='--version', regex=r'(\d+(\.\d+)*)'):
             stdout=PIPE,
             stderr=STDOUT,
             check=True,
+            env=env,
         )
         output = proc.stdout
     except FileNotFoundError as e:

src/ocrmypdf/exec/tesseract.py

@@ -60,18 +60,16 @@ HOCR_TEMPLATE = """<?xml version="1.0" encoding="UTF-8"?>
 """
 
 
-@lru_cache(maxsize=1)
-def version():
-    return get_version('tesseract', regex=r'tesseract\s(.+)')
+def version(tesseract_env=None):
+    return get_version('tesseract', regex=r'tesseract\s(.+)', env=tesseract_env)
 
 
-def v4():
+def v4(tesseract_env=None):
     "Is this Tesseract v4.0?"
-    return version() >= '4'
+    return version(tesseract_env) >= '4'
 
 
-@lru_cache(maxsize=1)
-def has_textonly_pdf():
+def has_textonly_pdf(tesseract_env=None):
     """Does Tesseract have textonly_pdf capability?
 
     Available in v4.00.00alpha since January 2017. Best to
@@ -80,7 +78,15 @@ def has_textonly_pdf():
     args_tess = ['tesseract', '--print-parameters', 'pdf']
     params = ''
     try:
-        params = check_output(args_tess, universal_newlines=True, stderr=STDOUT)
+        proc = run(
+            args_tess,
+            check=True,
+            universal_newlines=True,
+            stdout=PIPE,
+            stderr=STDOUT,
+            env=tesseract_env,
+        )
+        params = proc.stdout
     except CalledProcessError as e:
         print("Could not --print-parameters from tesseract", file=sys.stderr)
         raise MissingDependencyError from e
@@ -89,8 +95,7 @@ def has_textonly_pdf():
     return False
 
 
-@lru_cache(maxsize=1)
-def languages():
+def languages(tesseract_env=None):
     def lang_error(output):
         msg = dedent(
             """Tesseract failed to report available languages.
@@ -104,7 +109,12 @@ def languages():
     args_tess = ['tesseract', '--list-langs']
     try:
         proc = run(
-            args_tess, universal_newlines=True, stdout=PIPE, stderr=STDOUT, check=True
+            args_tess,
+            universal_newlines=True,
+            stdout=PIPE,
+            stderr=STDOUT,
+            check=True,
+            env=tesseract_env,
         )
         output = proc.stdout
     except CalledProcessError as e:
@@ -127,7 +137,7 @@ def tess_base_args(langs, engine_mode):
     return args
 
 
-def get_orientation(input_file, engine_mode, timeout: float, log):
+def get_orientation(input_file, engine_mode, timeout: float, log, tesseract_env=None):
     args_tesseract = tess_base_args(['osd'], engine_mode) + [
         '--psm',
         '0',
@@ -136,7 +146,15 @@ def get_orientation(input_file, engine_mode, timeout: float, log):
     ]
 
     try:
-        stdout = check_output(args_tesseract, stderr=STDOUT, timeout=timeout)
+        p = run(
+            args_tesseract,
+            stdout=PIPE,
+            stderr=STDOUT,
+            timeout=timeout,
+            check=True,
+            env=tesseract_env,
+        )
+        stdout = p.stdout
     except TimeoutExpired:
         return OrientationConfidence(angle=0, confidence=0.0)
     except CalledProcessError as e:
@@ -235,6 +253,7 @@ def generate_hocr(
     pagesegmode: int,
     user_words,
     user_patterns,
+    tesseract_env,
     log,
 ):
 
@@ -258,7 +277,15 @@ def generate_hocr(
     args_tesseract.extend([input_file, prefix, 'hocr', 'txt'] + tessconfig)
     try:
         log.debug(args_tesseract)
-        stdout = check_output(args_tesseract, stderr=STDOUT, timeout=timeout)
+        p = run(
+            args_tesseract,
+            stdout=PIPE,
+            stderr=STDOUT,
+            timeout=timeout,
+            check=True,
+            env=tesseract_env,
+        )
+        stdout = p.stdout
     except TimeoutExpired:
         # Generate a HOCR file with no recognized text if tesseract times out
         # Temporary workaround to hocrTransform not being able to function if
@@ -310,9 +337,10 @@ def generate_pdf(
     pagesegmode: int,
     user_words,
     user_patterns,
+    tesseract_env,
     log,
 ):
-    '''Use Tesseract to render a PDF.
+    """Use Tesseract to render a PDF.
 
     input_image -- image to analyze
     skip_pdf -- if we time out, use this file as output
@@ -324,14 +352,14 @@ def generate_pdf(
     tessconfig -- tesseract configuration
     timeout -- timeout (seconds)
     log -- logger object
-    '''
+    """
 
     args_tesseract = tess_base_args(language, engine_mode)
 
     if pagesegmode is not None:
         args_tesseract.extend(['--psm', str(pagesegmode)])
 
-    if text_only and has_textonly_pdf():
+    if text_only and has_textonly_pdf(tesseract_env):
         args_tesseract.extend(['-c', 'textonly_pdf=1'])
 
     if user_words:
@@ -348,7 +376,15 @@ def generate_pdf(
     args_tesseract.extend([input_image, prefix, 'pdf', 'txt'] + tessconfig)
     try:
         log.debug(args_tesseract)
-        stdout = check_output(args_tesseract, stderr=STDOUT, timeout=timeout)
+        p = run(
+            args_tesseract,
+            stdout=PIPE,
+            stderr=STDOUT,
+            timeout=timeout,
+            check=True,
+            env=tesseract_env,
+        )
+        stdout = p.stdout
         if os.path.exists(prefix + '.txt'):
             shutil.move(prefix + '.txt', output_text)
     except TimeoutExpired:

tests/cache/cmyk/__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt/stderr.bin

@@ -0,0 +1 @@
+Tesseract Open Source OCR Engine v4.0.0 with Leptonica

tests/cache/cmyk/__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt/txt.bin

@@ -0,0 +1,13 @@
+Portez ce vieux whisky au juge
+blond qui fume sur son Ile
+interieure, a cöte de l'alcöve
+ovoide, oU les büches se
+consument dans l'ätre, ce qui
+lui permet de penser & la
+caenogenese de |'etre dont il
+est question dans la cause
+ambigu& entendue a MoY, dans
+un capharnaüm qui, pense-t-il,
+diminue ca et la la qualite de son
+ceuvre.
+

tests/cache/lichtenstein/__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt/stderr.bin

@@ -0,0 +1 @@
+Tesseract Open Source OCR Engine v4.0.0 with Leptonica

tests/cache/lichtenstein/__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt/txt.bin

@@ -0,0 +1,13 @@
+Portez ce vieux whisky au juge
+blond qui fume sur son Ile
+interieure, a cöte de l'alcöve
+ovoide, oU les büches se
+consument dans l'ätre, ce qui
+lui permet de penser & la
+caenogenese de |'etre dont il
+est question dans la cause
+ambigu& entendue a MoY, dans
+un capharnaüm qui, pense-t-il,
+diminue ca et la la qualite de son
+ceuvre.
+

tests/cache/manifest.jsonl

@@ -63,3 +63,5 @@
 {"tesseract_version": "tesseract 4.0.0  leptonica-1.78.0   libgif 5.1.4 : libjpeg 9c : libpng 1.6.37 : libtiff 4.0.10 : zlib 1.2.11 : libwebp 1.0.2 : libopenjp2 2.3.1  Found AVX2  Found AVX  Found SSE ", "platform": "Darwin-18.5.0-x86_64-i386-64bit", "python": "3.7.3", "argv_slug": "__-l__eng__000002_ocr.png__000002_ocr_tess__pdf__txt", "sourcefile": "resources/multipage.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000002_ocr.png", "$TMPDIR/000002_ocr_tess", "pdf", "txt"]}
 {"tesseract_version": "tesseract 4.0.0  leptonica-1.78.0   libgif 5.1.4 : libjpeg 9c : libpng 1.6.37 : libtiff 4.0.10 : zlib 1.2.11 : libwebp 1.0.2 : libopenjp2 2.3.1  Found AVX2  Found AVX  Found SSE ", "platform": "Darwin-18.5.0-x86_64-i386-64bit", "python": "3.7.3", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/poster.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
 {"tesseract_version": "tesseract 4.0.0  leptonica-1.78.0   libgif 5.1.4 : libjpeg 9c : libpng 1.6.37 : libtiff 4.0.10 : zlib 1.2.11 : libwebp 1.0.2 : libopenjp2 2.3.1  Found AVX2  Found AVX  Found SSE ", "platform": "Darwin-18.5.0-x86_64-i386-64bit", "python": "3.7.3", "argv_slug": "__-l__eng__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/poster.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "tesseract 4.0.0  leptonica-1.78.0   libgif 5.1.4 : libjpeg 9c : libpng 1.6.37 : libtiff 4.0.10 : zlib 1.2.11 : libwebp 1.0.2 : libopenjp2 2.3.1  Found AVX2  Found AVX  Found SSE ", "platform": "Darwin-18.6.0-x86_64-i386-64bit", "python": "3.7.3", "argv_slug": "__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/cmyk.pdf", "args": ["-l", "deu", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "tesseract 4.0.0  leptonica-1.78.0   libgif 5.1.4 : libjpeg 9c : libpng 1.6.37 : libtiff 4.0.10 : zlib 1.2.11 : libwebp 1.0.2 : libopenjp2 2.3.1  Found AVX2  Found AVX  Found SSE ", "platform": "Darwin-18.6.0-x86_64-i386-64bit", "python": "3.7.3", "argv_slug": "__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/lichtenstein.pdf", "args": ["-l", "deu", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}

tests/conftest.py

@@ -21,6 +21,7 @@ import sys
 from contextlib import contextmanager
 from pathlib import Path
 from subprocess import PIPE, run
+from ocrmypdf import api, cli
 
 import pytest
 
@@ -185,18 +186,21 @@ def no_outpdf(tmp_path):
 def check_ocrmypdf(input_file, output_file, *args, env=None):
     """Run ocrmypdf and confirmed that a valid file was created"""
 
-    p, out, err = run_ocrmypdf(input_file, output_file, *args, env=env)
-    # ensure py.test collects the output, use -s to view
-    print(err, file=sys.stderr)
-    assert p.returncode == 0
+    # p, out, err = run_ocrmypdf(input_file, output_file, *args, env=env)
+
+    options = cli.parser.parse_args(
+        [str(input_file), str(output_file)] + [str(arg) for arg in args]
+    )
+    api.check_options(options)
+    if env:
+        options.tesseract_env = env
+        options.tesseract_env['_OCRMYPDF_TEST_INFILE'] = input_file
+    result = api.run_pipeline(options, api=True)
+
+    assert result == 0
     assert os.path.exists(str(output_file)), "Output file not created"
     assert os.stat(str(output_file)).st_size > 100, "PDF too small or empty"
-    assert out == "", (
-        "The following was written to stdout and should not have been: \n"
-        + "<stdout>\n"
-        + out
-        + "\n</stdout>"
-    )
+
     return output_file
 
 

tests/test_filters.py

@@ -26,7 +26,6 @@ from ocrmypdf.filters import invert, whiteout
 from ocrmypdf._plugins import load_plugin
 
 
-os_environ = pytest.helpers.os_environ
 check_ocrmypdf = pytest.helpers.check_ocrmypdf
 
 

tests/test_main.py

@@ -40,7 +40,6 @@ from ocrmypdf.pdfinfo import Colorspace, Encoding, PdfInfo
 check_ocrmypdf = pytest.helpers.check_ocrmypdf
 run_ocrmypdf = pytest.helpers.run_ocrmypdf
 spoof = pytest.helpers.spoof
-os_environ = pytest.helpers.os_environ
 
 
 RENDERERS = ['hocr', 'sandwich']
@@ -614,8 +613,10 @@ def test_closed_streams(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf):
 
 
 def test_masks(spoof_tesseract_noop, resources, outpdf):
-    with os_environ(spoof_tesseract_noop):
-        assert ocrmypdf(resources / 'masks.pdf', outpdf) == ExitCode.ok
+    assert (
+        ocrmypdf(resources / 'masks.pdf', outpdf, tesseract_env=spoof_tesseract_noop)
+        == ExitCode.ok
+    )
 
 
 def test_linearized_pdf_and_indirect_object(spoof_tesseract_noop, resources, outpdf):