Add "bad UTF8 output from Tesseract" test

docs/release_notes.rst

@@ -8,12 +8,15 @@ The OCRmyPDF package itself does not contain a public API, although it is fairly
 v5.4.4
 ------
 
--   Fix issue #181, fix final merge failure for PDFs with more pages than the system file handle limit (``ulimit -n``)
--   Fix issue #200, an uncommon syntax for formatting decimal numbers in a PDF would cause qpdf to issue a warning, which ocrmypdf treated as an error. Now this the warning is relayed.
+-   Fix issue #181: fix final merge failure for PDFs with more pages than the system file handle limit (``ulimit -n``)
+-   Fix issue #200: an uncommon syntax for formatting decimal numbers in a PDF would cause qpdf to issue a warning, which ocrmypdf treated as an error. Now this the warning is relayed.
 -   Fix an issue where intermediate PDFs would be created at version 1.3 instead of the version of the original file. It's possible but unlikely this had side effects.
+-   A warning is now issued when older versions of qpdf are used since issues like #200 cause qpdf to infinite-loop
+-   Address issue #140: if Tesseract outputs invalid UTF-8, escape it and print its message instead of aborting with a Unicode error 
 -   Adding previously unlisted setup requirement, pytest-runner
 -   Update documentation: fix an error in the example script for Synology with Docker images, improved security guidance, advised ``pip install --user``
 
+
 v5.4.3
 ------
 

ocrmypdf/exec/tesseract.py

@@ -250,9 +250,9 @@ def generate_hocr(input_file, output_files, language: list, engine_mode,
         _generate_null_hocr(output_hocr, output_sidecar, input_file)
     except CalledProcessError as e:
         tesseract_log_output(log, e.output, input_file)
-        if 'read_params_file: parameter not found' in e.output:
+        if b'read_params_file: parameter not found' in e.output:
             raise TesseractConfigError() from e
-        if 'Image too large' in e.output:
+        if b'Image too large' in e.output:
             _generate_null_hocr(output_hocr, output_sidecar, input_file)
             return
 
@@ -342,10 +342,10 @@ def generate_pdf(*, input_image, skip_pdf, output_pdf, output_text,
         use_skip_page(text_only, skip_pdf, output_pdf, output_text)
     except CalledProcessError as e:
         tesseract_log_output(log, e.output, input_image)
-        if 'read_params_file: parameter not found' in e.output:
+        if b'read_params_file: parameter not found' in e.output:
             raise TesseractConfigError() from e
 
-        if 'Image too large' in e.output:
+        if b'Image too large' in e.output:
             use_skip_page(text_only, skip_pdf, output_pdf, output_text)
             return
         raise e from e

tests/spoof/tesseract_badutf8.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+# © 2017 James R. Barlow: github.com/jbarlow83
+import sys
+import img2pdf
+import PyPDF2 as pypdf
+from PIL import Image
+
+
+"""Tesseract bad utf8 spoof
+
+In 'hocr' mode or 'pdf' mode, return error code 1 and some non-Unicode
+text because tesseract seems to do that in some cases related to 
+language pack version mismatches
+
+"""
+
+
+VERSION_STRING = '''tesseract 3.05.01
+ leptonica-1.72
+  libjpeg 8d : libpng 1.6.19 : libtiff 4.0.6 : zlib 1.2.5
+SPOOFED
+'''
+
+# Japanese "Invalid UTF-8" encoded in Shift JIS
+BAD_UTF8 = b'\x96\xb3\x8c\xf8\x82\xc8UTF-8\x0a'
+
+
+def main():
+    if sys.argv[1] == '--version':
+        print(VERSION_STRING, file=sys.stderr)
+        sys.exit(0)
+    elif sys.argv[1] == '--list-langs':
+        print('List of available languages (1):\neng', file=sys.stderr)
+        sys.exit(0)
+    elif sys.argv[1] == '--print-parameters':
+        print("Some parameters", file=sys.stderr)
+        print("textonly_pdf\t1\tSome help text")
+        sys.exit(0)
+    elif sys.argv[-2] in ('hocr', 'pdf'):
+        sys.stdout.buffer.write(BAD_UTF8)
+        sys.exit(1)
+    elif sys.argv[-1] == 'stdout':
+        inputf = sys.argv[-2]
+        print("""Orientation: 0
+Orientation in degrees: 0
+Orientation confidence: 100.00
+Script: 1
+Script confidence: 100.00""", file=sys.stderr)
+    else:
+        print("Spoof doesn't understand arguments", file=sys.stderr)
+        print(sys.argv, file=sys.stderr)
+        sys.exit(1)
+
+    sys.exit(0)
+
+
+if __name__ == '__main__':
+    main()

tests/test_main.py

@@ -55,6 +55,11 @@ def spoof_no_tess_gs_raster_fail():
     return spoof(tesseract='tesseract_noop.py', gs='gs_raster_failure.py')
 
 
+@pytest.fixture
+def spoof_tess_bad_utf8():
+    return spoof(tesseract='tesseract_badutf8.py')
+
+
 @pytest.fixture
 def spoof_qpdf_always_error():
     return spoof(qpdf='qpdf_dummy_return2.py')
@@ -1019,3 +1024,16 @@ def test_bad_locale():
     assert p.returncode != 0
     assert 'configured to use ASCII as encoding' in err, "should whine"
 
+
+@pytest.mark.parametrize('renderer', RENDERERS)
+def test_bad_utf8(spoof_tess_bad_utf8, renderer, resources, no_outpdf):
+    p, out, err = run_ocrmypdf(
+        resources / 'ccitt.pdf', no_outpdf,
+        '--pdf-renderer', renderer,
+        env=spoof_tess_bad_utf8
+    )
+
+    assert out == '', "stdout not clean"
+    assert p.returncode != 0
+    assert 'not utf-8' in err, "should whine about utf-8"
+    assert '\\x96' in err, 'should repeat backslash encoded output'