From 44a45fc3fb1cc4479215631aa0d7b2cac66926aa Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Wed, 29 Nov 2017 14:08:07 -0800 Subject: [PATCH] Add "bad UTF8 output from Tesseract" test --- docs/release_notes.rst | 7 ++-- ocrmypdf/exec/tesseract.py | 8 ++--- tests/spoof/tesseract_badutf8.py | 58 ++++++++++++++++++++++++++++++++ tests/test_main.py | 18 ++++++++++ 4 files changed, 85 insertions(+), 6 deletions(-) create mode 100755 tests/spoof/tesseract_badutf8.py diff --git a/docs/release_notes.rst b/docs/release_notes.rst index b8771061..63325328 100644 --- a/docs/release_notes.rst +++ b/docs/release_notes.rst @@ -8,12 +8,15 @@ The OCRmyPDF package itself does not contain a public API, although it is fairly v5.4.4 ------ -- Fix issue #181, fix final merge failure for PDFs with more pages than the system file handle limit (``ulimit -n``) -- Fix issue #200, an uncommon syntax for formatting decimal numbers in a PDF would cause qpdf to issue a warning, which ocrmypdf treated as an error. Now this the warning is relayed. +- Fix issue #181: fix final merge failure for PDFs with more pages than the system file handle limit (``ulimit -n``) +- Fix issue #200: an uncommon syntax for formatting decimal numbers in a PDF would cause qpdf to issue a warning, which ocrmypdf treated as an error. Now this the warning is relayed. - Fix an issue where intermediate PDFs would be created at version 1.3 instead of the version of the original file. It's possible but unlikely this had side effects. +- A warning is now issued when older versions of qpdf are used since issues like #200 cause qpdf to infinite-loop +- Address issue #140: if Tesseract outputs invalid UTF-8, escape it and print its message instead of aborting with a Unicode error - Adding previously unlisted setup requirement, pytest-runner - Update documentation: fix an error in the example script for Synology with Docker images, improved security guidance, advised ``pip install --user`` + v5.4.3 ------ diff --git a/ocrmypdf/exec/tesseract.py b/ocrmypdf/exec/tesseract.py index a8532705..0f49ce98 100644 --- a/ocrmypdf/exec/tesseract.py +++ b/ocrmypdf/exec/tesseract.py @@ -250,9 +250,9 @@ def generate_hocr(input_file, output_files, language: list, engine_mode, _generate_null_hocr(output_hocr, output_sidecar, input_file) except CalledProcessError as e: tesseract_log_output(log, e.output, input_file) - if 'read_params_file: parameter not found' in e.output: + if b'read_params_file: parameter not found' in e.output: raise TesseractConfigError() from e - if 'Image too large' in e.output: + if b'Image too large' in e.output: _generate_null_hocr(output_hocr, output_sidecar, input_file) return @@ -342,10 +342,10 @@ def generate_pdf(*, input_image, skip_pdf, output_pdf, output_text, use_skip_page(text_only, skip_pdf, output_pdf, output_text) except CalledProcessError as e: tesseract_log_output(log, e.output, input_image) - if 'read_params_file: parameter not found' in e.output: + if b'read_params_file: parameter not found' in e.output: raise TesseractConfigError() from e - if 'Image too large' in e.output: + if b'Image too large' in e.output: use_skip_page(text_only, skip_pdf, output_pdf, output_text) return raise e from e diff --git a/tests/spoof/tesseract_badutf8.py b/tests/spoof/tesseract_badutf8.py new file mode 100755 index 00000000..cd624cda --- /dev/null +++ b/tests/spoof/tesseract_badutf8.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +# © 2017 James R. Barlow: github.com/jbarlow83 +import sys +import img2pdf +import PyPDF2 as pypdf +from PIL import Image + + +"""Tesseract bad utf8 spoof + +In 'hocr' mode or 'pdf' mode, return error code 1 and some non-Unicode +text because tesseract seems to do that in some cases related to +language pack version mismatches + +""" + + +VERSION_STRING = '''tesseract 3.05.01 + leptonica-1.72 + libjpeg 8d : libpng 1.6.19 : libtiff 4.0.6 : zlib 1.2.5 +SPOOFED +''' + +# Japanese "Invalid UTF-8" encoded in Shift JIS +BAD_UTF8 = b'\x96\xb3\x8c\xf8\x82\xc8UTF-8\x0a' + + +def main(): + if sys.argv[1] == '--version': + print(VERSION_STRING, file=sys.stderr) + sys.exit(0) + elif sys.argv[1] == '--list-langs': + print('List of available languages (1):\neng', file=sys.stderr) + sys.exit(0) + elif sys.argv[1] == '--print-parameters': + print("Some parameters", file=sys.stderr) + print("textonly_pdf\t1\tSome help text") + sys.exit(0) + elif sys.argv[-2] in ('hocr', 'pdf'): + sys.stdout.buffer.write(BAD_UTF8) + sys.exit(1) + elif sys.argv[-1] == 'stdout': + inputf = sys.argv[-2] + print("""Orientation: 0 +Orientation in degrees: 0 +Orientation confidence: 100.00 +Script: 1 +Script confidence: 100.00""", file=sys.stderr) + else: + print("Spoof doesn't understand arguments", file=sys.stderr) + print(sys.argv, file=sys.stderr) + sys.exit(1) + + sys.exit(0) + + +if __name__ == '__main__': + main() diff --git a/tests/test_main.py b/tests/test_main.py index 0a206e8e..c21f2253 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -55,6 +55,11 @@ def spoof_no_tess_gs_raster_fail(): return spoof(tesseract='tesseract_noop.py', gs='gs_raster_failure.py') +@pytest.fixture +def spoof_tess_bad_utf8(): + return spoof(tesseract='tesseract_badutf8.py') + + @pytest.fixture def spoof_qpdf_always_error(): return spoof(qpdf='qpdf_dummy_return2.py') @@ -1019,3 +1024,16 @@ def test_bad_locale(): assert p.returncode != 0 assert 'configured to use ASCII as encoding' in err, "should whine" + +@pytest.mark.parametrize('renderer', RENDERERS) +def test_bad_utf8(spoof_tess_bad_utf8, renderer, resources, no_outpdf): + p, out, err = run_ocrmypdf( + resources / 'ccitt.pdf', no_outpdf, + '--pdf-renderer', renderer, + env=spoof_tess_bad_utf8 + ) + + assert out == '', "stdout not clean" + assert p.returncode != 0 + assert 'not utf-8' in err, "should whine about utf-8" + assert '\\x96' in err, 'should repeat backslash encoded output'