Update develop with master changes

We’re well out of the “trivial updates” zone

tests/resources/README.rst

@@ -34,6 +34,9 @@ In some cases they were converted from one image format to another without other
     *   - typewriter.png, 2400dpi.pdf
         - `Wikimedia: Triumph typewrtier text Linzensoep`_
         * Creative Commons BY-SA 2.5
+    *   - baiona.png
+        - `Wikimedia: Baionako udalerri mugakideak`_
+        - Creative Commons BY-SA 4.0
 
 
 Files generated for this project
@@ -85,6 +88,9 @@ under the terms of the license in LICENSE.rst.
     *   - overlay.pdf
         - @maxandersen
         - PDF file generated by PDFPen pro that triggered content stream parse errors
+    *   - no_conentes.pdf
+        - @jbarlow83
+        - synthetic PDF with a blank page that has no /Contents entry
 
 Assemblies
 ==========
@@ -115,4 +121,6 @@ These test resources are assemblies from other previously mentioned files, relea
 
 .. _`Linux (Wikipedia Article)`: https://de.wikipedia.org/wiki/Linux 
 
-.. _`Wikimedia: Triumph typewrtier text Linzensoep`: https://commons.wikimedia.org/wiki/File:Triumph.typewriter_text_Linzensoep.gif
+.. _`Wikimedia: Triumph typewrtier text Linzensoep`: https://commons.wikimedia.org/wiki/File:Triumph.typewriter_text_Linzensoep.gif
+
+.. _`Wikimedia: Baionako udalerri mugakideak`: https://commons.wikimedia.org/wiki/File:Baionako_udalerri_mugakideak.png

tests/resources/no_contents.pdf

@@ -0,0 +1,21 @@
+%PDF-1.3
+%¿÷¢þ
+1 0 obj
+<< /Pages 2 0 R /Type /Catalog >>
+endobj
+2 0 obj
+<< /Count 1 /Kids [ 3 0 R ] /Type /Pages >>
+endobj
+3 0 obj
+<< /MediaBox [ 0 0 720 720 ] /Parent 2 0 R /Resources << >> /Type /Page >>
+endobj
+xref
+0 4
+0000000000 65535 f 
+0000000015 00000 n 
+0000000064 00000 n 
+0000000123 00000 n 
+trailer << /Root 1 0 R /Size 4 /ID [<52bba3c78160d0c6e851b59110e5d076><52bba3c78160d0c6e851b59110e5d076>] >>
+startxref
+213
+%%EOF

tests/spoof/gs_raster_failure.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+import sys
+import os
+
+"""Replicate Ghostscript raster failure while allowing rendering"""
+
+
+def real_ghostscript(argv):
+    gs_args = ['gs'] + argv[1:]
+    os.execvp("gs", gs_args)
+    return  # Not reachable
+
+
+def main():
+    if '--version' in sys.argv:
+        print('9.20')
+        print('SPOOFED: ' + os.path.basename(__filename__))
+        sys.exit(0)
+
+    # For any rendering calls (device == pdfwrite) call real ghostscript
+    if '-sDEVICE=pdfwrite' in sys.argv:
+        real_ghostscript(sys.argv)
+        return
+
+    # Fail
+    print("ERROR: Ghost story archive not found")
+    sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

tests/spoof/gs_render_failure.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+import sys
+import os
+
+"""Replicate Ghostscript render failure while allowing rasterizing"""
+
+
+def real_ghostscript(argv):
+    gs_args = ['gs'] + argv[1:]
+    os.execvp("gs", gs_args)
+    return  # Not reachable
+
+
+def main():
+    if '--version' in sys.argv:
+        print('9.20')
+        print('SPOOFED: ' + os.path.basename(__filename__))
+        sys.exit(0)
+
+    # For any rasterize calls (device != pdfwrite) call real ghostscript
+    if '-sDEVICE=pdfwrite' not in sys.argv:
+        real_ghostscript(sys.argv)
+        return
+
+    # Fail
+    print("ERROR: Casper is not a friendly ghost")
+    sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

tests/spoof/tesseract_cache.py

@@ -36,6 +36,10 @@ def real_tesseract():
 
 def main():
     operation = sys.argv[-1]
+    sidecar = False
+    if sys.argv[-2] == 'txt':
+        sidecar = True
+
     # For anything unexpected operation, defer to real tesseract binary
     # Currently this includes all use of "--tesseract-config"
     if operation != 'hocr' and operation != 'pdf' and operation != 'stdout':
@@ -92,16 +96,22 @@ def main():
         return
 
     if operation == 'stdout':
+        # tesseract [--options] ... input stdout
         input_file = sys.argv[-2]
         output_file = 'stdout'
+        sidecar_file = ''
     else:
-        input_file = sys.argv[-3]
-        output_file = sys.argv[-2]
+        # tesseract [--options] ... input output txt hocr|pdf
+        input_file = sys.argv[-4]
+        output_file = sys.argv[-3]
+        sidecar_file = sys.argv[-3]
 
     if operation == 'hocr':
         output_file += '.hocr'
+        sidecar_file += '.txt'
     elif operation == 'pdf':
         output_file += '.pdf'
+        sidecar_file += '.txt'
 
     with open(input_file, 'rb') as f:
         m.update(f.read())
@@ -112,6 +122,8 @@ def main():
         print("Tesseract cache hit", file=sys.stderr)
         if operation != 'stdout':
             shutil.copy(cache_name, output_file)
+            if sidecar:
+                shutil.copy(cache_name + '.sidecar', sidecar_file)
 
         # Replicate output
         with open(cache_name + '.stdout', 'rb') as f:
@@ -149,6 +161,8 @@ def main():
             shutil.copy(output_file, cache_name)
         else:
             print("Could not find output file", file=sys.stderr)
+        if sidecar and os.path.exists(sidecar_file):
+            shutil.copy(sidecar_file, cache_name + '.sidecar')
     else:
         open(cache_name, 'w').close()
 

tests/spoof/tesseract_noop.py

@@ -53,18 +53,22 @@ def main():
         print('List of available languages (1):\neng', file=sys.stderr)
         sys.exit(0)
     elif sys.argv[-1] == 'hocr':
-        inputf = sys.argv[-3]
-        output = sys.argv[-2]
+        inputf = sys.argv[-4]
+        output = sys.argv[-3]
         with Image.open(inputf) as im, \
                 open(output + '.hocr', 'w', encoding='utf-8') as f:
             w, h = im.size
             f.write(HOCR_TEMPLATE.format(str(w), str(h)))
+        with open(output + '.txt', 'w') as f:
+            f.write('')
     elif sys.argv[-1] == 'pdf':
-        inputf = sys.argv[-3]
-        output = sys.argv[-2]
+        inputf = sys.argv[-4]
+        output = sys.argv[-3]
         pdf_bytes = img2pdf.convert([inputf], dpi=300)
         with open(output + '.pdf', 'wb') as f:
             f.write(pdf_bytes)
+        with open(output + '.txt', 'w') as f:
+            f.write('')
     elif sys.argv[-1] == 'stdout':
         inputf = sys.argv[-2]
         print("""Orientation: 0

tests/test_main.py

@@ -51,6 +51,16 @@ def spoof_no_tess_pdfa_warning():
     return spoof(tesseract='tesseract_noop.py', gs='gs_feature_elision.py')
 
 
+@pytest.fixture
+def spoof_no_tess_gs_render_fail():
+    return spoof(tesseract='tesseract_noop.py', gs='gs_render_failure.py')
+
+
+@pytest.fixture
+def spoof_no_tess_gs_raster_fail():
+    return spoof(tesseract='tesseract_noop.py', gs='gs_raster_failure.py')
+
+
 @pytest.fixture
 def spoof_qpdf_always_error():
     return spoof(qpdf='qpdf_dummy_return2.py')
@@ -137,14 +147,18 @@ def test_remove_background(spoof_tesseract_noop, resources, outdir):
 @pytest.mark.parametrize("output_type", ['pdf', 'pdfa'])
 def test_exotic_image(spoof_tesseract_cache, pdf, renderer, output_type,
                       resources, outdir):
+    outfile = outdir / 'test_{0}_{1}.pdf'.format(pdf, renderer)
     check_ocrmypdf(
         resources / pdf,
-        outdir / 'test_{0}_{1}.pdf'.format(pdf, renderer),
+        outfile,
         '-dc',
         '-v', '1',
         '--output-type', output_type,
+        '--sidecar',
         '--pdf-renderer', renderer, env=spoof_tesseract_cache)
 
+    assert outfile.with_suffix('.pdf.txt').exists()
+
 
 @pytest.mark.parametrize("output_type", [
     'pdfa', 'pdf'
@@ -738,6 +752,11 @@ def test_very_high_dpi(spoof_tesseract_cache, resources, outpdf):
     "Checks for a Decimal quantize error with high DPI, etc"
     check_ocrmypdf(resources / '2400dpi.pdf', outpdf,
                    env=spoof_tesseract_cache)
+    pdfinfo = pdf_get_all_pageinfo(outpdf)
+
+    image = pdfinfo[0]['images'][0]
+    assert image['dpi_w'] == image['dpi_h']
+    assert image['dpi_w'] == 2400
 
 
 def test_overlay(spoof_tesseract_noop, resources, outpdf):
@@ -835,3 +854,138 @@ def test_pagesize_consistency(renderer, resources, outpdf):
 
     assert isclose(before_dims[0], after_dims[0])
     assert isclose(before_dims[1], after_dims[1])
+
+
+def test_skip_big_with_no_images(spoof_tesseract_noop, resources, outpdf):
+    check_ocrmypdf(resources / 'blank.pdf', outpdf,
+                   '--skip-big', '5',
+                   '--force-ocr',
+                   env=spoof_tesseract_noop)
+
+
+def test_gs_render_failure(spoof_no_tess_gs_render_fail, resources, outpdf):
+    p, out, err = run_ocrmypdf(
+        resources / 'blank.pdf', outpdf,
+        env=spoof_no_tess_gs_render_fail)
+    print(err)
+    assert p.returncode == ExitCode.child_process_error
+
+
+def test_gs_raster_failure(spoof_no_tess_gs_raster_fail, resources, outpdf):
+    p, out, err = run_ocrmypdf(
+        resources / 'ccitt.pdf', outpdf,
+        env=spoof_no_tess_gs_raster_fail)
+    print(err)
+    assert p.returncode == ExitCode.child_process_error
+
+
+def test_no_contents(spoof_tesseract_noop, resources, outpdf):
+    check_ocrmypdf(resources / 'no_contents.pdf', outpdf, '--force-ocr',
+                   env=spoof_tesseract_noop)
+
+
+@pytest.mark.parametrize('image', [
+    'baiona.png',
+    'baiona_gray.png',
+    'congress.jpg'
+    ])
+def test_compression_preserved(spoof_tesseract_noop, ocrmypdf_exec,
+                               resources, image, outpdf):
+    from PIL import Image
+
+    input_file = str(resources / image)
+    output_file = str(outpdf)
+
+    im = Image.open(input_file)
+
+    # Runs: ocrmypdf - output.pdf < testfile
+    with open(input_file, 'rb') as input_stream:
+        p_args = ocrmypdf_exec + [
+            '--image-dpi', '150', '--output-type', 'pdf', '-', output_file]
+        p = Popen(
+            p_args, close_fds=True, stdout=PIPE, stderr=PIPE,
+            stdin=input_stream, env=spoof_tesseract_noop)
+        out, err = p.communicate()
+
+        assert p.returncode == ExitCode.ok
+
+    pdfinfo = pdf_get_all_pageinfo(output_file)
+
+    pdfimage = pdfinfo[0]['images'][0]
+
+    if input_file.endswith('.png'):
+        assert pdfimage['enc'] != 'jpeg', \
+            "Lossless compression changed to lossy!"
+    elif input_file.endswith('.jpg'):
+        assert pdfimage['enc'] == 'jpeg', \
+            "Lossy compression changed to lossless!"
+    if im.mode.startswith('RGB') or im.mode.startswith('BGR'):
+        assert pdfimage['color'] == 'rgb', \
+            "Colorspace changed"
+    elif im.mode.startswith('L'):
+        assert pdfimage['color'] == 'gray', \
+            "Colorspace changed"
+
+
+@pytest.mark.parametrize('image,compression', [
+    ('baiona.png', 'jpeg'),
+    ('baiona_gray.png', 'lossless'),
+    ('congress.jpg', 'lossless')
+    ])
+def test_compression_changed(spoof_tesseract_noop, ocrmypdf_exec,
+                             resources, image, compression, outpdf):
+    from PIL import Image
+
+    input_file = str(resources / image)
+    output_file = str(outpdf)
+
+    im = Image.open(input_file)
+
+    # Runs: ocrmypdf - output.pdf < testfile
+    with open(input_file, 'rb') as input_stream:
+        p_args = ocrmypdf_exec + [
+            '--image-dpi', '150', '--output-type', 'pdfa',
+            '--pdfa-image-compression', compression,
+            '-', output_file]
+        p = Popen(
+            p_args, close_fds=True, stdout=PIPE, stderr=PIPE,
+            stdin=input_stream, env=spoof_tesseract_noop)
+        out, err = p.communicate()
+
+        assert p.returncode == ExitCode.ok
+
+    pdfinfo = pdf_get_all_pageinfo(output_file)
+
+    pdfimage = pdfinfo[0]['images'][0]
+
+    if compression == 'jpeg':
+        assert pdfimage['enc'] == 'jpeg'
+    elif compression == 'lossless':
+        assert pdfimage['enc'] == 'image'
+
+    if im.mode.startswith('RGB') or im.mode.startswith('BGR'):
+        assert pdfimage['color'] == 'rgb', \
+            "Colorspace changed"
+    elif im.mode.startswith('L'):
+        assert pdfimage['color'] == 'gray', \
+            "Colorspace changed"
+
+
+def test_sidecar_pagecount(spoof_tesseract_cache, resources, outpdf):
+    sidecar = outpdf + '.txt'
+    check_ocrmypdf(
+        resources / 'multipage.pdf', outpdf,
+        '--skip-text',
+        '--sidecar', sidecar,
+        env=spoof_tesseract_cache)
+
+    pdfinfo = pdf_get_all_pageinfo(str(resources / 'multipage.pdf'))
+    num_pages = len(pdfinfo)
+
+    with open(sidecar, 'r') as f:
+        ocr_text = f.read()
+
+    # There should a formfeed between each pair of pages, so the count of
+    # formfeeds is the page count less one
+    assert ocr_text.count('\f') == num_pages - 1, \
+        "Sidecar page count does not match PDF page count"

tests/test_pageinfo.py

@@ -107,3 +107,11 @@ def test_form_xobject(resources):
     pdfinfo = pageinfo.pdf_get_all_pageinfo(str(filename))
     pdfimage = pdfinfo[0]['images'][0]
     assert pdfimage['width'] == 50
+
+
+def test_no_contents(resources):
+    filename = resources / 'no_contents.pdf'
+
+    pdfinfo = pageinfo.pdf_get_all_pageinfo(str(filename))
+    assert len(pdfinfo[0]['images']) == 0
+    assert pdfinfo[0]['has_text'] == False

tests/test_tess4.py

@@ -6,11 +6,42 @@ from ocrmypdf.exceptions import ExitCode
 from ocrmypdf.exec import tesseract
 from ocrmypdf import pageinfo
 import sys
+import os
+import PyPDF2 as pypdf
+
+
+spoof = pytest.helpers.spoof
+
+
+def tess4_possible_location():
+    """The location of tesseract 4 may be OCRMYPDF_TESS4, OCRMYPDF_TESSERACT,
+    or the installed version on PATH."""
+    return os.environ.get('OCRMYPDF_TESS4') or \
+           os.environ.get('OCRMYPDF_TESSERACT') or \
+            'tesseract'
+
+
+@pytest.fixture
+def ensure_tess4():
+    return spoof(tesseract=tess4_possible_location())
+
+
+def tess4_available():
+    """Check if a tesseract 4 binary is available, even if it's not the
+    official "tesseract" on PATH
+
+    """
+    old_environ = os.environ.copy()
+    try:
+        os.environ['OCRMYPDF_TESSERACT'] = tess4_possible_location()
+        return tesseract.v4() and tesseract.has_textonly_pdf()
+    finally:
+        os.environ = old_environ
 
 
 # Skip all tests in this file if not tesseract 4
 pytestmark = pytest.mark.skipif(
-    not (tesseract.v4() and tesseract.has_textonly_pdf()),
+    not tess4_available(),
     reason="tesseract 4.0 with textonly_pdf feature required")
 
 check_ocrmypdf = pytest.helpers.check_ocrmypdf
@@ -18,14 +49,16 @@ run_ocrmypdf = pytest.helpers.run_ocrmypdf
 spoof = pytest.helpers.spoof
 
 
-def test_textonly_pdf(resources, outdir):
+def test_textonly_pdf(ensure_tess4, resources, outdir):
     check_ocrmypdf(
         resources / 'linn.pdf',
-        outdir / 'linn_textonly.pdf', '--pdf-renderer', 'tess4')
+        outdir / 'linn_textonly.pdf', '--pdf-renderer', 'tess4',
+        '--sidecar', 'foo',
+        env=ensure_tess4)
 
 
 @pytest.mark.skipif(sys.version_info < (3, 5), reason="needs math.isclose")
-def test_pagesize_consistency_tess4(resources, outpdf):
+def test_pagesize_consistency_tess4(ensure_tess4, resources, outpdf):
     from math import isclose
 
     infile = resources / 'linn.pdf'
@@ -35,9 +68,47 @@ def test_pagesize_consistency_tess4(resources, outpdf):
     check_ocrmypdf(
         infile,
         outpdf, '--pdf-renderer', 'tess4',
-        '--clean', '--deskew', '--remove-background', '--clean-final')
+        '--clean', '--deskew', '--remove-background', '--clean-final',
+        env=ensure_tess4)
 
     after_dims = pytest.helpers.first_page_dimensions(outpdf)
 
     assert isclose(before_dims[0], after_dims[0])
     assert isclose(before_dims[1], after_dims[1])
+
+
+@pytest.mark.parametrize('basename', ['graph_ocred.pdf', 'cardinal.pdf'])
+def test_skip_pages_does_not_replicate(
+        ensure_tess4, resources, basename, outdir):
+    infile = resources / basename
+    outpdf = outdir / basename
+
+    check_ocrmypdf(
+        infile,
+        outpdf, '--pdf-renderer', 'tess4', '--force-ocr',
+        '--tesseract-timeout', '0',
+        env=ensure_tess4
+    )
+
+    info_in = pageinfo.pdf_get_all_pageinfo(str(infile))
+
+    info = pageinfo.pdf_get_all_pageinfo(str(outpdf))
+    for page in info:
+        assert len(page['images']) == 1, "skipped page was replicated"
+
+    for n in range(len(info_in)):
+        assert info[n]['width_inches'] == info_in[n]['width_inches']
+
+
+def test_content_preservation(ensure_tess4, resources, outpdf):
+    infile = resources / 'masks.pdf'
+
+    check_ocrmypdf(
+        infile,
+        outpdf, '--pdf-renderer', 'tess4', '--tesseract-timeout', '0',
+        env=ensure_tess4
+    )
+
+    info = pageinfo.pdf_get_all_pageinfo(str(outpdf))
+    page = info[0]
+    assert len(page['images']) > 1, "masked were rasterized"