v4.5.3 release notes update

If tess4 renderer needed to skip OCR on a page it would end up
duplicating the page contents onto the new page, rather than creating
a blank OCR layer and placing it on the output page. This created
duplicated content in output files.

RELEASE_NOTES.rst

@@ -4,6 +4,15 @@ RELEASE NOTES
 OCRmyPDF uses `semantic versioning <http://semver.org/>`_.
 
 
+v4.5.3
+======
+
+-  Added a workaround for Ghostscript 9.21 and probably earlier versions would fail with the error message "VMerror -25", due to a Ghostscript bug in XMP metadata handling
+-  High Unicode characters (U+10000 and up) are no longer accepted for setting metadata on the command line, as Ghostscript may not handle them correctly.
+-  Fixed an issue where the ``tess4`` renderer would duplicate content onto output pages if tesseract failed or timed out
+-  Fixed ``tess4`` renderer not recognized when lossless reconstruction is possible
+
+
 v4.5.2
 ======
 

ocrmypdf/__main__.py

@@ -53,7 +53,6 @@ if tesseract.version() < MINIMUM_TESS_VERSION:
             MINIMUM_TESS_VERSION, tesseract.version()))
     sys.exit(ExitCode.missing_dependency)
 
-
 # -------------
 # Parser
 
@@ -291,7 +290,7 @@ def check_options_output(options, log):
             "--pdf-renderer=tesseract.")
 
     lossless_reconstruction = False
-    if options.pdf_renderer == 'hocr':
+    if options.pdf_renderer in ('hocr', 'tess4'):
         if not any((options.deskew, options.clean_final, options.force_ocr,
                    options.remove_background)):
             lossless_reconstruction = True
@@ -344,13 +343,31 @@ def check_options_advanced(options, log):
             "commit 3d9fb3b or later")
 
 
+def check_options_metadata(options, log):
+    import unicodedata
+    metadata = [options.title, options.author, options.keywords,
+                options.subject]
+    for s in (m for m in metadata if m):
+        for c in s:
+            if unicodedata.category(c) == 'Co' or ord(c) >= 0x10000:
+                raise ValueError(
+                    "One of the metadata strings contains "
+                    "an unsupported Unicode character: '{}' (U+{})".format(
+                        c, hex(ord(c))[2:].upper()
+                ))
+
+
 def check_options(options, log):
     try:
         check_options_languages(options, log)
+        check_options_metadata(options, log)
         check_options_output(options, log)
         check_options_preprocessing(options, log)
         check_options_ocr_behavior(options, log)
         check_options_advanced(options, log)
+    except ValueError as e:
+        log.error(e)
+        sys.exit(ExitCode.bad_args)
     except argparse.ArgumentError as e:
         log.error(e)
         sys.exit(ExitCode.bad_args)

ocrmypdf/exec/ghostscript.py

@@ -81,7 +81,7 @@ def generate_pdfa(pdf_pages, output_file, log, threads=1):
                   universal_newlines=True)
         stdout, _ = p.communicate()
 
-        if 'error' in stdout:
+        if 'error' in stdout or 'ERROR' in stdout:
             log.error(stdout)
         elif 'overprint mode not set' in stdout:
             # Unless someone is going to print PDF/A documents on a

ocrmypdf/exec/tesseract.py

@@ -11,6 +11,7 @@ from ..helpers import page_number
 from . import get_program
 from collections import namedtuple
 from textwrap import dedent
+import PyPDF2 as pypdf
 
 from subprocess import Popen, PIPE, CalledProcessError, \
     TimeoutExpired, check_output, STDOUT, DEVNULL
@@ -186,6 +187,8 @@ def tesseract_log_output(log, stdout, input_file):
             log.warning(prefix + "unsure about page orientation")
         elif 'error' in line.lower() or 'exception' in line.lower():
             log.error(prefix + line.strip())
+        elif 'warning' in line.lower():
+            log.warning(prefix + line.strip())
         elif 'read_params_file' in line.lower():
             log.error(prefix + line.strip())
         else:
@@ -270,6 +273,24 @@ def generate_hocr(input_file, output_hocr, language: list, engine_mode,
                 f_out.write(line)
 
 
+def use_skip_page(text_only, skip_pdf, output_pdf):
+    if not text_only:
+        os.symlink(skip_pdf, output_pdf)
+        return
+
+    # For text only we must create a blank page with dimensions identical
+    # to the skip page because this is equivalent to a page with no text
+
+    pdf_in = pypdf.PdfFileReader(skip_pdf)
+    page0 = pdf_in.pages[0]
+
+    with open(output_pdf, 'wb') as out:
+        pdf_out = pypdf.PdfFileWriter()
+        w, h = page0.mediaBox.getWidth(), page0.mediaBox.getHeight()
+        pdf_out.addBlankPage(w, h)
+        pdf_out.write(out)
+
+
 def generate_pdf(input_image, skip_pdf, output_pdf, language: list,
                  engine_mode, text_only: bool,
                  tessconfig: list, timeout: float, pagesegmode: int, log):
@@ -307,14 +328,14 @@ def generate_pdf(input_image, skip_pdf, output_pdf, language: list,
             universal_newlines=True, timeout=timeout)
     except TimeoutExpired:
         page_timedout(log, input_image)
-        shutil.copy(skip_pdf, output_pdf)
+        use_skip_page(text_only, skip_pdf, output_pdf)
     except CalledProcessError as e:
         tesseract_log_output(log, e.output, input_image)
         if 'read_params_file: parameter not found' in e.output:
             raise TesseractConfigError() from e
 
         if 'Image too large' in e.output:
-            shutil.copy(skip_pdf, output_pdf)
+            use_skip_page(text_only, skip_pdf, output_pdf)
             return
         raise e from e
     else:

ocrmypdf/pdfa.py

@@ -26,11 +26,7 @@ pdfa_def_template = u"""%!
 /ICCProfile ($icc_profile)
 def
 
-[ /Title <$title>
-  /Author <$author>
-  /Subject <$subject>
-  /Keywords <$keywords>
-  /Creator <$creator>
+[$docinfo
   /DOCINFO pdfmark
 
 % Define an ICC profile :
@@ -89,20 +85,30 @@ def encode_text_string(s: str) -> str:
 
 
 def _get_pdfa_def(icc_profile, icc_identifier, pdfmark):
-    pdfmark_utf16 = {k: encode_text_string(v) for k, v in pdfmark.items()}
+    # Ghostscript <= 9.21 has a bug where null entries in DOCINFO might produce
+    # ERROR: VMerror (-25) on closing pdfwrite device.
+    # https://bugs.ghostscript.com/show_bug.cgi?id=697684
+    # Work around this by only adding keys that have a nontrivial value
+    docinfo_keys = ('/Title', '/Author', '/Subject', '/Creator', '/Keywords')
+    docinfo_line_template = '  {key} <{value}>'
+
+    def docinfo_gen():
+        for key in docinfo_keys:
+            if key in pdfmark and pdfmark[key].strip() != '':
+                line = docinfo_line_template.format(
+                    key=key, value=encode_text_string(pdfmark[key]))
+                yield line
+
+    docinfo = '\n'.join(docinfo_gen())
 
     t = Template(pdfa_def_template)
     result = t.substitute(icc_profile=icc_profile,
                           icc_identifier=icc_identifier,
-                          title=pdfmark_utf16.get('/Title', ''),
-                          author=pdfmark_utf16.get('/Author', ''),
-                          subject=pdfmark_utf16.get('/Subject', ''),
-                          creator=pdfmark_utf16.get('/Creator', ''),
-                          keywords=pdfmark_utf16.get('/Keywords', ''))
+                          docinfo=docinfo)
     return result
 
 
-def generate_pdfa_def(target_filename, pdfmark, icc='sRGB'):
+def generate_pdfa_ps(target_filename, pdfmark, icc='sRGB'):
     if icc == 'sRGB':
         icc_profile = SRGB_ICC_PROFILE
     else:
@@ -116,6 +122,10 @@ def generate_pdfa_def(target_filename, pdfmark, icc='sRGB'):
         f.write(ps)
 
 
+# The old name is generate_pdfa_def -- now deprecated
+generate_pdfa_def = generate_pdfa_ps
+
+
 def file_claims_pdfa(filename):
     """Determines if the file claims to be PDF/A compliant
 

ocrmypdf/pipeline.py

@@ -23,7 +23,7 @@ from ruffus import formatter, regex, Pipeline, suffix
 
 from .hocrtransform import HocrTransform
 from .pageinfo import pdf_get_all_pageinfo
-from .pdfa import generate_pdfa_def, file_claims_pdfa
+from .pdfa import generate_pdfa_ps, file_claims_pdfa
 from .helpers import re_symlink, is_iterable_notstr, page_number
 from .exec import ghostscript, tesseract, qpdf
 from .exceptions import *
@@ -767,13 +767,14 @@ def generate_postscript_stub(
     options = context.get_options()
     pdf = pypdf.PdfFileReader(input_file)
     pdfmark = get_pdfmark(pdf, options)
-    generate_pdfa_def(output_file, pdfmark)
+    generate_pdfa_ps(output_file, pdfmark)
 
 
 def skip_page(
         input_file,
         output_file,
-        log):
+        log,
+        context):
     # The purpose of this step is its filter to forward only the skipped
     # files (.skip.oriented.pdf) while disregarding the processed ones
     # (.ocr.oriented.pdf).  Alternative would be for merge_pages to filter
@@ -1036,7 +1037,7 @@ def build_pipeline(options, work_folder, log, context):
         task_func=generate_postscript_stub,
         input=task_repair_pdf,
         filter=formatter(r'\.repaired\.pdf'),
-        output=os.path.join(work_folder, 'pdfa_def.ps'),
+        output=os.path.join(work_folder, 'pdfa.ps'),
         extras=[log, context])
     task_generate_postscript_stub.active_if(options.output_type == 'pdfa')
 
@@ -1048,7 +1049,7 @@ def build_pipeline(options, work_folder, log, context):
         filter=suffix('.skip.oriented.pdf'),
         output='.done.pdf',
         output_dir=work_folder,
-        extras=[log])
+        extras=[log, context])
 
     # Merge pages
     task_merge_pages_ghostscript = main_pipeline.merge(

tests/test_main.py

@@ -167,55 +167,50 @@ def test_preserve_metadata(spoof_tesseract_noop, output_type,
     assert pdfa_info['output'] == output_type
 
 
-@pytest.mark.skipif(
-    pytest.helpers.is_linux() and not pytest.helpers.running_in_docker(),
-    reason="likely to fail if Linux locale is not configured correctly")
-@pytest.mark.skipif(
-    pytest.helpers.is_macos() and pytest.helpers.running_in_travis(),
-    reason="save Travis the trouble of installing poppler")
-@pytest.mark.xfail(
-    ghostscript.version() == '9.21',
-    reason="gs 9.21 has a regression that affects this"
-    )
 @pytest.mark.parametrize("output_type", [
     'pdfa', 'pdf'
     ])
 def test_override_metadata(spoof_tesseract_noop, output_type, resources,
                            outpdf):
     input_file = resources / 'c02-22.pdf'
-
     german = 'Du siehst den Wald vor lauter Bäumen nicht.'
     chinese = '孔子'
-    high_unicode = 'U+1030C is: 𐌌'
 
     p, out, err = run_ocrmypdf(
         input_file, outpdf,
         '--title', german,
         '--author', chinese,
-        '--subject', high_unicode,
         '--output-type', output_type,
         env=spoof_tesseract_noop)
 
     assert p.returncode == ExitCode.ok, err
 
-    pdf = str(outpdf)
+    reader = pypdf.PdfFileReader(outpdf)
 
-    out_pdfinfo = check_output(['pdfinfo', pdf], universal_newlines=True)
-    lines_pdfinfo = out_pdfinfo.splitlines()
-    pdfinfo = {}
-    for line in lines_pdfinfo:
-        k, v = line.strip().split(':', maxsplit=1)
-        pdfinfo[k.strip()] = v.strip()
+    assert reader.documentInfo['/Title'] == german
+    assert reader.documentInfo['/Author'] == chinese
+    assert reader.documentInfo.get('/Keywords', '') == ''
 
-    assert pdfinfo['Title'] == german
-    assert pdfinfo['Author'] == chinese
-    assert pdfinfo['Subject'] == high_unicode
-    assert pdfinfo.get('Keywords', '') == ''
-
-    pdfa_info = file_claims_pdfa(pdf)
+    pdfa_info = file_claims_pdfa(outpdf)
     assert pdfa_info['output'] == output_type
 
 
+def test_high_unicode(spoof_tesseract_noop, resources, no_outpdf):
+
+    # Ghostscript doesn't support high Unicode, so neither do we, to be
+    # safe
+    input_file = resources / 'c02-22.pdf'
+    high_unicode = 'U+1030C is: 𐌌'
+
+    p, out, err = run_ocrmypdf(
+        input_file, no_outpdf,
+        '--subject', high_unicode,
+        '--output-type', 'pdfa',
+        env=spoof_tesseract_noop)
+
+    assert p.returncode == ExitCode.bad_args, err
+
+
 @pytest.mark.parametrize('renderer', [
     'hocr',
     'tesseract',

tests/test_tess4.py

@@ -6,11 +6,42 @@ from ocrmypdf.exceptions import ExitCode
 from ocrmypdf.exec import tesseract
 from ocrmypdf import pageinfo
 import sys
+import os
+import PyPDF2 as pypdf
+
+
+spoof = pytest.helpers.spoof
+
+
+def tess4_possible_location():
+    """The location of tesseract 4 may be OCRMYPDF_TESS4, OCRMYPDF_TESSERACT,
+    or the installed version on PATH."""
+    return os.environ.get('OCRMYPDF_TESS4') or \
+           os.environ.get('OCRMYPDF_TESSERACT') or \
+            'tesseract'
+
+
+@pytest.fixture
+def ensure_tess4():
+    return spoof(tesseract=tess4_possible_location())
+
+
+def tess4_available():
+    """Check if a tesseract 4 binary is available, even if it's not the
+    official "tesseract" on PATH
+
+    """
+    old_environ = os.environ.copy()
+    try:
+        os.environ['OCRMYPDF_TESSERACT'] = tess4_possible_location()
+        return tesseract.v4() and tesseract.has_textonly_pdf()
+    finally:
+        os.environ = old_environ
 
 
 # Skip all tests in this file if not tesseract 4
 pytestmark = pytest.mark.skipif(
-    not (tesseract.v4() and tesseract.has_textonly_pdf()),
+    not tess4_available(),
     reason="tesseract 4.0 with textonly_pdf feature required")
 
 check_ocrmypdf = pytest.helpers.check_ocrmypdf
@@ -18,14 +49,15 @@ run_ocrmypdf = pytest.helpers.run_ocrmypdf
 spoof = pytest.helpers.spoof
 
 
-def test_textonly_pdf(resources, outdir):
+def test_textonly_pdf(ensure_tess4, resources, outdir):
     check_ocrmypdf(
         resources / 'linn.pdf',
-        outdir / 'linn_textonly.pdf', '--pdf-renderer', 'tess4')
+        outdir / 'linn_textonly.pdf', '--pdf-renderer', 'tess4',
+        env=ensure_tess4)
 
 
 @pytest.mark.skipif(sys.version_info < (3, 5), reason="needs math.isclose")
-def test_pagesize_consistency_tess4(resources, outpdf):
+def test_pagesize_consistency_tess4(ensure_tess4, resources, outpdf):
     from math import isclose
 
     infile = resources / 'linn.pdf'
@@ -35,9 +67,47 @@ def test_pagesize_consistency_tess4(resources, outpdf):
     check_ocrmypdf(
         infile,
         outpdf, '--pdf-renderer', 'tess4',
-        '--clean', '--deskew', '--remove-background', '--clean-final')
+        '--clean', '--deskew', '--remove-background', '--clean-final',
+        env=ensure_tess4)
 
     after_dims = pytest.helpers.first_page_dimensions(outpdf)
 
     assert isclose(before_dims[0], after_dims[0])
     assert isclose(before_dims[1], after_dims[1])
+
+
+@pytest.mark.parametrize('basename', ['graph_ocred.pdf', 'cardinal.pdf'])
+def test_skip_pages_does_not_replicate(
+        ensure_tess4, resources, basename, outdir):
+    infile = resources / basename
+    outpdf = outdir / basename
+
+    check_ocrmypdf(
+        infile,
+        outpdf, '--pdf-renderer', 'tess4', '--force-ocr',
+        '--tesseract-timeout', '0',
+        env=ensure_tess4
+    )
+
+    info_in = pageinfo.pdf_get_all_pageinfo(str(infile))
+
+    info = pageinfo.pdf_get_all_pageinfo(str(outpdf))
+    for page in info:
+        assert len(page['images']) == 1, "skipped page was replicated"
+
+    for n in range(len(info_in)):
+        assert info[n]['width_inches'] == info_in[n]['width_inches']
+
+
+def test_content_preservation(ensure_tess4, resources, outpdf):
+    infile = resources / 'masks.pdf'
+
+    check_ocrmypdf(
+        infile,
+        outpdf, '--pdf-renderer', 'tess4', '--tesseract-timeout', '0',
+        env=ensure_tess4
+    )
+
+    info = pageinfo.pdf_get_all_pageinfo(str(outpdf))
+    page = info[0]
+    assert len(page['images']) > 1, "masked were rasterized"