v6.2.5 release notes

Cherry-picked and resolved from c74f2ee

docs/release_notes.rst

@@ -10,6 +10,20 @@ The OCRmyPDF package itself does not contain a public API, although it is fairly
    replace: `#$1 <https://github.com/jbarlow83/OCRmyPDF/issues/$1>`_
 
 
+v6.2.5
+------
+
+-   Backport compatibility fixes for Tesseract 4.0.0-rcN from v7.2.0
+
+
+v6.2.4
+------
+
+-   Backport Ghostscript 9.25 compatibility fixes, which removes support for setting Unicode metadata
+-   Backport blacklisting Ghostscript 9.24
+-   Older versions of Ghostscript are still supported
+
+
 v6.2.3
 ------
 

src/ocrmypdf/__main__.py

@@ -587,9 +587,27 @@ def do_ruffus_exception(ruffus_five_tuple, options, log):
     description of the error message that occurred."""
     exit_code = None
 
-    task_name, job_name, exc_name, exc_value, exc_stack = ruffus_five_tuple
-    job_name = job_name  # unused
-    if exc_name == 'builtins.SystemExit':
+    _task_name, _job_name, exc_name, exc_value, exc_stack = ruffus_five_tuple
+
+    if isinstance(exc_name, type):
+        # ruffus is full of mystery... sometimes (probably when the process
+        # group leader is killed) exc_name is the class object of the exception,
+        # rather than a str. So reach into the object and get its name.
+        exc_name = exc_name.__name__
+
+    if exc_name.startswith('ocrmypdf.exceptions.'):
+        base_exc_name = exc_name.replace('ocrmypdf.exceptions.', '')
+        exc_class = getattr(ocrmypdf_exceptions, base_exc_name)
+        exit_code = getattr(exc_class, 'exit_code', ExitCode.other_error)
+        try:
+            if isinstance(exc_value, exc_class):
+                exc_msg = str(exc_value)
+            else:
+                exc_msg = str(exc_class())
+        except Exception:
+            exc_msg = "Unknown"
+
+    if exc_name in ('builtins.SystemExit', 'SystemExit'):
         match = re.search(r"\.(.+?)\)", exc_value)
         exit_code_name = match.groups()[0]
         exit_code = getattr(ExitCode, exit_code_name, 'other_error')
@@ -611,36 +629,9 @@ def do_ruffus_exception(ruffus_five_tuple, options, log):
         msg = "Error occurred while running this command:"
         log.error(msg + '\n' + exc_value)
         exit_code = ExitCode.child_process_error
-    elif (exc_name == 'PyPDF2.utils.PdfReadError' and \
-            'not been decrypted' in exc_value) or \
-            (exc_name == 'ocrmypdf.exceptions.EncryptedPdfError'):
-        log.error(textwrap.dedent("""\
-            Input PDF is encrypted. The encryption must be removed to
-            perform OCR.
-
-            For information about this PDF's security use
-                qpdf --show-encryption infilename
-
-            You can remove the encryption using
-                qpdf --decrypt [--password=[password]] infilename
-
-            """))
-        exit_code = ExitCode.encrypted_pdf
-    elif exc_name == 'ocrmypdf.exceptions.PdfMergeFailedError':
-        log.error(textwrap.dedent("""\
-            Failed to merge PDF image layer with OCR layer
-
-            Usually this happens because the input PDF file is mal-formed and
-            ocrmypdf cannot automatically correct the problem on its own.
-
-            Try using
-                ocrmypdf --pdf-renderer tesseract  [..other args..]
-            """))
-        exit_code = ExitCode.input_file
     elif exc_name.startswith('ocrmypdf.exceptions.'):
-        base_exc_name = exc_name.replace('ocrmypdf.exceptions.', '')
-        exc_class = getattr(ocrmypdf_exceptions, base_exc_name)
-        exit_code = exc_class.exit_code
+        if exc_msg:
+            log.error(exc_msg)
     elif exc_name == 'PIL.Image.DecompressionBombError':
         msg = cleanup_ruffus_error_message(exc_value)
         msg += ("\nUse the --max-image-mpixels argument to set increase the "
@@ -863,6 +854,14 @@ def run_pipeline():
             "security vulnerabilities with certain malformed PDFs. Consider "
             "upgrading to version 7.0.0 or newer.".format(qpdf.version()))
 
+    if ghostscript.version() == '9.24':
+        complain(
+            "Ghostscript 9.24 contains serious regressions and is not "
+            "supported. Please upgrade to Ghostscript 9.25 or use an older "
+            "version."
+        )
+        return ExitCode.missing_dependency
+
     # Any changes to options will not take effect for options that are already
     # bound to function parameters in the pipeline. (For example
     # options.input_file, options.pdf_renderer are already bound.)

src/ocrmypdf/exceptions.py

@@ -17,6 +17,7 @@
 
 
 from enum import IntEnum
+from textwrap import dedent
 
 class ExitCode(IntEnum):
     ok = 0
@@ -35,6 +36,13 @@ class ExitCode(IntEnum):
 
 class ExitCodeException(Exception):
     exit_code = ExitCode.other_error
+    message = ""
+
+    def __str__(self):
+        super_msg = super().__str__()  # Don't do str(super())
+        if self.message:
+            return self.message.format(super_msg)
+        return super_msg
 
 
 class BadArgsError(ExitCodeException):
@@ -43,7 +51,15 @@ class BadArgsError(ExitCodeException):
 
 class PdfMergeFailedError(ExitCodeException):
     exit_code = ExitCode.input_file
+    message = dedent('''\
+        Failed to merge PDF image layer with OCR layer
 
+        Usually this happens because the input PDF file is malformed and
+        ocrmypdf cannot automatically correct the problem on its own.
+
+        Try using
+            ocrmypdf --pdf-renderer sandwich  [..other args..]
+        ''')
 
 class MissingDependencyError(ExitCodeException):
     exit_code = ExitCode.missing_dependency
@@ -75,7 +91,18 @@ class SubprocessOutputError(ExitCodeException):
 
 class EncryptedPdfError(ExitCodeException):
     exit_code = ExitCode.encrypted_pdf
+    message = dedent('''\
+        Input PDF is encrypted. The encryption must be removed to
+        perform OCR.
+
+        For information about this PDF's security use
+            qpdf --show-encryption infilename
+
+        You can remove the encryption using
+            qpdf --decrypt [--password=[password]] infilename
+        ''')
 
 
 class TesseractConfigError(ExitCodeException):
     exit_code = ExitCode.invalid_config
+    message = "Error occurred while parsing a Tesseract configuration file"

src/ocrmypdf/exec/ghostscript.py

@@ -34,12 +34,20 @@ def version():
 
 def jpeg_passthrough_available():
     """
-    Ghostscript 9.23 introduced JPEG passthrough but it seems to corrupt the
-    last two bytes of certain images, for now we disable it for 9.23 and
-    do not mention it for < 9.23.
+    Returns True if the installed version of Ghostscript supports JPEG passthru
 
+    Prior to 9.23, Ghostscript decode and re-encoded JPEGs internally. In 9.23
+    it gained the ability to keep JPEGs unmodified. However, the 9.23
+    implementation was buggy and would deletes the last two bytes of images in
+    some cases, as reported here.
     https://bugs.ghostscript.com/show_bug.cgi?id=699216
 
+    The issue was fixed for 9.24, hence that is the first version we consider
+    the feature available. (However, we don't use 9.24 at all, so the first
+    version that allows JPEG passthrough is 9.25.
+
+    Regardless, in ocrmypdf 6.x we are ignoring this new feature entirely to
+    avoid new behavior.
     """
     return False
 
@@ -52,20 +60,20 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
                   pageno=1, page_dpi=None):
     """
     Rasterize one page of a PDF at resolution (xres, yres) in canvas units.
-    
-    The image is sized to match the integer pixels dimensions implied by 
+
+    The image is sized to match the integer pixels dimensions implied by
     (xres, yres) even if those numbers are noninteger. The image's DPI will
      be overridden with the values in page_dpi.
-    
+
     :param input_file: pathlike
     :param output_file: pathlike
     :param xres: resolution at which to rasterize page
-    :param yres: 
-    :param raster_device: 
-    :param log: 
+    :param yres:
+    :param raster_device:
+    :param log:
     :param pageno: page number to rasterize (beginning at page 1)
-    :param page_dpi: resolution tuple (x, y) overriding output image DPI 
-    :return: 
+    :param page_dpi: resolution tuple (x, y) overriding output image DPI
+    :return:
     """
     res = xres, yres
     int_res = round(xres), round(yres)
@@ -85,7 +93,7 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
             '-o', tmp.name,
             fspath(input_file)
         ]
-        
+
         p = run(args_gs, stdout=PIPE, stderr=STDOUT,
                 universal_newlines=True)
         if _gs_error_reported(p.stdout):
@@ -139,15 +147,17 @@ def generate_pdfa(pdf_pages, output_file, compression, log,
             "-dAutoFilterGrayImages=true",
         ]
 
-    # Older versions of Ghostscript expect a leading slash in 
-    # sColorConversionStrategy, newer ones should not have it. See Ghostscript 
+    # Older versions of Ghostscript expect a leading slash in
+    # sColorConversionStrategy, newer ones should not have it. See Ghostscript
     # git commit fe1c025d.
     strategy = 'RGB' if version() >= '9.19' else '/RGB'
 
-    if version() == '9.23':
+    if version() >= '9.23':
         # 9.23: new feature JPEG passthrough is broken in some cases, best to
         # disable it always
         # https://bugs.ghostscript.com/show_bug.cgi?id=699216
+        # fixed in 9.24, but to avoid changing expected behavior we disable it
+        # for ocrmypdf 6.x
         compression_args.append('-dPassThroughJPEGImages=false')
 
     with NamedTemporaryFile(delete=True) as gs_pdf:

src/ocrmypdf/exec/tesseract.py

@@ -67,8 +67,8 @@ def v4():
 @lru_cache(maxsize=1)
 def has_textonly_pdf():
     """Does Tesseract have textonly_pdf capability?
-    
-    Available in 3.05.01, and v4.00.00alpha since January 2017. Best to 
+
+    Available in 3.05.01, and v4.00.00alpha since January 2017. Best to
     parse the parameter list
     """
     args_tess = [
@@ -191,6 +191,14 @@ def tesseract_log_output(log, stdout, input_file):
             log.warning(prefix + "lots of diacritics - possibly poor OCR")
         elif line.startswith('OSD: Weak margin'):
             log.warning(prefix + "unsure about page orientation")
+        elif 'Error in pixScanForForeground' in line:
+            pass  # Appears to be spurious/problem with nonwhite borders
+        elif 'Error in boxClipToRectangle' in line:
+            pass  # Always appears with pixScanForForeground message
+        elif 'parameter not found: ' in line.lower():
+            log.error(prefix + line.strip())
+            problem = line.split('found: ')[1]
+            raise TesseractConfigError(problem)
         elif 'error' in line.lower() or 'exception' in line.lower():
             log.error(prefix + line.strip())
         elif 'warning' in line.lower():
@@ -264,8 +272,6 @@ def generate_hocr(input_file, output_files, language: list, engine_mode,
         _generate_null_hocr(output_hocr, output_sidecar, input_file)
     except CalledProcessError as e:
         tesseract_log_output(log, e.output, input_file)
-        if b'read_params_file: parameter not found' in e.output:
-            raise TesseractConfigError() from e
         if b'Image too large' in e.output:
             _generate_null_hocr(output_hocr, output_sidecar, input_file)
             return
@@ -362,9 +368,6 @@ def generate_pdf(*, input_image, skip_pdf, output_pdf, output_text,
         use_skip_page(text_only, skip_pdf, output_pdf, output_text)
     except CalledProcessError as e:
         tesseract_log_output(log, e.output, input_image)
-        if b'read_params_file: parameter not found' in e.output:
-            raise TesseractConfigError() from e
-
         if b'Image too large' in e.output:
             use_skip_page(text_only, skip_pdf, output_pdf, output_text)
             return

src/ocrmypdf/pdfa.py

@@ -15,7 +15,21 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
-# Generate a PDFA_def.ps file for Ghostscript >= 9.14
+"""
+Generate a PDFMARK file for Ghostscript >= 9.14, for PDF/A conversion
+
+pdfmark is an extension to the Postscript language that describes some PDF
+features like bookmarks and annotations. It was originally specified Adobe
+Distiller, for Postscript to PDF conversion:
+https://www.adobe.com/content/dam/acom/en/devnet/acrobat/pdfs/pdfmark_reference.pdf
+
+Ghostscript uses pdfmark for PDF to PDF/A conversion as well. To use Ghostscript
+to create a PDF/A, we need to create a pdfmark file with the necessary metadata.
+
+This takes care of the many version-specific bugs and pecularities in
+Ghostscript's handling of pdfmark.
+
+"""
 
 from string import Template
 from binascii import hexlify
@@ -78,7 +92,8 @@ def
 
 
 def encode_text_string(s: str) -> str:
-    '''Encode text string to hex string for use in a PDF
+    """
+    Encode text string to hex string for use in a PDF
 
     From PDF 32000-1:2008 a string object may be included in hexademical form
     if it is enclosed in angle brackets.  For general Unicode the string should
@@ -86,7 +101,7 @@ def encode_text_string(s: str) -> str:
     ASCII strings could be encoded as PdfDocEncoding literals provided
     that certain Postscript sequences are escaped.  But it's far simpler to
     encode everything as UTF-16.
-    '''
+    """
 
     # Sometimes lazy C programmers leave their NULs at the end of strings they
     # insert into PDFs
@@ -102,10 +117,30 @@ def encode_text_string(s: str) -> str:
     return ascii_hex_str
 
 
-def encode_pdf_date(d: datetime) -> str:
-    """Encode Python datetime object as PDF date string
+def _encode_ascii(s: str) -> str:
+    """
+    Aggressively strip non-ASCII and PDF escape sequences
 
-    From Adobe pdfmark manual:    
+    Ghostscript 9.24+ lost support for UTF-16BE in pdfmark files for reasons
+    given in GhostPDL commit e997c683. Our temporary workaround is use ASCII
+    and drop all non-ASCII characters. A slightly improved alternative would
+    be to implement PdfDocEncoding in pikepdf and encode to that, or handle
+    metadata there.
+    """
+    trans = str.maketrans({
+        '(': '',
+        ')': '',
+        '\\': '',
+        '\0': ''
+    })
+    return s.translate(trans).encode('ascii', errors='replace').decode()
+
+
+def encode_pdf_date(d: datetime) -> str:
+    """
+    Encode Python datetime object as PDF date string
+
+    From Adobe pdfmark manual:
     (D:YYYYMMDDHHmmSSOHH'mm')
     D: is an optional prefix. YYYY is the year. All fields after the year are
     optional. MM is the month (01-12), DD is the day (01-31), HH is the
@@ -137,16 +172,23 @@ def encode_pdf_date(d: datetime) -> str:
 
 
 def decode_pdf_date(s: str) -> datetime:
+    """
+    Decode a pdfmark date to a Python datetime object
+
+    A pdfmark date is a string in a paritcular format. See the pdfmark
+    Reference for the specification.
+
+    """
     if s.startswith('D:'):
         s = s[2:]
 
-    # Literal Z00'00', is incorrect but found in the wild, 
+    # Literal Z00'00', is incorrect but found in the wild,
     # probably made by OS X Quartz -- standardize
     if s.endswith("Z00'00'"):
         s = s.replace("Z00'00'", '+0000')
     elif s.endswith('Z'):
         s = s.replace('Z', '+0000')
-    
+
     s = s.replace("'", "")  # Remove apos from PDF time strings
 
     return datetime.strptime(s, r'%Y%m%d%H%M%S%z')
@@ -154,7 +196,7 @@ def decode_pdf_date(s: str) -> datetime:
 
 def _get_pdfmark_dates(pdfmark):
     """Encode dates for pdfmark Postscript.  The best way to deal with a
-    missing date entry is set it to null, because if the key is omitted 
+    missing date entry is set it to null, because if the key is omitted
     Ghostscript will set it to now - we do not want to erase the fact that
     the value was unknown.  Setting to an empty string breaks Ghostscript
     9.22 as reported here:
@@ -172,30 +214,47 @@ def _get_pdfmark_dates(pdfmark):
             date_str = date_str[2:]
         try:
             yield '  {} (D:{})'.format(
-                    key, 
+                    key,
                     encode_pdf_date(decode_pdf_date(date_str)))
         except ValueError:
             yield '  {} null'.format(key)
 
 
-def _get_pdfa_def(icc_profile, icc_identifier, pdfmark):
-    """Create a Postscript file for Ghostscript.  pdfmark contains the various
-    objects as strings; these must be encoded in ASCII, and dates have a 
-    special format."""
+def _get_pdfa_def(icc_profile, icc_identifier, pdfmark, ascii_docinfo=False):
+    """Create a Postscript pdfmark file for Ghostscript.
+
+    pdfmark contains the various objects as strings; these must be encoded in
+    ASCII, and dates have a special format.
+
+    :param icc_profile: filename of the ICC profile to include in pdfmark
+    :param icc_identifier: ICC identifier such as 'sRGB'
+    :param pdfmark: a dictionary containing keys to include the pdfmark
+    :param ascii_docinfo: if True, the docinfo block must be encoded in pure
+        ASCII and may not contain UTF-16BE-BOM-hex encoded strings, as
+        required for Ghostscript 9.24+
+
+    :returns: a string containing the entire pdfmark
+
+    """
 
     # Ghostscript <= 9.21 has a bug where null entries in DOCINFO might produce
     # ERROR: VMerror (-25) on closing pdfwrite device.
     # https://bugs.ghostscript.com/show_bug.cgi?id=697684
     # Work around this by only adding keys that have a nontrivial value
     docinfo_keys = ('/Title', '/Author', '/Subject', '/Creator', '/Keywords')
-    docinfo_line_template = '  {key} <{value}>'
 
     def docinfo_gen():
+        if not ascii_docinfo:
+            docinfo_line_template = '  {key} <{value}>'
+            encode = encode_text_string
+        else:
+            docinfo_line_template = '  {key} ({value})'
+            encode = _encode_ascii
         yield from _get_pdfmark_dates(pdfmark)
         for key in docinfo_keys:
             if key in pdfmark and pdfmark[key].strip() != '':
                 line = docinfo_line_template.format(
-                    key=key, value=encode_text_string(pdfmark[key]))
+                    key=key, value=encode(pdfmark[key]))
                 yield line
     docinfo = '\n'.join(docinfo_gen())
 
@@ -206,13 +265,13 @@ def _get_pdfa_def(icc_profile, icc_identifier, pdfmark):
     return result
 
 
-def generate_pdfa_ps(target_filename, pdfmark, icc='sRGB'):
+def generate_pdfa_ps(target_filename, pdfmark, icc='sRGB', ascii_docinfo=False):
     if icc == 'sRGB':
         icc_profile = SRGB_ICC_PROFILE
     else:
         raise NotImplementedError("Only supporting sRGB")
 
-    ps = _get_pdfa_def(icc_profile, icc, pdfmark)
+    ps = _get_pdfa_def(icc_profile, icc, pdfmark, ascii_docinfo=ascii_docinfo)
 
     # We should have encoded everything to pure ASCII by this point, and
     # to be safe, only allow ASCII in PostScript
@@ -263,4 +322,3 @@ def file_claims_pdfa(filename):
     pdfa_dict['conformance'] = conformance
 
     return pdfa_dict
-

src/ocrmypdf/pipeline.py

@@ -926,7 +926,20 @@ def generate_postscript_stub(
     options = context.get_options()
     pdf = pypdf.PdfFileReader(input_file)
     pdfmark = get_pdfmark(pdf, options)
-    generate_pdfa_ps(output_file, pdfmark)
+
+    ascii_docinfo = False
+    if ghostscript.version() >= '9.24':
+        ascii_docinfo = True
+        try:
+            for v in pdfmark.values():
+                v.encode('ascii', errors='strict')
+        except UnicodeEncodeError:
+            log.warning(
+                "Ghostscript 9.24+ does not support Unicode strings in "
+                " metadata. These will be converted to ASCII if possible."
+            )
+
+    generate_pdfa_ps(output_file, pdfmark, ascii_docinfo=ascii_docinfo)
 
 
 def skip_page(

tests/test_main.py

@@ -734,7 +734,7 @@ THIS FILE IS INVALID
         resources / 'ccitt.pdf', outdir / 'out.pdf',
         '--pdf-renderer', renderer,
         '--tesseract-config', cfg_file)
-    assert "parameter not found" in err, "No error message"
+    assert "parameter not found" in err.lower(), "No error message"
     assert p.returncode == ExitCode.invalid_config
 
 
@@ -908,13 +908,7 @@ def test_compression_changed(spoof_tesseract_noop, ocrmypdf_exec,
     if compression == "jpeg":
         assert pdfimage.enc == Encoding.jpeg
     else:
-        if ghostscript.jpeg_passthrough_available():
-            # Ghostscript 9.23 adds JPEG passthrough, which allows a JPEG to be
-            # copied without transcoding - so report
-            if image.endswith('jpg'):
-                assert pdfimage.enc == Encoding.jpeg
-        else:
-            assert pdfimage.enc not in (Encoding.jpeg, Encoding.jpeg2000)
+        assert pdfimage.enc not in (Encoding.jpeg, Encoding.jpeg2000)
 
     if im.mode.startswith('RGB') or im.mode.startswith('BGR'):
         assert pdfimage.color == Colorspace.rgb, \

tests/test_metadata.py

@@ -24,6 +24,12 @@ from datetime import timezone
 from ocrmypdf.pdfa import file_claims_pdfa, encode_pdf_date, decode_pdf_date
 from ocrmypdf.exceptions import ExitCode
 from ocrmypdf.lib import fitz
+from ocrmypdf.helpers import fspath
+from ocrmypdf.pdfa import (
+    file_claims_pdfa, encode_pdf_date, decode_pdf_date, generate_pdfa_ps,
+    SRGB_ICC_PROFILE
+)
+from ocrmypdf.exec import ghostscript
 
 # pytest.helpers is dynamic
 # pylint: disable=no-member
@@ -76,6 +82,9 @@ def test_override_metadata(spoof_tesseract_noop, output_type, resources,
     before = pypdf.PdfFileReader(str(input_file))
     after = pypdf.PdfFileReader(outpdf)
 
+    if ghostscript.version() >= '9.24':
+        pytest.xfail('Ghostscript 9.24+ does not support Unicode DOCINFO')
+
     assert after.documentInfo['/Title'] == german
     assert after.documentInfo['/Author'] == chinese
     assert after.documentInfo.get('/Keywords', '') == ''
@@ -136,7 +145,7 @@ def test_creation_date_preserved(spoof_tesseract_noop, output_type, resources,
 
     before = pypdf.PdfFileReader(str(input_file)).getDocumentInfo()
     check_ocrmypdf(
-        input_file, outpdf, '--output-type', output_type, 
+        input_file, outpdf, '--output-type', output_type,
         env=spoof_tesseract_noop)
     after = pypdf.PdfFileReader(str(outpdf)).getDocumentInfo()
 
@@ -157,5 +166,3 @@ def test_creation_date_preserved(spoof_tesseract_noop, output_type, resources,
     date_after = decode_pdf_date(after['/ModDate'])
     assert seconds_between_dates(
         date_after, datetime.datetime.now(timezone.utc)) < 1000
-
-

tests/test_tess4.py

@@ -29,8 +29,7 @@ from pathlib import Path
 spoof = pytest.helpers.spoof
 
 
-@pytest.fixture
-def ensure_tess4():
+def _ensure_tess4():
     if tesseract.v4():
         # "tesseract" on $PATH is already v4
         return os.environ.copy()
@@ -49,6 +48,11 @@ def ensure_tess4():
     raise EnvironmentError("Can't find Tesseract 4")
 
 
+@pytest.fixture
+def ensure_tess4():
+    return _ensure_tess4()
+
+
 @contextmanager
 def modified_os_environ(env):
     old_env = os.environ.copy()
@@ -63,8 +67,8 @@ def tess4_available():
 
     """
     try:
-        # ensure_tess4 locates the tess4 binary we are going to check
-        env = ensure_tess4()
+        # _ensure_tess4 locates the tess4 binary we are going to check
+        env = _ensure_tess4()
         with modified_os_environ(env):
             # Now jump into this environment and make sure it really is Tess4
             return tesseract.v4() and tesseract.has_textonly_pdf()