v6.2.4 release notes

This feature works for gs >= 9.24, but we don't want to change behavior in
a backport, so we're disabling it unconditionally.

docs/release_notes.rst

@@ -10,6 +10,14 @@ The OCRmyPDF package itself does not contain a public API, although it is fairly
    replace: `#$1 <https://github.com/jbarlow83/OCRmyPDF/issues/$1>`_
 
 
+v6.2.4
+------
+
+-   Backport Ghostscript 9.25 compatibility fixes, which removes support for setting Unicode metadata
+-   Backport blacklisting Ghostscript 9.24
+-   Older versions of Ghostscript are still supported
+
+
 v6.2.3
 ------
 

src/ocrmypdf/__main__.py

@@ -863,6 +863,14 @@ def run_pipeline():
             "security vulnerabilities with certain malformed PDFs. Consider "
             "upgrading to version 7.0.0 or newer.".format(qpdf.version()))
 
+    if ghostscript.version() == '9.24':
+        complain(
+            "Ghostscript 9.24 contains serious regressions and is not "
+            "supported. Please upgrade to Ghostscript 9.25 or use an older "
+            "version."
+        )
+        return ExitCode.missing_dependency
+
     # Any changes to options will not take effect for options that are already
     # bound to function parameters in the pipeline. (For example
     # options.input_file, options.pdf_renderer are already bound.)

src/ocrmypdf/exec/ghostscript.py

@@ -34,12 +34,20 @@ def version():
 
 def jpeg_passthrough_available():
     """
-    Ghostscript 9.23 introduced JPEG passthrough but it seems to corrupt the
+    Returns True if the installed version of Ghostscript supports JPEG passthru
-    last two bytes of certain images, for now we disable it for 9.23 and
-    do not mention it for < 9.23.
 
+    Prior to 9.23, Ghostscript decode and re-encoded JPEGs internally. In 9.23
+    it gained the ability to keep JPEGs unmodified. However, the 9.23
+    implementation was buggy and would deletes the last two bytes of images in
+    some cases, as reported here.
     https://bugs.ghostscript.com/show_bug.cgi?id=699216
 
+    The issue was fixed for 9.24, hence that is the first version we consider
+    the feature available. (However, we don't use 9.24 at all, so the first
+    version that allows JPEG passthrough is 9.25.
+
+    Regardless, in ocrmypdf 6.x we are ignoring this new feature entirely to
+    avoid new behavior.
     """
     return False
 
@@ -144,10 +152,12 @@ def generate_pdfa(pdf_pages, output_file, compression, log,
     # git commit fe1c025d.
     strategy = 'RGB' if version() >= '9.19' else '/RGB'
 
-    if version() == '9.23':
+    if version() >= '9.23':
         # 9.23: new feature JPEG passthrough is broken in some cases, best to
         # disable it always
         # https://bugs.ghostscript.com/show_bug.cgi?id=699216
+        # fixed in 9.24, but to avoid changing expected behavior we disable it
+        # for ocrmypdf 6.x
         compression_args.append('-dPassThroughJPEGImages=false')
 
     with NamedTemporaryFile(delete=True) as gs_pdf:

src/ocrmypdf/pdfa.py

@@ -15,7 +15,21 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
-# Generate a PDFA_def.ps file for Ghostscript >= 9.14
+"""
+Generate a PDFMARK file for Ghostscript >= 9.14, for PDF/A conversion
+
+pdfmark is an extension to the Postscript language that describes some PDF
+features like bookmarks and annotations. It was originally specified Adobe
+Distiller, for Postscript to PDF conversion:
+https://www.adobe.com/content/dam/acom/en/devnet/acrobat/pdfs/pdfmark_reference.pdf
+
+Ghostscript uses pdfmark for PDF to PDF/A conversion as well. To use Ghostscript
+to create a PDF/A, we need to create a pdfmark file with the necessary metadata.
+
+This takes care of the many version-specific bugs and pecularities in
+Ghostscript's handling of pdfmark.
+
+"""
 
 from string import Template
 from binascii import hexlify
@@ -78,7 +92,8 @@ def
 
 
 def encode_text_string(s: str) -> str:
-    '''Encode text string to hex string for use in a PDF
+    """
+    Encode text string to hex string for use in a PDF
 
     From PDF 32000-1:2008 a string object may be included in hexademical form
     if it is enclosed in angle brackets.  For general Unicode the string should
@@ -86,7 +101,7 @@ def encode_text_string(s: str) -> str:
     ASCII strings could be encoded as PdfDocEncoding literals provided
     that certain Postscript sequences are escaped.  But it's far simpler to
     encode everything as UTF-16.
-    '''
+    """
 
     # Sometimes lazy C programmers leave their NULs at the end of strings they
     # insert into PDFs
@@ -102,8 +117,28 @@ def encode_text_string(s: str) -> str:
     return ascii_hex_str
 
 
+def _encode_ascii(s: str) -> str:
+    """
+    Aggressively strip non-ASCII and PDF escape sequences
+
+    Ghostscript 9.24+ lost support for UTF-16BE in pdfmark files for reasons
+    given in GhostPDL commit e997c683. Our temporary workaround is use ASCII
+    and drop all non-ASCII characters. A slightly improved alternative would
+    be to implement PdfDocEncoding in pikepdf and encode to that, or handle
+    metadata there.
+    """
+    trans = str.maketrans({
+        '(': '',
+        ')': '',
+        '\\': '',
+        '\0': ''
+    })
+    return s.translate(trans).encode('ascii', errors='replace').decode()
+
+
 def encode_pdf_date(d: datetime) -> str:
-    """Encode Python datetime object as PDF date string
+    """
+    Encode Python datetime object as PDF date string
 
     From Adobe pdfmark manual:
     (D:YYYYMMDDHHmmSSOHH'mm')
@@ -137,6 +172,13 @@ def encode_pdf_date(d: datetime) -> str:
 
 
 def decode_pdf_date(s: str) -> datetime:
+    """
+    Decode a pdfmark date to a Python datetime object
+
+    A pdfmark date is a string in a paritcular format. See the pdfmark
+    Reference for the specification.
+
+    """
     if s.startswith('D:'):
         s = s[2:]
 
@@ -178,24 +220,41 @@ def _get_pdfmark_dates(pdfmark):
             yield '  {} null'.format(key)
 
 
-def _get_pdfa_def(icc_profile, icc_identifier, pdfmark):
+def _get_pdfa_def(icc_profile, icc_identifier, pdfmark, ascii_docinfo=False):
-    """Create a Postscript file for Ghostscript.  pdfmark contains the various
+    """Create a Postscript pdfmark file for Ghostscript.
-    objects as strings; these must be encoded in ASCII, and dates have a 
+
-    special format."""
+    pdfmark contains the various objects as strings; these must be encoded in
+    ASCII, and dates have a special format.
+
+    :param icc_profile: filename of the ICC profile to include in pdfmark
+    :param icc_identifier: ICC identifier such as 'sRGB'
+    :param pdfmark: a dictionary containing keys to include the pdfmark
+    :param ascii_docinfo: if True, the docinfo block must be encoded in pure
+        ASCII and may not contain UTF-16BE-BOM-hex encoded strings, as
+        required for Ghostscript 9.24+
+
+    :returns: a string containing the entire pdfmark
+
+    """
 
     # Ghostscript <= 9.21 has a bug where null entries in DOCINFO might produce
     # ERROR: VMerror (-25) on closing pdfwrite device.
     # https://bugs.ghostscript.com/show_bug.cgi?id=697684
     # Work around this by only adding keys that have a nontrivial value
     docinfo_keys = ('/Title', '/Author', '/Subject', '/Creator', '/Keywords')
-    docinfo_line_template = '  {key} <{value}>'
 
     def docinfo_gen():
+        if not ascii_docinfo:
+            docinfo_line_template = '  {key} <{value}>'
+            encode = encode_text_string
+        else:
+            docinfo_line_template = '  {key} ({value})'
+            encode = _encode_ascii
         yield from _get_pdfmark_dates(pdfmark)
         for key in docinfo_keys:
             if key in pdfmark and pdfmark[key].strip() != '':
                 line = docinfo_line_template.format(
-                    key=key, value=encode_text_string(pdfmark[key]))
+                    key=key, value=encode(pdfmark[key]))
                 yield line
     docinfo = '\n'.join(docinfo_gen())
 
@@ -206,13 +265,13 @@ def _get_pdfa_def(icc_profile, icc_identifier, pdfmark):
     return result
 
 
-def generate_pdfa_ps(target_filename, pdfmark, icc='sRGB'):
+def generate_pdfa_ps(target_filename, pdfmark, icc='sRGB', ascii_docinfo=False):
     if icc == 'sRGB':
         icc_profile = SRGB_ICC_PROFILE
     else:
         raise NotImplementedError("Only supporting sRGB")
 
-    ps = _get_pdfa_def(icc_profile, icc, pdfmark)
+    ps = _get_pdfa_def(icc_profile, icc, pdfmark, ascii_docinfo=ascii_docinfo)
 
     # We should have encoded everything to pure ASCII by this point, and
     # to be safe, only allow ASCII in PostScript
@@ -263,4 +322,3 @@ def file_claims_pdfa(filename):
     pdfa_dict['conformance'] = conformance
 
     return pdfa_dict
-

src/ocrmypdf/pipeline.py

@@ -926,7 +926,20 @@ def generate_postscript_stub(
     options = context.get_options()
     pdf = pypdf.PdfFileReader(input_file)
     pdfmark = get_pdfmark(pdf, options)
-    generate_pdfa_ps(output_file, pdfmark)
+
+    ascii_docinfo = False
+    if ghostscript.version() >= '9.24':
+        ascii_docinfo = True
+        try:
+            for v in pdfmark.values():
+                v.encode('ascii', errors='strict')
+        except UnicodeEncodeError:
+            log.warning(
+                "Ghostscript 9.24+ does not support Unicode strings in "
+                " metadata. These will be converted to ASCII if possible."
+            )
+
+    generate_pdfa_ps(output_file, pdfmark, ascii_docinfo=ascii_docinfo)
 
 
 def skip_page(

tests/test_main.py

@@ -908,13 +908,7 @@ def test_compression_changed(spoof_tesseract_noop, ocrmypdf_exec,
     if compression == "jpeg":
         assert pdfimage.enc == Encoding.jpeg
     else:
-        if ghostscript.jpeg_passthrough_available():
+        assert pdfimage.enc not in (Encoding.jpeg, Encoding.jpeg2000)
-            # Ghostscript 9.23 adds JPEG passthrough, which allows a JPEG to be
-            # copied without transcoding - so report
-            if image.endswith('jpg'):
-                assert pdfimage.enc == Encoding.jpeg
-        else:
-            assert pdfimage.enc not in (Encoding.jpeg, Encoding.jpeg2000)
 
     if im.mode.startswith('RGB') or im.mode.startswith('BGR'):
         assert pdfimage.color == Colorspace.rgb, \

tests/test_metadata.py

@@ -24,6 +24,12 @@ from datetime import timezone
 from ocrmypdf.pdfa import file_claims_pdfa, encode_pdf_date, decode_pdf_date
 from ocrmypdf.exceptions import ExitCode
 from ocrmypdf.lib import fitz
+from ocrmypdf.helpers import fspath
+from ocrmypdf.pdfa import (
+    file_claims_pdfa, encode_pdf_date, decode_pdf_date, generate_pdfa_ps,
+    SRGB_ICC_PROFILE
+)
+from ocrmypdf.exec import ghostscript
 
 # pytest.helpers is dynamic
 # pylint: disable=no-member
@@ -76,6 +82,9 @@ def test_override_metadata(spoof_tesseract_noop, output_type, resources,
     before = pypdf.PdfFileReader(str(input_file))
     after = pypdf.PdfFileReader(outpdf)
 
+    if ghostscript.version() >= '9.24':
+        pytest.xfail('Ghostscript 9.24+ does not support Unicode DOCINFO')
+
     assert after.documentInfo['/Title'] == german
     assert after.documentInfo['/Author'] == chinese
     assert after.documentInfo.get('/Keywords', '') == ''
@@ -157,5 +166,3 @@ def test_creation_date_preserved(spoof_tesseract_noop, output_type, resources,
     date_after = decode_pdf_date(after['/ModDate'])
     assert seconds_between_dates(
         date_after, datetime.datetime.now(timezone.utc)) < 1000
-
-