Fix travis.yml

This feature works for gs >= 9.24, but we don't want to change behavior in
a backport, so we're disabling it unconditionally.

.travis.yml

@@ -31,29 +31,14 @@ matrix:
       sudo: required
       language: python
       python: "3.5"
-      env: EXTRAS=
     - os: linux
       sudo: required
       language: python
       python: "3.6"
-      env: EXTRAS=
-    - os: linux
-      sudo: required
-      language: python
-      python: "3.6"
-      env: EXTRAS=[fitz]
     - os: linux
       sudo: required
       language: python
       python: "3.7-dev"
-    - os: osx
-      osx_image: xcode8
-      language: generic
-      env: EXTRAS=
-    - os: osx
-      osx_image: xcode8
-      language: generic
-      env: EXTRAS=[fitz]
 
 before_cache:
 - rm -f $HOME/.cache/pip/log/debug.log
@@ -72,7 +57,7 @@ before_install: |
 
 install:
 - pip3 install pycparser # py3.7 workaround for https://github.com/eliben/pycparser/issues/251
-- pip3 install ".$EXTRAS"
+- pip3 install .
 - pip3 install -r test_requirements.txt
 
 script:
@@ -94,5 +79,5 @@ deploy:
   on:
     branch: master
     tags: true
-    condition: $TRAVIS_PYTHON_VERSION == "3.6" && $TRAVIS_OS_NAME == "linux" && $EXTRAS == ""
+    condition: $TRAVIS_PYTHON_VERSION == "3.6" && $TRAVIS_OS_NAME == "linux"
   skip_upload_docs: true

docs/installation.rst

@@ -192,15 +192,13 @@ You can then install OCRmyPDF from PyPI, for the current user:
 
 .. code-block:: bash
 
-    pip3 install --user ocrmypdf[fitz]
+    pip3 install --user ocrmypdf
 
 or system-wide:
 
 .. code-block:: bash
 
-    pip3 install ocrmypdf[fitz]
+    pip3 install ocrmypdf
-
-``[fitz]`` includes the optional dependency on PyMuPDF, which improves OCRmyPDF's output in many cases.
 
 The command line program should now be available:
 
@@ -226,7 +224,7 @@ Then install ocrmypdf 6.1.5 for the local user and set the user's ``PATH`` to ch
 .. code-block:: bash
 
     export PATH=$HOME/.local/bin:$PATH
-    pip3 install --user ocrmypdf[fitz]
+    pip3 install --user ocrmypdf
 
 
 Installing on Ubuntu 16.04 LTS
@@ -249,13 +247,13 @@ If you wish install OCRmyPDF for the current user:
 
 .. code-block:: bash
 
-    pip3 install --user ocrmypdf[fitz]
+    pip3 install --user ocrmypdf
 
 Alternately, system-wide. Note that this may modify the system Python environment:
 
 .. code-block:: bash
 
-    sudo pip3 install ocrmypdf[fitz]
+    sudo pip3 install ocrmypdf
 
 If you wish to install OCRmyPDF to a virtual environment to isolate the system Python, you can follow these steps.
 
@@ -263,9 +261,7 @@ If you wish to install OCRmyPDF to a virtual environment to isolate the system P
 
     python3 -m venv venv-ocrmypdf
     source venv-ocrmypdf/bin/activate
-    pip3 install ocrmypdf[fitz]
+    pip3 install ocrmypdf
-
-``[fitz]`` includes the optional dependency on PyMuPDF, which improves OCRmyPDF's output in many cases.
 
 
 Installing on Ubuntu 14.04 LTS
@@ -316,7 +312,7 @@ Now we need to install ``pip`` and let it install ocrmypdf:
 .. code-block:: bash
 
     curl https://bootstrap.pypa.io/ez_setup.py -o - | python3.6 && python3.6 -m easy_install pip
-    pip3.6 install ocrmypdf[fitz]
+    pip3.6 install ocrmypdf
 
 The ``wget`` command will download a program and run it.
 
@@ -396,14 +392,11 @@ The following dependencies are recommended:
 - Ghostscript 9.22 or newer
 - qpdf 8.0.2 or newer
 - unpaper 6.1
-- PyMuPDF 1.12.5 or newer
 
 These are in addition to the Python packaging dependencies, meaning that unfortunately, the ``pip install`` command cannot satisfy all of them.
 
 Python 3.6 and Tesseract 4.0.0-beta.1 are recommended for best OCR results and best performance.
 
-The library PyMuPDF is not widely available in platform distributions, and it improves OCRmyPDF in certain conditions. Consider installing OCRmyPDF from the Python binary wheels, which include a precompiled version of this library.
-
 
 Installing HEAD revision from sources
 -------------------------------------

docs/release_notes.rst

@@ -10,6 +10,24 @@ The OCRmyPDF package itself does not contain a public API, although it is fairly
    replace: `#$1 <https://github.com/jbarlow83/OCRmyPDF/issues/$1>`_
 
 
+v6.2.5
+------
+
+-   Disable a failing test due to Tesseract 4.0rc1 behavior change. Previously, Tesseract would exit with an error message if its configuration was invalid, and OCRmyPDF would intercept this message. Now Tesseract issues a warning, which OCRmyPDF v6.2.5 may relay or ignore. (In v7.x, OCRmyPDF will respond to the warning.)
+
+-   This release branch no longer supports using the optional PyMuPDF installation, since it was removed in v7.x.
+
+-   This release branch no longer supports macOS. macOS users should upgrade to v7.x.
+
+
+v6.2.4
+------
+
+-   Backport Ghostscript 9.25 compatibility fixes, which removes support for setting Unicode metadata
+-   Backport blacklisting Ghostscript 9.24
+-   Older versions of Ghostscript are still supported
+
+
 v6.2.3
 ------
 

src/ocrmypdf/__main__.py

@@ -863,6 +863,14 @@ def run_pipeline():
             "security vulnerabilities with certain malformed PDFs. Consider "
             "upgrading to version 7.0.0 or newer.".format(qpdf.version()))
 
+    if ghostscript.version() == '9.24':
+        complain(
+            "Ghostscript 9.24 contains serious regressions and is not "
+            "supported. Please upgrade to Ghostscript 9.25 or use an older "
+            "version."
+        )
+        return ExitCode.missing_dependency
+
     # Any changes to options will not take effect for options that are already
     # bound to function parameters in the pipeline. (For example
     # options.input_file, options.pdf_renderer are already bound.)

src/ocrmypdf/exec/ghostscript.py

@@ -34,12 +34,20 @@ def version():
 
 def jpeg_passthrough_available():
     """
-    Ghostscript 9.23 introduced JPEG passthrough but it seems to corrupt the
+    Returns True if the installed version of Ghostscript supports JPEG passthru
-    last two bytes of certain images, for now we disable it for 9.23 and
-    do not mention it for < 9.23.
 
+    Prior to 9.23, Ghostscript decode and re-encoded JPEGs internally. In 9.23
+    it gained the ability to keep JPEGs unmodified. However, the 9.23
+    implementation was buggy and would deletes the last two bytes of images in
+    some cases, as reported here.
     https://bugs.ghostscript.com/show_bug.cgi?id=699216
 
+    The issue was fixed for 9.24, hence that is the first version we consider
+    the feature available. (However, we don't use 9.24 at all, so the first
+    version that allows JPEG passthrough is 9.25.
+
+    Regardless, in ocrmypdf 6.x we are ignoring this new feature entirely to
+    avoid new behavior.
     """
     return False
 
@@ -144,10 +152,12 @@ def generate_pdfa(pdf_pages, output_file, compression, log,
     # git commit fe1c025d.
     strategy = 'RGB' if version() >= '9.19' else '/RGB'
 
-    if version() == '9.23':
+    if version() >= '9.23':
         # 9.23: new feature JPEG passthrough is broken in some cases, best to
         # disable it always
         # https://bugs.ghostscript.com/show_bug.cgi?id=699216
+        # fixed in 9.24, but to avoid changing expected behavior we disable it
+        # for ocrmypdf 6.x
         compression_args.append('-dPassThroughJPEGImages=false')
 
     with NamedTemporaryFile(delete=True) as gs_pdf:

src/ocrmypdf/pdfa.py

@@ -15,7 +15,21 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
-# Generate a PDFA_def.ps file for Ghostscript >= 9.14
+"""
+Generate a PDFMARK file for Ghostscript >= 9.14, for PDF/A conversion
+
+pdfmark is an extension to the Postscript language that describes some PDF
+features like bookmarks and annotations. It was originally specified Adobe
+Distiller, for Postscript to PDF conversion:
+https://www.adobe.com/content/dam/acom/en/devnet/acrobat/pdfs/pdfmark_reference.pdf
+
+Ghostscript uses pdfmark for PDF to PDF/A conversion as well. To use Ghostscript
+to create a PDF/A, we need to create a pdfmark file with the necessary metadata.
+
+This takes care of the many version-specific bugs and pecularities in
+Ghostscript's handling of pdfmark.
+
+"""
 
 from string import Template
 from binascii import hexlify
@@ -78,7 +92,8 @@ def
 
 
 def encode_text_string(s: str) -> str:
-    '''Encode text string to hex string for use in a PDF
+    """
+    Encode text string to hex string for use in a PDF
 
     From PDF 32000-1:2008 a string object may be included in hexademical form
     if it is enclosed in angle brackets.  For general Unicode the string should
@@ -86,7 +101,7 @@ def encode_text_string(s: str) -> str:
     ASCII strings could be encoded as PdfDocEncoding literals provided
     that certain Postscript sequences are escaped.  But it's far simpler to
     encode everything as UTF-16.
-    '''
+    """
 
     # Sometimes lazy C programmers leave their NULs at the end of strings they
     # insert into PDFs
@@ -102,8 +117,28 @@ def encode_text_string(s: str) -> str:
     return ascii_hex_str
 
 
+def _encode_ascii(s: str) -> str:
+    """
+    Aggressively strip non-ASCII and PDF escape sequences
+
+    Ghostscript 9.24+ lost support for UTF-16BE in pdfmark files for reasons
+    given in GhostPDL commit e997c683. Our temporary workaround is use ASCII
+    and drop all non-ASCII characters. A slightly improved alternative would
+    be to implement PdfDocEncoding in pikepdf and encode to that, or handle
+    metadata there.
+    """
+    trans = str.maketrans({
+        '(': '',
+        ')': '',
+        '\\': '',
+        '\0': ''
+    })
+    return s.translate(trans).encode('ascii', errors='replace').decode()
+
+
 def encode_pdf_date(d: datetime) -> str:
-    """Encode Python datetime object as PDF date string
+    """
+    Encode Python datetime object as PDF date string
 
     From Adobe pdfmark manual:
     (D:YYYYMMDDHHmmSSOHH'mm')
@@ -137,6 +172,13 @@ def encode_pdf_date(d: datetime) -> str:
 
 
 def decode_pdf_date(s: str) -> datetime:
+    """
+    Decode a pdfmark date to a Python datetime object
+
+    A pdfmark date is a string in a paritcular format. See the pdfmark
+    Reference for the specification.
+
+    """
     if s.startswith('D:'):
         s = s[2:]
 
@@ -178,24 +220,41 @@ def _get_pdfmark_dates(pdfmark):
             yield '  {} null'.format(key)
 
 
-def _get_pdfa_def(icc_profile, icc_identifier, pdfmark):
+def _get_pdfa_def(icc_profile, icc_identifier, pdfmark, ascii_docinfo=False):
-    """Create a Postscript file for Ghostscript.  pdfmark contains the various
+    """Create a Postscript pdfmark file for Ghostscript.
-    objects as strings; these must be encoded in ASCII, and dates have a 
+
-    special format."""
+    pdfmark contains the various objects as strings; these must be encoded in
+    ASCII, and dates have a special format.
+
+    :param icc_profile: filename of the ICC profile to include in pdfmark
+    :param icc_identifier: ICC identifier such as 'sRGB'
+    :param pdfmark: a dictionary containing keys to include the pdfmark
+    :param ascii_docinfo: if True, the docinfo block must be encoded in pure
+        ASCII and may not contain UTF-16BE-BOM-hex encoded strings, as
+        required for Ghostscript 9.24+
+
+    :returns: a string containing the entire pdfmark
+
+    """
 
     # Ghostscript <= 9.21 has a bug where null entries in DOCINFO might produce
     # ERROR: VMerror (-25) on closing pdfwrite device.
     # https://bugs.ghostscript.com/show_bug.cgi?id=697684
     # Work around this by only adding keys that have a nontrivial value
     docinfo_keys = ('/Title', '/Author', '/Subject', '/Creator', '/Keywords')
-    docinfo_line_template = '  {key} <{value}>'
 
     def docinfo_gen():
+        if not ascii_docinfo:
+            docinfo_line_template = '  {key} <{value}>'
+            encode = encode_text_string
+        else:
+            docinfo_line_template = '  {key} ({value})'
+            encode = _encode_ascii
         yield from _get_pdfmark_dates(pdfmark)
         for key in docinfo_keys:
             if key in pdfmark and pdfmark[key].strip() != '':
                 line = docinfo_line_template.format(
-                    key=key, value=encode_text_string(pdfmark[key]))
+                    key=key, value=encode(pdfmark[key]))
                 yield line
     docinfo = '\n'.join(docinfo_gen())
 
@@ -206,13 +265,13 @@ def _get_pdfa_def(icc_profile, icc_identifier, pdfmark):
     return result
 
 
-def generate_pdfa_ps(target_filename, pdfmark, icc='sRGB'):
+def generate_pdfa_ps(target_filename, pdfmark, icc='sRGB', ascii_docinfo=False):
     if icc == 'sRGB':
         icc_profile = SRGB_ICC_PROFILE
     else:
         raise NotImplementedError("Only supporting sRGB")
 
-    ps = _get_pdfa_def(icc_profile, icc, pdfmark)
+    ps = _get_pdfa_def(icc_profile, icc, pdfmark, ascii_docinfo=ascii_docinfo)
 
     # We should have encoded everything to pure ASCII by this point, and
     # to be safe, only allow ASCII in PostScript
@@ -263,4 +322,3 @@ def file_claims_pdfa(filename):
     pdfa_dict['conformance'] = conformance
 
     return pdfa_dict
-

src/ocrmypdf/pipeline.py

@@ -926,7 +926,20 @@ def generate_postscript_stub(
     options = context.get_options()
     pdf = pypdf.PdfFileReader(input_file)
     pdfmark = get_pdfmark(pdf, options)
-    generate_pdfa_ps(output_file, pdfmark)
+
+    ascii_docinfo = False
+    if ghostscript.version() >= '9.24':
+        ascii_docinfo = True
+        try:
+            for v in pdfmark.values():
+                v.encode('ascii', errors='strict')
+        except UnicodeEncodeError:
+            log.warning(
+                "Ghostscript 9.24+ does not support Unicode strings in "
+                " metadata. These will be converted to ASCII if possible."
+            )
+
+    generate_pdfa_ps(output_file, pdfmark, ascii_docinfo=ascii_docinfo)
 
 
 def skip_page(

tests/test_main.py

@@ -723,6 +723,9 @@ def test_tesseract_config_notfound(renderer, resources, outdir):
 
 
 @pytest.mark.parametrize('renderer', RENDERERS)
+@pytest.mark.skipif(
+    tesseract.v4(),
+    reason='not valid due to behavior change in Tesseract >= 4.0-rc1')
 def test_tesseract_config_invalid(renderer, resources, outdir):
     cfg_file = outdir / 'test.cfg'
     with cfg_file.open('w') as f:
@@ -907,12 +910,6 @@ def test_compression_changed(spoof_tesseract_noop, ocrmypdf_exec,
 
     if compression == "jpeg":
         assert pdfimage.enc == Encoding.jpeg
-    else:
-        if ghostscript.jpeg_passthrough_available():
-            # Ghostscript 9.23 adds JPEG passthrough, which allows a JPEG to be
-            # copied without transcoding - so report
-            if image.endswith('jpg'):
-                assert pdfimage.enc == Encoding.jpeg
     else:
         assert pdfimage.enc not in (Encoding.jpeg, Encoding.jpeg2000)
 

tests/test_metadata.py

@@ -24,6 +24,12 @@ from datetime import timezone
 from ocrmypdf.pdfa import file_claims_pdfa, encode_pdf_date, decode_pdf_date
 from ocrmypdf.exceptions import ExitCode
 from ocrmypdf.lib import fitz
+from ocrmypdf.helpers import fspath
+from ocrmypdf.pdfa import (
+    file_claims_pdfa, encode_pdf_date, decode_pdf_date, generate_pdfa_ps,
+    SRGB_ICC_PROFILE
+)
+from ocrmypdf.exec import ghostscript
 
 # pytest.helpers is dynamic
 # pylint: disable=no-member
@@ -76,6 +82,9 @@ def test_override_metadata(spoof_tesseract_noop, output_type, resources,
     before = pypdf.PdfFileReader(str(input_file))
     after = pypdf.PdfFileReader(outpdf)
 
+    if ghostscript.version() >= '9.24':
+        pytest.xfail('Ghostscript 9.24+ does not support Unicode DOCINFO')
+
     assert after.documentInfo['/Title'] == german
     assert after.documentInfo['/Author'] == chinese
     assert after.documentInfo.get('/Keywords', '') == ''
@@ -157,5 +166,3 @@ def test_creation_date_preserved(spoof_tesseract_noop, output_type, resources,
     date_after = decode_pdf_date(after['/ModDate'])
     assert seconds_between_dates(
         date_after, datetime.datetime.now(timezone.utc)) < 1000
-
-