Fix travis.yml

This feature works for gs >= 9.24, but we don't want to change behavior in
a backport, so we're disabling it unconditionally.

.github/ISSUE_TEMPLATE

@@ -2,11 +2,4 @@ Please include the command line and a test file with your issue report.
 
 If possible, please use a test file that we can include in future test cases (no personal information, no copyrighted material).
 
-If you wish to encrypt a test file so that only the maintainer of OCRmyPDF can view it, you may use:
+If you wish to encrypt a test file for the OCRmyPDF maintainer only, see the [Wiki](https://github.com/jbarlow83/OCRmyPDF/wiki).
-
-```bash
-
-gpg --recv-keys 4434eb74c4a35f7f --keyserver pgp.mit.edu
-gpg --output test.pdf.gpg --encrypt --receipient 4434eb74c4a35f7f test.pdf
-
-```

.gitignore

@@ -2,12 +2,14 @@
 *.pyc
 *.sublime-*
 venv*/
+.venv/
 pyvenv.cfg
 tasks.py
 .bash_history
 .ruffus_history.sqlite
 .idea/
 .pytest_cache/
+.pylintrc
 
 # Package building
 *.egg-info/
@@ -42,3 +44,4 @@ pdfbox-app*.jar
 .vscode/
 IDEAS
 _Dockerfile.local
+/scratch.py

.travis.yml

@@ -1,47 +1,54 @@
 dist: trusty
 language: python
 cache:
-  ccache: true
   pip: true
   directories:
   - $HOME/Library/Caches/Homebrew
 
-env:
+addons:
-  global:
+  apt:
-    - secure: "hsf6MT+n2x3OiDM2fQyJZdV0/PWYmv81LdVqC6cfnHBE/8N3DloJRqQ7WfO14TxhiK9PEC7MpyCj0lSabUHEO7gSH6Vks6I1asoSkt8S9/bSMlhT4hei+pwVpeGEiU5xHVATNjY+D919VC3IFvc3XmjT74h/2SLhaZ+jhEmDggM="  # HOMEBREW_OCRMYPDF_TOKEN
+    update: true
+    sources:
+    - sourceline: 'ppa:alex-p/tesseract-ocr'
+    - sourceline: 'ppa:heyarje/libav-11'
+    - sourceline: 'ppa:vshn/ghostscript'
+    packages:
+    - ghostscript
+    - libavcodec56
+    - libavformat56
+    - libavutil54
+    - libffi-dev
+    - poppler-utils
+    - qpdf
+    - tesseract-ocr
+    - tesseract-ocr-deu
+    - tesseract-ocr-eng
+    - tesseract-ocr-fra
 
 matrix:
   include:
     - os: linux
       sudo: required
       language: python
-      python: 3.5
+      python: "3.5"
-      env: EXTRAS=
     - os: linux
       sudo: required
       language: python
-      python: 3.6
+      python: "3.6"
-      env: EXTRAS=
     - os: linux
       sudo: required
       language: python
-      python: 3.6
+      python: "3.7-dev"
-      env: EXTRAS=[fitz]
-    - os: osx
-      osx_image: xcode8
-      language: generic
-      env: EXTRAS=
-    - os: osx
-      osx_image: xcode8
-      language: generic
-      env: EXTRAS=[fitz]
 
 before_cache:
 - rm -f $HOME/.cache/pip/log/debug.log
 
 before_install: |
   if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
-    bash .travis/linux_before_install.sh
+    pip install --upgrade pip
+    mkdir -p packages
+    wget -q 'https://www.dropbox.com/s/vaq0kbwi6e6au80/unpaper_6.1-1.deb?raw=1' -O packages/unpaper_6.1-1.deb
+    sudo dpkg -i packages/unpaper_6.1-1.deb
   elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
     brew update && brew bundle --file=.travis/Brewfile
     pip3 install --upgrade pip
@@ -49,7 +56,8 @@ before_install: |
   fi
 
 install:
-- pip3 install ".$EXTRAS"
+- pip3 install pycparser # py3.7 workaround for https://github.com/eliben/pycparser/issues/251
+- pip3 install .
 - pip3 install -r test_requirements.txt
 
 script:
@@ -71,33 +79,5 @@ deploy:
   on:
     branch: master
     tags: true
-    condition: $TRAVIS_PYTHON_VERSION == "3.6" && $TRAVIS_OS_NAME == "linux" && $EXTRAS == ""
+    condition: $TRAVIS_PYTHON_VERSION == "3.6" && $TRAVIS_OS_NAME == "linux"
   skip_upload_docs: true
-
-  # test pypi
-- provider: pypi
-  server: https://testpypi.pypi.org/legacy/
-  user: ocrmypdf-travis
-  password:
-    secure: "DTFOmmNL6olA0+yXvp4u9jXZlZeqrJsJ0526jzqf4a3gZ6jnGTq5UI6WzRsslSyoMMfXKtHQebqHM6ogSgCZinyZ3ufHJo8fn9brxbEc2gsiWkbj5o3bGwdWMT1vNNE7XW0VCpw87rZ1EEwjl4FJHFudMlPR1yfU5+uq0k0PACo="
-  distributions: "sdist"
-  on:
-    branch: develop
-    tags: false
-    condition: $TRAVIS_OS_NAME == "osx"
-  skip_upload_docs: true
-
-  # null deploy for osx
-  # we really just want to run after_deploy *after* pypi upload is done, but
-  # after_deploy on runs if a given box deployed
-- provider: script
-  script: /usr/bin/true
-  on:
-    branch: master
-    tags: true
-    condition: $TRAVIS_OS_NAME == "osx"
-
-after_deploy: |
-  if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-    bash .travis/osx_brew.sh
-  fi

.travis/autobrew.py

@@ -1,93 +0,0 @@
-#!/usr/bin/env python3
-# © 2017-18 James R. Barlow: github.com/jbarlow83
-
-from string import Template
-from subprocess import run, PIPE
-import re
-
-recipe_template = Template("""
-class Ocrmypdf < Formula
-  include Language::Python::Virtualenv
-
-  desc "Adds an OCR text layer to scanned PDF files"
-  homepage "https://github.com/jbarlow83/OCRmyPDF"
-  ${ocrmypdf_url}
-  ${ocrmypdf_sha256}
-
-  depends_on "pkg-config" => :build
-  depends_on "mupdf-tools" => :build  # statically links libmupdf.a
-  depends_on "freetype"
-  depends_on "ghostscript"
-  depends_on "jpeg"
-  depends_on "libpng"
-  depends_on "python"
-  depends_on "qpdf"
-  depends_on "tesseract"
-  depends_on "unpaper"
-
-${resources}
-  def install
-    venv = virtualenv_create(libexec, "python3")
-
-    resource("Pillow").stage do
-      inreplace "setup.py" do |s|
-        sdkprefix = MacOS::CLT.installed? ? "" : MacOS.sdk_path
-        s.gsub! "openjpeg.h", "probably_not_a_header_called_this_eh.h"
-        s.gsub! "ZLIB_ROOT = None", "ZLIB_ROOT = ('#{sdkprefix}/usr/lib', '#{sdkprefix}/usr/include')"
-        s.gsub! "JPEG_ROOT = None", "JPEG_ROOT = ('#{Formula["jpeg"].opt_prefix}/lib', '#{Formula["jpeg"].opt_prefix}/include')"
-        s.gsub! "FREETYPE_ROOT = None", "FREETYPE_ROOT = ('#{Formula["freetype"].opt_prefix}/lib', '#{Formula["freetype"].opt_prefix}/include')"
-      end
-
-      # avoid triggering "helpful" distutils code that doesn't recognize Xcode 7 .tbd stubs
-      ENV.append "CFLAGS", "-I#{MacOS.sdk_path}/System/Library/Frameworks/Tk.framework/Versions/8.5/Headers" unless MacOS::CLT.installed?
-      venv.pip_install Pathname.pwd
-    end
-
-    res = resources.map(&:name).to_set - ["Pillow"]
-
-    res.each do |r|
-      venv.pip_install resource(r)
-    end
-
-    venv.pip_install_and_link buildpath
-  end
-
-  test do
-    # Since we use Python 3, we require a UTF-8 locale
-    ENV["LC_ALL"] = "en_US.UTF-8"
-
-    system "#{bin}/ocrmypdf", "-f", "-q", "--deskew",
-                              test_fixtures("test.pdf"), "ocr.pdf"
-    assert_predicate testpath/"ocr.pdf", :exist?
-  end
-end
-""")
-
-def main():
-    p = run(['poet', '--single', 'ocrmypdf'],
-            encoding='utf-8', stdout=PIPE, check=True)
-
-    ocrmypdf_lines = p.stdout.splitlines()
-    ocrmypdf_url = ocrmypdf_lines[1].strip()
-    ocrmypdf_sha256 = ocrmypdf_lines[2].strip()
-
-    ocrmypdf_version = re.search(
-        r'ocrmypdf-(.+)\.tar.*', ocrmypdf_url).group(1)
-    print(f"Autobrewing {ocrmypdf_version}")
-
-    p = run(['poet', '--resources', 'ocrmypdf'],
-            encoding='utf-8', stdout=PIPE, check=True)
-
-    poet_resources = p.stdout
-
-    # Remove the duplicate "ocrmypdf" resource block
-    all_resources = poet_resources.split('resource')
-    kept_resources = [block for block in all_resources if 'ocrmypdf' not in block]
-    resources = 'resource'.join(kept_resources)
-
-    with open('ocrmypdf.rb', 'w') as out:
-        out.write(recipe_template.substitute(**locals()))
-
-
-if __name__ == '__main__':
-    main()

.travis/linux_before_install.sh

@@ -1,42 +0,0 @@
-#!/bin/bash
-# © 2017 James R. Barlow: github.com/jbarlow83
-set -euo pipefail
-set -x
-
-sudo add-apt-repository ppa:vshn/ghostscript -y
-sudo add-apt-repository ppa:heyarje/libav-11 -y
-sudo apt-get update -qq
-sudo apt-get install -y \
-	ghostscript \
-	poppler-utils \
-	libavformat56 \
-	libavcodec56 \
-	libavutil54 \
-	libffi-dev
-
-sudo add-apt-repository ppa:alex-p/tesseract-ocr -y
-
-sudo apt-get update
-sudo apt-get autoremove -y
-sudo apt-get install -y --no-install-recommends \
-		tesseract-ocr \
-		tesseract-ocr-eng \
-		tesseract-ocr-fra \
-		tesseract-ocr-deu
-
-pip install --upgrade pip
-mkdir -p packages
-wget -q 'https://www.dropbox.com/s/vaq0kbwi6e6au80/unpaper_6.1-1.deb?raw=1' -O packages/unpaper_6.1-1.deb
-sudo dpkg -i packages/unpaper_6.1-1.deb
-
-if [ ! -f /usr/local/bin/qpdf ]; then
-	export QPDF_RELEASE='https://github.com/qpdf/qpdf/releases/download/release-qpdf-8.0.2/qpdf-8.0.2.tar.gz'
-	mkdir qpdf
-    wget -q $QPDF_RELEASE -O - | tar xz -C qpdf --strip-components=1
-    cd qpdf/
-	export PATH="/usr/local/opt/ccache/libexec:$PATH"
-    ./configure --prefix=/usr
-    make -j 2
-    sudo make install
-    cd ..
-fi

.travis/osx_brew.sh

@@ -1,23 +0,0 @@
-#!/bin/bash
-# © 2017 James R. Barlow: github.com/jbarlow83
-set -uo pipefail
-set -x
-
-pip3 install homebrew-pypi-poet
-python3 .travis/autobrew.py
-cat ocrmypdf.rb
-
-# brew audit crashes Travis
-#brew audit ocrmypdf.rb
-
-# Important: disable debug output so token is hidden
-set +x
-git clone https://$HOMEBREW_OCRMYPDF_TOKEN@github.com/jbarlow83/homebrew-ocrmypdf.git
-set -x
-
-pushd homebrew-ocrmypdf
-cp ../ocrmypdf.rb Formula/ocrmypdf.rb
-git add Formula/ocrmypdf.rb
-git commit -m "homebrew-ocrmypdf: automatic release $TRAVIS_BUILD_NUMBER $TRAVIS_TAG"
-git push origin master
-popd

README.rst

@@ -126,9 +126,7 @@ If you detect an issue, please:
 Requirements
 ------------
 
-Runs on CPython 3.6, and requires external program installations of Ghostscript, Tesseract OCR, QPDF, and Leptonica. ocrmypdf is pure Python, but uses CFFI to portably generate library bindings.
+Runs on CPython 3.5, 3.6 and 3.7. Requires external program installations of Ghostscript, Tesseract OCR, QPDF, and Leptonica. ocrmypdf is pure Python, but uses CFFI to portably generate library bindings.
-
-Python 3.5 is also supported.
 
 Press & Media
 -------------

docs/installation.rst

@@ -168,7 +168,7 @@ Install or upgrade the required Homebrew packages, if any are missing:
     brew install libxml2 libffi leptonica
     brew install unpaper   # optional
 
-Python 3.5 and 3.6 are supported.
+Python 3.5, 3.6 and 3.7 are supported.
 
 Install the required Tesseract OCR engine with the language packs you plan to use:
 
@@ -192,15 +192,13 @@ You can then install OCRmyPDF from PyPI, for the current user:
 
 .. code-block:: bash
 
-    pip3 install --user ocrmypdf[fitz]
+    pip3 install --user ocrmypdf
 
 or system-wide:
 
 .. code-block:: bash
 
-    pip3 install ocrmypdf[fitz]
+    pip3 install ocrmypdf
-
-``[fitz]`` includes the optional dependency on PyMuPDF, which improves OCRmyPDF's output in many cases.
 
 The command line program should now be available:
 
@@ -226,7 +224,7 @@ Then install ocrmypdf 6.1.5 for the local user and set the user's ``PATH`` to ch
 .. code-block:: bash
 
     export PATH=$HOME/.local/bin:$PATH
-    pip3 install --user ocrmypdf[fitz]
+    pip3 install --user ocrmypdf
 
 
 Installing on Ubuntu 16.04 LTS
@@ -249,13 +247,13 @@ If you wish install OCRmyPDF for the current user:
 
 .. code-block:: bash
 
-    pip3 install --user ocrmypdf[fitz]
+    pip3 install --user ocrmypdf
 
 Alternately, system-wide. Note that this may modify the system Python environment:
 
 .. code-block:: bash
 
-    sudo pip3 install ocrmypdf[fitz]
+    sudo pip3 install ocrmypdf
 
 If you wish to install OCRmyPDF to a virtual environment to isolate the system Python, you can follow these steps.
 
@@ -263,9 +261,7 @@ If you wish to install OCRmyPDF to a virtual environment to isolate the system P
 
     python3 -m venv venv-ocrmypdf
     source venv-ocrmypdf/bin/activate
-    pip3 install ocrmypdf[fitz]
+    pip3 install ocrmypdf
-
-``[fitz]`` includes the optional dependency on PyMuPDF, which improves OCRmyPDF's output in many cases.
 
 
 Installing on Ubuntu 14.04 LTS
@@ -316,7 +312,7 @@ Now we need to install ``pip`` and let it install ocrmypdf:
 .. code-block:: bash
 
     curl https://bootstrap.pypa.io/ez_setup.py -o - | python3.6 && python3.6 -m easy_install pip
-    pip3.6 install ocrmypdf[fitz]
+    pip3.6 install ocrmypdf
 
 The ``wget`` command will download a program and run it.
 
@@ -396,14 +392,11 @@ The following dependencies are recommended:
 - Ghostscript 9.22 or newer
 - qpdf 8.0.2 or newer
 - unpaper 6.1
-- PyMuPDF 1.12.5 or newer
 
 These are in addition to the Python packaging dependencies, meaning that unfortunately, the ``pip install`` command cannot satisfy all of them.
 
 Python 3.6 and Tesseract 4.0.0-beta.1 are recommended for best OCR results and best performance.
 
-The library PyMuPDF is not widely available in platform distributions, and it improves OCRmyPDF in certain conditions. Consider installing OCRmyPDF from the Python binary wheels, which include a precompiled version of this library.
-
 
 Installing HEAD revision from sources
 -------------------------------------

docs/release_notes.rst

@@ -9,6 +9,44 @@ The OCRmyPDF package itself does not contain a public API, although it is fairly
    find:    [^`]\#([0-9]{1,3})[^0-9]
    replace: `#$1 <https://github.com/jbarlow83/OCRmyPDF/issues/$1>`_
 
+
+v6.2.5
+------
+
+-   Disable a failing test due to Tesseract 4.0rc1 behavior change. Previously, Tesseract would exit with an error message if its configuration was invalid, and OCRmyPDF would intercept this message. Now Tesseract issues a warning, which OCRmyPDF v6.2.5 may relay or ignore. (In v7.x, OCRmyPDF will respond to the warning.)
+
+-   This release branch no longer supports using the optional PyMuPDF installation, since it was removed in v7.x.
+
+-   This release branch no longer supports macOS. macOS users should upgrade to v7.x.
+
+
+v6.2.4
+------
+
+-   Backport Ghostscript 9.25 compatibility fixes, which removes support for setting Unicode metadata
+-   Backport blacklisting Ghostscript 9.24
+-   Older versions of Ghostscript are still supported
+
+
+v6.2.3
+------
+
+-   Fix compatibility with img2pdf >= 0.3.0 by rejecting input images that have an alpha channel
+
+
+v6.2.2
+------
+
+-   Backport compatibility fixes for Python 3.7 and ruffus 2.7.0 from v7.0.0
+-   Backport fix to ignore masks when deciding what colors are on a page
+-   Backport some minor improvements from v7.0.0: better argument validation and warnings about the Tesseract 4.0.0 ``--user-words`` regression
+
+v6.2.1
+------
+
+-   Fix recent versions of Tesseract (after 4.0.0-beta1) not being detected as supporting the ``sandwich`` renderer (`#271 <https://github.com/ppjbarlow83/OCRmyPDF/issues/271>`_).
+
+
 v6.2.0
 ------
 

requirements.txt

@@ -1,10 +1,11 @@
 # requirements.txt can be used to replicate the developer's build environment
 # setup.py lists a separate set of requirements that are looser to simplify
 # installation
-ruffus == 2.6.3
+ruffus == 2.7.0
-Pillow == 5.1.0
+Pillow == 5.2.0
 reportlab == 3.4.0
 PyPDF2 == 1.26.0
 img2pdf == 0.2.4
 cffi == 1.11.5
 PyMuPDF == 1.12.5
+defusedxml == 0.5.0

setup.py

@@ -215,6 +215,7 @@ setup(
     classifiers=[
         "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
         "Development Status :: 5 - Production/Stable",
         "Environment :: Console",
         "Intended Audience :: End Users/Desktop",
@@ -248,7 +249,7 @@ setup(
                                   # block 5.1.0, broken wheels
         'PyPDF2 >= 1.26',         # pure Python, so track HEAD closely
         'reportlab >= 3.3.0',     # oldest released version with sane image handling
-        'ruffus == 2.6.3',        # pinned - ocrmypdf implements a 2.6.3 workaround
+        'ruffus >= 2.7.0',
     ],
     extras_require={
         'fitz': ['PyMuPDF >= 1.12.5']     # for table of contents bug

src/ocrmypdf/__main__.py

@@ -85,6 +85,20 @@ if tesseract.version() < MINIMUM_TESS_VERSION:
 # -------------
 # Parser
 
+def numeric(basetype, min_=None, max_=None):
+    "Validator for numeric params"
+    min_ = basetype(min_) if min_ is not None else None
+    max_ = basetype(max_) if max_ is not None else None
+    def _numeric(string):
+        value = basetype(string)
+        if (min_ is not None and value < min_
+                or max_ is not None and value > max_):
+            msg = "%r not in valid range %r" % (string, (min_, max_))
+            raise argparse.ArgumentTypeError(msg)
+        return value
+    return _numeric
+
+
 parser = argparse.ArgumentParser(
     prog=PROGRAM_NAME,
     fromfile_prefix_chars='@',
@@ -233,7 +247,7 @@ preprocessing.add_argument(
     help="Clean page as above, and incorporate the cleaned image in the final "
          "PDF.  Might remove desired content.")
 preprocessing.add_argument(
-    '--oversample', metavar='DPI', type=int, default=0,
+    '--oversample', metavar='DPI', type=numeric(int, 0, 5000), default=0,
     help="Oversample images to at least the specified DPI, to improve OCR "
          "results slightly")
 
@@ -255,7 +269,7 @@ ocrsettings.add_argument(
 #          "pages")
 
 ocrsettings.add_argument(
-    '--skip-big', type=float, metavar='MPixels',
+    '--skip-big', type=numeric(float, 0, 5000), metavar='MPixels',
     help="Skip OCR on pages larger than the specified amount of megapixels, "
          "but include skipped pages in final output")
 
@@ -263,7 +277,7 @@ advanced = parser.add_argument_group(
     "Advanced",
     "Advanced options to control Tesseract's OCR behavior")
 advanced.add_argument(
-    '--max-image-mpixels', action='store', type=float, metavar='MPixels',
+    '--max-image-mpixels', action='store', type=numeric(float, 0), metavar='MPixels',
     help="Set maximum number of pixels to unpack before treating an image as a "
          "decompression bomb",
     default=128.0)
@@ -296,11 +310,11 @@ advanced.add_argument(
          " of Ghostscript; deprecated"
     )
 advanced.add_argument(
-    '--tesseract-timeout', default=180.0, type=float, metavar='SECONDS',
+    '--tesseract-timeout', default=180.0, type=numeric(float, 0), metavar='SECONDS',
     help='Give up on OCR after the timeout, but copy the preprocessed page '
          'into the final output')
 advanced.add_argument(
-    '--rotate-pages-threshold', default=14.0, type=float, metavar='CONFIDENCE',
+    '--rotate-pages-threshold', default=14.0, type=numeric(float, max_=1000), metavar='CONFIDENCE',
     help="Only rotate pages when confidence is above this value (arbitrary "
          "units reported by tesseract)")
 advanced.add_argument(
@@ -491,6 +505,10 @@ def check_options_advanced(options, log):
             "--output-type is not 'pdfa', 'pdfa-1', or 'pdfa-2'"
         )
 
+    if tesseract.v4() and (options.user_words or options.user_patterns):
+        log.warning(
+            'Tesseract 4.x ignores --user-words, so this has no effect')
+
 
 def check_options_metadata(options, log):
     import unicodedata
@@ -638,33 +656,31 @@ def do_ruffus_exception(ruffus_five_tuple, options, log):
     return ExitCode.other_error
 
 
-def traverse_ruffus_exception(e_args, options, log):
+def traverse_ruffus_exception(exceptions, options, log):
-    """Walk through a RethrownJobError and find the first exception.
+    """Traverse a RethrownJobError and output the exceptions
 
-    Ruffus flattens exception to 5 element tuples. Because of a bug
+    Ruffus presents exceptions as 5 element tuples. The RethrownJobException
-    in <= 2.6.3 it may present either the single:
+    has a list of exceptions like
-      (task, job, exc, value, stack)
+        e.job_exceptions = [(5-tuple), (5-tuple), ...]
-    or something like:
-      [[(task, job, exc, value, stack)]]
 
-    Generally cross-process exception marshalling doesn't work well
+    ruffus < 2.7.0 had a bug with exception marshalling that would give
-    and ruffus doesn't support because BaseException has its own
+    different output whether the main or child process raised the exception.
-    implementation of __reduce__ that attempts to reconstruct the
+    We no longer support this.
-    exception based on e.__init__(e.args).
 
-    Attempting to log the exception directly marshalls it to the logger
+    Attempting to log the exception itself will re-marshall it to the logger
-    which is probably in another process, so it's better to log only
+    which is normally running in another process. It's better to avoid re-
-    data from the exception at this point.
+    marshalling.
 
     The exit code will be based on this, even if multiple exceptions occurred
     at the same time."""
 
-    if isinstance(e_args, Sequence) and isinstance(e_args[0], str) and \
+    exit_codes = []
-            len(e_args) == 5:
+    for exc in exceptions:
-        return do_ruffus_exception(e_args, options, log)
+        exit_code = do_ruffus_exception(exc, options, log)
-    elif is_iterable_notstr(e_args):
+        exit_codes.append(exit_code)
-        for exc in e_args:
+
-            return traverse_ruffus_exception(exc, options, log)
+    return exit_codes[0]  # Multiple codes are rare so take the first one
+
 
 
 def check_closed_streams(options):
@@ -847,6 +863,14 @@ def run_pipeline():
             "security vulnerabilities with certain malformed PDFs. Consider "
             "upgrading to version 7.0.0 or newer.".format(qpdf.version()))
 
+    if ghostscript.version() == '9.24':
+        complain(
+            "Ghostscript 9.24 contains serious regressions and is not "
+            "supported. Please upgrade to Ghostscript 9.25 or use an older "
+            "version."
+        )
+        return ExitCode.missing_dependency
+
     # Any changes to options will not take effect for options that are already
     # bound to function parameters in the pipeline. (For example
     # options.input_file, options.pdf_renderer are already bound.)
@@ -886,7 +910,8 @@ def run_pipeline():
     except ruffus_exceptions.RethrownJobError as e:
         if options.verbose:
             _log.debug(str(e))  # stringify exception so logger doesn't have to
-        exitcode = traverse_ruffus_exception(e.args, options, _log)
+        exceptions = e.job_exceptions
+        exitcode = traverse_ruffus_exception(exceptions, options, _log)
         if exitcode is None:
             _log.error("Unexpected ruffus exception: " + str(e))
             _log.error(repr(e))

src/ocrmypdf/_unicodefun.py

@@ -40,6 +40,11 @@ import codecs
 
 def verify_python3_env():
     """Ensures that the environment is good for unicode on Python 3."""
+
+    # PEP 538 changes in Python 3.7 should make this wrangling unnecessary
+    if sys.version_info[0:3] >= (3, 7, 0):
+        return
+
     try:
         import locale
         fs_enc = codecs.lookup(locale.getpreferredencoding()).name

src/ocrmypdf/exec/__init__.py

@@ -37,6 +37,10 @@ def get_version(program, *,
             args_prog, close_fds=True, universal_newlines=True,
             stdout=PIPE, stderr=STDOUT, check=True)
         output = proc.stdout
+    except FileNotFoundError as e:
+        raise MissingDependencyError(
+            "Could not find program '{}' on the PATH".format(
+                program)) from e
     except CalledProcessError as e:
         if e.returncode < 0:
             raise MissingDependencyError(

src/ocrmypdf/exec/ghostscript.py

@@ -34,12 +34,20 @@ def version():
 
 def jpeg_passthrough_available():
     """
-    Ghostscript 9.23 introduced JPEG passthrough but it seems to corrupt the
+    Returns True if the installed version of Ghostscript supports JPEG passthru
-    last two bytes of certain images, for now we disable it for 9.23 and
-    do not mention it for < 9.23.
 
+    Prior to 9.23, Ghostscript decode and re-encoded JPEGs internally. In 9.23
+    it gained the ability to keep JPEGs unmodified. However, the 9.23
+    implementation was buggy and would deletes the last two bytes of images in
+    some cases, as reported here.
     https://bugs.ghostscript.com/show_bug.cgi?id=699216
 
+    The issue was fixed for 9.24, hence that is the first version we consider
+    the feature available. (However, we don't use 9.24 at all, so the first
+    version that allows JPEG passthrough is 9.25.
+
+    Regardless, in ocrmypdf 6.x we are ignoring this new feature entirely to
+    avoid new behavior.
     """
     return False
 
@@ -144,10 +152,12 @@ def generate_pdfa(pdf_pages, output_file, compression, log,
     # git commit fe1c025d.
     strategy = 'RGB' if version() >= '9.19' else '/RGB'
 
-    if version() == '9.23':
+    if version() >= '9.23':
         # 9.23: new feature JPEG passthrough is broken in some cases, best to
         # disable it always
         # https://bugs.ghostscript.com/show_bug.cgi?id=699216
+        # fixed in 9.24, but to avoid changing expected behavior we disable it
+        # for ocrmypdf 6.x
         compression_args.append('-dPassThroughJPEGImages=false')
 
     with NamedTemporaryFile(delete=True) as gs_pdf:

src/ocrmypdf/exec/tesseract.py

@@ -73,7 +73,8 @@ def has_textonly_pdf():
     """
     args_tess = [
         'tesseract',
-        '--print-parameters'
+        '--print-parameters',
+        'pdf'
     ]
     params = ''
     try:
@@ -159,7 +160,7 @@ def get_orientation(input_file, language: list, engine_mode, timeout: float,
             assert 'Rotate' not in osd
             angle = -angle % 360
         else:
-            # Tesseract == 3.04.01, hopefully also Tesseract > 3.04.01
+            # Tesseract >= 3.04.01
             # reports "Orientation in degrees" as a clockwise angle
             assert 'Rotate' in osd
 

src/ocrmypdf/helpers.py

@@ -15,6 +15,8 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
+from PIL import Image
+
 from functools import partial
 from collections.abc import Iterable
 from contextlib import suppress, contextmanager

src/ocrmypdf/pdfa.py

@@ -15,7 +15,21 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
-# Generate a PDFA_def.ps file for Ghostscript >= 9.14
+"""
+Generate a PDFMARK file for Ghostscript >= 9.14, for PDF/A conversion
+
+pdfmark is an extension to the Postscript language that describes some PDF
+features like bookmarks and annotations. It was originally specified Adobe
+Distiller, for Postscript to PDF conversion:
+https://www.adobe.com/content/dam/acom/en/devnet/acrobat/pdfs/pdfmark_reference.pdf
+
+Ghostscript uses pdfmark for PDF to PDF/A conversion as well. To use Ghostscript
+to create a PDF/A, we need to create a pdfmark file with the necessary metadata.
+
+This takes care of the many version-specific bugs and pecularities in
+Ghostscript's handling of pdfmark.
+
+"""
 
 from string import Template
 from binascii import hexlify
@@ -78,7 +92,8 @@ def
 
 
 def encode_text_string(s: str) -> str:
-    '''Encode text string to hex string for use in a PDF
+    """
+    Encode text string to hex string for use in a PDF
 
     From PDF 32000-1:2008 a string object may be included in hexademical form
     if it is enclosed in angle brackets.  For general Unicode the string should
@@ -86,7 +101,7 @@ def encode_text_string(s: str) -> str:
     ASCII strings could be encoded as PdfDocEncoding literals provided
     that certain Postscript sequences are escaped.  But it's far simpler to
     encode everything as UTF-16.
-    '''
+    """
 
     # Sometimes lazy C programmers leave their NULs at the end of strings they
     # insert into PDFs
@@ -102,8 +117,28 @@ def encode_text_string(s: str) -> str:
     return ascii_hex_str
 
 
+def _encode_ascii(s: str) -> str:
+    """
+    Aggressively strip non-ASCII and PDF escape sequences
+
+    Ghostscript 9.24+ lost support for UTF-16BE in pdfmark files for reasons
+    given in GhostPDL commit e997c683. Our temporary workaround is use ASCII
+    and drop all non-ASCII characters. A slightly improved alternative would
+    be to implement PdfDocEncoding in pikepdf and encode to that, or handle
+    metadata there.
+    """
+    trans = str.maketrans({
+        '(': '',
+        ')': '',
+        '\\': '',
+        '\0': ''
+    })
+    return s.translate(trans).encode('ascii', errors='replace').decode()
+
+
 def encode_pdf_date(d: datetime) -> str:
-    """Encode Python datetime object as PDF date string
+    """
+    Encode Python datetime object as PDF date string
 
     From Adobe pdfmark manual:
     (D:YYYYMMDDHHmmSSOHH'mm')
@@ -137,6 +172,13 @@ def encode_pdf_date(d: datetime) -> str:
 
 
 def decode_pdf_date(s: str) -> datetime:
+    """
+    Decode a pdfmark date to a Python datetime object
+
+    A pdfmark date is a string in a paritcular format. See the pdfmark
+    Reference for the specification.
+
+    """
     if s.startswith('D:'):
         s = s[2:]
 
@@ -178,24 +220,41 @@ def _get_pdfmark_dates(pdfmark):
             yield '  {} null'.format(key)
 
 
-def _get_pdfa_def(icc_profile, icc_identifier, pdfmark):
+def _get_pdfa_def(icc_profile, icc_identifier, pdfmark, ascii_docinfo=False):
-    """Create a Postscript file for Ghostscript.  pdfmark contains the various
+    """Create a Postscript pdfmark file for Ghostscript.
-    objects as strings; these must be encoded in ASCII, and dates have a 
+
-    special format."""
+    pdfmark contains the various objects as strings; these must be encoded in
+    ASCII, and dates have a special format.
+
+    :param icc_profile: filename of the ICC profile to include in pdfmark
+    :param icc_identifier: ICC identifier such as 'sRGB'
+    :param pdfmark: a dictionary containing keys to include the pdfmark
+    :param ascii_docinfo: if True, the docinfo block must be encoded in pure
+        ASCII and may not contain UTF-16BE-BOM-hex encoded strings, as
+        required for Ghostscript 9.24+
+
+    :returns: a string containing the entire pdfmark
+
+    """
 
     # Ghostscript <= 9.21 has a bug where null entries in DOCINFO might produce
     # ERROR: VMerror (-25) on closing pdfwrite device.
     # https://bugs.ghostscript.com/show_bug.cgi?id=697684
     # Work around this by only adding keys that have a nontrivial value
     docinfo_keys = ('/Title', '/Author', '/Subject', '/Creator', '/Keywords')
-    docinfo_line_template = '  {key} <{value}>'
 
     def docinfo_gen():
+        if not ascii_docinfo:
+            docinfo_line_template = '  {key} <{value}>'
+            encode = encode_text_string
+        else:
+            docinfo_line_template = '  {key} ({value})'
+            encode = _encode_ascii
         yield from _get_pdfmark_dates(pdfmark)
         for key in docinfo_keys:
             if key in pdfmark and pdfmark[key].strip() != '':
                 line = docinfo_line_template.format(
-                    key=key, value=encode_text_string(pdfmark[key]))
+                    key=key, value=encode(pdfmark[key]))
                 yield line
     docinfo = '\n'.join(docinfo_gen())
 
@@ -206,13 +265,13 @@ def _get_pdfa_def(icc_profile, icc_identifier, pdfmark):
     return result
 
 
-def generate_pdfa_ps(target_filename, pdfmark, icc='sRGB'):
+def generate_pdfa_ps(target_filename, pdfmark, icc='sRGB', ascii_docinfo=False):
     if icc == 'sRGB':
         icc_profile = SRGB_ICC_PROFILE
     else:
         raise NotImplementedError("Only supporting sRGB")
 
-    ps = _get_pdfa_def(icc_profile, icc, pdfmark)
+    ps = _get_pdfa_def(icc_profile, icc, pdfmark, ascii_docinfo=ascii_docinfo)
 
     # We should have encoded everything to pure ASCII by this point, and
     # to be safe, only allow ASCII in PostScript
@@ -263,4 +322,3 @@ def file_claims_pdfa(filename):
     pdfa_dict['conformance'] = conformance
 
     return pdfa_dict
-

src/ocrmypdf/pipeline.py

@@ -19,6 +19,7 @@ from contextlib import suppress
 from shutil import copyfileobj
 from pathlib import Path
 from datetime import datetime, timezone
+from io import BytesIO
 import sys
 import os
 import shutil
@@ -144,6 +145,13 @@ def triage_image_file(input_file, output_file, log, options):
                 "image was scanned and specify it using --image-dpi.")
             raise DpiError()
 
+        if im.mode in ('RGBA', 'LA'):
+            log.error(
+                "The input image has an alpha channel. Remove the alpha "
+                "channel first."
+            )
+            raise UnsupportedImageFormatError()
+
         if 'iccprofile' not in im.info:
             if im.mode == 'RGB':
                 log.info('Input image has no ICC profile, assuming sRGB')
@@ -496,17 +504,23 @@ def rasterize_with_ghostscript(
     options = context.get_options()
     pageinfo = get_pageinfo(input_file, context)
 
-    device = 'png16m'  # 24-bit
+    colorspaces = ['pngmono', 'pnggray', 'png256', 'png16m']
-    if pageinfo.images:
+    device_idx = 0
-        if all(image.comp == 1 for image in pageinfo.images):
+    def at_least(cs):
-            if all(image.bpc == 1 for image in pageinfo.images):
+        return max(device_idx, colorspaces.index(cs))
-                device = 'pngmono'
+
-            elif all(image.bpc > 1 and image.color == Colorspace.index
+    for image in pageinfo.images:
-                     for image in pageinfo.images):
+        if image.type_ != 'image':
-                device = 'png256'
+            continue  # ignore masks
-            elif all(image.bpc > 1 and image.color == Colorspace.gray
+        if image.bpc > 1:
-                     for image in pageinfo.images):
+            if image.color == Colorspace.index:
-                device = 'pnggray'
+                device_idx = at_least('png256')
+            elif image.color == Colorspace.gray:
+                device_idx = at_least('pnggray')
+            else:
+                device_idx = at_least('png16m')
+
+    device = colorspaces[device_idx]
 
     log.debug("Rasterize {0} with {1}".format(
               os.path.basename(input_file), device))
@@ -912,7 +926,20 @@ def generate_postscript_stub(
     options = context.get_options()
     pdf = pypdf.PdfFileReader(input_file)
     pdfmark = get_pdfmark(pdf, options)
-    generate_pdfa_ps(output_file, pdfmark)
+
+    ascii_docinfo = False
+    if ghostscript.version() >= '9.24':
+        ascii_docinfo = True
+        try:
+            for v in pdfmark.values():
+                v.encode('ascii', errors='strict')
+        except UnicodeEncodeError:
+            log.warning(
+                "Ghostscript 9.24+ does not support Unicode strings in "
+                " metadata. These will be converted to ASCII if possible."
+            )
+
+    generate_pdfa_ps(output_file, pdfmark, ascii_docinfo=ascii_docinfo)
 
 
 def skip_page(

test_requirements.txt

@@ -2,4 +2,4 @@ pytest >= 3.2
 pytest-helpers-namespace
 pytest-xdist
 pytest-cov
-pytest-timeout
+pytest-timeout == 1.2.1

tests/spoof/tesseract_cache.py

@@ -106,6 +106,8 @@ def main():
     source = os.environ['_OCRMYPDF_TEST_INFILE']  # required
     args = parser.parse_args()
 
+    cache_disabled = os.environ.get('_OCRMYPDF_CACHE_DISABLED', False)
+
     if args.imagename == 'stdin':
         real_tesseract()
 
@@ -128,7 +130,7 @@ def main():
     print("Tesseract cache folder {} - ".format(cache_folder), end='', 
           file=sys.stderr)
 
-    if (cache_folder / 'stderr.bin').exists():
+    if (cache_folder / 'stderr.bin').exists() and not cache_disabled:
         # Cache hit
         print("HIT", file=sys.stderr)
 

tests/test_main.py

@@ -300,7 +300,8 @@ def test_autorotate_threshold(
 @pytest.mark.parametrize('renderer',RENDERERS)
 def test_ocr_timeout(renderer, resources, outpdf):
     out = check_ocrmypdf(resources / 'skew.pdf', outpdf,
-                         '--tesseract-timeout', '1.0')
+                         '--tesseract-timeout', '0.01',
+                         '--pdf-renderer', renderer)
     pdfinfo = PdfInfo(out)
     assert not pdfinfo[0].has_text
 
@@ -722,6 +723,9 @@ def test_tesseract_config_notfound(renderer, resources, outdir):
 
 
 @pytest.mark.parametrize('renderer', RENDERERS)
+@pytest.mark.skipif(
+    tesseract.v4(),
+    reason='not valid due to behavior change in Tesseract >= 4.0-rc1')
 def test_tesseract_config_invalid(renderer, resources, outdir):
     cfg_file = outdir / 'test.cfg'
     with cfg_file.open('w') as f:
@@ -828,6 +832,7 @@ def test_no_contents(spoof_tesseract_noop, resources, outpdf):
 @pytest.mark.parametrize('image', [
     'baiona.png',
     'baiona_gray.png',
+    'baiona_alpha.png',
     'congress.jpg'
     ])
 def test_compression_preserved(spoof_tesseract_noop, ocrmypdf_exec,
@@ -838,7 +843,6 @@ def test_compression_preserved(spoof_tesseract_noop, ocrmypdf_exec,
     output_file = str(outpdf)
 
     im = Image.open(input_file)
-
     # Runs: ocrmypdf - output.pdf < testfile
     with open(input_file, 'rb') as input_stream:
         p_args = ocrmypdf_exec + [
@@ -848,7 +852,12 @@ def test_compression_preserved(spoof_tesseract_noop, ocrmypdf_exec,
             stdin=input_stream, env=spoof_tesseract_noop)
         out, err = p.communicate()
 
-        assert p.returncode == ExitCode.ok
+        if im.mode in ('RGBA', 'LA'):
+            # If alpha image is input, expect an error
+            assert p.returncode != ExitCode.ok and b'alpha' in err
+            return
+
+        assert p.returncode == ExitCode.ok, err.decode('utf-8')
 
     pdfinfo = PdfInfo(output_file)
 
@@ -893,7 +902,7 @@ def test_compression_changed(spoof_tesseract_noop, ocrmypdf_exec,
             stdin=input_stream, env=spoof_tesseract_noop)
         out, err = p.communicate()
 
-        assert p.returncode == ExitCode.ok
+        assert p.returncode == ExitCode.ok, err
 
     pdfinfo = PdfInfo(output_file)
 
@@ -902,13 +911,7 @@ def test_compression_changed(spoof_tesseract_noop, ocrmypdf_exec,
     if compression == "jpeg":
         assert pdfimage.enc == Encoding.jpeg
     else:
-        if ghostscript.jpeg_passthrough_available():
+        assert pdfimage.enc not in (Encoding.jpeg, Encoding.jpeg2000)
-            # Ghostscript 9.23 adds JPEG passthrough, which allows a JPEG to be
-            # copied without transcoding - so report
-            if image.endswith('jpg'):
-                assert pdfimage.enc == Encoding.jpeg
-        else:
-            assert pdfimage.enc not in (Encoding.jpeg, Encoding.jpeg2000)
 
     if im.mode.startswith('RGB') or im.mode.startswith('BGR'):
         assert pdfimage.color == Colorspace.rgb, \
@@ -966,6 +969,7 @@ def test_pdfa_n(spoof_tesseract_cache, pdfa_level, resources, outpdf):
     assert pdfa_info['conformance'] == 'PDF/A-{}B'.format(pdfa_level)
 
 
+@pytest.mark.skipif(sys.version_info >= (3, 7, 0), reason='fixed')
 def test_bad_locale():
     env = os.environ.copy()
     env['LC_ALL'] = 'C'

tests/test_metadata.py

@@ -24,6 +24,12 @@ from datetime import timezone
 from ocrmypdf.pdfa import file_claims_pdfa, encode_pdf_date, decode_pdf_date
 from ocrmypdf.exceptions import ExitCode
 from ocrmypdf.lib import fitz
+from ocrmypdf.helpers import fspath
+from ocrmypdf.pdfa import (
+    file_claims_pdfa, encode_pdf_date, decode_pdf_date, generate_pdfa_ps,
+    SRGB_ICC_PROFILE
+)
+from ocrmypdf.exec import ghostscript
 
 # pytest.helpers is dynamic
 # pylint: disable=no-member
@@ -76,6 +82,9 @@ def test_override_metadata(spoof_tesseract_noop, output_type, resources,
     before = pypdf.PdfFileReader(str(input_file))
     after = pypdf.PdfFileReader(outpdf)
 
+    if ghostscript.version() >= '9.24':
+        pytest.xfail('Ghostscript 9.24+ does not support Unicode DOCINFO')
+
     assert after.documentInfo['/Title'] == german
     assert after.documentInfo['/Author'] == chinese
     assert after.documentInfo.get('/Keywords', '') == ''
@@ -157,5 +166,3 @@ def test_creation_date_preserved(spoof_tesseract_noop, output_type, resources,
     date_after = decode_pdf_date(after['/ModDate'])
     assert seconds_between_dates(
         date_after, datetime.datetime.now(timezone.utc)) < 1000
-
-