v6.2.4 release notes

This feature works for gs >= 9.24, but we don't want to change behavior in
a backport, so we're disabling it unconditionally.

.gitignore

@@ -2,6 +2,7 @@
 *.pyc
 *.sublime-*
 venv*/
+.venv/
 pyvenv.cfg
 tasks.py
 .bash_history

.travis.yml

@@ -1,32 +1,51 @@
 dist: trusty
 language: python
 cache:
-  ccache: true
   pip: true
   directories:
   - $HOME/Library/Caches/Homebrew
 
-env:
-  global:
-    - secure: "hsf6MT+n2x3OiDM2fQyJZdV0/PWYmv81LdVqC6cfnHBE/8N3DloJRqQ7WfO14TxhiK9PEC7MpyCj0lSabUHEO7gSH6Vks6I1asoSkt8S9/bSMlhT4hei+pwVpeGEiU5xHVATNjY+D919VC3IFvc3XmjT74h/2SLhaZ+jhEmDggM="  # HOMEBREW_OCRMYPDF_TOKEN
+addons:
+  apt:
+    update: true
+    sources:
+    - sourceline: 'ppa:alex-p/tesseract-ocr'
+    - sourceline: 'ppa:heyarje/libav-11'
+    - sourceline: 'ppa:vshn/ghostscript'
+    packages:
+    - ghostscript
+    - libavcodec56
+    - libavformat56
+    - libavutil54
+    - libffi-dev
+    - poppler-utils
+    - qpdf
+    - tesseract-ocr
+    - tesseract-ocr-deu
+    - tesseract-ocr-eng
+    - tesseract-ocr-fra
 
 matrix:
   include:
     - os: linux
       sudo: required
       language: python
-      python: 3.5
+      python: "3.5"
       env: EXTRAS=
     - os: linux
       sudo: required
       language: python
-      python: 3.6
+      python: "3.6"
       env: EXTRAS=
     - os: linux
       sudo: required
       language: python
-      python: 3.6
+      python: "3.6"
       env: EXTRAS=[fitz]
+    - os: linux
+      sudo: required
+      language: python
+      python: "3.7-dev"
     - os: osx
       osx_image: xcode8
       language: generic
@@ -41,7 +60,10 @@ before_cache:
 
 before_install: |
   if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
-    bash .travis/linux_before_install.sh
+    pip install --upgrade pip
+    mkdir -p packages
+    wget -q 'https://www.dropbox.com/s/vaq0kbwi6e6au80/unpaper_6.1-1.deb?raw=1' -O packages/unpaper_6.1-1.deb
+    sudo dpkg -i packages/unpaper_6.1-1.deb
   elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
     brew update && brew bundle --file=.travis/Brewfile
     pip3 install --upgrade pip
@@ -49,6 +71,7 @@ before_install: |
   fi
 
 install:
+- pip3 install pycparser # py3.7 workaround for https://github.com/eliben/pycparser/issues/251
 - pip3 install ".$EXTRAS"
 - pip3 install -r test_requirements.txt
 
@@ -73,31 +96,3 @@ deploy:
     tags: true
     condition: $TRAVIS_PYTHON_VERSION == "3.6" && $TRAVIS_OS_NAME == "linux" && $EXTRAS == ""
   skip_upload_docs: true
-
-  # test pypi
-- provider: pypi
-  server: https://testpypi.pypi.org/legacy/
-  user: ocrmypdf-travis
-  password:
-    secure: "DTFOmmNL6olA0+yXvp4u9jXZlZeqrJsJ0526jzqf4a3gZ6jnGTq5UI6WzRsslSyoMMfXKtHQebqHM6ogSgCZinyZ3ufHJo8fn9brxbEc2gsiWkbj5o3bGwdWMT1vNNE7XW0VCpw87rZ1EEwjl4FJHFudMlPR1yfU5+uq0k0PACo="
-  distributions: "sdist"
-  on:
-    branch: develop
-    tags: false
-    condition: $TRAVIS_OS_NAME == "osx"
-  skip_upload_docs: true
-
-  # null deploy for osx
-  # we really just want to run after_deploy *after* pypi upload is done, but
-  # after_deploy on runs if a given box deployed
-- provider: script
-  script: /usr/bin/true
-  on:
-    branch: master
-    tags: true
-    condition: $TRAVIS_OS_NAME == "osx"
-
-after_deploy: |
-  if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-    bash .travis/osx_brew.sh
-  fi

.travis/autobrew.py

@@ -1,93 +0,0 @@
-#!/usr/bin/env python3
-# © 2017-18 James R. Barlow: github.com/jbarlow83
-
-from string import Template
-from subprocess import run, PIPE
-import re
-
-recipe_template = Template("""
-class Ocrmypdf < Formula
-  include Language::Python::Virtualenv
-
-  desc "Adds an OCR text layer to scanned PDF files"
-  homepage "https://github.com/jbarlow83/OCRmyPDF"
-  ${ocrmypdf_url}
-  ${ocrmypdf_sha256}
-
-  depends_on "pkg-config" => :build
-  depends_on "mupdf-tools" => :build  # statically links libmupdf.a
-  depends_on "freetype"
-  depends_on "ghostscript"
-  depends_on "jpeg"
-  depends_on "libpng"
-  depends_on "python"
-  depends_on "qpdf"
-  depends_on "tesseract"
-  depends_on "unpaper"
-
-${resources}
-  def install
-    venv = virtualenv_create(libexec, "python3")
-
-    resource("Pillow").stage do
-      inreplace "setup.py" do |s|
-        sdkprefix = MacOS::CLT.installed? ? "" : MacOS.sdk_path
-        s.gsub! "openjpeg.h", "probably_not_a_header_called_this_eh.h"
-        s.gsub! "ZLIB_ROOT = None", "ZLIB_ROOT = ('#{sdkprefix}/usr/lib', '#{sdkprefix}/usr/include')"
-        s.gsub! "JPEG_ROOT = None", "JPEG_ROOT = ('#{Formula["jpeg"].opt_prefix}/lib', '#{Formula["jpeg"].opt_prefix}/include')"
-        s.gsub! "FREETYPE_ROOT = None", "FREETYPE_ROOT = ('#{Formula["freetype"].opt_prefix}/lib', '#{Formula["freetype"].opt_prefix}/include')"
-      end
-
-      # avoid triggering "helpful" distutils code that doesn't recognize Xcode 7 .tbd stubs
-      ENV.append "CFLAGS", "-I#{MacOS.sdk_path}/System/Library/Frameworks/Tk.framework/Versions/8.5/Headers" unless MacOS::CLT.installed?
-      venv.pip_install Pathname.pwd
-    end
-
-    res = resources.map(&:name).to_set - ["Pillow"]
-
-    res.each do |r|
-      venv.pip_install resource(r)
-    end
-
-    venv.pip_install_and_link buildpath
-  end
-
-  test do
-    # Since we use Python 3, we require a UTF-8 locale
-    ENV["LC_ALL"] = "en_US.UTF-8"
-
-    system "#{bin}/ocrmypdf", "-f", "-q", "--deskew",
-                              test_fixtures("test.pdf"), "ocr.pdf"
-    assert_predicate testpath/"ocr.pdf", :exist?
-  end
-end
-""")
-
-def main():
-    p = run(['poet', '--single', 'ocrmypdf'],
-            encoding='utf-8', stdout=PIPE, check=True)
-
-    ocrmypdf_lines = p.stdout.splitlines()
-    ocrmypdf_url = ocrmypdf_lines[1].strip()
-    ocrmypdf_sha256 = ocrmypdf_lines[2].strip()
-
-    ocrmypdf_version = re.search(
-        r'ocrmypdf-(.+)\.tar.*', ocrmypdf_url).group(1)
-    print(f"Autobrewing {ocrmypdf_version}")
-
-    p = run(['poet', '--resources', 'ocrmypdf'],
-            encoding='utf-8', stdout=PIPE, check=True)
-
-    poet_resources = p.stdout
-
-    # Remove the duplicate "ocrmypdf" resource block
-    all_resources = poet_resources.split('resource')
-    kept_resources = [block for block in all_resources if 'ocrmypdf' not in block]
-    resources = 'resource'.join(kept_resources)
-
-    with open('ocrmypdf.rb', 'w') as out:
-        out.write(recipe_template.substitute(**locals()))
-
-
-if __name__ == '__main__':
-    main()

.travis/linux_before_install.sh

@@ -1,31 +0,0 @@
-#!/bin/bash
-# © 2017 James R. Barlow: github.com/jbarlow83
-set -euo pipefail
-set -x
-
-sudo add-apt-repository ppa:vshn/ghostscript -y
-sudo add-apt-repository ppa:heyarje/libav-11 -y
-sudo apt-get update -qq
-sudo apt-get install -y \
-	ghostscript \
-	poppler-utils \
-	libavformat56 \
-	libavcodec56 \
-	libavutil54 \
-	libffi-dev \
-	qpdf
-
-sudo add-apt-repository ppa:alex-p/tesseract-ocr -y
-
-sudo apt-get update
-sudo apt-get autoremove -y
-sudo apt-get install -y --no-install-recommends \
-		tesseract-ocr \
-		tesseract-ocr-eng \
-		tesseract-ocr-fra \
-		tesseract-ocr-deu
-
-pip install --upgrade pip
-mkdir -p packages
-wget -q 'https://www.dropbox.com/s/vaq0kbwi6e6au80/unpaper_6.1-1.deb?raw=1' -O packages/unpaper_6.1-1.deb
-sudo dpkg -i packages/unpaper_6.1-1.deb

.travis/osx_brew.sh

@@ -1,23 +0,0 @@
-#!/bin/bash
-# © 2017 James R. Barlow: github.com/jbarlow83
-set -uo pipefail
-set -x
-
-pip3 install homebrew-pypi-poet
-python3 .travis/autobrew.py
-cat ocrmypdf.rb
-
-# brew audit crashes Travis
-#brew audit ocrmypdf.rb
-
-# Important: disable debug output so token is hidden
-set +x
-git clone https://$HOMEBREW_OCRMYPDF_TOKEN@github.com/jbarlow83/homebrew-ocrmypdf.git
-set -x
-
-pushd homebrew-ocrmypdf
-cp ../ocrmypdf.rb Formula/ocrmypdf.rb
-git add Formula/ocrmypdf.rb
-git commit -m "homebrew-ocrmypdf: automatic release $TRAVIS_BUILD_NUMBER $TRAVIS_TAG"
-git push origin master
-popd

README.rst

@@ -4,10 +4,10 @@ OCRmyPDF
 .. image:: https://travis-ci.org/jbarlow83/OCRmyPDF.svg?branch=master
     :target: https://travis-ci.org/jbarlow83/OCRmyPDF
 
-.. image:: https://img.shields.io/pypi/v/ocrmypdf.svg   
+.. image:: https://img.shields.io/pypi/v/ocrmypdf.svg
     :target: https://pypi.org/project/ocrmypdf/
 
-.. image:: https://img.shields.io/homebrew/v/ocrmypdf.svg   
+.. image:: https://img.shields.io/homebrew/v/ocrmypdf.svg
     :alt: homebrew
     :target: http://brewformulas.org/Ocrmypdf
 
@@ -54,7 +54,7 @@ Motivation
 I searched the web for a free command line tool to OCR PDF files on
 Linux/UNIX: I found many, but none of them were really satisfying.
 
--  Either they produced PDF files with misplaced text under the image (making copy/paste impossible) 
+-  Either they produced PDF files with misplaced text under the image (making copy/paste impossible)
 -  Or they did not handle accents and multilingual characters
 -  Or they changed the resolution of the embedded images
 -  Or they generated ridiculously large PDF files
@@ -63,7 +63,7 @@ Linux/UNIX: I found many, but none of them were really satisfying.
 -  On top of that none of them produced PDF/A files (format dedicated for long time storage)
 
 ...so I decided to develop my own tool (using various existing scripts
-as an inspiration). 
+as an inspiration).
 
 Installation
 ------------
@@ -97,7 +97,7 @@ you can often find packages that provide language packs:
 
    # Debian/Ubuntu users
    apt-get install tesseract-ocr-chi-sim  # Example: Install Chinese Simplified language back
-   
+
 You can then pass the ``-l LANG`` argument to OCRmyPDF to give a hint as to what languages it should search for. Multiple
 languages can be requested.
 
@@ -126,9 +126,7 @@ If you detect an issue, please:
 Requirements
 ------------
 
-Runs on CPython 3.6, and requires external program installations of Ghostscript, Tesseract OCR, QPDF, and Leptonica. ocrmypdf is pure Python, but uses CFFI to portably generate library bindings.
-
-Python 3.5 is also supported.
+Runs on CPython 3.5, 3.6 and 3.7. Requires external program installations of Ghostscript, Tesseract OCR, QPDF, and Leptonica. ocrmypdf is pure Python, but uses CFFI to portably generate library bindings.
 
 Press & Media
 -------------
@@ -142,7 +140,7 @@ Press & Media
 License
 -------
 
-The OCRmyPDF software is licensed under the GNU GPLv3. Certain files are covered by other licenses, as noted in their source files. 
+The OCRmyPDF software is licensed under the GNU GPLv3. Certain files are covered by other licenses, as noted in their source files.
 
 The license for each test file varies, and is noted in tests/resources/README.rst. The documentation is licensed under Creative Commons Attribution-ShareAlike 4.0 (CC-BY-SA 4.0).
 

docs/installation.rst

@@ -1,7 +1,7 @@
 Installation
 ============
 
-The easiest way to install OCRmyPDF to follow the steps for your operating system/platform. 
+The easiest way to install OCRmyPDF to follow the steps for your operating system/platform.
 
 If you want to use the latest version of OCRmyPDF, your best bet is to install the most recent version your platform provides, and then upgrade that version by installing the Python binary wheels.
 
@@ -23,7 +23,7 @@ To see what versions are available, check the `Debian Package Tracker <https://t
 Installing on macOS with Homebrew
 ---------------------------------
 
-.. image:: https://img.shields.io/homebrew/v/ocrmypdf.svg   
+.. image:: https://img.shields.io/homebrew/v/ocrmypdf.svg
     :alt: homebrew
     :target: http://brewformulas.org/Ocrmypdf
 
@@ -55,7 +55,7 @@ Follow the Docker installation instructions for your platform.  If you can run t
 .. code-block:: bash
 
     docker run hello-world
-   
+
 OCRmyPDF will use all available CPU cores.  By default, the VirtualBox machine instance on Windows and macOS has only a single CPU core enabled. Use the VirtualBox Manager to determine the name of your Docker engine host, and then follow these optional steps to enable multiple CPUs:
 
 .. code-block:: bash
@@ -103,14 +103,14 @@ You can then run ocrmypdf using the command:
 .. code-block:: bash
 
     docker run --rm ocrmypdf --help
-  
+
 To execute the OCRmyPDF on a local file, you must `provide a writable volume to the Docker image <https://docs.docker.com/userguide/dockervolumes/>`_, and both the input and output file must be inside the writable volume.  This example command uses the current working directory as the writable volume:
 
 .. code-block:: bash
 
     docker run --rm -v "$(pwd):/home/docker" <other docker arguments>   ocrmypdf <your arguments to ocrmypdf>
 
-In this worked example, the current working directory contains an input file called ``test.pdf`` and the output will go to ``output.pdf``: 
+In this worked example, the current working directory contains an input file called ``test.pdf`` and the output will go to ``output.pdf``:
 
 .. code-block:: bash
 
@@ -167,11 +167,11 @@ Install or upgrade the required Homebrew packages, if any are missing:
     brew install python3
     brew install libxml2 libffi leptonica
     brew install unpaper   # optional
-   
-Python 3.5 and 3.6 are supported.
+
+Python 3.5, 3.6 and 3.7 are supported.
 
 Install the required Tesseract OCR engine with the language packs you plan to use:
-   
+
 .. code-block:: bash
 
     brew install tesseract                       # Option 1: for English, French, German, Spanish
@@ -398,7 +398,7 @@ The following dependencies are recommended:
 - unpaper 6.1
 - PyMuPDF 1.12.5 or newer
 
-These are in addition to the Python packaging dependencies, meaning that unfortunately, the ``pip install`` command cannot satisfy all of them. 
+These are in addition to the Python packaging dependencies, meaning that unfortunately, the ``pip install`` command cannot satisfy all of them.
 
 Python 3.6 and Tesseract 4.0.0-beta.1 are recommended for best OCR results and best performance.
 

docs/release_notes.rst

@@ -10,6 +10,27 @@ The OCRmyPDF package itself does not contain a public API, although it is fairly
    replace: `#$1 <https://github.com/jbarlow83/OCRmyPDF/issues/$1>`_
 
 
+v6.2.4
+------
+
+-   Backport Ghostscript 9.25 compatibility fixes, which removes support for setting Unicode metadata
+-   Backport blacklisting Ghostscript 9.24
+-   Older versions of Ghostscript are still supported
+
+
+v6.2.3
+------
+
+-   Fix compatibility with img2pdf >= 0.3.0 by rejecting input images that have an alpha channel
+
+
+v6.2.2
+------
+
+-   Backport compatibility fixes for Python 3.7 and ruffus 2.7.0 from v7.0.0
+-   Backport fix to ignore masks when deciding what colors are on a page
+-   Backport some minor improvements from v7.0.0: better argument validation and warnings about the Tesseract 4.0.0 ``--user-words`` regression
+
 v6.2.1
 ------
 

requirements.txt

@@ -1,10 +1,11 @@
 # requirements.txt can be used to replicate the developer's build environment
 # setup.py lists a separate set of requirements that are looser to simplify
 # installation
-ruffus == 2.6.3
-Pillow == 5.1.0
+ruffus == 2.7.0
+Pillow == 5.2.0
 reportlab == 3.4.0
 PyPDF2 == 1.26.0
 img2pdf == 0.2.4
 cffi == 1.11.5
-PyMuPDF == 1.12.5
+PyMuPDF == 1.12.5
+defusedxml == 0.5.0

setup.py

@@ -215,6 +215,7 @@ setup(
     classifiers=[
         "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
         "Development Status :: 5 - Production/Stable",
         "Environment :: Console",
         "Intended Audience :: End Users/Desktop",
@@ -243,12 +244,12 @@ setup(
         'cffi >= 1.9.1',          # must be a setup and install requirement
         'defusedxml >= 0.5.0',    # pure Python, so track HEAD closely
         'img2pdf >= 0.2.4',       # pure Python, so track HEAD closely
-        'Pillow >= 4.0.0, != 5.1.0 ; sys_platform == "darwin"',        
+        'Pillow >= 4.0.0, != 5.1.0 ; sys_platform == "darwin"',
                                   # Pillow < 4 has BytesIO/TIFF bug w/img2pdf 0.2.3
                                   # block 5.1.0, broken wheels
         'PyPDF2 >= 1.26',         # pure Python, so track HEAD closely
         'reportlab >= 3.3.0',     # oldest released version with sane image handling
-        'ruffus == 2.6.3',        # pinned - ocrmypdf implements a 2.6.3 workaround
+        'ruffus >= 2.7.0',
     ],
     extras_require={
         'fitz': ['PyMuPDF >= 1.12.5']     # for table of contents bug

src/ocrmypdf/__main__.py

@@ -70,7 +70,7 @@ def complain(message):
 if 'IDE_PROJECT_ROOTS' in os.environ:
     os.environ['PATH'] = '/usr/local/bin:' + os.environ['PATH']
 
-# -------- 
+# --------
 # Critical environment tests
 
 verify_python3_env()
@@ -85,6 +85,20 @@ if tesseract.version() < MINIMUM_TESS_VERSION:
 # -------------
 # Parser
 
+def numeric(basetype, min_=None, max_=None):
+    "Validator for numeric params"
+    min_ = basetype(min_) if min_ is not None else None
+    max_ = basetype(max_) if max_ is not None else None
+    def _numeric(string):
+        value = basetype(string)
+        if (min_ is not None and value < min_
+                or max_ is not None and value > max_):
+            msg = "%r not in valid range %r" % (string, (min_, max_))
+            raise argparse.ArgumentTypeError(msg)
+        return value
+    return _numeric
+
+
 parser = argparse.ArgumentParser(
     prog=PROGRAM_NAME,
     fromfile_prefix_chars='@',
@@ -156,7 +170,7 @@ parser.add_argument(
     '--image-dpi', metavar='DPI', type=int,
     help="For input image instead of PDF, use this DPI instead of file's.")
 parser.add_argument(
-    '--output-type', choices=['pdfa', 'pdf', 'pdfa-1', 'pdfa-2', 'pdfa-3'], 
+    '--output-type', choices=['pdfa', 'pdf', 'pdfa-1', 'pdfa-2', 'pdfa-3'],
     default='pdfa',
     help="Choose output type. 'pdfa' creates a PDF/A-2b compliant file for "
          "long term archiving (default, recommended) but may not suitable "
@@ -233,7 +247,7 @@ preprocessing.add_argument(
     help="Clean page as above, and incorporate the cleaned image in the final "
          "PDF.  Might remove desired content.")
 preprocessing.add_argument(
-    '--oversample', metavar='DPI', type=int, default=0,
+    '--oversample', metavar='DPI', type=numeric(int, 0, 5000), default=0,
     help="Oversample images to at least the specified DPI, to improve OCR "
          "results slightly")
 
@@ -255,7 +269,7 @@ ocrsettings.add_argument(
 #          "pages")
 
 ocrsettings.add_argument(
-    '--skip-big', type=float, metavar='MPixels',
+    '--skip-big', type=numeric(float, 0, 5000), metavar='MPixels',
     help="Skip OCR on pages larger than the specified amount of megapixels, "
          "but include skipped pages in final output")
 
@@ -263,7 +277,7 @@ advanced = parser.add_argument_group(
     "Advanced",
     "Advanced options to control Tesseract's OCR behavior")
 advanced.add_argument(
-    '--max-image-mpixels', action='store', type=float, metavar='MPixels',
+    '--max-image-mpixels', action='store', type=numeric(float, 0), metavar='MPixels',
     help="Set maximum number of pixels to unpack before treating an image as a "
          "decompression bomb",
     default=128.0)
@@ -296,11 +310,11 @@ advanced.add_argument(
          " of Ghostscript; deprecated"
     )
 advanced.add_argument(
-    '--tesseract-timeout', default=180.0, type=float, metavar='SECONDS',
+    '--tesseract-timeout', default=180.0, type=numeric(float, 0), metavar='SECONDS',
     help='Give up on OCR after the timeout, but copy the preprocessed page '
          'into the final output')
 advanced.add_argument(
-    '--rotate-pages-threshold', default=14.0, type=float, metavar='CONFIDENCE',
+    '--rotate-pages-threshold', default=14.0, type=numeric(float, max_=1000), metavar='CONFIDENCE',
     help="Only rotate pages when confidence is above this value (arbitrary "
          "units reported by tesseract)")
 advanced.add_argument(
@@ -491,6 +505,10 @@ def check_options_advanced(options, log):
             "--output-type is not 'pdfa', 'pdfa-1', or 'pdfa-2'"
         )
 
+    if tesseract.v4() and (options.user_words or options.user_patterns):
+        log.warning(
+            'Tesseract 4.x ignores --user-words, so this has no effect')
+
 
 def check_options_metadata(options, log):
     import unicodedata
@@ -574,7 +592,7 @@ def do_ruffus_exception(ruffus_five_tuple, options, log):
     if exc_name == 'builtins.SystemExit':
         match = re.search(r"\.(.+?)\)", exc_value)
         exit_code_name = match.groups()[0]
-        exit_code = getattr(ExitCode, exit_code_name, 'other_error')        
+        exit_code = getattr(ExitCode, exit_code_name, 'other_error')
     elif exc_name == 'ruffus.ruffus_exceptions.MissingInputFileError':
         log.error(cleanup_ruffus_error_message(exc_value))
         exit_code = ExitCode.input_file
@@ -598,7 +616,7 @@ def do_ruffus_exception(ruffus_five_tuple, options, log):
             (exc_name == 'ocrmypdf.exceptions.EncryptedPdfError'):
         log.error(textwrap.dedent("""\
             Input PDF is encrypted. The encryption must be removed to
-            perform OCR. 
+            perform OCR.
 
             For information about this PDF's security use
                 qpdf --show-encryption infilename
@@ -607,7 +625,7 @@ def do_ruffus_exception(ruffus_five_tuple, options, log):
                 qpdf --decrypt [--password=[password]] infilename
 
             """))
-        exit_code = ExitCode.encrypted_pdf        
+        exit_code = ExitCode.encrypted_pdf
     elif exc_name == 'ocrmypdf.exceptions.PdfMergeFailedError':
         log.error(textwrap.dedent("""\
             Failed to merge PDF image layer with OCR layer
@@ -638,33 +656,31 @@ def do_ruffus_exception(ruffus_five_tuple, options, log):
     return ExitCode.other_error
 
 
-def traverse_ruffus_exception(e_args, options, log):
-    """Walk through a RethrownJobError and find the first exception.
+def traverse_ruffus_exception(exceptions, options, log):
+    """Traverse a RethrownJobError and output the exceptions
 
-    Ruffus flattens exception to 5 element tuples. Because of a bug
-    in <= 2.6.3 it may present either the single:
-      (task, job, exc, value, stack)
-    or something like:
-      [[(task, job, exc, value, stack)]]
-    
-    Generally cross-process exception marshalling doesn't work well
-    and ruffus doesn't support because BaseException has its own
-    implementation of __reduce__ that attempts to reconstruct the
-    exception based on e.__init__(e.args).
-    
-    Attempting to log the exception directly marshalls it to the logger
-    which is probably in another process, so it's better to log only
-    data from the exception at this point.
+    Ruffus presents exceptions as 5 element tuples. The RethrownJobException
+    has a list of exceptions like
+        e.job_exceptions = [(5-tuple), (5-tuple), ...]
+
+    ruffus < 2.7.0 had a bug with exception marshalling that would give
+    different output whether the main or child process raised the exception.
+    We no longer support this.
+
+    Attempting to log the exception itself will re-marshall it to the logger
+    which is normally running in another process. It's better to avoid re-
+    marshalling.
 
     The exit code will be based on this, even if multiple exceptions occurred
     at the same time."""
 
-    if isinstance(e_args, Sequence) and isinstance(e_args[0], str) and \
-            len(e_args) == 5:
-        return do_ruffus_exception(e_args, options, log)
-    elif is_iterable_notstr(e_args):
-        for exc in e_args:
-            return traverse_ruffus_exception(exc, options, log)
+    exit_codes = []
+    for exc in exceptions:
+        exit_code = do_ruffus_exception(exc, options, log)
+        exit_codes.append(exit_code)
+
+    return exit_codes[0]  # Multiple codes are rare so take the first one
+
 
 
 def check_closed_streams(options):
@@ -749,7 +765,7 @@ def check_environ(options, _log):
     for k in old_envvars:
         if k in os.environ:
             _log.warning(textwrap.dedent("""\
-                OCRmyPDF no longer uses the environment variable {}. 
+                OCRmyPDF no longer uses the environment variable {}.
                 Change PATH to select alternate programs.""".format(k)))
 
 
@@ -792,14 +808,14 @@ def report_output_file_size(options, _log, input_file, output_file):
     ratio = output_size / input_size
     if ratio < 1.35 or input_size < 25000:
         return  # Seems fine
-    
+
     reasons = []
     if not fitz:
         reasons.append("The optional dependency PyMuPDF is not installed.")
     image_preproc = {
-        'deskew', 
-        'clean_final', 
-        'remove_background', 
+        'deskew',
+        'clean_final',
+        'remove_background',
         'oversample',
         'force_ocr'
     }
@@ -847,6 +863,14 @@ def run_pipeline():
             "security vulnerabilities with certain malformed PDFs. Consider "
             "upgrading to version 7.0.0 or newer.".format(qpdf.version()))
 
+    if ghostscript.version() == '9.24':
+        complain(
+            "Ghostscript 9.24 contains serious regressions and is not "
+            "supported. Please upgrade to Ghostscript 9.25 or use an older "
+            "version."
+        )
+        return ExitCode.missing_dependency
+
     # Any changes to options will not take effect for options that are already
     # bound to function parameters in the pipeline. (For example
     # options.input_file, options.pdf_renderer are already bound.)
@@ -886,7 +910,8 @@ def run_pipeline():
     except ruffus_exceptions.RethrownJobError as e:
         if options.verbose:
             _log.debug(str(e))  # stringify exception so logger doesn't have to
-        exitcode = traverse_ruffus_exception(e.args, options, _log)
+        exceptions = e.job_exceptions
+        exitcode = traverse_ruffus_exception(exceptions, options, _log)
         if exitcode is None:
             _log.error("Unexpected ruffus exception: " + str(e))
             _log.error(repr(e))
@@ -919,7 +944,7 @@ def run_pipeline():
             _log.warning('Output file: The generated PDF is INVALID')
             return ExitCode.invalid_output_pdf
 
-        report_output_file_size(options, _log, start_input_file, 
+        report_output_file_size(options, _log, start_input_file,
                                 options.output_file)
 
     pdfinfo = context.get_pdfinfo()

src/ocrmypdf/_unicodefun.py

@@ -40,6 +40,11 @@ import codecs
 
 def verify_python3_env():
     """Ensures that the environment is good for unicode on Python 3."""
+
+    # PEP 538 changes in Python 3.7 should make this wrangling unnecessary
+    if sys.version_info[0:3] >= (3, 7, 0):
+        return
+
     try:
         import locale
         fs_enc = codecs.lookup(locale.getpreferredencoding()).name

src/ocrmypdf/exec/__init__.py

@@ -25,7 +25,7 @@ from ..exceptions import MissingDependencyError
 
 
 
-def get_version(program, *, 
+def get_version(program, *,
         version_arg='--version', regex=r'(\d+(\.\d+)*)'):
     "Get the version of the specified program"
     args_prog = [
@@ -37,6 +37,10 @@ def get_version(program, *,
             args_prog, close_fds=True, universal_newlines=True,
             stdout=PIPE, stderr=STDOUT, check=True)
         output = proc.stdout
+    except FileNotFoundError as e:
+        raise MissingDependencyError(
+            "Could not find program '{}' on the PATH".format(
+                program)) from e
     except CalledProcessError as e:
         if e.returncode < 0:
             raise MissingDependencyError(

src/ocrmypdf/exec/ghostscript.py

@@ -34,12 +34,20 @@ def version():
 
 def jpeg_passthrough_available():
     """
-    Ghostscript 9.23 introduced JPEG passthrough but it seems to corrupt the
-    last two bytes of certain images, for now we disable it for 9.23 and
-    do not mention it for < 9.23.
+    Returns True if the installed version of Ghostscript supports JPEG passthru
 
+    Prior to 9.23, Ghostscript decode and re-encoded JPEGs internally. In 9.23
+    it gained the ability to keep JPEGs unmodified. However, the 9.23
+    implementation was buggy and would deletes the last two bytes of images in
+    some cases, as reported here.
     https://bugs.ghostscript.com/show_bug.cgi?id=699216
 
+    The issue was fixed for 9.24, hence that is the first version we consider
+    the feature available. (However, we don't use 9.24 at all, so the first
+    version that allows JPEG passthrough is 9.25.
+
+    Regardless, in ocrmypdf 6.x we are ignoring this new feature entirely to
+    avoid new behavior.
     """
     return False
 
@@ -52,20 +60,20 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
                   pageno=1, page_dpi=None):
     """
     Rasterize one page of a PDF at resolution (xres, yres) in canvas units.
-    
-    The image is sized to match the integer pixels dimensions implied by 
+
+    The image is sized to match the integer pixels dimensions implied by
     (xres, yres) even if those numbers are noninteger. The image's DPI will
      be overridden with the values in page_dpi.
-    
+
     :param input_file: pathlike
     :param output_file: pathlike
     :param xres: resolution at which to rasterize page
-    :param yres: 
-    :param raster_device: 
-    :param log: 
+    :param yres:
+    :param raster_device:
+    :param log:
     :param pageno: page number to rasterize (beginning at page 1)
-    :param page_dpi: resolution tuple (x, y) overriding output image DPI 
-    :return: 
+    :param page_dpi: resolution tuple (x, y) overriding output image DPI
+    :return:
     """
     res = xres, yres
     int_res = round(xres), round(yres)
@@ -85,7 +93,7 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
             '-o', tmp.name,
             fspath(input_file)
         ]
-        
+
         p = run(args_gs, stdout=PIPE, stderr=STDOUT,
                 universal_newlines=True)
         if _gs_error_reported(p.stdout):
@@ -139,15 +147,17 @@ def generate_pdfa(pdf_pages, output_file, compression, log,
             "-dAutoFilterGrayImages=true",
         ]
 
-    # Older versions of Ghostscript expect a leading slash in 
-    # sColorConversionStrategy, newer ones should not have it. See Ghostscript 
+    # Older versions of Ghostscript expect a leading slash in
+    # sColorConversionStrategy, newer ones should not have it. See Ghostscript
     # git commit fe1c025d.
     strategy = 'RGB' if version() >= '9.19' else '/RGB'
 
-    if version() == '9.23':
+    if version() >= '9.23':
         # 9.23: new feature JPEG passthrough is broken in some cases, best to
         # disable it always
         # https://bugs.ghostscript.com/show_bug.cgi?id=699216
+        # fixed in 9.24, but to avoid changing expected behavior we disable it
+        # for ocrmypdf 6.x
         compression_args.append('-dPassThroughJPEGImages=false')
 
     with NamedTemporaryFile(delete=True) as gs_pdf:

src/ocrmypdf/exec/tesseract.py

@@ -160,7 +160,7 @@ def get_orientation(input_file, language: list, engine_mode, timeout: float,
             assert 'Rotate' not in osd
             angle = -angle % 360
         else:
-            # Tesseract == 3.04.01, hopefully also Tesseract > 3.04.01
+            # Tesseract >= 3.04.01
             # reports "Orientation in degrees" as a clockwise angle
             assert 'Rotate' in osd
 

src/ocrmypdf/helpers.py

@@ -15,6 +15,8 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
+from PIL import Image
+
 from functools import partial
 from collections.abc import Iterable
 from contextlib import suppress, contextmanager
@@ -103,7 +105,7 @@ def is_file_writable(test_file):
 
     if p.is_symlink():
         # Python 3.5 does not accept parameters for Path.resolve() and behaves
-        # as if strict=True (throws an exception on failure). Python 3.6 
+        # as if strict=True (throws an exception on failure). Python 3.6
         # defaults to strict=False. This implements strict=False like behavior
         # for Python 3.5.
         if sys.version_info[0:2] <= (3, 5):

src/ocrmypdf/pdfa.py

@@ -15,7 +15,21 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
-# Generate a PDFA_def.ps file for Ghostscript >= 9.14
+"""
+Generate a PDFMARK file for Ghostscript >= 9.14, for PDF/A conversion
+
+pdfmark is an extension to the Postscript language that describes some PDF
+features like bookmarks and annotations. It was originally specified Adobe
+Distiller, for Postscript to PDF conversion:
+https://www.adobe.com/content/dam/acom/en/devnet/acrobat/pdfs/pdfmark_reference.pdf
+
+Ghostscript uses pdfmark for PDF to PDF/A conversion as well. To use Ghostscript
+to create a PDF/A, we need to create a pdfmark file with the necessary metadata.
+
+This takes care of the many version-specific bugs and pecularities in
+Ghostscript's handling of pdfmark.
+
+"""
 
 from string import Template
 from binascii import hexlify
@@ -78,7 +92,8 @@ def
 
 
 def encode_text_string(s: str) -> str:
-    '''Encode text string to hex string for use in a PDF
+    """
+    Encode text string to hex string for use in a PDF
 
     From PDF 32000-1:2008 a string object may be included in hexademical form
     if it is enclosed in angle brackets.  For general Unicode the string should
@@ -86,7 +101,7 @@ def encode_text_string(s: str) -> str:
     ASCII strings could be encoded as PdfDocEncoding literals provided
     that certain Postscript sequences are escaped.  But it's far simpler to
     encode everything as UTF-16.
-    '''
+    """
 
     # Sometimes lazy C programmers leave their NULs at the end of strings they
     # insert into PDFs
@@ -102,10 +117,30 @@ def encode_text_string(s: str) -> str:
     return ascii_hex_str
 
 
-def encode_pdf_date(d: datetime) -> str:
-    """Encode Python datetime object as PDF date string
+def _encode_ascii(s: str) -> str:
+    """
+    Aggressively strip non-ASCII and PDF escape sequences
 
-    From Adobe pdfmark manual:    
+    Ghostscript 9.24+ lost support for UTF-16BE in pdfmark files for reasons
+    given in GhostPDL commit e997c683. Our temporary workaround is use ASCII
+    and drop all non-ASCII characters. A slightly improved alternative would
+    be to implement PdfDocEncoding in pikepdf and encode to that, or handle
+    metadata there.
+    """
+    trans = str.maketrans({
+        '(': '',
+        ')': '',
+        '\\': '',
+        '\0': ''
+    })
+    return s.translate(trans).encode('ascii', errors='replace').decode()
+
+
+def encode_pdf_date(d: datetime) -> str:
+    """
+    Encode Python datetime object as PDF date string
+
+    From Adobe pdfmark manual:
     (D:YYYYMMDDHHmmSSOHH'mm')
     D: is an optional prefix. YYYY is the year. All fields after the year are
     optional. MM is the month (01-12), DD is the day (01-31), HH is the
@@ -137,16 +172,23 @@ def encode_pdf_date(d: datetime) -> str:
 
 
 def decode_pdf_date(s: str) -> datetime:
+    """
+    Decode a pdfmark date to a Python datetime object
+
+    A pdfmark date is a string in a paritcular format. See the pdfmark
+    Reference for the specification.
+
+    """
     if s.startswith('D:'):
         s = s[2:]
 
-    # Literal Z00'00', is incorrect but found in the wild, 
+    # Literal Z00'00', is incorrect but found in the wild,
     # probably made by OS X Quartz -- standardize
     if s.endswith("Z00'00'"):
         s = s.replace("Z00'00'", '+0000')
     elif s.endswith('Z'):
         s = s.replace('Z', '+0000')
-    
+
     s = s.replace("'", "")  # Remove apos from PDF time strings
 
     return datetime.strptime(s, r'%Y%m%d%H%M%S%z')
@@ -154,7 +196,7 @@ def decode_pdf_date(s: str) -> datetime:
 
 def _get_pdfmark_dates(pdfmark):
     """Encode dates for pdfmark Postscript.  The best way to deal with a
-    missing date entry is set it to null, because if the key is omitted 
+    missing date entry is set it to null, because if the key is omitted
     Ghostscript will set it to now - we do not want to erase the fact that
     the value was unknown.  Setting to an empty string breaks Ghostscript
     9.22 as reported here:
@@ -172,30 +214,47 @@ def _get_pdfmark_dates(pdfmark):
             date_str = date_str[2:]
         try:
             yield '  {} (D:{})'.format(
-                    key, 
+                    key,
                     encode_pdf_date(decode_pdf_date(date_str)))
         except ValueError:
             yield '  {} null'.format(key)
 
 
-def _get_pdfa_def(icc_profile, icc_identifier, pdfmark):
-    """Create a Postscript file for Ghostscript.  pdfmark contains the various
-    objects as strings; these must be encoded in ASCII, and dates have a 
-    special format."""
+def _get_pdfa_def(icc_profile, icc_identifier, pdfmark, ascii_docinfo=False):
+    """Create a Postscript pdfmark file for Ghostscript.
+
+    pdfmark contains the various objects as strings; these must be encoded in
+    ASCII, and dates have a special format.
+
+    :param icc_profile: filename of the ICC profile to include in pdfmark
+    :param icc_identifier: ICC identifier such as 'sRGB'
+    :param pdfmark: a dictionary containing keys to include the pdfmark
+    :param ascii_docinfo: if True, the docinfo block must be encoded in pure
+        ASCII and may not contain UTF-16BE-BOM-hex encoded strings, as
+        required for Ghostscript 9.24+
+
+    :returns: a string containing the entire pdfmark
+
+    """
 
     # Ghostscript <= 9.21 has a bug where null entries in DOCINFO might produce
     # ERROR: VMerror (-25) on closing pdfwrite device.
     # https://bugs.ghostscript.com/show_bug.cgi?id=697684
     # Work around this by only adding keys that have a nontrivial value
     docinfo_keys = ('/Title', '/Author', '/Subject', '/Creator', '/Keywords')
-    docinfo_line_template = '  {key} <{value}>'
 
     def docinfo_gen():
+        if not ascii_docinfo:
+            docinfo_line_template = '  {key} <{value}>'
+            encode = encode_text_string
+        else:
+            docinfo_line_template = '  {key} ({value})'
+            encode = _encode_ascii
         yield from _get_pdfmark_dates(pdfmark)
         for key in docinfo_keys:
             if key in pdfmark and pdfmark[key].strip() != '':
                 line = docinfo_line_template.format(
-                    key=key, value=encode_text_string(pdfmark[key]))
+                    key=key, value=encode(pdfmark[key]))
                 yield line
     docinfo = '\n'.join(docinfo_gen())
 
@@ -206,13 +265,13 @@ def _get_pdfa_def(icc_profile, icc_identifier, pdfmark):
     return result
 
 
-def generate_pdfa_ps(target_filename, pdfmark, icc='sRGB'):
+def generate_pdfa_ps(target_filename, pdfmark, icc='sRGB', ascii_docinfo=False):
     if icc == 'sRGB':
         icc_profile = SRGB_ICC_PROFILE
     else:
         raise NotImplementedError("Only supporting sRGB")
 
-    ps = _get_pdfa_def(icc_profile, icc, pdfmark)
+    ps = _get_pdfa_def(icc_profile, icc, pdfmark, ascii_docinfo=ascii_docinfo)
 
     # We should have encoded everything to pure ASCII by this point, and
     # to be safe, only allow ASCII in PostScript
@@ -263,4 +322,3 @@ def file_claims_pdfa(filename):
     pdfa_dict['conformance'] = conformance
 
     return pdfa_dict
-

src/ocrmypdf/pipeline.py

@@ -19,6 +19,7 @@ from contextlib import suppress
 from shutil import copyfileobj
 from pathlib import Path
 from datetime import datetime, timezone
+from io import BytesIO
 import sys
 import os
 import shutil
@@ -144,6 +145,13 @@ def triage_image_file(input_file, output_file, log, options):
                 "image was scanned and specify it using --image-dpi.")
             raise DpiError()
 
+        if im.mode in ('RGBA', 'LA'):
+            log.error(
+                "The input image has an alpha channel. Remove the alpha "
+                "channel first."
+            )
+            raise UnsupportedImageFormatError()
+
         if 'iccprofile' not in im.info:
             if im.mode == 'RGB':
                 log.info('Input image has no ICC profile, assuming sRGB')
@@ -172,7 +180,7 @@ def triage_image_file(input_file, output_file, log, options):
 
 def _pdf_guess_version(input_file, search_window=1024):
     """Try to find version signature at start of file.
-    
+
     Not robust enough to deal with appended files.
 
     Returns empty string if not found, indicating file is probably not PDF.
@@ -231,7 +239,7 @@ def repair_and_parse_pdf(
             "output these files.)  Use --output-type=pdf instead."
         )
         raise InputFileError()
-    
+
     if len(pdfinfo.pages) > 2000 and sys.version_info[0:2] <= (3, 5):
         log.warning(
             "Performance regressions are known occur with Python 3.5 for "
@@ -298,7 +306,7 @@ def is_ocr_required(pageinfo, log, options):
         # We found a page with no images and no text. That means it may
         # have vector art that the user wants to OCR. If we determined
         # lossless reconstruction is not possible then we have to rasterize
-        # the image. So if OCR is being forced, take that to mean YES, go 
+        # the image. So if OCR is being forced, take that to mean YES, go
         # ahead and rasterize. If not forced, then pretend there's no text
         # on the page at all so we don't lose anything.
         # This could be made smarter by explicitly searching for vector art.
@@ -496,17 +504,23 @@ def rasterize_with_ghostscript(
     options = context.get_options()
     pageinfo = get_pageinfo(input_file, context)
 
-    device = 'png16m'  # 24-bit
-    if pageinfo.images:
-        if all(image.comp == 1 for image in pageinfo.images):
-            if all(image.bpc == 1 for image in pageinfo.images):
-                device = 'pngmono'
-            elif all(image.bpc > 1 and image.color == Colorspace.index
-                     for image in pageinfo.images):
-                device = 'png256'
-            elif all(image.bpc > 1 and image.color == Colorspace.gray
-                     for image in pageinfo.images):
-                device = 'pnggray'
+    colorspaces = ['pngmono', 'pnggray', 'png256', 'png16m']
+    device_idx = 0
+    def at_least(cs):
+        return max(device_idx, colorspaces.index(cs))
+
+    for image in pageinfo.images:
+        if image.type_ != 'image':
+            continue  # ignore masks
+        if image.bpc > 1:
+            if image.color == Colorspace.index:
+                device_idx = at_least('png256')
+            elif image.color == Colorspace.gray:
+                device_idx = at_least('pnggray')
+            else:
+                device_idx = at_least('png16m')
+
+    device = colorspaces[device_idx]
 
     log.debug("Rasterize {0} with {1}".format(
               os.path.basename(input_file), device))
@@ -878,7 +892,7 @@ def get_pdfmark(base_pdf, options):
         except (KeyError, TypeError):
             return ''
 
-    pdfmark = {k: from_document_info(k) for k in 
+    pdfmark = {k: from_document_info(k) for k in
         ('/Title', '/Author', '/Keywords', '/Subject', '/CreationDate')}
     if options.title:
         pdfmark['/Title'] = options.title
@@ -912,7 +926,20 @@ def generate_postscript_stub(
     options = context.get_options()
     pdf = pypdf.PdfFileReader(input_file)
     pdfmark = get_pdfmark(pdf, options)
-    generate_pdfa_ps(output_file, pdfmark)
+
+    ascii_docinfo = False
+    if ghostscript.version() >= '9.24':
+        ascii_docinfo = True
+        try:
+            for v in pdfmark.values():
+                v.encode('ascii', errors='strict')
+        except UnicodeEncodeError:
+            log.warning(
+                "Ghostscript 9.24+ does not support Unicode strings in "
+                " metadata. These will be converted to ASCII if possible."
+            )
+
+    generate_pdfa_ps(output_file, pdfmark, ascii_docinfo=ascii_docinfo)
 
 
 def skip_page(
@@ -1069,7 +1096,7 @@ def merge_sidecars(
             if txt_file:
                 with open(txt_file, 'r', encoding="utf-8") as in_:
                     txt = in_.read()
-                    # Tesseract v4 alpha started adding form feeds in 
+                    # Tesseract v4 alpha started adding form feeds in
                     # commit aa6eb6b
                     # No obvious way to detect what binaries will do this, so
                     # for consistency just ignore its form feeds and insert our
@@ -1103,7 +1130,7 @@ def copy_final(
             sys.stdout.flush()
         else:
             # At this point we overwrite the output_file specified by the user
-            # use copyfileobj because then we use open() to create the file and 
+            # use copyfileobj because then we use open() to create the file and
             # get the appropriate umask, ownership, etc.
             with open(output_file, 'wb') as output_stream:
                 copyfileobj(input_stream, output_stream)
@@ -1278,7 +1305,7 @@ def build_pipeline(options, work_folder, log, context):
         output=os.path.join(work_folder, r'\1.rendered.pdf'),
         extras=[log, context])
     task_combine_layers.graphviz(fillcolor='"#00cc66"')
-    task_combine_layers.active_if(options.pdf_renderer == 'hocr' or 
+    task_combine_layers.active_if(options.pdf_renderer == 'hocr' or
                                   options.pdf_renderer == 'sandwich')
 
     # Tesseract OCR+PDF

tests/spoof/tesseract_cache.py

@@ -106,6 +106,8 @@ def main():
     source = os.environ['_OCRMYPDF_TEST_INFILE']  # required
     args = parser.parse_args()
 
+    cache_disabled = os.environ.get('_OCRMYPDF_CACHE_DISABLED', False)
+
     if args.imagename == 'stdin':
         real_tesseract()
 
@@ -128,7 +130,7 @@ def main():
     print("Tesseract cache folder {} - ".format(cache_folder), end='', 
           file=sys.stderr)
 
-    if (cache_folder / 'stderr.bin').exists():
+    if (cache_folder / 'stderr.bin').exists() and not cache_disabled:
         # Cache hit
         print("HIT", file=sys.stderr)
 

tests/test_main.py

@@ -300,7 +300,8 @@ def test_autorotate_threshold(
 @pytest.mark.parametrize('renderer',RENDERERS)
 def test_ocr_timeout(renderer, resources, outpdf):
     out = check_ocrmypdf(resources / 'skew.pdf', outpdf,
-                         '--tesseract-timeout', '1.0')
+                         '--tesseract-timeout', '0.01',
+                         '--pdf-renderer', renderer)
     pdfinfo = PdfInfo(out)
     assert not pdfinfo[0].has_text
 
@@ -472,7 +473,7 @@ def test_tesseract_image_too_big(renderer, spoof_tesseract_big_image_error,
                                  resources, outpdf):
     check_ocrmypdf(
         resources / 'hugemono.pdf', outpdf, '-r',
-        '--pdf-renderer', renderer, 
+        '--pdf-renderer', renderer,
         '--max-image-mpixels', '0',
         env=spoof_tesseract_big_image_error)
 
@@ -828,6 +829,7 @@ def test_no_contents(spoof_tesseract_noop, resources, outpdf):
 @pytest.mark.parametrize('image', [
     'baiona.png',
     'baiona_gray.png',
+    'baiona_alpha.png',
     'congress.jpg'
     ])
 def test_compression_preserved(spoof_tesseract_noop, ocrmypdf_exec,
@@ -838,7 +840,6 @@ def test_compression_preserved(spoof_tesseract_noop, ocrmypdf_exec,
     output_file = str(outpdf)
 
     im = Image.open(input_file)
-
     # Runs: ocrmypdf - output.pdf < testfile
     with open(input_file, 'rb') as input_stream:
         p_args = ocrmypdf_exec + [
@@ -848,7 +849,12 @@ def test_compression_preserved(spoof_tesseract_noop, ocrmypdf_exec,
             stdin=input_stream, env=spoof_tesseract_noop)
         out, err = p.communicate()
 
-        assert p.returncode == ExitCode.ok
+        if im.mode in ('RGBA', 'LA'):
+            # If alpha image is input, expect an error
+            assert p.returncode != ExitCode.ok and b'alpha' in err
+            return
+
+        assert p.returncode == ExitCode.ok, err.decode('utf-8')
 
     pdfinfo = PdfInfo(output_file)
 
@@ -893,7 +899,7 @@ def test_compression_changed(spoof_tesseract_noop, ocrmypdf_exec,
             stdin=input_stream, env=spoof_tesseract_noop)
         out, err = p.communicate()
 
-        assert p.returncode == ExitCode.ok
+        assert p.returncode == ExitCode.ok, err
 
     pdfinfo = PdfInfo(output_file)
 
@@ -902,13 +908,7 @@ def test_compression_changed(spoof_tesseract_noop, ocrmypdf_exec,
     if compression == "jpeg":
         assert pdfimage.enc == Encoding.jpeg
     else:
-        if ghostscript.jpeg_passthrough_available():
-            # Ghostscript 9.23 adds JPEG passthrough, which allows a JPEG to be
-            # copied without transcoding - so report
-            if image.endswith('jpg'):
-                assert pdfimage.enc == Encoding.jpeg
-        else:
-            assert pdfimage.enc not in (Encoding.jpeg, Encoding.jpeg2000)
+        assert pdfimage.enc not in (Encoding.jpeg, Encoding.jpeg2000)
 
     if im.mode.startswith('RGB') or im.mode.startswith('BGR'):
         assert pdfimage.color == Colorspace.rgb, \
@@ -966,6 +966,7 @@ def test_pdfa_n(spoof_tesseract_cache, pdfa_level, resources, outpdf):
     assert pdfa_info['conformance'] == 'PDF/A-{}B'.format(pdfa_level)
 
 
+@pytest.mark.skipif(sys.version_info >= (3, 7, 0), reason='fixed')
 def test_bad_locale():
     env = os.environ.copy()
     env['LC_ALL'] = 'C'
@@ -993,7 +994,7 @@ def test_bad_utf8(spoof_tess_bad_utf8, renderer, resources, no_outpdf):
 
 
 @pytest.mark.skipif(
-    not tesseract.has_textonly_pdf(), 
+    not tesseract.has_textonly_pdf(),
     reason="issue only affects sandwich")
 def test_rotate_deskew_timeout(resources, outdir):
     check_ocrmypdf(
@@ -1043,10 +1044,10 @@ def test_text_curves(spoof_tesseract_noop, resources, outpdf):
     check_ocrmypdf(
         resources / 'vector.pdf', outpdf, '--force-ocr',
         env=spoof_tesseract_noop)
-    
+
     info = PdfInfo(outpdf)
     assert len(info.pages[0].images) != 0, "force did not rasterize"
-    
+
 
 def test_dev_null(spoof_tesseract_noop, resources):
     p, out, err = run_ocrmypdf(

tests/test_metadata.py

@@ -24,6 +24,12 @@ from datetime import timezone
 from ocrmypdf.pdfa import file_claims_pdfa, encode_pdf_date, decode_pdf_date
 from ocrmypdf.exceptions import ExitCode
 from ocrmypdf.lib import fitz
+from ocrmypdf.helpers import fspath
+from ocrmypdf.pdfa import (
+    file_claims_pdfa, encode_pdf_date, decode_pdf_date, generate_pdfa_ps,
+    SRGB_ICC_PROFILE
+)
+from ocrmypdf.exec import ghostscript
 
 # pytest.helpers is dynamic
 # pylint: disable=no-member
@@ -76,6 +82,9 @@ def test_override_metadata(spoof_tesseract_noop, output_type, resources,
     before = pypdf.PdfFileReader(str(input_file))
     after = pypdf.PdfFileReader(outpdf)
 
+    if ghostscript.version() >= '9.24':
+        pytest.xfail('Ghostscript 9.24+ does not support Unicode DOCINFO')
+
     assert after.documentInfo['/Title'] == german
     assert after.documentInfo['/Author'] == chinese
     assert after.documentInfo.get('/Keywords', '') == ''
@@ -136,7 +145,7 @@ def test_creation_date_preserved(spoof_tesseract_noop, output_type, resources,
 
     before = pypdf.PdfFileReader(str(input_file)).getDocumentInfo()
     check_ocrmypdf(
-        input_file, outpdf, '--output-type', output_type, 
+        input_file, outpdf, '--output-type', output_type,
         env=spoof_tesseract_noop)
     after = pypdf.PdfFileReader(str(outpdf)).getDocumentInfo()
 
@@ -157,5 +166,3 @@ def test_creation_date_preserved(spoof_tesseract_noop, output_type, resources,
     date_after = decode_pdf_date(after['/ModDate'])
     assert seconds_between_dates(
         date_after, datetime.datetime.now(timezone.utc)) < 1000
-
-