Update setup.py for version changes

Decided on this for simplicity of old release branch.

Modifies baiona.png by stripping
alpha, adds baiona_alpha which
includes the alpha.

.docker/Dockerfile

@@ -3,28 +3,49 @@
 FROM      ubuntu:18.04
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
+  build-essential autoconf automake libtool \
+  libleptonica-dev \
+  zlib1g-dev \
+  libexempi3 \
   ocrmypdf \
-  unpaper \
+  pngquant \
+  python3-pip \
+  python3-venv \
+  tesseract-ocr \
+  tesseract-ocr-chi-sim \
+  tesseract-ocr-deu \
   tesseract-ocr-eng \
   tesseract-ocr-fra \
-  tesseract-ocr-deu \
-  tesseract-ocr-spa \
   tesseract-ocr-por \
-  tesseract-ocr-chi-sim \
-  python3-venv \
-  python3-pip
+  tesseract-ocr-spa \
+  unpaper \
+  wget
+
 
 ENV LANG=C.UTF-8
 
+# Compile and install jbig2
+# Needs libleptonica-dev, zlib1g-dev
+RUN \
+  mkdir jbig2 \
+  && wget -q https://github.com/agl/jbig2enc/archive/0.29.tar.gz -O - | \
+      tar xz -C jbig2 --strip-components=1 \
+  && cd jbig2 \
+  && ./autogen.sh && ./configure && make && make install \
+  && cd .. \
+  && rm -rf jbig2
+
+RUN apt-get remove -y autoconf automake libtool
+
 RUN python3 -m venv --system-site-packages /appenv
 
 # This installs the latest binary wheel instead of the code in the current
 # folder. Installing from source will fail, apparently because cffi needs
-# build-essentials (gcc) to do a source installation 
+# build-essentials (gcc) to do a source installation
 # (i.e. "pip install ."). It's unclear to me why this is the case.
 RUN . /appenv/bin/activate; \
   pip install --upgrade pip \
-  && pip install ocrmypdf[fitz]
+  && pip install --upgrade ocrmypdf
 
 # Now copy the application in, mainly to get the test suite.
 # Do this now to make the best use of Docker cache.
@@ -35,6 +56,7 @@ RUN . /appenv/bin/activate; \
 # Remove the junk, including the source version of application since it was
 # already installed
 RUN rm -rf /tmp/* /var/tmp/* /root/* /application/ocrmypdf \
+  && apt-get remove -y build-essential \
   && apt-get autoremove -y \
   && apt-get autoclean -y
 

.travis.yml

@@ -27,10 +27,6 @@ addons:
     - tesseract-ocr-eng
     - tesseract-ocr-fra
 
-env:
-  global:
-    - secure: "hsf6MT+n2x3OiDM2fQyJZdV0/PWYmv81LdVqC6cfnHBE/8N3DloJRqQ7WfO14TxhiK9PEC7MpyCj0lSabUHEO7gSH6Vks6I1asoSkt8S9/bSMlhT4hei+pwVpeGEiU5xHVATNjY+D919VC3IFvc3XmjT74h/2SLhaZ+jhEmDggM="  # HOMEBREW_OCRMYPDF_TOKEN
-
 matrix:
   include:
     - os: linux
@@ -94,18 +90,3 @@ deploy:
     tags: true
     condition: $TRAVIS_PYTHON_VERSION == "3.6" && $TRAVIS_OS_NAME == "linux"
   skip_upload_docs: true
-
-  # null deploy for osx
-  # we really just want to run after_deploy *after* pypi upload is done, but
-  # after_deploy on runs if a given box deployed
-- provider: script
-  script: /usr/bin/true
-  on:
-    branch: master
-    tags: true
-    condition: $TRAVIS_OS_NAME == "osx"
-
-after_deploy: |
-  if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-    bash .travis/osx_brew.sh
-  fi

.travis/Brewfile

@@ -2,10 +2,8 @@ brew 'python'
 
 brew 'exempi'
 brew 'ghostscript'
-brew 'jbig2dec'
 brew 'jbig2enc'
 brew 'leptonica'
-brew 'libffi'
 brew 'libtiff'
 brew 'openjpeg'
 brew 'pngquant'

.travis/autobrew.py

@@ -1,93 +0,0 @@
-#!/usr/bin/env python3
-# © 2017-18 James R. Barlow: github.com/jbarlow83
-
-from string import Template
-from subprocess import run, PIPE
-import re
-
-recipe_template = Template("""
-class Ocrmypdf < Formula
-  include Language::Python::Virtualenv
-
-  desc "Adds an OCR text layer to scanned PDF files"
-  homepage "https://github.com/jbarlow83/OCRmyPDF"
-  ${ocrmypdf_url}
-  ${ocrmypdf_sha256}
-
-  depends_on "pkg-config" => :build
-  depends_on "mupdf-tools" => :build  # statically links libmupdf.a
-  depends_on "freetype"
-  depends_on "ghostscript"
-  depends_on "jpeg"
-  depends_on "libpng"
-  depends_on "python"
-  depends_on "qpdf"
-  depends_on "tesseract"
-  depends_on "unpaper"
-
-${resources}
-  def install
-    venv = virtualenv_create(libexec, "python3")
-
-    resource("Pillow").stage do
-      inreplace "setup.py" do |s|
-        sdkprefix = MacOS::CLT.installed? ? "" : MacOS.sdk_path
-        s.gsub! "openjpeg.h", "probably_not_a_header_called_this_eh.h"
-        s.gsub! "ZLIB_ROOT = None", "ZLIB_ROOT = ('#{sdkprefix}/usr/lib', '#{sdkprefix}/usr/include')"
-        s.gsub! "JPEG_ROOT = None", "JPEG_ROOT = ('#{Formula["jpeg"].opt_prefix}/lib', '#{Formula["jpeg"].opt_prefix}/include')"
-        s.gsub! "FREETYPE_ROOT = None", "FREETYPE_ROOT = ('#{Formula["freetype"].opt_prefix}/lib', '#{Formula["freetype"].opt_prefix}/include')"
-      end
-
-      # avoid triggering "helpful" distutils code that doesn't recognize Xcode 7 .tbd stubs
-      ENV.append "CFLAGS", "-I#{MacOS.sdk_path}/System/Library/Frameworks/Tk.framework/Versions/8.5/Headers" unless MacOS::CLT.installed?
-      venv.pip_install Pathname.pwd
-    end
-
-    res = resources.map(&:name).to_set - ["Pillow"]
-
-    res.each do |r|
-      venv.pip_install resource(r)
-    end
-
-    venv.pip_install_and_link buildpath
-  end
-
-  test do
-    # Since we use Python 3, we require a UTF-8 locale
-    ENV["LC_ALL"] = "en_US.UTF-8"
-
-    system "#{bin}/ocrmypdf", "-f", "-q", "--deskew",
-                              test_fixtures("test.pdf"), "ocr.pdf"
-    assert_predicate testpath/"ocr.pdf", :exist?
-  end
-end
-""")
-
-def main():
-    p = run(['poet', '--single', 'ocrmypdf'],
-            encoding='utf-8', stdout=PIPE, check=True)
-
-    ocrmypdf_lines = p.stdout.splitlines()
-    ocrmypdf_url = ocrmypdf_lines[1].strip()
-    ocrmypdf_sha256 = ocrmypdf_lines[2].strip()
-
-    ocrmypdf_version = re.search(
-        r'ocrmypdf-(.+)\.tar.*', ocrmypdf_url).group(1)
-    print(f"Autobrewing {ocrmypdf_version}")
-
-    p = run(['poet', '--resources', 'ocrmypdf'],
-            encoding='utf-8', stdout=PIPE, check=True)
-
-    poet_resources = p.stdout
-
-    # Remove the duplicate "ocrmypdf" resource block
-    all_resources = poet_resources.split('resource')
-    kept_resources = [block for block in all_resources if 'ocrmypdf' not in block]
-    resources = 'resource'.join(kept_resources)
-
-    with open('ocrmypdf.rb', 'w') as out:
-        out.write(recipe_template.substitute(**locals()))
-
-
-if __name__ == '__main__':
-    main()

.travis/osx_brew.sh

@@ -1,23 +0,0 @@
-#!/bin/bash
-# © 2017 James R. Barlow: github.com/jbarlow83
-set -uo pipefail
-set -x
-
-pip3 install homebrew-pypi-poet
-python3 .travis/autobrew.py
-cat ocrmypdf.rb
-
-# brew audit crashes Travis
-#brew audit ocrmypdf.rb
-
-# Important: disable debug output so token is hidden
-set +x
-git clone https://$HOMEBREW_OCRMYPDF_TOKEN@github.com/jbarlow83/homebrew-ocrmypdf.git
-set -x
-
-pushd homebrew-ocrmypdf
-cp ../ocrmypdf.rb Formula/ocrmypdf.rb
-git add Formula/ocrmypdf.rb
-git commit -m "homebrew-ocrmypdf: automatic release $TRAVIS_BUILD_NUMBER $TRAVIS_TAG"
-git push origin master
-popd

docs/conf.py

@@ -60,6 +60,24 @@ author = 'James R. Barlow'
 # built documents.
 #
 # The short X.Y version.
+
+import os
+on_rtd = os.environ.get('READTHEDOCS') == 'True'
+
+if on_rtd:
+    # Help ReadTheDocs avoid having to install any binary extension modules
+    import sys
+    from unittest.mock import MagicMock
+
+    class Mock(MagicMock):
+        @classmethod
+        def __getattr__(cls, name):
+            return MagicMock()
+
+    MOCK_MODULES = ['pikepdf', 'libxmp', 'libxmp.utils']
+    sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES)
+
+
 from ocrmypdf import __version__ as OCRMYPDF_VERSION
 
 _version_parts = OCRMYPDF_VERSION.split('.')

docs/installation.rst

@@ -1,6 +1,11 @@
 Installation
 ============
 
+.. |latest| image:: https://img.shields.io/pypi/v/ocrmypdf.svg
+    :alt: OCRmyPDF latest released version on PyPI
+
+|latest|
+
 The easiest way to install OCRmyPDF to follow the steps for your operating system/platform.
 
 If you want to use the latest version of OCRmyPDF, your best bet is to install the most recent version your platform provides, and then upgrade that version by installing the Python binary wheels.
@@ -12,13 +17,49 @@ If you want to use the latest version of OCRmyPDF, your best bet is to install t
 Installing on Debian and Ubuntu 16.10 or newer
 ----------------------------------------------
 
+.. |deb-stable| image:: https://repology.org/badge/version-for-repo/debian_stable/ocrmypdf.svg
+    :alt: Debian 9 stable ("stretch")
+
+.. |deb-testing| image:: https://repology.org/badge/version-for-repo/debian_testing/ocrmypdf.svg
+    :alt: Debian 10 testing ("buster")
+
+.. |deb-unstable| image:: https://repology.org/badge/version-for-repo/debian_unstable/ocrmypdf.svg
+    :alt: Debian unstable
+
+.. |ubu-1710| image:: https://repology.org/badge/version-for-repo/ubuntu_17_10/ocrmypdf.svg
+    :alt: Ubuntu 17.10
+
+.. |ubu-1804| image:: https://repology.org/badge/version-for-repo/ubuntu_18_04/ocrmypdf.svg
+    :alt: Ubuntu 18.04 LTS
+
+.. |ubu-1810| image:: https://repology.org/badge/version-for-repo/ubuntu_18_10/ocrmypdf.svg
+    :alt: Ubuntu 18.10
+
+
++-------------------------------------------+
+| **OCRmyPDF version**                      |
++-------------------------------------------+
+| |latest|                                  |
++-------------------------------------------+
+| |deb-stable| |deb-testing| |deb-unstable| |
++-------------------------------------------+
+| |ubu-1710| |ubu-1804| |ubu-1810|          |
++-------------------------------------------+
+
 Users of Debian 9 ("stretch") or later or Ubuntu 16.10 or later may simply
 
 .. code-block:: bash
 
     apt-get install ocrmypdf
 
-To see what versions are available, check the `Debian Package Tracker <https://tracker.debian.org/pkg/ocrmypdf>`_ or `Ubuntu launchpad.net <https://launchpad.net/ocrmypdf>`_.
+For full details on version availability, check the `Debian Package Tracker <https://tracker.debian.org/pkg/ocrmypdf>`_ or `Ubuntu launchpad.net <https://launchpad.net/ocrmypdf>`_.
+
+If the version available for your platform is out of date, you could opt to install the latest version from source. See `Installing HEAD revision from sources`_.
+
+.. note::
+
+    OCRmyPDF for Debian and Ubuntu currently omit the JBIG2 encoder. OCRmyPDF works fine without it but will produce larger output files. If you build jbig2enc from source, ocrmypdf 7.0.0 and later will automatically detect it on the ``PATH``. See `Optional: installing the JBIG2 encoder`_.
+
 
 Installing on macOS with Homebrew
 ---------------------------------
@@ -41,12 +82,12 @@ OCRmyPDF is now a standard `Homebrew <https://brew.sh>`_ formula. To install on
 
     Users who previously installed OCRmyPDF from the private tap should switch to the mainline version (``brew untap jbarlow83/ocrmypdf``) and install from there.
 
-.. _Docker-install:
+.. _docker-install:
 
 Installing the Docker image
 ---------------------------
 
-For many users, installing the Docker image will be easier than installing all of OCRmyPDF's dependencies. For Windows, it is the only option.
+For some users, installing the Docker image will be easier than installing all of OCRmyPDF's dependencies. For Windows, it is the only option.
 
 If you have `Docker <https://docs.docker.com/>`_ installed on your system, you can install a Docker image of the latest release.
 
@@ -142,7 +183,7 @@ Or in the wonderful `fish shell <https://fishshell.com/>`_:
 
 .. note::
 
-    The ocrmypdf Docker containers are designed to be used for a single OCR job. The ``docker run --rm`` argument tells Docker to delete temporary storage associated with container when it is done executing.
+    The ocrmypdf Docker images are designed for application delivery, to enable use of OCRmyPDF without fussing with dependencies. ``docker run --rm`` argument tells Docker to delete the container after it runs, because each container is only good for a single job. The Docker image is not designed for use as a persistent web service or for use on Amazon EC2 Container Service (AWS ECS).
 
 Manual installation on macOS
 ----------------------------
@@ -157,24 +198,14 @@ Update Homebrew:
 
     brew update
 
-Install or upgrade the required Homebrew packages, if any are missing:
+Install or upgrade the required Homebrew packages, if any are missing. To do this, download the ``Brewfile`` that lists all of the dependencies to the current directory, and run ``brew bundle`` to process them (installing or upgrading as needed). ``Brewfile`` is a plain text file.
 
 .. code-block:: bash
 
-    brew install libpng openjpeg jbig2dec libtiff     # image libraries
-    brew install qpdf
-    brew install ghostscript
-    brew install python3
-    brew install libxml2 libffi leptonica
-    brew install unpaper   # optional
+    wget https://github.com/jbarlow83/OCRmyPDF/raw/master/.travis/Brewfile
+    brew bundle
 
-Python 3.5, 3.6 and 3.7 are supported.
-
-Install the required Tesseract OCR engine with the language packs you plan to use:
-
-.. code-block:: bash
-
-    brew install tesseract                       # Option 1: for English, French, German, Spanish
+This will include the English, French, German and Spanish language packs. If you need other languages you can optionally install them all:
 
 .. _macos-all-languages:
 
@@ -210,7 +241,7 @@ Installing the latest version on Ubuntu 18.04 LTS
 -------------------------------------------------
 
 Ubuntu 18.04 includes ocrmypdf 6.1.2. To install a more recent version, first
-install the system version to get all the dependencies:
+install the system version to get most of the dependencies:
 
 .. code-block:: bash
 
@@ -219,6 +250,15 @@ install the system version to get all the dependencies:
         ocrmypdf \
         python3-pip
 
+There are a few dependency changes between ocrmypdf 6.1.2 and 7.x. Let's get
+these, too.
+
+.. code-block:: bash
+
+    sudo apt-get install \
+        libexempi3 \
+        pngquant
+
 Then install the most recent ocrmypdf for the local user and set the user's ``PATH`` to check for the user's Python packages.
 
 .. code-block:: bash
@@ -226,6 +266,7 @@ Then install the most recent ocrmypdf for the local user and set the user's ``PA
     export PATH=$HOME/.local/bin:$PATH
     pip3 install --user ocrmypdf
 
+To add JBIG2 encoding, see `Optional: installing the JBIG2 encoder`_.
 
 Installing on Ubuntu 16.04 LTS
 ------------------------------
@@ -236,32 +277,39 @@ No package is currently available for Ubuntu 16.04, but you can install the depe
 
     sudo apt-get update
     sudo apt-get install \
-        unpaper \
         ghostscript \
-        tesseract-ocr \
-        qpdf \
+        libexempi3 \
+        pngquant \
+        python3-cffi \
         python3-pip \
-        python3-cffi
+        qpdf \
+        tesseract-ocr \
+        unpaper
 
-If you wish install OCRmyPDF for the current user:
+If you wish install OCRmyPDF for the current user, and ensure that the ``PATH``
+environment variable contains ``$HOME/.local/bin``.
 
 .. code-block:: bash
 
+    export PATH=$HOME/.local/bin:$PATH
     pip3 install --user ocrmypdf
 
-Alternately, system-wide. Note that this may modify the system Python environment:
+Alternately, you can install ocrmypdf system-wide. (Not recommended.)
 
 .. code-block:: bash
 
     sudo pip3 install ocrmypdf
 
-If you wish to install OCRmyPDF to a virtual environment to isolate the system Python, you can follow these steps.
+At your option, you may upgrade Ubuntu 16.04 LTS to Tesseract 4.0 for improved OCR results.
 
 .. code-block:: bash
 
-    python3 -m venv venv-ocrmypdf
-    source venv-ocrmypdf/bin/activate
-    pip3 install ocrmypdf
+    sudo apt-get install -y software-properties-common python-software-properties
+    sudo add-apt-repository ppa:alex-p/tesseract-ocr -y
+    sudo apt-get update
+    sudo apt-get upgrade tesseract-ocr
+
+To add JBIG2 encoding, see `Optional: installing the JBIG2 encoder`_.
 
 Installing on Ubuntu 14.04 LTS
 ------------------------------
@@ -281,11 +329,13 @@ Install system dependencies:
     sudo apt-get install \
         software-properties-common python-software-properties \
         zlib1g-dev \
+        libexempi3 \
         libjpeg-dev \
         libffi-dev \
+        pngquant \
         qpdf
 
-We will need backports of Ghostscript 9.16, libav-11 (for unpaper 6.1), Tesseract 4.00 (alpha), and Python 3.6. This will replace Ghostscript and Tesseract 3.x on your system. Python 3.6 will be installed alongside the system Python 3.
+We will need backports of Ghostscript 9.16, libav-11 (for unpaper 6.1), Tesseract 4.00 (alpha), and Python 3.6. This will replace Ghostscript and Tesseract 3.x on your system. Python 3.6 will be installed alongside the system Python 3.4.
 
 If you prefer to not modify your system in this matter, consider using a Docker container.
 
@@ -322,6 +372,8 @@ These installation instructions omit the optional dependency ``unpaper``, which
     wget -q 'https://www.dropbox.com/s/vaq0kbwi6e6au80/unpaper_6.1-1.deb?raw=1' -O unpaper_6.1-1.deb
     sudo dpkg -i unpaper_6.1-1.deb
 
+To add JBIG2 encoding, see `Optional: installing the JBIG2 encoder`_.
+
 
 Installing on ArchLinux
 -----------------------
@@ -336,7 +388,7 @@ The author is aware of an `ArchLinux package for ocrmypdf <https://aur.archlinux
 Installing on Windows
 ---------------------
 
-Direct installation on Windows is not possible.  Install the _`Docker` container as described above.  Ensure that your command prompt can run the docker "hello world" container.
+Direct installation on Windows is not possible.  `Install the Docker <docker-install_>`_ container as described above.  Ensure that your command prompt can run the docker "hello world" container.
 
 It would probably not be too difficult to run on Windows.  The main reason this has been avoided is the difficulty of packaging and installing the various non-Python dependencies: Tesseract, QPDF, Ghostscript, Leptonica.  Pull requests to add or improve Windows support would be quite welcome.
 
@@ -377,27 +429,36 @@ Since ``pip3 install --user`` does not work correctly on some platforms, notably
 Requirements for pip and HEAD install
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-OCRmyPDF currently requires these external programs to be installed:
+OCRmyPDF currently requires these external programs and libraries to be installed:
 
 - Python 3.5 or newer
-- Tesseract 3.04 or newer
 - Ghostscript 9.15 or newer
+- libexempi3 2.2.0 or newer
 - qpdf 7.0.0 or newer
+- Tesseract 3.04 or newer
 
-The following dependencies are recommended:
+As of ocrmypdf 7.0.0, the following versions are recommended:
 
-- Python 3.6
-- Tesseract 4.00 or newer
-- Ghostscript 9.22 or newer
-- qpdf 8.0.2 or newer
+- Python 3.7
+- Ghostscript 9.23
+- libexempi3 2.4.5
+- qpdf 8.1.0
+- Tesseract 4.0.0-beta1
+- jbig2enc 0.29 or newer
+- pngquant 2.5 or newer
 - unpaper 6.1
-- PyMuPDF 1.12.5 or newer
+
+jbig2enc, pngquant, and unpaper are optional. If missing certain features are disabled. OCRmyPDF will discover them as soon as they are available.
+
+**jbig2enc**, if present, will be used to optimize the encoding of monochrome images.  This can significantly reduce the file size of the output file.  It is not required.  `jbig2enc <https://github.com/agl/jbig2enc>`_ is not generally available for Ubuntu or Debian due to lingering concerns about patent issues, but can easily be built from source. To add JBIG2 encoding, see `Optional: installing the JBIG2 encoder`_.
+
+**pngquant**, if present, is optionally used to optimize the encoding of PNG-style images in PDFs (actually, any that are that losslessly encoded) by lossily quantizing to a smaller color palette. It is only activated then the ``--optimize`` argument is ``2`` or ``3``.
+
+**unpaper**, if present, enables the ``--clean`` and ``--clean-final`` command line options.
 
 These are in addition to the Python packaging dependencies, meaning that unfortunately, the ``pip install`` command cannot satisfy all of them.
 
-Python 3.6 and Tesseract 4.0.0-beta.1 are recommended for best OCR results and best performance.
-
-The library PyMuPDF is not widely available in platform distributions, and it improves OCRmyPDF in certain conditions. Consider installing OCRmyPDF from the Python binary wheels, which include a precompiled version of this library.
+Python 3.7 and Tesseract 4.0.0-beta.1 are recommended for best OCR results and best performance.
 
 
 Installing HEAD revision from sources
@@ -441,8 +502,27 @@ need to be installed. The script requires specific versions of the
 dependencies. Older version than the ones mentioned in the release notes
 are likely not to be compatible to OCRmyPDF.
 
+To add JBIG2 encoding, see `Optional: installing the JBIG2 encoder`_.
 
 Other Linux packages
 --------------------
 
 See the `Repology <https://repology.org/metapackage/ocrmypdf/versions>`_ page.
+
+
+Optional: installing the JBIG2 encoder
+--------------------------------------
+
+Most Linux distributions do not include a JBIG2 encoder since JBIG2 encoding was patented for a long time. All known JBIG2 US patents have expired as of 2017, but it is possible that unknown patents exist.
+
+To build a JBIG2 encoder from source:
+
+.. code-block:: bash
+
+    git clone https://github.com/agl/jbig2enc
+    cd jbig2enc
+    ./autogen.sh
+    ./configure && make
+    [sudo] make install
+
+On macOS, Homebrew packages jbig2enc and OCRmyPDF includes it by default.

docs/introduction.rst

@@ -1,9 +1,7 @@
 Introduction
 ============
-
 OCRmyPDF is a Python 3 package that adds OCR layers to PDFs.
 
-
 About OCR
 ---------
 
@@ -66,6 +64,8 @@ In the case of a PDF that is nothing other than a container of images (no rotati
 
 OCRmyPDF uses several strategies depending on input options and the input PDF itself, but generally speaking it rasterizes a page for OCR and then grafts the OCR back onto the original. As such it can handle complex PDFs and still preserve their contents as much as possible.
 
+OCRmyPDF also supports a many, many edge cases that have cropped over several years of development. We support PDF features like images inside of Form XObjects, and pages with UserUnit scaling. We support rare image formats like non-monochrome 1-bit images. Thanks to pikepdf and QPDF, we auto-repair PDFs that are damaged. (Not that you need to know what any of these are! You should be able to throw any PDF at it.)
+
 
 Limitations
 -----------
@@ -78,7 +78,6 @@ OCRmyPDF is limited by the Tesseract OCR engine.  As such it experiences these l
 * If a document contains languages outside of those given in the ``-l LANG`` arguments, results may be poor.
 * It is not always good at analyzing the natural reading order of documents. For example, it may fail to recognize that a document contains two columns and join text across the columns.
 * Poor quality scans may produce poor quality OCR. Garbage in, garbage out.
-* PDFs that use transparent layers are not currently checked in the test suite, so they may not work correctly.
 * It does not expose information about what font family text belongs to.
 
 OCRmyPDF is also limited by the PDF specification:
@@ -91,9 +90,11 @@ Ghostscript also imposes some limitations:
 * PDFs containing JBIG2-encoded content will be converted to CCITT Group4 encoding, which has lower compression ratios, if Ghostscript PDF/A is enabled.
 * PDFs containing JPEG 2000-encoded content will be converted to JPEG encoding, which may introduce compression artifacts, if Ghostscript PDF/A is enabled.
 * Ghostscript may transcode grayscale and color images, either lossy to lossless or lossless to lossy, based on an internal algorithm. This behavior can be suppressed by setting ``--pdfa-image-compression`` to ``jpeg`` or ``lossless`` to set all images to one type or the other. Ghostscript has no option to maintain the input image's format.
-  
-OCRmyPDF is currently not designed to be used as a Python API; it is designed to be run as a command line tool. ``import ocrmypf`` currently attempts to process the command line on ``sys.argv`` at import time so it has side effects that will interfere with its use as a package. The API it presents should not be considered stable.
 
+Regarding OCRmyPDF itself:
+
+* PDFs that use transparency are not currently represented in the test suite
+* The Python API exported by ``import ocrmypdf`` is design to help scripts that use OCRmyPDF but is not currently capable of running OCRmyPDF jobs due to limitations in an underlying library.
 
 Similar programs
 ----------------
@@ -112,4 +113,3 @@ Web front-ends
 * `OCRmyPDF-web <https://github.com/sseemayer/OCRmyPDF-web>`_, a micro web-frontend for OCRmyPDF (third-party, not actively maintained)
 
 Bear in mind that OCRmyPDF is not designed to be secure against malware-bearing PDFs (see `Using OCRmyPDF online`_).
-

docs/release_notes.rst

@@ -13,12 +13,21 @@ Note that it is licensed under GPLv3, so scripts that ``import ocrmypdf`` and ar
    find:    [^`]\#([0-9]{1,3})[^0-9]
    replace: `#$1 <https://github.com/jbarlow83/OCRmyPDF/issues/$1>`_
 
+v7.0.1
+------
+
+-   Fix compatibility with img2pdf >= 0.3.0 by rejecting input images that have an alpha channel
+
+-   Add forward compatibility for pikepdf 0.3.0 (unrelated to img2pdf)
+
+-   Various documentation updates for v7.0.0 changes
+
 v7.0.0
 ------
 
 -   The core algorithm for combining OCR layers with existing PDF pages has been rewritten and improved considerably.  PDFs are no longer split into single page PDFs for processing; instead, images are rendered and the OCR results are grafted onto the input PDF.  The new algorithm uses less temporary disk space and is much more performant especially for large files.
 
--   New dependency: `pikepdf <https://github.com/pikepdf/pikepdf>`_. pikepdf is a  powerful new Python PDF library driving the latest OCRmyPDF features, built on the QPDF C++ library (libqpdf).
+-   New dependency: `pikepdf <https://github.com/pikepdf/pikepdf>`_. pikepdf is a powerful new Python PDF library driving the latest OCRmyPDF features, built on the QPDF C++ library (libqpdf).
 
 -   New feature: PDF optimization with ``-O`` or ``--optimize``.  After OCR, OCRmyPDF will perform image optimizations relevant to OCR PDFs.
 
@@ -61,6 +70,18 @@ v7.0.0
 
     +   It may be necessary to separately ``pip install pycparser`` to avoid `another Python 3.7 issue <https://github.com/eliben/pycparser/pull/135>`_.
 
+v6.2.3
+------
+
+-   Fix compatibility with img2pdf >= 0.3.0 by rejecting input images that have an alpha channel
+
+v6.2.2
+------
+
+-   Backport compatibility fixes for Python 3.7 and ruffus 2.7.0 from v7.0.0
+-   Backport fix to ignore masks when deciding what colors are on a page
+-   Backport some minor improvements from v7.0.0: better argument validation and warnings about the Tesseract 4.0.0 ``--user-words`` regression
+
 v6.2.1
 ------
 

requirements.txt

@@ -8,4 +8,4 @@ Pillow >= 5.0.0, != 5.1.0 ; sys_platform == "darwin"
 pycparser == 2.18
 python-xmp-toolkit == 2.0.1
 reportlab == 3.4.0
-ruffus == 2.7
+ruffus == 2.7.0

setup.py

@@ -248,14 +248,14 @@ setup(
     ],
     install_requires=[
         'cffi >= 1.9.1',          # must be a setup and install requirement
-        'img2pdf >= 0.2.4',       # pure Python, so track HEAD closely
-        'pikepdf >= 0.2.2, < 0.3',
+        'img2pdf >= 0.2.4, < 0.4',       # pure Python, so track HEAD closely
+        'pikepdf >= 0.2.2, < 0.4',
         'Pillow >= 4.0.0, != 5.1.0 ; sys_platform == "darwin"',
                                   # Pillow < 4 has BytesIO/TIFF bug w/img2pdf 0.2.3
                                   # block 5.1.0, broken wheels
         'python-xmp-toolkit >= 2, < 3',
         'reportlab >= 3.3.0',     # oldest released version with sane image handling
-        'ruffus >= 2.7',
+        'ruffus >= 2.7.0',
     ],
     extras_require={
         'fitz': [],  # Backward compatibility

src/ocrmypdf/_pipeline.py

@@ -19,7 +19,6 @@ from contextlib import suppress
 from shutil import copyfileobj, copyfile
 from pathlib import Path
 from datetime import datetime, timezone
-
 import sys
 import os
 import re
@@ -85,6 +84,13 @@ def triage_image_file(input_file, output_file, log, options):
                 "image was scanned and specify it using --image-dpi.")
             raise DpiError()
 
+        if im.mode in ('RGBA', 'LA'):
+            log.error(
+                "The input image has an alpha channel. Remove the alpha "
+                "channel first."
+            )
+            raise UnsupportedImageFormatError()
+
         if 'iccprofile' not in im.info:
             if im.mode == 'RGB':
                 log.info('Input image has no ICC profile, assuming sRGB')

src/ocrmypdf/optimize.py

@@ -24,6 +24,11 @@ import sys
 from PIL import Image
 
 import pikepdf
+try:
+    from pikepdf import Null  # pikepdf <= 0.2.2
+except ImportError:
+    def Null():               # pikepdf > 0.2.2
+        return None
 
 from ._jobcontext import JobContext
 from . import leptonica
@@ -241,7 +246,7 @@ def transcode_jpegs(pike, jpegs, root, log, options):
         im_obj = pike.get_object(xref, 0)
         im_obj.write(
             compdata.read(), pikepdf.Name('/DCTDecode'),
-            pikepdf.Null()
+            Null()
         )
 
 
@@ -281,7 +286,7 @@ def transcode_pngs(pike, pngs, root, log, options):
         if len(compdata) > int(im_obj.stream_dict.Length):
             continue  # If we produced a larger image, don't use
 
-        predictor = pikepdf.Null()
+        predictor = Null()
         if compdata.predictor > 0:
             predictor = pikepdf.Dictionary({'/Predictor': compdata.predictor})
 

tests/resources/README.rst

@@ -146,8 +146,9 @@ Assemblies
 
 These test resources are assemblies or derivatives from other previously mentioned files, released under the same license terms as their input files.
 
-- baiona_gray.png (from baiona.png)
-- baiona_colormapped.png (from baiona.png)
+- baiona_gray.png (from baiona.png, grayscale version)
+- baiona_colormapped.png (from baiona.png, palette version)
+- baiona_alpha.png (from baiona.png, RGB+A version)
 - cardinal.pdf (four cardinal directions, baked-in rotated copies of linn.png)
 - ccitt.pdf (linn.png, converted to CCITT encoding)
 - encrypted_algo4.pdf (congress.jpg, encrypted with algorithm 4 - not supported by PyPDF2)

tests/test_main.py

@@ -691,6 +691,7 @@ def test_no_contents(spoof_tesseract_noop, resources, outpdf):
 @pytest.mark.parametrize('image', [
     'baiona.png',
     'baiona_gray.png',
+    'baiona_alpha.png',
     'congress.jpg'
     ])
 def test_compression_preserved(spoof_tesseract_noop, ocrmypdf_exec,
@@ -699,7 +700,6 @@ def test_compression_preserved(spoof_tesseract_noop, ocrmypdf_exec,
     output_file = str(outpdf)
 
     im = Image.open(input_file)
-
     # Runs: ocrmypdf - output.pdf < testfile
     with open(input_file, 'rb') as input_stream:
         p_args = ocrmypdf_exec + [
@@ -710,7 +710,12 @@ def test_compression_preserved(spoof_tesseract_noop, ocrmypdf_exec,
             stdin=input_stream, env=spoof_tesseract_noop)
         out, err = p.communicate()
 
-        assert p.returncode == ExitCode.ok
+        if im.mode in ('RGBA', 'LA'):
+            # If alpha image is input, expect an error
+            assert p.returncode != ExitCode.ok and b'alpha' in err
+            return
+
+        assert p.returncode == ExitCode.ok, err.decode('utf-8')
 
     pdfinfo = PdfInfo(output_file)
 
@@ -754,7 +759,7 @@ def test_compression_changed(spoof_tesseract_noop, ocrmypdf_exec,
             stdin=input_stream, env=spoof_tesseract_noop)
         out, err = p.communicate()
 
-        assert p.returncode == ExitCode.ok
+        assert p.returncode == ExitCode.ok, err
 
     pdfinfo = PdfInfo(output_file)