Remove pikepdf < 0.3 compatibility shims since > 0.3.1 is now required

[ci skip]

.docker/Dockerfile

@@ -3,28 +3,49 @@
 FROM      ubuntu:18.04
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
+  build-essential autoconf automake libtool \
+  libleptonica-dev \
+  zlib1g-dev \
+  libexempi3 \
   ocrmypdf \
-  unpaper \
+  pngquant \
+  python3-pip \
+  python3-venv \
+  tesseract-ocr \
+  tesseract-ocr-chi-sim \
+  tesseract-ocr-deu \
   tesseract-ocr-eng \
   tesseract-ocr-fra \
-  tesseract-ocr-deu \
-  tesseract-ocr-spa \
   tesseract-ocr-por \
-  tesseract-ocr-chi-sim \
-  python3-venv \
-  python3-pip
+  tesseract-ocr-spa \
+  unpaper \
+  wget
+
 
 ENV LANG=C.UTF-8
 
+# Compile and install jbig2
+# Needs libleptonica-dev, zlib1g-dev
+RUN \
+  mkdir jbig2 \
+  && wget -q https://github.com/agl/jbig2enc/archive/0.29.tar.gz -O - | \
+      tar xz -C jbig2 --strip-components=1 \
+  && cd jbig2 \
+  && ./autogen.sh && ./configure && make && make install \
+  && cd .. \
+  && rm -rf jbig2
+
+RUN apt-get remove -y autoconf automake libtool
+
 RUN python3 -m venv --system-site-packages /appenv
 
 # This installs the latest binary wheel instead of the code in the current
 # folder. Installing from source will fail, apparently because cffi needs
-# build-essentials (gcc) to do a source installation 
+# build-essentials (gcc) to do a source installation
 # (i.e. "pip install ."). It's unclear to me why this is the case.
 RUN . /appenv/bin/activate; \
   pip install --upgrade pip \
-  && pip install ocrmypdf[fitz]
+  && pip install --upgrade ocrmypdf
 
 # Now copy the application in, mainly to get the test suite.
 # Do this now to make the best use of Docker cache.
@@ -35,6 +56,7 @@ RUN . /appenv/bin/activate; \
 # Remove the junk, including the source version of application since it was
 # already installed
 RUN rm -rf /tmp/* /var/tmp/* /root/* /application/ocrmypdf \
+  && apt-get remove -y build-essential \
   && apt-get autoremove -y \
   && apt-get autoclean -y
 

.travis.yml

@@ -1,50 +1,70 @@
-dist: trusty
-language: python
 cache:
   pip: true
   directories:
   - $HOME/Library/Caches/Homebrew
 
-addons:
-  apt:
-    update: true
-    sources:
-    - sourceline: 'ppa:alex-p/tesseract-ocr'
-    - sourceline: 'ppa:heyarje/libav-11'
-    - sourceline: 'ppa:vshn/ghostscript'
-    packages:
-    - ghostscript
-    - libavcodec56
-    - libavformat56
-    - libavutil54
-    - libexempi3
-    - libffi-dev
-    - pngquant
-    - poppler-utils
-    - qpdf
-    - tesseract-ocr
-    - tesseract-ocr-deu
-    - tesseract-ocr-eng
-    - tesseract-ocr-fra
-
-env:
-  global:
-    - secure: "hsf6MT+n2x3OiDM2fQyJZdV0/PWYmv81LdVqC6cfnHBE/8N3DloJRqQ7WfO14TxhiK9PEC7MpyCj0lSabUHEO7gSH6Vks6I1asoSkt8S9/bSMlhT4hei+pwVpeGEiU5xHVATNjY+D919VC3IFvc3XmjT74h/2SLhaZ+jhEmDggM="  # HOMEBREW_OCRMYPDF_TOKEN
-
 matrix:
   include:
     - os: linux
+      dist: trusty
       sudo: required
       language: python
       python: "3.5"
+      env:
+        - DIST=trusty
+      addons: &trusty_apt
+        apt:
+          update: true
+          sources:
+          - sourceline: 'ppa:alex-p/tesseract-ocr'
+          - sourceline: 'ppa:heyarje/libav-11'
+          - sourceline: 'ppa:vshn/ghostscript'
+          packages:
+          - ghostscript
+          - libavcodec56
+          - libavformat56
+          - libavutil54
+          - libexempi3
+          - libffi-dev
+          - pngquant
+          - poppler-utils
+          - qpdf
+          - tesseract-ocr
+          - tesseract-ocr-deu
+          - tesseract-ocr-eng
+          - tesseract-ocr-fra
     - os: linux
+      dist: trusty
       sudo: required
       language: python
       python: "3.6"
+      env:
+        - DIST=trusty
+      addons: *trusty_apt
     - os: linux
+      dist: xenial
       sudo: required
       language: python
-      python: "3.7-dev"
+      python: "3.7"
+      env:
+        - DIST=xenial
+      addons:
+        apt:
+          update: true
+          sources:
+          - sourceline: 'ppa:alex-p/tesseract-ocr'
+          packages:
+          - ghostscript
+          - libexempi3
+          - libffi-dev
+          - pngquant
+          - poppler-utils
+          - qpdf
+          - tesseract-ocr
+          - tesseract-ocr-deu
+          - tesseract-ocr-eng
+          - tesseract-ocr-fra
+          - unpaper
     - os: osx
       osx_image: xcode8
       language: generic
@@ -57,9 +77,11 @@ before_install: |
   if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
     pip3 install --upgrade pip
     pip3 install --upgrade wheel
-    mkdir -p packages
-    wget -q 'https://www.dropbox.com/s/vaq0kbwi6e6au80/unpaper_6.1-1.deb?raw=1' -O packages/unpaper_6.1-1.deb
-    sudo dpkg -i packages/unpaper_6.1-1.deb
+    if [[ "$DIST" == "trusty" ]]; then
+      mkdir -p packages
+      wget -q 'https://www.dropbox.com/s/vaq0kbwi6e6au80/unpaper_6.1-1.deb?raw=1' -O packages/unpaper_6.1-1.deb
+      sudo dpkg -i packages/unpaper_6.1-1.deb
+    fi
   elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
     brew update && brew bundle --file=.travis/Brewfile
     pip3 install --upgrade pip
@@ -94,18 +116,3 @@ deploy:
     tags: true
     condition: $TRAVIS_PYTHON_VERSION == "3.6" && $TRAVIS_OS_NAME == "linux"
   skip_upload_docs: true
-
-  # null deploy for osx
-  # we really just want to run after_deploy *after* pypi upload is done, but
-  # after_deploy on runs if a given box deployed
-- provider: script
-  script: /usr/bin/true
-  on:
-    branch: master
-    tags: true
-    condition: $TRAVIS_OS_NAME == "osx"
-
-after_deploy: |
-  if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-    bash .travis/osx_brew.sh
-  fi

.travis/Brewfile

@@ -2,10 +2,8 @@ brew 'python'
 
 brew 'exempi'
 brew 'ghostscript'
-brew 'jbig2dec'
 brew 'jbig2enc'
 brew 'leptonica'
-brew 'libffi'
 brew 'libtiff'
 brew 'openjpeg'
 brew 'pngquant'

.travis/autobrew.py

@@ -1,93 +0,0 @@
-#!/usr/bin/env python3
-# © 2017-18 James R. Barlow: github.com/jbarlow83
-
-from string import Template
-from subprocess import run, PIPE
-import re
-
-recipe_template = Template("""
-class Ocrmypdf < Formula
-  include Language::Python::Virtualenv
-
-  desc "Adds an OCR text layer to scanned PDF files"
-  homepage "https://github.com/jbarlow83/OCRmyPDF"
-  ${ocrmypdf_url}
-  ${ocrmypdf_sha256}
-
-  depends_on "pkg-config" => :build
-  depends_on "mupdf-tools" => :build  # statically links libmupdf.a
-  depends_on "freetype"
-  depends_on "ghostscript"
-  depends_on "jpeg"
-  depends_on "libpng"
-  depends_on "python"
-  depends_on "qpdf"
-  depends_on "tesseract"
-  depends_on "unpaper"
-
-${resources}
-  def install
-    venv = virtualenv_create(libexec, "python3")
-
-    resource("Pillow").stage do
-      inreplace "setup.py" do |s|
-        sdkprefix = MacOS::CLT.installed? ? "" : MacOS.sdk_path
-        s.gsub! "openjpeg.h", "probably_not_a_header_called_this_eh.h"
-        s.gsub! "ZLIB_ROOT = None", "ZLIB_ROOT = ('#{sdkprefix}/usr/lib', '#{sdkprefix}/usr/include')"
-        s.gsub! "JPEG_ROOT = None", "JPEG_ROOT = ('#{Formula["jpeg"].opt_prefix}/lib', '#{Formula["jpeg"].opt_prefix}/include')"
-        s.gsub! "FREETYPE_ROOT = None", "FREETYPE_ROOT = ('#{Formula["freetype"].opt_prefix}/lib', '#{Formula["freetype"].opt_prefix}/include')"
-      end
-
-      # avoid triggering "helpful" distutils code that doesn't recognize Xcode 7 .tbd stubs
-      ENV.append "CFLAGS", "-I#{MacOS.sdk_path}/System/Library/Frameworks/Tk.framework/Versions/8.5/Headers" unless MacOS::CLT.installed?
-      venv.pip_install Pathname.pwd
-    end
-
-    res = resources.map(&:name).to_set - ["Pillow"]
-
-    res.each do |r|
-      venv.pip_install resource(r)
-    end
-
-    venv.pip_install_and_link buildpath
-  end
-
-  test do
-    # Since we use Python 3, we require a UTF-8 locale
-    ENV["LC_ALL"] = "en_US.UTF-8"
-
-    system "#{bin}/ocrmypdf", "-f", "-q", "--deskew",
-                              test_fixtures("test.pdf"), "ocr.pdf"
-    assert_predicate testpath/"ocr.pdf", :exist?
-  end
-end
-""")
-
-def main():
-    p = run(['poet', '--single', 'ocrmypdf'],
-            encoding='utf-8', stdout=PIPE, check=True)
-
-    ocrmypdf_lines = p.stdout.splitlines()
-    ocrmypdf_url = ocrmypdf_lines[1].strip()
-    ocrmypdf_sha256 = ocrmypdf_lines[2].strip()
-
-    ocrmypdf_version = re.search(
-        r'ocrmypdf-(.+)\.tar.*', ocrmypdf_url).group(1)
-    print(f"Autobrewing {ocrmypdf_version}")
-
-    p = run(['poet', '--resources', 'ocrmypdf'],
-            encoding='utf-8', stdout=PIPE, check=True)
-
-    poet_resources = p.stdout
-
-    # Remove the duplicate "ocrmypdf" resource block
-    all_resources = poet_resources.split('resource')
-    kept_resources = [block for block in all_resources if 'ocrmypdf' not in block]
-    resources = 'resource'.join(kept_resources)
-
-    with open('ocrmypdf.rb', 'w') as out:
-        out.write(recipe_template.substitute(**locals()))
-
-
-if __name__ == '__main__':
-    main()

.travis/osx_brew.sh

@@ -1,23 +0,0 @@
-#!/bin/bash
-# © 2017 James R. Barlow: github.com/jbarlow83
-set -uo pipefail
-set -x
-
-pip3 install homebrew-pypi-poet
-python3 .travis/autobrew.py
-cat ocrmypdf.rb
-
-# brew audit crashes Travis
-#brew audit ocrmypdf.rb
-
-# Important: disable debug output so token is hidden
-set +x
-git clone https://$HOMEBREW_OCRMYPDF_TOKEN@github.com/jbarlow83/homebrew-ocrmypdf.git
-set -x
-
-pushd homebrew-ocrmypdf
-cp ../ocrmypdf.rb Formula/ocrmypdf.rb
-git add Formula/ocrmypdf.rb
-git commit -m "homebrew-ocrmypdf: automatic release $TRAVIS_BUILD_NUMBER $TRAVIS_TAG"
-git push origin master
-popd

docs/conf.py

@@ -60,6 +60,24 @@ author = 'James R. Barlow'
 # built documents.
 #
 # The short X.Y version.
+
+import os
+on_rtd = os.environ.get('READTHEDOCS') == 'True'
+
+if on_rtd:
+    # Help ReadTheDocs avoid having to install any binary extension modules
+    import sys
+    from unittest.mock import MagicMock
+
+    class Mock(MagicMock):
+        @classmethod
+        def __getattr__(cls, name):
+            return MagicMock()
+
+    MOCK_MODULES = ['pikepdf', 'libxmp', 'libxmp.utils']
+    sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES)
+
+
 from ocrmypdf import __version__ as OCRMYPDF_VERSION
 
 _version_parts = OCRMYPDF_VERSION.split('.')

docs/cookbook.rst

@@ -61,7 +61,7 @@ If the page is "just a little off horizontal", like a crooked picture, then you
 OCR languages other than English
 """"""""""""""""""""""""""""""""
 
-By default OCRmyPDF assumes the document is English. 
+By default OCRmyPDF assumes the document is English.
 
 .. code-block:: bash
 
@@ -127,7 +127,7 @@ OCRmyPDF perform some image processing on each page of a PDF, if desired.  The s
 * ``--remove-background`` attempts to detect and remove a noisy background from grayscale or color images.  Monochrome images are ignored. This should not be used on documents that contain color photos as it may remove them.
 
 * ``--deskew`` will correct pages were scanned at a skewed angle by rotating them back into place.  Skew determination and correction is performed using `Postl's variance of line sums <http://www.leptonica.com/skew-measurement.html>`_ algorithm as implemented in `Leptonica <http://www.leptonica.com/index.html>`_.
-  
+
 * ``--clean`` uses `unpaper <https://www.flameeyes.eu/projects/unpaper>`_ to clean up pages before OCR, but does not alter the final output.  This makes it less likely that OCR will try to find text in background noise.
 
 * ``--clean-final`` uses unpaper to clean up pages before OCR and inserts the page into the final output.  You will want to review each page to ensure that unpaper did not remove something important.
@@ -176,7 +176,7 @@ To redo OCR on a file OCRed with other OCR software or a previous version of OCR
 
     ocrmypdf --force-ocr input.pdf output.pdf
 
-Note that the method above will force rasterization of all pages, potentially reducing quality or losing vector content. 
+Note that the method above will force rasterization of all pages, potentially reducing quality or losing vector content.
 
 To ensure quality is preserved, one could extract all of the images and rebuild the PDF for a lossless transformation. This recipe does not work when PDFs contain multiple images per page, as many do in practice. It will also lose any page rotation information.
 
@@ -198,3 +198,19 @@ The `Image processing`_ features can improve OCR quality.
 Rotating pages and deskewing helps to ensure that the page orientation is correct before OCR begins. Removing the background and/or cleaning the page can also improve results. The ``--oversample DPI`` argument can be specified to resample images to higher resolution before attempting OCR; this can improve results as well.
 
 OCR quality will suffer if the resolution of input images is not correct (since the range of pixel sizes that will be checked for possible fonts will also be incorrect).
+
+
+PDF optimization
+----------------
+
+By default OCRmyPDF will attempt to perform lossless optimizations on the images inside PDFs after OCR is complete. Optimization is performed even if no OCR text is found.
+
+The ``--optimize N`` (short form ``-O``) argument controls optimization, where ``N`` ranges from 0 to 3. ``--optimize 0`` disables optimizations. ``1`` enables lossless optimizations that can be performed safely with no quality loss. ``2`` enables lossy optimizations such as image color quantizations. ``3`` enables more aggressive optimizations and targets a lower JPEG quality.
+
+Optimization is improved when a JBIG2 encoder is available and when ``pngquant`` is installed. If either of these components are missing, then some types of images will not be optimized.
+
+Currently optimization attempts to find more efficient encodings for images. The types of optimization available may expand over time. By default, OCRmyPDF compresses data streams inside PDFs, and will change inefficient encodings to more modern versions. A program like ``qpdf`` can be used to change encodings, e.g. to inspect the internals fo a PDF.
+
+.. code-block:: bash
+
+    ocrmypdf --optimize 3 in.pdf out.pdf  # Make it as small as possible

docs/index.rst

@@ -18,6 +18,7 @@ PDF is the best format for storing and exchanging scanned documents.  Unfortunat
    release_notes
    installation
    languages
+   jbig2
 
 .. toctree::
    :caption: Usage
@@ -36,4 +37,3 @@ Indices and tables
 * :ref:`genindex`
 * :ref:`modindex`
 * :ref:`search`
-

docs/installation.rst

@@ -1,16 +1,53 @@
 Installation
 ============
 
+.. |latest| image:: https://img.shields.io/pypi/v/ocrmypdf.svg
+    :alt: OCRmyPDF latest released version on PyPI
+
+|latest|
+
 The easiest way to install OCRmyPDF to follow the steps for your operating system/platform.
 
 If you want to use the latest version of OCRmyPDF, your best bet is to install the most recent version your platform provides, and then upgrade that version by installing the Python binary wheels.
 
 .. contents:: Platform-specific steps
-    :depth: 1
+    :depth: 2
     :local:
 
-Installing on Debian and Ubuntu 16.10 or newer
-----------------------------------------------
+Installing on Linux
+-------------------
+
+Debian and Ubuntu 16.10 or newer
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. |deb-stable| image:: https://repology.org/badge/version-for-repo/debian_stable/ocrmypdf.svg
+    :alt: Debian 9 stable ("stretch")
+
+.. |deb-testing| image:: https://repology.org/badge/version-for-repo/debian_testing/ocrmypdf.svg
+    :alt: Debian 10 testing ("buster")
+
+.. |deb-unstable| image:: https://repology.org/badge/version-for-repo/debian_unstable/ocrmypdf.svg
+    :alt: Debian unstable
+
+.. |ubu-1710| image:: https://repology.org/badge/version-for-repo/ubuntu_17_10/ocrmypdf.svg
+    :alt: Ubuntu 17.10
+
+.. |ubu-1804| image:: https://repology.org/badge/version-for-repo/ubuntu_18_04/ocrmypdf.svg
+    :alt: Ubuntu 18.04 LTS
+
+.. |ubu-1810| image:: https://repology.org/badge/version-for-repo/ubuntu_18_10/ocrmypdf.svg
+    :alt: Ubuntu 18.10
+
+
++-------------------------------------------+
+| **OCRmyPDF version**                      |
++-------------------------------------------+
+| |latest|                                  |
++-------------------------------------------+
+| |deb-stable| |deb-testing| |deb-unstable| |
++-------------------------------------------+
+| |ubu-1710| |ubu-1804| |ubu-1810|          |
++-------------------------------------------+
 
 Users of Debian 9 ("stretch") or later or Ubuntu 16.10 or later may simply
 
@@ -18,10 +55,176 @@ Users of Debian 9 ("stretch") or later or Ubuntu 16.10 or later may simply
 
     apt-get install ocrmypdf
 
-To see what versions are available, check the `Debian Package Tracker <https://tracker.debian.org/pkg/ocrmypdf>`_ or `Ubuntu launchpad.net <https://launchpad.net/ocrmypdf>`_.
+For full details on version availability, check the `Debian Package Tracker <https://tracker.debian.org/pkg/ocrmypdf>`_ or `Ubuntu launchpad.net <https://launchpad.net/ocrmypdf>`_.
 
-Installing on macOS with Homebrew
----------------------------------
+If the version available for your platform is out of date, you could opt to install the latest version from source. See `Installing HEAD revision from sources`_.
+
+.. note::
+
+    OCRmyPDF for Debian and Ubuntu currently omit the JBIG2 encoder. OCRmyPDF works fine without it but will produce larger output files. If you build jbig2enc from source, ocrmypdf 7.0.0 and later will automatically detect it on the ``PATH``. To add JBIG2 encoding, see `Installing the JBIG2 encoder`_.
+
+Installing the latest version on Ubuntu 18.04 LTS
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Ubuntu 18.04 includes ocrmypdf 6.1.2. To install a more recent version, first
+install the system version to get most of the dependencies:
+
+.. code-block:: bash
+
+    sudo apt-get update
+    sudo apt-get install \
+        ocrmypdf \
+        python3-pip
+
+There are a few dependency changes between ocrmypdf 6.1.2 and 7.x. Let's get
+these, too.
+
+.. code-block:: bash
+
+    sudo apt-get install \
+        libexempi3 \
+        pngquant
+
+Then install the most recent ocrmypdf for the local user and set the user's ``PATH`` to check for the user's Python packages.
+
+.. code-block:: bash
+
+    export PATH=$HOME/.local/bin:$PATH
+    pip3 install --user ocrmypdf
+
+To add JBIG2 encoding, see `Installing the JBIG2 encoder`_.
+
+Ubuntu 16.04 LTS
+^^^^^^^^^^^^^^^^
+
+No package is currently available for Ubuntu 16.04, but you can install the dependencies manually:
+
+.. code-block:: bash
+
+    sudo apt-get update
+    sudo apt-get install \
+        ghostscript \
+        libexempi3 \
+        pngquant \
+        python3-cffi \
+        python3-pip \
+        qpdf \
+        tesseract-ocr \
+        unpaper
+
+If you wish install OCRmyPDF for the current user, and ensure that the ``PATH``
+environment variable contains ``$HOME/.local/bin``.
+
+.. code-block:: bash
+
+    export PATH=$HOME/.local/bin:$PATH
+    pip3 install --user ocrmypdf
+
+Alternately, you can install ocrmypdf system-wide. (Not recommended.)
+
+.. code-block:: bash
+
+    sudo pip3 install ocrmypdf
+
+At your option, you may upgrade Ubuntu 16.04 LTS to Tesseract 4.0 for improved OCR results.
+
+.. code-block:: bash
+
+    sudo apt-get install -y software-properties-common python-software-properties
+    sudo add-apt-repository ppa:alex-p/tesseract-ocr -y
+    sudo apt-get update
+    sudo apt-get upgrade tesseract-ocr
+
+To add JBIG2 encoding, see `Installing the JBIG2 encoder`_.
+
+Ubuntu 14.04 LTS
+^^^^^^^^^^^^^^^^
+
+Installing on Ubuntu 14.04 LTS (trusty) is more difficult than some other options, because it is older and does not provide ``pip``.
+
+Update apt-get:
+
+.. code-block:: bash
+
+    sudo apt-get update
+
+Install system dependencies:
+
+.. code-block:: bash
+
+    sudo apt-get install \
+        software-properties-common python-software-properties \
+        zlib1g-dev \
+        libexempi3 \
+        libjpeg-dev \
+        libffi-dev \
+        pngquant \
+        qpdf
+
+We will need backports of Ghostscript 9.16, libav-11 (for unpaper 6.1), Tesseract 4.00 (alpha), and Python 3.6. This will replace Ghostscript and Tesseract 3.x on your system. Python 3.6 will be installed alongside the system Python 3.4.
+
+If you prefer to not modify your system in this matter, consider using a Docker container.
+
+.. code-block:: bash
+
+    sudo add-apt-repository ppa:vshn/ghostscript -y
+    sudo add-apt-repository ppa:heyarje/libav-11 -y
+    sudo add-apt-repository ppa:alex-p/tesseract-ocr -y
+    sudo add-apt-repository ppa:jonathonf/python-3.6 -y
+
+    sudo apt-get update
+
+    sudo apt-get install \
+        python3.6-dev \
+        ghostscript \
+        tesseract-ocr \
+        tesseract-ocr-eng \
+        libavformat56 libavcodec56 libavutil54 \
+        wget
+
+Now we need to install ``pip`` and let it install ocrmypdf:
+
+.. code-block:: bash
+
+    curl https://bootstrap.pypa.io/ez_setup.py -o - | python3.6 && python3.6 -m easy_install pip
+    pip3.6 install ocrmypdf
+
+The ``wget`` command will download a program and run it.
+
+These installation instructions omit the optional dependency ``unpaper``, which is only available at version 0.4.2 in Ubuntu 14.04. The author could not find a backport of ``unpaper``, and created a .deb package to do the job of installing unpaper 6.1 (for x86 64-bit only):
+
+.. code-block:: bash
+
+    wget -q 'https://www.dropbox.com/s/vaq0kbwi6e6au80/unpaper_6.1-1.deb?raw=1' -O unpaper_6.1-1.deb
+    sudo dpkg -i unpaper_6.1-1.deb
+
+To add JBIG2 encoding, see `Installing the JBIG2 encoder`_.
+
+ArchLinux
+^^^^^^^^^
+
+.. image:: https://repology.org/badge/version-for-repo/aur/ocrmypdf.svg
+    :alt: ArchLinux
+    :target: https://repology.org/metapackage/ocrmypdf
+
+The author is aware of an `ArchLinux package for ocrmypdf <https://aur.archlinux.org/packages/ocrmypdf/>`_. It seems like the following command might work.
+
+.. code-block:: bash
+
+    pacman -S ocrmypdf
+
+Other Linux packages
+^^^^^^^^^^^^^^^^^^^^
+
+See the `Repology <https://repology.org/metapackage/ocrmypdf/versions>`_ page.
+
+In general, first install the OCRmyPDF package for your system, then optionally use the procedure `Installing with Python pip`_ to install a more recent version.
+
+Installing on macOS
+-------------------
+
+Homebrew
+^^^^^^^^
 
 .. image:: https://img.shields.io/homebrew/v/ocrmypdf.svg
     :alt: homebrew
@@ -41,12 +244,64 @@ OCRmyPDF is now a standard `Homebrew <https://brew.sh>`_ formula. To install on
 
     Users who previously installed OCRmyPDF from the private tap should switch to the mainline version (``brew untap jbarlow83/ocrmypdf``) and install from there.
 
-.. _Docker-install:
+Manual installation on macOS
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+These instructions probably work on all macOS supported by Homebrew.
+
+If it's not already present, `install Homebrew <http://brew.sh/>`_.
+
+Update Homebrew:
+
+.. code-block:: bash
+
+    brew update
+
+Install or upgrade the required Homebrew packages, if any are missing. To do this, download the ``Brewfile`` that lists all of the dependencies to the current directory, and run ``brew bundle`` to process them (installing or upgrading as needed). ``Brewfile`` is a plain text file.
+
+.. code-block:: bash
+
+    wget https://github.com/jbarlow83/OCRmyPDF/raw/master/.travis/Brewfile
+    brew bundle
+
+This will include the English, French, German and Spanish language packs. If you need other languages you can optionally install them all:
+
+.. _macos-all-languages:
+
+.. code-block:: bash
+
+    brew install tesseract --with-all-languages  # Option 2: for all language packs
+
+Update the homebrew pip:
+
+.. code-block:: bash
+
+    pip3 install --upgrade pip
+
+You can then install OCRmyPDF from PyPI, for the current user:
+
+.. code-block:: bash
+
+    pip3 install --user ocrmypdf
+
+or system-wide:
+
+.. code-block:: bash
+
+    pip3 install ocrmypdf
+
+The command line program should now be available:
+
+.. code-block:: bash
+
+    ocrmypdf --help
+
+.. _docker-install:
 
 Installing the Docker image
 ---------------------------
 
-For many users, installing the Docker image will be easier than installing all of OCRmyPDF's dependencies. For Windows, it is the only option.
+For some users, installing the Docker image will be easier than installing all of OCRmyPDF's dependencies. For Windows, it is the only option.
 
 If you have `Docker <https://docs.docker.com/>`_ installed on your system, you can install a Docker image of the latest release.
 
@@ -142,201 +397,12 @@ Or in the wonderful `fish shell <https://fishshell.com/>`_:
 
 .. note::
 
-    The ocrmypdf Docker containers are designed to be used for a single OCR job. The ``docker run --rm`` argument tells Docker to delete temporary storage associated with container when it is done executing.
-
-Manual installation on macOS
-----------------------------
-
-These instructions probably work on all macOS supported by Homebrew.
-
-If it's not already present, `install Homebrew <http://brew.sh/>`_.
-
-Update Homebrew:
-
-.. code-block:: bash
-
-    brew update
-
-Install or upgrade the required Homebrew packages, if any are missing:
-
-.. code-block:: bash
-
-    brew install libpng openjpeg jbig2dec libtiff     # image libraries
-    brew install qpdf
-    brew install ghostscript
-    brew install python3
-    brew install libxml2 libffi leptonica
-    brew install unpaper   # optional
-
-Python 3.5, 3.6 and 3.7 are supported.
-
-Install the required Tesseract OCR engine with the language packs you plan to use:
-
-.. code-block:: bash
-
-    brew install tesseract                       # Option 1: for English, French, German, Spanish
-
-.. _macos-all-languages:
-
-.. code-block:: bash
-
-    brew install tesseract --with-all-languages  # Option 2: for all language packs
-
-Update the homebrew pip:
-
-.. code-block:: bash
-
-    pip3 install --upgrade pip
-
-You can then install OCRmyPDF from PyPI, for the current user:
-
-.. code-block:: bash
-
-    pip3 install --user ocrmypdf
-
-or system-wide:
-
-.. code-block:: bash
-
-    pip3 install ocrmypdf
-
-The command line program should now be available:
-
-.. code-block:: bash
-
-    ocrmypdf --help
-
-Installing the latest version on Ubuntu 18.04 LTS
--------------------------------------------------
-
-Ubuntu 18.04 includes ocrmypdf 6.1.2. To install a more recent version, first
-install the system version to get all the dependencies:
-
-.. code-block:: bash
-
-    sudo apt-get update
-    sudo apt-get install \
-        ocrmypdf \
-        python3-pip
-
-Then install the most recent ocrmypdf for the local user and set the user's ``PATH`` to check for the user's Python packages.
-
-.. code-block:: bash
-
-    export PATH=$HOME/.local/bin:$PATH
-    pip3 install --user ocrmypdf
-
-
-Installing on Ubuntu 16.04 LTS
-------------------------------
-
-No package is currently available for Ubuntu 16.04, but you can install the dependencies manually:
-
-.. code-block:: bash
-
-    sudo apt-get update
-    sudo apt-get install \
-        unpaper \
-        ghostscript \
-        tesseract-ocr \
-        qpdf \
-        python3-pip \
-        python3-cffi
-
-If you wish install OCRmyPDF for the current user:
-
-.. code-block:: bash
-
-    pip3 install --user ocrmypdf
-
-Alternately, system-wide. Note that this may modify the system Python environment:
-
-.. code-block:: bash
-
-    sudo pip3 install ocrmypdf
-
-If you wish to install OCRmyPDF to a virtual environment to isolate the system Python, you can follow these steps.
-
-.. code-block:: bash
-
-    python3 -m venv venv-ocrmypdf
-    source venv-ocrmypdf/bin/activate
-    pip3 install ocrmypdf
-
-Installing on Ubuntu 14.04 LTS
-------------------------------
-
-Installing on Ubuntu 14.04 LTS (trusty) is more difficult than some other options, because it is older and does not provide ``pip``.
-
-Update apt-get:
-
-.. code-block:: bash
-
-    sudo apt-get update
-
-Install system dependencies:
-
-.. code-block:: bash
-
-    sudo apt-get install \
-        software-properties-common python-software-properties \
-        zlib1g-dev \
-        libjpeg-dev \
-        libffi-dev \
-        qpdf
-
-We will need backports of Ghostscript 9.16, libav-11 (for unpaper 6.1), Tesseract 4.00 (alpha), and Python 3.6. This will replace Ghostscript and Tesseract 3.x on your system. Python 3.6 will be installed alongside the system Python 3.
-
-If you prefer to not modify your system in this matter, consider using a Docker container.
-
-.. code-block:: bash
-
-    sudo add-apt-repository ppa:vshn/ghostscript -y
-    sudo add-apt-repository ppa:heyarje/libav-11 -y
-    sudo add-apt-repository ppa:alex-p/tesseract-ocr -y
-    sudo add-apt-repository ppa:jonathonf/python-3.6 -y
-
-    sudo apt-get update
-
-    sudo apt-get install \
-        python3.6-dev \
-        ghostscript \
-        tesseract-ocr \
-        tesseract-ocr-eng \
-        libavformat56 libavcodec56 libavutil54 \
-        wget
-
-Now we need to install ``pip`` and let it install ocrmypdf:
-
-.. code-block:: bash
-
-    curl https://bootstrap.pypa.io/ez_setup.py -o - | python3.6 && python3.6 -m easy_install pip
-    pip3.6 install ocrmypdf
-
-The ``wget`` command will download a program and run it.
-
-These installation instructions omit the optional dependency ``unpaper``, which is only available at version 0.4.2 in Ubuntu 14.04. The author could not find a backport of ``unpaper``, and created a .deb package to do the job of installing unpaper 6.1 (for x86 64-bit only):
-
-.. code-block:: bash
-
-    wget -q 'https://www.dropbox.com/s/vaq0kbwi6e6au80/unpaper_6.1-1.deb?raw=1' -O unpaper_6.1-1.deb
-    sudo dpkg -i unpaper_6.1-1.deb
-
-
-Installing on ArchLinux
------------------------
-
-The author is aware of an `ArchLinux package for ocrmypdf <https://aur.archlinux.org/packages/ocrmypdf/>`_. It seems like the following command might work.
-
-.. code-block:: bash
-
-    pacman -S ocrmypdf
-
+    The ocrmypdf Docker images are designed for application delivery, to enable use of OCRmyPDF without fussing with dependencies. ``docker run --rm`` argument tells Docker to delete the container after it runs, because each container is only good for a single job. The Docker image is not designed for use as a persistent web service or for use on Amazon EC2 Container Service (AWS ECS).
 
 Installing on Windows
 ---------------------
 
-Direct installation on Windows is not possible.  Install the _`Docker` container as described above.  Ensure that your command prompt can run the docker "hello world" container.
+Direct installation on Windows is not possible.  `Install the Docker <docker-install_>`_ container as described above.  Ensure that your command prompt can run the docker "hello world" container.
 
 It would probably not be too difficult to run on Windows.  The main reason this has been avoided is the difficulty of packaging and installing the various non-Python dependencies: Tesseract, QPDF, Ghostscript, Leptonica.  Pull requests to add or improve Windows support would be quite welcome.
 
@@ -358,7 +424,11 @@ where /c/Users/sampleuser is a Unix representation of the Windows path C:\\Users
 Installing with Python pip
 --------------------------
 
-First, install `your platform's version <https://repology.org/metapackage/ocrmypdf/versions>`_ of ``ocrmypdf``, if available, as a way of ensuring that external dependencies are (mostly) satisified, even though the platform version may be out of date. Use ``ocrmypdf --version`` to confirm what version was installed.
+OCRmyPDF is delivered by PyPI because it is a convenient way to install the latest version. However, PyPI and ``pip`` cannot address the fact that ``ocrmypdf`` depends on certain non-Python system libraries and programs being instsalled.
+
+For best results, first install `your platform's version <https://repology.org/metapackage/ocrmypdf/versions>`_ of ``ocrmypdf``, using the instructions elsewhere in this document. Then you can use ``pip`` to get the latest version if your platform version is out of date. Chances are that this will satisfy most dependencies.
+
+Use ``ocrmypdf --version`` to confirm what version was installed.
 
 Then you can install the latest OCRmyPDF from the Python wheels. First try:
 
@@ -377,27 +447,36 @@ Since ``pip3 install --user`` does not work correctly on some platforms, notably
 Requirements for pip and HEAD install
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-OCRmyPDF currently requires these external programs to be installed:
+OCRmyPDF currently requires these external programs and libraries to be installed:
 
 - Python 3.5 or newer
-- Tesseract 3.04 or newer
 - Ghostscript 9.15 or newer
+- libexempi3 2.2.0 or newer
 - qpdf 7.0.0 or newer
+- Tesseract 3.04 or newer
 
-The following dependencies are recommended:
+As of ocrmypdf 7.0.0, the following versions are recommended:
 
-- Python 3.6
-- Tesseract 4.00 or newer
-- Ghostscript 9.22 or newer
-- qpdf 8.0.2 or newer
+- Python 3.7
+- Ghostscript 9.23
+- libexempi3 2.4.5
+- qpdf 8.1.0
+- Tesseract 4.0.0-beta1
+- jbig2enc 0.29 or newer
+- pngquant 2.5 or newer
 - unpaper 6.1
-- PyMuPDF 1.12.5 or newer
+
+jbig2enc, pngquant, and unpaper are optional. If missing certain features are disabled. OCRmyPDF will discover them as soon as they are available.
+
+**jbig2enc**, if present, will be used to optimize the encoding of monochrome images.  This can significantly reduce the file size of the output file.  It is not required.  `jbig2enc <https://github.com/agl/jbig2enc>`_ is not generally available for Ubuntu or Debian due to lingering concerns about patent issues, but can easily be built from source. To add JBIG2 encoding, see `Optional: installing the JBIG2 encoder`_.
+
+**pngquant**, if present, is optionally used to optimize the encoding of PNG-style images in PDFs (actually, any that are that losslessly encoded) by lossily quantizing to a smaller color palette. It is only activated then the ``--optimize`` argument is ``2`` or ``3``.
+
+**unpaper**, if present, enables the ``--clean`` and ``--clean-final`` command line options.
 
 These are in addition to the Python packaging dependencies, meaning that unfortunately, the ``pip install`` command cannot satisfy all of them.
 
-Python 3.6 and Tesseract 4.0.0-beta.1 are recommended for best OCR results and best performance.
-
-The library PyMuPDF is not widely available in platform distributions, and it improves OCRmyPDF in certain conditions. Consider installing OCRmyPDF from the Python binary wheels, which include a precompiled version of this library.
+Python 3.7 and Tesseract 4.0.0-beta.1 are recommended for best OCR results and best performance.
 
 
 Installing HEAD revision from sources
@@ -427,8 +506,8 @@ You may find it easiest to install in a virtual environment, rather than system-
     cd OCRmyPDF
     pip3 install .
 
-However, ``ocrmypdf`` will only be accessible on the system PATH after
-you activate the virtual environment.
+However, ``ocrmypdf`` will only be accessible on the system PATH
+when you activate the virtual environment.
 
 To run the program:
 
@@ -441,8 +520,18 @@ need to be installed. The script requires specific versions of the
 dependencies. Older version than the ones mentioned in the release notes
 are likely not to be compatible to OCRmyPDF.
 
+For development
+~~~~~~~~~~~~~~~
 
-Other Linux packages
---------------------
+To install all of the development and test requirements:
 
-See the `Repology <https://repology.org/metapackage/ocrmypdf/versions>`_ page.
+.. code-block:: bash
+
+    git clone -b master https://github.com/jbarlow83/OCRmyPDF.git
+    python3 -m venv
+    source venv/bin/activate
+    cd OCRmyPDF
+    pip install -e .
+    pip install -r dev_requirements.txt -r test_requirements.txt
+
+To add JBIG2 encoding, see `Installing the JBIG2 encoder`_.

docs/introduction.rst

@@ -1,9 +1,7 @@
 Introduction
 ============
-
 OCRmyPDF is a Python 3 package that adds OCR layers to PDFs.
 
-
 About OCR
 ---------
 
@@ -66,6 +64,8 @@ In the case of a PDF that is nothing other than a container of images (no rotati
 
 OCRmyPDF uses several strategies depending on input options and the input PDF itself, but generally speaking it rasterizes a page for OCR and then grafts the OCR back onto the original. As such it can handle complex PDFs and still preserve their contents as much as possible.
 
+OCRmyPDF also supports a many, many edge cases that have cropped over several years of development. We support PDF features like images inside of Form XObjects, and pages with UserUnit scaling. We support rare image formats like non-monochrome 1-bit images. Thanks to pikepdf and QPDF, we auto-repair PDFs that are damaged. (Not that you need to know what any of these are! You should be able to throw any PDF at it.)
+
 
 Limitations
 -----------
@@ -78,7 +78,6 @@ OCRmyPDF is limited by the Tesseract OCR engine.  As such it experiences these l
 * If a document contains languages outside of those given in the ``-l LANG`` arguments, results may be poor.
 * It is not always good at analyzing the natural reading order of documents. For example, it may fail to recognize that a document contains two columns and join text across the columns.
 * Poor quality scans may produce poor quality OCR. Garbage in, garbage out.
-* PDFs that use transparent layers are not currently checked in the test suite, so they may not work correctly.
 * It does not expose information about what font family text belongs to.
 
 OCRmyPDF is also limited by the PDF specification:
@@ -91,9 +90,11 @@ Ghostscript also imposes some limitations:
 * PDFs containing JBIG2-encoded content will be converted to CCITT Group4 encoding, which has lower compression ratios, if Ghostscript PDF/A is enabled.
 * PDFs containing JPEG 2000-encoded content will be converted to JPEG encoding, which may introduce compression artifacts, if Ghostscript PDF/A is enabled.
 * Ghostscript may transcode grayscale and color images, either lossy to lossless or lossless to lossy, based on an internal algorithm. This behavior can be suppressed by setting ``--pdfa-image-compression`` to ``jpeg`` or ``lossless`` to set all images to one type or the other. Ghostscript has no option to maintain the input image's format.
-  
-OCRmyPDF is currently not designed to be used as a Python API; it is designed to be run as a command line tool. ``import ocrmypf`` currently attempts to process the command line on ``sys.argv`` at import time so it has side effects that will interfere with its use as a package. The API it presents should not be considered stable.
 
+Regarding OCRmyPDF itself:
+
+* PDFs that use transparency are not currently represented in the test suite
+* The Python API exported by ``import ocrmypdf`` is design to help scripts that use OCRmyPDF but is not currently capable of running OCRmyPDF jobs due to limitations in an underlying library.
 
 Similar programs
 ----------------
@@ -112,4 +113,3 @@ Web front-ends
 * `OCRmyPDF-web <https://github.com/sseemayer/OCRmyPDF-web>`_, a micro web-frontend for OCRmyPDF (third-party, not actively maintained)
 
 Bear in mind that OCRmyPDF is not designed to be secure against malware-bearing PDFs (see `Using OCRmyPDF online`_).
-

docs/jbig2.rst

@@ -0,0 +1,20 @@
+Installing the JBIG2 encoder
+==========================
+
+Most Linux distributions do not include a JBIG2 encoder since JBIG2 encoding was patented for a long time. All known JBIG2 US patents have expired as of 2017, but it is possible that unknown patents exist.
+
+JBIG2 encoding is recommended for OCRmyPDF and is used to losslessly create smaller PDFs. If JBIG2 encoding not available, lower quality encodings will be used.
+
+JBIG2 decoding is not patented and is performed automatically by most PDF viewers. It is widely supported has been part of the PDF specification since 2001.
+
+On macOS, Homebrew packages jbig2enc and OCRmyPDF includes it by default. The Docker image for OCRmyPDF also builds its own JBIG2 encoder from source.
+
+For all other Linux, you must build a JBIG2 encoder from source:
+
+.. code-block:: bash
+
+    git clone https://github.com/agl/jbig2enc
+    cd jbig2enc
+    ./autogen.sh
+    ./configure && make
+    [sudo] make install

docs/release_notes.rst

@@ -13,12 +13,33 @@ Note that it is licensed under GPLv3, so scripts that ``import ocrmypdf`` and ar
    find:    [^`]\#([0-9]{1,3})[^0-9]
    replace: `#$1 <https://github.com/jbarlow83/OCRmyPDF/issues/$1>`_
 
+v7.0.3
+------
+
+-   Fix issue `#284 <https://github.com/jbarlow83/OCRmyPDF/issues/284>`_, an error when parsing inline images that have are also image masks, by upgrading pikepdf to 0.3.1
+
+v7.0.2
+------
+
+-   Fix a regression with ``--rotate-pages`` on pages that already had rotations applied. (`#279 <https://github.com/jbarlow83/OCRmyPDF/issues/279>`_)
+
+-   Improve quality of page rotation in some cases by rasterizing a higher quality preview image. (`#281 <https://github.com/jbarlow83/OCRmyPDF/issues/281>`_)
+
+v7.0.1
+------
+
+-   Fix compatibility with img2pdf >= 0.3.0 by rejecting input images that have an alpha channel
+
+-   Add forward compatibility for pikepdf 0.3.0 (unrelated to img2pdf)
+
+-   Various documentation updates for v7.0.0 changes
+
 v7.0.0
 ------
 
 -   The core algorithm for combining OCR layers with existing PDF pages has been rewritten and improved considerably.  PDFs are no longer split into single page PDFs for processing; instead, images are rendered and the OCR results are grafted onto the input PDF.  The new algorithm uses less temporary disk space and is much more performant especially for large files.
 
--   New dependency: `pikepdf <https://github.com/pikepdf/pikepdf>`_. pikepdf is a  powerful new Python PDF library driving the latest OCRmyPDF features, built on the QPDF C++ library (libqpdf).
+-   New dependency: `pikepdf <https://github.com/pikepdf/pikepdf>`_. pikepdf is a powerful new Python PDF library driving the latest OCRmyPDF features, built on the QPDF C++ library (libqpdf).
 
 -   New feature: PDF optimization with ``-O`` or ``--optimize``.  After OCR, OCRmyPDF will perform image optimizations relevant to OCR PDFs.
 
@@ -48,8 +69,7 @@ v7.0.0
 
     +   ``PyMuPDF``
 
--   The ``sandwich`` PDF renderer can be used with all supported versions of Tesseract, including that those prior to v3.05 which don't support
-``-c textonly``. (Tesseract v4.0.0 is recommended and more efficient.)
+-   The ``sandwich`` PDF renderer can be used with all supported versions of Tesseract, including that those prior to v3.05 which don't support ``-c textonly``. (Tesseract v4.0.0 is recommended and more efficient.)
 
 -   ``--pdf-renderer auto`` option and the diagnostics used to select a PDF renderer now work better with old versions, but may make different decisions than past versions.
 
@@ -61,6 +81,18 @@ v7.0.0
 
     +   It may be necessary to separately ``pip install pycparser`` to avoid `another Python 3.7 issue <https://github.com/eliben/pycparser/pull/135>`_.
 
+v6.2.3
+------
+
+-   Fix compatibility with img2pdf >= 0.3.0 by rejecting input images that have an alpha channel
+
+v6.2.2
+------
+
+-   Backport compatibility fixes for Python 3.7 and ruffus 2.7.0 from v7.0.0
+-   Backport fix to ignore masks when deciding what colors are on a page
+-   Backport some minor improvements from v7.0.0: better argument validation and warnings about the Tesseract 4.0.0 ``--user-words`` regression
+
 v6.2.1
 ------
 
@@ -176,7 +208,7 @@ v6.0.0
 v5.7.0
 ------
 
--   Fixed an issue that caused poor CPU utilization on machines more than 4 cores when running Tesseract 4. (Related to issue `#217 <https://github.com/jbarlow83/OCRmyPDF/issues/217>`_.)
+-   Fixed an issue that caused poor CPU utilization on machines with more than 4 cores when running Tesseract 4. (Related to issue `#217 <https://github.com/jbarlow83/OCRmyPDF/issues/217>`_.)
 
 -   The 'hocr' renderer has been improved. The 'sandwich' and 'tesseract' renderers are still better for most use cases, but 'hocr' may be useful for people who work with the PDF.js renderer in English/ASCII languages. (`#225 <https://github.com/jbarlow83/OCRmyPDF/issues/225>`_)
 

requirements.txt

@@ -2,10 +2,10 @@
 # setup.py lists a separate set of requirements that are looser to simplify
 # installation
 cffi == 1.11.5
-img2pdf == 0.2.4
-pikepdf == 0.2.2
+img2pdf == 0.3.0
+pikepdf == 0.3.0
 Pillow >= 5.0.0, != 5.1.0 ; sys_platform == "darwin"
 pycparser == 2.18
 python-xmp-toolkit == 2.0.1
 reportlab == 3.4.0
-ruffus == 2.7
+ruffus == 2.7.0

setup.py

@@ -248,14 +248,14 @@ setup(
     ],
     install_requires=[
         'cffi >= 1.9.1',          # must be a setup and install requirement
-        'img2pdf >= 0.2.4',       # pure Python, so track HEAD closely
-        'pikepdf >= 0.2.2, < 0.3',
+        'img2pdf >= 0.2.4, < 0.4',       # pure Python, so track HEAD closely
+        'pikepdf >= 0.3.1, < 0.4',
         'Pillow >= 4.0.0, != 5.1.0 ; sys_platform == "darwin"',
                                   # Pillow < 4 has BytesIO/TIFF bug w/img2pdf 0.2.3
                                   # block 5.1.0, broken wheels
         'python-xmp-toolkit >= 2, < 3',
         'reportlab >= 3.3.0',     # oldest released version with sane image handling
-        'ruffus >= 2.7',
+        'ruffus >= 2.7.0',
     ],
     extras_require={
         'fitz': [],  # Backward compatibility

src/ocrmypdf/_pipeline.py

@@ -19,7 +19,6 @@ from contextlib import suppress
 from shutil import copyfileobj, copyfile
 from pathlib import Path
 from datetime import datetime, timezone
-
 import sys
 import os
 import re
@@ -85,6 +84,13 @@ def triage_image_file(input_file, output_file, log, options):
                 "image was scanned and specify it using --image-dpi.")
             raise DpiError()
 
+        if im.mode in ('RGBA', 'LA'):
+            log.error(
+                "The input image has an alpha channel. Remove the alpha "
+                "channel first."
+            )
+            raise UnsupportedImageFormatError()
+
         if 'iccprofile' not in im.info:
             if im.mode == 'RGB':
                 log.info('Input image has no ICC profile, assuming sRGB')
@@ -312,16 +318,6 @@ def marker_pages(
         page.symlink_to(input_file)  # pylint: disable=E1101
 
 
-def split_page(
-        placeholder_file,
-        output_file,
-        log,
-        context):
-    pageno = page_number(placeholder_file) - 1
-    input_pdf = context.get_pdfinfo().filename
-    qpdf.extract_page(input_pdf, output_file, pageno)
-
-
 def ocr_or_skip(
         input_files,
         output_files,
@@ -353,13 +349,13 @@ def rasterize_preview(
         context):
     pageinfo = get_pageinfo(input_file, context)
     options = context.get_options()
-    canvas_dpi = get_canvas_square_dpi(pageinfo, options) / 2
-    page_dpi = get_page_square_dpi(pageinfo, options) / 2
+    canvas_dpi = get_canvas_square_dpi(pageinfo, options)
+    page_dpi = get_page_square_dpi(pageinfo, options)
 
     ghostscript.rasterize_pdf(
         input_file, output_file, xres=canvas_dpi, yres=canvas_dpi,
         raster_device='jpeggray', log=log, page_dpi=(page_dpi, page_dpi),
-        pageno=page_number(input_file), rotation=pageinfo.rotation)
+        pageno=page_number(input_file))
 
 
 def orient_page(
@@ -367,6 +363,22 @@ def orient_page(
         output_file,
         log,
         context):
+    """
+    Work out orientation correct for each page.
+
+    We ask Ghostscript to draw a preview page, which will rasterize with the
+    current /Rotate applied, and then ask Tesseract which way the page is
+    oriented. If the value of /Rotate is correct (e.g., a user already
+    manually fixed rotation), then Tesseract will say the page is pointing
+    up and the correction is zero. Otherwise, the orientation found by
+    Tesseract represents the clockwise rotation, or the counterclockwise
+    correction to rotation.
+
+    When we draw the real page for OCR, we rotate it by the CCW correction,
+    which points it (hopefully) upright. _weave.py takes care of the orienting
+    the image and text layers.
+
+    """
 
     options = context.get_options()
     page_pdf = next(ii for ii in infiles if ii.endswith('.page.pdf'))
@@ -389,33 +401,43 @@ def orient_page(
         270: '⇦'
     }
 
+    pageno = page_number(page_pdf) - 1
+    pdfinfo = context.get_pdfinfo()
+    existing_rotation = pdfinfo[pageno].rotation
+
+    correction = orient_conf.angle % 360
+
     apply_correction = False
-    description = ''
+    action = ''
     if orient_conf.confidence >= options.rotate_pages_threshold:
-        if orient_conf.angle != 0:
+        if correction != 0:
             apply_correction = True
-            description = ' - will rotate'
+            action = ' - will rotate'
         else:
-            description = ' - rotation appears correct'
+            action = ' - rotation appears correct'
     else:
-        if orient_conf.angle != 0:
-            description = ' - confidence too low to rotate'
+        if correction != 0:
+            action = ' - confidence too low to rotate'
         else:
-            description = ' - no change'
+            action = ' - no change'
+
+    facing = ''
+    if existing_rotation != 0:
+        facing = 'with existing rotation {}, '.format(direction.get(
+                existing_rotation, '?'))
+    facing += 'page is facing {}'.format(direction.get(
+            orient_conf.angle, '?'))
 
     log.info(
-        '{0:4d}: page is facing {1}, confidence {2:.2f}{3}'.format(
-            page_number(preview),
-            direction.get(orient_conf.angle, '?'),
-            orient_conf.confidence,
-            description)
+        '{pagenum:4d}: {facing}, confidence {conf:.2f}{action}'.format(
+            pagenum=page_number(preview),
+            facing=facing,
+            conf=orient_conf.confidence,
+            action=action)
     )
 
     re_symlink(page_pdf, output_file, log)
     if apply_correction:
-        pageno = page_number(page_pdf) - 1
-        pdfinfo = context.get_pdfinfo()
-        correction = (orient_conf.angle - pdfinfo[pageno].rotation) % 360
         context.set_rotation(pageno, correction)
 
 

src/ocrmypdf/exec/ghostscript.py

@@ -113,8 +113,6 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
     if not page_dpi:
         page_dpi = res
 
-    autorotate = '/PageByPage' if rotation is None else '/None'
-
     with NamedTemporaryFile(delete=True) as tmp:
         args_gs = [
             'gs',
@@ -127,7 +125,7 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
             '-dLastPage=%i' % pageno,
             '-r{0}x{1}'.format(str(int_res[0]), str(int_res[1])),
             '-o', tmp.name,
-            '-dAutoRotatePages=%s' % autorotate,
+            '-dAutoRotatePages=/None',  # Probably has no effect on raster
             '-f',
             fspath(input_file)
         ]

src/ocrmypdf/optimize.py

@@ -240,8 +240,7 @@ def transcode_jpegs(pike, jpegs, root, log, options):
         compdata = leptonica.CompressedData.open(opt_jpg)
         im_obj = pike.get_object(xref, 0)
         im_obj.write(
-            compdata.read(), pikepdf.Name('/DCTDecode'),
-            pikepdf.Null()
+            compdata.read(), filter=pikepdf.Name('/DCTDecode')
         )
 
 
@@ -281,7 +280,7 @@ def transcode_pngs(pike, pngs, root, log, options):
         if len(compdata) > int(im_obj.stream_dict.Length):
             continue  # If we produced a larger image, don't use
 
-        predictor = pikepdf.Null()
+        predictor = None
         if compdata.predictor > 0:
             predictor = pikepdf.Dictionary({'/Predictor': compdata.predictor})
 
@@ -304,7 +303,10 @@ def transcode_pngs(pike, pngs, root, log, options):
             elif compdata.spp == 4:
                 cs = pikepdf.Name('/DeviceCMYK')
         im_obj.ColorSpace = cs
-        im_obj.write(compdata.read(), pikepdf.Name('/FlateDecode'), predictor)
+        im_obj.write(
+            compdata.read(),
+            filter=pikepdf.Name('/FlateDecode'), decode_parms=predictor
+        )
 
 
 def optimize(

tests/conftest.py

@@ -145,7 +145,8 @@ def check_ocrmypdf(input_file, output_file, *args, env=None):
 
 
 @pytest.helpers.register
-def run_ocrmypdf(input_file, output_file, *args, env=None):
+def run_ocrmypdf(input_file, output_file, *args, env=None,
+        universal_newlines=True):
     "Run ocrmypdf and let caller deal with results"
 
     if env is None:
@@ -155,7 +156,7 @@ def run_ocrmypdf(input_file, output_file, *args, env=None):
              [str(input_file), str(output_file)]
     p = Popen(
         p_args, close_fds=True, stdout=PIPE, stderr=PIPE,
-        universal_newlines=True, env=env)
+        universal_newlines=universal_newlines, env=env)
     out, err = p.communicate()
     #print(err)
 
@@ -168,3 +169,21 @@ def first_page_dimensions(pdf):
     info = pdfinfo.PdfInfo(pdf)
     page0 = info[0]
     return (page0.width_inches, page0.height_inches)
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--runslow", action="store_true", default=False,
+        help=("run slow tests only useful for development (unlikely to be "
+              "useful for downstream packagers)")
+    )
+
+
+def pytest_collection_modifyitems(config, items):
+    if config.getoption("--runslow"):
+        # --runslow given in cli: do not skip slow tests
+        return
+    skip_slow = pytest.mark.skip(reason="need --runslow option to run")
+    for item in items:
+        if "slow" in item.keywords:
+            item.add_marker(skip_slow)

tests/resources/README.rst

@@ -146,8 +146,9 @@ Assemblies
 
 These test resources are assemblies or derivatives from other previously mentioned files, released under the same license terms as their input files.
 
-- baiona_gray.png (from baiona.png)
-- baiona_colormapped.png (from baiona.png)
+- baiona_gray.png (from baiona.png, grayscale version)
+- baiona_colormapped.png (from baiona.png, palette version)
+- baiona_alpha.png (from baiona.png, RGB+A version)
 - cardinal.pdf (four cardinal directions, baked-in rotated copies of linn.png)
 - ccitt.pdf (linn.png, converted to CCITT encoding)
 - encrypted_algo4.pdf (congress.jpg, encrypted with algorithm 4 - not supported by PyPDF2)

tests/test_main.py

@@ -691,6 +691,7 @@ def test_no_contents(spoof_tesseract_noop, resources, outpdf):
 @pytest.mark.parametrize('image', [
     'baiona.png',
     'baiona_gray.png',
+    'baiona_alpha.png',
     'congress.jpg'
     ])
 def test_compression_preserved(spoof_tesseract_noop, ocrmypdf_exec,
@@ -699,7 +700,6 @@ def test_compression_preserved(spoof_tesseract_noop, ocrmypdf_exec,
     output_file = str(outpdf)
 
     im = Image.open(input_file)
-
     # Runs: ocrmypdf - output.pdf < testfile
     with open(input_file, 'rb') as input_stream:
         p_args = ocrmypdf_exec + [
@@ -710,7 +710,12 @@ def test_compression_preserved(spoof_tesseract_noop, ocrmypdf_exec,
             stdin=input_stream, env=spoof_tesseract_noop)
         out, err = p.communicate()
 
-        assert p.returncode == ExitCode.ok
+        if im.mode in ('RGBA', 'LA'):
+            # If alpha image is input, expect an error
+            assert p.returncode != ExitCode.ok and b'alpha' in err
+            return
+
+        assert p.returncode == ExitCode.ok, err.decode('utf-8')
 
     pdfinfo = PdfInfo(output_file)
 
@@ -754,7 +759,7 @@ def test_compression_changed(spoof_tesseract_noop, ocrmypdf_exec,
             stdin=input_stream, env=spoof_tesseract_noop)
         out, err = p.communicate()
 
-        assert p.returncode == ExitCode.ok
+        assert p.returncode == ExitCode.ok, err
 
     pdfinfo = PdfInfo(output_file)
 

tests/test_rotation.py

@@ -16,12 +16,17 @@
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
 import logging
+from io import BytesIO
 
+from PIL import Image
 import pytest
+import img2pdf
+import pikepdf
 
 from ocrmypdf import leptonica
 from ocrmypdf.pdfinfo import PdfInfo
 from ocrmypdf.exec import ghostscript
+from ocrmypdf.helpers import fspath
 
 
 # pytest.helpers is dynamic
@@ -51,7 +56,8 @@ def check_monochrome_correlation(
             return
         ghostscript.rasterize_pdf(
             pdf, png, xres=100, yres=100,
-            raster_device='pngmono', log=gslog, pageno=pageno)
+            raster_device='pngmono', log=gslog, pageno=pageno,
+            rotation=0)
 
     rasterize(reference_pdf, reference_pageno, reference_png)
     rasterize(test_pdf, test_pageno, test_png)
@@ -83,6 +89,7 @@ def test_monochrome_correlation(resources, outdir):
     assert corr > 0.90
 
 
+@pytest.mark.slow
 @pytest.mark.parametrize('renderer', RENDERERS)
 def test_autorotate(spoof_tesseract_cache, renderer, resources, outdir):
     # cardinal.pdf contains four copies of an image rotated in each cardinal
@@ -171,3 +178,46 @@ def test_rotate_deskew_timeout(resources, outdir):
 
     # Confirm that the page still got deskewed
     assert correlation > 0.50
+
+
+@pytest.mark.slow
+@pytest.mark.parametrize('page_angle', (0, 90, 180, 270))
+@pytest.mark.parametrize('image_angle', (0, 90, 180, 270))
+def test_rotate_page_level(image_angle, page_angle, resources, outdir):
+
+    def make_rotate_test(prefix, image_angle, page_angle):
+        im = Image.open(fspath(resources / 'typewriter.png'))
+        if image_angle != 0:
+            ccw_angle = -image_angle % 360
+            im = im.transpose(getattr(Image, 'ROTATE_{}'.format(ccw_angle)))
+        memimg = BytesIO()
+        im.save(memimg, format='PNG')
+        memimg.seek(0)
+        mempdf = BytesIO()
+        img2pdf.convert(
+            memimg.read(),
+            layout_fun=img2pdf.get_fixed_dpi_layout_fun((200, 200)),
+            outputstream=mempdf
+        )
+        mempdf.seek(0)
+        pike = pikepdf.open(mempdf)
+        pike.pages[0].Rotate = page_angle
+        target = outdir / '{}_{}_{}.pdf'.format(prefix, image_angle, page_angle)
+        pike.save(target)
+        return target
+
+    reference = make_rotate_test('ref', 0, 0)
+    test = make_rotate_test('test', image_angle, page_angle)
+    out = test.with_suffix('.out.pdf')
+
+    p, _, err = run_ocrmypdf(
+        test, out,
+        '-O0',
+        '--rotate-pages',
+        '--rotate-pages-threshold', '0.001',
+        universal_newlines=False
+    )
+    err = err.decode('utf-8', errors='replace')
+    assert p.returncode == 0, err
+
+    assert check_monochrome_correlation(outdir, reference, 1, out, 1) > 0.2