v7.0.4 notes

Error was
TypeError: unsupported operand type(s) for -: 'tuple' and 'int'

.travis.yml

@@ -1,46 +1,70 @@
-dist: trusty
-language: python
 cache:
   pip: true
   directories:
   - $HOME/Library/Caches/Homebrew
 
-addons:
-  apt:
-    update: true
-    sources:
-    - sourceline: 'ppa:alex-p/tesseract-ocr'
-    - sourceline: 'ppa:heyarje/libav-11'
-    - sourceline: 'ppa:vshn/ghostscript'
-    packages:
-    - ghostscript
-    - libavcodec56
-    - libavformat56
-    - libavutil54
-    - libexempi3
-    - libffi-dev
-    - pngquant
-    - poppler-utils
-    - qpdf
-    - tesseract-ocr
-    - tesseract-ocr-deu
-    - tesseract-ocr-eng
-    - tesseract-ocr-fra
-
 matrix:
   include:
     - os: linux
+      dist: trusty
       sudo: required
       language: python
       python: "3.5"
+      env:
+        - DIST=trusty
+      addons: &trusty_apt
+        apt:
+          update: true
+          sources:
+          - sourceline: 'ppa:alex-p/tesseract-ocr'
+          - sourceline: 'ppa:heyarje/libav-11'
+          - sourceline: 'ppa:vshn/ghostscript'
+          packages:
+          - ghostscript
+          - libavcodec56
+          - libavformat56
+          - libavutil54
+          - libexempi3
+          - libffi-dev
+          - pngquant
+          - poppler-utils
+          - qpdf
+          - tesseract-ocr
+          - tesseract-ocr-deu
+          - tesseract-ocr-eng
+          - tesseract-ocr-fra
     - os: linux
+      dist: trusty
       sudo: required
       language: python
       python: "3.6"
+      env:
+        - DIST=trusty
+      addons: *trusty_apt
     - os: linux
+      dist: xenial
       sudo: required
       language: python
-      python: "3.7-dev"
+      python: "3.7"
+      env:
+        - DIST=xenial
+      addons:
+        apt:
+          update: true
+          sources:
+          - sourceline: 'ppa:alex-p/tesseract-ocr'
+          packages:
+          - ghostscript
+          - libexempi3
+          - libffi-dev
+          - pngquant
+          - poppler-utils
+          - qpdf
+          - tesseract-ocr
+          - tesseract-ocr-deu
+          - tesseract-ocr-eng
+          - tesseract-ocr-fra
+          - unpaper
     - os: osx
       osx_image: xcode8
       language: generic
@@ -53,9 +77,11 @@ before_install: |
   if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
     pip3 install --upgrade pip
     pip3 install --upgrade wheel
-    mkdir -p packages
-    wget -q 'https://www.dropbox.com/s/vaq0kbwi6e6au80/unpaper_6.1-1.deb?raw=1' -O packages/unpaper_6.1-1.deb
-    sudo dpkg -i packages/unpaper_6.1-1.deb
+    if [[ "$DIST" == "trusty" ]]; then
+      mkdir -p packages
+      wget -q 'https://www.dropbox.com/s/vaq0kbwi6e6au80/unpaper_6.1-1.deb?raw=1' -O packages/unpaper_6.1-1.deb
+      sudo dpkg -i packages/unpaper_6.1-1.deb
+    fi
   elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
     brew update && brew bundle --file=.travis/Brewfile
     pip3 install --upgrade pip

docs/cookbook.rst

@@ -61,7 +61,7 @@ If the page is "just a little off horizontal", like a crooked picture, then you
 OCR languages other than English
 """"""""""""""""""""""""""""""""
 
-By default OCRmyPDF assumes the document is English. 
+By default OCRmyPDF assumes the document is English.
 
 .. code-block:: bash
 
@@ -127,7 +127,7 @@ OCRmyPDF perform some image processing on each page of a PDF, if desired.  The s
 * ``--remove-background`` attempts to detect and remove a noisy background from grayscale or color images.  Monochrome images are ignored. This should not be used on documents that contain color photos as it may remove them.
 
 * ``--deskew`` will correct pages were scanned at a skewed angle by rotating them back into place.  Skew determination and correction is performed using `Postl's variance of line sums <http://www.leptonica.com/skew-measurement.html>`_ algorithm as implemented in `Leptonica <http://www.leptonica.com/index.html>`_.
-  
+
 * ``--clean`` uses `unpaper <https://www.flameeyes.eu/projects/unpaper>`_ to clean up pages before OCR, but does not alter the final output.  This makes it less likely that OCR will try to find text in background noise.
 
 * ``--clean-final`` uses unpaper to clean up pages before OCR and inserts the page into the final output.  You will want to review each page to ensure that unpaper did not remove something important.
@@ -176,7 +176,7 @@ To redo OCR on a file OCRed with other OCR software or a previous version of OCR
 
     ocrmypdf --force-ocr input.pdf output.pdf
 
-Note that the method above will force rasterization of all pages, potentially reducing quality or losing vector content. 
+Note that the method above will force rasterization of all pages, potentially reducing quality or losing vector content.
 
 To ensure quality is preserved, one could extract all of the images and rebuild the PDF for a lossless transformation. This recipe does not work when PDFs contain multiple images per page, as many do in practice. It will also lose any page rotation information.
 
@@ -198,3 +198,19 @@ The `Image processing`_ features can improve OCR quality.
 Rotating pages and deskewing helps to ensure that the page orientation is correct before OCR begins. Removing the background and/or cleaning the page can also improve results. The ``--oversample DPI`` argument can be specified to resample images to higher resolution before attempting OCR; this can improve results as well.
 
 OCR quality will suffer if the resolution of input images is not correct (since the range of pixel sizes that will be checked for possible fonts will also be incorrect).
+
+
+PDF optimization
+----------------
+
+By default OCRmyPDF will attempt to perform lossless optimizations on the images inside PDFs after OCR is complete. Optimization is performed even if no OCR text is found.
+
+The ``--optimize N`` (short form ``-O``) argument controls optimization, where ``N`` ranges from 0 to 3. ``--optimize 0`` disables optimizations. ``1`` enables lossless optimizations that can be performed safely with no quality loss. ``2`` enables lossy optimizations such as image color quantizations. ``3`` enables more aggressive optimizations and targets a lower JPEG quality.
+
+Optimization is improved when a JBIG2 encoder is available and when ``pngquant`` is installed. If either of these components are missing, then some types of images will not be optimized.
+
+Currently optimization attempts to find more efficient encodings for images. The types of optimization available may expand over time. By default, OCRmyPDF compresses data streams inside PDFs, and will change inefficient encodings to more modern versions. A program like ``qpdf`` can be used to change encodings, e.g. to inspect the internals fo a PDF.
+
+.. code-block:: bash
+
+    ocrmypdf --optimize 3 in.pdf out.pdf  # Make it as small as possible

docs/index.rst

@@ -18,6 +18,7 @@ PDF is the best format for storing and exchanging scanned documents.  Unfortunat
    release_notes
    installation
    languages
+   jbig2
 
 .. toctree::
    :caption: Usage
@@ -36,4 +37,3 @@ Indices and tables
 * :ref:`genindex`
 * :ref:`modindex`
 * :ref:`search`
-

docs/installation.rst

@@ -11,11 +11,14 @@ The easiest way to install OCRmyPDF to follow the steps for your operating syste
 If you want to use the latest version of OCRmyPDF, your best bet is to install the most recent version your platform provides, and then upgrade that version by installing the Python binary wheels.
 
 .. contents:: Platform-specific steps
-    :depth: 1
+    :depth: 2
     :local:
 
-Installing on Debian and Ubuntu 16.10 or newer
-----------------------------------------------
+Installing on Linux
+-------------------
+
+Debian and Ubuntu 16.10 or newer
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 .. |deb-stable| image:: https://repology.org/badge/version-for-repo/debian_stable/ocrmypdf.svg
     :alt: Debian 9 stable ("stretch")
@@ -52,17 +55,176 @@ Users of Debian 9 ("stretch") or later or Ubuntu 16.10 or later may simply
 
     apt-get install ocrmypdf
 
-For full details on version availability, check the `Debian Package Tracker <https://tracker.debian.org/pkg/ocrmypdf>`_ or `Ubuntu launchpad.net <https://launchpad.net/ocrmypdf>`_.
+As indicated in the table above, Debian and Ubuntu releases may lag behind the latest version. If the version available for your platform is out of date, you could opt to install the latest version from source. See `Installing HEAD revision from sources`_.
 
-If the version available for your platform is out of date, you could opt to install the latest version from source. See `Installing HEAD revision from sources`_.
+For full details on version availability for your platform, check the `Debian Package Tracker <https://tracker.debian.org/pkg/ocrmypdf>`_ or `Ubuntu launchpad.net <https://launchpad.net/ocrmypdf>`_.
 
 .. note::
 
-    OCRmyPDF for Debian and Ubuntu currently omit the JBIG2 encoder. OCRmyPDF works fine without it but will produce larger output files. If you build jbig2enc from source, ocrmypdf 7.0.0 and later will automatically detect it on the ``PATH``. See `Optional: installing the JBIG2 encoder`_.
+    OCRmyPDF for Debian and Ubuntu currently omit the JBIG2 encoder. OCRmyPDF works fine without it but will produce larger output files. If you build jbig2enc from source, ocrmypdf 7.0.0 and later will automatically detect it (specifically the ``jbig2`` binary) on the ``PATH``. To add JBIG2 encoding, see `Installing the JBIG2 encoder`_.
 
+Installing the latest version on Ubuntu 18.04 LTS
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Installing on macOS with Homebrew
----------------------------------
+Ubuntu 18.04 includes ocrmypdf 6.1.2. To install a more recent version, first
+install the system version to get most of the dependencies:
+
+.. code-block:: bash
+
+    sudo apt-get update
+    sudo apt-get install \
+        ocrmypdf \
+        python3-pip
+
+There are a few dependency changes between ocrmypdf 6.1.2 and 7.x. Let's get
+these, too.
+
+.. code-block:: bash
+
+    sudo apt-get install \
+        libexempi3 \
+        pngquant
+
+Then install the most recent ocrmypdf for the local user and set the user's ``PATH`` to check for the user's Python packages.
+
+.. code-block:: bash
+
+    export PATH=$HOME/.local/bin:$PATH
+    pip3 install --user ocrmypdf
+
+To add JBIG2 encoding, see `Installing the JBIG2 encoder`_.
+
+Ubuntu 16.04 LTS
+^^^^^^^^^^^^^^^^
+
+No package is currently available for Ubuntu 16.04, but you can install the dependencies manually:
+
+.. code-block:: bash
+
+    sudo apt-get update
+    sudo apt-get install \
+        ghostscript \
+        libexempi3 \
+        pngquant \
+        python3-cffi \
+        python3-pip \
+        qpdf \
+        tesseract-ocr \
+        unpaper
+
+If you wish install OCRmyPDF for the current user, and ensure that the ``PATH``
+environment variable contains ``$HOME/.local/bin``.
+
+.. code-block:: bash
+
+    export PATH=$HOME/.local/bin:$PATH
+    pip3 install --user ocrmypdf
+
+Alternately, you can install ocrmypdf system-wide. (Not recommended.)
+
+.. code-block:: bash
+
+    sudo pip3 install ocrmypdf
+
+At your option, you may upgrade Ubuntu 16.04 LTS to Tesseract 4.0 for improved OCR results.
+
+.. code-block:: bash
+
+    sudo apt-get install -y software-properties-common python-software-properties
+    sudo add-apt-repository ppa:alex-p/tesseract-ocr -y
+    sudo apt-get update
+    sudo apt-get upgrade tesseract-ocr
+
+To add JBIG2 encoding, see `Installing the JBIG2 encoder`_.
+
+Ubuntu 14.04 LTS
+^^^^^^^^^^^^^^^^
+
+Installing on Ubuntu 14.04 LTS (trusty) is more difficult than some other options, because it is older and does not provide ``pip``.
+
+Update apt-get:
+
+.. code-block:: bash
+
+    sudo apt-get update
+
+Install system dependencies:
+
+.. code-block:: bash
+
+    sudo apt-get install \
+        software-properties-common python-software-properties \
+        zlib1g-dev \
+        libexempi3 \
+        libjpeg-dev \
+        libffi-dev \
+        pngquant \
+        qpdf
+
+We will need backports of Ghostscript 9.16, libav-11 (for unpaper 6.1), Tesseract 4.00 (alpha), and Python 3.6. This will replace Ghostscript and Tesseract 3.x on your system. Python 3.6 will be installed alongside the system Python 3.4.
+
+If you prefer to not modify your system in this matter, consider using a Docker container.
+
+.. code-block:: bash
+
+    sudo add-apt-repository ppa:vshn/ghostscript -y
+    sudo add-apt-repository ppa:heyarje/libav-11 -y
+    sudo add-apt-repository ppa:alex-p/tesseract-ocr -y
+    sudo add-apt-repository ppa:jonathonf/python-3.6 -y
+
+    sudo apt-get update
+
+    sudo apt-get install \
+        python3.6-dev \
+        ghostscript \
+        tesseract-ocr \
+        tesseract-ocr-eng \
+        libavformat56 libavcodec56 libavutil54 \
+        wget
+
+Now we need to install ``pip`` and let it install ocrmypdf:
+
+.. code-block:: bash
+
+    curl https://bootstrap.pypa.io/ez_setup.py -o - | python3.6 && python3.6 -m easy_install pip
+    pip3.6 install ocrmypdf
+
+The ``wget`` command will download a program and run it.
+
+These installation instructions omit the optional dependency ``unpaper``, which is only available at version 0.4.2 in Ubuntu 14.04. The author could not find a backport of ``unpaper``, and created a .deb package to do the job of installing unpaper 6.1 (for x86 64-bit only):
+
+.. code-block:: bash
+
+    wget -q 'https://www.dropbox.com/s/vaq0kbwi6e6au80/unpaper_6.1-1.deb?raw=1' -O unpaper_6.1-1.deb
+    sudo dpkg -i unpaper_6.1-1.deb
+
+To add JBIG2 encoding, see `Installing the JBIG2 encoder`_.
+
+ArchLinux
+^^^^^^^^^
+
+.. image:: https://repology.org/badge/version-for-repo/aur/ocrmypdf.svg
+    :alt: ArchLinux
+    :target: https://repology.org/metapackage/ocrmypdf
+
+The author is aware of an `ArchLinux package for ocrmypdf <https://aur.archlinux.org/packages/ocrmypdf/>`_. It seems like the following command might work.
+
+.. code-block:: bash
+
+    pacman -S ocrmypdf
+
+Other Linux packages
+^^^^^^^^^^^^^^^^^^^^
+
+See the `Repology <https://repology.org/metapackage/ocrmypdf/versions>`_ page.
+
+In general, first install the OCRmyPDF package for your system, then optionally use the procedure `Installing with Python pip`_ to install a more recent version.
+
+Installing on macOS
+-------------------
+
+Homebrew
+^^^^^^^^
 
 .. image:: https://img.shields.io/homebrew/v/ocrmypdf.svg
     :alt: homebrew
@@ -82,6 +244,58 @@ OCRmyPDF is now a standard `Homebrew <https://brew.sh>`_ formula. To install on
 
     Users who previously installed OCRmyPDF from the private tap should switch to the mainline version (``brew untap jbarlow83/ocrmypdf``) and install from there.
 
+Manual installation on macOS
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+These instructions probably work on all macOS supported by Homebrew.
+
+If it's not already present, `install Homebrew <http://brew.sh/>`_.
+
+Update Homebrew:
+
+.. code-block:: bash
+
+    brew update
+
+Install or upgrade the required Homebrew packages, if any are missing. To do this, download the ``Brewfile`` that lists all of the dependencies to the current directory, and run ``brew bundle`` to process them (installing or upgrading as needed). ``Brewfile`` is a plain text file.
+
+.. code-block:: bash
+
+    wget https://github.com/jbarlow83/OCRmyPDF/raw/master/.travis/Brewfile
+    brew bundle
+
+This will include the English, French, German and Spanish language packs. If you need other languages you can optionally install them all:
+
+.. _macos-all-languages:
+
+.. code-block:: bash
+
+    brew install tesseract --with-all-languages  # Option 2: for all language packs
+
+Update the homebrew pip:
+
+.. code-block:: bash
+
+    pip3 install --upgrade pip
+
+You can then install OCRmyPDF from PyPI, for the current user:
+
+.. code-block:: bash
+
+    pip3 install --user ocrmypdf
+
+or system-wide:
+
+.. code-block:: bash
+
+    pip3 install ocrmypdf
+
+The command line program should now be available:
+
+.. code-block:: bash
+
+    ocrmypdf --help
+
 .. _docker-install:
 
 Installing the Docker image
@@ -185,206 +399,6 @@ Or in the wonderful `fish shell <https://fishshell.com/>`_:
 
     The ocrmypdf Docker images are designed for application delivery, to enable use of OCRmyPDF without fussing with dependencies. ``docker run --rm`` argument tells Docker to delete the container after it runs, because each container is only good for a single job. The Docker image is not designed for use as a persistent web service or for use on Amazon EC2 Container Service (AWS ECS).
 
-Manual installation on macOS
-----------------------------
-
-These instructions probably work on all macOS supported by Homebrew.
-
-If it's not already present, `install Homebrew <http://brew.sh/>`_.
-
-Update Homebrew:
-
-.. code-block:: bash
-
-    brew update
-
-Install or upgrade the required Homebrew packages, if any are missing. To do this, download the ``Brewfile`` that lists all of the dependencies to the current directory, and run ``brew bundle`` to process them (installing or upgrading as needed). ``Brewfile`` is a plain text file.
-
-.. code-block:: bash
-
-    wget https://github.com/jbarlow83/OCRmyPDF/raw/master/.travis/Brewfile
-    brew bundle
-
-This will include the English, French, German and Spanish language packs. If you need other languages you can optionally install them all:
-
-.. _macos-all-languages:
-
-.. code-block:: bash
-
-    brew install tesseract --with-all-languages  # Option 2: for all language packs
-
-Update the homebrew pip:
-
-.. code-block:: bash
-
-    pip3 install --upgrade pip
-
-You can then install OCRmyPDF from PyPI, for the current user:
-
-.. code-block:: bash
-
-    pip3 install --user ocrmypdf
-
-or system-wide:
-
-.. code-block:: bash
-
-    pip3 install ocrmypdf
-
-The command line program should now be available:
-
-.. code-block:: bash
-
-    ocrmypdf --help
-
-Installing the latest version on Ubuntu 18.04 LTS
--------------------------------------------------
-
-Ubuntu 18.04 includes ocrmypdf 6.1.2. To install a more recent version, first
-install the system version to get most of the dependencies:
-
-.. code-block:: bash
-
-    sudo apt-get update
-    sudo apt-get install \
-        ocrmypdf \
-        python3-pip
-
-There are a few dependency changes between ocrmypdf 6.1.2 and 7.x. Let's get
-these, too.
-
-.. code-block:: bash
-
-    sudo apt-get install \
-        libexempi3 \
-        pngquant
-
-Then install the most recent ocrmypdf for the local user and set the user's ``PATH`` to check for the user's Python packages.
-
-.. code-block:: bash
-
-    export PATH=$HOME/.local/bin:$PATH
-    pip3 install --user ocrmypdf
-
-To add JBIG2 encoding, see `Optional: installing the JBIG2 encoder`_.
-
-Installing on Ubuntu 16.04 LTS
-------------------------------
-
-No package is currently available for Ubuntu 16.04, but you can install the dependencies manually:
-
-.. code-block:: bash
-
-    sudo apt-get update
-    sudo apt-get install \
-        ghostscript \
-        libexempi3 \
-        pngquant \
-        python3-cffi \
-        python3-pip \
-        qpdf \
-        tesseract-ocr \
-        unpaper
-
-If you wish install OCRmyPDF for the current user, and ensure that the ``PATH``
-environment variable contains ``$HOME/.local/bin``.
-
-.. code-block:: bash
-
-    export PATH=$HOME/.local/bin:$PATH
-    pip3 install --user ocrmypdf
-
-Alternately, you can install ocrmypdf system-wide. (Not recommended.)
-
-.. code-block:: bash
-
-    sudo pip3 install ocrmypdf
-
-At your option, you may upgrade Ubuntu 16.04 LTS to Tesseract 4.0 for improved OCR results.
-
-.. code-block:: bash
-
-    sudo apt-get install -y software-properties-common python-software-properties
-    sudo add-apt-repository ppa:alex-p/tesseract-ocr -y
-    sudo apt-get update
-    sudo apt-get upgrade tesseract-ocr
-
-To add JBIG2 encoding, see `Optional: installing the JBIG2 encoder`_.
-
-Installing on Ubuntu 14.04 LTS
-------------------------------
-
-Installing on Ubuntu 14.04 LTS (trusty) is more difficult than some other options, because it is older and does not provide ``pip``.
-
-Update apt-get:
-
-.. code-block:: bash
-
-    sudo apt-get update
-
-Install system dependencies:
-
-.. code-block:: bash
-
-    sudo apt-get install \
-        software-properties-common python-software-properties \
-        zlib1g-dev \
-        libexempi3 \
-        libjpeg-dev \
-        libffi-dev \
-        pngquant \
-        qpdf
-
-We will need backports of Ghostscript 9.16, libav-11 (for unpaper 6.1), Tesseract 4.00 (alpha), and Python 3.6. This will replace Ghostscript and Tesseract 3.x on your system. Python 3.6 will be installed alongside the system Python 3.4.
-
-If you prefer to not modify your system in this matter, consider using a Docker container.
-
-.. code-block:: bash
-
-    sudo add-apt-repository ppa:vshn/ghostscript -y
-    sudo add-apt-repository ppa:heyarje/libav-11 -y
-    sudo add-apt-repository ppa:alex-p/tesseract-ocr -y
-    sudo add-apt-repository ppa:jonathonf/python-3.6 -y
-
-    sudo apt-get update
-
-    sudo apt-get install \
-        python3.6-dev \
-        ghostscript \
-        tesseract-ocr \
-        tesseract-ocr-eng \
-        libavformat56 libavcodec56 libavutil54 \
-        wget
-
-Now we need to install ``pip`` and let it install ocrmypdf:
-
-.. code-block:: bash
-
-    curl https://bootstrap.pypa.io/ez_setup.py -o - | python3.6 && python3.6 -m easy_install pip
-    pip3.6 install ocrmypdf
-
-The ``wget`` command will download a program and run it.
-
-These installation instructions omit the optional dependency ``unpaper``, which is only available at version 0.4.2 in Ubuntu 14.04. The author could not find a backport of ``unpaper``, and created a .deb package to do the job of installing unpaper 6.1 (for x86 64-bit only):
-
-.. code-block:: bash
-
-    wget -q 'https://www.dropbox.com/s/vaq0kbwi6e6au80/unpaper_6.1-1.deb?raw=1' -O unpaper_6.1-1.deb
-    sudo dpkg -i unpaper_6.1-1.deb
-
-To add JBIG2 encoding, see `Optional: installing the JBIG2 encoder`_.
-
-
-Installing on ArchLinux
------------------------
-
-The author is aware of an `ArchLinux package for ocrmypdf <https://aur.archlinux.org/packages/ocrmypdf/>`_. It seems like the following command might work.
-
-.. code-block:: bash
-
-    pacman -S ocrmypdf
-
-
 Installing on Windows
 ---------------------
 
@@ -410,7 +424,11 @@ where /c/Users/sampleuser is a Unix representation of the Windows path C:\\Users
 Installing with Python pip
 --------------------------
 
-First, install `your platform's version <https://repology.org/metapackage/ocrmypdf/versions>`_ of ``ocrmypdf``, if available, as a way of ensuring that external dependencies are (mostly) satisified, even though the platform version may be out of date. Use ``ocrmypdf --version`` to confirm what version was installed.
+OCRmyPDF is delivered by PyPI because it is a convenient way to install the latest version. However, PyPI and ``pip`` cannot address the fact that ``ocrmypdf`` depends on certain non-Python system libraries and programs being instsalled.
+
+For best results, first install `your platform's version <https://repology.org/metapackage/ocrmypdf/versions>`_ of ``ocrmypdf``, using the instructions elsewhere in this document. Then you can use ``pip`` to get the latest version if your platform version is out of date. Chances are that this will satisfy most dependencies.
+
+Use ``ocrmypdf --version`` to confirm what version was installed.
 
 Then you can install the latest OCRmyPDF from the Python wheels. First try:
 
@@ -429,12 +447,12 @@ Since ``pip3 install --user`` does not work correctly on some platforms, notably
 Requirements for pip and HEAD install
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-OCRmyPDF currently requires these external programs and libraries to be installed:
+OCRmyPDF currently requires these external programs and libraries to be installed, and must be satisfied using the operating system package manager. ``pip`` cannot provide them.
 
 - Python 3.5 or newer
 - Ghostscript 9.15 or newer
 - libexempi3 2.2.0 or newer
-- qpdf 7.0.0 or newer
+- qpdf 8.1.0 or newer
 - Tesseract 3.04 or newer
 
 As of ocrmypdf 7.0.0, the following versions are recommended:
@@ -466,6 +484,8 @@ Installing HEAD revision from sources
 
 If you have ``git`` and Python 3.5 or newer installed, you can install from source. When the ``pip`` installer runs, it will alert you if dependencies are missing.
 
+If you prefer to build every from source, you will need to `build pikepdf from source <https://pikepdf.readthedocs.io/en/latest/installation.html#building-from-source>`_. First ensure you can build and install pikepdf.
+
 To install the HEAD revision from sources in the current Python 3 environment:
 
 .. code-block:: bash
@@ -488,8 +508,8 @@ You may find it easiest to install in a virtual environment, rather than system-
     cd OCRmyPDF
     pip3 install .
 
-However, ``ocrmypdf`` will only be accessible on the system PATH after
-you activate the virtual environment.
+However, ``ocrmypdf`` will only be accessible on the system PATH
+when you activate the virtual environment.
 
 To run the program:
 
@@ -502,27 +522,18 @@ need to be installed. The script requires specific versions of the
 dependencies. Older version than the ones mentioned in the release notes
 are likely not to be compatible to OCRmyPDF.
 
-To add JBIG2 encoding, see `Optional: installing the JBIG2 encoder`_.
+For development
+~~~~~~~~~~~~~~~
 
-Other Linux packages
---------------------
-
-See the `Repology <https://repology.org/metapackage/ocrmypdf/versions>`_ page.
-
-
-Optional: installing the JBIG2 encoder
---------------------------------------
-
-Most Linux distributions do not include a JBIG2 encoder since JBIG2 encoding was patented for a long time. All known JBIG2 US patents have expired as of 2017, but it is possible that unknown patents exist.
-
-To build a JBIG2 encoder from source:
+To install all of the development and test requirements:
 
 .. code-block:: bash
 
-    git clone https://github.com/agl/jbig2enc
-    cd jbig2enc
-    ./autogen.sh
-    ./configure && make
-    [sudo] make install
+    git clone -b master https://github.com/jbarlow83/OCRmyPDF.git
+    python3 -m venv
+    source venv/bin/activate
+    cd OCRmyPDF
+    pip install -e .
+    pip install -r dev_requirements.txt -r test_requirements.txt
 
-On macOS, Homebrew packages jbig2enc and OCRmyPDF includes it by default.
+To add JBIG2 encoding, see `Installing the JBIG2 encoder`_.

docs/jbig2.rst

@@ -0,0 +1,20 @@
+Installing the JBIG2 encoder
+==========================
+
+Most Linux distributions do not include a JBIG2 encoder since JBIG2 encoding was patented for a long time. All known JBIG2 US patents have expired as of 2017, but it is possible that unknown patents exist.
+
+JBIG2 encoding is recommended for OCRmyPDF and is used to losslessly create smaller PDFs. If JBIG2 encoding not available, lower quality encodings will be used.
+
+JBIG2 decoding is not patented and is performed automatically by most PDF viewers. It is widely supported has been part of the PDF specification since 2001.
+
+On macOS, Homebrew packages jbig2enc and OCRmyPDF includes it by default. The Docker image for OCRmyPDF also builds its own JBIG2 encoder from source.
+
+For all other Linux, you must build a JBIG2 encoder from source:
+
+.. code-block:: bash
+
+    git clone https://github.com/agl/jbig2enc
+    cd jbig2enc
+    ./autogen.sh
+    ./configure && make
+    [sudo] make install

docs/release_notes.rst

@@ -13,6 +13,28 @@ Note that it is licensed under GPLv3, so scripts that ``import ocrmypdf`` and ar
    find:    [^`]\#([0-9]{1,3})[^0-9]
    replace: `#$1 <https://github.com/jbarlow83/OCRmyPDF/issues/$1>`_
 
+v7.0.4
+------
+
+-   Fix exception thrown when trying to optimize a certain type of PNG embedded
+in a PDF with the ``-O2``
+
+-   Update to pikepdf 0.3.2, to gain support for optimizing some additional
+image types that were previously excluded from optimization (CMYK and grayscale).
+Fixes `#285 <https://github.com/jbarlow83/OCRmyPDF/issues/285>`_.
+
+v7.0.3
+------
+
+-   Fix issue `#284 <https://github.com/jbarlow83/OCRmyPDF/issues/284>`_, an error when parsing inline images that have are also image masks, by upgrading pikepdf to 0.3.1
+
+v7.0.2
+------
+
+-   Fix a regression with ``--rotate-pages`` on pages that already had rotations applied. (`#279 <https://github.com/jbarlow83/OCRmyPDF/issues/279>`_)
+
+-   Improve quality of page rotation in some cases by rasterizing a higher quality preview image. (`#281 <https://github.com/jbarlow83/OCRmyPDF/issues/281>`_)
+
 v7.0.1
 ------
 
@@ -57,8 +79,7 @@ v7.0.0
 
     +   ``PyMuPDF``
 
--   The ``sandwich`` PDF renderer can be used with all supported versions of Tesseract, including that those prior to v3.05 which don't support
-``-c textonly``. (Tesseract v4.0.0 is recommended and more efficient.)
+-   The ``sandwich`` PDF renderer can be used with all supported versions of Tesseract, including that those prior to v3.05 which don't support ``-c textonly``. (Tesseract v4.0.0 is recommended and more efficient.)
 
 -   ``--pdf-renderer auto`` option and the diagnostics used to select a PDF renderer now work better with old versions, but may make different decisions than past versions.
 
@@ -197,7 +218,7 @@ v6.0.0
 v5.7.0
 ------
 
--   Fixed an issue that caused poor CPU utilization on machines more than 4 cores when running Tesseract 4. (Related to issue `#217 <https://github.com/jbarlow83/OCRmyPDF/issues/217>`_.)
+-   Fixed an issue that caused poor CPU utilization on machines with more than 4 cores when running Tesseract 4. (Related to issue `#217 <https://github.com/jbarlow83/OCRmyPDF/issues/217>`_.)
 
 -   The 'hocr' renderer has been improved. The 'sandwich' and 'tesseract' renderers are still better for most use cases, but 'hocr' may be useful for people who work with the PDF.js renderer in English/ASCII languages. (`#225 <https://github.com/jbarlow83/OCRmyPDF/issues/225>`_)
 

requirements.txt

@@ -2,8 +2,8 @@
 # setup.py lists a separate set of requirements that are looser to simplify
 # installation
 cffi == 1.11.5
-img2pdf == 0.2.4
-pikepdf == 0.2.2
+img2pdf == 0.3.0
+pikepdf == 0.3.0
 Pillow >= 5.0.0, != 5.1.0 ; sys_platform == "darwin"
 pycparser == 2.18
 python-xmp-toolkit == 2.0.1

setup.py

@@ -241,6 +241,7 @@ setup(
         'cffi >= 1.9.1',        # to build the leptonica module
         'pytest-runner',        # to enable python setup.py test
         'setuptools_scm',       # so that version will work
+        'setuptools_scm_git_archive'    # enable version from github tarballs
     ],
     use_scm_version={'version_scheme': 'post-release'},
     cffi_modules=[
@@ -249,7 +250,7 @@ setup(
     install_requires=[
         'cffi >= 1.9.1',          # must be a setup and install requirement
         'img2pdf >= 0.2.4, < 0.4',       # pure Python, so track HEAD closely
-        'pikepdf >= 0.2.2, < 0.4',
+        'pikepdf >= 0.3.2, < 0.4',
         'Pillow >= 4.0.0, != 5.1.0 ; sys_platform == "darwin"',
                                   # Pillow < 4 has BytesIO/TIFF bug w/img2pdf 0.2.3
                                   # block 5.1.0, broken wheels

src/ocrmypdf/_pipeline.py

@@ -318,16 +318,6 @@ def marker_pages(
         page.symlink_to(input_file)  # pylint: disable=E1101
 
 
-def split_page(
-        placeholder_file,
-        output_file,
-        log,
-        context):
-    pageno = page_number(placeholder_file) - 1
-    input_pdf = context.get_pdfinfo().filename
-    qpdf.extract_page(input_pdf, output_file, pageno)
-
-
 def ocr_or_skip(
         input_files,
         output_files,
@@ -359,13 +349,13 @@ def rasterize_preview(
         context):
     pageinfo = get_pageinfo(input_file, context)
     options = context.get_options()
-    canvas_dpi = get_canvas_square_dpi(pageinfo, options) / 2
-    page_dpi = get_page_square_dpi(pageinfo, options) / 2
+    canvas_dpi = get_canvas_square_dpi(pageinfo, options)
+    page_dpi = get_page_square_dpi(pageinfo, options)
 
     ghostscript.rasterize_pdf(
         input_file, output_file, xres=canvas_dpi, yres=canvas_dpi,
         raster_device='jpeggray', log=log, page_dpi=(page_dpi, page_dpi),
-        pageno=page_number(input_file), rotation=pageinfo.rotation)
+        pageno=page_number(input_file))
 
 
 def orient_page(
@@ -373,6 +363,22 @@ def orient_page(
         output_file,
         log,
         context):
+    """
+    Work out orientation correct for each page.
+
+    We ask Ghostscript to draw a preview page, which will rasterize with the
+    current /Rotate applied, and then ask Tesseract which way the page is
+    oriented. If the value of /Rotate is correct (e.g., a user already
+    manually fixed rotation), then Tesseract will say the page is pointing
+    up and the correction is zero. Otherwise, the orientation found by
+    Tesseract represents the clockwise rotation, or the counterclockwise
+    correction to rotation.
+
+    When we draw the real page for OCR, we rotate it by the CCW correction,
+    which points it (hopefully) upright. _weave.py takes care of the orienting
+    the image and text layers.
+
+    """
 
     options = context.get_options()
     page_pdf = next(ii for ii in infiles if ii.endswith('.page.pdf'))
@@ -395,33 +401,43 @@ def orient_page(
         270: '⇦'
     }
 
+    pageno = page_number(page_pdf) - 1
+    pdfinfo = context.get_pdfinfo()
+    existing_rotation = pdfinfo[pageno].rotation
+
+    correction = orient_conf.angle % 360
+
     apply_correction = False
-    description = ''
+    action = ''
     if orient_conf.confidence >= options.rotate_pages_threshold:
-        if orient_conf.angle != 0:
+        if correction != 0:
             apply_correction = True
-            description = ' - will rotate'
+            action = ' - will rotate'
         else:
-            description = ' - rotation appears correct'
+            action = ' - rotation appears correct'
     else:
-        if orient_conf.angle != 0:
-            description = ' - confidence too low to rotate'
+        if correction != 0:
+            action = ' - confidence too low to rotate'
         else:
-            description = ' - no change'
+            action = ' - no change'
+
+    facing = ''
+    if existing_rotation != 0:
+        facing = 'with existing rotation {}, '.format(direction.get(
+                existing_rotation, '?'))
+    facing += 'page is facing {}'.format(direction.get(
+            orient_conf.angle, '?'))
 
     log.info(
-        '{0:4d}: page is facing {1}, confidence {2:.2f}{3}'.format(
-            page_number(preview),
-            direction.get(orient_conf.angle, '?'),
-            orient_conf.confidence,
-            description)
+        '{pagenum:4d}: {facing}, confidence {conf:.2f}{action}'.format(
+            pagenum=page_number(preview),
+            facing=facing,
+            conf=orient_conf.confidence,
+            action=action)
     )
 
     re_symlink(page_pdf, output_file, log)
     if apply_correction:
-        pageno = page_number(page_pdf) - 1
-        pdfinfo = context.get_pdfinfo()
-        correction = (orient_conf.angle - pdfinfo[pageno].rotation) % 360
         context.set_rotation(pageno, correction)
 
 

src/ocrmypdf/exec/ghostscript.py

@@ -113,8 +113,6 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
     if not page_dpi:
         page_dpi = res
 
-    autorotate = '/PageByPage' if rotation is None else '/None'
-
     with NamedTemporaryFile(delete=True) as tmp:
         args_gs = [
             'gs',
@@ -127,7 +125,7 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
             '-dLastPage=%i' % pageno,
             '-r{0}x{1}'.format(str(int_res[0]), str(int_res[1])),
             '-o', tmp.name,
-            '-dAutoRotatePages=%s' % autorotate,
+            '-dAutoRotatePages=/None',  # Probably has no effect on raster
             '-f',
             fspath(input_file)
         ]

src/ocrmypdf/optimize.py

@@ -24,11 +24,6 @@ import sys
 from PIL import Image
 
 import pikepdf
-try:
-    from pikepdf import Null  # pikepdf <= 0.2.2
-except ImportError:
-    def Null():               # pikepdf > 0.2.2
-        return None
 
 from ._jobcontext import JobContext
 from . import leptonica
@@ -37,7 +32,7 @@ from .exec import pngquant, jbig2enc
 
 PAGE_GROUP_SIZE = 10
 DEFAULT_JPEG_QUALITY = 75
-DEFAULT_PNG_QUALITY = (65, 75)
+DEFAULT_PNG_QUALITY = 70
 
 
 def img_name(root, xref, ext):
@@ -245,8 +240,7 @@ def transcode_jpegs(pike, jpegs, root, log, options):
         compdata = leptonica.CompressedData.open(opt_jpg)
         im_obj = pike.get_object(xref, 0)
         im_obj.write(
-            compdata.read(), pikepdf.Name('/DCTDecode'),
-            Null()
+            compdata.read(), filter=pikepdf.Name('/DCTDecode')
         )
 
 
@@ -286,7 +280,7 @@ def transcode_pngs(pike, pngs, root, log, options):
         if len(compdata) > int(im_obj.stream_dict.Length):
             continue  # If we produced a larger image, don't use
 
-        predictor = Null()
+        predictor = None
         if compdata.predictor > 0:
             predictor = pikepdf.Dictionary({'/Predictor': compdata.predictor})
 
@@ -309,7 +303,10 @@ def transcode_pngs(pike, pngs, root, log, options):
             elif compdata.spp == 4:
                 cs = pikepdf.Name('/DeviceCMYK')
         im_obj.ColorSpace = cs
-        im_obj.write(compdata.read(), pikepdf.Name('/FlateDecode'), predictor)
+        im_obj.write(
+            compdata.read(),
+            filter=pikepdf.Name('/FlateDecode'), decode_parms=predictor
+        )
 
 
 def optimize(

tests/conftest.py

@@ -145,7 +145,8 @@ def check_ocrmypdf(input_file, output_file, *args, env=None):
 
 
 @pytest.helpers.register
-def run_ocrmypdf(input_file, output_file, *args, env=None):
+def run_ocrmypdf(input_file, output_file, *args, env=None,
+        universal_newlines=True):
     "Run ocrmypdf and let caller deal with results"
 
     if env is None:
@@ -155,7 +156,7 @@ def run_ocrmypdf(input_file, output_file, *args, env=None):
              [str(input_file), str(output_file)]
     p = Popen(
         p_args, close_fds=True, stdout=PIPE, stderr=PIPE,
-        universal_newlines=True, env=env)
+        universal_newlines=universal_newlines, env=env)
     out, err = p.communicate()
     #print(err)
 
@@ -168,3 +169,21 @@ def first_page_dimensions(pdf):
     info = pdfinfo.PdfInfo(pdf)
     page0 = info[0]
     return (page0.width_inches, page0.height_inches)
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--runslow", action="store_true", default=False,
+        help=("run slow tests only useful for development (unlikely to be "
+              "useful for downstream packagers)")
+    )
+
+
+def pytest_collection_modifyitems(config, items):
+    if config.getoption("--runslow"):
+        # --runslow given in cli: do not skip slow tests
+        return
+    skip_slow = pytest.mark.skip(reason="need --runslow option to run")
+    for item in items:
+        if "slow" in item.keywords:
+            item.add_marker(skip_slow)

tests/test_rotation.py

@@ -16,12 +16,17 @@
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
 import logging
+from io import BytesIO
 
+from PIL import Image
 import pytest
+import img2pdf
+import pikepdf
 
 from ocrmypdf import leptonica
 from ocrmypdf.pdfinfo import PdfInfo
 from ocrmypdf.exec import ghostscript
+from ocrmypdf.helpers import fspath
 
 
 # pytest.helpers is dynamic
@@ -51,7 +56,8 @@ def check_monochrome_correlation(
             return
         ghostscript.rasterize_pdf(
             pdf, png, xres=100, yres=100,
-            raster_device='pngmono', log=gslog, pageno=pageno)
+            raster_device='pngmono', log=gslog, pageno=pageno,
+            rotation=0)
 
     rasterize(reference_pdf, reference_pageno, reference_png)
     rasterize(test_pdf, test_pageno, test_png)
@@ -83,6 +89,7 @@ def test_monochrome_correlation(resources, outdir):
     assert corr > 0.90
 
 
+@pytest.mark.slow
 @pytest.mark.parametrize('renderer', RENDERERS)
 def test_autorotate(spoof_tesseract_cache, renderer, resources, outdir):
     # cardinal.pdf contains four copies of an image rotated in each cardinal
@@ -171,3 +178,46 @@ def test_rotate_deskew_timeout(resources, outdir):
 
     # Confirm that the page still got deskewed
     assert correlation > 0.50
+
+
+@pytest.mark.slow
+@pytest.mark.parametrize('page_angle', (0, 90, 180, 270))
+@pytest.mark.parametrize('image_angle', (0, 90, 180, 270))
+def test_rotate_page_level(image_angle, page_angle, resources, outdir):
+
+    def make_rotate_test(prefix, image_angle, page_angle):
+        im = Image.open(fspath(resources / 'typewriter.png'))
+        if image_angle != 0:
+            ccw_angle = -image_angle % 360
+            im = im.transpose(getattr(Image, 'ROTATE_{}'.format(ccw_angle)))
+        memimg = BytesIO()
+        im.save(memimg, format='PNG')
+        memimg.seek(0)
+        mempdf = BytesIO()
+        img2pdf.convert(
+            memimg.read(),
+            layout_fun=img2pdf.get_fixed_dpi_layout_fun((200, 200)),
+            outputstream=mempdf
+        )
+        mempdf.seek(0)
+        pike = pikepdf.open(mempdf)
+        pike.pages[0].Rotate = page_angle
+        target = outdir / '{}_{}_{}.pdf'.format(prefix, image_angle, page_angle)
+        pike.save(target)
+        return target
+
+    reference = make_rotate_test('ref', 0, 0)
+    test = make_rotate_test('test', image_angle, page_angle)
+    out = test.with_suffix('.out.pdf')
+
+    p, _, err = run_ocrmypdf(
+        test, out,
+        '-O0',
+        '--rotate-pages',
+        '--rotate-pages-threshold', '0.001',
+        universal_newlines=False
+    )
+    err = err.decode('utf-8', errors='replace')
+    assert p.returncode == 0, err
+
+    assert check_monochrome_correlation(outdir, reference, 1, out, 1) > 0.2