optimize: use Decode to invert 1bpp PNGs for now

v8.2.0 release notes: optimizer
Merge branch 'feature/optimization-fixes'
2019-03-03 17:50:12 -08:00 · 2019-03-03 15:26:01 -08:00 · 2019-03-03 15:00:20 -08:00 · 2019-03-03 14:59:59 -08:00 · 2019-03-03 14:57:15 -08:00 · 2019-03-03 14:15:20 -08:00
31 changed files with 753 additions and 520 deletions
@@ -0,0 +1,84 @@
+FROM alpine:3.9 as base
+
+FROM base as builder
+
+ENV LANG=C.UTF-8
+
+RUN \
+  echo '@testing http://nl.alpinelinux.org/alpine/edge/testing' >> /etc/apk/repositories \
+  # Add runtime dependencies
+  && apk add --update \
+    python3-dev \
+    py3-setuptools \
+    jbig2enc@testing \
+    ghostscript \
+    qpdf \
+    tesseract-ocr \
+    unpaper \
+    pngquant \
+    libxml2-dev \
+    libxslt-dev \
+    zlib-dev \
+    qpdf-dev \
+    libffi-dev \
+    leptonica-dev \
+    binutils \
+  # Install pybind11 for pikepdf
+  && pip3 install pybind11 \
+  # Install flask for the webservice
+  && pip3 install flask \
+  # Add build dependencies
+  && apk add --virtual build-dependencies \
+    build-base \
+    git
+
+COPY . /app
+
+WORKDIR /app
+
+RUN pip3 install .
+
+FROM base
+
+ENV LANG=C.UTF-8
+
+RUN \
+  echo '@testing http://nl.alpinelinux.org/alpine/edge/testing' >> /etc/apk/repositories \
+  # Add runtime dependencies
+  && apk add --update \
+    python3 \
+    jbig2enc@testing \
+    ghostscript \
+    qpdf \
+    tesseract-ocr \
+    tesseract-ocr-data-deu \
+    tesseract-ocr-data-chi_sim \
+    unpaper \
+    pngquant \
+    libxml2 \
+    libxslt \
+    zlib \
+    qpdf \
+    libffi \
+    leptonica-dev \
+    binutils \
+  && mkdir /app
+
+WORKDIR /app
+
+# Copy build artifacts (python site-packages9
+COPY --from=builder /usr/lib/python3.6/site-packages /usr/lib/python3.6/site-packages
+COPY --from=builder /usr/bin/ocrmypdf /usr/bin/dumppdf.py /usr/bin/latin2ascii.py /usr/bin/pdf2txt.py /usr/bin/img2pdf /usr/bin/chardetect /usr/bin/
+
+# Copy
+COPY --from=builder /app/.docker/webservice.py /app/
+
+# Copy minimal project files to get the test suite.
+COPY --from=builder /app/setup.cfg /app/setup.py /app/README.md /app/
+COPY --from=builder /app/requirements /app/requirements
+COPY --from=builder /app/tests /app/tests
+COPY --from=builder /app/src /app/src
+# Copy PKG-INFO from build artifact in app dir to make setuptools-scm happy
+RUN cp /usr/lib/python3.6/site-packages/ocrmypdf-*.egg-info/PKG-INFO /app
+
+ENTRYPOINT ["/usr/bin/ocrmypdf"]
@@ -1,5 +1,5 @@
 # webservice.py wrapper for OCRmyPDF
-# Copyright (C) 2018 James R. Barlow: github.com/jbarlow83
+# Copyright (C) 2019 James R. Barlow: github.com/jbarlow83
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -17,13 +17,22 @@
 """This is a simple web service/HTTP wrapper for OCRmyPDF

 This may be more convenient than the command line tool for some Docker users.
-Note that OCRmyPDF uses Ghostscript, which is licensed under AGPL3+. While
-OCRmyPDF is under GPL3, this file is distributed under the Affero GPL3+ license,
+Note that OCRmyPDF uses Ghostscript, which is licensed under AGPLv3+. While
+OCRmyPDF is under GPLv3, this file is distributed under the Affero GPLv3+ license,
 to emphasize that SaaS deployments should make sure they comply with
 Ghostscript's license as well as OCRmyPDF's.
 """

-from flask import Flask, Response, flash, request, redirect, url_for, abort, send_from_directory
+from flask import (
+    Flask,
+    Response,
+    flash,
+    request,
+    redirect,
+    url_for,
+    abort,
+    send_from_directory,
+)
 from subprocess import run, PIPE
 from tempfile import TemporaryDirectory
 from werkzeug.utils import secure_filename
@@ -81,7 +90,7 @@ def upload_file():

    return """
    <!doctype html>
-    <title>OCRmyPDF webapp</title>
+    <title>OCRmyPDF webservice</title>
    <h1>Upload a PDF (debug UI)</h1>
    <form method=post enctype=multipart/form-data>
      <label for="args">Command line parameters</label>
@@ -90,7 +99,27 @@ def upload_file():
      <input type=file name=file>
      <input type=submit value=Upload>
    </form>
+    <h4>Notice</h2>
+    <div style="font-size: 70%; max-width: 34em;">
+    <p>This is a webservice wrapper for OCRmyPDF.</p>
+    <p>Copyright 2019 James R. Barlow</p>
+    <p>This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+    </p>
+    <p>This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+    </p>
+    <p>
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see &lt;http://www.gnu.org/licenses/&gt;.
+    </p>
+    </div>
    """

+
 if __name__ == "__main__":
-    app.run(host='0.0.0.0')
+    app.run(host='0.0.0.0', port=5000)
@@ -5,6 +5,7 @@
 *.sublime*
 **/*.pyc
 .*/
+!.git/
 !.docker/
 .ruffus_history.sqlite
 bin/
@@ -225,7 +225,7 @@ OCRmyPDF normally saves its intermediate results to a temporary folder and delet

 If the ``-k`` argument is issued on the command line, OCRmyPDF will keep the temporary folder and print the location, whether it succeeded or failed (provided the Python interpreter did not crash). An example message is:

-.. code-block::
+.. code-block:: none

    Temporary working files saved at:
    /tmp/com.github.ocrmypdf.u20wpz07
@@ -52,7 +52,9 @@ master_doc = 'index'

 # General information about the project.
 project = 'ocrmypdf'
-copyright = '2018, James R. Barlow. Licensed under Creative Commons Attribution-ShareAlike 4.0.'
+copyright = (
+    '2019, James R. Barlow. Licensed under Creative Commons Attribution-ShareAlike 4.0.'
+)
 author = 'James R. Barlow'

 # The version info for the project you're documenting, acts as replacement for
@@ -62,6 +64,7 @@ author = 'James R. Barlow'
 # The short X.Y version.

 import os
+
 on_rtd = os.environ.get('READTHEDOCS') == 'True'

 if on_rtd:
@@ -78,18 +81,16 @@ if on_rtd:
        'pikepdf',
        'pikepdf.models',
        'pikepdf.models.metadata',
-        'ocrmypdf.leptonica'
+        'ocrmypdf.leptonica',
    ]
    sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES)


-from ocrmypdf import __version__ as OCRMYPDF_VERSION
+from pkg_resources import get_distribution, DistributionNotFound

-_version_parts = OCRMYPDF_VERSION.split('.')
-
-version = '.'.join(_version_parts[0:2])
 # The full version, including alpha/beta/rc tags.
-release = OCRMYPDF_VERSION
+release = get_distribution('ocrmypdf').version
+version = '.'.join(release.split('.')[:2])

 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -158,9 +159,7 @@ html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 # further.  For a list of options available for each theme, see the
 # documentation.
 #
-html_theme_options = {
-    'display_version': False
-}
+html_theme_options = {'display_version': False}

 # Add any paths that contain custom themes here, relative to this directory.
 # html_theme_path = []
@@ -273,29 +272,25 @@ htmlhelp_basename = 'ocrmypdfdoc'
 # -- Options for LaTeX output ---------------------------------------------

 latex_elements = {
-     # The paper size ('letterpaper' or 'a4paper').
-     #
-     # 'papersize': 'letterpaper',
-
-     # The font size ('10pt', '11pt' or '12pt').
-     #
-     # 'pointsize': '10pt',
-
-     # Additional stuff for the LaTeX preamble.
-     #
-     # 'preamble': '',
-
-     # Latex figure (float) alignment
-     #
-     # 'figure_align': 'htbp',
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
 }

 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, 'ocrmypdf.tex', 'ocrmypdf Documentation',
-     'James R. Barlow', 'manual'),
+    (master_doc, 'ocrmypdf.tex', 'ocrmypdf Documentation', 'James R. Barlow', 'manual')
 ]

 # The name of an image file (relative to this directory) to place at the top of
@@ -335,10 +330,7 @@ latex_documents = [

 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [
-    (master_doc, 'ocrmypdf', 'ocrmypdf Documentation',
-     [author], 1)
-]
+man_pages = [(master_doc, 'ocrmypdf', 'ocrmypdf Documentation', [author], 1)]

 # If true, show URL addresses after external links.
 #
@@ -351,9 +343,15 @@ man_pages = [
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    (master_doc, 'ocrmypdf', 'ocrmypdf Documentation',
-     author, 'ocrmypdf', 'One line description of project.',
-     'Miscellaneous'),
+    (
+        master_doc,
+        'ocrmypdf',
+        'ocrmypdf Documentation',
+        author,
+        'ocrmypdf',
+        'One line description of project.',
+        'Miscellaneous',
+    )
 ]

 # Documents to append as an appendix to all manuals.
@@ -5,7 +5,7 @@ Basic examples
 --------------

 Help!
-"""""
+^^^^^

 ocrmypdf has built-in help.

@@ -15,28 +15,28 @@ ocrmypdf has built-in help.


 Add an OCR layer and convert to PDF/A
-"""""""""""""""""""""""""""""""""""""
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

 .. code-block:: bash

    ocrmypdf input.pdf output.pdf

 Add an OCR layer and output a standard PDF
-""""""""""""""""""""""""""""""""""""""""""
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

 .. code-block:: bash

    ocrmypdf --output-type pdf input.pdf output.pdf

 Create a PDF/A with all color and grayscale images converted to JPEG
-""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

 .. code-block:: bash

    ocrmypdf --output-type pdfa --pdfa-image-compression jpeg input.pdf output.pdf

 Modify a file in place
-""""""""""""""""""""""
+^^^^^^^^^^^^^^^^^^^^^^

 The file will only be overwritten if OCRmyPDF is successful.

@@ -45,7 +45,7 @@ The file will only be overwritten if OCRmyPDF is successful.
    ocrmypdf myfile.pdf myfile.pdf

 Correct page rotation
-"""""""""""""""""""""
+^^^^^^^^^^^^^^^^^^^^^

 OCR will attempt to automatic correct the rotation of each page. This can help fix a scanning job that contains a mix of landscape and portrait pages.

@@ -58,9 +58,9 @@ You can increase (decrease) the parameter ``--rotate-pages-threshold`` to make p
 If the page is "just a little off horizontal", like a crooked picture, then you want ``--deskew``. ``--rotate-pages`` is for when the cardinal angle is wrong.

 OCR languages other than English
-""""""""""""""""""""""""""""""""
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

-By default OCRmyPDF assumes the document is English.
+OCRmyPDF assumes the document is in English unless told otherwise. OCR quality may be poor if the wrong language is used.

 .. code-block:: bash

@@ -69,8 +69,10 @@ By default OCRmyPDF assumes the document is English.

 Language packs must be installed for all languages specified. See :ref:`Installing additional language packs <lang-packs>`.

+Unfortunately, the Tesseract OCR engine has no ability to detect the language when it is unknown.
+
 Produce PDF and text file containing OCR text
-"""""""""""""""""""""""""""""""""""""""""""""
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

 This produces a file named "output.pdf" and a companion text file named "output.txt".

@@ -79,7 +81,10 @@ This produces a file named "output.pdf" and a companion text file named "output.
    ocrmypdf --sidecar output.txt input.pdf output.pdf

 OCR images, not PDFs
--------------------
+^^^^^^^^^^^^^^^^^^^^
+
+Option: use Tesseract
+"""""""""""""""""""""

 If you are starting with images, you can just use Tesseract directly to convert images to PDFs:

@@ -92,9 +97,12 @@ If you are starting with images, you can just use Tesseract directly to convert
    # When there are multiple images
    tesseract text-file-containing-list-of-image-filenames.txt output-prefix pdf

-Tesseract's PDF output is quite good – OCRmyPDF uses it internally by default. However, OCRmyPDF has many features not available in Tesseract like like image processing, metadata control, and PDF/A generation.
+Tesseract's PDF output is quite good – OCRmyPDF uses it internally, in some cases. However, OCRmyPDF has many features not available in Tesseract like image processing, metadata control, and PDF/A generation.

-Use a program like `img2pdf <https://gitlab.mister-muffin.de/josch/img2pdf>`_ to convert your images to PDFs, and then pipe the results to run ocrmypdf.  The `-` tells ocrmypdf to read standard input.
+Option: use img2pdf
+"""""""""""""""""""
+
+You can also use a program like `img2pdf <https://gitlab.mister-muffin.de/josch/img2pdf>`_ to convert your images to PDFs, and then pipe the results to run ocrmypdf.  The ``-`` tells ocrmypdf to read standard input.

 .. code-block:: bash

@@ -102,6 +110,9 @@ Use a program like `img2pdf <https://gitlab.mister-muffin.de/josch/img2pdf>`_ to

 ``img2pdf`` is recommended because it does an excellent job at generating PDFs without transcoding images.

+Option: use OCRmyPDF (single images only)
+"""""""""""""""""""""""""""""""""""""""""
+
 For convenience, OCRmyPDF can also convert single images to PDFs on its own. If the resolution (dots per inch, DPI) of an image is not set or is incorrect, it can be overridden with ``--image-dpi``. (As 1 inch is 2.54 cm, 1 dpi = 0.39 dpcm).

 .. code-block:: bash
@@ -110,9 +121,10 @@ For convenience, OCRmyPDF can also convert single images to PDFs on its own. If

 If you have multiple images, you must use ``img2pdf`` to convert the images to PDF.

-.. note::
+Not recommended
+"""""""""""""""

-    ImageMagick ``convert`` can also convert a group of images to PDF, but in the author's experience it takes a long time, transcodes unnecessarily and gives poor results.
+We caution against using ImageMagick or Ghostscript to convert images to PDF, since they may transcode images or produce downsampled images, sometimes without warning.

 Image processing
 ----------------
@@ -129,7 +141,7 @@ OCRmyPDF perform some image processing on each page of a PDF, if desired.  The s

 * ``--clean-final`` uses unpaper to clean up pages before OCR and inserts the page into the final output.  You will want to review each page to ensure that unpaper did not remove something important.

-* ``--mask-barcodes`` will "cover up" any barcodes detected in the image of a page. Barcodes are known to confuse Tesseract OCR and interfere with the recognition of text on the same baseline as a barcode. The output file will contain the unaltered image of the barcode.
+* ``--mask-barcodes`` will suppress any barcodes detected in a page image. Barcodes are known to confuse Tesseract OCR and interfere with the recognition of text on the same baseline as a barcode. The output file will contain the unaltered image of the barcode.

 .. note::

@@ -139,8 +151,8 @@ OCRmyPDF perform some image processing on each page of a PDF, if desired.  The s

    ``--clean-final`` and ``-remove-background`` may leave undesirable visual artifacts in some images where their algorithms have shortcomings. Files should be visually reviewed after using these options.

-OCR and correct document skew (crooked scan)
-""""""""""""""""""""""""""""""""""""""""""""
+Example: OCR and correct document skew (crooked scan)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

 Deskew:

@@ -156,7 +168,7 @@ Image processing commands can be combined. The order in which options are given


 Don't actually OCR my PDF
-"""""""""""""""""""""""""
+-------------------------

 If you set ``--tesseract-timeout 0`` OCRmyPDF will apply its image processing without performing OCR, if all you want to is to apply image processing or PDF/A conversion.

@@ -166,7 +178,7 @@ If you set ``--tesseract-timeout 0`` OCRmyPDF will apply its image processing wi


 Redo existing OCR
-"""""""""""""""""
+-----------------

 To redo OCR on a file OCRed with other OCR software or a previous version of OCRmyPDF and/or Tesseract, you may use the ``--redo-ocr`` argument. (Normally, OCRmyPDF will exit with an error if asked to modify a file with OCR.)

@@ -196,11 +208,28 @@ PDF optimization

 By default OCRmyPDF will attempt to perform lossless optimizations on the images inside PDFs after OCR is complete. Optimization is performed even if no OCR text is found.

-The ``--optimize N`` (short form ``-O``) argument controls optimization, where ``N`` ranges from 0 to 3. ``--optimize 0`` disables optimizations. ``1`` enables lossless optimizations that can be performed safely with no quality loss. ``2`` enables lossy optimizations such as image color quantizations. ``3`` enables more aggressive optimizations and targets a lower JPEG quality.
+The ``--optimize N`` (short form ``-O``) argument controls optimization, where ``N`` ranges from 0 to 3 inclusive, analogous to the optimization levels in the GCC compiler.

-Optimization is improved when a JBIG2 encoder is available and when ``pngquant`` is installed. If either of these components are missing, then some types of images will not be optimized.
+.. list-table::
+    :widths: auto
+    :header-rows: 1

-Currently optimization attempts to find more efficient encodings for images. The types of optimization available may expand over time. By default, OCRmyPDF compresses data streams inside PDFs, and will change inefficient encodings to more modern versions. A program like ``qpdf`` can be used to change encodings, e.g. to inspect the internals fo a PDF.
+    *   - Level
+        - Comments
+    *   - ``--optimize 0``
+        - Disables optimization.
+    *   - ``--optimize 1``
+        - Enables lossless optimizations, such as transcoding images to more
+          efficient formats. Also compress other uncompressed objects in the
+          PDF and enables the more efficient "object streams" within the PDF.
+    *   - ``--optimize 2``
+        - All of the above, and enables lossy optimizations and color quantization.
+    *   - ``--optimize 3``
+        - All of the above, and enables more aggressive optimizations and targets lower image quality.
+
+Optimization is improved when a JBIG2 encoder is available and when ``pngquant`` is installed. If either of these components are missing, then some types of images cannot be optimized.
+
+The types of optimization available may expand over time. By default, OCRmyPDF compresses data streams inside PDFs, and will change inefficient compression modes to more modern versions. A program like ``qpdf`` can be used to change encodings, e.g. to inspect the internals fo a PDF.

 .. code-block:: bash

@@ -0,0 +1,155 @@
+OCRmyPDF Docker image
+=====================
+
+OCRmyPDF is also available in a Docker image that packages recent versions of all dependencies.
+
+For users who already have Docker installed this may be an easy and convenient option. However, it is less performant than a system installation and may require Docker engine configuration.
+
+OCRmyPDF needs a generous amount of RAM, CPU cores, and temporary storage space.
+
+.. _docker-install:
+
+Installing the Docker image
+---------------------------
+
+If you have `Docker <https://docs.docker.com/>`_ installed on your system, you can install a Docker image of the latest release.
+
+The recommended OCRmyPDF Docker image is currently named ``ocrmypdf-alpine``:
+
+.. code-block:: bash
+
+    docker pull jbarlow83/ocrmypdf-alpine
+
+Follow the Docker installation instructions for your platform.  If you can run this command successfully, your system is ready to download and execute the image:
+
+.. code-block:: bash
+
+    docker run hello-world
+
+OCRmyPDF will use all available CPU cores.  By default, the VirtualBox machine instance on Windows and macOS has only a single CPU core enabled. Use the VirtualBox Manager to determine the name of your Docker engine host, and then follow these optional steps to enable multiple CPUs:
+
+.. code-block:: bash
+
+    # Optional step for Mac OS X users
+    docker-machine stop "yourVM"
+    VBoxManage modifyvm "yourVM" --cpus 2  # or whatever number of core is desired
+    docker-machine start "yourVM"
+    eval $(docker-machine env "yourVM")
+
+Using the Docker image on the command line
+------------------------------------------
+
+**Unlike typical Docker containers**, in this mode we are using the OCRmyPDF Docker container is intended to be emphemeral – it runs for one OCR job and then terminates, just like a command line program. We are using Docker as a way of delivering an application, not a server.
+
+To start a Docker container (instance of the image):
+
+.. code-block:: bash
+
+    docker tag jbarlow83/ocrmypdf-alpine ocrmypdf
+    docker run --rm ocrmypdf (... all other arguments here...)
+
+For convenience, create a shell alias to hide the Docker command:
+
+.. code-block:: bash
+
+    alias ocrmypdf='docker run --rm -v "$(pwd):/home/docker" ocrmypdf'
+    ocrmypdf --version  # runs docker version
+
+Or in the wonderful `fish shell <https://fishshell.com/>`_:
+
+.. code-block:: fish
+
+    alias ocrmypdf 'docker run --rm ocrmypdf'
+    funcsave ocrmypdf
+
+.. _docker-lang-packs:
+
+Adding languages to the Docker image
+------------------------------------
+
+By default the Docker image includes English, German and Simplified Chinese, the most popular languages for OCRmyPDF users based on feedback. You may add other languages by creating a new Dockerfile based on the public one:
+
+.. code-block:: dockerfile
+
+    FROM jbarlow83/ocrmypdf-alpine
+
+    # Add French
+    RUN apk add tesseract-ocr-data-fra
+
+Executing the test suite
+------------------------
+
+The OCRmyPDF test suite is installed with image.  To run it:
+
+.. code-block:: bash
+
+    docker run --entrypoint python3 jbarlow83/ocrmypdf-alpine setup.py test
+
+Using the OCRmyPDF web service wrapper
+--------------------------------------
+
+The OCRmyPDF Docker image includes an example, barebones HTTP web service. The webservice may be launched as follows:
+
+.. code-block:: bash
+
+    docker run --entrypoint python3 -p 5000:5000 jbarlow83/ocrmypdf-alpine webservice.py
+
+Unlike command line usage this program will open a socket and wait for connections.
+
+.. warning::
+
+    The OCRmyPDF web service wrapper is intended for demonstration or development. It provides no security, no authentication, no protection against denial of service attacks, and no load balancing. The default Flask WSGI server is used, which is intended for development only. The server is single-threaded and so can respond to only one client at a time. It cannot respond to clients while busy with OCR.
+
+Clients must keep their open connection while waiting for OCR to complete. This may entail setting a long timeout; this interface is more useful for internal HTTP API calls.
+
+Unlike the rest of OCRmyPDF, this web service is licensed under the Affero GPLv3 (AGPLv3) since Ghostscript, a dependency of OCRmyPDF, is also licensed in this way.
+
+In addition to the above, please read our :ref:`general remarks on using OCRmyPDF as a service <ocr-service>`_.
+
+Legacy Ubuntu Docker images
+---------------------------
+
+Previously OCRmyPDF was delivered in several Docker images for different purposes, based on Ubuntu.
+
+The Ubuntu-based images will be maintained for some time but should not be used for new deployments. They are as follows:
+
+.. list-table::
+    :widths: auto
+    :header-rows: 1
+
+    *   - Image name
+        - Download command
+        - Notes
+    *   - ocrmypdf
+        - ``docker pull jbarlow83/ocrmypdf``
+        - Latest ocrmypdf with Tesseract 4.0.0-beta1 on Ubuntu 18.04. Includes English, French, German, Spanish, Portugeuse and Simplified Chinese.
+    *   - ocrmypdf-polyglot
+        - ``docker pull jbarlow83/ocrmypdf-polyglot``
+        - As above, with all available language packs.
+    *   - ocrmypdf-webservice
+        - ``docker pull jbarlow83/ocrmypdf-webservice``
+        - All language packs, and a simple HTTP wrapper allowing OCRmyPDF to be used as a web service. Note that this component is licensed under AGPLv3.
+
+To execute the Ubuntu-based OCRmyPDF on a local file, you must `provide a writable volume to the Docker image <https://docs.docker.com/userguide/dockervolumes/>`_, and both the input and output file must be inside the writable volume. This limitation applies only to the legacy images.
+
+This example command uses the current working directory as the writable volume:
+
+.. code-block:: bash
+
+    docker run --rm -v "$(pwd):/home/docker" <other docker arguments>   ocrmypdf <your arguments to ocrmypdf>
+
+In this worked example, the current working directory contains an input file called ``test.pdf`` and the output will go to ``output.pdf``:
+
+.. code-block:: bash
+
+    docker run --rm -v "$(pwd):/home/docker"   ocrmypdf --skip-text test.pdf output.pdf
+
+.. note:: The working directory should be a writable local volume or Docker may not have permission to access it.
+
+Note that ``ocrmypdf`` has its own separate ``-v VERBOSITYLEVEL`` argument to control debug verbosity. All Docker arguments should before the ``ocrmypdf`` image name and all arguments to ``ocrmypdf`` should be listed after.
+
+In some environments the permissions associated with Docker can be complex to configure. The process that executes Docker may end up not having the permissions to write the specified file system. In that case one can stream the file into and out of the Docker process and avoid all permission hassles, using ``-`` as the input and output filename:
+
+.. code-block:: bash
+
+    docker run --rm -i   ocrmypdf <other arguments to ocrmypdf> - - <input.pdf >output.pdf
@@ -25,6 +25,7 @@ PDF is the best format for storing and exchanging scanned documents.  Unfortunat
   :maxdepth: 2

   cookbook
+   docker
   advanced
   batch
   security
@@ -1,5 +1,5 @@
-Installation
-============
+Installing OCRmyPDF
+===================

 .. |latest| image:: https://img.shields.io/pypi/v/ocrmypdf.svg
    :alt: OCRmyPDF latest released version on PyPI
@@ -103,7 +103,7 @@ sources`_.
    OCRmyPDF works fine without it but will produce larger output files. If you
    build jbig2enc from source, ocrmypdf 7.0.0 and later will automatically
    detect it on the ``PATH``. To add JBIG2 encoding, see `Installing the JBIG2
-    encoder`_.
+    encoder <jbig2>`_.

 Installing the latest version on Ubuntu 18.04 LTS
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -298,6 +298,12 @@ OCRmyPDF is now a standard `Homebrew <https://brew.sh>`_ formula. To install on

    brew install ocrmypdf

+This will include only the English language pack. If you need other languages you can optionally install them all:
+
+.. code-block:: bash
+
+    brew install tesseract-lang  # Optional: Install all language packs
+
 .. note::

    Users who previously installed OCRmyPDF on macOS using ``pip install ocrmypdf`` should remove the pip version (``pip3 uninstall ocrmypdf``) before switching to the Homebrew version.
@@ -358,133 +364,19 @@ The command line program should now be available:

    ocrmypdf --help

-.. _docker-install:
-
 Installing the Docker image
 ---------------------------

 For some users, installing the Docker image will be easier than installing all of OCRmyPDF's dependencies. For Windows, it is the only option.

-If you have `Docker <https://docs.docker.com/>`_ installed on your system, you can install a Docker image of the latest release.
-
-Follow the Docker installation instructions for your platform.  If you can run this command successfully, your system is ready to download and execute the image:
-
-.. code-block:: bash
-
-    docker run hello-world
-
-OCRmyPDF will use all available CPU cores.  By default, the VirtualBox machine instance on Windows and macOS has only a single CPU core enabled. Use the VirtualBox Manager to determine the name of your Docker engine host, and then follow these optional steps to enable multiple CPUs:
-
-.. code-block:: bash
-
-    # Optional step for Mac OS X users
-    docker-machine stop "yourVM"
-    VBoxManage modifyvm "yourVM" --cpus 2  # or whatever number of core is desired
-    docker-machine start "yourVM"
-    eval $(docker-machine env "yourVM")
-
-Assuming you have a Docker engine running, you can download one of the three available images:
-
-.. list-table::
-    :widths: auto
-    :header-rows: 1
-
-    *   - Image name
-        - Download command
-        - Notes
-    *   - ocrmypdf
-        - ``docker pull jbarlow83/ocrmypdf``
-        - Latest ocrmypdf with Tesseract 4.0.0-beta1 on Ubuntu 18.04. Includes English, French, German, Spanish, Portugeuse and Simplified Chinese.
-    *   - ocrmypdf-polyglot
-        - ``docker pull jbarlow83/ocrmypdf-polyglot``
-        - As above, with all available language packs.
-    *   - ocrmypdf-webservice
-        - ``docker pull jbarlow83/ocrmypdf-webservice``
-        - All language packs, and a simple HTTP wrapper allowing OCRmyPDF to be used as a web service. Note that this component is licensed under AGPLv3.
-
-For example:
-
-.. code-block:: bash
-
-    docker pull jbarlow83/ocrmypdf
-
-Then tag it to give a more convenient name, just ocrmypdf:
-
-.. code-block:: bash
-
-    docker tag jbarlow83/ocrmypdf ocrmypdf
-
-.. _docker-polyglot:
-
-The alternative "polyglot" image provides `all available language packs <https://github.com/tesseract-ocr/tesseract/blob/master/doc/tesseract.1.asc#languages>`_.
-
-You can then run ocrmypdf using the command:
-
-.. code-block:: bash
-
-    docker run --rm ocrmypdf --help
-
-To execute the OCRmyPDF on a local file, you must `provide a writable volume to the Docker image <https://docs.docker.com/userguide/dockervolumes/>`_, and both the input and output file must be inside the writable volume.  This example command uses the current working directory as the writable volume:
-
-.. code-block:: bash
-
-    docker run --rm -v "$(pwd):/home/docker" <other docker arguments>   ocrmypdf <your arguments to ocrmypdf>
-
-In this worked example, the current working directory contains an input file called ``test.pdf`` and the output will go to ``output.pdf``:
-
-.. code-block:: bash
-
-    docker run --rm -v "$(pwd):/home/docker"   ocrmypdf --skip-text test.pdf output.pdf
-
-.. note:: The working directory should be a writable local volume or Docker may not have permission to access it.
-
-Note that ``ocrmypdf`` has its own separate ``-v VERBOSITYLEVEL`` argument to control debug verbosity. All Docker arguments should before the ``ocrmypdf`` image name and all arguments to ``ocrmypdf`` should be listed after.
-
-In some environments the permissions associated with Docker can be complex to configure. The process that executes Docker may end up not having the permissions to write the specified file system. In that case one can stream the file into and out of the Docker process and avoid all permission hassles, using ``-`` as the input and output filename:
-
-.. code-block:: bash
-
-    docker run --rm -i   ocrmypdf <other arguments to ocrmypdf> - - <input.pdf >output.pdf
-
-For convenience, a shell alias can hide the docker command:
-
-.. code-block:: bash
-
-    alias ocrmypdf='docker run --rm -v "$(pwd):/home/docker" ocrmypdf'
-    ocrmypdf --version  # runs docker version
-
-Or in the wonderful `fish shell <https://fishshell.com/>`_:
-
-.. code-block:: fish
-
-    alias ocrmypdf 'docker run --rm -v (pwd):/home/docker ocrmypdf'
-    funcsave ocrmypdf
-
-.. note::
-
-    The ocrmypdf Docker images are designed for application delivery, to enable use of OCRmyPDF without fussing with dependencies. ``docker run --rm`` argument tells Docker to delete the container after it runs, because each container is only good for a single job. The Docker image is not designed for use as a persistent web service or for use on Amazon EC2 Container Service (AWS ECS).
+See `OCRmyPDF Docker Image <docker>`_ for more information.

 Installing on Windows
 ---------------------

-Direct installation on Windows is not possible.  `Install the Docker <docker-install_>`_ container as described above.  Ensure that your command prompt can run the docker "hello world" container.
-
-It would probably not be too difficult to run on Windows.  The main reason this has been avoided is the difficulty of packaging and installing the various non-Python dependencies: Tesseract, QPDF, Ghostscript, Leptonica.  Pull requests to add or improve Windows support would be quite welcome.
-
-
-Running on Windows
-~~~~~~~~~~~~~~~~~~
-
-The command line syntax to run ocrmypdf from a command prompt will resemble:
-
-.. code-block:: bat
-
-    docker run -v /c/Users/sampleuser:/home/docker ocrmypdf --skip-text test.pdf output.pdf
-
-where /c/Users/sampleuser is a Unix representation of the Windows path C:\\Users\\sampleuser, assuming a user named "sampleuser" is running ocrmypdf on a file in their home directory, and the files "test.pdf" and "output.pdf" are in the sampleuser folder. The Windows user must have read and write permissions.
-
-`Bash on Ubuntu on Windows <https://github.com/Microsoft/BashOnWindows>`_ should also be a viable route for running the OCRmyPDF Docker container.
+Direct installation on Windows is not possible.  `Install the Docker <docker-install>`_ container as described above.  Ensure that your command prompt can run the docker "hello world" container.

+It would probably not be too difficult to port on Windows.  The main reason this has been avoided is the difficulty of packaging and installing the various non-Python dependencies: Tesseract, QPDF, Ghostscript, Leptonica.  Pull requests to add or improve Windows support would be quite welcome.

 Installing with Python pip
 --------------------------
@@ -510,7 +402,7 @@ Since ``pip3 install --user`` does not work correctly on some platforms, notably
    pip3 install ocrmypdf

 Requirements for pip and HEAD install
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

 OCRmyPDF currently requires these external programs and libraries to be installed, and must be satisfied using the operating system package manager. ``pip`` cannot provide them.

@@ -584,7 +476,7 @@ dependencies. Older version than the ones mentioned in the release notes
 are likely not to be compatible to OCRmyPDF.

 For development
-~~~~~~~~~~~~~~~
+^^^^^^^^^^^^^^^

 To install all of the development and test requirements:

@@ -110,10 +110,10 @@ To the author's knowledge, OCRmyPDF is the most feature-rich and thoroughly test
 Web front-ends
 --------------

-The Docker image ocrmypdf-webservice provides a web service front-end that allows files to submitted over HTTP and the results "downloaded". This is an HTTP server intended to simplify web services deployments; it is not intended to be deployed on the public internet and no real security measures to speak of.
+The Docker image ``ocrmypdf-alpine`` provides a web service front-end that allows files to submitted over HTTP and the results "downloaded". This is an HTTP server intended to simplify web services deployments; it is not intended to be deployed on the public internet and no real security measures to speak of.

-In addition, the following integrations are available:
+In addition, the following third-party integrations are available:

 * `Nextcloud OCR <https://github.com/janis91/ocr>`_ is a free software plugin for the Nextcloud private cloud software

-Bear in mind that OCRmyPDF is not designed to be secure against malware-bearing PDFs (see `Using OCRmyPDF online`_). Users should ensure they comply with OCRmyPDF's licenses and the licenses of all dependencies. In particular, OCRmyPDF requires Ghostscript, which is licensed under AGPLv3.
+OCRmyPDF is not designed to be secure against malware-bearing PDFs (see `Using OCRmyPDF online <ocr-service>`_). Users should ensure they comply with OCRmyPDF's licenses and the licenses of all dependencies. In particular, OCRmyPDF requires Ghostscript, which is licensed under AGPLv3.
@@ -32,4 +32,4 @@ JBIG2 lossy mode does achieve higher compression ratios than any other monochrom

 To turn on JBIG2 lossy mode, add the argument ``--jbig2-lossy``. ``--optimize {1,2,3}`` are necessary for the argument to take effect also required. Also, a JBIG2 encoder must be installed as described in the previous section.

-*ocrmypdf v7.0 and v7.1 used lossy mode by default.*
+*Due to an oversight, ocrmypdf v7.0 and v7.1 used lossy mode by default.*
@@ -3,7 +3,7 @@
 Installing additional language packs
 ====================================

-OCRmyPDF uses Tesseract for OCR, and relies on its language packs for languages other than English. 
+OCRmyPDF uses Tesseract for OCR, and relies on its language packs for languages other than English.

 Tesseract supports `most languages <https://github.com/tesseract-ocr/tesseract/blob/master/doc/tesseract.1.asc#languages>`_.

@@ -46,31 +46,4 @@ You can install additional language packs by :ref:`installing Tesseract using Ho
 Docker users
 ------------

-Users of the Docker image may use the alternative :ref:`"polyglot" container <docker-polyglot>` which includes all languages.
-
-Adding individual language packs to a Docker image
-""""""""""""""""""""""""""""""""""""""""""""""""""
-
-If you wish to add a single language pack, you could do the following:
-
-* Download the desired ``.trainedata`` file from the `tessdata <https://github.com/tesseract-ocr/tessdata>`_ repository. Let's use Hebrew in this example (``heb.traineddata``)
-
-* Copy the file to ``/home/user/downloads/heb.traineddata``.
-
-* Create a new container based on the ocrmypdf-tess4 image and jump into it with a terminal:
-
-.. code-block:: bash
-
-	host$ docker run  -v /home/user/downloads:/home/docker -it --entrypoint /bin/bash ocrmypdf-tess4
-
-* Put the file where Tesseract expects it:
-
-.. code-block:: bash
-
-	docker$ cp /home/docker/heb.traineddata /usr/share/tesseract-ocr/tessdata
-
-* Note the container id, and save it as a new image (in this example, ``ocrmypdf-tess4-heb``)
-
-.. code-block:: bash
-
-    host$ docker commit <container_id> ocrmypdf-tess4-heb
+Users of the OCRmyPDF Docker image should install language packs into a derived Docker image as :ref:`described in that section <docker-lang-packs>`.
@@ -13,6 +13,23 @@ Note that it is licensed under GPLv3, so scripts that ``import ocrmypdf`` and ar
   find:    [^`]\#([0-9]{1,3})[^0-9]
   replace: `#$1 <https://github.com/jbarlow83/OCRmyPDF/issues/$1>`_

+v8.2.0
+------
+
+-   A major improvement to our Docker image is now available thanks to hard work contributed by @mawi12345. The new Docker image, ocrmypdf-alpine, is based on Alpine Linux, and includes most of the functionality of three existed images in a smaller package. This image will replace the main Docker image eventually but for now all are being built. `See documentation for details <https://ocrmypdf.readthedocs.io/en/latest/docker.html>`_.
+
+-   Documentation reorganized especially around the use of Docker images.
+
+-   Fixed a problem with PDF image optimization, where the optimizer would unnecessarily decompress and recompress PNG images, in some cases losing the benefits of the quantization it just had just performed. The optimizer is now capable of embedding PNG images into PDFs without transcoding them.
+
+-   Fixed a minor regression with lossy JBIG2 image optimization. All JBIG2 candidates images were incorrectly placed into a single optimization group for the whole file, instead of grouping pages together. This usually makes a larger JBIG2Globals dictionary and results in inferior compression, so it worked less well than designed. However, quality would not be impacted. Lossless JBIG2 was entirely unaffected.
+
+-   Updated dependencies, including pikepdf to 1.1.0. This fixes `#358 <https://github.com/jbarlow83/OCRmyPDF/issues/358>`_.
+
+-   The install-time version checks for certain external programs have been removed from setup.py. These tests are now performed at run-time.
+
+-   The non-standard option to override install-time checks (``setup.py install --force``) is now deprecated and prints a warning. It will be removed in a future release.
+
 v8.1.0
 ------

@@ -25,12 +25,14 @@ Finally, OCRmyPDF rasterizes each page of the PDF using `Ghostscript <http://gho

 Depending on the options specified, OCRmyPDF may graft the OCR layer into the existing PDF or it may essentially reconstruct ("re-fry") a visually identical PDF that may be quite different at the binary level. That said, OCRmyPDF is not a tool designed for sanitizing PDFs.

+.. _ocr-service:
+
 Using OCRmyPDF online or as a service
 -------------------------------------

-OCRmyPDF should not be deployed as a public-facing service, such as a website where a potential attacker could upload a PDF of their choice for OCR. OCRmyPDF is not designed to be secure against PDF malware. Another concern is PDFs specifically designed to be a denial of service attack: PDFs can contain recursive data structures that sometimes send parsers into infinite loops, and issue complex graphics drawing commands.
+OCRmyPDF is not designed for use as a public web service where a malicious user could upload a chosen PDF. In particular, it is not necessarily secure against PDF malware or PDFs that cause denial of service. OCRmyPDF relies on Ghostscript, and therefore, if deployed online one should be prepared to comply with Ghostscript's Affero GPL license, OCRmyPDF's GPL license, and any other licenses.

-Setting aside these concerns, a side effect of OCRmyPDF is it may incidentally sanitize PDFs that contain malware. It runs ``qpdf`` to repair the PDF, which could correct malformed PDF structures that are part of an attack. When PDF/A output is selected (the default), the input PDF is partially reconstructed by Ghostscript. When ``--force-ocr`` is used, all pages are rasterized and reconverted to PDF, which could remove malware in embedded images. No guarantees.
+Setting aside these concerns, a side effect of OCRmyPDF is it may incidentally sanitize PDFs that contain certain types of malware. It runs ``qpdf`` to repair the PDF, which could correct malformed PDF structures that are part of an attack. When PDF/A output is selected (the default), the input PDF is partially reconstructed by Ghostscript. When ``--force-ocr`` is used, all pages are rasterized and reconverted to PDF, which could remove malware in embedded images.

 OCRmyPDF should be relatively safe to use in a trusted intranet, with some considerations:

@@ -1,3 +1,13 @@
+[build-system]
+requires = [
+  "setuptools >= 30.3.0",
+  "wheel",
+  "cffi",
+  "setuptools_scm",
+  "setuptools_scm_git_archive"
+]
+build-backend = "setuptools.build_meta"
+
 [tool.black]
 line-length = 88
 py36 = true
@@ -2,12 +2,12 @@
 # setup.py lists a separate set of requirements that are looser to simplify
 # installation
 chardet == 3.0.4
-cffi == 1.11.5
-img2pdf == 0.3.1
+cffi == 1.12.2
+img2pdf == 0.3.3
 pdfminer.six == 20181108
-pikepdf == 1.0.2
+pikepdf == 1.1.0
 Pillow >= 5.0.0, != 5.1.0 ; sys_platform == "darwin"
 pycparser == 2.19
 python-xmp-toolkit == 2.0.1
-reportlab == 3.5.9
-ruffus == 2.8.0
+reportlab == 3.5.13
+ruffus == 2.8.1
@@ -1,7 +1,7 @@
-pytest == 3.9.3
-pytest-helpers-namespace
+pytest == 4.3.0
+pytest-helpers-namespace >= 2019.1.8
 pytest-xdist
-pytest-cov
+pytest-cov >= 2.6.1
 python-xmp-toolkit   # requires apt-get install libexempi3
                     # or brew install exempi
 PyPDF2 >= 1.26.0
@@ -20,6 +20,7 @@
 from __future__ import print_function, unicode_literals

 import sys
+
 if sys.version_info < (3, 6):
    print("Python 3.6 or newer is required", file=sys.stderr)
    sys.exit(1)
@@ -31,171 +32,19 @@ import re

 # pylint: disable=w0613

-missing_program = '''
-The program '{program}' could not be executed or was not found on your
-system PATH.
-'''
-
-unknown_version = '''
-OCRmyPDF requires '{program}' {need_version} or higher.  Your system has
-'{program}' but we cannot tell what version is installed.  Contact the
-package maintainer.
-'''
-
-old_version = '''
-OCRmyPDF requires '{program}' {need_version} or higher.  Your system appears
-to have {found_version}.  Please update this program.
-'''
-
-okay_its_optional = '''
-This program is OPTIONAL, so installation of OCRmyPDF can proceed, but
-some functionality may be missing.
-'''
-
-not_okay_its_required = '''
-This program is REQUIRED for OCRmyPDF to work.  Installation will abort.
-'''
-
-osx_install_advice = '''
-If you have homebrew installed, try these command to install the missing
-packages:
-    brew update
-    brew upgrade
-    brew install {package}
-'''
-
-linux_install_advice = '''
-On systems with the aptitude package manager (Debian, Ubuntu), try these
-commands:
-    sudo apt-get update
-    sudo apt-get install {package}
-
-On RPM-based systems (Red Hat, Fedora), search for instructions on
-installing the RPM for {program}.
-'''
-
-
-def get_platform():
-    if sys.platform.startswith('freebsd'):
-        return 'freebsd'
-    elif sys.platform.startswith('linux'):
-        return 'linux'
-    return sys.platform
-
-
-def _error_trailer(program, package, optional, **kwargs):
-    if optional:
-        print(okay_its_optional.format(**locals()), file=sys.stderr)
-    else:
-        print(not_okay_its_required.format(**locals()), file=sys.stderr)
-
-    if isinstance(package, Mapping):
-        package = package[get_platform()]
-
-    if get_platform() == 'darwin':
-        print(osx_install_advice.format(**locals()), file=sys.stderr)
-    elif get_platform() == 'linux':
-        print(linux_install_advice.format(**locals()), file=sys.stderr)
-
-
-def error_missing_program(
-        program,
-        package,
-        optional
-        ):
-    print(missing_program.format(**locals()), file=sys.stderr)
-    _error_trailer(**locals())
-
-
-def error_unknown_version(
-        program,
-        package,
-        optional,
-        need_version
-        ):
-    print(unknown_version.format(**locals()), file=sys.stderr)
-    _error_trailer(**locals())
-
-
-def error_old_version(
-        program,
-        package,
-        optional,
-        need_version,
-        found_version
-        ):
-    print(old_version.format(**locals()), file=sys.stderr)
-    _error_trailer(**locals())
-
-
-def check_external_program(
-        program,
-        need_version,
-        package,
-        version_check_args=None,
-        version_scrape_regex=re.compile(r'(\d+\.\d+(?:\.\d+)?)'),
-        optional=False):
-    if not version_check_args:
-        version_check_args = ['--version']
-    print(f'Checking for {program} >= {need_version}...')
-    try:
-        result = check_output(
-                [program] + version_check_args,
-                universal_newlines=True, stderr=STDOUT)
-    except (CalledProcessError, FileNotFoundError):
-        error_missing_program(program, package, optional)
-        if not optional:
-            sys.exit(1)
-        print(f'Continuing install without {program}')
-        return
-
-    try:
-        found_version = version_scrape_regex.search(result).group(1)
-    except AttributeError:
-        error_unknown_version(program, package, optional, need_version)
-        sys.exit(1)
-
-    if found_version < need_version:
-        error_old_version(program, package, optional, need_version,
-                          found_version)
-
-    print(f'Found {program} {found_version}')
-

 command = next((arg for arg in sys.argv[1:] if not arg.startswith('-')), '')
-forced = ('--force' in sys.argv)
+if command.startswith('install') or command in [
+    'check',
+    'test',
+    'nosetests',
+    'easy_install',
+]:
+    forced = '--force' in sys.argv
+    if forced:
+        print("The argument --force is deprecated. Please discontinue use.")


-if not forced and command.startswith('install') or \
-        command in ['check', 'test', 'nosetests', 'easy_install']:
-    check_external_program(
-        program='tesseract',
-        need_version='4.0.0',  # using backport for Travis CI
-        package={'darwin': 'tesseract', 'linux': 'tesseract-ocr'}
-    )
-    check_external_program(
-        program='gs',
-        need_version='9.15',  # limited by Travis CI / Ubuntu 14.04 backports
-        package='ghostscript'
-    )
-    check_external_program(
-        program='unpaper',
-        need_version='6.1',   # latest sane version
-        package='unpaper',
-        optional=True
-    )
-    check_external_program(
-        program='qpdf',
-        need_version='8.0.2', # test suite known to fail on 5.1.1
-        package='qpdf'
-    )
-    check_external_program(
-        program='pngquant',
-        need_version='2.0.0',
-        package='pngquant',
-        optional=True
-    )
-
 if 'upload' in sys.argv[1:]:
    print('Use twine to upload the package - setup.py upload is insecure')
    sys.exit(1)
@@ -207,6 +56,7 @@ def readme():
    with open('README.md', encoding='utf-8') as f:
        return f.read()

+
 setup(
    name='ocrmypdf',
    description='OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched',
@@ -234,45 +84,37 @@ setup(
        "Topic :: Scientific/Engineering :: Image Recognition",
        "Topic :: Text Processing :: Indexing",
        "Topic :: Text Processing :: Linguistic",
-        ],
+    ],
    python_requires=' >= 3.6',
-    setup_requires=[
-        'cffi >= 1.9.1',        # to build the leptonica module
-        'pytest-runner',        # to enable python setup.py test
-        'setuptools_scm',       # so that version will work
-        'setuptools_scm_git_archive'    # enable version from github tarballs
+    setup_requires=[  # can be removed whenever we can drop pip 9 support
+        'cffi >= 1.9.1',  # to build the leptonica module
+        'pytest-runner',  # to enable python setup.py test
+        'setuptools_scm',  # so that version will work
+        'setuptools_scm_git_archive',  # enable version from github tarballs
    ],
    use_scm_version={'version_scheme': 'post-release'},
-    cffi_modules=[
-        'src/ocrmypdf/lib/compile_leptonica.py:ffibuilder'
-    ],
+    cffi_modules=['src/ocrmypdf/lib/compile_leptonica.py:ffibuilder'],
    install_requires=[
        'chardet >= 3.0.4, < 4',  # unlisted requirement of pdfminer.six 20181108
-        'cffi >= 1.9.1',          # must be a setup and install requirement
-        'img2pdf >= 0.3.0, < 0.4',       # pure Python, so track HEAD closely
+        'cffi >= 1.9.1',  # must be a setup and install requirement
+        'img2pdf >= 0.3.0, < 0.4',  # pure Python, so track HEAD closely
        'pdfminer.six == 20181108 ; sys_platform != "darwin"',
-        'pikepdf >= 1.0.5, < 2',
+        'pikepdf >= 1.1.0, < 2',
        'Pillow >= 4.0.0, != 5.1.0 ; sys_platform == "darwin"',
-                                  # Pillow < 4 has BytesIO/TIFF bug w/img2pdf 0.2.3
-                                  # block 5.1.0, broken wheels
-        'reportlab >= 3.3.0',     # oldest released version with sane image handling
+        # Pillow < 4 has BytesIO/TIFF bug w/img2pdf 0.2.3
+        # block 5.1.0, broken wheels
+        'reportlab >= 3.3.0',  # oldest released version with sane image handling
        'ruffus >= 2.7.0',
    ],
-    extras_require={
-        'pdfminer': ['pdfminer.six == 20181108'],
-    },
+    extras_require={'pdfminer': ['pdfminer.six == 20181108']},
    tests_require=tests_require,
-    entry_points={
-        'console_scripts': [
-            'ocrmypdf = ocrmypdf.__main__:run_pipeline'
-        ],
-    },
+    entry_points={'console_scripts': ['ocrmypdf = ocrmypdf.__main__:run_pipeline']},
    package_data={'ocrmypdf': ['data/sRGB.icc']},
    include_package_data=True,
    zip_safe=False,
    project_urls={
        'Documentation': 'https://ocrmypdf.readthedocs.io/',
        'Source': 'https://github.com/jbarlow83/ocrmypdf',
-        'Tracker': 'https://github.com/jbarlow83/ocrmypdf/issues'
-    }
+        'Tracker': 'https://github.com/jbarlow83/ocrmypdf/issues',
+    },
 )
@@ -44,7 +44,14 @@ from .exceptions import (
    MissingDependencyError,
    OutputFileAccessError,
 )
-from .exec import ghostscript, qpdf, tesseract
+from .exec import (
+    ghostscript,
+    qpdf,
+    tesseract,
+    check_external_program,
+    unpaper,
+    pngquant,
+)
 from .helpers import available_cpu_count, is_file_writable, re_symlink
 from .pdfa import file_claims_pdfa

@@ -67,13 +74,6 @@ if 'IDE_PROJECT_ROOTS' in os.environ:

 verify_python3_env()

-if not tesseract.v4:
-    complain(
-        f"Please install tesseract 4.0.0 or newer "
-        f"(currently installed version is {tesseract.version()})"
-    )
-    sys.exit(ExitCode.missing_dependency)
-
 # -------------
 # Parser

@@ -633,9 +633,7 @@ def check_options_preprocessing(options, log):
    if options.clean_final:
        options.clean = True
    if options.unpaper_args and not options.clean:
-        raise argparse.ArgumentError(
-            None, "--clean is required for --unpaper-args"
-        )
+        raise argparse.ArgumentError(None, "--clean is required for --unpaper-args")
    if any((options.clean, options.clean_final)):
        from .exec import unpaper

@@ -930,9 +928,6 @@ def log_page_orientations(pdfinfo, _log):

 def preamble(_log):
    _log.debug('ocrmypdf ' + VERSION)
-    _log.debug('tesseract ' + tesseract.version())
-    _log.debug('qpdf ' + qpdf.version())
-    _log.debug('gs ' + ghostscript.version())


 def check_environ(options, _log):
@@ -1032,6 +1027,54 @@ def report_output_file_size(options, _log, input_file, output_file):
    )


+def check_dependency_versions(log):
+    check_external_program(
+        log=log,
+        program='tesseract',
+        package={'darwin': 'tesseract', 'linux': 'tesseract-ocr'},
+        version_checker=tesseract.version,
+        need_version='4.0.0',  # using backport for Travis CI
+    )
+    check_external_program(
+        log=log,
+        program='gs',
+        package='ghostscript',
+        version_checker=ghostscript.version,
+        need_version='9.15',  # limited by Travis CI / Ubuntu 14.04 backports
+    )
+    if ghostscript.version() == '9.24':
+        complain(
+            "Ghostscript 9.24 contains serious regressions and is not "
+            "supported. Please upgrade to Ghostscript 9.25 or use an older "
+            "version."
+        )
+        return ExitCode.missing_dependency
+    check_external_program(
+        log=log,
+        program='unpaper',
+        package='unpaper',
+        version_checker=unpaper.version,
+        need_version='6.1',  # latest sane version
+        optional=True,
+    )
+    if os.environ.get('TRAVIS') != 'true':  # Suppress for Ubuntu trusty
+        check_external_program(
+            log=log,
+            program='qpdf',
+            package='qpdf',
+            version_checker=qpdf.version,
+            need_version='8.0.2',
+        )
+    check_external_program(
+        log=log,
+        program='pngquant',
+        package='pngquant',
+        version_checker=pngquant.version,
+        need_version='2.0.0',
+        optional=True,
+    )
+
+
 def run_pipeline(args=None):
    options = parser.parse_args(args=args)
    options.verbose_abbreviated_path = 1
@@ -1048,24 +1091,7 @@ def run_pipeline(args=None):
    )
    preamble(_log)
    check_options(options, _log)
-
-    # Complain about qpdf version < 7.0.0
-    # Suppress the warning if in the test suite, since there are no PPAs
-    # for qpdf 7.0.0 for Ubuntu trusty (i.e. Travis)
-    if qpdf.version() < '7.0.0' and not os.environ.get('PYTEST_CURRENT_TEST'):
-        complain(
-            f"You are using qpdf version {qpdf.version()} which has known issues including "
-            f"security vulnerabilities with certain malformed PDFs. Consider "
-            f"upgrading to version 7.0.0 or newer."
-        )
-
-    if ghostscript.version() == '9.24':
-        complain(
-            "Ghostscript 9.24 contains serious regressions and is not "
-            "supported. Please upgrade to Ghostscript 9.25 or use an older "
-            "version."
-        )
-        return ExitCode.missing_dependency
+    check_dependency_versions(_log)

    # Any changes to options will not take effect for options that are already
    # bound to function parameters in the pipeline. (For example
@@ -181,8 +181,6 @@ def _find_font(text, pdf_base):
            break
    if pdf_text_font:
        font = pdf_base.copy_foreign(pdf_text_font)
-    if font_key is None:
-        print('font_key is None')
    return font, font_key


@@ -22,6 +22,7 @@ import re
 import sys
 from subprocess import run, STDOUT, PIPE, CalledProcessError
 from ..exceptions import MissingDependencyError
+from collections.abc import Mapping


 def get_version(program, *, version_arg='--version', regex=r'(\d+(\.\d+)*)'):
@@ -58,3 +59,102 @@ def get_version(program, *, version_arg='--version', regex=r'(\d+(\.\d+)*)'):
        )

    return version
+
+
+missing_program = '''
+The program '{program}' could not be executed or was not found on your
+system PATH.
+'''
+
+unknown_version = '''
+OCRmyPDF requires '{program}' {need_version} or higher.  Your system has
+'{program}' but we cannot tell what version is installed.  Contact the
+package maintainer.
+'''
+
+old_version = '''
+OCRmyPDF requires '{program}' {need_version} or higher.  Your system appears
+to have {found_version}.  Please update this program.
+'''
+
+okay_its_optional = '''
+This program is OPTIONAL, so installation of OCRmyPDF can proceed, but
+some functionality may be missing.
+'''
+
+not_okay_its_required = '''
+This program is REQUIRED for OCRmyPDF to work.  Installation will abort.
+'''
+
+osx_install_advice = '''
+If you have homebrew installed, try these command to install the missing
+packages:
+    brew update
+    brew upgrade
+    brew install {package}
+'''
+
+linux_install_advice = '''
+On systems with the aptitude package manager (Debian, Ubuntu), try these
+commands:
+    sudo apt-get update
+    sudo apt-get install {package}
+
+On RPM-based systems (Red Hat, Fedora), search for instructions on
+installing the RPM for {program}.
+'''
+
+
+def get_platform():
+    if sys.platform.startswith('freebsd'):
+        return 'freebsd'
+    elif sys.platform.startswith('linux'):
+        return 'linux'
+    return sys.platform
+
+
+def _error_trailer(log, program, package, optional, **kwargs):
+    if optional:
+        log.error(okay_its_optional.format(**locals()), file=sys.stderr)
+    else:
+        log.error(not_okay_its_required.format(**locals()), file=sys.stderr)
+
+    if isinstance(package, Mapping):
+        package = package[get_platform()]
+
+    if get_platform() == 'darwin':
+        log.error(osx_install_advice.format(**locals()), file=sys.stderr)
+    elif get_platform() == 'linux':
+        log.error(linux_install_advice.format(**locals()), file=sys.stderr)
+
+
+def error_missing_program(log, program, package, optional):
+    log.error(missing_program.format(**locals()), file=sys.stderr)
+    _error_trailer(log, **locals())
+
+
+def error_unknown_version(log, program, package, optional, need_version):
+    log.error(unknown_version.format(**locals()), file=sys.stderr)
+    _error_trailer(log, **locals())
+
+
+def error_old_version(log, program, package, optional, need_version, found_version):
+    log.error(old_version.format(**locals()), file=sys.stderr)
+    _error_trailer(log, **locals())
+
+
+def check_external_program(
+    log, program, package, version_checker, need_version, optional=False
+):
+    try:
+        found_version = version_checker()
+    except (CalledProcessError, FileNotFoundError, MissingDependencyError):
+        error_missing_program(log, program, package, optional)
+        if not optional:
+            sys.exit(1)
+        return
+
+    if found_version < need_version:
+        error_old_version(log, program, package, optional, need_version, found_version)
+
+    log.debug(f'Found {program} {found_version}')
@@ -17,6 +17,9 @@

 from functools import lru_cache
 from subprocess import run
+from tempfile import NamedTemporaryFile
+
+from PIL import Image

 from . import get_version
 from ..exceptions import MissingDependencyError
@@ -36,16 +39,32 @@ def available():


 def quantize(input_file, output_file, quality_min, quality_max):
-    args = [
-        'pngquant',
-        '--force',
-        '--skip-if-larger',
-        '--output',
-        output_file,
-        '--quality',
-        f'{quality_min}-{quality_max}',
-        '--',
-        input_file,
-    ]
-    proc = run(args)
-    proc.check_returncode()
+    if input_file.endswith('.jpg'):
+        im = Image.open(input_file)
+        with NamedTemporaryFile(suffix='.png') as tmp:
+            im.save(tmp)
+            args = [
+                'pngquant',
+                '--force',
+                '--skip-if-larger',
+                '--output',
+                output_file,
+                '--quality',
+                f'{quality_min}-{quality_max}',
+                '--',
+                tmp.name,
+            ]
+            run(args)
+    else:
+        args = [
+            'pngquant',
+            '--force',
+            '--skip-if-larger',
+            '--output',
+            output_file,
+            '--quality',
+            f'{quality_min}-{quality_max}',
+            '--',
+            input_file,
+        ]
+        run(args)
@@ -25,6 +25,7 @@ from pathlib import Path
 from PIL import Image

 import pikepdf
+from pikepdf import Name, Dictionary, Array

 from . import leptonica
 from ._jobcontext import JobContext
@@ -52,7 +53,7 @@ def tif_name(root, xref):


 def extract_image_filter(pike, root, log, image, xref):
-    if image.Subtype != '/Image':
+    if image.Subtype != Name.Image:
        return None
    if image.Length < 100:
        log.debug("Skipping small image, xref %s", xref)
@@ -68,7 +69,7 @@ def extract_image_filter(pike, root, log, image, xref):
    if pim.bits_per_component > 8:
        return None  # Don't mess with wide gamut images

-    if filtdp[0] == '/JPXDecode':
+    if filtdp[0] == Name.JPXDecode:
        return None  # Don't do JPEG2000

    return pim, filtdp
@@ -82,7 +83,7 @@ def extract_image_jbig2(*, pike, root, log, image, xref, options):

    if (
        pim.bits_per_component == 1
-        and filtdp != '/JBIG2Decode'
+        and filtdp != Name.JBIG2Decode
        and jbig2enc.available()
    ):
        try:
@@ -102,7 +103,7 @@ def extract_image_generic(*, pike, root, log, image, xref, options):
        return None
    pim, filtdp = result

-    if filtdp[0] == '/DCTDecode' and options.optimize >= 2:
+    if filtdp[0] == Name.DCTDecode and options.optimize >= 2:
        # This is a simple heuristic derived from some training data, that has
        # about a 70% chance of guessing whether the JPEG is high quality,
        # and possibly recompressible, or not. The number itself doesn't mean
@@ -148,11 +149,22 @@ def extract_image_generic(*, pike, root, log, image, xref, options):
 def extract_images(pike, root, log, options, extract_fn):
    """Extract image using extract_fn

-    extract_fn decides whether the image is interesting in this case
+    Enumerate images on each page, lookup their xref/ID number in the PDF.
+    Exclude images that are soft masks (i.e. alpha transparency related).
+    Record the page number on which an image is first used, since images may be
+    used on multiple pages (or multiple times on the same page).
+
+    Current we do not check Form XObjects or other objects that may contain
+    images, and we don't evaluate alternate images or thumbnails.
+
+    extract_fn must decide if wants to extract the image in this context. If
+    it does a tuple should be returned: (xref, ext) where .ext is the file
+    extension. extract_fn must also extract the file it finds interesting.
    """

    include_xrefs = set()
    exclude_xrefs = set()
+    pageno_for_xref = {}
    errors = 0
    for pageno, page in enumerate(pike.pages):
        try:
@@ -168,6 +180,8 @@ def extract_images(pike, root, log, options, extract_fn):
                smask_xref = image.SMask.objgen[0]
                exclude_xrefs.add(smask_xref)
            include_xrefs.add(xref)
+            if xref not in pageno_for_xref:
+                pageno_for_xref[xref] = pageno

    working_xrefs = include_xrefs - exclude_xrefs
    for xref in working_xrefs:
@@ -177,13 +191,12 @@ def extract_images(pike, root, log, options, extract_fn):
                pike=pike, root=root, log=log, image=image, xref=xref, options=options
            )
        except Exception as e:
-            log.debug("Image xref %s", xref)
-            log.debug(repr(e))
+            log.debug("Image xref %s, error %s", xref, repr(e))
            errors += 1
        else:
            if result:
                _, ext = result
-                yield pageno, xref, ext
+                yield pageno_for_xref[xref], xref, ext


 def extract_images_generic(pike, root, log, options):
@@ -197,7 +210,7 @@ def extract_images_generic(pike, root, log, options):
            pngs.append(xref)
        elif ext == '.jpg':
            jpegs.append(xref)
-    log.debug("Optimizable images: " "JPEGs: %s PNGs: %s", len(jpegs), len(pngs))
+    log.debug("Optimizable images: JPEGs: %s PNGs: %s", len(jpegs), len(pngs))
    return jpegs, pngs


@@ -215,7 +228,7 @@ def extract_images_jbig2(pike, root, log, options):
    jbig2_groups = {
        group: xrefs for group, xrefs in jbig2_groups.items() if len(xrefs) > 0
    }
-    log.debug("Optimizable images: " "JBIG2 groups: %s", (len(jbig2_groups),))
+    log.debug("Optimizable images: JBIG2 groups: %s", (len(jbig2_groups),))
    return jbig2_groups


@@ -270,7 +283,7 @@ def convert_to_jbig2(pike, jbig2_groups, root, log, options):
    must be lossy encoding since jbig2enc does not support refinement coding.

    When the JBIG2 symbolic coder is not used, each JBIG2 stands on its own
-    and needs no dictionary. Currently this is must be lossless JBIG2.
+    and needs no dictionary. Currently this must be lossless JBIG2.
    """

    _produce_jbig2_images(jbig2_groups, root, log, options)
@@ -281,7 +294,7 @@ def convert_to_jbig2(pike, jbig2_groups, root, log, options):
        if jbig2_symfile.exists():
            jbig2_globals_data = jbig2_symfile.read_bytes()
            jbig2_globals = pikepdf.Stream(pike, jbig2_globals_data)
-            jbig2_globals_dict = pikepdf.Dictionary({'/JBIG2Globals': jbig2_globals})
+            jbig2_globals_dict = Dictionary(JBIG2Globals=jbig2_globals)
        elif options.jbig2_page_group_size == 1:
            jbig2_globals_dict = None
        else:
@@ -293,9 +306,7 @@ def convert_to_jbig2(pike, jbig2_groups, root, log, options):
            jbig2_im_data = jbig2_im_file.read_bytes()
            im_obj = pike.get_object(xref, 0)
            im_obj.write(
-                jbig2_im_data,
-                filter=pikepdf.Name('/JBIG2Decode'),
-                decode_parms=jbig2_globals_dict,
+                jbig2_im_data, filter=Name.JBIG2Decode, decode_parms=jbig2_globals_dict
            )


@@ -310,17 +321,17 @@ def transcode_jpegs(pike, jpegs, root, log, options):
        # https://github.com/python-pillow/Pillow/issues/1144
        with Image.open(fspath(in_jpg)) as im:
            im.save(fspath(opt_jpg), optimize=True, quality=options.jpeg_quality)
-        # pylint: disable=no-member
+
        if opt_jpg.stat().st_size > in_jpg.stat().st_size:
            log.debug("xref %s, jpeg, made larger - skip", xref)
            continue

        compdata = leptonica.CompressedData.open(opt_jpg)
        im_obj = pike.get_object(xref, 0)
-        im_obj.write(compdata.read(), filter=pikepdf.Name('/DCTDecode'))
+        im_obj.write(compdata.read(), filter=Name.DCTDecode)


-def transcode_pngs(pike, pngs, root, log, options):
+def transcode_pngs(pike, images, image_name_fn, root, log, options):
    if options.optimize >= 2:
        png_quality = (
            max(10, options.png_quality - 10),
@@ -329,65 +340,91 @@ def transcode_pngs(pike, pngs, root, log, options):
        with concurrent.futures.ThreadPoolExecutor(
            max_workers=options.jobs
        ) as executor:
-            for xref in pngs:
+            for xref in images:
+                log.debug(image_name_fn(root, xref))
                executor.submit(
                    pngquant.quantize,
-                    png_name(root, xref),
+                    image_name_fn(root, xref),
                    png_name(root, xref),
                    png_quality[0],
                    png_quality[1],
                )

-    for xref in pngs:
+    for xref in images:
        im_obj = pike.get_object(xref, 0)
-
-        # Open, transcode (!), package for PDF
        try:
-            pix = leptonica.Pix.open(png_name(root, xref))
-            if pix.depth == 1:
-                pix = pix.invert()  # PDF assumes 1 is black for monochrome
-            compdata = pix.generate_pdf_ci_data(leptonica.lept.L_FLATE_ENCODE, 0)
+            compdata = leptonica.CompressedData.open(png_name(root, xref))
        except leptonica.LeptonicaError as e:
+            # Most likely this means file not found, i.e. quantize did not
+            # produce an improved version
            log.error(e)
            continue

-        # This is what we should be doing: open the compressed data without
-        # transcoding. However this shifts each pixel row by one for some
-        # reason.
-        # compdata = leptonica.CompressedData.open(png_name(root, xref))
+        # If re-coded image is larger don't use it - we test here because
+        # pngquant knows the size of the temporary output file but not the actual
+        # object in the PDF
        if len(compdata) > int(im_obj.stream_dict.Length):
-            continue  # If we produced a larger image, don't use
+            log.debug(
+                f"pngquant: pngquant did not improve over original image "
+                f"{len(compdata)} > {int(im_obj.stream_dict.Length)}"
+            )
+            continue

-        predictor = None
-        if compdata.predictor > 0:
-            predictor = pikepdf.Dictionary({'/Predictor': compdata.predictor})
+        # When a PNG is inserted into a PDF, we more or less copy the IDAT section from
+        # the PDF and transfer the rest of the PNG headers to PDF image metadata.
+        # One thing we have to do is tell the PDF reader whether a predictor was used
+        # on the image before Flate encoding. (Typically one is.)
+        # According to Leptonica source, PDF readers don't actually need us
+        # to specify the correct predictor, they just need a value of either:
+        #   1 - no predictor
+        #   10-14 - there is a predictor
+        # Leptonica's compdata->predictor only tells TRUE or FALSE
+        # From there the PNG decoder can infer the rest from the file.
+        # In practice the predictor should be Paeth, 14, so we'll use that.
+        # See:
+        #   - PDF RM 7.4.4.4 Table 10
+        #   - https://github.com/DanBloomberg/leptonica/blob/master/src/pdfio2.c#L757
+        predictor = 14 if compdata.predictor > 0 else 1
+        dparms = Dictionary(Predictor=predictor)
+        if predictor > 1:
+            dparms.BitsPerComponent = compdata.bps  # Yes, this is redundant
+            dparms.Colors = compdata.spp
+            dparms.Columns = compdata.w

        im_obj.BitsPerComponent = compdata.bps
        im_obj.Width = compdata.w
        im_obj.Height = compdata.h

        if compdata.ncolors > 0:
+            # .ncolors is the number of colors in the palette, not the number of
+            # colors used in a true color image
            palette_pdf_string = compdata.get_palette_pdf_string()
            palette_data = pikepdf.Object.parse(palette_pdf_string)
            palette_stream = pikepdf.Stream(pike, bytes(palette_data))
            palette = [
-                pikepdf.Name('/Indexed'),
-                pikepdf.Name('/DeviceRGB'),
+                Name.Indexed,
+                Name.DeviceRGB,
                compdata.ncolors - 1,
                palette_stream,
            ]
            cs = palette
        else:
            if compdata.spp == 1:
-                cs = pikepdf.Name('/DeviceGray')
+                # PDF interprets binary-1 as black in 1bpp, but PNG sets
+                # black to 0 for 1bpp. Create a palette that informs the PDF
+                # of the mapping - seems cleaner to go this way but pikepdf
+                # needs to be patched to support it.
+                # palette = [Name.Indexed, Name.DeviceGray, 1, b"\xff\x00"]
+                # cs = palette
+                cs = Name.DeviceGray
            elif compdata.spp == 3:
-                cs = pikepdf.Name('/DeviceRGB')
+                cs = Name.DeviceRGB
            elif compdata.spp == 4:
-                cs = pikepdf.Name('/DeviceCMYK')
+                cs = Name.DeviceCMYK
+        if compdata.bps == 1:
+            im_obj.Decode = [1, 0]  # Bit of a kludge but this inverts photometric too
        im_obj.ColorSpace = cs
-        im_obj.write(
-            compdata.read(), filter=pikepdf.Name('/FlateDecode'), decode_parms=predictor
-        )
+        im_obj.write(compdata.read(), filter=Name.FlateDecode, decode_parms=dparms)


 def optimize(input_file, output_file, log, context):
@@ -407,11 +444,14 @@ def optimize(input_file, output_file, log, context):
    pike = pikepdf.Pdf.open(input_file)

    root = Path(output_file).parent / 'images'
-    root.mkdir(exist_ok=True)  # pylint: disable=no-member
+    root.mkdir(exist_ok=True)

    jpegs, pngs = extract_images_generic(pike, root, log, options)
    transcode_jpegs(pike, jpegs, root, log, options)
-    transcode_pngs(pike, pngs, root, log, options)
+    # if options.optimize >= 2:
+    # Try pngifying the jpegs
+    #    transcode_pngs(pike, jpegs, jpg_name, root, log, options)
+    transcode_pngs(pike, pngs, png_name, root, log, options)

    jbig2_groups = extract_images_jbig2(pike, root, log, options)
    convert_to_jbig2(pike, jbig2_groups, root, log, options)
@@ -0,0 +1 @@
+Tesseract Open Source OCR Engine v4.0.0 with Leptonica
@@ -0,0 +1,13 @@
+Portez ce vieux whisky au juge
+blond qui fume sur son ile
+interieure, a cöte de l'alcöve
+ovolde, oU les büches se
+consument dans l'ätre, ce qui
+lui permet de penser ä la
+cgnogenese de l'&tre dont il
+est question dans la cause
+ambigu6 entendue ä Moy, dans
+un capharnaüÜm qui, pense-t-il,
+diminue ca et 13 la qualite de son
+ceuvre.
+
@@ -44,3 +44,4 @@
 {"tesseract_version": "tesseract 4.0.0  leptonica-1.77.0   libjpeg 9c : libpng 1.6.35 : libtiff 4.0.10 : zlib 1.2.11 : libopenjp2 2.3.0  Found AVX2  Found AVX  Found SSE ", "platform": "Darwin-18.0.0-x86_64-i386-64bit", "python": "3.7.1", "argv_slug": "__-l__eng__000004.ocr.png__000004__hocr__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "$TMPDIR/000004.ocr.png", "$TMPDIR/000004", "hocr", "txt"]}
 {"tesseract_version": "tesseract 4.0.0  leptonica-1.77.0   libjpeg 9c : libpng 1.6.35 : libtiff 4.0.10 : zlib 1.2.11 : libopenjp2 2.3.0  Found AVX2  Found AVX  Found SSE ", "platform": "Darwin-18.0.0-x86_64-i386-64bit", "python": "3.7.1", "argv_slug": "__-l__eng__000003.ocr.png__000003__hocr__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "$TMPDIR/000003.ocr.png", "$TMPDIR/000003", "hocr", "txt"]}
 {"tesseract_version": "tesseract 4.0.0  leptonica-1.77.0   libjpeg 9c : libpng 1.6.35 : libtiff 4.0.10 : zlib 1.2.11 : libopenjp2 2.3.0  Found AVX2  Found AVX  Found SSE ", "platform": "Darwin-18.0.0-x86_64-i386-64bit", "python": "3.7.1", "argv_slug": "__-l__eng__000002.ocr.png__000002__hocr__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "$TMPDIR/000002.ocr.png", "$TMPDIR/000002", "hocr", "txt"]}
+{"tesseract_version": "tesseract 4.0.0  leptonica-1.77.0   libgif 5.1.4 : libjpeg 9c : libpng 1.6.36 : libtiff 4.0.10 : zlib 1.2.11 : libwebp 1.0.2 : libopenjp2 2.3.0  Found AVX2  Found AVX  Found SSE ", "platform": "Darwin-18.2.0-x86_64-i386-64bit", "python": "3.7.2", "argv_slug": "__-l__deu__000001.ocr.png__000001.text__pdf__txt", "sourcefile": "resources/francais.pdf", "args": ["-l", "deu", "-c", "textonly_pdf=1", "$TMPDIR/000001.ocr.png", "$TMPDIR/000001.text", "pdf", "txt"]}
@@ -338,15 +338,16 @@ def test_force_ocr_on_pdf_with_no_images(spoof_tesseract_crash, resources, no_ou
    pytest.helpers.is_macos() and pytest.helpers.running_in_travis(),
    reason="takes too long to install language packs in Travis macOS homebrew",
 )
-def test_french(spoof_tesseract_cache, resources, outdir):
+def test_german(spoof_tesseract_cache, resources, outdir):
    # Produce a sidecar too - implicit test that system locale is set up
-    # properly
+    # properly. It is fine that we are testing -l deu on a French file because
+    # we are exercising the functionality not going for accuracy.
    sidecar = outdir / 'francais.txt'
    p, out, err = run_ocrmypdf(
        resources / 'francais.pdf',
        outdir / 'francais.pdf',
        '-l',
-        'fra',
+        'deu',  # more commonly installed
        '--sidecar',
        sidecar,
        env=spoof_tesseract_cache,
@@ -998,6 +999,9 @@ def test_pdfa_n(spoof_tesseract_cache, pdfa_level, resources, outpdf):


@pytest.mark.skipif(sys.version_info >= (3, 7, 0), reason='better utf-8')
+@pytest.mark.skipif(
+    Path('/etc/alpine-release').exists(), reason="invalid test on alpine"
+)
 def test_bad_locale():
    env = os.environ.copy()
    env['LC_ALL'] = 'C'
@@ -188,9 +188,8 @@ def test_xml_metadata_preserved(spoof_tesseract_noop, output_type, resources, ou
    input_file = resources / 'graph.pdf'

    try:
-        import libxmp
-        from libxmp.utils import file_to_dict
        from libxmp import consts
+        from libxmp.utils import file_to_dict
    except Exception:
        pytest.skip("libxmp not available or libexempi3 not installed")

@@ -331,16 +330,13 @@ def test_prevent_gs_invalid_xml(resources, outdir):
    from ocrmypdf.pdfinfo import PdfInfo

    generate_pdfa_ps(outdir / 'pdfa.ps')
-    input_files = [
-        str(outdir / 'layers.rendered.pdf'),
-        str(outdir / 'pdfa.ps'),
-    ]
+    input_files = [str(outdir / 'layers.rendered.pdf'), str(outdir / 'pdfa.ps')]
    copyfile(resources / 'enron1.pdf', outdir / 'layers.rendered.pdf')
    log = logging.getLogger()
    context = JobContext()

-    options = parser.parse_args(args=[
-        '-j', '1', '--output-type', 'pdfa-2', 'a.pdf', 'b.pdf']
+    options = parser.parse_args(
+        args=['-j', '1', '--output-type', 'pdfa-2', 'a.pdf', 'b.pdf']
    )
    context.options = options
    context.pdfinfo = PdfInfo(resources / 'enron1.pdf')
@@ -349,11 +345,13 @@ def test_prevent_gs_invalid_xml(resources, outdir):
        input_files_groups=input_files,
        output_file=outdir / 'pdfa.pdf',
        log=log,
-        context=context
+        context=context,
    )

    with open(outdir / 'pdfa.pdf', 'rb') as f:
-        with mmap.mmap(f.fileno(), 0, flags=mmap.MAP_PRIVATE, prot=mmap.PROT_READ) as mm:
+        with mmap.mmap(
+            f.fileno(), 0, flags=mmap.MAP_PRIVATE, prot=mmap.PROT_READ
+        ) as mm:
            # Since the XML may be invalid, we scan instead of actually feeding it
            # to a parser.
            XMP_MAGIC = b'W5M0MpCehiHzreSzNTczkc9d'
@@ -120,7 +120,7 @@ def test_flate_to_jbig2(resources, outdir, spoof_tesseract_noop):
        '--image-dpi',
        '100',
        '--png-quality',
-        '10',
+        '50',
        '--optimize',
        '3',
        env=spoof_tesseract_noop,
Author	SHA1	Message	Date
James R. Barlow	a27ee3ee8c	optimize: use Decode to invert 1bpp PNGs for now	2019-03-03 17:50:12 -08:00
James R. Barlow	c2f316c2c5	v8.2.0 release notes: optimizer	2019-03-03 15:26:01 -08:00
James R. Barlow	974979b0a0	Merge branch 'feature/optimization-fixes'	2019-03-03 15:00:20 -08:00
James R. Barlow	66586bdaab	optimize: Disable jpg->png migration Needs more testing before release	2019-03-03 14:59:59 -08:00
James R. Barlow	01d2ea309f	Fix Predictor name and photometric flip	2019-03-03 14:57:15 -08:00
James R. Barlow	e918480351	v8.2.0 release notes	2019-03-03 14:15:20 -08:00
James R. Barlow	52fd84fa95	Remove debug message	2019-03-03 13:31:10 -08:00
James R. Barlow	2c56b0935c	docs: minor	2019-03-03 03:28:17 -08:00
James R. Barlow	4f69ace868	optimize: fix all JBIG2 images binned on last page During some past refactor it appears we now end up treating all JBIG2 images as if they appeared on the last page in the file. This bug had no visual side ffects but probably led to suboptimal JBIG2 encoding.	2019-03-03 03:28:17 -08:00
James R. Barlow	497c531112	optimize: update comments	2019-03-03 03:28:17 -08:00
James R. Barlow	b27b92fbf3	optimize: on aggressive settings try JPG to PNG transcoding If the color count of an image is low such as when black and white documents are scanned in color, PNG with lossy quantization may produce a superior encoding to JPEG. This is expensive to test however.	2019-03-03 03:28:17 -08:00
James R. Barlow	2e6ba2df8c	optimize: fix recoding of PNGs Previously we opened pngquant-compressed PNGs with transcoding because the transcode free function in Leptonica didn't seem to work. This mean Leptonica may have thrown away the hard of pngquant if didn't understand the encoding. This change resolves the issue and allows us to open PNG encoded data and insert it into a PDF without transcoding. Should improve encoding quality.	2019-03-03 03:28:17 -08:00
James R. Barlow	67a405c6b7	Move install-time external program checks out of setup.py We did runtime tests for several of them anyway, and it's better to do at runtime since config may change after installation.	2019-03-03 03:26:56 -08:00
James R. Barlow	58e6663806	Update test cache for french->german change	2019-03-03 03:23:59 -08:00
James R. Barlow	602570fcf9	Update requirements	2019-03-03 02:27:56 -08:00
James R. Barlow	691f8ce254	Docs: reorganize for new docker-alpine image	2019-03-01 23:15:32 -08:00
James R. Barlow	22812e74b9	Merge branch 'master' of github.com:jbarlow83/OCRmyPDF	2019-02-26 13:01:59 -08:00
Martin Windandjbarlow83	9d824e723d	Add Dockerfile based on alpine:3.9 (#354 ) * Do not exclude .git from docker build * Use multi-stage builds to keep the image size down * Copy project files to get the test suite. * Add webservice * Add tesseract language data for German and Chinese Simplified	2019-02-26 13:01:38 -08:00
James R. Barlow	5dad800d85	Add version to build-system declaration	2019-02-26 12:58:44 -08:00
James R. Barlow	56a56a4dcb	docs: avoid importing ocrmypdf	2019-02-26 12:57:50 -08:00
James R. Barlow	3f1d9ef99c	Fix tests for move to Alpine dockerfile	2019-02-26 12:30:21 -08:00
James R. Barlow	92c8a5885e	Declare build system in pyproject.toml	2019-02-26 12:23:33 -08:00
James R. Barlow	7749d14252	Merge branch 'master' of github.com:jbarlow83/OCRmyPDF	2019-02-24 01:56:47 -08:00
Julien Maandjbarlow83	9b92af5aed	README: install other language packs on macOS (#352 ) The default homebrew formula installs only the English language pack. Another brew formula exists to install all other language packs. This makes it easier than having to do the whole install manually.	2019-02-19 10:13:36 -08:00
James R. Barlow	0bf26b03ae	optimize: Modernize pikepdf usage	2019-02-16 14:03:10 -08:00
				`@@ -0,0 +1 @@`
				`Tesseract Open Source OCR Engine v4.0.0 with Leptonica`