Further fixes to external program version testing

Needs more testing before release

.docker/alpine.dockerfile

@@ -0,0 +1,84 @@
+FROM alpine:3.9 as base
+
+FROM base as builder
+
+ENV LANG=C.UTF-8
+
+RUN \
+  echo '@testing http://nl.alpinelinux.org/alpine/edge/testing' >> /etc/apk/repositories \
+  # Add runtime dependencies
+  && apk add --update \
+    python3-dev \
+    py3-setuptools \
+    jbig2enc@testing \
+    ghostscript \
+    qpdf \
+    tesseract-ocr \
+    unpaper \
+    pngquant \
+    libxml2-dev \
+    libxslt-dev \
+    zlib-dev \
+    qpdf-dev \
+    libffi-dev \
+    leptonica-dev \
+    binutils \
+  # Install pybind11 for pikepdf
+  && pip3 install pybind11 \
+  # Install flask for the webservice
+  && pip3 install flask \
+  # Add build dependencies
+  && apk add --virtual build-dependencies \
+    build-base \
+    git
+
+COPY . /app
+
+WORKDIR /app
+
+RUN pip3 install .
+
+FROM base
+
+ENV LANG=C.UTF-8
+
+RUN \
+  echo '@testing http://nl.alpinelinux.org/alpine/edge/testing' >> /etc/apk/repositories \
+  # Add runtime dependencies
+  && apk add --update \
+    python3 \
+    jbig2enc@testing \
+    ghostscript \
+    qpdf \
+    tesseract-ocr \
+    tesseract-ocr-data-deu \
+    tesseract-ocr-data-chi_sim \
+    unpaper \
+    pngquant \
+    libxml2 \
+    libxslt \
+    zlib \
+    qpdf \
+    libffi \
+    leptonica-dev \
+    binutils \
+  && mkdir /app
+
+WORKDIR /app
+
+# Copy build artifacts (python site-packages9
+COPY --from=builder /usr/lib/python3.6/site-packages /usr/lib/python3.6/site-packages
+COPY --from=builder /usr/bin/ocrmypdf /usr/bin/dumppdf.py /usr/bin/latin2ascii.py /usr/bin/pdf2txt.py /usr/bin/img2pdf /usr/bin/chardetect /usr/bin/
+
+# Copy
+COPY --from=builder /app/.docker/webservice.py /app/
+
+# Copy minimal project files to get the test suite.
+COPY --from=builder /app/setup.cfg /app/setup.py /app/README.md /app/
+COPY --from=builder /app/requirements /app/requirements
+COPY --from=builder /app/tests /app/tests
+COPY --from=builder /app/src /app/src
+# Copy PKG-INFO from build artifact in app dir to make setuptools-scm happy
+RUN cp /usr/lib/python3.6/site-packages/ocrmypdf-*.egg-info/PKG-INFO /app
+
+ENTRYPOINT ["/usr/bin/ocrmypdf"]

.docker/webservice.dockerfile

@@ -16,4 +16,9 @@ COPY .docker/webservice.py /application
 
 USER docker
 
+VOLUME ["/config"]
+
+# This config file is optional
+ENV OCRMYPDF_WEBSERVICE_SETTINGS "/config/config.py"
+
 ENTRYPOINT ["python3", "/application/webservice.py"]

.docker/webservice.py

@@ -1,5 +1,5 @@
 # webservice.py wrapper for OCRmyPDF
-# Copyright (C) 2018 James R. Barlow: github.com/jbarlow83
+# Copyright (C) 2019 James R. Barlow: github.com/jbarlow83
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -17,13 +17,22 @@
 """This is a simple web service/HTTP wrapper for OCRmyPDF
 
 This may be more convenient than the command line tool for some Docker users.
-Note that OCRmyPDF uses Ghostscript, which is licensed under AGPL3+. While
-OCRmyPDF is under GPL3, this file is distributed under the Affero GPL3+ license,
+Note that OCRmyPDF uses Ghostscript, which is licensed under AGPLv3+. While
+OCRmyPDF is under GPLv3, this file is distributed under the Affero GPLv3+ license,
 to emphasize that SaaS deployments should make sure they comply with
 Ghostscript's license as well as OCRmyPDF's.
 """
 
-from flask import Flask, Response, flash, request, redirect, url_for, abort, send_from_directory
+from flask import (
+    Flask,
+    Response,
+    flash,
+    request,
+    redirect,
+    url_for,
+    abort,
+    send_from_directory,
+)
 from subprocess import run, PIPE
 from tempfile import TemporaryDirectory
 from werkzeug.utils import secure_filename
@@ -32,11 +41,9 @@ import shlex
 
 app = Flask(__name__)
 app.secret_key = "secret"
+app.config['MAX_CONTENT_LENGTH'] = 50_000_000
+app.config.from_envvar("OCRMYPDF_WEBSERVICE_SETTINGS", silent=True)
 
-uploaddir = TemporaryDirectory(prefix="ocrmypdf-upload")
-downloaddir = TemporaryDirectory(prefix="ocrmypdf-download")
-
-app.config["UPLOAD_FOLDER"] = uploaddir
 ALLOWED_EXTENSIONS = set(["pdf"])
 
 
@@ -45,9 +52,13 @@ def allowed_file(filename):
 
 
 def do_ocrmypdf(file):
+    uploaddir = TemporaryDirectory(prefix="ocrmypdf-upload")
+    downloaddir = TemporaryDirectory(prefix="ocrmypdf-download")
+
     filename = secure_filename(file.filename)
     up_file = os.path.join(uploaddir.name, filename)
     file.save(up_file)
+
     down_file = os.path.join(downloaddir.name, filename)
 
     cmd_args = [arg for arg in shlex.split(request.form["params"])]
@@ -79,7 +90,7 @@ def upload_file():
 
     return """
     <!doctype html>
-    <title>OCRmyPDF webapp</title>
+    <title>OCRmyPDF webservice</title>
     <h1>Upload a PDF (debug UI)</h1>
     <form method=post enctype=multipart/form-data>
       <label for="args">Command line parameters</label>
@@ -88,7 +99,27 @@ def upload_file():
       <input type=file name=file>
       <input type=submit value=Upload>
     </form>
+    <h4>Notice</h2>
+    <div style="font-size: 70%; max-width: 34em;">
+    <p>This is a webservice wrapper for OCRmyPDF.</p>
+    <p>Copyright 2019 James R. Barlow</p>
+    <p>This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+    </p>
+    <p>This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+    </p>
+    <p>
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see &lt;http://www.gnu.org/licenses/&gt;.
+    </p>
+    </div>
     """
 
+
 if __name__ == "__main__":
-    app.run(host='0.0.0.0')
+    app.run(host='0.0.0.0', port=5000)

.dockerignore

@@ -5,6 +5,7 @@
 *.sublime*
 **/*.pyc
 .*/
+!.git/
 !.docker/
 .ruffus_history.sqlite
 bin/

.pre-commit-config.yaml

@@ -0,0 +1,6 @@
+repos:
+-   repo: https://github.com/ambv/black
+    rev: stable
+    hooks:
+    - id: black
+      language_version: python3.6

docs/advanced.rst

@@ -1,6 +1,34 @@
 Advanced features
 =================
 
+Control of unpaper
+------------------
+
+OCRmyPDF uses ``unpaper`` to provide the implementation of the ``--clean`` and ``--clean-final`` arguments. `unpaper <https://github.com/Flameeyes/unpaper/blob/master/doc/basic-concepts.md>`_ provides a variety of image processing filters to improve images.
+
+By default, OCRmyPDF uses only ``unpaper`` arguments that were found to be safe to use on almost all files without having to inspect every page of the file afterwards. This is particularly true when only ``--clean`` is used, since that instructs OCRmyPDF to only clean the image before OCR and not the final image.
+
+However, if you wish to use the more aggressive options in ``unpaper``, you may use ``--unpaper-args '...'`` to override the OCRmyPDF's defaults and forward other arguments to unpaper. This option will forward arguments to ``unpaper`` without any knowledge of what that program considers to be valid arguments. The string of arguments must be quoted as shown in the examples below. No filename arguments may be included. OCRmyPDF will assume it can append input and output filename of intermediate images to the ``--unpaper-args`` string.
+
+In this example, we tell ``unpaper`` to expect two pages of text on a sheet (image), such as occurs when two facing pages of a book are scanned. ``unpaper`` uses this information to deskew each independently and clean up the margins of both.
+
+.. code-block:: bash
+
+    ocrmypdf --clean --clean-final --unpaper-args '--layout double' input.pdf output.pdf
+    ocrmypdf --clean --clean-final --unpaper-args '--layout double --no-noisefilter' input.pdf output.pdf
+
+.. warning::
+
+    Some ``unpaper`` features will reposition text within the image. ``--clean-final`` is recommended to avoid this issue.
+
+.. warning::
+
+    Some ``unpaper`` features cause multiple input or output files to be consumed or produced. OCRmyPDF requires ``unpaper`` to consume one file and produce one file. An deviation from that condition will result in errors.
+
+.. note::
+
+    ``unpaper`` uses uncompressed PBM/PGM/PPM files for its intermediate files. For large images or documents, it can take a lot of temporary disk space.
+
 Control of OCR options
 ----------------------
 
@@ -17,7 +45,6 @@ If ``--redo-ocr`` is issued, then a detailed text analysis is performed. Text is
 
 If ``--force-ocr`` is issued, then all pages will be rasterized to images, discarding any hidden OCR text, and rasterizing any printable text. This is useful for redoing OCR, for fixing OCR text with a damaged character map (text is selectable but not searchable), and destroying redacted information. Any forms and vector graphics will be rasterized as well.
 
-
 Time and image size limits
 """"""""""""""""""""""""""
 
@@ -58,7 +85,6 @@ For example, if you have a development build of Tesseract don't wish to use the
 
 In this example ``TESSDATA_PREFIX`` is required to redirect Tesseract to an alternate folder for its "tessdata" files.
 
-
 Overriding other support programs
 """""""""""""""""""""""""""""""""
 
@@ -198,7 +224,7 @@ OCRmyPDF normally saves its intermediate results to a temporary folder and delet
 
 If the ``-k`` argument is issued on the command line, OCRmyPDF will keep the temporary folder and print the location, whether it succeeded or failed (provided the Python interpreter did not crash). An example message is:
 
-.. code-block::
+.. code-block:: none
 
     Temporary working files saved at:
     /tmp/com.github.ocrmypdf.u20wpz07

docs/batch.rst

@@ -18,12 +18,12 @@ The ``--tag`` argument tells parallel to print the filename as a prefix whenever
 
 	parallel --tag -j 2 ocrmypdf '{}' 'output/{}' ::: *.pdf
 
-OCRmyPDF automatically repairs PDFs before parsing and gathering information from them.  If you are already repairing PDFs with ``qpdf`` prior to attempting OCR, or you can use ``--skip-repair`` to skip this step.  It may improve performance for large files, since repairing PDFs is single-threaded.
+OCRmyPDF automatically repairs PDFs before parsing and gathering information from them.
 
 Directory trees
 ---------------
 
-This will walk through a directory tree and run OCR on all files in place, printing the output in a way that makes 
+This will walk through a directory tree and run OCR on all files in place, printing the output in a way that makes
 
 .. code-block:: bash
 
@@ -66,7 +66,7 @@ This user contributed script also provides an example of batch processing.
 	    log_file = script_dir + '/ocr-tree.log'
 
 	logging.basicConfig(
-			level=logging.INFO, format='%(asctime)s %(message)s', 
+			level=logging.INFO, format='%(asctime)s %(message)s',
 			filename=log_file, filemode='w')
 
 	for dir_name, subdirs, file_list in os.walk(start_dir):
@@ -80,9 +80,9 @@ This user contributed script also provides an example of batch processing.
 	            print(full_path)
 	            cmd = ["ocrmypdf",  "--deskew", filename, filename]
 	            logging.info(cmd)
-	            proc = subprocess.Popen(
+	            proc = subprocess.run(
 	            	cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-	            result = proc.stdout.read()
+	            result = proc.stdout
 	            if proc.returncode == 6:
 	                print("Skipped document because it already contained text")
 	            elif proc.returncode == 0:
@@ -144,14 +144,14 @@ This is only possible for x86-based Synology products. Some Synology products us
 	            timestamp_OCR = time.strftime("%Y-%m-%d-%H%M_OCR_")
 	            filename_OCR = timestamp_OCR + file_noext + '.pdf'
 	            docker_mount = dir_name + ':/home/docker'
-	# create string for pdf processing 
+	# create string for pdf processing
 	# diskstation needs a user:group docker:docker. find uid:gid of your diskstation docker:docker with id docker.
 	# use this uid:gid in -u flag
 	# rw rights for docker:docker at source dir are also necessary
-	# the script is processed as root user via chron 
+	# the script is processed as root user via chron
 	            cmd = ['docker', 'run', '--rm', '-v', docker_mount, '-u=1030:65538', 'jbarlow83/ocrmypdf', , '--deskew' , filename, filename_OCR]
 	            logging.info(cmd)
-	            proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+	            proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 	            result = proc.stdout.read()
 	            logging.info(result)
 	            full_path_OCR = dir_name + '/' + filename_OCR
@@ -210,5 +210,3 @@ Alternatives
 """"""""""""
 
 * `Watchman <https://facebook.github.io/watchman/>`_ is a more powerful alternative to ``watchmedo``.
-
-

docs/conf.py

@@ -52,7 +52,9 @@ master_doc = 'index'
 
 # General information about the project.
 project = 'ocrmypdf'
-copyright = '2018, James R. Barlow. Licensed under Creative Commons Attribution-ShareAlike 4.0.'
+copyright = (
+    '2019, James R. Barlow. Licensed under Creative Commons Attribution-ShareAlike 4.0.'
+)
 author = 'James R. Barlow'
 
 # The version info for the project you're documenting, acts as replacement for
@@ -62,6 +64,7 @@ author = 'James R. Barlow'
 # The short X.Y version.
 
 import os
+
 on_rtd = os.environ.get('READTHEDOCS') == 'True'
 
 if on_rtd:
@@ -78,18 +81,16 @@ if on_rtd:
         'pikepdf',
         'pikepdf.models',
         'pikepdf.models.metadata',
-        'ocrmypdf.leptonica'
+        'ocrmypdf.leptonica',
     ]
     sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES)
 
 
-from ocrmypdf import __version__ as OCRMYPDF_VERSION
+from pkg_resources import get_distribution, DistributionNotFound
 
-_version_parts = OCRMYPDF_VERSION.split('.')
-
-version = '.'.join(_version_parts[0:2])
 # The full version, including alpha/beta/rc tags.
-release = OCRMYPDF_VERSION
+release = get_distribution('ocrmypdf').version
+version = '.'.join(release.split('.')[:2])
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -158,9 +159,7 @@ html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 # further.  For a list of options available for each theme, see the
 # documentation.
 #
-html_theme_options = {
-    'display_version': False
-}
+html_theme_options = {'display_version': False}
 
 # Add any paths that contain custom themes here, relative to this directory.
 # html_theme_path = []
@@ -273,29 +272,25 @@ htmlhelp_basename = 'ocrmypdfdoc'
 # -- Options for LaTeX output ---------------------------------------------
 
 latex_elements = {
-     # The paper size ('letterpaper' or 'a4paper').
-     #
-     # 'papersize': 'letterpaper',
-
-     # The font size ('10pt', '11pt' or '12pt').
-     #
-     # 'pointsize': '10pt',
-
-     # Additional stuff for the LaTeX preamble.
-     #
-     # 'preamble': '',
-
-     # Latex figure (float) alignment
-     #
-     # 'figure_align': 'htbp',
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
 }
 
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, 'ocrmypdf.tex', 'ocrmypdf Documentation',
-     'James R. Barlow', 'manual'),
+    (master_doc, 'ocrmypdf.tex', 'ocrmypdf Documentation', 'James R. Barlow', 'manual')
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
@@ -335,10 +330,7 @@ latex_documents = [
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [
-    (master_doc, 'ocrmypdf', 'ocrmypdf Documentation',
-     [author], 1)
-]
+man_pages = [(master_doc, 'ocrmypdf', 'ocrmypdf Documentation', [author], 1)]
 
 # If true, show URL addresses after external links.
 #
@@ -351,9 +343,15 @@ man_pages = [
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    (master_doc, 'ocrmypdf', 'ocrmypdf Documentation',
-     author, 'ocrmypdf', 'One line description of project.',
-     'Miscellaneous'),
+    (
+        master_doc,
+        'ocrmypdf',
+        'ocrmypdf Documentation',
+        author,
+        'ocrmypdf',
+        'One line description of project.',
+        'Miscellaneous',
+    )
 ]
 
 # Documents to append as an appendix to all manuals.

docs/cookbook.rst

@@ -5,7 +5,7 @@ Basic examples
 --------------
 
 Help!
-"""""
+^^^^^
 
 ocrmypdf has built-in help.
 
@@ -15,28 +15,28 @@ ocrmypdf has built-in help.
 
 
 Add an OCR layer and convert to PDF/A
-"""""""""""""""""""""""""""""""""""""
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 .. code-block:: bash
 
     ocrmypdf input.pdf output.pdf
 
 Add an OCR layer and output a standard PDF
-""""""""""""""""""""""""""""""""""""""""""
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 .. code-block:: bash
 
     ocrmypdf --output-type pdf input.pdf output.pdf
 
 Create a PDF/A with all color and grayscale images converted to JPEG
-""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 .. code-block:: bash
 
     ocrmypdf --output-type pdfa --pdfa-image-compression jpeg input.pdf output.pdf
 
 Modify a file in place
-""""""""""""""""""""""
+^^^^^^^^^^^^^^^^^^^^^^
 
 The file will only be overwritten if OCRmyPDF is successful.
 
@@ -45,7 +45,7 @@ The file will only be overwritten if OCRmyPDF is successful.
     ocrmypdf myfile.pdf myfile.pdf
 
 Correct page rotation
-"""""""""""""""""""""
+^^^^^^^^^^^^^^^^^^^^^
 
 OCR will attempt to automatic correct the rotation of each page. This can help fix a scanning job that contains a mix of landscape and portrait pages.
 
@@ -58,9 +58,9 @@ You can increase (decrease) the parameter ``--rotate-pages-threshold`` to make p
 If the page is "just a little off horizontal", like a crooked picture, then you want ``--deskew``. ``--rotate-pages`` is for when the cardinal angle is wrong.
 
 OCR languages other than English
-""""""""""""""""""""""""""""""""
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-By default OCRmyPDF assumes the document is English.
+OCRmyPDF assumes the document is in English unless told otherwise. OCR quality may be poor if the wrong language is used.
 
 .. code-block:: bash
 
@@ -69,8 +69,10 @@ By default OCRmyPDF assumes the document is English.
 
 Language packs must be installed for all languages specified. See :ref:`Installing additional language packs <lang-packs>`.
 
+Unfortunately, the Tesseract OCR engine has no ability to detect the language when it is unknown.
+
 Produce PDF and text file containing OCR text
-"""""""""""""""""""""""""""""""""""""""""""""
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 This produces a file named "output.pdf" and a companion text file named "output.txt".
 
@@ -79,7 +81,10 @@ This produces a file named "output.pdf" and a companion text file named "output.
     ocrmypdf --sidecar output.txt input.pdf output.pdf
 
 OCR images, not PDFs
---------------------
+^^^^^^^^^^^^^^^^^^^^
+
+Option: use Tesseract
+"""""""""""""""""""""
 
 If you are starting with images, you can just use Tesseract directly to convert images to PDFs:
 
@@ -92,9 +97,12 @@ If you are starting with images, you can just use Tesseract directly to convert
     # When there are multiple images
     tesseract text-file-containing-list-of-image-filenames.txt output-prefix pdf
 
-Tesseract's PDF output is quite good – OCRmyPDF uses it internally by default. However, OCRmyPDF has many features not available in Tesseract like like image processing, metadata control, and PDF/A generation.
+Tesseract's PDF output is quite good – OCRmyPDF uses it internally, in some cases. However, OCRmyPDF has many features not available in Tesseract like image processing, metadata control, and PDF/A generation.
 
-Use a program like `img2pdf <https://gitlab.mister-muffin.de/josch/img2pdf>`_ to convert your images to PDFs, and then pipe the results to run ocrmypdf.  The `-` tells ocrmypdf to read standard input.
+Option: use img2pdf
+"""""""""""""""""""
+
+You can also use a program like `img2pdf <https://gitlab.mister-muffin.de/josch/img2pdf>`_ to convert your images to PDFs, and then pipe the results to run ocrmypdf.  The ``-`` tells ocrmypdf to read standard input.
 
 .. code-block:: bash
 
@@ -102,6 +110,9 @@ Use a program like `img2pdf <https://gitlab.mister-muffin.de/josch/img2pdf>`_ to
 
 ``img2pdf`` is recommended because it does an excellent job at generating PDFs without transcoding images.
 
+Option: use OCRmyPDF (single images only)
+"""""""""""""""""""""""""""""""""""""""""
+
 For convenience, OCRmyPDF can also convert single images to PDFs on its own. If the resolution (dots per inch, DPI) of an image is not set or is incorrect, it can be overridden with ``--image-dpi``. (As 1 inch is 2.54 cm, 1 dpi = 0.39 dpcm).
 
 .. code-block:: bash
@@ -110,9 +121,10 @@ For convenience, OCRmyPDF can also convert single images to PDFs on its own. If
 
 If you have multiple images, you must use ``img2pdf`` to convert the images to PDF.
 
-.. note::
+Not recommended
+"""""""""""""""
 
-    ImageMagick ``convert`` can also convert a group of images to PDF, but in the author's experience it takes a long time, transcodes unnecessarily and gives poor results.
+We caution against using ImageMagick or Ghostscript to convert images to PDF, since they may transcode images or produce downsampled images, sometimes without warning.
 
 Image processing
 ----------------
@@ -129,7 +141,7 @@ OCRmyPDF perform some image processing on each page of a PDF, if desired.  The s
 
 * ``--clean-final`` uses unpaper to clean up pages before OCR and inserts the page into the final output.  You will want to review each page to ensure that unpaper did not remove something important.
 
-* ``--mask-barcodes`` will "cover up" any barcodes detected in the image of a page. Barcodes are known to confuse Tesseract OCR and interfere with the recognition of text on the same baseline as a barcode. The output file will contain the unaltered image of the barcode.
+* ``--mask-barcodes`` will suppress any barcodes detected in a page image. Barcodes are known to confuse Tesseract OCR and interfere with the recognition of text on the same baseline as a barcode. The output file will contain the unaltered image of the barcode.
 
 .. note::
 
@@ -139,8 +151,8 @@ OCRmyPDF perform some image processing on each page of a PDF, if desired.  The s
 
     ``--clean-final`` and ``-remove-background`` may leave undesirable visual artifacts in some images where their algorithms have shortcomings. Files should be visually reviewed after using these options.
 
-OCR and correct document skew (crooked scan)
-""""""""""""""""""""""""""""""""""""""""""""
+Example: OCR and correct document skew (crooked scan)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Deskew:
 
@@ -156,7 +168,7 @@ Image processing commands can be combined. The order in which options are given
 
 
 Don't actually OCR my PDF
-"""""""""""""""""""""""""
+-------------------------
 
 If you set ``--tesseract-timeout 0`` OCRmyPDF will apply its image processing without performing OCR, if all you want to is to apply image processing or PDF/A conversion.
 
@@ -166,7 +178,7 @@ If you set ``--tesseract-timeout 0`` OCRmyPDF will apply its image processing wi
 
 
 Redo existing OCR
-"""""""""""""""""
+-----------------
 
 To redo OCR on a file OCRed with other OCR software or a previous version of OCRmyPDF and/or Tesseract, you may use the ``--redo-ocr`` argument. (Normally, OCRmyPDF will exit with an error if asked to modify a file with OCR.)
 
@@ -196,11 +208,28 @@ PDF optimization
 
 By default OCRmyPDF will attempt to perform lossless optimizations on the images inside PDFs after OCR is complete. Optimization is performed even if no OCR text is found.
 
-The ``--optimize N`` (short form ``-O``) argument controls optimization, where ``N`` ranges from 0 to 3. ``--optimize 0`` disables optimizations. ``1`` enables lossless optimizations that can be performed safely with no quality loss. ``2`` enables lossy optimizations such as image color quantizations. ``3`` enables more aggressive optimizations and targets a lower JPEG quality.
+The ``--optimize N`` (short form ``-O``) argument controls optimization, where ``N`` ranges from 0 to 3 inclusive, analogous to the optimization levels in the GCC compiler.
 
-Optimization is improved when a JBIG2 encoder is available and when ``pngquant`` is installed. If either of these components are missing, then some types of images will not be optimized.
+.. list-table::
+    :widths: auto
+    :header-rows: 1
 
-Currently optimization attempts to find more efficient encodings for images. The types of optimization available may expand over time. By default, OCRmyPDF compresses data streams inside PDFs, and will change inefficient encodings to more modern versions. A program like ``qpdf`` can be used to change encodings, e.g. to inspect the internals fo a PDF.
+    *   - Level
+        - Comments
+    *   - ``--optimize 0``
+        - Disables optimization.
+    *   - ``--optimize 1``
+        - Enables lossless optimizations, such as transcoding images to more
+          efficient formats. Also compress other uncompressed objects in the
+          PDF and enables the more efficient "object streams" within the PDF.
+    *   - ``--optimize 2``
+        - All of the above, and enables lossy optimizations and color quantization.
+    *   - ``--optimize 3``
+        - All of the above, and enables more aggressive optimizations and targets lower image quality.
+
+Optimization is improved when a JBIG2 encoder is available and when ``pngquant`` is installed. If either of these components are missing, then some types of images cannot be optimized.
+
+The types of optimization available may expand over time. By default, OCRmyPDF compresses data streams inside PDFs, and will change inefficient compression modes to more modern versions. A program like ``qpdf`` can be used to change encodings, e.g. to inspect the internals fo a PDF.
 
 .. code-block:: bash
 

docs/docker.rst

@@ -0,0 +1,155 @@
+OCRmyPDF Docker image
+=====================
+
+OCRmyPDF is also available in a Docker image that packages recent versions of all dependencies.
+
+For users who already have Docker installed this may be an easy and convenient option. However, it is less performant than a system installation and may require Docker engine configuration.
+
+OCRmyPDF needs a generous amount of RAM, CPU cores, and temporary storage space.
+
+.. _docker-install:
+
+Installing the Docker image
+---------------------------
+
+If you have `Docker <https://docs.docker.com/>`_ installed on your system, you can install a Docker image of the latest release.
+
+The recommended OCRmyPDF Docker image is currently named ``ocrmypdf-alpine``:
+
+.. code-block:: bash
+
+    docker pull jbarlow83/ocrmypdf-alpine
+
+Follow the Docker installation instructions for your platform.  If you can run this command successfully, your system is ready to download and execute the image:
+
+.. code-block:: bash
+
+    docker run hello-world
+
+OCRmyPDF will use all available CPU cores.  By default, the VirtualBox machine instance on Windows and macOS has only a single CPU core enabled. Use the VirtualBox Manager to determine the name of your Docker engine host, and then follow these optional steps to enable multiple CPUs:
+
+.. code-block:: bash
+
+    # Optional step for Mac OS X users
+    docker-machine stop "yourVM"
+    VBoxManage modifyvm "yourVM" --cpus 2  # or whatever number of core is desired
+    docker-machine start "yourVM"
+    eval $(docker-machine env "yourVM")
+
+Using the Docker image on the command line
+------------------------------------------
+
+**Unlike typical Docker containers**, in this mode we are using the OCRmyPDF Docker container is intended to be emphemeral – it runs for one OCR job and then terminates, just like a command line program. We are using Docker as a way of delivering an application, not a server.
+
+To start a Docker container (instance of the image):
+
+.. code-block:: bash
+
+    docker tag jbarlow83/ocrmypdf-alpine ocrmypdf
+    docker run --rm ocrmypdf (... all other arguments here...)
+
+For convenience, create a shell alias to hide the Docker command:
+
+.. code-block:: bash
+
+    alias ocrmypdf='docker run --rm -v "$(pwd):/home/docker" ocrmypdf'
+    ocrmypdf --version  # runs docker version
+
+Or in the wonderful `fish shell <https://fishshell.com/>`_:
+
+.. code-block:: fish
+
+    alias ocrmypdf 'docker run --rm ocrmypdf'
+    funcsave ocrmypdf
+
+.. _docker-lang-packs:
+
+Adding languages to the Docker image
+------------------------------------
+
+By default the Docker image includes English, German and Simplified Chinese, the most popular languages for OCRmyPDF users based on feedback. You may add other languages by creating a new Dockerfile based on the public one:
+
+.. code-block:: dockerfile
+
+    FROM jbarlow83/ocrmypdf-alpine
+
+    # Add French
+    RUN apk add tesseract-ocr-data-fra
+
+Executing the test suite
+------------------------
+
+The OCRmyPDF test suite is installed with image.  To run it:
+
+.. code-block:: bash
+
+    docker run --entrypoint python3 jbarlow83/ocrmypdf-alpine setup.py test
+
+Using the OCRmyPDF web service wrapper
+--------------------------------------
+
+The OCRmyPDF Docker image includes an example, barebones HTTP web service. The webservice may be launched as follows:
+
+.. code-block:: bash
+
+    docker run --entrypoint python3 -p 5000:5000 jbarlow83/ocrmypdf-alpine webservice.py
+
+Unlike command line usage this program will open a socket and wait for connections.
+
+.. warning::
+
+    The OCRmyPDF web service wrapper is intended for demonstration or development. It provides no security, no authentication, no protection against denial of service attacks, and no load balancing. The default Flask WSGI server is used, which is intended for development only. The server is single-threaded and so can respond to only one client at a time. It cannot respond to clients while busy with OCR.
+
+Clients must keep their open connection while waiting for OCR to complete. This may entail setting a long timeout; this interface is more useful for internal HTTP API calls.
+
+Unlike the rest of OCRmyPDF, this web service is licensed under the Affero GPLv3 (AGPLv3) since Ghostscript, a dependency of OCRmyPDF, is also licensed in this way.
+
+In addition to the above, please read our :ref:`general remarks on using OCRmyPDF as a service <ocr-service>`_.
+
+Legacy Ubuntu Docker images
+---------------------------
+
+Previously OCRmyPDF was delivered in several Docker images for different purposes, based on Ubuntu.
+
+The Ubuntu-based images will be maintained for some time but should not be used for new deployments. They are as follows:
+
+.. list-table::
+    :widths: auto
+    :header-rows: 1
+
+    *   - Image name
+        - Download command
+        - Notes
+    *   - ocrmypdf
+        - ``docker pull jbarlow83/ocrmypdf``
+        - Latest ocrmypdf with Tesseract 4.0.0-beta1 on Ubuntu 18.04. Includes English, French, German, Spanish, Portugeuse and Simplified Chinese.
+    *   - ocrmypdf-polyglot
+        - ``docker pull jbarlow83/ocrmypdf-polyglot``
+        - As above, with all available language packs.
+    *   - ocrmypdf-webservice
+        - ``docker pull jbarlow83/ocrmypdf-webservice``
+        - All language packs, and a simple HTTP wrapper allowing OCRmyPDF to be used as a web service. Note that this component is licensed under AGPLv3.
+
+To execute the Ubuntu-based OCRmyPDF on a local file, you must `provide a writable volume to the Docker image <https://docs.docker.com/userguide/dockervolumes/>`_, and both the input and output file must be inside the writable volume. This limitation applies only to the legacy images.
+
+This example command uses the current working directory as the writable volume:
+
+.. code-block:: bash
+
+    docker run --rm -v "$(pwd):/home/docker" <other docker arguments>   ocrmypdf <your arguments to ocrmypdf>
+
+In this worked example, the current working directory contains an input file called ``test.pdf`` and the output will go to ``output.pdf``:
+
+.. code-block:: bash
+
+    docker run --rm -v "$(pwd):/home/docker"   ocrmypdf --skip-text test.pdf output.pdf
+
+.. note:: The working directory should be a writable local volume or Docker may not have permission to access it.
+
+Note that ``ocrmypdf`` has its own separate ``-v VERBOSITYLEVEL`` argument to control debug verbosity. All Docker arguments should before the ``ocrmypdf`` image name and all arguments to ``ocrmypdf`` should be listed after.
+
+In some environments the permissions associated with Docker can be complex to configure. The process that executes Docker may end up not having the permissions to write the specified file system. In that case one can stream the file into and out of the Docker process and avoid all permission hassles, using ``-`` as the input and output filename:
+
+.. code-block:: bash
+
+    docker run --rm -i   ocrmypdf <other arguments to ocrmypdf> - - <input.pdf >output.pdf

docs/index.rst

@@ -25,6 +25,7 @@ PDF is the best format for storing and exchanging scanned documents.  Unfortunat
    :maxdepth: 2
 
    cookbook
+   docker
    advanced
    batch
    security

docs/installation.rst

@@ -1,5 +1,5 @@
-Installation
-============
+Installing OCRmyPDF
+===================
 
 .. |latest| image:: https://img.shields.io/pypi/v/ocrmypdf.svg
     :alt: OCRmyPDF latest released version on PyPI
@@ -103,7 +103,7 @@ sources`_.
     OCRmyPDF works fine without it but will produce larger output files. If you
     build jbig2enc from source, ocrmypdf 7.0.0 and later will automatically
     detect it on the ``PATH``. To add JBIG2 encoding, see `Installing the JBIG2
-    encoder`_.
+    encoder <jbig2>`_.
 
 Installing the latest version on Ubuntu 18.04 LTS
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -260,14 +260,14 @@ These installation instructions omit the optional dependency ``unpaper``, which
 
 To add JBIG2 encoding, see :ref:`jbig2`.
 
-ArchLinux
-^^^^^^^^^
+ArchLinux (AUR)
+^^^^^^^^^^^^^^^
 
 .. image:: https://repology.org/badge/version-for-repo/aur/ocrmypdf.svg
     :alt: ArchLinux
     :target: https://repology.org/metapackage/ocrmypdf
 
-The author is aware of an `ArchLinux User Repository package for ocrmypdf <https://aur.archlinux.org/packages/ocrmypdf/>`_. You can use the following command.
+There is an `ArchLinux User Repository package for ocrmypdf <https://aur.archlinux.org/packages/ocrmypdf/>`_. You can use the following command.
 
 .. code-block:: bash
 
@@ -298,6 +298,12 @@ OCRmyPDF is now a standard `Homebrew <https://brew.sh>`_ formula. To install on
 
     brew install ocrmypdf
 
+This will include only the English language pack. If you need other languages you can optionally install them all:
+
+.. code-block:: bash
+
+    brew install tesseract-lang  # Optional: Install all language packs
+
 .. note::
 
     Users who previously installed OCRmyPDF on macOS using ``pip install ocrmypdf`` should remove the pip version (``pip3 uninstall ocrmypdf``) before switching to the Homebrew version.
@@ -358,133 +364,19 @@ The command line program should now be available:
 
     ocrmypdf --help
 
-.. _docker-install:
-
 Installing the Docker image
 ---------------------------
 
 For some users, installing the Docker image will be easier than installing all of OCRmyPDF's dependencies. For Windows, it is the only option.
 
-If you have `Docker <https://docs.docker.com/>`_ installed on your system, you can install a Docker image of the latest release.
-
-Follow the Docker installation instructions for your platform.  If you can run this command successfully, your system is ready to download and execute the image:
-
-.. code-block:: bash
-
-    docker run hello-world
-
-OCRmyPDF will use all available CPU cores.  By default, the VirtualBox machine instance on Windows and macOS has only a single CPU core enabled. Use the VirtualBox Manager to determine the name of your Docker engine host, and then follow these optional steps to enable multiple CPUs:
-
-.. code-block:: bash
-
-    # Optional step for Mac OS X users
-    docker-machine stop "yourVM"
-    VBoxManage modifyvm "yourVM" --cpus 2  # or whatever number of core is desired
-    docker-machine start "yourVM"
-    eval $(docker-machine env "yourVM")
-
-Assuming you have a Docker engine running, you can download one of the three available images:
-
-.. list-table::
-    :widths: auto
-    :header-rows: 1
-
-    *   - Image name
-        - Download command
-        - Notes
-    *   - ocrmypdf
-        - ``docker pull jbarlow83/ocrmypdf``
-        - Latest ocrmypdf with Tesseract 4.0.0-beta1 on Ubuntu 18.04. Includes English, French, German, Spanish, Portugeuse and Simplified Chinese.
-    *   - ocrmypdf-polyglot
-        - ``docker pull jbarlow83/ocrmypdf-polyglot``
-        - As above, with all available language packs.
-    *   - ocrmypdf-webservice
-        - ``docker pull jbarlow83/ocrmypdf-polyglot``
-        - All language packs, and a simple HTTP wrapper allowing OCRmyPDF to be used as a web service. Note that this component is licensed under AGPLv3.
-
-For example:
-
-.. code-block:: bash
-
-    docker pull jbarlow83/ocrmypdf
-
-Then tag it to give a more convenient name, just ocrmypdf:
-
-.. code-block:: bash
-
-    docker tag jbarlow83/ocrmypdf ocrmypdf
-
-.. _docker-polyglot:
-
-The alternative "polyglot" image provides `all available language packs <https://github.com/tesseract-ocr/tesseract/blob/master/doc/tesseract.1.asc#languages>`_.
-
-You can then run ocrmypdf using the command:
-
-.. code-block:: bash
-
-    docker run --rm ocrmypdf --help
-
-To execute the OCRmyPDF on a local file, you must `provide a writable volume to the Docker image <https://docs.docker.com/userguide/dockervolumes/>`_, and both the input and output file must be inside the writable volume.  This example command uses the current working directory as the writable volume:
-
-.. code-block:: bash
-
-    docker run --rm -v "$(pwd):/home/docker" <other docker arguments>   ocrmypdf <your arguments to ocrmypdf>
-
-In this worked example, the current working directory contains an input file called ``test.pdf`` and the output will go to ``output.pdf``:
-
-.. code-block:: bash
-
-    docker run --rm -v "$(pwd):/home/docker"   ocrmypdf --skip-text test.pdf output.pdf
-
-.. note:: The working directory should be a writable local volume or Docker may not have permission to access it.
-
-Note that ``ocrmypdf`` has its own separate ``-v VERBOSITYLEVEL`` argument to control debug verbosity. All Docker arguments should before the ``ocrmypdf`` image name and all arguments to ``ocrmypdf`` should be listed after.
-
-In some environments the permissions associated with Docker can be complex to configure. The process that executes Docker may end up not having the permissions to write the specified file system. In that case one can stream the file into and out of the Docker process and avoid all permission hassles, using ``-`` as the input and output filename:
-
-.. code-block:: bash
-
-    docker run --rm -i   ocrmypdf <other arguments to ocrmypdf> - - <input.pdf >output.pdf
-
-For convenience, a shell alias can hide the docker command:
-
-.. code-block:: bash
-
-    alias ocrmypdf='docker run --rm -v "$(pwd):/home/docker" ocrmypdf'
-    ocrmypdf --version  # runs docker version
-
-Or in the wonderful `fish shell <https://fishshell.com/>`_:
-
-.. code-block:: fish
-
-    alias ocrmypdf 'docker run --rm -v (pwd):/home/docker ocrmypdf'
-    funcsave ocrmypdf
-
-.. note::
-
-    The ocrmypdf Docker images are designed for application delivery, to enable use of OCRmyPDF without fussing with dependencies. ``docker run --rm`` argument tells Docker to delete the container after it runs, because each container is only good for a single job. The Docker image is not designed for use as a persistent web service or for use on Amazon EC2 Container Service (AWS ECS).
+See `OCRmyPDF Docker Image <docker>`_ for more information.
 
 Installing on Windows
 ---------------------
 
-Direct installation on Windows is not possible.  `Install the Docker <docker-install_>`_ container as described above.  Ensure that your command prompt can run the docker "hello world" container.
-
-It would probably not be too difficult to run on Windows.  The main reason this has been avoided is the difficulty of packaging and installing the various non-Python dependencies: Tesseract, QPDF, Ghostscript, Leptonica.  Pull requests to add or improve Windows support would be quite welcome.
-
-
-Running on Windows
-~~~~~~~~~~~~~~~~~~
-
-The command line syntax to run ocrmypdf from a command prompt will resemble:
-
-.. code-block:: bat
-
-    docker run -v /c/Users/sampleuser:/home/docker ocrmypdf --skip-text test.pdf output.pdf
-
-where /c/Users/sampleuser is a Unix representation of the Windows path C:\\Users\\sampleuser, assuming a user named "sampleuser" is running ocrmypdf on a file in their home directory, and the files "test.pdf" and "output.pdf" are in the sampleuser folder. The Windows user must have read and write permissions.
-
-`Bash on Ubuntu on Windows <https://github.com/Microsoft/BashOnWindows>`_ should also be a viable route for running the OCRmyPDF Docker container.
+Direct installation on Windows is not possible.  `Install the Docker <docker-install>`_ container as described above.  Ensure that your command prompt can run the docker "hello world" container.
 
+It would probably not be too difficult to port on Windows.  The main reason this has been avoided is the difficulty of packaging and installing the various non-Python dependencies: Tesseract, QPDF, Ghostscript, Leptonica.  Pull requests to add or improve Windows support would be quite welcome.
 
 Installing with Python pip
 --------------------------
@@ -510,7 +402,7 @@ Since ``pip3 install --user`` does not work correctly on some platforms, notably
     pip3 install ocrmypdf
 
 Requirements for pip and HEAD install
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 OCRmyPDF currently requires these external programs and libraries to be installed, and must be satisfied using the operating system package manager. ``pip`` cannot provide them.
 
@@ -584,7 +476,7 @@ dependencies. Older version than the ones mentioned in the release notes
 are likely not to be compatible to OCRmyPDF.
 
 For development
-~~~~~~~~~~~~~~~
+^^^^^^^^^^^^^^^
 
 To install all of the development and test requirements:
 

docs/introduction.rst

@@ -110,10 +110,10 @@ To the author's knowledge, OCRmyPDF is the most feature-rich and thoroughly test
 Web front-ends
 --------------
 
-The Docker image ocrmypdf-webservice provides a web service front-end that allows files to submitted over HTTP and the results "downloaded". This is an HTTP server intended to simplify web services deployments; it is not intended to be deployed on the public internet and no real security measures to speak of.
+The Docker image ``ocrmypdf-alpine`` provides a web service front-end that allows files to submitted over HTTP and the results "downloaded". This is an HTTP server intended to simplify web services deployments; it is not intended to be deployed on the public internet and no real security measures to speak of.
 
-In addition, the following integrations are available:
+In addition, the following third-party integrations are available:
 
 * `Nextcloud OCR <https://github.com/janis91/ocr>`_ is a free software plugin for the Nextcloud private cloud software
 
-Bear in mind that OCRmyPDF is not designed to be secure against malware-bearing PDFs (see `Using OCRmyPDF online`_). Users should ensure they comply with OCRmyPDF's licenses and the licenses of all dependencies. In particular, OCRmyPDF requires Ghostscript, which is licensed under AGPLv3.
+OCRmyPDF is not designed to be secure against malware-bearing PDFs (see `Using OCRmyPDF online <ocr-service>`_). Users should ensure they comply with OCRmyPDF's licenses and the licenses of all dependencies. In particular, OCRmyPDF requires Ghostscript, which is licensed under AGPLv3.

docs/jbig2.rst

@@ -32,4 +32,4 @@ JBIG2 lossy mode does achieve higher compression ratios than any other monochrom
 
 To turn on JBIG2 lossy mode, add the argument ``--jbig2-lossy``. ``--optimize {1,2,3}`` are necessary for the argument to take effect also required. Also, a JBIG2 encoder must be installed as described in the previous section.
 
-*ocrmypdf v7.0 and v7.1 used lossy mode by default.*
+*Due to an oversight, ocrmypdf v7.0 and v7.1 used lossy mode by default.*

docs/languages.rst

@@ -3,7 +3,7 @@
 Installing additional language packs
 ====================================
 
-OCRmyPDF uses Tesseract for OCR, and relies on its language packs for languages other than English. 
+OCRmyPDF uses Tesseract for OCR, and relies on its language packs for languages other than English.
 
 Tesseract supports `most languages <https://github.com/tesseract-ocr/tesseract/blob/master/doc/tesseract.1.asc#languages>`_.
 
@@ -46,31 +46,4 @@ You can install additional language packs by :ref:`installing Tesseract using Ho
 Docker users
 ------------
 
-Users of the Docker image may use the alternative :ref:`"polyglot" container <docker-polyglot>` which includes all languages.
-
-Adding individual language packs to a Docker image
-""""""""""""""""""""""""""""""""""""""""""""""""""
-
-If you wish to add a single language pack, you could do the following:
-
-* Download the desired ``.trainedata`` file from the `tessdata <https://github.com/tesseract-ocr/tessdata>`_ repository. Let's use Hebrew in this example (``heb.traineddata``)
-
-* Copy the file to ``/home/user/downloads/heb.traineddata``.
-
-* Create a new container based on the ocrmypdf-tess4 image and jump into it with a terminal:
-
-.. code-block:: bash
-
-	host$ docker run  -v /home/user/downloads:/home/docker -it --entrypoint /bin/bash ocrmypdf-tess4
-
-* Put the file where Tesseract expects it:
-
-.. code-block:: bash
-
-	docker$ cp /home/docker/heb.traineddata /usr/share/tesseract-ocr/tessdata
-
-* Note the container id, and save it as a new image (in this example, ``ocrmypdf-tess4-heb``)
-
-.. code-block:: bash
-
-    host$ docker commit <container_id> ocrmypdf-tess4-heb
+Users of the OCRmyPDF Docker image should install language packs into a derived Docker image as :ref:`described in that section <docker-lang-packs>`.

docs/release_notes.rst

@@ -13,6 +13,47 @@ Note that it is licensed under GPLv3, so scripts that ``import ocrmypdf`` and ar
    find:    [^`]\#([0-9]{1,3})[^0-9]
    replace: `#$1 <https://github.com/jbarlow83/OCRmyPDF/issues/$1>`_
 
+v8.2.2
+------
+
+-   Fixed a regression from v8.2.0, an exception that occurred while attempting to report that ``unpaper`` or another optional dependency was unavailable.
+
+-   In some cases, ``ocrmypdf [-c|--clean]`` failed to exit with an error when ``unpaper`` is not installed.
+
+v8.2.1
+------
+
+-   This release was canceled.
+
+v8.2.0
+------
+
+-   A major improvement to our Docker image is now available thanks to hard work contributed by @mawi12345. The new Docker image, ocrmypdf-alpine, is based on Alpine Linux, and includes most of the functionality of three existed images in a smaller package. This image will replace the main Docker image eventually but for now all are being built. `See documentation for details <https://ocrmypdf.readthedocs.io/en/latest/docker.html>`_.
+
+-   Documentation reorganized especially around the use of Docker images.
+
+-   Fixed a problem with PDF image optimization, where the optimizer would unnecessarily decompress and recompress PNG images, in some cases losing the benefits of the quantization it just had just performed. The optimizer is now capable of embedding PNG images into PDFs without transcoding them.
+
+-   Fixed a minor regression with lossy JBIG2 image optimization. All JBIG2 candidates images were incorrectly placed into a single optimization group for the whole file, instead of grouping pages together. This usually makes a larger JBIG2Globals dictionary and results in inferior compression, so it worked less well than designed. However, quality would not be impacted. Lossless JBIG2 was entirely unaffected.
+
+-   Updated dependencies, including pikepdf to 1.1.0. This fixes `#358 <https://github.com/jbarlow83/OCRmyPDF/issues/358>`_.
+
+-   The install-time version checks for certain external programs have been removed from setup.py. These tests are now performed at run-time.
+
+-   The non-standard option to override install-time checks (``setup.py install --force``) is now deprecated and prints a warning. It will be removed in a future release.
+
+v8.1.0
+------
+
+-   Added a feature, ``--unpaper-args``, which allows passing arbitrary arguments to ``unpaper`` when using ``--clean`` or ``--clean-final``. The default, very conservative unpaper settings are suppressed.
+
+-   The argument ``--clean-final`` now implies ``--clean``. It was possible to issue ``--clean-final`` on its before this, but it would have no useful effect.
+
+-   Fixed an exception on traversing corrupt table of contents entries (specifically, those with invalid destination objects)
+
+-   Fixed an issue when using ``--tesseract-timeout`` and image processing features on a file with more than 100 pages. `#347 <https://github.com/jbarlow83/OCRmyPDF/issues/347>`_
+
+-   OCRmyPDF now always calls ``os.nice(5)`` to signal to operating systems that it is a background process.
 
 v8.0.1
 ------

docs/security.rst

@@ -25,12 +25,14 @@ Finally, OCRmyPDF rasterizes each page of the PDF using `Ghostscript <http://gho
 
 Depending on the options specified, OCRmyPDF may graft the OCR layer into the existing PDF or it may essentially reconstruct ("re-fry") a visually identical PDF that may be quite different at the binary level. That said, OCRmyPDF is not a tool designed for sanitizing PDFs.
 
+.. _ocr-service:
+
 Using OCRmyPDF online or as a service
 -------------------------------------
 
-OCRmyPDF should not be deployed as a public-facing service, such as a website where a potential attacker could upload a PDF of their choice for OCR. OCRmyPDF is not designed to be secure against PDF malware. Another concern is PDFs specifically designed to be a denial of service attack: PDFs can contain recursive data structures that sometimes send parsers into infinite loops, and issue complex graphics drawing commands.
+OCRmyPDF is not designed for use as a public web service where a malicious user could upload a chosen PDF. In particular, it is not necessarily secure against PDF malware or PDFs that cause denial of service. OCRmyPDF relies on Ghostscript, and therefore, if deployed online one should be prepared to comply with Ghostscript's Affero GPL license, OCRmyPDF's GPL license, and any other licenses.
 
-Setting aside these concerns, a side effect of OCRmyPDF is it may incidentally sanitize PDFs that contain malware. It runs ``qpdf`` to repair the PDF, which could correct malformed PDF structures that are part of an attack. When PDF/A output is selected (the default), the input PDF is partially reconstructed by Ghostscript. When ``--force-ocr`` is used, all pages are rasterized and reconverted to PDF, which could remove malware in embedded images. No guarantees.
+Setting aside these concerns, a side effect of OCRmyPDF is it may incidentally sanitize PDFs that contain certain types of malware. It runs ``qpdf`` to repair the PDF, which could correct malformed PDF structures that are part of an attack. When PDF/A output is selected (the default), the input PDF is partially reconstructed by Ghostscript. When ``--force-ocr`` is used, all pages are rasterized and reconverted to PDF, which could remove malware in embedded images.
 
 OCRmyPDF should be relatively safe to use in a trusted intranet, with some considerations:
 

pyproject.toml

@@ -0,0 +1,32 @@
+[build-system]
+requires = [
+  "setuptools >= 30.3.0",
+  "wheel",
+  "cffi",
+  "setuptools_scm",
+  "setuptools_scm_git_archive"
+]
+build-backend = "setuptools.build_meta"
+
+[tool.black]
+line-length = 88
+py36 = true
+skip-string-normalization = true
+include = '\.pyi?$'
+exclude = '''
+/(
+    \.eggs
+  | \.git
+  | \.hg
+  | \.mypy_cache
+  | \.tox
+  | \.venv
+  | _build
+  | buck-out
+  | build
+  | dist
+  | docs
+  | misc
+  | \.egg-info
+)/
+'''

requirements/main.txt

@@ -2,12 +2,12 @@
 # setup.py lists a separate set of requirements that are looser to simplify
 # installation
 chardet == 3.0.4
-cffi == 1.11.5
-img2pdf == 0.3.1
+cffi == 1.12.2
+img2pdf == 0.3.3
 pdfminer.six == 20181108
-pikepdf == 1.0.2
+pikepdf == 1.1.0
 Pillow >= 5.0.0, != 5.1.0 ; sys_platform == "darwin"
 pycparser == 2.19
 python-xmp-toolkit == 2.0.1
-reportlab == 3.5.9
-ruffus == 2.8.0
+reportlab == 3.5.13
+ruffus == 2.8.1

requirements/test.txt

@@ -1,7 +1,7 @@
-pytest == 3.9.3
-pytest-helpers-namespace
+pytest == 4.3.0
+pytest-helpers-namespace >= 2019.1.8
 pytest-xdist
-pytest-cov
+pytest-cov >= 2.6.1
 python-xmp-toolkit   # requires apt-get install libexempi3
                      # or brew install exempi
 PyPDF2 >= 1.26.0

setup.py

@@ -20,6 +20,7 @@
 from __future__ import print_function, unicode_literals
 
 import sys
+
 if sys.version_info < (3, 6):
     print("Python 3.6 or newer is required", file=sys.stderr)
     sys.exit(1)
@@ -31,171 +32,19 @@ import re
 
 # pylint: disable=w0613
 
-missing_program = '''
-The program '{program}' could not be executed or was not found on your
-system PATH.
-'''
-
-unknown_version = '''
-OCRmyPDF requires '{program}' {need_version} or higher.  Your system has
-'{program}' but we cannot tell what version is installed.  Contact the
-package maintainer.
-'''
-
-old_version = '''
-OCRmyPDF requires '{program}' {need_version} or higher.  Your system appears
-to have {found_version}.  Please update this program.
-'''
-
-okay_its_optional = '''
-This program is OPTIONAL, so installation of OCRmyPDF can proceed, but
-some functionality may be missing.
-'''
-
-not_okay_its_required = '''
-This program is REQUIRED for OCRmyPDF to work.  Installation will abort.
-'''
-
-osx_install_advice = '''
-If you have homebrew installed, try these command to install the missing
-packages:
-    brew update
-    brew upgrade
-    brew install {package}
-'''
-
-linux_install_advice = '''
-On systems with the aptitude package manager (Debian, Ubuntu), try these
-commands:
-    sudo apt-get update
-    sudo apt-get install {package}
-
-On RPM-based systems (Red Hat, Fedora), search for instructions on
-installing the RPM for {program}.
-'''
-
-
-def get_platform():
-    if sys.platform.startswith('freebsd'):
-        return 'freebsd'
-    elif sys.platform.startswith('linux'):
-        return 'linux'
-    return sys.platform
-
-
-def _error_trailer(program, package, optional, **kwargs):
-    if optional:
-        print(okay_its_optional.format(**locals()), file=sys.stderr)
-    else:
-        print(not_okay_its_required.format(**locals()), file=sys.stderr)
-
-    if isinstance(package, Mapping):
-        package = package[get_platform()]
-
-    if get_platform() == 'darwin':
-        print(osx_install_advice.format(**locals()), file=sys.stderr)
-    elif get_platform() == 'linux':
-        print(linux_install_advice.format(**locals()), file=sys.stderr)
-
-
-def error_missing_program(
-        program,
-        package,
-        optional
-        ):
-    print(missing_program.format(**locals()), file=sys.stderr)
-    _error_trailer(**locals())
-
-
-def error_unknown_version(
-        program,
-        package,
-        optional,
-        need_version
-        ):
-    print(unknown_version.format(**locals()), file=sys.stderr)
-    _error_trailer(**locals())
-
-
-def error_old_version(
-        program,
-        package,
-        optional,
-        need_version,
-        found_version
-        ):
-    print(old_version.format(**locals()), file=sys.stderr)
-    _error_trailer(**locals())
-
-
-def check_external_program(
-        program,
-        need_version,
-        package,
-        version_check_args=None,
-        version_scrape_regex=re.compile(r'(\d+\.\d+(?:\.\d+)?)'),
-        optional=False):
-    if not version_check_args:
-        version_check_args = ['--version']
-    print(f'Checking for {program} >= {need_version}...')
-    try:
-        result = check_output(
-                [program] + version_check_args,
-                universal_newlines=True, stderr=STDOUT)
-    except (CalledProcessError, FileNotFoundError):
-        error_missing_program(program, package, optional)
-        if not optional:
-            sys.exit(1)
-        print(f'Continuing install without {program}')
-        return
-
-    try:
-        found_version = version_scrape_regex.search(result).group(1)
-    except AttributeError:
-        error_unknown_version(program, package, optional, need_version)
-        sys.exit(1)
-
-    if found_version < need_version:
-        error_old_version(program, package, optional, need_version,
-                          found_version)
-
-    print(f'Found {program} {found_version}')
-
 
 command = next((arg for arg in sys.argv[1:] if not arg.startswith('-')), '')
-forced = ('--force' in sys.argv)
+if command.startswith('install') or command in [
+    'check',
+    'test',
+    'nosetests',
+    'easy_install',
+]:
+    forced = '--force' in sys.argv
+    if forced:
+        print("The argument --force is deprecated. Please discontinue use.")
 
 
-if not forced and command.startswith('install') or \
-        command in ['check', 'test', 'nosetests', 'easy_install']:
-    check_external_program(
-        program='tesseract',
-        need_version='4.0.0',  # using backport for Travis CI
-        package={'darwin': 'tesseract', 'linux': 'tesseract-ocr'}
-    )
-    check_external_program(
-        program='gs',
-        need_version='9.15',  # limited by Travis CI / Ubuntu 14.04 backports
-        package='ghostscript'
-    )
-    check_external_program(
-        program='unpaper',
-        need_version='6.1',   # latest sane version
-        package='unpaper',
-        optional=True
-    )
-    check_external_program(
-        program='qpdf',
-        need_version='8.0.2', # test suite known to fail on 5.1.1
-        package='qpdf'
-    )
-    check_external_program(
-        program='pngquant',
-        need_version='2.0.0',
-        package='pngquant',
-        optional=True
-    )
-
 if 'upload' in sys.argv[1:]:
     print('Use twine to upload the package - setup.py upload is insecure')
     sys.exit(1)
@@ -207,6 +56,7 @@ def readme():
     with open('README.md', encoding='utf-8') as f:
         return f.read()
 
+
 setup(
     name='ocrmypdf',
     description='OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched',
@@ -234,45 +84,37 @@ setup(
         "Topic :: Scientific/Engineering :: Image Recognition",
         "Topic :: Text Processing :: Indexing",
         "Topic :: Text Processing :: Linguistic",
-        ],
+    ],
     python_requires=' >= 3.6',
-    setup_requires=[
-        'cffi >= 1.9.1',        # to build the leptonica module
-        'pytest-runner',        # to enable python setup.py test
-        'setuptools_scm',       # so that version will work
-        'setuptools_scm_git_archive'    # enable version from github tarballs
+    setup_requires=[  # can be removed whenever we can drop pip 9 support
+        'cffi >= 1.9.1',  # to build the leptonica module
+        'pytest-runner',  # to enable python setup.py test
+        'setuptools_scm',  # so that version will work
+        'setuptools_scm_git_archive',  # enable version from github tarballs
     ],
     use_scm_version={'version_scheme': 'post-release'},
-    cffi_modules=[
-        'src/ocrmypdf/lib/compile_leptonica.py:ffibuilder'
-    ],
+    cffi_modules=['src/ocrmypdf/lib/compile_leptonica.py:ffibuilder'],
     install_requires=[
         'chardet >= 3.0.4, < 4',  # unlisted requirement of pdfminer.six 20181108
-        'cffi >= 1.9.1',          # must be a setup and install requirement
-        'img2pdf >= 0.3.0, < 0.4',       # pure Python, so track HEAD closely
+        'cffi >= 1.9.1',  # must be a setup and install requirement
+        'img2pdf >= 0.3.0, < 0.4',  # pure Python, so track HEAD closely
         'pdfminer.six == 20181108 ; sys_platform != "darwin"',
-        'pikepdf >= 1.0.5, < 2',
+        'pikepdf >= 1.1.0, < 2',
         'Pillow >= 4.0.0, != 5.1.0 ; sys_platform == "darwin"',
-                                  # Pillow < 4 has BytesIO/TIFF bug w/img2pdf 0.2.3
-                                  # block 5.1.0, broken wheels
-        'reportlab >= 3.3.0',     # oldest released version with sane image handling
+        # Pillow < 4 has BytesIO/TIFF bug w/img2pdf 0.2.3
+        # block 5.1.0, broken wheels
+        'reportlab >= 3.3.0',  # oldest released version with sane image handling
         'ruffus >= 2.7.0',
     ],
-    extras_require={
-        'pdfminer': ['pdfminer.six == 20181108'],
-    },
+    extras_require={'pdfminer': ['pdfminer.six == 20181108']},
     tests_require=tests_require,
-    entry_points={
-        'console_scripts': [
-            'ocrmypdf = ocrmypdf.__main__:run_pipeline'
-        ],
-    },
+    entry_points={'console_scripts': ['ocrmypdf = ocrmypdf.__main__:run_pipeline']},
     package_data={'ocrmypdf': ['data/sRGB.icc']},
     include_package_data=True,
     zip_safe=False,
     project_urls={
         'Documentation': 'https://ocrmypdf.readthedocs.io/',
         'Source': 'https://github.com/jbarlow83/ocrmypdf',
-        'Tracker': 'https://github.com/jbarlow83/ocrmypdf/issues'
-    }
+        'Tracker': 'https://github.com/jbarlow83/ocrmypdf/issues',
+    },
 )

src/ocrmypdf/__main__.py

@@ -44,7 +44,15 @@ from .exceptions import (
     MissingDependencyError,
     OutputFileAccessError,
 )
-from .exec import ghostscript, qpdf, tesseract
+from .exec import (
+    ghostscript,
+    jbig2enc,
+    qpdf,
+    tesseract,
+    check_external_program,
+    unpaper,
+    pngquant,
+)
 from .helpers import available_cpu_count, is_file_writable, re_symlink
 from .pdfa import file_claims_pdfa
 
@@ -67,13 +75,6 @@ if 'IDE_PROJECT_ROOTS' in os.environ:
 
 verify_python3_env()
 
-if not tesseract.v4:
-    complain(
-        f"Please install tesseract 4.0.0 or newer "
-        f"(currently installed version is {tesseract.version()})"
-    )
-    sys.exit(ExitCode.missing_dependency)
-
 # -------------
 # Parser
 
@@ -278,6 +279,13 @@ preprocessing.add_argument(
     help="Clean page as above, and incorporate the cleaned image in the final "
     "PDF.  Might remove desired content.",
 )
+preprocessing.add_argument(
+    '--unpaper-args',
+    type=str,
+    default=None,
+    help="A quoted string of arguments to pass to unpaper. Requires --clean. "
+    "Example: --unpaper-args '--layout double'.",
+)
 preprocessing.add_argument(
     '--oversample',
     metavar='DPI',
@@ -595,40 +603,27 @@ def check_options_sidecar(options, log):
         options.sidecar = options.output_file + '.txt'
 
 
-def _optional_program_required(name, version_fn, min_version, for_argument):
-    try:
-        if version_fn() < min_version:
-            raise MissingDependencyError(
-                f"The installed '{name}' is not supported. "
-                f"Install version {min_version} or newer."
-            )
-    except (FileNotFoundError, MissingDependencyError):
-        raise MissingDependencyError(
-            f"Install the '{name}' program to use {for_argument}."
-        )
-
-
-def _optional_program_recommended(name, version_fn, min_version, for_argument):
-    try:
-        if version_fn() < min_version:
-            raise MissingDependencyError(
-                f"The installed '{name}' is not supported. "
-                f"Install version {min_version} or newer."
-            )
-    except (FileNotFoundError, MissingDependencyError):
-        complain(
-            f"For best results, install the optional program '{name}' to use the "
-            f"argument {for_argument}."
-        )
-
-
 def check_options_preprocessing(options, log):
-    if any((options.clean, options.clean_final)):
-        from .exec import unpaper
-
-        _optional_program_required(
-            'unpaper', unpaper.version, '6.1', '--clean, --clean-final'
+    if options.clean_final:
+        options.clean = True
+    if options.unpaper_args and not options.clean:
+        raise argparse.ArgumentError(None, "--clean is required for --unpaper-args")
+    if options.clean:
+        check_external_program(
+            log=log,
+            program='unpaper',
+            package='unpaper',
+            version_checker=unpaper.version,
+            need_version='6.1',
+            required_for=['--clean, --clean-final'],
         )
+        try:
+            if options.unpaper_args:
+                options.unpaper_args = unpaper.validate_custom_args(
+                    options.unpaper_args
+                )
+        except Exception as e:
+            raise argparse.ArgumentError(None, str(e))
 
 
 def check_options_ocr_behavior(options, log):
@@ -646,19 +641,26 @@ def check_options_ocr_behavior(options, log):
 
 def check_options_optimizing(options, log):
     if options.optimize >= 2:
-        from .exec import pngquant, jbig2enc
-
-        _optional_program_required(
-            'pngquant', pngquant.version, '2.0.1', '--optimize {2,3}'
+        check_external_program(
+            log=log,
+            program='pngquant',
+            package='pngquant',
+            version_checker=pngquant.version,
+            need_version='2.0.1',
+            required_for='--optimize {2,3}',
         )
 
-    if options.jbig2_lossy:
-        _optional_program_required('jbig2', jbig2enc.version, '0.28', '--jbig2-lossy')
-    elif options.optimize >= 2:
+    if options.optimize >= 2:
         # Although we use JBIG2 for optimize=1, don't nag about it unless the
         # user is asking for more optimization
-        _optional_program_recommended(
-            'jbig2', jbig2enc.version, '0.28', '--optimize {2,3}'
+        check_external_program(
+            log=log,
+            program='jbig2',
+            package='jbig2enc',
+            version_checker=jbig2enc.version,
+            need_version='0.28',
+            required_for='--optimize {2,3} | --jbig2-lossy',
+            recommended=True if not options.jbig2_lossy else False,
         )
 
     if options.optimize == 0 and any(
@@ -910,9 +912,6 @@ def log_page_orientations(pdfinfo, _log):
 
 def preamble(_log):
     _log.debug('ocrmypdf ' + VERSION)
-    _log.debug('tesseract ' + tesseract.version())
-    _log.debug('qpdf ' + qpdf.version())
-    _log.debug('gs ' + ghostscript.version())
 
 
 def check_environ(options, _log):
@@ -1012,6 +1011,37 @@ def report_output_file_size(options, _log, input_file, output_file):
     )
 
 
+def check_dependency_versions(options, log):
+    check_external_program(
+        log=log,
+        program='tesseract',
+        package={'darwin': 'tesseract', 'linux': 'tesseract-ocr'},
+        version_checker=tesseract.version,
+        need_version='4.0.0',  # using backport for Travis CI
+    )
+    check_external_program(
+        log=log,
+        program='gs',
+        package='ghostscript',
+        version_checker=ghostscript.version,
+        need_version='9.15',  # limited by Travis CI / Ubuntu 14.04 backports
+    )
+    if ghostscript.version() == '9.24':
+        complain(
+            "Ghostscript 9.24 contains serious regressions and is not "
+            "supported. Please upgrade to Ghostscript 9.25 or use an older "
+            "version."
+        )
+        return ExitCode.missing_dependency
+    check_external_program(
+        log=log,
+        program='qpdf',
+        package='qpdf',
+        version_checker=qpdf.version,
+        need_version='8.0.2',
+    )
+
+
 def run_pipeline(args=None):
     options = parser.parse_args(args=args)
     options.verbose_abbreviated_path = 1
@@ -1028,24 +1058,7 @@ def run_pipeline(args=None):
     )
     preamble(_log)
     check_options(options, _log)
-
-    # Complain about qpdf version < 7.0.0
-    # Suppress the warning if in the test suite, since there are no PPAs
-    # for qpdf 7.0.0 for Ubuntu trusty (i.e. Travis)
-    if qpdf.version() < '7.0.0' and not os.environ.get('PYTEST_CURRENT_TEST'):
-        complain(
-            f"You are using qpdf version {qpdf.version()} which has known issues including "
-            f"security vulnerabilities with certain malformed PDFs. Consider "
-            f"upgrading to version 7.0.0 or newer."
-        )
-
-    if ghostscript.version() == '9.24':
-        complain(
-            "Ghostscript 9.24 contains serious regressions and is not "
-            "supported. Please upgrade to Ghostscript 9.25 or use an older "
-            "version."
-        )
-        return ExitCode.missing_dependency
+    check_dependency_versions(options, _log)
 
     # Any changes to options will not take effect for options that are already
     # bound to function parameters in the pipeline. (For example
@@ -1081,6 +1094,8 @@ def run_pipeline(args=None):
 
         build_pipeline(options, work_folder, _log, context)
         atexit.register(cleanup_working_files, work_folder, options)
+        if hasattr(os, 'nice'):
+            os.nice(5)
         cmdline.run(options)
     except ruffus_exceptions.RethrownJobError as e:
         if options.verbose:

src/ocrmypdf/_pipeline.py

@@ -566,7 +566,7 @@ def preprocess_clean(input_file, output_file, log, context):
     pageinfo = get_pageinfo(input_file, context)
     dpi = get_page_square_dpi(pageinfo, options)
 
-    unpaper.clean(input_file, output_file, dpi, log)
+    unpaper.clean(input_file, output_file, dpi, log, options.unpaper_args)
 
 
 def select_ocr_image(infiles, output_file, log, context):
@@ -648,7 +648,7 @@ def ocr_tesseract_hocr(input_file, output_files, log, context):
 
 
 def select_visible_page_image(infiles, output_file, log, context):
-    "Selects a whole page image that we can show the user (if necessary)"
+    """Selects a whole page image that we can show the user (if necessary)"""
 
     options = context.get_options()
     if options.clean_final:

src/ocrmypdf/_unicodefun.py

@@ -58,9 +58,9 @@ def verify_python3_env():  # pragma: no cover
     if os.name == 'posix':
         import subprocess
 
-        rv = subprocess.Popen(
+        rv = subprocess.run(
             ['locale', '-a'], stdout=subprocess.PIPE, stderr=subprocess.PIPE
-        ).communicate()[0]
+        ).stdout
         good_locales = set()
         has_c_utf8 = False
 

src/ocrmypdf/_weave.py

@@ -17,6 +17,7 @@
 
 from itertools import groupby
 from pathlib import Path
+import os
 
 import pikepdf
 
@@ -24,8 +25,11 @@ from .exec import tesseract
 from .helpers import flatten_groups, page_number
 
 
+MAX_OPEN_PAGE_PDFS = int(os.environ.get('_OCRMYPDF_MAX_OPEN_PAGE_PDFS', 100))
+
+
 def _update_page_resources(*, page, font, font_key, procset):
-    "Update this page's fonts with a reference to the Glyphless font"
+    """Update this page's fonts with a reference to the Glyphless font"""
 
     if '/Resources' not in page:
         page['/Resources'] = pikepdf.Dictionary({})
@@ -34,7 +38,7 @@ def _update_page_resources(*, page, font, font_key, procset):
         fonts = resources['/Font']
     except KeyError:
         fonts = pikepdf.Dictionary({})
-    if font_key not in fonts:
+    if font_key is not None and font_key not in fonts:
         fonts[font_key] = font
     resources['/Font'] = fonts
 
@@ -160,7 +164,7 @@ def _weave_layers_graft(
 
 
 def _find_font(text, pdf_base):
-    "Copy a font from the filename text into pdf_base"
+    """Copy a font from the filename text into pdf_base"""
 
     font, font_key = None, None
     possible_font_names = ('/f-0-0', '/F1')
@@ -245,12 +249,14 @@ def _fix_toc(pdf_base, pageref_remap, log):
         Inner helper function: change the objgen for any page from the old we
         invalidated to its new one.
         """
-        if not isinstance(dest_node, pikepdf.Array):
-            return
-        pageref = dest_node[0]
-        if pageref['/Type'] == '/Page' and pageref.objgen in pageref_remap:
-            new_objgen = pageref_remap[pageref.objgen]
-            dest_node[0] = pdf_base.get_object(new_objgen)
+        try:
+            pageref = dest_node[0]
+            if pageref['/Type'] == '/Page' and pageref.objgen in pageref_remap:
+                new_objgen = pageref_remap[pageref.objgen]
+                dest_node[0] = pdf_base.get_object(new_objgen)
+        except (IndexError, TypeError) as e:
+            log.warning("This file may contain invalid table of contents entries")
+            log.debug(e)
 
     def visit_remap_dest(pdf_base, node, log):
         """
@@ -390,7 +396,7 @@ def weave_layers(infiles, output_file, log, context):
             content_rotation - autorotate_correction
         ) % 360
 
-        if len(keep_open) > 100:
+        if len(keep_open) > MAX_OPEN_PAGE_PDFS:
             # qpdf limitations require us to keep files open when we intend
             # to copy content from them before saving. However, we want to keep
             # a lid on file handles and memory usage, so for big files we're
@@ -407,7 +413,7 @@ def weave_layers(infiles, output_file, log, context):
 
             pdf_base = pikepdf.open(interim)
             procset = pdf_base.pages[0].Resources.ProcSet
-            font = pdf_base.pages[0].Resources.Font.get(font_key)
+            font, font_key = None, None  # Reacquire this information
 
     _fix_toc(pdf_base, pagerefs, log)
     pdf_base.save(output_file)

src/ocrmypdf/exec/__init__.py

@@ -21,7 +21,8 @@ import os
 import re
 import sys
 from subprocess import run, STDOUT, PIPE, CalledProcessError
-from ..exceptions import MissingDependencyError
+from ..exceptions import MissingDependencyError, ExitCode
+from collections.abc import Mapping
 
 
 def get_version(program, *, version_arg='--version', regex=r'(\d+(\.\d+)*)'):
@@ -42,7 +43,7 @@ def get_version(program, *, version_arg='--version', regex=r'(\d+(\.\d+)*)'):
             f"Could not find program '{program}' on the PATH"
         ) from e
     except CalledProcessError as e:
-        if e.returncode < 0:
+        if e.returncode != 0:
             raise MissingDependencyError(
                 f"Ran program '{program}' but it exited with an error:\n{e.output}"
             ) from e
@@ -58,3 +59,115 @@ def get_version(program, *, version_arg='--version', regex=r'(\d+(\.\d+)*)'):
         )
 
     return version
+
+
+missing_program = '''
+The program '{program}' could not be executed or was not found on your
+system PATH.
+'''
+
+missing_optional_program = '''
+The program '{program}' could not be executed or was not found on your
+system PATH.  This program is required when you use the
+{required_for} arguments.  You could try omitting these arguments, or install
+the package.
+'''
+
+missing_recommend_program = '''
+The program '{program}' could not be executed or was not found on your
+system PATH.  This program is recommended when using the {required_for} arguments,
+but not required, so we will proceed.  For best results, install the program.
+'''
+
+old_version = '''
+OCRmyPDF requires '{program}' {need_version} or higher.  Your system appears
+to have {found_version}.  Please update this program.
+'''
+
+old_version_required_for = '''
+OCRmyPDF requires '{program}' {need_version} or higher when run with the
+{required_for} arguments.  If you omit these arguments, OCRmyPDF may be able to
+proceed.  For best results, install the program.
+'''
+
+osx_install_advice = '''
+If you have homebrew installed, try these command to install the missing
+package:
+    brew install {package}
+'''
+
+linux_install_advice = '''
+On systems with the aptitude package manager (Debian, Ubuntu), try these
+commands:
+    sudo apt-get update
+    sudo apt-get install {package}
+
+On RPM-based systems (Red Hat, Fedora), search for instructions on
+installing the RPM for {program}.
+'''
+
+
+def _get_platform():
+    if sys.platform.startswith('freebsd'):
+        return 'freebsd'
+    elif sys.platform.startswith('linux'):
+        return 'linux'
+    return sys.platform
+
+
+def _error_trailer(log, program, package, **kwargs):
+    if isinstance(package, Mapping):
+        package = package[_get_platform()]
+
+    if _get_platform() == 'darwin':
+        log.info(osx_install_advice.format(**locals()))
+    elif _get_platform() == 'linux':
+        log.info(linux_install_advice.format(**locals()))
+
+
+def _error_missing_program(log, program, package, required_for, recommended):
+    if required_for:
+        log.error(missing_optional_program.format(**locals()))
+    elif recommended:
+        log.info(missing_recommend_program.format(**locals()))
+    else:
+        log.error(missing_program.format(**locals()))
+    _error_trailer(**locals())
+
+
+def _error_old_version(
+    log, program, package, need_version, found_version, required_for
+):
+    if required_for:
+        log.error(old_version_required_for.format(**locals()))
+    else:
+        log.error(old_version.format(**locals()))
+    _error_trailer(**locals())
+
+
+def check_external_program(
+    *,
+    log,
+    program,
+    package,
+    version_checker,
+    need_version,
+    required_for=None,
+    recommended=False,
+):
+    try:
+        found_version = version_checker()
+    except (CalledProcessError, FileNotFoundError, MissingDependencyError):
+        _error_missing_program(log, program, package, required_for, recommended)
+        if not recommended:
+            sys.exit(ExitCode.missing_dependency)
+        return
+
+    if found_version < need_version:
+        _error_old_version(
+            log, program, package, need_version, found_version, required_for
+        )
+        if not recommended:
+            sys.exit(ExitCode.missing_dependency)
+
+    log.debug(f'Found {program} {found_version}')

src/ocrmypdf/exec/pngquant.py

@@ -17,6 +17,9 @@
 
 from functools import lru_cache
 from subprocess import run
+from tempfile import NamedTemporaryFile
+
+from PIL import Image
 
 from . import get_version
 from ..exceptions import MissingDependencyError
@@ -36,16 +39,32 @@ def available():
 
 
 def quantize(input_file, output_file, quality_min, quality_max):
-    args = [
-        'pngquant',
-        '--force',
-        '--skip-if-larger',
-        '--output',
-        output_file,
-        '--quality',
-        f'{quality_min}-{quality_max}',
-        '--',
-        input_file,
-    ]
-    proc = run(args)
-    proc.check_returncode()
+    if input_file.endswith('.jpg'):
+        im = Image.open(input_file)
+        with NamedTemporaryFile(suffix='.png') as tmp:
+            im.save(tmp)
+            args = [
+                'pngquant',
+                '--force',
+                '--skip-if-larger',
+                '--output',
+                output_file,
+                '--quality',
+                f'{quality_min}-{quality_max}',
+                '--',
+                tmp.name,
+            ]
+            run(args)
+    else:
+        args = [
+            'pngquant',
+            '--force',
+            '--skip-if-larger',
+            '--output',
+            output_file,
+            '--quality',
+            f'{quality_min}-{quality_max}',
+            '--',
+            input_file,
+        ]
+        run(args)

src/ocrmypdf/exec/tesseract.py

@@ -200,7 +200,9 @@ def tesseract_log_output(log, stdout, input_file):
             log.info(prefix + line.strip())
 
 
-def page_timedout(log, input_file):
+def page_timedout(log, input_file, timeout):
+    if timeout == 0:
+        return
     prefix = f"{(page_number(input_file)):4d}: [tesseract] "
     log.warning(prefix + " took too long to OCR - skipping")
 
@@ -257,7 +259,7 @@ def generate_hocr(
         # Generate a HOCR file with no recognized text if tesseract times out
         # Temporary workaround to hocrTransform not being able to function if
         # it does not have a valid hOCR file.
-        page_timedout(log, input_file)
+        page_timedout(log, input_file, timeout)
         _generate_null_hocr(output_hocr, output_sidecar, input_file)
     except CalledProcessError as e:
         tesseract_log_output(log, e.output, input_file)
@@ -347,7 +349,7 @@ def generate_pdf(
         if os.path.exists(prefix + '.txt'):
             shutil.move(prefix + '.txt', output_text)
     except TimeoutExpired:
-        page_timedout(log, input_image)
+        page_timedout(log, input_image, timeout)
         use_skip_page(text_only, skip_pdf, output_pdf, output_text)
     except CalledProcessError as e:
         tesseract_log_output(log, e.output, input_image)

src/ocrmypdf/exec/unpaper.py

@@ -19,13 +19,15 @@
 # https://github.com/Flameeyes/unpaper/blob/master/doc/basic-concepts.md
 
 import os
+import shlex
+import subprocess
 import sys
 from functools import lru_cache
-from subprocess import STDOUT, CalledProcessError, check_output
-from tempfile import NamedTemporaryFile
+from subprocess import PIPE, STDOUT, CalledProcessError
+from tempfile import TemporaryDirectory
 
 from . import get_version
-from ..exceptions import MissingDependencyError
+from ..exceptions import MissingDependencyError, SubprocessOutputError
 
 try:
     from PIL import Image
@@ -64,43 +66,64 @@ def run(input_file, output_file, dpi, log, mode_args):
         im.close()
         raise MissingDependencyError() from e
 
-    with NamedTemporaryFile(suffix=suffix) as input_pnm, NamedTemporaryFile(
-        suffix=suffix, mode="r+b"
-    ) as output_pnm:
+    with TemporaryDirectory() as tmpdir:
+        input_pnm = os.path.join(tmpdir, f'input{suffix}')
+        output_pnm = os.path.join(tmpdir, f'output{suffix}')
         im.save(input_pnm, format='PPM')
         im.close()
 
-        os.unlink(output_pnm.name)
-
-        args_unpaper.extend([input_pnm.name, output_pnm.name])
+        # To prevent any shenanigans from accepting arbitrary parameters in
+        # --unpaper-args, we:
+        # 1) run with cwd set to a tmpdir with only unpaper's files
+        # 2) forbid the use of '/' in arguments, to prevent changing paths
+        # 3) append absolute paths for the input and output file
+        # This should ensure that a user cannot clobber some other file with
+        # their unpaper arguments (whether intentionally or otherwise)
+        args_unpaper.extend([input_pnm, output_pnm])
         try:
-            stdout = check_output(
-                args_unpaper, close_fds=True, universal_newlines=True, stderr=STDOUT
+            proc = subprocess.run(
+                args_unpaper,
+                check=True,
+                close_fds=True,
+                universal_newlines=True,
+                stderr=STDOUT,
+                cwd=tmpdir,
+                stdout=PIPE,
             )
         except CalledProcessError as e:
             log.debug(e.output)
             raise e from e
         else:
-            log.debug(stdout)
-            # unpaper sets dpi to 72
-            Image.open(output_pnm.name).save(output_file, dpi=(dpi, dpi))
+            log.debug(proc.stdout)
+            # unpaper sets dpi to 72; fix this
+            try:
+                Image.open(output_pnm).save(output_file, dpi=(dpi, dpi))
+            except (FileNotFoundError, OSError):
+                raise SubprocessOutputError(
+                    "unpaper: failed to produce the expected output file. Called with: "
+                    + str(args_unpaper)
+                ) from None
 
 
-def clean(input_file, output_file, dpi, log):
-    run(
-        input_file,
-        output_file,
-        dpi,
-        log,
-        [
-            '--layout',
-            'none',
-            '--mask-scan-size',
-            '100',  # don't blank out narrow columns
-            '--no-border-align',  # don't align visible content to borders
-            '--no-mask-center',  # don't center visible content within page
-            '--no-grayfilter',  # don't remove light gray areas
-            '--no-blackfilter',  # don't remove solid black areas
-            '--no-deskew',  # don't deskew
-        ],
-    )
+def validate_custom_args(args: str):
+    unpaper_args = shlex.split(args)
+    if any('/' in arg for arg in unpaper_args):
+        raise ValueError('No filenames allowed in --unpaper-args')
+    return unpaper_args
+
+
+def clean(input_file, output_file, dpi, log, unpaper_args=None):
+    default_args = [
+        '--layout',
+        'none',
+        '--mask-scan-size',
+        '100',  # don't blank out narrow columns
+        '--no-border-align',  # don't align visible content to borders
+        '--no-mask-center',  # don't center visible content within page
+        '--no-grayfilter',  # don't remove light gray areas
+        '--no-blackfilter',  # don't remove solid black areas
+        '--no-deskew',  # don't deskew
+    ]
+    if not unpaper_args:
+        unpaper_args = default_args
+    run(input_file, output_file, dpi, log, unpaper_args)

src/ocrmypdf/optimize.py

@@ -25,6 +25,7 @@ from pathlib import Path
 from PIL import Image
 
 import pikepdf
+from pikepdf import Name, Dictionary, Array
 
 from . import leptonica
 from ._jobcontext import JobContext
@@ -52,7 +53,7 @@ def tif_name(root, xref):
 
 
 def extract_image_filter(pike, root, log, image, xref):
-    if image.Subtype != '/Image':
+    if image.Subtype != Name.Image:
         return None
     if image.Length < 100:
         log.debug("Skipping small image, xref %s", xref)
@@ -68,7 +69,7 @@ def extract_image_filter(pike, root, log, image, xref):
     if pim.bits_per_component > 8:
         return None  # Don't mess with wide gamut images
 
-    if filtdp[0] == '/JPXDecode':
+    if filtdp[0] == Name.JPXDecode:
         return None  # Don't do JPEG2000
 
     return pim, filtdp
@@ -82,7 +83,7 @@ def extract_image_jbig2(*, pike, root, log, image, xref, options):
 
     if (
         pim.bits_per_component == 1
-        and filtdp != '/JBIG2Decode'
+        and filtdp != Name.JBIG2Decode
         and jbig2enc.available()
     ):
         try:
@@ -102,7 +103,7 @@ def extract_image_generic(*, pike, root, log, image, xref, options):
         return None
     pim, filtdp = result
 
-    if filtdp[0] == '/DCTDecode' and options.optimize >= 2:
+    if filtdp[0] == Name.DCTDecode and options.optimize >= 2:
         # This is a simple heuristic derived from some training data, that has
         # about a 70% chance of guessing whether the JPEG is high quality,
         # and possibly recompressible, or not. The number itself doesn't mean
@@ -148,11 +149,22 @@ def extract_image_generic(*, pike, root, log, image, xref, options):
 def extract_images(pike, root, log, options, extract_fn):
     """Extract image using extract_fn
 
-    extract_fn decides whether the image is interesting in this case
+    Enumerate images on each page, lookup their xref/ID number in the PDF.
+    Exclude images that are soft masks (i.e. alpha transparency related).
+    Record the page number on which an image is first used, since images may be
+    used on multiple pages (or multiple times on the same page).
+
+    Current we do not check Form XObjects or other objects that may contain
+    images, and we don't evaluate alternate images or thumbnails.
+
+    extract_fn must decide if wants to extract the image in this context. If
+    it does a tuple should be returned: (xref, ext) where .ext is the file
+    extension. extract_fn must also extract the file it finds interesting.
     """
 
     include_xrefs = set()
     exclude_xrefs = set()
+    pageno_for_xref = {}
     errors = 0
     for pageno, page in enumerate(pike.pages):
         try:
@@ -168,6 +180,8 @@ def extract_images(pike, root, log, options, extract_fn):
                 smask_xref = image.SMask.objgen[0]
                 exclude_xrefs.add(smask_xref)
             include_xrefs.add(xref)
+            if xref not in pageno_for_xref:
+                pageno_for_xref[xref] = pageno
 
     working_xrefs = include_xrefs - exclude_xrefs
     for xref in working_xrefs:
@@ -177,13 +191,12 @@ def extract_images(pike, root, log, options, extract_fn):
                 pike=pike, root=root, log=log, image=image, xref=xref, options=options
             )
         except Exception as e:
-            log.debug("Image xref %s", xref)
-            log.debug(repr(e))
+            log.debug("Image xref %s, error %s", xref, repr(e))
             errors += 1
         else:
             if result:
                 _, ext = result
-                yield pageno, xref, ext
+                yield pageno_for_xref[xref], xref, ext
 
 
 def extract_images_generic(pike, root, log, options):
@@ -197,7 +210,7 @@ def extract_images_generic(pike, root, log, options):
             pngs.append(xref)
         elif ext == '.jpg':
             jpegs.append(xref)
-    log.debug("Optimizable images: " "JPEGs: %s PNGs: %s", len(jpegs), len(pngs))
+    log.debug("Optimizable images: JPEGs: %s PNGs: %s", len(jpegs), len(pngs))
     return jpegs, pngs
 
 
@@ -215,7 +228,7 @@ def extract_images_jbig2(pike, root, log, options):
     jbig2_groups = {
         group: xrefs for group, xrefs in jbig2_groups.items() if len(xrefs) > 0
     }
-    log.debug("Optimizable images: " "JBIG2 groups: %s", (len(jbig2_groups),))
+    log.debug("Optimizable images: JBIG2 groups: %s", (len(jbig2_groups),))
     return jbig2_groups
 
 
@@ -270,7 +283,7 @@ def convert_to_jbig2(pike, jbig2_groups, root, log, options):
     must be lossy encoding since jbig2enc does not support refinement coding.
 
     When the JBIG2 symbolic coder is not used, each JBIG2 stands on its own
-    and needs no dictionary. Currently this is must be lossless JBIG2.
+    and needs no dictionary. Currently this must be lossless JBIG2.
     """
 
     _produce_jbig2_images(jbig2_groups, root, log, options)
@@ -281,7 +294,7 @@ def convert_to_jbig2(pike, jbig2_groups, root, log, options):
         if jbig2_symfile.exists():
             jbig2_globals_data = jbig2_symfile.read_bytes()
             jbig2_globals = pikepdf.Stream(pike, jbig2_globals_data)
-            jbig2_globals_dict = pikepdf.Dictionary({'/JBIG2Globals': jbig2_globals})
+            jbig2_globals_dict = Dictionary(JBIG2Globals=jbig2_globals)
         elif options.jbig2_page_group_size == 1:
             jbig2_globals_dict = None
         else:
@@ -293,9 +306,7 @@ def convert_to_jbig2(pike, jbig2_groups, root, log, options):
             jbig2_im_data = jbig2_im_file.read_bytes()
             im_obj = pike.get_object(xref, 0)
             im_obj.write(
-                jbig2_im_data,
-                filter=pikepdf.Name('/JBIG2Decode'),
-                decode_parms=jbig2_globals_dict,
+                jbig2_im_data, filter=Name.JBIG2Decode, decode_parms=jbig2_globals_dict
             )
 
 
@@ -310,17 +321,17 @@ def transcode_jpegs(pike, jpegs, root, log, options):
         # https://github.com/python-pillow/Pillow/issues/1144
         with Image.open(fspath(in_jpg)) as im:
             im.save(fspath(opt_jpg), optimize=True, quality=options.jpeg_quality)
-        # pylint: disable=no-member
+
         if opt_jpg.stat().st_size > in_jpg.stat().st_size:
             log.debug("xref %s, jpeg, made larger - skip", xref)
             continue
 
         compdata = leptonica.CompressedData.open(opt_jpg)
         im_obj = pike.get_object(xref, 0)
-        im_obj.write(compdata.read(), filter=pikepdf.Name('/DCTDecode'))
+        im_obj.write(compdata.read(), filter=Name.DCTDecode)
 
 
-def transcode_pngs(pike, pngs, root, log, options):
+def transcode_pngs(pike, images, image_name_fn, root, log, options):
     if options.optimize >= 2:
         png_quality = (
             max(10, options.png_quality - 10),
@@ -329,65 +340,91 @@ def transcode_pngs(pike, pngs, root, log, options):
         with concurrent.futures.ThreadPoolExecutor(
             max_workers=options.jobs
         ) as executor:
-            for xref in pngs:
+            for xref in images:
+                log.debug(image_name_fn(root, xref))
                 executor.submit(
                     pngquant.quantize,
-                    png_name(root, xref),
+                    image_name_fn(root, xref),
                     png_name(root, xref),
                     png_quality[0],
                     png_quality[1],
                 )
 
-    for xref in pngs:
+    for xref in images:
         im_obj = pike.get_object(xref, 0)
-
-        # Open, transcode (!), package for PDF
         try:
-            pix = leptonica.Pix.open(png_name(root, xref))
-            if pix.depth == 1:
-                pix = pix.invert()  # PDF assumes 1 is black for monochrome
-            compdata = pix.generate_pdf_ci_data(leptonica.lept.L_FLATE_ENCODE, 0)
+            compdata = leptonica.CompressedData.open(png_name(root, xref))
         except leptonica.LeptonicaError as e:
+            # Most likely this means file not found, i.e. quantize did not
+            # produce an improved version
             log.error(e)
             continue
 
-        # This is what we should be doing: open the compressed data without
-        # transcoding. However this shifts each pixel row by one for some
-        # reason.
-        # compdata = leptonica.CompressedData.open(png_name(root, xref))
+        # If re-coded image is larger don't use it - we test here because
+        # pngquant knows the size of the temporary output file but not the actual
+        # object in the PDF
         if len(compdata) > int(im_obj.stream_dict.Length):
-            continue  # If we produced a larger image, don't use
+            log.debug(
+                f"pngquant: pngquant did not improve over original image "
+                f"{len(compdata)} > {int(im_obj.stream_dict.Length)}"
+            )
+            continue
 
-        predictor = None
-        if compdata.predictor > 0:
-            predictor = pikepdf.Dictionary({'/Predictor': compdata.predictor})
+        # When a PNG is inserted into a PDF, we more or less copy the IDAT section from
+        # the PDF and transfer the rest of the PNG headers to PDF image metadata.
+        # One thing we have to do is tell the PDF reader whether a predictor was used
+        # on the image before Flate encoding. (Typically one is.)
+        # According to Leptonica source, PDF readers don't actually need us
+        # to specify the correct predictor, they just need a value of either:
+        #   1 - no predictor
+        #   10-14 - there is a predictor
+        # Leptonica's compdata->predictor only tells TRUE or FALSE
+        # From there the PNG decoder can infer the rest from the file.
+        # In practice the predictor should be Paeth, 14, so we'll use that.
+        # See:
+        #   - PDF RM 7.4.4.4 Table 10
+        #   - https://github.com/DanBloomberg/leptonica/blob/master/src/pdfio2.c#L757
+        predictor = 14 if compdata.predictor > 0 else 1
+        dparms = Dictionary(Predictor=predictor)
+        if predictor > 1:
+            dparms.BitsPerComponent = compdata.bps  # Yes, this is redundant
+            dparms.Colors = compdata.spp
+            dparms.Columns = compdata.w
 
         im_obj.BitsPerComponent = compdata.bps
         im_obj.Width = compdata.w
         im_obj.Height = compdata.h
 
         if compdata.ncolors > 0:
+            # .ncolors is the number of colors in the palette, not the number of
+            # colors used in a true color image
             palette_pdf_string = compdata.get_palette_pdf_string()
             palette_data = pikepdf.Object.parse(palette_pdf_string)
             palette_stream = pikepdf.Stream(pike, bytes(palette_data))
             palette = [
-                pikepdf.Name('/Indexed'),
-                pikepdf.Name('/DeviceRGB'),
+                Name.Indexed,
+                Name.DeviceRGB,
                 compdata.ncolors - 1,
                 palette_stream,
             ]
             cs = palette
         else:
             if compdata.spp == 1:
-                cs = pikepdf.Name('/DeviceGray')
+                # PDF interprets binary-1 as black in 1bpp, but PNG sets
+                # black to 0 for 1bpp. Create a palette that informs the PDF
+                # of the mapping - seems cleaner to go this way but pikepdf
+                # needs to be patched to support it.
+                # palette = [Name.Indexed, Name.DeviceGray, 1, b"\xff\x00"]
+                # cs = palette
+                cs = Name.DeviceGray
             elif compdata.spp == 3:
-                cs = pikepdf.Name('/DeviceRGB')
+                cs = Name.DeviceRGB
             elif compdata.spp == 4:
-                cs = pikepdf.Name('/DeviceCMYK')
+                cs = Name.DeviceCMYK
+        if compdata.bps == 1:
+            im_obj.Decode = [1, 0]  # Bit of a kludge but this inverts photometric too
         im_obj.ColorSpace = cs
-        im_obj.write(
-            compdata.read(), filter=pikepdf.Name('/FlateDecode'), decode_parms=predictor
-        )
+        im_obj.write(compdata.read(), filter=Name.FlateDecode, decode_parms=dparms)
 
 
 def optimize(input_file, output_file, log, context):
@@ -407,11 +444,14 @@ def optimize(input_file, output_file, log, context):
     pike = pikepdf.Pdf.open(input_file)
 
     root = Path(output_file).parent / 'images'
-    root.mkdir(exist_ok=True)  # pylint: disable=no-member
+    root.mkdir(exist_ok=True)
 
     jpegs, pngs = extract_images_generic(pike, root, log, options)
     transcode_jpegs(pike, jpegs, root, log, options)
-    transcode_pngs(pike, pngs, root, log, options)
+    # if options.optimize >= 2:
+    # Try pngifying the jpegs
+    #    transcode_pngs(pike, jpegs, jpg_name, root, log, options)
+    transcode_pngs(pike, pngs, png_name, root, log, options)
 
     jbig2_groups = extract_images_jbig2(pike, root, log, options)
     convert_to_jbig2(pike, jbig2_groups, root, log, options)

tests/cache/francais/__-l__deu__000001.ocr.png__000001.text__pdf__txt/stderr.bin

@@ -0,0 +1 @@
+Tesseract Open Source OCR Engine v4.0.0 with Leptonica

tests/cache/francais/__-l__deu__000001.ocr.png__000001.text__pdf__txt/txt.bin

@@ -0,0 +1,13 @@
+Portez ce vieux whisky au juge
+blond qui fume sur son ile
+interieure, a cöte de l'alcöve
+ovolde, oU les büches se
+consument dans l'ätre, ce qui
+lui permet de penser ä la
+cgnogenese de l'&tre dont il
+est question dans la cause
+ambigu6 entendue ä Moy, dans
+un capharnaüÜm qui, pense-t-il,
+diminue ca et 13 la qualite de son
+ceuvre.
+

tests/cache/manifest.jsonl

@@ -44,3 +44,4 @@
 {"tesseract_version": "tesseract 4.0.0  leptonica-1.77.0   libjpeg 9c : libpng 1.6.35 : libtiff 4.0.10 : zlib 1.2.11 : libopenjp2 2.3.0  Found AVX2  Found AVX  Found SSE ", "platform": "Darwin-18.0.0-x86_64-i386-64bit", "python": "3.7.1", "argv_slug": "__-l__eng__000004.ocr.png__000004__hocr__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "$TMPDIR/000004.ocr.png", "$TMPDIR/000004", "hocr", "txt"]}
 {"tesseract_version": "tesseract 4.0.0  leptonica-1.77.0   libjpeg 9c : libpng 1.6.35 : libtiff 4.0.10 : zlib 1.2.11 : libopenjp2 2.3.0  Found AVX2  Found AVX  Found SSE ", "platform": "Darwin-18.0.0-x86_64-i386-64bit", "python": "3.7.1", "argv_slug": "__-l__eng__000003.ocr.png__000003__hocr__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "$TMPDIR/000003.ocr.png", "$TMPDIR/000003", "hocr", "txt"]}
 {"tesseract_version": "tesseract 4.0.0  leptonica-1.77.0   libjpeg 9c : libpng 1.6.35 : libtiff 4.0.10 : zlib 1.2.11 : libopenjp2 2.3.0  Found AVX2  Found AVX  Found SSE ", "platform": "Darwin-18.0.0-x86_64-i386-64bit", "python": "3.7.1", "argv_slug": "__-l__eng__000002.ocr.png__000002__hocr__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "$TMPDIR/000002.ocr.png", "$TMPDIR/000002", "hocr", "txt"]}
+{"tesseract_version": "tesseract 4.0.0  leptonica-1.77.0   libgif 5.1.4 : libjpeg 9c : libpng 1.6.36 : libtiff 4.0.10 : zlib 1.2.11 : libwebp 1.0.2 : libopenjp2 2.3.0  Found AVX2  Found AVX  Found SSE ", "platform": "Darwin-18.2.0-x86_64-i386-64bit", "python": "3.7.2", "argv_slug": "__-l__deu__000001.ocr.png__000001.text__pdf__txt", "sourcefile": "resources/francais.pdf", "args": ["-l", "deu", "-c", "textonly_pdf=1", "$TMPDIR/000001.ocr.png", "$TMPDIR/000001.text", "pdf", "txt"]}

tests/conftest.py

@@ -19,7 +19,7 @@ import os
 import platform
 import sys
 from pathlib import Path
-from subprocess import PIPE, Popen
+from subprocess import PIPE, run
 
 import pytest
 
@@ -72,6 +72,17 @@ def needs_pdfminer(fn):
     return fn
 
 
+@pytest.helpers.register
+def have_unpaper():
+    try:
+        from ocrmypdf.exec import unpaper
+
+        unpaper.version()
+    except Exception:
+        return False
+    return True
+
+
 TESTS_ROOT = os.path.abspath(os.path.dirname(__file__))
 SPOOF_PATH = os.path.join(TESTS_ROOT, 'spoof')
 PROJECT_ROOT = os.path.dirname(TESTS_ROOT)
@@ -147,7 +158,7 @@ def no_outpdf(tmpdir):
 
 @pytest.helpers.register
 def check_ocrmypdf(input_file, output_file, *args, env=None):
-    "Run ocrmypdf and confirmed that a valid file was created"
+    """Run ocrmypdf and confirmed that a valid file was created"""
 
     p, out, err = run_ocrmypdf(input_file, output_file, *args, env=env)
     # ensure py.test collects the output, use -s to view
@@ -171,19 +182,16 @@ def run_ocrmypdf(input_file, output_file, *args, env=None, universal_newlines=Tr
     if env is None:
         env = os.environ
 
-    p_args = OCRMYPDF + [str(arg) for arg in args] + [str(input_file), str(output_file)]
-    p = Popen(
-        p_args,
-        close_fds=True,
-        stdout=PIPE,
-        stderr=PIPE,
-        universal_newlines=universal_newlines,
-        env=env,
+    p_args = (
+        OCRMYPDF
+        + [str(arg) for arg in args if arg is not None]
+        + [str(input_file), str(output_file)]
     )
-    out, err = p.communicate()
-    # print(err)
-
-    return p, out, err
+    p = run(
+        p_args, stdout=PIPE, stderr=PIPE, universal_newlines=universal_newlines, env=env
+    )
+    # print(p.stderr)
+    return p, p.stdout, p.stderr
 
 
 @pytest.helpers.register

tests/test_main.py

@@ -21,14 +21,14 @@ import shutil
 import sys
 from math import isclose
 from pathlib import Path
-from subprocess import DEVNULL, PIPE, Popen
+from subprocess import DEVNULL, PIPE, run, Popen
 
 import PIL
 import pytest
 from PIL import Image
 
 from ocrmypdf.exceptions import ExitCode, MissingDependencyError
-from ocrmypdf.exec import ghostscript, qpdf, tesseract
+from ocrmypdf.exec import ghostscript, qpdf, tesseract, unpaper
 from ocrmypdf.leptonica import Pix
 from ocrmypdf.pdfa import file_claims_pdfa
 from ocrmypdf.pdfinfo import Colorspace, Encoding, PdfInfo
@@ -164,7 +164,7 @@ def test_exotic_image(
     check_ocrmypdf(
         resources / pdf,
         outfile,
-        '-dc',
+        '-dc' if pytest.helpers.have_unpaper() else '-d',
         '-v',
         '1',
         '--output-type',
@@ -282,8 +282,7 @@ def test_maximum_options(
         resources / 'multipage.pdf',
         outpdf,
         '-d',
-        '-c',
-        '-i',
+        '-ci' if pytest.helpers.have_unpaper() else None,
         '-f',
         '-k',
         '--oversample',
@@ -338,15 +337,16 @@ def test_force_ocr_on_pdf_with_no_images(spoof_tesseract_crash, resources, no_ou
     pytest.helpers.is_macos() and pytest.helpers.running_in_travis(),
     reason="takes too long to install language packs in Travis macOS homebrew",
 )
-def test_french(spoof_tesseract_cache, resources, outdir):
+def test_german(spoof_tesseract_cache, resources, outdir):
     # Produce a sidecar too - implicit test that system locale is set up
-    # properly
+    # properly. It is fine that we are testing -l deu on a French file because
+    # we are exercising the functionality not going for accuracy.
     sidecar = outdir / 'francais.txt'
     p, out, err = run_ocrmypdf(
         resources / 'francais.pdf',
         outdir / 'francais.pdf',
         '-l',
-        'fra',
+        'deu',  # more commonly installed
         '--sidecar',
         sidecar,
         env=spoof_tesseract_cache,
@@ -541,7 +541,6 @@ def test_jbig2_passthrough(spoof_tesseract_cache, resources, outpdf):
         'hocr',
         env=spoof_tesseract_cache,
     )
-
     out_pageinfo = PdfInfo(out)
     assert out_pageinfo[0].images[0].enc == Encoding.jbig2
 
@@ -553,16 +552,13 @@ def test_stdin(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf):
     # Runs: ocrmypdf - output.pdf < testfile.pdf
     with open(input_file, 'rb') as input_stream:
         p_args = ocrmypdf_exec + ['-', output_file]
-        p = Popen(
+        p = run(
             p_args,
-            close_fds=True,
             stdout=PIPE,
             stderr=PIPE,
             stdin=input_stream,
             env=spoof_tesseract_noop,
         )
-        out, err = p.communicate()
-
         assert p.returncode == ExitCode.ok
 
 
@@ -573,16 +569,13 @@ def test_stdout(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf):
     # Runs: ocrmypdf francais.pdf - > test_stdout.pdf
     with open(output_file, 'wb') as output_stream:
         p_args = ocrmypdf_exec + [input_file, '-']
-        p = Popen(
+        p = run(
             p_args,
-            close_fds=True,
             stdout=output_stream,
             stderr=PIPE,
             stdin=DEVNULL,
             env=spoof_tesseract_noop,
         )
-        out, err = p.communicate()
-
         assert p.returncode == ExitCode.ok
 
     assert qpdf.check(output_file, log=None)
@@ -780,10 +773,10 @@ def test_pagesize_consistency(renderer, resources, outpdf):
         outpdf,
         '--pdf-renderer',
         renderer,
-        '--clean',
+        '--clean' if pytest.helpers.have_unpaper() else None,
         '--deskew',
         '--remove-background',
-        '--clean-final',
+        '--clean-final' if pytest.helpers.have_unpaper() else None,
     )
 
     after_dims = first_page_dimensions(outpdf)
@@ -851,22 +844,21 @@ def test_compression_preserved(
             '-',
             output_file,
         ]
-        p = Popen(
+        p = run(
             p_args,
-            close_fds=True,
             stdout=PIPE,
             stderr=PIPE,
             stdin=input_stream,
+            universal_newlines=True,
             env=spoof_tesseract_noop,
         )
-        out, err = p.communicate()
 
         if im.mode in ('RGBA', 'LA'):
             # If alpha image is input, expect an error
-            assert p.returncode != ExitCode.ok and b'alpha' in err
+            assert p.returncode != ExitCode.ok and 'alpha' in p.stderr
             return
 
-        assert p.returncode == ExitCode.ok, err.decode('utf-8')
+        assert p.returncode == ExitCode.ok, p.stderr
 
     pdfinfo = PdfInfo(output_file)
 
@@ -912,17 +904,15 @@ def test_compression_changed(
             '-',
             output_file,
         ]
-        p = Popen(
+        p = run(
             p_args,
-            close_fds=True,
             stdout=PIPE,
             stderr=PIPE,
             stdin=input_stream,
+            universal_newlines=True,
             env=spoof_tesseract_noop,
         )
-        out, err = p.communicate()
-
-        assert p.returncode == ExitCode.ok, err
+        assert p.returncode == ExitCode.ok, p.stderr
 
     pdfinfo = PdfInfo(output_file)
 
@@ -998,6 +988,9 @@ def test_pdfa_n(spoof_tesseract_cache, pdfa_level, resources, outpdf):
 
 
 @pytest.mark.skipif(sys.version_info >= (3, 7, 0), reason='better utf-8')
+@pytest.mark.skipif(
+    Path('/etc/alpine-release').exists(), reason="invalid test on alpine"
+)
 def test_bad_locale():
     env = os.environ.copy()
     env['LC_ALL'] = 'C'

tests/test_metadata.py

@@ -188,9 +188,8 @@ def test_xml_metadata_preserved(spoof_tesseract_noop, output_type, resources, ou
     input_file = resources / 'graph.pdf'
 
     try:
-        import libxmp
-        from libxmp.utils import file_to_dict
         from libxmp import consts
+        from libxmp.utils import file_to_dict
     except Exception:
         pytest.skip("libxmp not available or libexempi3 not installed")
 
@@ -331,16 +330,13 @@ def test_prevent_gs_invalid_xml(resources, outdir):
     from ocrmypdf.pdfinfo import PdfInfo
 
     generate_pdfa_ps(outdir / 'pdfa.ps')
-    input_files = [
-        str(outdir / 'layers.rendered.pdf'),
-        str(outdir / 'pdfa.ps'),
-    ]
+    input_files = [str(outdir / 'layers.rendered.pdf'), str(outdir / 'pdfa.ps')]
     copyfile(resources / 'enron1.pdf', outdir / 'layers.rendered.pdf')
     log = logging.getLogger()
     context = JobContext()
 
-    options = parser.parse_args(args=[
-        '-j', '1', '--output-type', 'pdfa-2', 'a.pdf', 'b.pdf']
+    options = parser.parse_args(
+        args=['-j', '1', '--output-type', 'pdfa-2', 'a.pdf', 'b.pdf']
     )
     context.options = options
     context.pdfinfo = PdfInfo(resources / 'enron1.pdf')
@@ -349,11 +345,13 @@ def test_prevent_gs_invalid_xml(resources, outdir):
         input_files_groups=input_files,
         output_file=outdir / 'pdfa.pdf',
         log=log,
-        context=context
+        context=context,
     )
 
     with open(outdir / 'pdfa.pdf', 'rb') as f:
-        with mmap.mmap(f.fileno(), 0, flags=mmap.MAP_PRIVATE, prot=mmap.PROT_READ) as mm:
+        with mmap.mmap(
+            f.fileno(), 0, flags=mmap.MAP_PRIVATE, prot=mmap.PROT_READ
+        ) as mm:
             # Since the XML may be invalid, we scan instead of actually feeding it
             # to a parser.
             XMP_MAGIC = b'W5M0MpCehiHzreSzNTczkc9d'

tests/test_optimize.py

@@ -120,7 +120,7 @@ def test_flate_to_jbig2(resources, outdir, spoof_tesseract_noop):
         '--image-dpi',
         '100',
         '--png-quality',
-        '10',
+        '50',
         '--optimize',
         '3',
         env=spoof_tesseract_noop,

tests/test_tess4.py

@@ -117,10 +117,10 @@ def test_pagesize_consistency_tess4(ensure_tess4, resources, outpdf):
         outpdf,
         '--pdf-renderer',
         'sandwich',
-        '--clean',
+        '--clean' if pytest.helpers.have_unpaper() else None,
         '--deskew',
         '--remove-background',
-        '--clean-final',
+        '--clean-final' if pytest.helpers.have_unpaper() else None,
         env=ensure_tess4,
     )
 

tests/test_unpaper.py

@@ -15,11 +15,16 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
+import argparse
+from os import fspath
 from pathlib import Path
+from unittest.mock import MagicMock, patch
 
 import pytest
 
+from ocrmypdf import __main__ as main
 from ocrmypdf.exceptions import ExitCode
+from ocrmypdf.exec import unpaper
 
 # pytest.helpers is dynamic
 # pylint: disable=no-member
@@ -30,26 +35,79 @@ run_ocrmypdf = pytest.helpers.run_ocrmypdf
 spoof = pytest.helpers.spoof
 
 
-@pytest.fixture(scope='session')
+def have_unpaper():
+    try:
+        unpaper.version()
+    except Exception:
+        return False
+    else:
+        return True
+
+
+@pytest.fixture(scope="session")
 def spoof_unpaper_oldversion(tmpdir_factory):
-    return spoof(tmpdir_factory, unpaper='unpaper_oldversion.py')
+    return spoof(tmpdir_factory, unpaper="unpaper_oldversion.py")
 
 
-@pytest.mark.skipif(True, reason="needs new fixture implementation")
 def test_no_unpaper(resources, no_outpdf):
-    # <disable unpaper here>
-    p, out, err = run_ocrmypdf(
-        resources / 'c02-22.pdf', no_outpdf, '--clean', env=os.environ
-    )
-    assert p.returncode == ExitCode.missing_dependency
+    input_ = fspath(resources / "c02-22.pdf")
+    output = fspath(no_outpdf)
+    options = main.parser.parse_args(args=["--clean", input_, output])
+
+    with patch("ocrmypdf.exec.unpaper.version") as mock_unpaper_version:
+        mock_unpaper_version.side_effect = FileNotFoundError("unpaper")
+        with pytest.raises(SystemExit):
+            main.check_options(options, log=MagicMock())
 
 
 def test_old_unpaper(spoof_unpaper_oldversion, resources, no_outpdf):
     p, out, err = run_ocrmypdf(
-        resources / 'c02-22.pdf', no_outpdf, '--clean', env=spoof_unpaper_oldversion
+        resources / "c02-22.pdf", no_outpdf, "--clean", env=spoof_unpaper_oldversion
     )
     assert p.returncode == ExitCode.missing_dependency
 
 
+@pytest.mark.skipif(not have_unpaper(), reason="requires unpaper")
 def test_clean(spoof_tesseract_noop, resources, outpdf):
-    check_ocrmypdf(resources / 'skew.pdf', outpdf, '-c', env=spoof_tesseract_noop)
+    check_ocrmypdf(resources / "skew.pdf", outpdf, "-c", env=spoof_tesseract_noop)
+
+
+@pytest.mark.skipif(not have_unpaper(), reason="requires unpaper")
+def test_unpaper_args_valid(spoof_tesseract_noop, resources, outpdf):
+    check_ocrmypdf(
+        resources / "skew.pdf",
+        outpdf,
+        "-c",
+        "--unpaper-args",
+        "--layout double",  # Spaces required here
+        env=spoof_tesseract_noop,
+    )
+
+
+@pytest.mark.skipif(not have_unpaper(), reason="requires unpaper")
+def test_unpaper_args_invalid_filename(spoof_tesseract_noop, resources, outpdf):
+    p, out, err = run_ocrmypdf(
+        resources / "skew.pdf",
+        outpdf,
+        "-c",
+        "--unpaper-args",
+        "/etc/passwd",
+        env=spoof_tesseract_noop,
+    )
+    assert "No filenames allowed" in err
+    assert p.returncode == ExitCode.bad_args
+
+
+@pytest.mark.skipif(not have_unpaper(), reason="requires unpaper")
+def test_unpaper_args_invalid(spoof_tesseract_noop, resources, outpdf):
+    p, out, err = run_ocrmypdf(
+        resources / "skew.pdf",
+        outpdf,
+        "-c",
+        "--unpaper-args",
+        "unpaper is not going to like these arguments",
+        env=spoof_tesseract_noop,
+    )
+    # Can't tell difference between unpaper choking on bad arguments or some
+    # other unpaper failure
+    assert p.returncode == ExitCode.child_process_error

tests/test_userunit.py

@@ -15,20 +15,12 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
-import logging
-import os
-import shutil
 from math import isclose
-from subprocess import DEVNULL, PIPE, Popen, check_call, check_output
 
-import PyPDF2 as pypdf
 import pytest
 
-from ocrmypdf import leptonica
 from ocrmypdf.exceptions import ExitCode
-from ocrmypdf.exec import ghostscript
-from ocrmypdf.pdfa import file_claims_pdfa
-from ocrmypdf.pdfinfo import Colorspace, Encoding, PdfInfo
+from ocrmypdf.pdfinfo import PdfInfo
 
 check_ocrmypdf = pytest.helpers.check_ocrmypdf
 run_ocrmypdf = pytest.helpers.run_ocrmypdf

tests/test_weave.py

@@ -0,0 +1,64 @@
+# © 2019 James R. Barlow: github.com/jbarlow83
+#
+# This file is part of OCRmyPDF.
+#
+# OCRmyPDF is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# OCRmyPDF is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
+
+from unittest.mock import MagicMock
+import logging
+import os
+
+import pytest
+
+import pikepdf
+from ocrmypdf._weave import _fix_toc, _update_page_resources
+
+check_ocrmypdf = pytest.helpers.check_ocrmypdf
+
+
+def test_invalid_toc(resources, outdir, caplog):
+    pdf = pikepdf.open(resources / 'toc.pdf')
+
+    # Corrupt a TOC entry
+    pdf.Root.Outlines.Last.Dest = pikepdf.Array([None, 0.0, 0.1, 0.2])
+    pdf.save(outdir / 'test.pdf')
+
+    pdf = pikepdf.open(outdir / 'test.pdf')
+    remap = {}
+    remap[pdf.pages[0].objgen] = pdf.pages[0].objgen  # Dummy remap
+
+    # Confirm we complain about the TOC and don't throw an exception
+    log = logging.getLogger()
+    _fix_toc(pdf, remap, log)
+    assert 'invalid table of contents entries' in caplog.text
+
+
+def test_no_glyphless_weave(resources, outdir):
+    pdf = pikepdf.open(resources / 'francais.pdf')
+    pdf_aspect = pikepdf.open(resources / 'aspect.pdf')
+    pdf_cmyk = pikepdf.open(resources / 'cmyk.pdf')
+    pdf.pages.extend(pdf_aspect.pages)
+    pdf.pages.extend(pdf_cmyk.pages)
+    pdf.save(outdir / 'test.pdf')
+
+    env = os.environ.copy()
+    env['_OCRMYPDF_MAX_OPEN_PAGE_PDFS'] = '2'
+    check_ocrmypdf(
+        outdir / 'test.pdf',
+        outdir / 'out.pdf',
+        '--deskew',
+        '--tesseract-timeout',
+        '0',
+        env=env,
+    )