Drop support for omitting pdfminer.six

[ci skip]

.docker/alpine.dockerfile

@@ -0,0 +1,84 @@
+FROM alpine:3.9 as base
+
+FROM base as builder
+
+ENV LANG=C.UTF-8
+
+RUN \
+  echo '@testing http://nl.alpinelinux.org/alpine/edge/testing' >> /etc/apk/repositories \
+  # Add runtime dependencies
+  && apk add --update \
+    python3-dev \
+    py3-setuptools \
+    jbig2enc@testing \
+    ghostscript \
+    qpdf \
+    tesseract-ocr \
+    unpaper \
+    pngquant \
+    libxml2-dev \
+    libxslt-dev \
+    zlib-dev \
+    qpdf-dev \
+    libffi-dev \
+    leptonica-dev \
+    binutils \
+  # Install pybind11 for pikepdf
+  && pip3 install pybind11 \
+  # Install flask for the webservice
+  && pip3 install flask \
+  # Add build dependencies
+  && apk add --virtual build-dependencies \
+    build-base \
+    git
+
+COPY . /app
+
+WORKDIR /app
+
+RUN pip3 install .
+
+FROM base
+
+ENV LANG=C.UTF-8
+
+RUN \
+  echo '@testing http://nl.alpinelinux.org/alpine/edge/testing' >> /etc/apk/repositories \
+  # Add runtime dependencies
+  && apk add --update \
+    python3 \
+    jbig2enc@testing \
+    ghostscript \
+    qpdf \
+    tesseract-ocr \
+    tesseract-ocr-data-deu \
+    tesseract-ocr-data-chi_sim \
+    unpaper \
+    pngquant \
+    libxml2 \
+    libxslt \
+    zlib \
+    qpdf \
+    libffi \
+    leptonica-dev \
+    binutils \
+  && mkdir /app
+
+WORKDIR /app
+
+# Copy build artifacts (python site-packages9
+COPY --from=builder /usr/lib/python3.6/site-packages /usr/lib/python3.6/site-packages
+COPY --from=builder /usr/bin/ocrmypdf /usr/bin/dumppdf.py /usr/bin/latin2ascii.py /usr/bin/pdf2txt.py /usr/bin/img2pdf /usr/bin/chardetect /usr/bin/
+
+# Copy
+COPY --from=builder /app/.docker/webservice.py /app/
+
+# Copy minimal project files to get the test suite.
+COPY --from=builder /app/setup.cfg /app/setup.py /app/README.md /app/
+COPY --from=builder /app/requirements /app/requirements
+COPY --from=builder /app/tests /app/tests
+COPY --from=builder /app/src /app/src
+# Copy PKG-INFO from build artifact in app dir to make setuptools-scm happy
+RUN cp /usr/lib/python3.6/site-packages/ocrmypdf-*.egg-info/PKG-INFO /app
+
+ENTRYPOINT ["/usr/bin/ocrmypdf"]

.docker/webservice.dockerfile

@@ -16,4 +16,9 @@ COPY .docker/webservice.py /application
 
 USER docker
 
+VOLUME ["/config"]
+
+# This config file is optional
+ENV OCRMYPDF_WEBSERVICE_SETTINGS "/config/config.py"
+
 ENTRYPOINT ["python3", "/application/webservice.py"]

.docker/webservice.py

@@ -1,5 +1,5 @@
 # webservice.py wrapper for OCRmyPDF
-# Copyright (C) 2018 James R. Barlow: github.com/jbarlow83
+# Copyright (C) 2019 James R. Barlow: github.com/jbarlow83
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -17,13 +17,22 @@
 """This is a simple web service/HTTP wrapper for OCRmyPDF
 
 This may be more convenient than the command line tool for some Docker users.
-Note that OCRmyPDF uses Ghostscript, which is licensed under AGPL3+. While
-OCRmyPDF is under GPL3, this file is distributed under the Affero GPL3+ license,
+Note that OCRmyPDF uses Ghostscript, which is licensed under AGPLv3+. While
+OCRmyPDF is under GPLv3, this file is distributed under the Affero GPLv3+ license,
 to emphasize that SaaS deployments should make sure they comply with
 Ghostscript's license as well as OCRmyPDF's.
 """
 
-from flask import Flask, Response, flash, request, redirect, url_for, abort, send_from_directory
+from flask import (
+    Flask,
+    Response,
+    flash,
+    request,
+    redirect,
+    url_for,
+    abort,
+    send_from_directory,
+)
 from subprocess import run, PIPE
 from tempfile import TemporaryDirectory
 from werkzeug.utils import secure_filename
@@ -32,11 +41,9 @@ import shlex
 
 app = Flask(__name__)
 app.secret_key = "secret"
+app.config['MAX_CONTENT_LENGTH'] = 50_000_000
+app.config.from_envvar("OCRMYPDF_WEBSERVICE_SETTINGS", silent=True)
 
-uploaddir = TemporaryDirectory(prefix="ocrmypdf-upload")
-downloaddir = TemporaryDirectory(prefix="ocrmypdf-download")
-
-app.config["UPLOAD_FOLDER"] = uploaddir
 ALLOWED_EXTENSIONS = set(["pdf"])
 
 
@@ -45,9 +52,13 @@ def allowed_file(filename):
 
 
 def do_ocrmypdf(file):
+    uploaddir = TemporaryDirectory(prefix="ocrmypdf-upload")
+    downloaddir = TemporaryDirectory(prefix="ocrmypdf-download")
+
     filename = secure_filename(file.filename)
     up_file = os.path.join(uploaddir.name, filename)
     file.save(up_file)
+
     down_file = os.path.join(downloaddir.name, filename)
 
     cmd_args = [arg for arg in shlex.split(request.form["params"])]
@@ -79,7 +90,7 @@ def upload_file():
 
     return """
     <!doctype html>
-    <title>OCRmyPDF webapp</title>
+    <title>OCRmyPDF webservice</title>
     <h1>Upload a PDF (debug UI)</h1>
     <form method=post enctype=multipart/form-data>
       <label for="args">Command line parameters</label>
@@ -88,7 +99,27 @@ def upload_file():
       <input type=file name=file>
       <input type=submit value=Upload>
     </form>
+    <h4>Notice</h2>
+    <div style="font-size: 70%; max-width: 34em;">
+    <p>This is a webservice wrapper for OCRmyPDF.</p>
+    <p>Copyright 2019 James R. Barlow</p>
+    <p>This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+    </p>
+    <p>This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+    </p>
+    <p>
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see &lt;http://www.gnu.org/licenses/&gt;.
+    </p>
+    </div>
     """
 
+
 if __name__ == "__main__":
-    app.run(host='0.0.0.0')
+    app.run(host='0.0.0.0', port=5000)

.dockerignore

@@ -5,6 +5,7 @@
 *.sublime*
 **/*.pyc
 .*/
+!.git/
 !.docker/
 .ruffus_history.sqlite
 bin/

.gitattributes

@@ -9,5 +9,6 @@
 *.png	binary
 *.jpg	binary
 *.bin   binary
+*.afdesign  binary
 
 .git_archival.txt  export-subst

.gitignore

@@ -3,9 +3,10 @@
 .pylintrc
 .pytest_cache/
 .ruffus_history.sqlite
-.venv/
+.venv*/
 *.pyc
 *.sublime-*
+*.DS_Store
 
 # Package building
 .eggs/
@@ -13,6 +14,7 @@
 build/
 dist/
 wheelhouse/
+pip-wheel-metadata/
 
 # Automatically generated files
 docs/_build/
@@ -41,3 +43,4 @@ tests/resources/private/
 tmp/
 /debug_tests.py
 *.traineddata
+/private

.pre-commit-config.yaml

@@ -0,0 +1,6 @@
+repos:
+-   repo: https://github.com/ambv/black
+    rev: stable
+    hooks:
+    - id: black
+      language_version: python3.7

README.md

@@ -38,9 +38,9 @@ Main features
 - If requested deskews and/or cleans the image before performing OCR
 - Validates input and output files
 - Distributes work across all available CPU cores
-- Uses [Tesseract OCR](https://github.com/tesseract-ocr/tesseract) engine
-- Supports more than [100 languages](https://github.com/tesseract-ocr/tessdata) recognized by Tesseract
-- Battle-tested on thousands of PDFs, a test suite and continuous integration
+- Uses [Tesseract OCR](https://github.com/tesseract-ocr/tesseract) engine to recognize more than [100 languages](https://github.com/tesseract-ocr/tessdata)
+- Scales properly to handle files with thousands of pages
+- Battle-tested on millions of PDFs
 
 For details: please consult the [documentation](https://ocrmypdf.readthedocs.io/en/latest/).
 
@@ -131,6 +131,11 @@ Press & Media
 - [c't 1-2014, page 59](http://heise.de/-2279695): Detailed presentation of OCRmyPDF v1.0 in the leading German IT magazine c't
 - [heise Open Source, 09/2014: Texterkennung mit OCRmyPDF](http://heise.de/-2356670)
 
+Business enquiries
+------------------
+
+OCRmyPDF would not be the software that it is today is without companies and users choosing to provide support for feature development and consulting enquiries. We are happy to discuss all enquiries, whether for extending the existing feature set, or integrating OCRmyPDF into a larger system.
+
 License
 -------
 
@@ -138,7 +143,7 @@ The OCRmyPDF software is licensed under the GNU GPLv3. Certain files are covered
 
 The license for each test file varies, and is noted in tests/resources/README.rst. The documentation is licensed under Creative Commons Attribution-ShareAlike 4.0 (CC-BY-SA 4.0).
 
-OCRmyPDF versions prior to 6.0 were licensed under the MIT License.
+OCRmyPDF versions prior to 6.0 were distributed under the MIT License.
 
 Disclaimer
 ----------

docs/advanced.rst

@@ -1,6 +1,34 @@
 Advanced features
 =================
 
+Control of unpaper
+------------------
+
+OCRmyPDF uses ``unpaper`` to provide the implementation of the ``--clean`` and ``--clean-final`` arguments. `unpaper <https://github.com/Flameeyes/unpaper/blob/master/doc/basic-concepts.md>`_ provides a variety of image processing filters to improve images.
+
+By default, OCRmyPDF uses only ``unpaper`` arguments that were found to be safe to use on almost all files without having to inspect every page of the file afterwards. This is particularly true when only ``--clean`` is used, since that instructs OCRmyPDF to only clean the image before OCR and not the final image.
+
+However, if you wish to use the more aggressive options in ``unpaper``, you may use ``--unpaper-args '...'`` to override the OCRmyPDF's defaults and forward other arguments to unpaper. This option will forward arguments to ``unpaper`` without any knowledge of what that program considers to be valid arguments. The string of arguments must be quoted as shown in the examples below. No filename arguments may be included. OCRmyPDF will assume it can append input and output filename of intermediate images to the ``--unpaper-args`` string.
+
+In this example, we tell ``unpaper`` to expect two pages of text on a sheet (image), such as occurs when two facing pages of a book are scanned. ``unpaper`` uses this information to deskew each independently and clean up the margins of both.
+
+.. code-block:: bash
+
+    ocrmypdf --clean --clean-final --unpaper-args '--layout double' input.pdf output.pdf
+    ocrmypdf --clean --clean-final --unpaper-args '--layout double --no-noisefilter' input.pdf output.pdf
+
+.. warning::
+
+    Some ``unpaper`` features will reposition text within the image. ``--clean-final`` is recommended to avoid this issue.
+
+.. warning::
+
+    Some ``unpaper`` features cause multiple input or output files to be consumed or produced. OCRmyPDF requires ``unpaper`` to consume one file and produce one file. An deviation from that condition will result in errors.
+
+.. note::
+
+    ``unpaper`` uses uncompressed PBM/PGM/PPM files for its intermediate files. For large images or documents, it can take a lot of temporary disk space.
+
 Control of OCR options
 ----------------------
 
@@ -17,7 +45,6 @@ If ``--redo-ocr`` is issued, then a detailed text analysis is performed. Text is
 
 If ``--force-ocr`` is issued, then all pages will be rasterized to images, discarding any hidden OCR text, and rasterizing any printable text. This is useful for redoing OCR, for fixing OCR text with a damaged character map (text is selectable but not searchable), and destroying redacted information. Any forms and vector graphics will be rasterized as well.
 
-
 Time and image size limits
 """"""""""""""""""""""""""
 
@@ -58,7 +85,6 @@ For example, if you have a development build of Tesseract don't wish to use the
 
 In this example ``TESSDATA_PREFIX`` is required to redirect Tesseract to an alternate folder for its "tessdata" files.
 
-
 Overriding other support programs
 """""""""""""""""""""""""""""""""
 
@@ -189,3 +215,25 @@ user interface.  They may be imported from ``ocrmypdf.exceptions``.
     *	- 130
         - ``ExitCode.ctrl_c``
         - The program was interrupted by pressing Ctrl+C.
+
+
+Debugging the intermediate files
+--------------------------------
+
+OCRmyPDF normally saves its intermediate results to a temporary folder and deletes this folder when it exits, whether it succeeded or failed.
+
+If the ``-k`` argument is issued on the command line, OCRmyPDF will keep the temporary folder and print the location, whether it succeeded or failed (provided the Python interpreter did not crash). An example message is:
+
+.. code-block:: none
+
+    Temporary working files saved at:
+    /tmp/com.github.ocrmypdf.u20wpz07
+
+The organization of this folder is an implementation detail and subject to change between releases. However the general organization is that working files on a per page basis have the page number as a prefix (starting with page 1), an infix indicates the processing stage, and a suffix indicates the file type. Some important files include:
+
+* ``.page.png`` - what the input page looks like
+* ``.image`` - the image we will show the user if we are in a mode that changes the final appearance; may be in one of several image formats
+* ``.text.pdf`` - the OCR file; this will load as a blank page but should have visible text if checked with a tool like pdftotext or pdfminder.six
+* ``.ocr.png`` - the file that is sent to Tesseract for OCR; depending on arguments this may differ from the presentation image
+* ``layers.rendered.pdf`` - the composite PDF, before metadata repair and optimization
+* ``images/*`` - images extracted during the optimization process; here the prefix indicates a PDF object ID not a page number

docs/batch.rst

@@ -18,16 +18,22 @@ The ``--tag`` argument tells parallel to print the filename as a prefix whenever
 
 	parallel --tag -j 2 ocrmypdf '{}' 'output/{}' ::: *.pdf
 
-OCRmyPDF automatically repairs PDFs before parsing and gathering information from them.  If you are already repairing PDFs with ``qpdf`` prior to attempting OCR, or you can use ``--skip-repair`` to skip this step.  It may improve performance for large files, since repairing PDFs is single-threaded.
+OCRmyPDF automatically repairs PDFs before parsing and gathering information from them.
 
 Directory trees
 ---------------
 
-This will walk through a directory tree and run OCR on all files in place, printing the output in a way that makes 
+This will walk through a directory tree and run OCR on all files in place, printing the output in a way that makes
 
 .. code-block:: bash
 
-	find . --printf '%p' -name '*.pdf' -exec ocrmypdf '{}' '{}' \;
+	find . -printf '%p' -name '*.pdf' -exec ocrmypdf '{}' '{}' \;
+	
+Alternatively, with a docker container (mounts a volume to the container where the PDFs are stored):
+
+.. code-block:: bash
+
+	find . -printf '%p' -name '*.pdf' -exec docker run --rm -v <host dir>:<container dir> jbarlow83/ocrmypdf-alpine '<container dir>/{}' '<container dir>/{}' \;
 
 This only runs one ``ocrmypdf`` process at a time. This variation uses ``find`` to create a directory list and ``parallel`` to parallelize runs of ``ocrmypdf``, again updating files in place.
 
@@ -66,7 +72,7 @@ This user contributed script also provides an example of batch processing.
 	    log_file = script_dir + '/ocr-tree.log'
 
 	logging.basicConfig(
-			level=logging.INFO, format='%(asctime)s %(message)s', 
+			level=logging.INFO, format='%(asctime)s %(message)s',
 			filename=log_file, filemode='w')
 
 	for dir_name, subdirs, file_list in os.walk(start_dir):
@@ -80,9 +86,9 @@ This user contributed script also provides an example of batch processing.
 	            print(full_path)
 	            cmd = ["ocrmypdf",  "--deskew", filename, filename]
 	            logging.info(cmd)
-	            proc = subprocess.Popen(
+	            proc = subprocess.run(
 	            	cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-	            result = proc.stdout.read()
+	            result = proc.stdout
 	            if proc.returncode == 6:
 	                print("Skipped document because it already contained text")
 	            elif proc.returncode == 0:
@@ -144,14 +150,14 @@ This is only possible for x86-based Synology products. Some Synology products us
 	            timestamp_OCR = time.strftime("%Y-%m-%d-%H%M_OCR_")
 	            filename_OCR = timestamp_OCR + file_noext + '.pdf'
 	            docker_mount = dir_name + ':/home/docker'
-	# create string for pdf processing 
+	# create string for pdf processing
 	# diskstation needs a user:group docker:docker. find uid:gid of your diskstation docker:docker with id docker.
 	# use this uid:gid in -u flag
 	# rw rights for docker:docker at source dir are also necessary
-	# the script is processed as root user via chron 
+	# the script is processed as root user via chron
 	            cmd = ['docker', 'run', '--rm', '-v', docker_mount, '-u=1030:65538', 'jbarlow83/ocrmypdf', , '--deskew' , filename, filename_OCR]
 	            logging.info(cmd)
-	            proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+	            proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 	            result = proc.stdout.read()
 	            logging.info(result)
 	            full_path_OCR = dir_name + '/' + filename_OCR
@@ -163,14 +169,11 @@ This is only possible for x86-based Synology products. Some Synology products us
 	            shutil.move(full_path, full_path_archive)
 	logging.info('Finished.\n')
 
-
-
 Huge batch jobs
 """""""""""""""
 
 If you have thousands of files to work with, contact the author. Consulting work related to OCRmyPDF helps fund this open source project and all inquiries are appreciated.
 
-
 Hot (watched) folders
 ---------------------
 
@@ -211,4 +214,12 @@ Alternatives
 
 * `Watchman <https://facebook.github.io/watchman/>`_ is a more powerful alternative to ``watchmedo``.
 
+macOS Automator
+---------------
 
+You can use the Automator app with macOS, to create a Workflow or Quick Action. Use a *Run Shell Script* action in your workflow. In the context of Automator, the ``PATH`` may be set differently your Terminal's ``PATH``; you may need to explicitly set the PATH to include ``ocrmypdf``. The following example may serve as a starting point:
+
+.. image:: images/macos-workflow.png
+	:alt: Example macOS Automator script
+
+You may customize the command sent to ocrmypdf.

docs/conf.py

@@ -52,7 +52,9 @@ master_doc = 'index'
 
 # General information about the project.
 project = 'ocrmypdf'
-copyright = '2018, James R. Barlow. Licensed under Creative Commons Attribution-ShareAlike 4.0.'
+copyright = (
+    '2019, James R. Barlow. Licensed under Creative Commons Attribution-ShareAlike 4.0.'
+)
 author = 'James R. Barlow'
 
 # The version info for the project you're documenting, acts as replacement for
@@ -62,6 +64,7 @@ author = 'James R. Barlow'
 # The short X.Y version.
 
 import os
+
 on_rtd = os.environ.get('READTHEDOCS') == 'True'
 
 if on_rtd:
@@ -78,18 +81,16 @@ if on_rtd:
         'pikepdf',
         'pikepdf.models',
         'pikepdf.models.metadata',
-        'ocrmypdf.leptonica'
+        'ocrmypdf.leptonica',
     ]
     sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES)
 
 
-from ocrmypdf import __version__ as OCRMYPDF_VERSION
+from pkg_resources import get_distribution, DistributionNotFound
 
-_version_parts = OCRMYPDF_VERSION.split('.')
-
-version = '.'.join(_version_parts[0:2])
 # The full version, including alpha/beta/rc tags.
-release = OCRMYPDF_VERSION
+release = get_distribution('ocrmypdf').version
+version = '.'.join(release.split('.')[:2])
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -158,9 +159,7 @@ html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 # further.  For a list of options available for each theme, see the
 # documentation.
 #
-html_theme_options = {
-    'display_version': False
-}
+html_theme_options = {'display_version': False}
 
 # Add any paths that contain custom themes here, relative to this directory.
 # html_theme_path = []
@@ -273,29 +272,25 @@ htmlhelp_basename = 'ocrmypdfdoc'
 # -- Options for LaTeX output ---------------------------------------------
 
 latex_elements = {
-     # The paper size ('letterpaper' or 'a4paper').
-     #
-     # 'papersize': 'letterpaper',
-
-     # The font size ('10pt', '11pt' or '12pt').
-     #
-     # 'pointsize': '10pt',
-
-     # Additional stuff for the LaTeX preamble.
-     #
-     # 'preamble': '',
-
-     # Latex figure (float) alignment
-     #
-     # 'figure_align': 'htbp',
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
 }
 
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, 'ocrmypdf.tex', 'ocrmypdf Documentation',
-     'James R. Barlow', 'manual'),
+    (master_doc, 'ocrmypdf.tex', 'ocrmypdf Documentation', 'James R. Barlow', 'manual')
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
@@ -335,10 +330,7 @@ latex_documents = [
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [
-    (master_doc, 'ocrmypdf', 'ocrmypdf Documentation',
-     [author], 1)
-]
+man_pages = [(master_doc, 'ocrmypdf', 'ocrmypdf Documentation', [author], 1)]
 
 # If true, show URL addresses after external links.
 #
@@ -351,9 +343,15 @@ man_pages = [
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    (master_doc, 'ocrmypdf', 'ocrmypdf Documentation',
-     author, 'ocrmypdf', 'One line description of project.',
-     'Miscellaneous'),
+    (
+        master_doc,
+        'ocrmypdf',
+        'ocrmypdf Documentation',
+        author,
+        'ocrmypdf',
+        'One line description of project.',
+        'Miscellaneous',
+    )
 ]
 
 # Documents to append as an appendix to all manuals.

docs/cookbook.rst

@@ -5,7 +5,7 @@ Basic examples
 --------------
 
 Help!
-"""""
+^^^^^
 
 ocrmypdf has built-in help.
 
@@ -15,28 +15,28 @@ ocrmypdf has built-in help.
 
 
 Add an OCR layer and convert to PDF/A
-"""""""""""""""""""""""""""""""""""""
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 .. code-block:: bash
 
     ocrmypdf input.pdf output.pdf
 
 Add an OCR layer and output a standard PDF
-""""""""""""""""""""""""""""""""""""""""""
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 .. code-block:: bash
 
     ocrmypdf --output-type pdf input.pdf output.pdf
 
 Create a PDF/A with all color and grayscale images converted to JPEG
-""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 .. code-block:: bash
 
     ocrmypdf --output-type pdfa --pdfa-image-compression jpeg input.pdf output.pdf
 
 Modify a file in place
-""""""""""""""""""""""
+^^^^^^^^^^^^^^^^^^^^^^
 
 The file will only be overwritten if OCRmyPDF is successful.
 
@@ -45,7 +45,7 @@ The file will only be overwritten if OCRmyPDF is successful.
     ocrmypdf myfile.pdf myfile.pdf
 
 Correct page rotation
-"""""""""""""""""""""
+^^^^^^^^^^^^^^^^^^^^^
 
 OCR will attempt to automatic correct the rotation of each page. This can help fix a scanning job that contains a mix of landscape and portrait pages.
 
@@ -58,9 +58,9 @@ You can increase (decrease) the parameter ``--rotate-pages-threshold`` to make p
 If the page is "just a little off horizontal", like a crooked picture, then you want ``--deskew``. ``--rotate-pages`` is for when the cardinal angle is wrong.
 
 OCR languages other than English
-""""""""""""""""""""""""""""""""
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-By default OCRmyPDF assumes the document is English.
+OCRmyPDF assumes the document is in English unless told otherwise. OCR quality may be poor if the wrong language is used.
 
 .. code-block:: bash
 
@@ -69,8 +69,10 @@ By default OCRmyPDF assumes the document is English.
 
 Language packs must be installed for all languages specified. See :ref:`Installing additional language packs <lang-packs>`.
 
+Unfortunately, the Tesseract OCR engine has no ability to detect the language when it is unknown.
+
 Produce PDF and text file containing OCR text
-"""""""""""""""""""""""""""""""""""""""""""""
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 This produces a file named "output.pdf" and a companion text file named "output.txt".
 
@@ -79,7 +81,10 @@ This produces a file named "output.pdf" and a companion text file named "output.
     ocrmypdf --sidecar output.txt input.pdf output.pdf
 
 OCR images, not PDFs
---------------------
+^^^^^^^^^^^^^^^^^^^^
+
+Option: use Tesseract
+"""""""""""""""""""""
 
 If you are starting with images, you can just use Tesseract directly to convert images to PDFs:
 
@@ -92,9 +97,12 @@ If you are starting with images, you can just use Tesseract directly to convert
     # When there are multiple images
     tesseract text-file-containing-list-of-image-filenames.txt output-prefix pdf
 
-Tesseract's PDF output is quite good – OCRmyPDF uses it internally by default. However, OCRmyPDF has many features not available in Tesseract like like image processing, metadata control, and PDF/A generation.
+Tesseract's PDF output is quite good – OCRmyPDF uses it internally, in some cases. However, OCRmyPDF has many features not available in Tesseract like image processing, metadata control, and PDF/A generation.
 
-Use a program like `img2pdf <https://gitlab.mister-muffin.de/josch/img2pdf>`_ to convert your images to PDFs, and then pipe the results to run ocrmypdf.  The `-` tells ocrmypdf to read standard input.
+Option: use img2pdf
+"""""""""""""""""""
+
+You can also use a program like `img2pdf <https://gitlab.mister-muffin.de/josch/img2pdf>`_ to convert your images to PDFs, and then pipe the results to run ocrmypdf.  The ``-`` tells ocrmypdf to read standard input.
 
 .. code-block:: bash
 
@@ -102,6 +110,9 @@ Use a program like `img2pdf <https://gitlab.mister-muffin.de/josch/img2pdf>`_ to
 
 ``img2pdf`` is recommended because it does an excellent job at generating PDFs without transcoding images.
 
+Option: use OCRmyPDF (single images only)
+"""""""""""""""""""""""""""""""""""""""""
+
 For convenience, OCRmyPDF can also convert single images to PDFs on its own. If the resolution (dots per inch, DPI) of an image is not set or is incorrect, it can be overridden with ``--image-dpi``. (As 1 inch is 2.54 cm, 1 dpi = 0.39 dpcm).
 
 .. code-block:: bash
@@ -110,9 +121,10 @@ For convenience, OCRmyPDF can also convert single images to PDFs on its own. If
 
 If you have multiple images, you must use ``img2pdf`` to convert the images to PDF.
 
-.. note::
+Not recommended
+"""""""""""""""
 
-    ImageMagick ``convert`` can also convert a group of images to PDF, but in the author's experience it takes a long time, transcodes unnecessarily and gives poor results.
+We caution against using ImageMagick or Ghostscript to convert images to PDF, since they may transcode images or produce downsampled images, sometimes without warning.
 
 Image processing
 ----------------
@@ -129,7 +141,7 @@ OCRmyPDF perform some image processing on each page of a PDF, if desired.  The s
 
 * ``--clean-final`` uses unpaper to clean up pages before OCR and inserts the page into the final output.  You will want to review each page to ensure that unpaper did not remove something important.
 
-* ``--mask-barcodes`` will "cover up" any barcodes detected in the image of a page. Barcodes are known to confuse Tesseract OCR and interfere with the recognition of text on the same baseline as a barcode. The output file will contain the unaltered image of the barcode.
+* ``--mask-barcodes`` will suppress any barcodes detected in a page image. Barcodes are known to confuse Tesseract OCR and interfere with the recognition of text on the same baseline as a barcode. The output file will contain the unaltered image of the barcode.
 
 .. note::
 
@@ -139,8 +151,8 @@ OCRmyPDF perform some image processing on each page of a PDF, if desired.  The s
 
     ``--clean-final`` and ``-remove-background`` may leave undesirable visual artifacts in some images where their algorithms have shortcomings. Files should be visually reviewed after using these options.
 
-OCR and correct document skew (crooked scan)
-""""""""""""""""""""""""""""""""""""""""""""
+Example: OCR and correct document skew (crooked scan)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Deskew:
 
@@ -156,7 +168,7 @@ Image processing commands can be combined. The order in which options are given
 
 
 Don't actually OCR my PDF
-"""""""""""""""""""""""""
+-------------------------
 
 If you set ``--tesseract-timeout 0`` OCRmyPDF will apply its image processing without performing OCR, if all you want to is to apply image processing or PDF/A conversion.
 
@@ -166,7 +178,7 @@ If you set ``--tesseract-timeout 0`` OCRmyPDF will apply its image processing wi
 
 
 Redo existing OCR
-"""""""""""""""""
+-----------------
 
 To redo OCR on a file OCRed with other OCR software or a previous version of OCRmyPDF and/or Tesseract, you may use the ``--redo-ocr`` argument. (Normally, OCRmyPDF will exit with an error if asked to modify a file with OCR.)
 
@@ -196,11 +208,28 @@ PDF optimization
 
 By default OCRmyPDF will attempt to perform lossless optimizations on the images inside PDFs after OCR is complete. Optimization is performed even if no OCR text is found.
 
-The ``--optimize N`` (short form ``-O``) argument controls optimization, where ``N`` ranges from 0 to 3. ``--optimize 0`` disables optimizations. ``1`` enables lossless optimizations that can be performed safely with no quality loss. ``2`` enables lossy optimizations such as image color quantizations. ``3`` enables more aggressive optimizations and targets a lower JPEG quality.
+The ``--optimize N`` (short form ``-O``) argument controls optimization, where ``N`` ranges from 0 to 3 inclusive, analogous to the optimization levels in the GCC compiler.
 
-Optimization is improved when a JBIG2 encoder is available and when ``pngquant`` is installed. If either of these components are missing, then some types of images will not be optimized.
+.. list-table::
+    :widths: auto
+    :header-rows: 1
 
-Currently optimization attempts to find more efficient encodings for images. The types of optimization available may expand over time. By default, OCRmyPDF compresses data streams inside PDFs, and will change inefficient encodings to more modern versions. A program like ``qpdf`` can be used to change encodings, e.g. to inspect the internals fo a PDF.
+    *   - Level
+        - Comments
+    *   - ``--optimize 0``
+        - Disables optimization.
+    *   - ``--optimize 1``
+        - Enables lossless optimizations, such as transcoding images to more
+          efficient formats. Also compress other uncompressed objects in the
+          PDF and enables the more efficient "object streams" within the PDF.
+    *   - ``--optimize 2``
+        - All of the above, and enables lossy optimizations and color quantization.
+    *   - ``--optimize 3``
+        - All of the above, and enables more aggressive optimizations and targets lower image quality.
+
+Optimization is improved when a JBIG2 encoder is available and when ``pngquant`` is installed. If either of these components are missing, then some types of images cannot be optimized.
+
+The types of optimization available may expand over time. By default, OCRmyPDF compresses data streams inside PDFs, and will change inefficient compression modes to more modern versions. A program like ``qpdf`` can be used to change encodings, e.g. to inspect the internals fo a PDF.
 
 .. code-block:: bash
 

docs/docker.rst

@@ -0,0 +1,155 @@
+OCRmyPDF Docker image
+=====================
+
+OCRmyPDF is also available in a Docker image that packages recent versions of all dependencies.
+
+For users who already have Docker installed this may be an easy and convenient option. However, it is less performant than a system installation and may require Docker engine configuration.
+
+OCRmyPDF needs a generous amount of RAM, CPU cores, and temporary storage space.
+
+.. _docker-install:
+
+Installing the Docker image
+---------------------------
+
+If you have `Docker <https://docs.docker.com/>`_ installed on your system, you can install a Docker image of the latest release.
+
+The recommended OCRmyPDF Docker image is currently named ``ocrmypdf-alpine``:
+
+.. code-block:: bash
+
+    docker pull jbarlow83/ocrmypdf-alpine
+
+Follow the Docker installation instructions for your platform.  If you can run this command successfully, your system is ready to download and execute the image:
+
+.. code-block:: bash
+
+    docker run hello-world
+
+OCRmyPDF will use all available CPU cores.  By default, the VirtualBox machine instance on Windows and macOS has only a single CPU core enabled. Use the VirtualBox Manager to determine the name of your Docker engine host, and then follow these optional steps to enable multiple CPUs:
+
+.. code-block:: bash
+
+    # Optional step for Mac OS X users
+    docker-machine stop "yourVM"
+    VBoxManage modifyvm "yourVM" --cpus 2  # or whatever number of core is desired
+    docker-machine start "yourVM"
+    eval $(docker-machine env "yourVM")
+
+Using the Docker image on the command line
+------------------------------------------
+
+**Unlike typical Docker containers**, in this mode we are using the OCRmyPDF Docker container is intended to be emphemeral – it runs for one OCR job and then terminates, just like a command line program. We are using Docker as a way of delivering an application, not a server.
+
+To start a Docker container (instance of the image):
+
+.. code-block:: bash
+
+    docker tag jbarlow83/ocrmypdf-alpine ocrmypdf
+    docker run --rm ocrmypdf (... all other arguments here...)
+
+For convenience, create a shell alias to hide the Docker command:
+
+.. code-block:: bash
+
+    alias ocrmypdf='docker run --rm -v "$(pwd):/home/docker" ocrmypdf'
+    ocrmypdf --version  # runs docker version
+
+Or in the wonderful `fish shell <https://fishshell.com/>`_:
+
+.. code-block:: fish
+
+    alias ocrmypdf 'docker run --rm ocrmypdf'
+    funcsave ocrmypdf
+
+.. _docker-lang-packs:
+
+Adding languages to the Docker image
+------------------------------------
+
+By default the Docker image includes English, German and Simplified Chinese, the most popular languages for OCRmyPDF users based on feedback. You may add other languages by creating a new Dockerfile based on the public one:
+
+.. code-block:: dockerfile
+
+    FROM jbarlow83/ocrmypdf-alpine
+
+    # Add French
+    RUN apk add tesseract-ocr-data-fra
+
+Executing the test suite
+------------------------
+
+The OCRmyPDF test suite is installed with image.  To run it:
+
+.. code-block:: bash
+
+    docker run --entrypoint python3 jbarlow83/ocrmypdf-alpine setup.py test
+
+Using the OCRmyPDF web service wrapper
+--------------------------------------
+
+The OCRmyPDF Docker image includes an example, barebones HTTP web service. The webservice may be launched as follows:
+
+.. code-block:: bash
+
+    docker run --entrypoint python3 -p 5000:5000 jbarlow83/ocrmypdf-alpine webservice.py
+
+Unlike command line usage this program will open a socket and wait for connections.
+
+.. warning::
+
+    The OCRmyPDF web service wrapper is intended for demonstration or development. It provides no security, no authentication, no protection against denial of service attacks, and no load balancing. The default Flask WSGI server is used, which is intended for development only. The server is single-threaded and so can respond to only one client at a time. It cannot respond to clients while busy with OCR.
+
+Clients must keep their open connection while waiting for OCR to complete. This may entail setting a long timeout; this interface is more useful for internal HTTP API calls.
+
+Unlike the rest of OCRmyPDF, this web service is licensed under the Affero GPLv3 (AGPLv3) since Ghostscript, a dependency of OCRmyPDF, is also licensed in this way.
+
+In addition to the above, please read our :ref:`general remarks on using OCRmyPDF as a service <ocr-service>`.
+
+Legacy Ubuntu Docker images
+---------------------------
+
+Previously OCRmyPDF was delivered in several Docker images for different purposes, based on Ubuntu.
+
+The Ubuntu-based images will be maintained for some time but should not be used for new deployments. They are as follows:
+
+.. list-table::
+    :widths: auto
+    :header-rows: 1
+
+    *   - Image name
+        - Download command
+        - Notes
+    *   - ocrmypdf
+        - ``docker pull jbarlow83/ocrmypdf``
+        - Latest ocrmypdf with Tesseract 4.0.0-beta1 on Ubuntu 18.04. Includes English, French, German, Spanish, Portugeuse and Simplified Chinese.
+    *   - ocrmypdf-polyglot
+        - ``docker pull jbarlow83/ocrmypdf-polyglot``
+        - As above, with all available language packs.
+    *   - ocrmypdf-webservice
+        - ``docker pull jbarlow83/ocrmypdf-webservice``
+        - All language packs, and a simple HTTP wrapper allowing OCRmyPDF to be used as a web service. Note that this component is licensed under AGPLv3.
+
+To execute the Ubuntu-based OCRmyPDF on a local file, you must `provide a writable volume to the Docker image <https://docs.docker.com/userguide/dockervolumes/>`_, and both the input and output file must be inside the writable volume. This limitation applies only to the legacy images.
+
+This example command uses the current working directory as the writable volume:
+
+.. code-block:: bash
+
+    docker run --rm -v "$(pwd):/home/docker" <other docker arguments>   ocrmypdf <your arguments to ocrmypdf>
+
+In this worked example, the current working directory contains an input file called ``test.pdf`` and the output will go to ``output.pdf``:
+
+.. code-block:: bash
+
+    docker run --rm -v "$(pwd):/home/docker"   ocrmypdf --skip-text test.pdf output.pdf
+
+.. note:: The working directory should be a writable local volume or Docker may not have permission to access it.
+
+Note that ``ocrmypdf`` has its own separate ``-v VERBOSITYLEVEL`` argument to control debug verbosity. All Docker arguments should before the ``ocrmypdf`` image name and all arguments to ``ocrmypdf`` should be listed after.
+
+In some environments the permissions associated with Docker can be complex to configure. The process that executes Docker may end up not having the permissions to write the specified file system. In that case one can stream the file into and out of the Docker process and avoid all permission hassles, using ``-`` as the input and output filename:
+
+.. code-block:: bash
+
+    docker run --rm -i   ocrmypdf <other arguments to ocrmypdf> - - <input.pdf >output.pdf

docs/index.rst

@@ -25,6 +25,7 @@ PDF is the best format for storing and exchanging scanned documents.  Unfortunat
    :maxdepth: 2
 
    cookbook
+   docker
    advanced
    batch
    security

docs/installation.rst

@@ -1,14 +1,17 @@
-Installation
-============
+Installing OCRmyPDF
+===================
 
 .. |latest| image:: https://img.shields.io/pypi/v/ocrmypdf.svg
     :alt: OCRmyPDF latest released version on PyPI
 
 |latest|
 
-The easiest way to install OCRmyPDF to follow the steps for your operating system/platform.
+The easiest way to install OCRmyPDF is to follow the steps for your operating
+system/platform, although sometimes this version may be out of date.
 
-If you want to use the latest version of OCRmyPDF, your best bet is to install the most recent version your platform provides, and then upgrade that version by installing the Python binary wheels.
+If you want to use the latest version of OCRmyPDF, your best bet is to install
+the most recent version your platform provides, and then upgrade that version by
+installing the Python binary wheels.
 
 .. contents:: Platform-specific steps
     :depth: 2
@@ -100,7 +103,7 @@ sources`_.
     OCRmyPDF works fine without it but will produce larger output files. If you
     build jbig2enc from source, ocrmypdf 7.0.0 and later will automatically
     detect it on the ``PATH``. To add JBIG2 encoding, see `Installing the JBIG2
-    encoder`_.
+    encoder <jbig2>`_.
 
 Installing the latest version on Ubuntu 18.04 LTS
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -136,22 +139,56 @@ To add JBIG2 encoding, see :ref:`jbig2`.
 Ubuntu 16.04 LTS
 ^^^^^^^^^^^^^^^^
 
-No package is currently available for Ubuntu 16.04, but you can install the dependencies manually:
+No package is available for Ubuntu 16.04. OCRmyPDF 8.0 and newer require Python
+3.6. Ubuntu 16.04 ships Python 3.5, but you can install Python 3.6 on it. Or,
+you can skip Python 3.6 and install OCRmyPDF 7.x or older - for that procedure,
+please see the installation documentation for the version of OCRmyPDF you plan
+to use.
+
+**Install system packages for OCRmyPDF**
 
 .. code-block:: bash
 
     sudo apt-get update
-    sudo apt-get install \
+    sudo apt-get install -y software-properties-common python-software-properties
+    sudo add-apt-repository -y \
+        ppa:jonathonf/python-3.6 \
+        ppa:alex-p/tesseract-ocr
+    sudo apt-get update
+    sudo apt-get install -y \
         ghostscript \
         libexempi3 \
+        libffi6 \
         pngquant \
-        python3-cffi \
-        python3-pip \
+        python3.6 \
         qpdf \
         tesseract-ocr \
         unpaper
 
-If you wish install OCRmyPDF for the current user, and ensure that the ``PATH``
+This will install a Python 3.6 binary at ``/usr/bin/python3.6`` alongside the
+system's Python 3.5. Do not remove the system Python. This will also install
+Tesseract 4.0 from a PPA, since the version available in Ubuntu 16.04 is too old
+for OCRmyPDF.
+
+Now install pip for Python 3.6. This will install the Python 3.6 version of
+``pip`` at ``/usr/local/bin/pip``.
+
+.. code-block:: bash
+
+    curl https://bootstrap.pypa.io/get-pip.py | sudo python3.6
+
+**Install OCRmyPDF**
+
+OCRmyPDF requires the locale to be set for UTF-8. **On some minimal Ubuntu
+installations systems**, it may be necessary to set the locale.
+
+.. code-block:: bash
+
+    # Optional: Only need to set these if they are not already set
+    export LC_ALL=C.UTF-8
+    export LANG=C.UTF-8
+
+Now install OCRmyPDF for the current user, and ensure that the ``PATH``
 environment variable contains ``$HOME/.local/bin``.
 
 .. code-block:: bash
@@ -159,38 +196,20 @@ environment variable contains ``$HOME/.local/bin``.
     export PATH=$HOME/.local/bin:$PATH
     pip3 install --user ocrmypdf
 
-Alternately, you can install ocrmypdf system-wide. (Not recommended.)
-
-.. code-block:: bash
-
-    sudo pip3 install ocrmypdf
-
-At your option, you may upgrade Ubuntu 16.04 LTS to Tesseract 4.0 for improved OCR results.
-
-.. code-block:: bash
-
-    sudo apt-get install -y software-properties-common python-software-properties
-    sudo add-apt-repository ppa:alex-p/tesseract-ocr -y
-    sudo apt-get update
-    sudo apt-get upgrade tesseract-ocr
-
 To add JBIG2 encoding, see :ref:`jbig2`.
 
 Ubuntu 14.04 LTS
 ^^^^^^^^^^^^^^^^
 
-Installing on Ubuntu 14.04 LTS (trusty) is more difficult than some other options, because it is older and does not provide ``pip``.
-
-Update apt-get:
-
-.. code-block:: bash
-
-    sudo apt-get update
+Installing on Ubuntu 14.04 LTS (trusty) is more difficult than some other
+options, because of its age. Several backports are required. For explanations of
+some steps of this procedure, see the similar steps for Ubuntu 16.04.
 
 Install system dependencies:
 
 .. code-block:: bash
 
+    sudo apt-get update
     sudo apt-get install \
         software-properties-common python-software-properties \
         zlib1g-dev \
@@ -200,9 +219,13 @@ Install system dependencies:
         pngquant \
         qpdf
 
-We will need backports of Ghostscript 9.16, libav-11 (for unpaper 6.1), Tesseract 4.00 (alpha), and Python 3.6. This will replace Ghostscript and Tesseract 3.x on your system. Python 3.6 will be installed alongside the system Python 3.4.
+We will need backports of Ghostscript 9.16, libav-11 (for unpaper 6.1),
+Tesseract 4.00 (alpha), and Python 3.6. This will replace Ghostscript and
+Tesseract 3.x on your system. Python 3.6 will be installed alongside the system
+Python 3.4.
 
-If you prefer to not modify your system in this matter, consider using a Docker container.
+If you prefer to not modify your system in this matter, consider using a Docker
+container.
 
 .. code-block:: bash
 
@@ -228,8 +251,6 @@ Now we need to install ``pip`` and let it install ocrmypdf:
     curl https://bootstrap.pypa.io/ez_setup.py -o - | python3.6 && python3.6 -m easy_install pip
     pip3.6 install ocrmypdf
 
-The ``wget`` command will download a program and run it.
-
 These installation instructions omit the optional dependency ``unpaper``, which is only available at version 0.4.2 in Ubuntu 14.04. The author could not find a backport of ``unpaper``, and created a .deb package to do the job of installing unpaper 6.1 (for x86 64-bit only):
 
 .. code-block:: bash
@@ -239,14 +260,14 @@ These installation instructions omit the optional dependency ``unpaper``, which
 
 To add JBIG2 encoding, see :ref:`jbig2`.
 
-ArchLinux
-^^^^^^^^^
+ArchLinux (AUR)
+^^^^^^^^^^^^^^^
 
 .. image:: https://repology.org/badge/version-for-repo/aur/ocrmypdf.svg
     :alt: ArchLinux
     :target: https://repology.org/metapackage/ocrmypdf
 
-The author is aware of an `ArchLinux User Repository package for ocrmypdf <https://aur.archlinux.org/packages/ocrmypdf/>`_. You can use the following command.
+There is an `ArchLinux User Repository package for ocrmypdf <https://aur.archlinux.org/packages/ocrmypdf/>`_. You can use the following command.
 
 .. code-block:: bash
 
@@ -277,6 +298,12 @@ OCRmyPDF is now a standard `Homebrew <https://brew.sh>`_ formula. To install on
 
     brew install ocrmypdf
 
+This will include only the English language pack. If you need other languages you can optionally install them all:
+
+.. code-block:: bash
+
+    brew install tesseract-lang  # Optional: Install all language packs
+
 .. note::
 
     Users who previously installed OCRmyPDF on macOS using ``pip install ocrmypdf`` should remove the pip version (``pip3 uninstall ocrmypdf``) before switching to the Homebrew version.
@@ -337,133 +364,39 @@ The command line program should now be available:
 
     ocrmypdf --help
 
-.. _docker-install:
+Installing on FreeBSD
+---------------------
+
+FreeBSD 11.2 is known to work. Other versions likely work but have not been tested.
+
+In general it should work to:
+
+#. `Install and build pikepdf <https://pikepdf.readthedocs.io/en/latest/installation.html#installing-on-freebsd-11-2>`_.
+#. Install the equivalent list of dependencies for Linux.
 
 Installing the Docker image
 ---------------------------
 
 For some users, installing the Docker image will be easier than installing all of OCRmyPDF's dependencies. For Windows, it is the only option.
 
-If you have `Docker <https://docs.docker.com/>`_ installed on your system, you can install a Docker image of the latest release.
-
-Follow the Docker installation instructions for your platform.  If you can run this command successfully, your system is ready to download and execute the image:
-
-.. code-block:: bash
-
-    docker run hello-world
-
-OCRmyPDF will use all available CPU cores.  By default, the VirtualBox machine instance on Windows and macOS has only a single CPU core enabled. Use the VirtualBox Manager to determine the name of your Docker engine host, and then follow these optional steps to enable multiple CPUs:
-
-.. code-block:: bash
-
-    # Optional step for Mac OS X users
-    docker-machine stop "yourVM"
-    VBoxManage modifyvm "yourVM" --cpus 2  # or whatever number of core is desired
-    docker-machine start "yourVM"
-    eval $(docker-machine env "yourVM")
-
-Assuming you have a Docker engine running, you can download one of the three available images:
-
-.. list-table::
-    :widths: auto
-    :header-rows: 1
-
-    *   - Image name
-        - Download command
-        - Notes
-    *   - ocrmypdf
-        - ``docker pull jbarlow83/ocrmypdf``
-        - Latest ocrmypdf with Tesseract 4.0.0-beta1 on Ubuntu 18.04. Includes English, French, German, Spanish, Portugeuse and Simplified Chinese.
-    *   - ocrmypdf-polyglot
-        - ``docker pull jbarlow83/ocrmypdf-polyglot``
-        - As above, with all available language packs.
-    *   - ocrmypdf-webservice
-        - ``docker pull jbarlow83/ocrmypdf-polyglot``
-        - All language packs, and a simple HTTP wrapper allowing OCRmyPDF to be used as a web service. Note that this component is licensed under AGPLv3.
-
-For example:
-
-.. code-block:: bash
-
-    docker pull jbarlow83/ocrmypdf
-
-Then tag it to give a more convenient name, just ocrmypdf:
-
-.. code-block:: bash
-
-    docker tag jbarlow83/ocrmypdf ocrmypdf
-
-.. _docker-polyglot:
-
-The alternative "polyglot" image provides `all available language packs <https://github.com/tesseract-ocr/tesseract/blob/master/doc/tesseract.1.asc#languages>`_.
-
-You can then run ocrmypdf using the command:
-
-.. code-block:: bash
-
-    docker run --rm ocrmypdf --help
-
-To execute the OCRmyPDF on a local file, you must `provide a writable volume to the Docker image <https://docs.docker.com/userguide/dockervolumes/>`_, and both the input and output file must be inside the writable volume.  This example command uses the current working directory as the writable volume:
-
-.. code-block:: bash
-
-    docker run --rm -v "$(pwd):/home/docker" <other docker arguments>   ocrmypdf <your arguments to ocrmypdf>
-
-In this worked example, the current working directory contains an input file called ``test.pdf`` and the output will go to ``output.pdf``:
-
-.. code-block:: bash
-
-    docker run --rm -v "$(pwd):/home/docker"   ocrmypdf --skip-text test.pdf output.pdf
-
-.. note:: The working directory should be a writable local volume or Docker may not have permission to access it.
-
-Note that ``ocrmypdf`` has its own separate ``-v VERBOSITYLEVEL`` argument to control debug verbosity. All Docker arguments should before the ``ocrmypdf`` image name and all arguments to ``ocrmypdf`` should be listed after.
-
-In some environments the permissions associated with Docker can be complex to configure. The process that executes Docker may end up not having the permissions to write the specified file system. In that case one can stream the file into and out of the Docker process and avoid all permission hassles, using ``-`` as the input and output filename:
-
-.. code-block:: bash
-
-    docker run --rm -i   ocrmypdf <other arguments to ocrmypdf> - - <input.pdf >output.pdf
-
-For convenience, a shell alias can hide the docker command:
-
-.. code-block:: bash
-
-    alias ocrmypdf='docker run --rm -v "$(pwd):/home/docker" ocrmypdf'
-    ocrmypdf --version  # runs docker version
-
-Or in the wonderful `fish shell <https://fishshell.com/>`_:
-
-.. code-block:: fish
-
-    alias ocrmypdf 'docker run --rm -v (pwd):/home/docker ocrmypdf'
-    funcsave ocrmypdf
-
-.. note::
-
-    The ocrmypdf Docker images are designed for application delivery, to enable use of OCRmyPDF without fussing with dependencies. ``docker run --rm`` argument tells Docker to delete the container after it runs, because each container is only good for a single job. The Docker image is not designed for use as a persistent web service or for use on Amazon EC2 Container Service (AWS ECS).
+See `OCRmyPDF Docker Image <docker>`_ for more information.
 
 Installing on Windows
 ---------------------
 
-Direct installation on Windows is not possible.  `Install the Docker <docker-install_>`_ container as described above.  Ensure that your command prompt can run the docker "hello world" container.
+Direct installation on Windows is not possible, because there are a
+POSIX dependencies. Your options are:
 
-It would probably not be too difficult to run on Windows.  The main reason this has been avoided is the difficulty of packaging and installing the various non-Python dependencies: Tesseract, QPDF, Ghostscript, Leptonica.  Pull requests to add or improve Windows support would be quite welcome.
-
-
-Running on Windows
-~~~~~~~~~~~~~~~~~~
-
-The command line syntax to run ocrmypdf from a command prompt will resemble:
-
-.. code-block:: bat
-
-    docker run -v /c/Users/sampleuser:/home/docker ocrmypdf --skip-text test.pdf output.pdf
-
-where /c/Users/sampleuser is a Unix representation of the Windows path C:\\Users\\sampleuser, assuming a user named "sampleuser" is running ocrmypdf on a file in their home directory, and the files "test.pdf" and "output.pdf" are in the sampleuser folder. The Windows user must have read and write permissions.
-
-`Bash on Ubuntu on Windows <https://github.com/Microsoft/BashOnWindows>`_ should also be a viable route for running the OCRmyPDF Docker container.
+* Install Ubuntu 18.04 in Windows 10 Subsystem for Linux, then follow
+  the Ubuntu 18.04 procedure.
+* `Install the Docker <docker-install>`__ container. Ensure that
+  your command prompt can run the docker "hello world" container.
 
+It would probably not be too difficult to port on Windows. The main
+reason this has been avoided is the difficulty of packaging and
+installing the various non-Python dependencies: Tesseract, QPDF,
+Ghostscript, Leptonica. Pull requests to add or improve Windows support
+would be quite welcome.
 
 Installing with Python pip
 --------------------------
@@ -489,7 +422,7 @@ Since ``pip3 install --user`` does not work correctly on some platforms, notably
     pip3 install ocrmypdf
 
 Requirements for pip and HEAD install
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 OCRmyPDF currently requires these external programs and libraries to be installed, and must be satisfied using the operating system package manager. ``pip`` cannot provide them.
 
@@ -563,7 +496,7 @@ dependencies. Older version than the ones mentioned in the release notes
 are likely not to be compatible to OCRmyPDF.
 
 For development
-~~~~~~~~~~~~~~~
+^^^^^^^^^^^^^^^
 
 To install all of the development and test requirements:
 
@@ -577,3 +510,17 @@ To install all of the development and test requirements:
     pip install -r requirements/dev.txt -r requirements/test.txt
 
 To add JBIG2 encoding, see :ref:`jbig2`.
+
+Shell completions
+-----------------
+
+Completions for ``bash`` and ``fish`` are available in the project's
+``misc/completion`` folder. The ``bash`` completions are likely ``zsh``
+compatible but this has not been confirmed. Package maintainers, please install
+these at the appropriate locations for your system.
+
+To manually install the ``bash`` completion, copy ``misc/completion/ocrmypdf.bash`` to
+``/etc/bash_completion.d/ocrmypdf`` (rename the file).
+
+To manually install the ``fish`` completion, copy ``misc/completion/ocrmypdf.fish`` to
+``~/.config/fish/completions/ocrmypdf.fish``.

docs/introduction.rst

@@ -16,7 +16,7 @@ About PDFs
 
 PDFs are page description files that attempts to preserve a layout exactly. They  contain `vector graphics <http://vector-conversions.com/vectorizing/raster_vs_vector.html>`_ that can contain raster objects such as scanned images. Because PDFs can contain multiple pages (unlike many image formats) and can contain fonts and text, it is a good formats for exchanging scanned documents.
 
-.. image:: bitmap_vs_svg.svg
+.. image:: images/bitmap_vs_svg.svg
 
 A PDF page might contain multiple images, even if it only appears to have one image.  Some scanners or scanning software will segment pages into monochromatic text and color regions for example, to improve the compression ratio and appearance of the page.
 
@@ -110,10 +110,10 @@ To the author's knowledge, OCRmyPDF is the most feature-rich and thoroughly test
 Web front-ends
 --------------
 
-The Docker image ocrmypdf-webservice provides a web service front-end that allows files to submitted over HTTP and the results "downloaded". This is an HTTP server intended to simplify web services deployments; it is not intended to be deployed on the public internet and no real security measures to speak of.
+The Docker image ``ocrmypdf-alpine`` provides a web service front-end that allows files to submitted over HTTP and the results "downloaded". This is an HTTP server intended to simplify web services deployments; it is not intended to be deployed on the public internet and no real security measures to speak of.
 
-In addition, the following integrations are available:
+In addition, the following third-party integrations are available:
 
 * `Nextcloud OCR <https://github.com/janis91/ocr>`_ is a free software plugin for the Nextcloud private cloud software
 
-Bear in mind that OCRmyPDF is not designed to be secure against malware-bearing PDFs (see `Using OCRmyPDF online`_). Users should ensure they comply with OCRmyPDF's licenses and the licenses of all dependencies. In particular, OCRmyPDF requires Ghostscript, which is licensed under AGPLv3.
+OCRmyPDF is not designed to be secure against malware-bearing PDFs (see `Using OCRmyPDF online <ocr-service>`_). Users should ensure they comply with OCRmyPDF's licenses and the licenses of all dependencies. In particular, OCRmyPDF requires Ghostscript, which is licensed under AGPLv3.

docs/jbig2.rst

@@ -32,4 +32,4 @@ JBIG2 lossy mode does achieve higher compression ratios than any other monochrom
 
 To turn on JBIG2 lossy mode, add the argument ``--jbig2-lossy``. ``--optimize {1,2,3}`` are necessary for the argument to take effect also required. Also, a JBIG2 encoder must be installed as described in the previous section.
 
-*ocrmypdf v7.0 and v7.1 used lossy mode by default.*
+*Due to an oversight, ocrmypdf v7.0 and v7.1 used lossy mode by default.*

docs/languages.rst

@@ -3,7 +3,7 @@
 Installing additional language packs
 ====================================
 
-OCRmyPDF uses Tesseract for OCR, and relies on its language packs for languages other than English. 
+OCRmyPDF uses Tesseract for OCR, and relies on its language packs for languages other than English.
 
 Tesseract supports `most languages <https://github.com/tesseract-ocr/tesseract/blob/master/doc/tesseract.1.asc#languages>`_.
 
@@ -46,31 +46,4 @@ You can install additional language packs by :ref:`installing Tesseract using Ho
 Docker users
 ------------
 
-Users of the Docker image may use the alternative :ref:`"polyglot" container <docker-polyglot>` which includes all languages.
-
-Adding individual language packs to a Docker image
-""""""""""""""""""""""""""""""""""""""""""""""""""
-
-If you wish to add a single language pack, you could do the following:
-
-* Download the desired ``.trainedata`` file from the `tessdata <https://github.com/tesseract-ocr/tessdata>`_ repository. Let's use Hebrew in this example (``heb.traineddata``)
-
-* Copy the file to ``/home/user/downloads/heb.traineddata``.
-
-* Create a new container based on the ocrmypdf-tess4 image and jump into it with a terminal:
-
-.. code-block:: bash
-
-	host$ docker run  -v /home/user/downloads:/home/docker -it --entrypoint /bin/bash ocrmypdf-tess4
-
-* Put the file where Tesseract expects it:
-
-.. code-block:: bash
-
-	docker$ cp /home/docker/heb.traineddata /usr/share/tesseract-ocr/tessdata
-
-* Note the container id, and save it as a new image (in this example, ``ocrmypdf-tess4-heb``)
-
-.. code-block:: bash
-
-    host$ docker commit <container_id> ocrmypdf-tess4-heb
+Users of the OCRmyPDF Docker image should install language packs into a derived Docker image as :ref:`described in that section <docker-lang-packs>`.

docs/release_notes.rst

@@ -13,6 +13,102 @@ Note that it is licensed under GPLv3, so scripts that ``import ocrmypdf`` and ar
    find:    [^`]\#([0-9]{1,3})[^0-9]
    replace: `#$1 <https://github.com/jbarlow83/OCRmyPDF/issues/$1>`_
 
+v8.3.2
+------
+
+-   Dropped workaround for macOS that allowed it work without pdfminer.six,
+    now a proper sdist release of pdfminer.six is available.
+
+-   pikepdf 1.5.0 is now required.
+
+v8.3.1
+------
+
+-   Fixed an issue where PDFs with malformed metadata would be rendered as blank pages. `#398 <https://github.com/jbarlow83/OCRmyPDF/issues/398>`_.
+
+v8.3.0
+------
+
+-   Improved the strategy for updating pages when a new image of the page was produced. We know attempt to preserve more content from the original file, for annotations in particular.
+
+-   For PDFs with more than 100 pages and a sequence where one PDF page was replaced and one or more subsequent ones were skipped, an intermediate file would be corrupted while grafting OCR text, causing processing to fail.
+
+-   Previously, we resized the images produced by Ghostscript by a small number of pixels to ensure the output image size was an exactly what we wanted. Having discovered a way to get Ghostscript to produce the exact image sizes we require, we eliminated the resizing step.
+
+-   Command line completions for ``bash`` are now available, in addition to ``fish``, both in ``misc/completion``. Package maintainers, please install these so users can take advantage.
+
+-   Updated requirements.
+
+-   pikepdf 1.3.0 is now required.
+
+v8.2.4
+------
+
+-   Fixed a false positive while checking for a certain type of PDF that only Acrobat can read. We now more accurately detect Acrobat-only PDFs.
+
+-   OCRmyPDF holds fewer open file handles and is more prompt about releasing those it no longer needs.
+
+-   Minor optimization: we no longer traverse the table of contents to ensure all references in it are resolved, as changes to libqpdf have made this unnecessary.
+
+-   pikepdf 1.2.0 is now required.
+
+v8.2.3
+------
+
+-   Fixed that ``--mask-barcodes`` would occasionally leave a unwanted temporary file named ``junkpixt`` in the current working folder.
+
+-   Fixed (hopefully) handling of Leptonica errors in an environment where a non-standard ``sys.stderr`` is present.
+
+-   Improved help text for ``--verbose``.
+
+v8.2.2
+------
+
+-   Fixed a regression from v8.2.0, an exception that occurred while attempting to report that ``unpaper`` or another optional dependency was unavailable.
+
+-   In some cases, ``ocrmypdf [-c|--clean]`` failed to exit with an error when ``unpaper`` is not installed.
+
+v8.2.1
+------
+
+-   This release was canceled.
+
+v8.2.0
+------
+
+-   A major improvement to our Docker image is now available thanks to hard work contributed by @mawi12345. The new Docker image, ocrmypdf-alpine, is based on Alpine Linux, and includes most of the functionality of three existed images in a smaller package. This image will replace the main Docker image eventually but for now all are being built. `See documentation for details <https://ocrmypdf.readthedocs.io/en/latest/docker.html>`_.
+
+-   Documentation reorganized especially around the use of Docker images.
+
+-   Fixed a problem with PDF image optimization, where the optimizer would unnecessarily decompress and recompress PNG images, in some cases losing the benefits of the quantization it just had just performed. The optimizer is now capable of embedding PNG images into PDFs without transcoding them.
+
+-   Fixed a minor regression with lossy JBIG2 image optimization. All JBIG2 candidates images were incorrectly placed into a single optimization group for the whole file, instead of grouping pages together. This usually makes a larger JBIG2Globals dictionary and results in inferior compression, so it worked less well than designed. However, quality would not be impacted. Lossless JBIG2 was entirely unaffected.
+
+-   Updated dependencies, including pikepdf to 1.1.0. This fixes `#358 <https://github.com/jbarlow83/OCRmyPDF/issues/358>`_.
+
+-   The install-time version checks for certain external programs have been removed from setup.py. These tests are now performed at run-time.
+
+-   The non-standard option to override install-time checks (``setup.py install --force``) is now deprecated and prints a warning. It will be removed in a future release.
+
+v8.1.0
+------
+
+-   Added a feature, ``--unpaper-args``, which allows passing arbitrary arguments to ``unpaper`` when using ``--clean`` or ``--clean-final``. The default, very conservative unpaper settings are suppressed.
+
+-   The argument ``--clean-final`` now implies ``--clean``. It was possible to issue ``--clean-final`` on its before this, but it would have no useful effect.
+
+-   Fixed an exception on traversing corrupt table of contents entries (specifically, those with invalid destination objects)
+
+-   Fixed an issue when using ``--tesseract-timeout`` and image processing features on a file with more than 100 pages. `#347 <https://github.com/jbarlow83/OCRmyPDF/issues/347>`_
+
+-   OCRmyPDF now always calls ``os.nice(5)`` to signal to operating systems that it is a background process.
+
+v8.0.1
+------
+
+-   Fixed an exception when parsing PDFs that are missing a required field. `#325 <https://github.com/jbarlow83/OCRmyPDF/issues/325>`_
+
+-   pikepdf 1.0.5 is now required, to address some other PDF parsing issues.
 
 v8.0.0
 ------

docs/security.rst

@@ -25,12 +25,14 @@ Finally, OCRmyPDF rasterizes each page of the PDF using `Ghostscript <http://gho
 
 Depending on the options specified, OCRmyPDF may graft the OCR layer into the existing PDF or it may essentially reconstruct ("re-fry") a visually identical PDF that may be quite different at the binary level. That said, OCRmyPDF is not a tool designed for sanitizing PDFs.
 
+.. _ocr-service:
+
 Using OCRmyPDF online or as a service
 -------------------------------------
 
-OCRmyPDF should not be deployed as a public-facing service, such as a website where a potential attacker could upload a PDF of their choice for OCR. OCRmyPDF is not designed to be secure against PDF malware. Another concern is PDFs specifically designed to be a denial of service attack: PDFs can contain recursive data structures that sometimes send parsers into infinite loops, and issue complex graphics drawing commands.
+OCRmyPDF is not designed for use as a public web service where a malicious user could upload a chosen PDF. In particular, it is not necessarily secure against PDF malware or PDFs that cause denial of service. OCRmyPDF relies on Ghostscript, and therefore, if deployed online one should be prepared to comply with Ghostscript's Affero GPL license, OCRmyPDF's GPL license, and any other licenses.
 
-Setting aside these concerns, a side effect of OCRmyPDF is it may incidentally sanitize PDFs that contain malware. It runs ``qpdf`` to repair the PDF, which could correct malformed PDF structures that are part of an attack. When PDF/A output is selected (the default), the input PDF is partially reconstructed by Ghostscript. When ``--force-ocr`` is used, all pages are rasterized and reconverted to PDF, which could remove malware in embedded images. No guarantees.
+Setting aside these concerns, a side effect of OCRmyPDF is it may incidentally sanitize PDFs that contain certain types of malware. It runs ``qpdf`` to repair the PDF, which could correct malformed PDF structures that are part of an attack. When PDF/A output is selected (the default), the input PDF is partially reconstructed by Ghostscript. When ``--force-ocr`` is used, all pages are rasterized and reconverted to PDF, which could remove malware in embedded images.
 
 OCRmyPDF should be relatively safe to use in a trusted intranet, with some considerations:
 

misc/completion/ocrmypdf.bash

@@ -0,0 +1,87 @@
+# ocrmypdf completion                                     -*- shell-script -*-
+
+_ocrmypdf()
+{
+    local cur prev cword words split
+    _init_completion -s || return
+
+    case $prev in
+        --version|-h|--help)
+            return
+            ;;
+        --user-words|--user-patterns|--tesseract-config)
+            _filedir
+            return
+            ;;
+        --output-type)
+            COMPREPLY=( $( compgen -W 'pdfa pdf pdfa-1 pdfa-2 pdfa-3' -- \
+                "$cur" ) )
+            return
+            ;;
+        --pdf-renderer)
+            COMPREPLY=( $( compgen -W 'auto hocr sandwich' -- "$cur" ) )
+            return
+            ;;
+        --pdfa-image-compression)
+            COMPREPLY=( $( compgen -W 'auto jpeg lossless' -- "$cur" ) )
+            return
+            ;;
+        -O|--optimize|--tesseract-oem)
+            COMPREPLY=( $( compgen -W '{0..3}' -- "$cur" ) )
+            return
+            ;;
+        --jpeg-quality|--png-quality)
+            COMPREPLY=( $( compgen -W '{0..100}' -- "$cur" ) )
+            return
+            ;;
+        -l|--language)
+            COMPREPLY=$( command tesseract --list-langs 2>/dev/null )
+            COMPREPLY=( $( compgen -W '${COMPREPLY[@]##*:}' -- "$cur" ) )
+            return
+            ;;
+        --image-dpi|--oversample|--skip-big|--max-image-mpixels|\
+        --tesseract-timeout|--rotate-pages-threshold)
+            COMPREPLY=( $( compgen -P "$cur" -W '{0..9}' ) )
+            return
+            ;;
+        -j|--jobs)
+            COMPREPLY=( $( compgen -W '{1..'$( _ncpus )'}' -- "$cur" ) )
+            return
+            ;;
+        -v|--verbose)
+            COMPREPLY=( $( compgen -W '{1..9}' -- "$cur" ) ) # max level ?
+            return
+            ;;
+        --tesseract-pagesegmode)
+            COMPREPLY=( $( compgen -W '{1..13}' -- "$cur" ) )
+            return
+            ;;
+        --sidecar|--title|--author|--subject|--keywords|--unpaper-args)
+            # argument required but no completions available
+            return
+            ;;
+    esac
+
+    $split && return
+
+    if [[ $cur == -* ]]; then
+        COMPREPLY=( $( compgen -W '--language --image-dpi --output-type
+            --sidecar --version --jobs --quiet --verbose --title --author
+            --subject --keywords --rotate-pages --remove-background --deskew
+            --clean --clean-final --unpaper-args --oversample --remove-vectors
+            --mask-barcodes --threshold --force-ocr --skip-text --redo-ocr
+            --skip-big --jpeg-quality --png-quality --jbig2-lossy
+            --max-image-mpixels --tesseract-config --tesseract-pagesegmode
+            --help --tesseract-oem --pdf-renderer --tesseract-timeout
+            --rotate-pages-threshold --pdfa-image-compression --user-words
+            --user-patterns --keep-temporary-files --flowchart --output-type' \
+            --  "$cur" ) )
+        return
+    else
+        _filedir
+        return
+    fi
+} &&
+complete -F _ocrmypdf ocrmypdf
+
+# ex: filetype=sh

pyproject.toml

@@ -0,0 +1,32 @@
+[build-system]
+requires = [
+  "setuptools >= 30.3.0",
+  "wheel",
+  "cffi",
+  "setuptools_scm",
+  "setuptools_scm_git_archive"
+]
+build-backend = "setuptools.build_meta"
+
+[tool.black]
+line-length = 88
+py36 = true
+skip-string-normalization = true
+include = '\.pyi?$'
+exclude = '''
+/(
+    \.eggs
+  | \.git
+  | \.hg
+  | \.mypy_cache
+  | \.tox
+  | \.venv
+  | _build
+  | buck-out
+  | build
+  | dist
+  | docs
+  | misc
+  | \.egg-info
+)/
+'''

requirements/main.txt

@@ -2,12 +2,12 @@
 # setup.py lists a separate set of requirements that are looser to simplify
 # installation
 chardet == 3.0.4
-cffi == 1.11.5
-img2pdf == 0.3.1
+cffi == 1.12.2
+img2pdf == 0.3.3
 pdfminer.six == 20181108
-pikepdf == 1.0.2
+pikepdf == 1.5.0.post0
 Pillow >= 5.0.0, != 5.1.0 ; sys_platform == "darwin"
 pycparser == 2.19
 python-xmp-toolkit == 2.0.1
-reportlab == 3.5.9
-ruffus == 2.8.0
+reportlab == 3.5.13
+ruffus == 2.8.1

requirements/test.txt

@@ -1,7 +1,7 @@
-pytest == 3.9.3
-pytest-helpers-namespace
-pytest-xdist
-pytest-cov
+pytest >= 4.4.1, < 5
+pytest-helpers-namespace >= 2019.1.8
+pytest-xdist == 1.28.0
+pytest-cov >= 2.6.1
 python-xmp-toolkit   # requires apt-get install libexempi3
                      # or brew install exempi
 PyPDF2 >= 1.26.0

setup.py

@@ -20,6 +20,7 @@
 from __future__ import print_function, unicode_literals
 
 import sys
+
 if sys.version_info < (3, 6):
     print("Python 3.6 or newer is required", file=sys.stderr)
     sys.exit(1)
@@ -31,171 +32,19 @@ import re
 
 # pylint: disable=w0613
 
-missing_program = '''
-The program '{program}' could not be executed or was not found on your
-system PATH.
-'''
-
-unknown_version = '''
-OCRmyPDF requires '{program}' {need_version} or higher.  Your system has
-'{program}' but we cannot tell what version is installed.  Contact the
-package maintainer.
-'''
-
-old_version = '''
-OCRmyPDF requires '{program}' {need_version} or higher.  Your system appears
-to have {found_version}.  Please update this program.
-'''
-
-okay_its_optional = '''
-This program is OPTIONAL, so installation of OCRmyPDF can proceed, but
-some functionality may be missing.
-'''
-
-not_okay_its_required = '''
-This program is REQUIRED for OCRmyPDF to work.  Installation will abort.
-'''
-
-osx_install_advice = '''
-If you have homebrew installed, try these command to install the missing
-packages:
-    brew update
-    brew upgrade
-    brew install {package}
-'''
-
-linux_install_advice = '''
-On systems with the aptitude package manager (Debian, Ubuntu), try these
-commands:
-    sudo apt-get update
-    sudo apt-get install {package}
-
-On RPM-based systems (Red Hat, Fedora), search for instructions on
-installing the RPM for {program}.
-'''
-
-
-def get_platform():
-    if sys.platform.startswith('freebsd'):
-        return 'freebsd'
-    elif sys.platform.startswith('linux'):
-        return 'linux'
-    return sys.platform
-
-
-def _error_trailer(program, package, optional, **kwargs):
-    if optional:
-        print(okay_its_optional.format(**locals()), file=sys.stderr)
-    else:
-        print(not_okay_its_required.format(**locals()), file=sys.stderr)
-
-    if isinstance(package, Mapping):
-        package = package[get_platform()]
-
-    if get_platform() == 'darwin':
-        print(osx_install_advice.format(**locals()), file=sys.stderr)
-    elif get_platform() == 'linux':
-        print(linux_install_advice.format(**locals()), file=sys.stderr)
-
-
-def error_missing_program(
-        program,
-        package,
-        optional
-        ):
-    print(missing_program.format(**locals()), file=sys.stderr)
-    _error_trailer(**locals())
-
-
-def error_unknown_version(
-        program,
-        package,
-        optional,
-        need_version
-        ):
-    print(unknown_version.format(**locals()), file=sys.stderr)
-    _error_trailer(**locals())
-
-
-def error_old_version(
-        program,
-        package,
-        optional,
-        need_version,
-        found_version
-        ):
-    print(old_version.format(**locals()), file=sys.stderr)
-    _error_trailer(**locals())
-
-
-def check_external_program(
-        program,
-        need_version,
-        package,
-        version_check_args=None,
-        version_scrape_regex=re.compile(r'(\d+\.\d+(?:\.\d+)?)'),
-        optional=False):
-    if not version_check_args:
-        version_check_args = ['--version']
-    print(f'Checking for {program} >= {need_version}...')
-    try:
-        result = check_output(
-                [program] + version_check_args,
-                universal_newlines=True, stderr=STDOUT)
-    except (CalledProcessError, FileNotFoundError):
-        error_missing_program(program, package, optional)
-        if not optional:
-            sys.exit(1)
-        print(f'Continuing install without {program}')
-        return
-
-    try:
-        found_version = version_scrape_regex.search(result).group(1)
-    except AttributeError:
-        error_unknown_version(program, package, optional, need_version)
-        sys.exit(1)
-
-    if found_version < need_version:
-        error_old_version(program, package, optional, need_version,
-                          found_version)
-
-    print(f'Found {program} {found_version}')
-
 
 command = next((arg for arg in sys.argv[1:] if not arg.startswith('-')), '')
-forced = ('--force' in sys.argv)
+if command.startswith('install') or command in [
+    'check',
+    'test',
+    'nosetests',
+    'easy_install',
+]:
+    forced = '--force' in sys.argv
+    if forced:
+        print("The argument --force is deprecated. Please discontinue use.")
 
 
-if not forced and command.startswith('install') or \
-        command in ['check', 'test', 'nosetests', 'easy_install']:
-    check_external_program(
-        program='tesseract',
-        need_version='4.0.0',  # using backport for Travis CI
-        package={'darwin': 'tesseract', 'linux': 'tesseract-ocr'}
-    )
-    check_external_program(
-        program='gs',
-        need_version='9.15',  # limited by Travis CI / Ubuntu 14.04 backports
-        package='ghostscript'
-    )
-    check_external_program(
-        program='unpaper',
-        need_version='6.1',   # latest sane version
-        package='unpaper',
-        optional=True
-    )
-    check_external_program(
-        program='qpdf',
-        need_version='8.0.2', # test suite known to fail on 5.1.1
-        package='qpdf'
-    )
-    check_external_program(
-        program='pngquant',
-        need_version='2.0.0',
-        package='pngquant',
-        optional=True
-    )
-
 if 'upload' in sys.argv[1:]:
     print('Use twine to upload the package - setup.py upload is insecure')
     sys.exit(1)
@@ -207,6 +56,7 @@ def readme():
     with open('README.md', encoding='utf-8') as f:
         return f.read()
 
+
 setup(
     name='ocrmypdf',
     description='OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched',
@@ -234,45 +84,36 @@ setup(
         "Topic :: Scientific/Engineering :: Image Recognition",
         "Topic :: Text Processing :: Indexing",
         "Topic :: Text Processing :: Linguistic",
-        ],
+    ],
     python_requires=' >= 3.6',
-    setup_requires=[
-        'cffi >= 1.9.1',        # to build the leptonica module
-        'pytest-runner',        # to enable python setup.py test
-        'setuptools_scm',       # so that version will work
-        'setuptools_scm_git_archive'    # enable version from github tarballs
+    setup_requires=[  # can be removed whenever we can drop pip 9 support
+        'cffi >= 1.9.1',  # to build the leptonica module
+        'pytest-runner',  # to enable python setup.py test
+        'setuptools_scm',  # so that version will work
+        'setuptools_scm_git_archive',  # enable version from github tarballs
     ],
     use_scm_version={'version_scheme': 'post-release'},
-    cffi_modules=[
-        'src/ocrmypdf/lib/compile_leptonica.py:ffibuilder'
-    ],
+    cffi_modules=['src/ocrmypdf/lib/compile_leptonica.py:ffibuilder'],
     install_requires=[
         'chardet >= 3.0.4, < 4',  # unlisted requirement of pdfminer.six 20181108
-        'cffi >= 1.9.1',          # must be a setup and install requirement
-        'img2pdf >= 0.3.0, < 0.4',       # pure Python, so track HEAD closely
-        'pdfminer.six == 20181108 ; sys_platform != "darwin"',
-        'pikepdf >= 1.0.2, < 2',
+        'cffi >= 1.9.1',  # must be a setup and install requirement
+        'img2pdf >= 0.3.0, < 0.4',  # pure Python, so track HEAD closely
+        'pdfminer.six == 20181108',
+        'pikepdf >= 1.5.0, < 2',
         'Pillow >= 4.0.0, != 5.1.0 ; sys_platform == "darwin"',
-                                  # Pillow < 4 has BytesIO/TIFF bug w/img2pdf 0.2.3
-                                  # block 5.1.0, broken wheels
-        'reportlab >= 3.3.0',     # oldest released version with sane image handling
+        # Pillow < 4 has BytesIO/TIFF bug w/img2pdf 0.2.3
+        # block 5.1.0, broken wheels
+        'reportlab >= 3.3.0',  # oldest released version with sane image handling
         'ruffus >= 2.7.0',
     ],
-    extras_require={
-        'pdfminer': ['pdfminer.six == 20181108'],
-    },
     tests_require=tests_require,
-    entry_points={
-        'console_scripts': [
-            'ocrmypdf = ocrmypdf.__main__:run_pipeline'
-        ],
-    },
+    entry_points={'console_scripts': ['ocrmypdf = ocrmypdf.__main__:run_pipeline']},
     package_data={'ocrmypdf': ['data/sRGB.icc']},
     include_package_data=True,
     zip_safe=False,
     project_urls={
         'Documentation': 'https://ocrmypdf.readthedocs.io/',
         'Source': 'https://github.com/jbarlow83/ocrmypdf',
-        'Tracker': 'https://github.com/jbarlow83/ocrmypdf/issues'
-    }
+        'Tracker': 'https://github.com/jbarlow83/ocrmypdf/issues',
+    },
 )

src/ocrmypdf/__main__.py

@@ -44,7 +44,15 @@ from .exceptions import (
     MissingDependencyError,
     OutputFileAccessError,
 )
-from .exec import ghostscript, qpdf, tesseract
+from .exec import (
+    ghostscript,
+    jbig2enc,
+    qpdf,
+    tesseract,
+    check_external_program,
+    unpaper,
+    pngquant,
+)
 from .helpers import available_cpu_count, is_file_writable, re_symlink
 from .pdfa import file_claims_pdfa
 
@@ -58,22 +66,11 @@ def complain(message):
     print(*textwrap.wrap(message), file=sys.stderr)
 
 
-# Hack to help debugger context find /usr/local/bin
-if 'IDE_PROJECT_ROOTS' in os.environ:
-    os.environ['PATH'] = '/usr/local/bin:' + os.environ['PATH']
-
 # --------
 # Critical environment tests
 
 verify_python3_env()
 
-if not tesseract.v4:
-    complain(
-        f"Please install tesseract 4.0.0 or newer "
-        f"(currently installed version is {tesseract.version()})"
-    )
-    sys.exit(ExitCode.missing_dependency)
-
 # -------------
 # Parser
 
@@ -230,7 +227,9 @@ jobcontrol.add_argument(
     default=[],
     nargs='?',
     action="append",
-    help="Print more verbose messages for each additional verbose level",
+    help="Print more verbose messages for each additional verbose level. Use "
+    "`-v 1` typically for much more detailed logging. Higher numbers "
+    "are probably only useful in debugging.",
 )
 
 metadata = parser.add_argument_group(
@@ -278,6 +277,13 @@ preprocessing.add_argument(
     help="Clean page as above, and incorporate the cleaned image in the final "
     "PDF.  Might remove desired content.",
 )
+preprocessing.add_argument(
+    '--unpaper-args',
+    type=str,
+    default=None,
+    help="A quoted string of arguments to pass to unpaper. Requires --clean. "
+    "Example: --unpaper-args '--layout double'.",
+)
 preprocessing.add_argument(
     '--oversample',
     metavar='DPI',
@@ -595,40 +601,27 @@ def check_options_sidecar(options, log):
         options.sidecar = options.output_file + '.txt'
 
 
-def _optional_program_required(name, version_fn, min_version, for_argument):
-    try:
-        if version_fn() < min_version:
-            raise MissingDependencyError(
-                f"The installed '{name}' is not supported. "
-                f"Install version {min_version} or newer."
-            )
-    except (FileNotFoundError, MissingDependencyError):
-        raise MissingDependencyError(
-            f"Install the '{name}' program to use {for_argument}."
-        )
-
-
-def _optional_program_recommended(name, version_fn, min_version, for_argument):
-    try:
-        if version_fn() < min_version:
-            raise MissingDependencyError(
-                f"The installed '{name}' is not supported. "
-                f"Install version {min_version} or newer."
-            )
-    except (FileNotFoundError, MissingDependencyError):
-        complain(
-            f"For best results, install the optional program '{name}' to use the "
-            f"argument {for_argument}."
-        )
-
-
 def check_options_preprocessing(options, log):
-    if any((options.clean, options.clean_final)):
-        from .exec import unpaper
-
-        _optional_program_required(
-            'unpaper', unpaper.version, '6.1', '--clean, --clean-final'
+    if options.clean_final:
+        options.clean = True
+    if options.unpaper_args and not options.clean:
+        raise argparse.ArgumentError(None, "--clean is required for --unpaper-args")
+    if options.clean:
+        check_external_program(
+            log=log,
+            program='unpaper',
+            package='unpaper',
+            version_checker=unpaper.version,
+            need_version='6.1',
+            required_for=['--clean, --clean-final'],
         )
+        try:
+            if options.unpaper_args:
+                options.unpaper_args = unpaper.validate_custom_args(
+                    options.unpaper_args
+                )
+        except Exception as e:
+            raise argparse.ArgumentError(None, str(e))
 
 
 def check_options_ocr_behavior(options, log):
@@ -646,19 +639,26 @@ def check_options_ocr_behavior(options, log):
 
 def check_options_optimizing(options, log):
     if options.optimize >= 2:
-        from .exec import pngquant, jbig2enc
-
-        _optional_program_required(
-            'pngquant', pngquant.version, '2.0.1', '--optimize {2,3}'
+        check_external_program(
+            log=log,
+            program='pngquant',
+            package='pngquant',
+            version_checker=pngquant.version,
+            need_version='2.0.1',
+            required_for='--optimize {2,3}',
         )
 
-    if options.jbig2_lossy:
-        _optional_program_required('jbig2', jbig2enc.version, '0.28', '--jbig2-lossy')
-    elif options.optimize >= 2:
+    if options.optimize >= 2:
         # Although we use JBIG2 for optimize=1, don't nag about it unless the
         # user is asking for more optimization
-        _optional_program_recommended(
-            'jbig2', jbig2enc.version, '0.28', '--optimize {2,3}'
+        check_external_program(
+            log=log,
+            program='jbig2',
+            package='jbig2enc',
+            version_checker=jbig2enc.version,
+            need_version='0.28',
+            required_for='--optimize {2,3} | --jbig2-lossy',
+            recommended=True if not options.jbig2_lossy else False,
         )
 
     if options.optimize == 0 and any(
@@ -910,9 +910,6 @@ def log_page_orientations(pdfinfo, _log):
 
 def preamble(_log):
     _log.debug('ocrmypdf ' + VERSION)
-    _log.debug('tesseract ' + tesseract.version())
-    _log.debug('qpdf ' + qpdf.version())
-    _log.debug('gs ' + ghostscript.version())
 
 
 def check_environ(options, _log):
@@ -1012,6 +1009,37 @@ def report_output_file_size(options, _log, input_file, output_file):
     )
 
 
+def check_dependency_versions(options, log):
+    check_external_program(
+        log=log,
+        program='tesseract',
+        package={'darwin': 'tesseract', 'linux': 'tesseract-ocr'},
+        version_checker=tesseract.version,
+        need_version='4.0.0',  # using backport for Travis CI
+    )
+    check_external_program(
+        log=log,
+        program='gs',
+        package='ghostscript',
+        version_checker=ghostscript.version,
+        need_version='9.15',  # limited by Travis CI / Ubuntu 14.04 backports
+    )
+    if ghostscript.version() == '9.24':
+        complain(
+            "Ghostscript 9.24 contains serious regressions and is not "
+            "supported. Please upgrade to Ghostscript 9.25 or use an older "
+            "version."
+        )
+        return ExitCode.missing_dependency
+    check_external_program(
+        log=log,
+        program='qpdf',
+        package='qpdf',
+        version_checker=qpdf.version,
+        need_version='8.0.2',
+    )
+
+
 def run_pipeline(args=None):
     options = parser.parse_args(args=args)
     options.verbose_abbreviated_path = 1
@@ -1028,24 +1056,7 @@ def run_pipeline(args=None):
     )
     preamble(_log)
     check_options(options, _log)
-
-    # Complain about qpdf version < 7.0.0
-    # Suppress the warning if in the test suite, since there are no PPAs
-    # for qpdf 7.0.0 for Ubuntu trusty (i.e. Travis)
-    if qpdf.version() < '7.0.0' and not os.environ.get('PYTEST_CURRENT_TEST'):
-        complain(
-            f"You are using qpdf version {qpdf.version()} which has known issues including "
-            f"security vulnerabilities with certain malformed PDFs. Consider "
-            f"upgrading to version 7.0.0 or newer."
-        )
-
-    if ghostscript.version() == '9.24':
-        complain(
-            "Ghostscript 9.24 contains serious regressions and is not "
-            "supported. Please upgrade to Ghostscript 9.25 or use an older "
-            "version."
-        )
-        return ExitCode.missing_dependency
+    check_dependency_versions(options, _log)
 
     # Any changes to options will not take effect for options that are already
     # bound to function parameters in the pipeline. (For example
@@ -1081,6 +1092,8 @@ def run_pipeline(args=None):
 
         build_pipeline(options, work_folder, _log, context)
         atexit.register(cleanup_working_files, work_folder, options)
+        if hasattr(os, 'nice'):
+            os.nice(5)
         cmdline.run(options)
     except ruffus_exceptions.RethrownJobError as e:
         if options.verbose:

src/ocrmypdf/_pipeline.py

@@ -566,7 +566,7 @@ def preprocess_clean(input_file, output_file, log, context):
     pageinfo = get_pageinfo(input_file, context)
     dpi = get_page_square_dpi(pageinfo, options)
 
-    unpaper.clean(input_file, output_file, dpi, log)
+    unpaper.clean(input_file, output_file, dpi, log, options.unpaper_args)
 
 
 def select_ocr_image(infiles, output_file, log, context):
@@ -648,7 +648,7 @@ def ocr_tesseract_hocr(input_file, output_files, log, context):
 
 
 def select_visible_page_image(infiles, output_file, log, context):
-    "Selects a whole page image that we can show the user (if necessary)"
+    """Selects a whole page image that we can show the user (if necessary)"""
 
     options = context.get_options()
     if options.clean_final:
@@ -816,16 +816,17 @@ def convert_to_pdfa(input_files_groups, output_file, log, context):
     # NULs in DocumentInfo seem to be common since older Acrobats included them.
     # pikepdf can deal with this, but we make the world a better place by
     # stamping them out as soon as possible.
-    pdf_layers_file = pikepdf.open(layers_file)
-    if pdf_layers_file.docinfo:
-        modified = False
-        for k, v in pdf_layers_file.docinfo.items():
-            if b'\x00' in bytes(v):
-                pdf_layers_file.docinfo[k] = bytes(v).replace(b'\x00', b'')
-                modified = True
+    modified = False
+    with pikepdf.open(layers_file) as pdf_layers_file:
+        if pdf_layers_file.docinfo:
+            for k, v in pdf_layers_file.docinfo.items():
+                if b'\x00' in bytes(v):
+                    pdf_layers_file.docinfo[k] = bytes(v).replace(b'\x00', b'')
+                    modified = True
         if modified:
-            pdf_layers_file.save(layers_file)
-    del pdf_layers_file
+            pdf_layers_file.save(layers_file + '_')
+    if modified:
+        os.replace(layers_file + '_', layers_file)
 
     ps = next((ii for ii in input_files if ii.endswith('.ps')), None)
     ghostscript.generate_pdfa(
@@ -880,6 +881,8 @@ def metadata_fixup(input_files_groups, output_file, log, context):
         compress_streams=True,
         object_stream_mode=pikepdf.ObjectStreamMode.generate,
     )
+    original.close()
+    pdf.close()
 
 
 def optimize_pdf(input_file, output_file, log, context):

src/ocrmypdf/_unicodefun.py

@@ -58,9 +58,9 @@ def verify_python3_env():  # pragma: no cover
     if os.name == 'posix':
         import subprocess
 
-        rv = subprocess.Popen(
+        rv = subprocess.run(
             ['locale', '-a'], stdout=subprocess.PIPE, stderr=subprocess.PIPE
-        ).communicate()[0]
+        ).stdout
         good_locales = set()
         has_c_utf8 = False
 

src/ocrmypdf/_weave.py

@@ -15,8 +15,10 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
+from contextlib import suppress
 from itertools import groupby
 from pathlib import Path
+import os
 
 import pikepdf
 
@@ -24,8 +26,11 @@ from .exec import tesseract
 from .helpers import flatten_groups, page_number
 
 
+MAX_REPLACE_PAGES = int(os.environ.get('_OCRMYPDF_MAX_REPLACE_PAGES', 100))
+
+
 def _update_page_resources(*, page, font, font_key, procset):
-    "Update this page's fonts with a reference to the Glyphless font"
+    """Update this page's fonts with a reference to the Glyphless font"""
 
     if '/Resources' not in page:
         page['/Resources'] = pikepdf.Dictionary({})
@@ -34,7 +39,7 @@ def _update_page_resources(*, page, font, font_key, procset):
         fonts = resources['/Font']
     except KeyError:
         fonts = pikepdf.Dictionary({})
-    if font_key not in fonts:
+    if font_key is not None and font_key not in fonts:
         fonts[font_key] = font
     resources['/Font'] = fonts
 
@@ -157,119 +162,32 @@ def _weave_layers_graft(
     _update_page_resources(
         page=base_page, font=font, font_key=font_key, procset=procset
     )
+    pdf_text.close()
 
 
 def _find_font(text, pdf_base):
-    "Copy a font from the filename text into pdf_base"
+    """Copy a font from the filename text into pdf_base"""
 
     font, font_key = None, None
     possible_font_names = ('/f-0-0', '/F1')
     try:
-        pdf_text = pikepdf.open(text)
-        pdf_text_fonts = pdf_text.pages[0].Resources.get('/Font', {})
-    except Exception:
+        with pikepdf.open(text) as pdf_text:
+            try:
+                pdf_text_fonts = pdf_text.pages[0].Resources.get('/Font', {})
+            except (AttributeError, IndexError, KeyError):
+                return None, None
+            for f in possible_font_names:
+                pdf_text_font = pdf_text_fonts.get(f, None)
+                if pdf_text_font is not None:
+                    font_key = f
+                    break
+            if pdf_text_font:
+                font = pdf_base.copy_foreign(pdf_text_font)
+            return font, font_key
+    except (FileNotFoundError, pikepdf.PdfError):
+        # PdfError occurs if a 0-length file is written e.g. due to OCR timeout
         return None, None
 
-    for f in possible_font_names:
-        pdf_text_font = pdf_text_fonts.get(f, None)
-        if pdf_text_font is not None:
-            font_key = f
-            break
-    if pdf_text_font:
-        font = pdf_base.copy_foreign(pdf_text_font)
-    return font, font_key
-
-
-def _traverse_toc(pdf_base, visitor_fn, log):
-    """
-    Walk the table of contents, calling visitor_fn() at each node
-
-    The /Outlines data structure is a messy data structure, but rather than
-    navigating hierarchically we just track unique nodes.  Enqueue nodes when
-    we find them, and never visit them again.  set() is awesome.  We look for
-    the two types of object in the table of contents that can be page bookmarks
-    and update the page entry.
-
-    """
-
-    visited = set()
-    queue = set()
-    link_keys = ('/Parent', '/First', '/Last', '/Prev', '/Next')
-
-    if not '/Outlines' in pdf_base.root:
-        return
-
-    queue.add(pdf_base.root.Outlines.objgen)
-    while queue:
-        objgen = queue.pop()
-        visited.add(objgen)
-        node = pdf_base.get_object(objgen)
-        log.debug('fix toc: exploring outline entries at %r', objgen)
-
-        # Enumerate other nodes we could visit from here
-        for key in link_keys:
-            if key not in node:
-                continue
-            item = node[key]
-            if not item.is_indirect:
-                # Direct references are not allowed here, but it's not clear
-                # what we should do if we find any. Removing them is an option:
-                # node[key] = pdf_base.make_indirect(None)
-                continue
-            objgen = item.objgen
-            if objgen not in visited:
-                queue.add(objgen)
-
-        if visitor_fn:
-            visitor_fn(pdf_base, node, log)
-
-
-def _fix_toc(pdf_base, pageref_remap, log):
-    """Repair the table of contents
-
-    Whenever we replace a page wholesale, it gets assigned a new objgen number
-    and other references to it within the PDF become invalid, most notably in
-    the table of contents (/Outlines in PDF-speak).  In weave_layers we collect
-    pageref_remap, a mapping that describes the new objgen number given an old
-    one.  (objgen is a tuple, and the gen is almost always zero.)
-
-    It may ultimately be better to find a way to rebuild a page in place.
-
-    """
-
-    if not pageref_remap:
-        return
-
-    def remap_dest(dest_node):
-        """
-        Inner helper function: change the objgen for any page from the old we
-        invalidated to its new one.
-        """
-        if not isinstance(dest_node, pikepdf.Array):
-            return
-        pageref = dest_node[0]
-        if pageref['/Type'] == '/Page' and pageref.objgen in pageref_remap:
-            new_objgen = pageref_remap[pageref.objgen]
-            dest_node[0] = pdf_base.get_object(new_objgen)
-
-    def visit_remap_dest(pdf_base, node, log):
-        """
-        Visitor function to fix ToC entries
-
-        Test for the two types of references to pages that can occur in ToCs.
-        Both types have the same final format (an indirect reference to the
-        target page).
-        """
-        if '/Dest' in node:
-            # /Dest reference to another page (old method)
-            remap_dest(node['/Dest'])
-        elif '/A' in node:
-            # /A (action) command set to "GoTo" (newer method)
-            if '/S' in node['/A'] and node['/A']['/S'] == '/GoTo':
-                remap_dest(node['/A']['/D'])
-
-    _traverse_toc(pdf_base, visit_remap_dest, log)
-
 
 def weave_layers(infiles, output_file, log, context):
     """Apply text layer and/or image layer changes to baseline file
@@ -311,21 +229,16 @@ def weave_layers(infiles, output_file, log, context):
     base = list(basegroup)[0]
     path_base = Path(base).resolve()
     pdf_base = pikepdf.open(path_base)
-    keep_open = []
     font, font_key, procset = None, None, None
     pdfinfo = context.get_pdfinfo()
-    pagerefs = {}
-
-    # Walk the table of contents first, to trigger pikepdf/qpdf to resolve all
-    # page references in the table of contents. Some PDF generators put invalid
-    # references in the ToC, so we want to resolve them to null before we
-    # create any references, or the ToC will be corrupted
-    _traverse_toc(pdf_base, None, log)
 
     procset = pdf_base.make_indirect(
         pikepdf.Object.parse(b'[ /PDF /Text /ImageB /ImageC /ImageI ]')
     )
 
+    emplacements = 1
+    interim_count = 0
+
     # Iterate rest
     for page_num, layers in groups:
         layers = list(layers)
@@ -338,30 +251,25 @@ def weave_layers(infiles, output_file, log, context):
         if text and not font:
             font, font_key = _find_font(text, pdf_base)
 
-        replacing = False
+        emplaced_page = False
         content_rotation = pdfinfo[page_num - 1].rotation
 
         path_image = Path(image).resolve() if image else None
         if path_image is not None and path_image != path_base:
-            # We are replacing the old page with a rasterized PDF of the new
-            # page
-            log.debug("Replace")
-            old_objgen = pdf_base.pages[page_num - 1].objgen
-
-            pdf_image = pikepdf.open(image)
-            keep_open.append(pdf_image)
-            image_page = pdf_image.pages[0]
-            pdf_base.pages[page_num - 1] = image_page
-
-            # We're adding a new page, which will get a new objgen number pair,
-            # so we need to update any references to it.  qpdf did not like
-            # my attempt to update the old object in place, but that is an
-            # option to consider
-            pagerefs[old_objgen] = pdf_base.pages[page_num - 1].objgen
-            replacing = True
+            # We are updating the old page with a rasterized PDF of the new
+            # page (without changing objgen, to preserve references)
+            log.debug("Emplacement update")
+            with pikepdf.open(image) as pdf_image:
+                emplacements += 1
+                foreign_image_page = pdf_image.pages[0]
+                pdf_base.pages.append(foreign_image_page)
+                local_image_page = pdf_base.pages[-1]
+                pdf_base.pages[page_num - 1].emplace(local_image_page)
+                del pdf_base.pages[-1]
+            emplaced_page = True
 
         autorotate_correction = context.get_rotation(page_num - 1)
-        if replacing:
+        if emplaced_page:
             content_rotation = autorotate_correction
         text_rotation = autorotate_correction
         text_misaligned = (text_rotation - content_rotation) % 360
@@ -390,24 +298,35 @@ def weave_layers(infiles, output_file, log, context):
             content_rotation - autorotate_correction
         ) % 360
 
-        if len(keep_open) > 100:
-            # qpdf limitations require us to keep files open when we intend
-            # to copy content from them before saving. However, we want to keep
-            # a lid on file handles and memory usage, so for big files we're
-            # going to stop and save periodically. Attach the font to page 1
-            # even if page 1 doesn't use it, so we have a way to get it back.
+        if emplacements % MAX_REPLACE_PAGES == 0:
+            # Periodically save and reload the Pdf object. This will keep a
+            # lid on our memory usage for very large files. Attach the font to
+            # page 1 even if page 1 doesn't use it, so we have a way to get it
+            # back.
+            # TODO refactor this to outside the loop
             page0 = pdf_base.pages[0]
             _update_page_resources(
                 page=page0, font=font, font_key=font_key, procset=procset
             )
-            interim = output_file + f'_working{page_num}.pdf'
-            pdf_base.save(interim)
-            del pdf_base
-            keep_open = []
 
-            pdf_base = pikepdf.open(interim)
+            # We cannot read and write the same file, that will corrupt it
+            # but we don't to keep more copies than we need to. Delete intermediates.
+            # {interim_count} is the opened file we were updateing
+            # {interim_count - 1} can be deleted
+            # {interim_count + 1} is the new file will produce and open
+            old_file = output_file + f'_working{interim_count - 1}.pdf'
+            if not context.get_options().keep_temporary_files:
+                with suppress(FileNotFoundError):
+                    os.unlink(old_file)
+
+            next_file = output_file + f'_working{interim_count + 1}.pdf'
+            pdf_base.save(next_file)
+            pdf_base.close()
+
+            pdf_base = pikepdf.open(next_file)
             procset = pdf_base.pages[0].Resources.ProcSet
-            font = pdf_base.pages[0].Resources.Font.get(font_key)
+            font, font_key = None, None  # Ensure we reacquire this information
+            interim_count += 1
 
-    _fix_toc(pdf_base, pagerefs, log)
     pdf_base.save(output_file)
+    pdf_base.close()

src/ocrmypdf/exec/__init__.py

@@ -21,7 +21,8 @@ import os
 import re
 import sys
 from subprocess import run, STDOUT, PIPE, CalledProcessError
-from ..exceptions import MissingDependencyError
+from ..exceptions import MissingDependencyError, ExitCode
+from collections.abc import Mapping
 
 
 def get_version(program, *, version_arg='--version', regex=r'(\d+(\.\d+)*)'):
@@ -42,7 +43,7 @@ def get_version(program, *, version_arg='--version', regex=r'(\d+(\.\d+)*)'):
             f"Could not find program '{program}' on the PATH"
         ) from e
     except CalledProcessError as e:
-        if e.returncode < 0:
+        if e.returncode != 0:
             raise MissingDependencyError(
                 f"Ran program '{program}' but it exited with an error:\n{e.output}"
             ) from e
@@ -58,3 +59,115 @@ def get_version(program, *, version_arg='--version', regex=r'(\d+(\.\d+)*)'):
         )
 
     return version
+
+
+missing_program = '''
+The program '{program}' could not be executed or was not found on your
+system PATH.
+'''
+
+missing_optional_program = '''
+The program '{program}' could not be executed or was not found on your
+system PATH.  This program is required when you use the
+{required_for} arguments.  You could try omitting these arguments, or install
+the package.
+'''
+
+missing_recommend_program = '''
+The program '{program}' could not be executed or was not found on your
+system PATH.  This program is recommended when using the {required_for} arguments,
+but not required, so we will proceed.  For best results, install the program.
+'''
+
+old_version = '''
+OCRmyPDF requires '{program}' {need_version} or higher.  Your system appears
+to have {found_version}.  Please update this program.
+'''
+
+old_version_required_for = '''
+OCRmyPDF requires '{program}' {need_version} or higher when run with the
+{required_for} arguments.  If you omit these arguments, OCRmyPDF may be able to
+proceed.  For best results, install the program.
+'''
+
+osx_install_advice = '''
+If you have homebrew installed, try these command to install the missing
+package:
+    brew install {package}
+'''
+
+linux_install_advice = '''
+On systems with the aptitude package manager (Debian, Ubuntu), try these
+commands:
+    sudo apt-get update
+    sudo apt-get install {package}
+
+On RPM-based systems (Red Hat, Fedora), search for instructions on
+installing the RPM for {program}.
+'''
+
+
+def _get_platform():
+    if sys.platform.startswith('freebsd'):
+        return 'freebsd'
+    elif sys.platform.startswith('linux'):
+        return 'linux'
+    return sys.platform
+
+
+def _error_trailer(log, program, package, **kwargs):
+    if isinstance(package, Mapping):
+        package = package[_get_platform()]
+
+    if _get_platform() == 'darwin':
+        log.info(osx_install_advice.format(**locals()))
+    elif _get_platform() == 'linux':
+        log.info(linux_install_advice.format(**locals()))
+
+
+def _error_missing_program(log, program, package, required_for, recommended):
+    if required_for:
+        log.error(missing_optional_program.format(**locals()))
+    elif recommended:
+        log.info(missing_recommend_program.format(**locals()))
+    else:
+        log.error(missing_program.format(**locals()))
+    _error_trailer(**locals())
+
+
+def _error_old_version(
+    log, program, package, need_version, found_version, required_for
+):
+    if required_for:
+        log.error(old_version_required_for.format(**locals()))
+    else:
+        log.error(old_version.format(**locals()))
+    _error_trailer(**locals())
+
+
+def check_external_program(
+    *,
+    log,
+    program,
+    package,
+    version_checker,
+    need_version,
+    required_for=None,
+    recommended=False,
+):
+    try:
+        found_version = version_checker()
+    except (CalledProcessError, FileNotFoundError, MissingDependencyError):
+        _error_missing_program(log, program, package, required_for, recommended)
+        if not recommended:
+            sys.exit(ExitCode.missing_dependency)
+        return
+
+    if found_version < need_version:
+        _error_old_version(
+            log, program, package, need_version, found_version, required_for
+        )
+        if not recommended:
+            sys.exit(ExitCode.missing_dependency)
+
+    log.debug(f'Found {program} {found_version}')

src/ocrmypdf/exec/ghostscript.py

@@ -129,8 +129,7 @@ def rasterize_pdf(
     :param filter_vector: if True, remove vector graphics objects
     :return:
     """
-    res = xres, yres
-    int_res = round(xres), round(yres)
+    res = round(xres, 6), round(yres, 6)
     if not page_dpi:
         page_dpi = res
 
@@ -145,7 +144,7 @@ def rasterize_pdf(
                 f'-sDEVICE={raster_device}',
                 f'-dFirstPage={pageno}',
                 f'-dLastPage={pageno}',
-                f'-r{str(int_res[0])}x{str(int_res[1])}',
+                f'-r{res[0]:f}x{res[1]:f}',
             ]
             + (['-dFILTERVECTOR'] if filter_vector else [])
             + [
@@ -168,23 +167,8 @@ def rasterize_pdf(
             log.error('Ghostscript rasterizing failed')
             raise SubprocessOutputError()
 
-        # Ghostscript only accepts integers for output resolution
-        # if the resolution happens to be fractional, then the discrepancy
-        # would change the size of the output page, especially if the DPI
-        # is quite low. Resize the image to the expected size
-
         tmp.seek(0)
         with Image.open(tmp) as im:
-            expected_size = (
-                round(im.size[0] / int_res[0] * res[0]),
-                round(im.size[1] / int_res[1] * res[1]),
-            )
-            if expected_size != im.size or page_dpi != (xres, yres):
-                log.debug(
-                    f"Ghostscript: resize output image {im.size} -> {expected_size}"
-                )
-                im = im.resize(expected_size)
-
             if rotation is not None:
                 log.debug("Rotating output by %i", rotation)
                 # rotation is a clockwise angle and Image.ROTATE_* is
@@ -269,7 +253,6 @@ def generate_pdfa(
                 "-dBATCH",
                 "-dNOPAUSE",
                 "-dCompatibilityLevel=" + str(pdf_version),
-                "-dNumRenderingThreads=" + str(threads),
                 "-sDEVICE=pdfwrite",
                 "-dAutoRotatePages=/None",
                 "-sColorConversionStrategy=" + strategy,

src/ocrmypdf/exec/pngquant.py

@@ -17,6 +17,9 @@
 
 from functools import lru_cache
 from subprocess import run
+from tempfile import NamedTemporaryFile
+
+from PIL import Image
 
 from . import get_version
 from ..exceptions import MissingDependencyError
@@ -36,16 +39,32 @@ def available():
 
 
 def quantize(input_file, output_file, quality_min, quality_max):
-    args = [
-        'pngquant',
-        '--force',
-        '--skip-if-larger',
-        '--output',
-        output_file,
-        '--quality',
-        f'{quality_min}-{quality_max}',
-        '--',
-        input_file,
-    ]
-    proc = run(args)
-    proc.check_returncode()
+    if input_file.endswith('.jpg'):
+        im = Image.open(input_file)
+        with NamedTemporaryFile(suffix='.png') as tmp:
+            im.save(tmp)
+            args = [
+                'pngquant',
+                '--force',
+                '--skip-if-larger',
+                '--output',
+                output_file,
+                '--quality',
+                f'{quality_min}-{quality_max}',
+                '--',
+                tmp.name,
+            ]
+            run(args)
+    else:
+        args = [
+            'pngquant',
+            '--force',
+            '--skip-if-larger',
+            '--output',
+            output_file,
+            '--quality',
+            f'{quality_min}-{quality_max}',
+            '--',
+            input_file,
+        ]
+        run(args)

src/ocrmypdf/exec/tesseract.py

@@ -200,7 +200,9 @@ def tesseract_log_output(log, stdout, input_file):
             log.info(prefix + line.strip())
 
 
-def page_timedout(log, input_file):
+def page_timedout(log, input_file, timeout):
+    if timeout == 0:
+        return
     prefix = f"{(page_number(input_file)):4d}: [tesseract] "
     log.warning(prefix + " took too long to OCR - skipping")
 
@@ -257,7 +259,7 @@ def generate_hocr(
         # Generate a HOCR file with no recognized text if tesseract times out
         # Temporary workaround to hocrTransform not being able to function if
         # it does not have a valid hOCR file.
-        page_timedout(log, input_file)
+        page_timedout(log, input_file, timeout)
         _generate_null_hocr(output_hocr, output_sidecar, input_file)
     except CalledProcessError as e:
         tesseract_log_output(log, e.output, input_file)
@@ -347,7 +349,7 @@ def generate_pdf(
         if os.path.exists(prefix + '.txt'):
             shutil.move(prefix + '.txt', output_text)
     except TimeoutExpired:
-        page_timedout(log, input_image)
+        page_timedout(log, input_image, timeout)
         use_skip_page(text_only, skip_pdf, output_pdf, output_text)
     except CalledProcessError as e:
         tesseract_log_output(log, e.output, input_image)

src/ocrmypdf/exec/unpaper.py

@@ -19,13 +19,15 @@
 # https://github.com/Flameeyes/unpaper/blob/master/doc/basic-concepts.md
 
 import os
+import shlex
+import subprocess
 import sys
 from functools import lru_cache
-from subprocess import STDOUT, CalledProcessError, check_output
-from tempfile import NamedTemporaryFile
+from subprocess import PIPE, STDOUT, CalledProcessError
+from tempfile import TemporaryDirectory
 
 from . import get_version
-from ..exceptions import MissingDependencyError
+from ..exceptions import MissingDependencyError, SubprocessOutputError
 
 try:
     from PIL import Image
@@ -64,43 +66,64 @@ def run(input_file, output_file, dpi, log, mode_args):
         im.close()
         raise MissingDependencyError() from e
 
-    with NamedTemporaryFile(suffix=suffix) as input_pnm, NamedTemporaryFile(
-        suffix=suffix, mode="r+b"
-    ) as output_pnm:
+    with TemporaryDirectory() as tmpdir:
+        input_pnm = os.path.join(tmpdir, f'input{suffix}')
+        output_pnm = os.path.join(tmpdir, f'output{suffix}')
         im.save(input_pnm, format='PPM')
         im.close()
 
-        os.unlink(output_pnm.name)
-
-        args_unpaper.extend([input_pnm.name, output_pnm.name])
+        # To prevent any shenanigans from accepting arbitrary parameters in
+        # --unpaper-args, we:
+        # 1) run with cwd set to a tmpdir with only unpaper's files
+        # 2) forbid the use of '/' in arguments, to prevent changing paths
+        # 3) append absolute paths for the input and output file
+        # This should ensure that a user cannot clobber some other file with
+        # their unpaper arguments (whether intentionally or otherwise)
+        args_unpaper.extend([input_pnm, output_pnm])
         try:
-            stdout = check_output(
-                args_unpaper, close_fds=True, universal_newlines=True, stderr=STDOUT
+            proc = subprocess.run(
+                args_unpaper,
+                check=True,
+                close_fds=True,
+                universal_newlines=True,
+                stderr=STDOUT,
+                cwd=tmpdir,
+                stdout=PIPE,
             )
         except CalledProcessError as e:
             log.debug(e.output)
             raise e from e
         else:
-            log.debug(stdout)
-            # unpaper sets dpi to 72
-            Image.open(output_pnm.name).save(output_file, dpi=(dpi, dpi))
+            log.debug(proc.stdout)
+            # unpaper sets dpi to 72; fix this
+            try:
+                Image.open(output_pnm).save(output_file, dpi=(dpi, dpi))
+            except (FileNotFoundError, OSError):
+                raise SubprocessOutputError(
+                    "unpaper: failed to produce the expected output file. Called with: "
+                    + str(args_unpaper)
+                ) from None
 
 
-def clean(input_file, output_file, dpi, log):
-    run(
-        input_file,
-        output_file,
-        dpi,
-        log,
-        [
-            '--layout',
-            'none',
-            '--mask-scan-size',
-            '100',  # don't blank out narrow columns
-            '--no-border-align',  # don't align visible content to borders
-            '--no-mask-center',  # don't center visible content within page
-            '--no-grayfilter',  # don't remove light gray areas
-            '--no-blackfilter',  # don't remove solid black areas
-            '--no-deskew',  # don't deskew
-        ],
-    )
+def validate_custom_args(args: str):
+    unpaper_args = shlex.split(args)
+    if any('/' in arg for arg in unpaper_args):
+        raise ValueError('No filenames allowed in --unpaper-args')
+    return unpaper_args
+
+
+def clean(input_file, output_file, dpi, log, unpaper_args=None):
+    default_args = [
+        '--layout',
+        'none',
+        '--mask-scan-size',
+        '100',  # don't blank out narrow columns
+        '--no-border-align',  # don't align visible content to borders
+        '--no-mask-center',  # don't center visible content within page
+        '--no-grayfilter',  # don't remove light gray areas
+        '--no-blackfilter',  # don't remove solid black areas
+        '--no-deskew',  # don't deskew
+    ]
+    if not unpaper_args:
+        unpaper_args = default_args
+    run(input_file, output_file, dpi, log, unpaper_args)

src/ocrmypdf/leptonica.py

@@ -43,11 +43,6 @@ lept = ffi.dlopen(find_library('lept'))
 lept.setMsgSeverity(lept.L_SEVERITY_WARNING)
 
 
-def stderr(*objs):
-    """Shorthand print to stderr."""
-    print("leptonica.py:", *objs, file=sys.stderr)
-
-
 class _LeptonicaErrorTrap:
     """
     Context manager to trap errors reported by Leptonica.
@@ -66,6 +61,7 @@ class _LeptonicaErrorTrap:
     def __init__(self):
         self.tmpfile = None
         self.copy_of_stderr = -1
+        self.no_stderr = False
 
     def __enter__(self):
         from io import UnsupportedOperation
@@ -73,33 +69,40 @@ class _LeptonicaErrorTrap:
         self.tmpfile = TemporaryFile()
 
         # Save the old stderr, and redirect stderr to temporary file
-        sys.stderr.flush()
+        with suppress(AttributeError):
+            sys.stderr.flush()
         try:
             self.copy_of_stderr = os.dup(sys.stderr.fileno())
             os.dup2(self.tmpfile.fileno(), sys.stderr.fileno(), inheritable=False)
+        except AttributeError:
+            # We are in some unusual context where our Python process does not
+            # have a sys.stderr. Leptonica still expects to write to file
+            # descriptor 2, so we are going to ensure it is redirected.
+            self.copy_of_stderr = None
+            self.no_stderr = True
+            os.dup2(self.tmpfile.fileno(), 2, inheritable=False)
         except UnsupportedOperation:
             self.copy_of_stderr = None
         return
 
     def __exit__(self, exc_type, exc_value, traceback):
         # Restore old stderr
-        sys.stderr.flush()
+        with suppress(AttributeError):
+            sys.stderr.flush()
+
         if self.copy_of_stderr is not None:
             os.dup2(self.copy_of_stderr, sys.stderr.fileno())
             os.close(self.copy_of_stderr)
+        if self.no_stderr:
+            os.close(2)
 
-        # Get data from tmpfile (in with block to ensure it is closed)
-        with self.tmpfile as tmpfile:
-            tmpfile.seek(0)  # Cursor will be at end, so move back to beginning
-            leptonica_output = tmpfile.read().decode(errors='replace')
-
-        assert self.tmpfile.closed
-        assert not sys.stderr.closed
-
-        # If there are Python errors, let them bubble up
+        # Get data from tmpfile
+        self.tmpfile.seek(0)  # Cursor will be at end, so move back to beginning
+        leptonica_output = self.tmpfile.read().decode(errors='replace')
+        self.tmpfile.close()
+        # If there are Python errors, record them
         if exc_type:
             logger.warning(leptonica_output)
-            return False
 
         # If there are Leptonica errors, wrap them in Python excpetions
         if 'Error' in leptonica_output:
@@ -614,9 +617,10 @@ class Pix(LeptonicaObject):
         except (LeptonicaError, ValueError, IndexError):
             return
         finally:
-            with suppress(FileNotFoundError):
-                os.unlink('junkpixt.png')  # leptonica may produce this
-                os.unlink('junkpixt')
+            leptonica_junk = ('junkpixt.png', 'junkpixt')
+            for junk in leptonica_junk:
+                with suppress(FileNotFoundError):
+                    os.unlink(junk)  # leptonica may produce this
 
         for n, s in enumerate(sarray):
             decoded = s.decode()

src/ocrmypdf/optimize.py

@@ -25,6 +25,7 @@ from pathlib import Path
 from PIL import Image
 
 import pikepdf
+from pikepdf import Name, Dictionary, Array
 
 from . import leptonica
 from ._jobcontext import JobContext
@@ -52,7 +53,7 @@ def tif_name(root, xref):
 
 
 def extract_image_filter(pike, root, log, image, xref):
-    if image.Subtype != '/Image':
+    if image.Subtype != Name.Image:
         return None
     if image.Length < 100:
         log.debug("Skipping small image, xref %s", xref)
@@ -68,7 +69,7 @@ def extract_image_filter(pike, root, log, image, xref):
     if pim.bits_per_component > 8:
         return None  # Don't mess with wide gamut images
 
-    if filtdp[0] == '/JPXDecode':
+    if filtdp[0] == Name.JPXDecode:
         return None  # Don't do JPEG2000
 
     return pim, filtdp
@@ -82,7 +83,7 @@ def extract_image_jbig2(*, pike, root, log, image, xref, options):
 
     if (
         pim.bits_per_component == 1
-        and filtdp != '/JBIG2Decode'
+        and filtdp != Name.JBIG2Decode
         and jbig2enc.available()
     ):
         try:
@@ -102,7 +103,7 @@ def extract_image_generic(*, pike, root, log, image, xref, options):
         return None
     pim, filtdp = result
 
-    if filtdp[0] == '/DCTDecode' and options.optimize >= 2:
+    if filtdp[0] == Name.DCTDecode and options.optimize >= 2:
         # This is a simple heuristic derived from some training data, that has
         # about a 70% chance of guessing whether the JPEG is high quality,
         # and possibly recompressible, or not. The number itself doesn't mean
@@ -148,11 +149,22 @@ def extract_image_generic(*, pike, root, log, image, xref, options):
 def extract_images(pike, root, log, options, extract_fn):
     """Extract image using extract_fn
 
-    extract_fn decides whether the image is interesting in this case
+    Enumerate images on each page, lookup their xref/ID number in the PDF.
+    Exclude images that are soft masks (i.e. alpha transparency related).
+    Record the page number on which an image is first used, since images may be
+    used on multiple pages (or multiple times on the same page).
+
+    Current we do not check Form XObjects or other objects that may contain
+    images, and we don't evaluate alternate images or thumbnails.
+
+    extract_fn must decide if wants to extract the image in this context. If
+    it does a tuple should be returned: (xref, ext) where .ext is the file
+    extension. extract_fn must also extract the file it finds interesting.
     """
 
     include_xrefs = set()
     exclude_xrefs = set()
+    pageno_for_xref = {}
     errors = 0
     for pageno, page in enumerate(pike.pages):
         try:
@@ -168,6 +180,8 @@ def extract_images(pike, root, log, options, extract_fn):
                 smask_xref = image.SMask.objgen[0]
                 exclude_xrefs.add(smask_xref)
             include_xrefs.add(xref)
+            if xref not in pageno_for_xref:
+                pageno_for_xref[xref] = pageno
 
     working_xrefs = include_xrefs - exclude_xrefs
     for xref in working_xrefs:
@@ -177,13 +191,12 @@ def extract_images(pike, root, log, options, extract_fn):
                 pike=pike, root=root, log=log, image=image, xref=xref, options=options
             )
         except Exception as e:
-            log.debug("Image xref %s", xref)
-            log.debug(repr(e))
+            log.debug("Image xref %s, error %s", xref, repr(e))
             errors += 1
         else:
             if result:
                 _, ext = result
-                yield pageno, xref, ext
+                yield pageno_for_xref[xref], xref, ext
 
 
 def extract_images_generic(pike, root, log, options):
@@ -197,7 +210,7 @@ def extract_images_generic(pike, root, log, options):
             pngs.append(xref)
         elif ext == '.jpg':
             jpegs.append(xref)
-    log.debug("Optimizable images: " "JPEGs: %s PNGs: %s", len(jpegs), len(pngs))
+    log.debug("Optimizable images: JPEGs: %s PNGs: %s", len(jpegs), len(pngs))
     return jpegs, pngs
 
 
@@ -215,7 +228,7 @@ def extract_images_jbig2(pike, root, log, options):
     jbig2_groups = {
         group: xrefs for group, xrefs in jbig2_groups.items() if len(xrefs) > 0
     }
-    log.debug("Optimizable images: " "JBIG2 groups: %s", (len(jbig2_groups),))
+    log.debug("Optimizable images: JBIG2 groups: %s", (len(jbig2_groups),))
     return jbig2_groups
 
 
@@ -270,7 +283,7 @@ def convert_to_jbig2(pike, jbig2_groups, root, log, options):
     must be lossy encoding since jbig2enc does not support refinement coding.
 
     When the JBIG2 symbolic coder is not used, each JBIG2 stands on its own
-    and needs no dictionary. Currently this is must be lossless JBIG2.
+    and needs no dictionary. Currently this must be lossless JBIG2.
     """
 
     _produce_jbig2_images(jbig2_groups, root, log, options)
@@ -281,7 +294,7 @@ def convert_to_jbig2(pike, jbig2_groups, root, log, options):
         if jbig2_symfile.exists():
             jbig2_globals_data = jbig2_symfile.read_bytes()
             jbig2_globals = pikepdf.Stream(pike, jbig2_globals_data)
-            jbig2_globals_dict = pikepdf.Dictionary({'/JBIG2Globals': jbig2_globals})
+            jbig2_globals_dict = Dictionary(JBIG2Globals=jbig2_globals)
         elif options.jbig2_page_group_size == 1:
             jbig2_globals_dict = None
         else:
@@ -293,9 +306,7 @@ def convert_to_jbig2(pike, jbig2_groups, root, log, options):
             jbig2_im_data = jbig2_im_file.read_bytes()
             im_obj = pike.get_object(xref, 0)
             im_obj.write(
-                jbig2_im_data,
-                filter=pikepdf.Name('/JBIG2Decode'),
-                decode_parms=jbig2_globals_dict,
+                jbig2_im_data, filter=Name.JBIG2Decode, decode_parms=jbig2_globals_dict
             )
 
 
@@ -310,17 +321,17 @@ def transcode_jpegs(pike, jpegs, root, log, options):
         # https://github.com/python-pillow/Pillow/issues/1144
         with Image.open(fspath(in_jpg)) as im:
             im.save(fspath(opt_jpg), optimize=True, quality=options.jpeg_quality)
-        # pylint: disable=no-member
+
         if opt_jpg.stat().st_size > in_jpg.stat().st_size:
             log.debug("xref %s, jpeg, made larger - skip", xref)
             continue
 
         compdata = leptonica.CompressedData.open(opt_jpg)
         im_obj = pike.get_object(xref, 0)
-        im_obj.write(compdata.read(), filter=pikepdf.Name('/DCTDecode'))
+        im_obj.write(compdata.read(), filter=Name.DCTDecode)
 
 
-def transcode_pngs(pike, pngs, root, log, options):
+def transcode_pngs(pike, images, image_name_fn, root, log, options):
     if options.optimize >= 2:
         png_quality = (
             max(10, options.png_quality - 10),
@@ -329,65 +340,91 @@ def transcode_pngs(pike, pngs, root, log, options):
         with concurrent.futures.ThreadPoolExecutor(
             max_workers=options.jobs
         ) as executor:
-            for xref in pngs:
+            for xref in images:
+                log.debug(image_name_fn(root, xref))
                 executor.submit(
                     pngquant.quantize,
-                    png_name(root, xref),
+                    image_name_fn(root, xref),
                     png_name(root, xref),
                     png_quality[0],
                     png_quality[1],
                 )
 
-    for xref in pngs:
+    for xref in images:
         im_obj = pike.get_object(xref, 0)
-
-        # Open, transcode (!), package for PDF
         try:
-            pix = leptonica.Pix.open(png_name(root, xref))
-            if pix.depth == 1:
-                pix = pix.invert()  # PDF assumes 1 is black for monochrome
-            compdata = pix.generate_pdf_ci_data(leptonica.lept.L_FLATE_ENCODE, 0)
+            compdata = leptonica.CompressedData.open(png_name(root, xref))
         except leptonica.LeptonicaError as e:
+            # Most likely this means file not found, i.e. quantize did not
+            # produce an improved version
             log.error(e)
             continue
 
-        # This is what we should be doing: open the compressed data without
-        # transcoding. However this shifts each pixel row by one for some
-        # reason.
-        # compdata = leptonica.CompressedData.open(png_name(root, xref))
+        # If re-coded image is larger don't use it - we test here because
+        # pngquant knows the size of the temporary output file but not the actual
+        # object in the PDF
         if len(compdata) > int(im_obj.stream_dict.Length):
-            continue  # If we produced a larger image, don't use
+            log.debug(
+                f"pngquant: pngquant did not improve over original image "
+                f"{len(compdata)} > {int(im_obj.stream_dict.Length)}"
+            )
+            continue
 
-        predictor = None
-        if compdata.predictor > 0:
-            predictor = pikepdf.Dictionary({'/Predictor': compdata.predictor})
+        # When a PNG is inserted into a PDF, we more or less copy the IDAT section from
+        # the PDF and transfer the rest of the PNG headers to PDF image metadata.
+        # One thing we have to do is tell the PDF reader whether a predictor was used
+        # on the image before Flate encoding. (Typically one is.)
+        # According to Leptonica source, PDF readers don't actually need us
+        # to specify the correct predictor, they just need a value of either:
+        #   1 - no predictor
+        #   10-14 - there is a predictor
+        # Leptonica's compdata->predictor only tells TRUE or FALSE
+        # From there the PNG decoder can infer the rest from the file.
+        # In practice the predictor should be Paeth, 14, so we'll use that.
+        # See:
+        #   - PDF RM 7.4.4.4 Table 10
+        #   - https://github.com/DanBloomberg/leptonica/blob/master/src/pdfio2.c#L757
+        predictor = 14 if compdata.predictor > 0 else 1
+        dparms = Dictionary(Predictor=predictor)
+        if predictor > 1:
+            dparms.BitsPerComponent = compdata.bps  # Yes, this is redundant
+            dparms.Colors = compdata.spp
+            dparms.Columns = compdata.w
 
         im_obj.BitsPerComponent = compdata.bps
         im_obj.Width = compdata.w
         im_obj.Height = compdata.h
 
         if compdata.ncolors > 0:
+            # .ncolors is the number of colors in the palette, not the number of
+            # colors used in a true color image
             palette_pdf_string = compdata.get_palette_pdf_string()
             palette_data = pikepdf.Object.parse(palette_pdf_string)
             palette_stream = pikepdf.Stream(pike, bytes(palette_data))
             palette = [
-                pikepdf.Name('/Indexed'),
-                pikepdf.Name('/DeviceRGB'),
+                Name.Indexed,
+                Name.DeviceRGB,
                 compdata.ncolors - 1,
                 palette_stream,
             ]
             cs = palette
         else:
             if compdata.spp == 1:
-                cs = pikepdf.Name('/DeviceGray')
+                # PDF interprets binary-1 as black in 1bpp, but PNG sets
+                # black to 0 for 1bpp. Create a palette that informs the PDF
+                # of the mapping - seems cleaner to go this way but pikepdf
+                # needs to be patched to support it.
+                # palette = [Name.Indexed, Name.DeviceGray, 1, b"\xff\x00"]
+                # cs = palette
+                cs = Name.DeviceGray
             elif compdata.spp == 3:
-                cs = pikepdf.Name('/DeviceRGB')
+                cs = Name.DeviceRGB
             elif compdata.spp == 4:
-                cs = pikepdf.Name('/DeviceCMYK')
+                cs = Name.DeviceCMYK
+        if compdata.bps == 1:
+            im_obj.Decode = [1, 0]  # Bit of a kludge but this inverts photometric too
         im_obj.ColorSpace = cs
-        im_obj.write(
-            compdata.read(), filter=pikepdf.Name('/FlateDecode'), decode_parms=predictor
-        )
+        im_obj.write(compdata.read(), filter=Name.FlateDecode, decode_parms=dparms)
 
 
 def optimize(input_file, output_file, log, context):
@@ -407,11 +444,14 @@ def optimize(input_file, output_file, log, context):
     pike = pikepdf.Pdf.open(input_file)
 
     root = Path(output_file).parent / 'images'
-    root.mkdir(exist_ok=True)  # pylint: disable=no-member
+    root.mkdir(exist_ok=True)
 
     jpegs, pngs = extract_images_generic(pike, root, log, options)
     transcode_jpegs(pike, jpegs, root, log, options)
-    transcode_pngs(pike, pngs, root, log, options)
+    # if options.optimize >= 2:
+    # Try pngifying the jpegs
+    #    transcode_pngs(pike, jpegs, jpg_name, root, log, options)
+    transcode_pngs(pike, pngs, png_name, root, log, options)
 
     jbig2_groups = extract_images_jbig2(pike, root, log, options)
     convert_to_jbig2(pike, jbig2_groups, root, log, options)

src/ocrmypdf/pdfa.py

@@ -131,19 +131,19 @@ def file_claims_pdfa(filename):
     do full PDF/A validation.
     """
 
-    pdf = pikepdf.open(filename)
-    pdfmeta = pdf.open_metadata()
-    if not pdfmeta.pdfa_status:
-        return {
-            'pass': False,
-            'output': 'pdf',
-            'conformance': 'No PDF/A metadata in XMP',
-        }
-    valid_part_conforms = {'1A', '1B', '2A', '2B', '2U', '3A', '3B', '3U'}
-    conformance = f'PDF/A-{pdfmeta.pdfa_status}'
-    pdfa_dict = {}
-    if pdfmeta.pdfa_status in valid_part_conforms:
-        pdfa_dict['pass'] = True
-        pdfa_dict['output'] = 'pdfa'
-    pdfa_dict['conformance'] = conformance
+    with pikepdf.open(filename) as pdf:
+        pdfmeta = pdf.open_metadata()
+        if not pdfmeta.pdfa_status:
+            return {
+                'pass': False,
+                'output': 'pdf',
+                'conformance': 'No PDF/A metadata in XMP',
+            }
+        valid_part_conforms = {'1A', '1B', '2A', '2B', '2U', '3A', '3B', '3U'}
+        conformance = f'PDF/A-{pdfmeta.pdfa_status}'
+        pdfa_dict = {}
+        if pdfmeta.pdfa_status in valid_part_conforms:
+            pdfa_dict['pass'] = True
+            pdfa_dict['output'] = 'pdfa'
+        pdfa_dict['conformance'] = conformance
     return pdfa_dict

src/ocrmypdf/pdfinfo/__init__.py

@@ -30,6 +30,7 @@ from pikepdf import PdfMatrix
 import pikepdf
 
 from . import ghosttext
+from .layout import get_page_analysis, get_text_boxes
 
 from ..exceptions import EncryptedPdfError, MissingDependencyError
 
@@ -391,6 +392,8 @@ def _image_xobjects(container):
     xobjs = resources['/XObject'].as_dict()
     for xobj in xobjs:
         candidate = xobjs[xobj]
+        if not '/Subtype' in candidate:
+            continue
         if candidate['/Subtype'] == '/Image':
             pdfimage = candidate
             yield (pdfimage, xobj)
@@ -562,14 +565,6 @@ def _pdf_get_pageinfo(pdf, pageno: int, infile, xmltext):
         )
         pageinfo['bboxes'] = bboxes
     else:
-        # pdfminer required for this section
-        try:
-            from .layout import get_page_analysis, get_text_boxes
-        except ImportError:
-            raise MissingDependencyError(
-                "pdfminer is required for this feature. Your distribution "
-                "may not have installed it."
-            )
         pscript5_mode = str(pdf.docinfo.get('/Creator')).startswith('PScript5')
         miner = get_page_analysis(infile, pageno, pscript5_mode)
         pageinfo['textboxes'] = list(simplify_textboxes(miner, get_text_boxes))
@@ -616,8 +611,9 @@ def _pdf_get_all_pageinfo(infile, detailed_analysis=False, log=None):
     if not log:
         log = Mock()
 
-    pdf = pikepdf.open(infile)
+    pdf = pikepdf.open(infile)  # Do not close in this function
     if pdf.is_encrypted:
+        pdf.close()
         raise EncryptedPdfError()  # Triggered by encryption with empty passwd
     if detailed_analysis:
         pages_xml = None
@@ -753,7 +749,13 @@ class PdfInfo:
             infile, detailed_page_analysis, log=log
         )
         self._needs_rendering = pdf.root.get('/NeedsRendering', False)
-        self._has_acroform = '/AcroForm' in pdf.root
+        self._has_acroform = False
+        if '/AcroForm' in pdf.root:
+            if len(pdf.root.AcroForm.get('/Fields', [])) > 0:
+                self._has_acroform = True
+            elif '/XFA' in pdf.root.AcroForm:
+                self._has_acroform = True
+        pdf.close()
 
     @property
     def pages(self):

tests/cache/francais/__-l__deu__000001.ocr.png__000001.text__pdf__txt/stderr.bin

@@ -0,0 +1 @@
+Tesseract Open Source OCR Engine v4.0.0 with Leptonica

tests/cache/francais/__-l__deu__000001.ocr.png__000001.text__pdf__txt/txt.bin

@@ -0,0 +1,13 @@
+Portez ce vieux whisky au juge
+blond qui fume sur son ile
+interieure, a cöte de l'alcöve
+ovolde, oU les büches se
+consument dans l'ätre, ce qui
+lui permet de penser ä la
+cgnogenese de l'&tre dont il
+est question dans la cause
+ambigu6 entendue ä Moy, dans
+un capharnaüÜm qui, pense-t-il,
+diminue ca et 13 la qualite de son
+ceuvre.
+

tests/cache/manifest.jsonl

@@ -44,3 +44,4 @@
 {"tesseract_version": "tesseract 4.0.0  leptonica-1.77.0   libjpeg 9c : libpng 1.6.35 : libtiff 4.0.10 : zlib 1.2.11 : libopenjp2 2.3.0  Found AVX2  Found AVX  Found SSE ", "platform": "Darwin-18.0.0-x86_64-i386-64bit", "python": "3.7.1", "argv_slug": "__-l__eng__000004.ocr.png__000004__hocr__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "$TMPDIR/000004.ocr.png", "$TMPDIR/000004", "hocr", "txt"]}
 {"tesseract_version": "tesseract 4.0.0  leptonica-1.77.0   libjpeg 9c : libpng 1.6.35 : libtiff 4.0.10 : zlib 1.2.11 : libopenjp2 2.3.0  Found AVX2  Found AVX  Found SSE ", "platform": "Darwin-18.0.0-x86_64-i386-64bit", "python": "3.7.1", "argv_slug": "__-l__eng__000003.ocr.png__000003__hocr__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "$TMPDIR/000003.ocr.png", "$TMPDIR/000003", "hocr", "txt"]}
 {"tesseract_version": "tesseract 4.0.0  leptonica-1.77.0   libjpeg 9c : libpng 1.6.35 : libtiff 4.0.10 : zlib 1.2.11 : libopenjp2 2.3.0  Found AVX2  Found AVX  Found SSE ", "platform": "Darwin-18.0.0-x86_64-i386-64bit", "python": "3.7.1", "argv_slug": "__-l__eng__000002.ocr.png__000002__hocr__txt", "sourcefile": "resources/cardinal.pdf", "args": ["-l", "eng", "$TMPDIR/000002.ocr.png", "$TMPDIR/000002", "hocr", "txt"]}
+{"tesseract_version": "tesseract 4.0.0  leptonica-1.77.0   libgif 5.1.4 : libjpeg 9c : libpng 1.6.36 : libtiff 4.0.10 : zlib 1.2.11 : libwebp 1.0.2 : libopenjp2 2.3.0  Found AVX2  Found AVX  Found SSE ", "platform": "Darwin-18.2.0-x86_64-i386-64bit", "python": "3.7.2", "argv_slug": "__-l__deu__000001.ocr.png__000001.text__pdf__txt", "sourcefile": "resources/francais.pdf", "args": ["-l", "deu", "-c", "textonly_pdf=1", "$TMPDIR/000001.ocr.png", "$TMPDIR/000001.text", "pdf", "txt"]}

tests/conftest.py

@@ -19,7 +19,7 @@ import os
 import platform
 import sys
 from pathlib import Path
-from subprocess import PIPE, Popen
+from subprocess import PIPE, run
 
 import pytest
 
@@ -63,13 +63,14 @@ def running_in_travis():
 
 
 @pytest.helpers.register
-def needs_pdfminer(fn):
+def have_unpaper():
     try:
-        import pdfminer
-    except ImportError:
-        skip = pytest.mark.skipif(True, reason="pdfminer not available")
-        return skip(fn)
-    return fn
+        from ocrmypdf.exec import unpaper
+
+        unpaper.version()
+    except Exception:
+        return False
+    return True
 
 
 TESTS_ROOT = os.path.abspath(os.path.dirname(__file__))
@@ -147,7 +148,7 @@ def no_outpdf(tmpdir):
 
 @pytest.helpers.register
 def check_ocrmypdf(input_file, output_file, *args, env=None):
-    "Run ocrmypdf and confirmed that a valid file was created"
+    """Run ocrmypdf and confirmed that a valid file was created"""
 
     p, out, err = run_ocrmypdf(input_file, output_file, *args, env=env)
     # ensure py.test collects the output, use -s to view
@@ -171,19 +172,16 @@ def run_ocrmypdf(input_file, output_file, *args, env=None, universal_newlines=Tr
     if env is None:
         env = os.environ
 
-    p_args = OCRMYPDF + [str(arg) for arg in args] + [str(input_file), str(output_file)]
-    p = Popen(
-        p_args,
-        close_fds=True,
-        stdout=PIPE,
-        stderr=PIPE,
-        universal_newlines=universal_newlines,
-        env=env,
+    p_args = (
+        OCRMYPDF
+        + [str(arg) for arg in args if arg is not None]
+        + [str(input_file), str(output_file)]
     )
-    out, err = p.communicate()
-    # print(err)
-
-    return p, out, err
+    p = run(
+        p_args, stdout=PIPE, stderr=PIPE, universal_newlines=universal_newlines, env=env
+    )
+    # print(p.stderr)
+    return p, p.stdout, p.stderr
 
 
 @pytest.helpers.register

tests/test_ghostscript.py

@@ -0,0 +1,81 @@
+# © 2019 James R. Barlow: github.com/jbarlow83
+#
+# This file is part of OCRmyPDF.
+#
+# OCRmyPDF is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# OCRmyPDF is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+from decimal import Decimal
+
+import pikepdf
+import pytest
+from PIL import Image
+
+from ocrmypdf.exec.ghostscript import rasterize_pdf
+
+
+@pytest.fixture
+def linn(resources):
+    path = resources / 'linn.pdf'
+    return path, pikepdf.open(path)
+
+
+def test_rasterize_size(linn, outdir, caplog):
+    path, pdf = linn
+    page_size_pts = (pdf.pages[0].MediaBox[2], pdf.pages[0].MediaBox[3])
+    assert pdf.pages[0].MediaBox[0] == pdf.pages[0].MediaBox[1] == 0
+    page_size = (page_size_pts[0] / Decimal(72), page_size_pts[1] / Decimal(72))
+    target_size = Decimal('200.0'), Decimal('150.0')
+    target_dpi = 42.0, 4242.0
+
+    log = logging.getLogger()
+    rasterize_pdf(
+        path,
+        outdir / 'out.png',
+        target_size[0] / page_size[0],
+        target_size[1] / page_size[1],
+        raster_device='pngmono',
+        log=log,
+        page_dpi=target_dpi,
+    )
+
+    with Image.open(outdir / 'out.png') as im:
+        assert im.size == target_size
+        assert im.info['dpi'] == target_dpi
+
+
+def test_rasterize_rotated(linn, outdir, caplog):
+    path, pdf = linn
+    page_size_pts = (pdf.pages[0].MediaBox[2], pdf.pages[0].MediaBox[3])
+    assert pdf.pages[0].MediaBox[0] == pdf.pages[0].MediaBox[1] == 0
+    page_size = (page_size_pts[0] / Decimal(72), page_size_pts[1] / Decimal(72))
+    target_size = Decimal('200.0'), Decimal('150.0')
+    target_dpi = 42.0, 4242.0
+
+    log = logging.getLogger()
+    caplog.set_level(logging.DEBUG)
+    rasterize_pdf(
+        path,
+        outdir / 'out.png',
+        target_size[0] / page_size[0],
+        target_size[1] / page_size[1],
+        raster_device='pngmono',
+        log=log,
+        page_dpi=target_dpi,
+        rotation=90,
+    )
+
+    with Image.open(outdir / 'out.png') as im:
+        assert im.size == (target_size[1], target_size[0])
+        assert im.info['dpi'] == (target_dpi[1], target_dpi[0])

tests/test_lept.py

@@ -17,7 +17,9 @@
 
 
 from os import fspath
+import os
 from pickle import dumps, loads
+from unittest.mock import patch
 
 import pytest
 from PIL import Image, ImageChops
@@ -85,3 +87,18 @@ def test_leptonica_compile(tmpdir):
     # existing compiled library. Also compile in API mode so that we test
     # the interfaces, even though we use it ABI mode.
     ffibuilder.compile(tmpdir=fspath(tmpdir), target=fspath(tmpdir / 'lepttest.*'))
+
+
+def test_with_stderr(capsys):
+    # pytest redirects stderr too; we must disable this for the test to be valid
+    with capsys.disabled():
+        with pytest.raises(FileNotFoundError):
+            lept.Pix.open("does_not_exist1")
+
+
+def test_without_stderr(capsys):
+    # pytest redirects stderr too; we must disable this for the test to be valid
+    with capsys.disabled():
+        with patch('sys.stderr', new=None):
+            with pytest.raises(FileNotFoundError):
+                lept.Pix.open("does_not_exist2")

tests/test_main.py

@@ -21,14 +21,14 @@ import shutil
 import sys
 from math import isclose
 from pathlib import Path
-from subprocess import DEVNULL, PIPE, Popen
+from subprocess import DEVNULL, PIPE, run, Popen
 
 import PIL
 import pytest
 from PIL import Image
 
 from ocrmypdf.exceptions import ExitCode, MissingDependencyError
-from ocrmypdf.exec import ghostscript, qpdf, tesseract
+from ocrmypdf.exec import ghostscript, qpdf, tesseract, unpaper
 from ocrmypdf.leptonica import Pix
 from ocrmypdf.pdfa import file_claims_pdfa
 from ocrmypdf.pdfinfo import Colorspace, Encoding, PdfInfo
@@ -164,7 +164,7 @@ def test_exotic_image(
     check_ocrmypdf(
         resources / pdf,
         outfile,
-        '-dc',
+        '-dc' if pytest.helpers.have_unpaper() else '-d',
         '-v',
         '1',
         '--output-type',
@@ -219,7 +219,6 @@ def test_skip_ocr(spoof_tesseract_cache, resources, outpdf):
     assert pdfinfo[0].has_text
 
 
-@pytest.helpers.needs_pdfminer
 def test_redo_ocr(spoof_tesseract_cache, resources, outpdf):
     in_ = resources / 'graph_ocred.pdf'
     before = PdfInfo(in_, detailed_page_analysis=True)
@@ -282,8 +281,7 @@ def test_maximum_options(
         resources / 'multipage.pdf',
         outpdf,
         '-d',
-        '-c',
-        '-i',
+        '-ci' if pytest.helpers.have_unpaper() else None,
         '-f',
         '-k',
         '--oversample',
@@ -338,15 +336,16 @@ def test_force_ocr_on_pdf_with_no_images(spoof_tesseract_crash, resources, no_ou
     pytest.helpers.is_macos() and pytest.helpers.running_in_travis(),
     reason="takes too long to install language packs in Travis macOS homebrew",
 )
-def test_french(spoof_tesseract_cache, resources, outdir):
+def test_german(spoof_tesseract_cache, resources, outdir):
     # Produce a sidecar too - implicit test that system locale is set up
-    # properly
+    # properly. It is fine that we are testing -l deu on a French file because
+    # we are exercising the functionality not going for accuracy.
     sidecar = outdir / 'francais.txt'
     p, out, err = run_ocrmypdf(
         resources / 'francais.pdf',
         outdir / 'francais.pdf',
         '-l',
-        'fra',
+        'deu',  # more commonly installed
         '--sidecar',
         sidecar,
         env=spoof_tesseract_cache,
@@ -541,7 +540,6 @@ def test_jbig2_passthrough(spoof_tesseract_cache, resources, outpdf):
         'hocr',
         env=spoof_tesseract_cache,
     )
-
     out_pageinfo = PdfInfo(out)
     assert out_pageinfo[0].images[0].enc == Encoding.jbig2
 
@@ -553,16 +551,13 @@ def test_stdin(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf):
     # Runs: ocrmypdf - output.pdf < testfile.pdf
     with open(input_file, 'rb') as input_stream:
         p_args = ocrmypdf_exec + ['-', output_file]
-        p = Popen(
+        p = run(
             p_args,
-            close_fds=True,
             stdout=PIPE,
             stderr=PIPE,
             stdin=input_stream,
             env=spoof_tesseract_noop,
         )
-        out, err = p.communicate()
-
         assert p.returncode == ExitCode.ok
 
 
@@ -573,16 +568,13 @@ def test_stdout(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf):
     # Runs: ocrmypdf francais.pdf - > test_stdout.pdf
     with open(output_file, 'wb') as output_stream:
         p_args = ocrmypdf_exec + [input_file, '-']
-        p = Popen(
+        p = run(
             p_args,
-            close_fds=True,
             stdout=output_stream,
             stderr=PIPE,
             stdin=DEVNULL,
             env=spoof_tesseract_noop,
         )
-        out, err = p.communicate()
-
         assert p.returncode == ExitCode.ok
 
     assert qpdf.check(output_file, log=None)
@@ -780,10 +772,10 @@ def test_pagesize_consistency(renderer, resources, outpdf):
         outpdf,
         '--pdf-renderer',
         renderer,
-        '--clean',
+        '--clean' if pytest.helpers.have_unpaper() else None,
         '--deskew',
         '--remove-background',
-        '--clean-final',
+        '--clean-final' if pytest.helpers.have_unpaper() else None,
     )
 
     after_dims = first_page_dimensions(outpdf)
@@ -851,22 +843,21 @@ def test_compression_preserved(
             '-',
             output_file,
         ]
-        p = Popen(
+        p = run(
             p_args,
-            close_fds=True,
             stdout=PIPE,
             stderr=PIPE,
             stdin=input_stream,
+            universal_newlines=True,
             env=spoof_tesseract_noop,
         )
-        out, err = p.communicate()
 
         if im.mode in ('RGBA', 'LA'):
             # If alpha image is input, expect an error
-            assert p.returncode != ExitCode.ok and b'alpha' in err
+            assert p.returncode != ExitCode.ok and 'alpha' in p.stderr
             return
 
-        assert p.returncode == ExitCode.ok, err.decode('utf-8')
+        assert p.returncode == ExitCode.ok, p.stderr
 
     pdfinfo = PdfInfo(output_file)
 
@@ -912,17 +903,15 @@ def test_compression_changed(
             '-',
             output_file,
         ]
-        p = Popen(
+        p = run(
             p_args,
-            close_fds=True,
             stdout=PIPE,
             stderr=PIPE,
             stdin=input_stream,
+            universal_newlines=True,
             env=spoof_tesseract_noop,
         )
-        out, err = p.communicate()
-
-        assert p.returncode == ExitCode.ok, err
+        assert p.returncode == ExitCode.ok, p.stderr
 
     pdfinfo = PdfInfo(output_file)
 
@@ -998,6 +987,9 @@ def test_pdfa_n(spoof_tesseract_cache, pdfa_level, resources, outpdf):
 
 
 @pytest.mark.skipif(sys.version_info >= (3, 7, 0), reason='better utf-8')
+@pytest.mark.skipif(
+    Path('/etc/alpine-release').exists(), reason="invalid test on alpine"
+)
 def test_bad_locale():
     env = os.environ.copy()
     env['LC_ALL'] = 'C'

tests/test_metadata.py

@@ -22,7 +22,7 @@ import logging
 import mmap
 from os import fspath
 from pathlib import Path
-from shutil import copyfile
+from shutil import copyfile, move
 from unittest.mock import MagicMock, patch
 
 import pytest
@@ -188,9 +188,8 @@ def test_xml_metadata_preserved(spoof_tesseract_noop, output_type, resources, ou
     input_file = resources / 'graph.pdf'
 
     try:
-        import libxmp
-        from libxmp.utils import file_to_dict
         from libxmp import consts
+        from libxmp.utils import file_to_dict
     except Exception:
         pytest.skip("libxmp not available or libexempi3 not installed")
 
@@ -286,7 +285,7 @@ def test_kodak_toc(resources, outpdf, spoof_tesseract_noop):
         assert isinstance(p.root.Outlines.First, pikepdf.Dictionary)
 
 
-def test_metadata_fixup_warning(resources, outdir):
+def test_metadata_fixup_warning(resources, outdir, caplog):
     from ocrmypdf._pipeline import metadata_fixup
 
     input_files = [
@@ -297,7 +296,7 @@ def test_metadata_fixup_warning(resources, outdir):
     for f in input_files:
         copyfile(resources / 'graph.pdf', f)
 
-    log = MagicMock()
+    log = logging.getLogger()
     context = MagicMock()
     metadata_fixup(
         input_files_groups=input_files,
@@ -305,15 +304,17 @@ def test_metadata_fixup_warning(resources, outdir):
         log=log,
         context=context,
     )
-    log.warning.assert_not_called()
+    for record in caplog.records:
+        assert record.levelname != 'WARNING'
 
     # Now add some metadata that will not be copyable
-    graph = pikepdf.open(outdir / 'graph.repaired.pdf')
-    with graph.open_metadata() as meta:
-        meta['prism2:publicationName'] = 'OCRmyPDF Test'
-    graph.save(outdir / 'graph.repaired.pdf')
+    with pikepdf.open(outdir / 'graph.repaired.pdf') as graph:
+        with graph.open_metadata() as meta:
+            meta['prism2:publicationName'] = 'OCRmyPDF Test'
+        graph.save(outdir / 'graph.repaired.modified.pdf')
+    move(outdir / 'graph.repaired.modified.pdf', outdir / 'graph.repaired.pdf')
 
-    log = MagicMock()
+    log = logging.getLogger()
     context = MagicMock()
     metadata_fixup(
         input_files_groups=input_files,
@@ -321,7 +322,7 @@ def test_metadata_fixup_warning(resources, outdir):
         log=log,
         context=context,
     )
-    log.warning.assert_called_once()
+    assert any(record.levelname == 'WARNING' for record in caplog.records)
 
 
 def test_prevent_gs_invalid_xml(resources, outdir):
@@ -331,16 +332,13 @@ def test_prevent_gs_invalid_xml(resources, outdir):
     from ocrmypdf.pdfinfo import PdfInfo
 
     generate_pdfa_ps(outdir / 'pdfa.ps')
-    input_files = [
-        str(outdir / 'layers.rendered.pdf'),
-        str(outdir / 'pdfa.ps'),
-    ]
+    input_files = [str(outdir / 'layers.rendered.pdf'), str(outdir / 'pdfa.ps')]
     copyfile(resources / 'enron1.pdf', outdir / 'layers.rendered.pdf')
     log = logging.getLogger()
     context = JobContext()
 
-    options = parser.parse_args(args=[
-        '-j', '1', '--output-type', 'pdfa-2', 'a.pdf', 'b.pdf']
+    options = parser.parse_args(
+        args=['-j', '1', '--output-type', 'pdfa-2', 'a.pdf', 'b.pdf']
     )
     context.options = options
     context.pdfinfo = PdfInfo(resources / 'enron1.pdf')
@@ -349,11 +347,13 @@ def test_prevent_gs_invalid_xml(resources, outdir):
         input_files_groups=input_files,
         output_file=outdir / 'pdfa.pdf',
         log=log,
-        context=context
+        context=context,
     )
 
     with open(outdir / 'pdfa.pdf', 'rb') as f:
-        with mmap.mmap(f.fileno(), 0, flags=mmap.MAP_PRIVATE, prot=mmap.PROT_READ) as mm:
+        with mmap.mmap(
+            f.fileno(), 0, flags=mmap.MAP_PRIVATE, prot=mmap.PROT_READ
+        ) as mm:
             # Since the XML may be invalid, we scan instead of actually feeding it
             # to a parser.
             XMP_MAGIC = b'W5M0MpCehiHzreSzNTczkc9d'

tests/test_optimize.py

@@ -120,7 +120,7 @@ def test_flate_to_jbig2(resources, outdir, spoof_tesseract_noop):
         '--image-dpi',
         '100',
         '--png-quality',
-        '10',
+        '50',
         '--optimize',
         '3',
         env=spoof_tesseract_noop,

tests/test_pdfinfo.py

@@ -183,12 +183,7 @@ def test_ocr_detection(resources):
 @pytest.mark.parametrize(
     'testfile', ('truetype_font_nomapping.pdf', 'type3_font_nomapping.pdf')
 )
-@pytest.helpers.needs_pdfminer  # pylint: disable=e1101
 def test_corrupt_font_detection(resources, testfile):
-    try:
-        import pdfminer
-    except ImportError:
-        pytest.skip("Needs pdfminer")
     filename = resources / testfile
     with pytest.raises(NotImplementedError):
         pdf = pdfinfo.PdfInfo(filename)

tests/test_tess4.py

@@ -117,10 +117,10 @@ def test_pagesize_consistency_tess4(ensure_tess4, resources, outpdf):
         outpdf,
         '--pdf-renderer',
         'sandwich',
-        '--clean',
+        '--clean' if pytest.helpers.have_unpaper() else None,
         '--deskew',
         '--remove-background',
-        '--clean-final',
+        '--clean-final' if pytest.helpers.have_unpaper() else None,
         env=ensure_tess4,
     )
 

tests/test_unpaper.py

@@ -15,11 +15,17 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
+import argparse
+import logging
+from os import fspath
 from pathlib import Path
+from unittest.mock import patch
 
 import pytest
 
+from ocrmypdf import __main__ as main
 from ocrmypdf.exceptions import ExitCode
+from ocrmypdf.exec import unpaper
 
 # pytest.helpers is dynamic
 # pylint: disable=no-member
@@ -30,26 +36,79 @@ run_ocrmypdf = pytest.helpers.run_ocrmypdf
 spoof = pytest.helpers.spoof
 
 
-@pytest.fixture(scope='session')
+def have_unpaper():
+    try:
+        unpaper.version()
+    except Exception:
+        return False
+    else:
+        return True
+
+
+@pytest.fixture(scope="session")
 def spoof_unpaper_oldversion(tmpdir_factory):
-    return spoof(tmpdir_factory, unpaper='unpaper_oldversion.py')
+    return spoof(tmpdir_factory, unpaper="unpaper_oldversion.py")
 
 
-@pytest.mark.skipif(True, reason="needs new fixture implementation")
 def test_no_unpaper(resources, no_outpdf):
-    # <disable unpaper here>
-    p, out, err = run_ocrmypdf(
-        resources / 'c02-22.pdf', no_outpdf, '--clean', env=os.environ
-    )
-    assert p.returncode == ExitCode.missing_dependency
+    input_ = fspath(resources / "c02-22.pdf")
+    output = fspath(no_outpdf)
+    options = main.parser.parse_args(args=["--clean", input_, output])
+
+    with patch("ocrmypdf.exec.unpaper.version") as mock_unpaper_version:
+        mock_unpaper_version.side_effect = FileNotFoundError("unpaper")
+        with pytest.raises(SystemExit):
+            main.check_options(options, log=logging.getLogger())
 
 
 def test_old_unpaper(spoof_unpaper_oldversion, resources, no_outpdf):
     p, out, err = run_ocrmypdf(
-        resources / 'c02-22.pdf', no_outpdf, '--clean', env=spoof_unpaper_oldversion
+        resources / "c02-22.pdf", no_outpdf, "--clean", env=spoof_unpaper_oldversion
     )
     assert p.returncode == ExitCode.missing_dependency
 
 
+@pytest.mark.skipif(not have_unpaper(), reason="requires unpaper")
 def test_clean(spoof_tesseract_noop, resources, outpdf):
-    check_ocrmypdf(resources / 'skew.pdf', outpdf, '-c', env=spoof_tesseract_noop)
+    check_ocrmypdf(resources / "skew.pdf", outpdf, "-c", env=spoof_tesseract_noop)
+
+
+@pytest.mark.skipif(not have_unpaper(), reason="requires unpaper")
+def test_unpaper_args_valid(spoof_tesseract_noop, resources, outpdf):
+    check_ocrmypdf(
+        resources / "skew.pdf",
+        outpdf,
+        "-c",
+        "--unpaper-args",
+        "--layout double",  # Spaces required here
+        env=spoof_tesseract_noop,
+    )
+
+
+@pytest.mark.skipif(not have_unpaper(), reason="requires unpaper")
+def test_unpaper_args_invalid_filename(spoof_tesseract_noop, resources, outpdf):
+    p, out, err = run_ocrmypdf(
+        resources / "skew.pdf",
+        outpdf,
+        "-c",
+        "--unpaper-args",
+        "/etc/passwd",
+        env=spoof_tesseract_noop,
+    )
+    assert "No filenames allowed" in err
+    assert p.returncode == ExitCode.bad_args
+
+
+@pytest.mark.skipif(not have_unpaper(), reason="requires unpaper")
+def test_unpaper_args_invalid(spoof_tesseract_noop, resources, outpdf):
+    p, out, err = run_ocrmypdf(
+        resources / "skew.pdf",
+        outpdf,
+        "-c",
+        "--unpaper-args",
+        "unpaper is not going to like these arguments",
+        env=spoof_tesseract_noop,
+    )
+    # Can't tell difference between unpaper choking on bad arguments or some
+    # other unpaper failure
+    assert p.returncode == ExitCode.child_process_error

tests/test_userunit.py

@@ -15,20 +15,12 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
-import logging
-import os
-import shutil
 from math import isclose
-from subprocess import DEVNULL, PIPE, Popen, check_call, check_output
 
-import PyPDF2 as pypdf
 import pytest
 
-from ocrmypdf import leptonica
 from ocrmypdf.exceptions import ExitCode
-from ocrmypdf.exec import ghostscript
-from ocrmypdf.pdfa import file_claims_pdfa
-from ocrmypdf.pdfinfo import Colorspace, Encoding, PdfInfo
+from ocrmypdf.pdfinfo import PdfInfo
 
 check_ocrmypdf = pytest.helpers.check_ocrmypdf
 run_ocrmypdf = pytest.helpers.run_ocrmypdf

tests/test_weave.py

@@ -0,0 +1,61 @@
+# © 2019 James R. Barlow: github.com/jbarlow83
+#
+# This file is part of OCRmyPDF.
+#
+# OCRmyPDF is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# OCRmyPDF is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+
+import pytest
+
+import pikepdf
+
+check_ocrmypdf = pytest.helpers.check_ocrmypdf
+
+
+def test_no_glyphless_weave(resources, outdir):
+    pdf = pikepdf.open(resources / 'francais.pdf')
+    pdf_aspect = pikepdf.open(resources / 'aspect.pdf')
+    pdf_cmyk = pikepdf.open(resources / 'cmyk.pdf')
+    pdf.pages.extend(pdf_aspect.pages)
+    pdf.pages.extend(pdf_cmyk.pages)
+    pdf.save(outdir / 'test.pdf')
+
+    env = os.environ.copy()
+    env['_OCRMYPDF_MAX_REPLACE_PAGES'] = '2'
+    check_ocrmypdf(
+        outdir / 'test.pdf',
+        outdir / 'out.pdf',
+        '--deskew',
+        '--tesseract-timeout',
+        '0',
+        env=env,
+    )
+
+
+def test_links(resources, outpdf):
+    check_ocrmypdf(
+        resources / 'link.pdf',
+        outpdf,
+        '--redo-ocr',
+        '--oversample',
+        '200',
+        '--output-type',
+        'pdf',
+    )
+    pdf = pikepdf.open(outpdf)
+    p1 = pdf.pages[0]
+    p2 = pdf.pages[1]
+    assert p1.Annots[0].A.D[0].objgen == p2.objgen
+    assert p2.Annots[0].A.D[0].objgen == p1.objgen