v5.6.3 notes

Did not account for API difference in pathlib

.coveragerc

@@ -0,0 +1,16 @@
+# Coverage isn't really compatible with subprocesses so results are unreliable
+
+[run]
+branch = True
+#concurrency = multiprocessing
+source = ocrmypdf/
+
+[report]
+exclude_lines =
+	pragma: no cover
+	def __repr__
+	raise AssertionError
+	raise NotImplementedError
+	if 0:
+	if False:
+	if __name__ == .__main__.:

.docker/Dockerfile

@@ -1,8 +1,6 @@
 # OCRmyPDF
 #
-# VERSION               4.4.2
-FROM      ubuntu:16.10
-MAINTAINER James R. Barlow <jim@purplerock.ca>
+FROM      ubuntu:17.10
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
   software-properties-common python-software-properties \
@@ -20,6 +18,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   tesseract-ocr-spa \
   tesseract-ocr-deu
 
+ENV LANG=C.UTF-8
+
 RUN python3 -m venv --system-site-packages /appenv
 
 # This installs the latest binary wheel instead of the code in the current
@@ -51,4 +51,4 @@ WORKDIR /home/docker
 
 # Must use array form of ENTRYPOINT
 # Non-array form does not append other arguments, because that is "intuitive"
-ENTRYPOINT ["/application/docker-wrapper.sh"]
+ENTRYPOINT ["/application/.docker/docker-wrapper.sh"]

.docker/Dockerfile.polyglot

@@ -1,8 +1,6 @@
 # OCRmyPDF polyglot
 #
-# VERSION               4.4.2
 FROM      jbarlow83/ocrmypdf:latest
-MAINTAINER James R. Barlow <jim@purplerock.ca>
 
 USER root
 
@@ -16,4 +14,4 @@ USER docker
 
 # Must use array form of ENTRYPOINT
 # Non-array form does not append other arguments, because that is "intuitive"
-ENTRYPOINT ["/application/docker-wrapper.sh"]
+ENTRYPOINT ["/application/.docker/docker-wrapper.sh"]

.docker/Dockerfile.tess4

@@ -1,8 +1,6 @@
 # OCRmyPDF
 #
-# VERSION               4.4.2
-FROM 	ubuntu:16.10
-MAINTAINER James R. Barlow <jim@purplerock.ca>
+FROM 	ubuntu:17.10
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
   software-properties-common python-software-properties \
@@ -30,6 +28,8 @@ RUN apt-get update \
 		tesseract-ocr-rus \
 		tesseract-ocr-chi-sim 
 
+ENV LANG=C.UTF-8
+
 RUN python3 -m venv --system-site-packages /appenv
 
 # This installs the latest binary wheel instead of the code in the current
@@ -61,4 +61,4 @@ WORKDIR /home/docker
 
 # Must use array form of ENTRYPOINT
 # Non-array form does not append other arguments, because that is "intuitive"
-ENTRYPOINT ["/application/docker-wrapper.sh"]
+ENTRYPOINT ["/application/.docker/docker-wrapper.sh"]

.dockerignore

@@ -3,18 +3,9 @@
 *.pyc
 *.rst
 *.sublime*
-*/*.pyc
-*/*/*.pyc
-*/*/*/*.pyc
-*/*/*/*/*.pyc
-*/*/*/*/*/*.pyc
-*/*/*/*/*/*/*.pyc
-*/*/*/*/*/*/*/*.pyc
-.cache/
-.eggs/
-.github/
-.git/
-.ipynb_checkpoints/
+**/*.pyc
+.*/
+!.docker/
 .ruffus_history.sqlite
 bin/
 build/

.gitignore

@@ -38,4 +38,7 @@ tests/cache/
 tests/output/
 tests/resources/private/
 tmp/
-pdfbox-app*.jar
+pdfbox-app*.jar
+.vscode/
+IDEAS
+_Dockerfile.local

.travis.yml

@@ -34,7 +34,9 @@ before_install: |
   if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
     bash .travis/linux_before_install.sh
   elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-    bash .travis/osx_before_install.sh
+    brew update && brew bundle --file=.travis/Brewfile
+    pip3 install --upgrade pip
+    pip3 install wheel
   fi
 
 install:
@@ -43,7 +45,9 @@ install:
 
 script:
 - mv ocrmypdf dont_import_this_ocrmypdf
-- pytest -n auto
+- tesseract --version
+- qpdf --version
+- pytest
 - mv dont_import_this_ocrmypdf ocrmypdf
 
 deploy:
@@ -84,11 +88,11 @@ deploy:
   script: /usr/bin/true
   on:
     branch: master
-    tags: false
+    tags: true
     condition: $TRAVIS_OS_NAME == "osx"
 
 after_deploy: |
-  if [[ "$TRAVIS_OS_NAME" == "osx" && "$TRAVIS_BRANCH" == "master" ]]; then
+  if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
     bash .travis/osx_brew.sh
   elif [[ "$TRAVIS_PYTHON_VERSION" == "3.6" && "$TRAVIS_OS_NAME" == "linux" ]]; then
     curl -H "Content-Type: application/json" --data '{"build": true}' -X POST https://registry.hub.docker.com/u/jbarlow83/ocrmypdf/trigger/$DOCKERHUB_OCRMYPDF_TOKEN/

.travis/Brewfile

@@ -0,0 +1,12 @@
+brew 'python'
+
+brew 'ghostscript'
+brew 'jbig2dec'
+brew 'leptonica'
+brew 'libffi'
+brew 'libtiff'
+brew 'libxml2'
+brew 'openjpeg'
+brew 'qpdf'
+brew 'tesseract'
+brew 'unpaper'

.travis/autobrew.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# © 2016 James R. Barlow: github.com/jbarlow83
+# © 2017-18 James R. Barlow: github.com/jbarlow83
 
 from string import Template
 from subprocess import run, PIPE
@@ -14,37 +14,50 @@ class Ocrmypdf < Formula
   ${ocrmypdf_url}
   ${ocrmypdf_sha256}
 
-  depends_on :python3
   depends_on "pkg-config" => :build
-  depends_on "libffi"
-  depends_on "tesseract"
-  depends_on "ghostscript"
-  depends_on "unpaper"
-  depends_on "qpdf"
-
-  # For Pillow source install
-  depends_on "openjpeg"
   depends_on "freetype"
+  depends_on "ghostscript"
+  depends_on "jpeg"
   depends_on "libpng"
-  depends_on "libjpeg"
-  depends_on "webp"
-  depends_on "little-cms2"
+  depends_on "python"
+  depends_on "qpdf"
+  depends_on "tesseract"
+  depends_on "unpaper"
 
 ${resources}
   def install
-    ENV.append ["SETUPTOOLS_SCM_PRETEND_VERSION"], "v${ocrmypdf_version}"
-    ENV.each do |key, value|
-      puts "#{key}:#{value}"
+    venv = virtualenv_create(libexec, "python3")
+
+    resource("Pillow").stage do
+      inreplace "setup.py" do |s|
+        sdkprefix = MacOS::CLT.installed? ? "" : MacOS.sdk_path
+        s.gsub! "openjpeg.h", "probably_not_a_header_called_this_eh.h"
+        s.gsub! "ZLIB_ROOT = None", "ZLIB_ROOT = ('#{sdkprefix}/usr/lib', '#{sdkprefix}/usr/include')"
+        s.gsub! "JPEG_ROOT = None", "JPEG_ROOT = ('#{Formula["jpeg"].opt_prefix}/lib', '#{Formula["jpeg"].opt_prefix}/include')"
+        s.gsub! "FREETYPE_ROOT = None", "FREETYPE_ROOT = ('#{Formula["freetype"].opt_prefix}/lib', '#{Formula["freetype"].opt_prefix}/include')"
+      end
+
+      # avoid triggering "helpful" distutils code that doesn't recognize Xcode 7 .tbd stubs
+      ENV.append "CFLAGS", "-I#{MacOS.sdk_path}/System/Library/Frameworks/Tk.framework/Versions/8.5/Headers" unless MacOS::CLT.installed?
+      venv.pip_install Pathname.pwd
     end
-    virtualenv_install_with_resources
+
+    res = resources.map(&:name).to_set - ["Pillow"]
+
+    res.each do |r|
+      venv.pip_install resource(r)
+    end
+
+    venv.pip_install_and_link buildpath
   end
 
   test do
-    # `test do` will create, run in and delete a temporary directory.
-    #
-    # The installed folder is not in the path, so use the entire path to any
-    # executables being tested: `system "#{bin}/program", "do", "something"`.
-    system "#{bin}/ocrmypdf", "--version"
+    # Since we use Python 3, we require a UTF-8 locale
+    ENV["LC_ALL"] = "en_US.UTF-8"
+
+    system "#{bin}/ocrmypdf", "-f", "-q", "--deskew",
+                              test_fixtures("test.pdf"), "ocr.pdf"
+    assert_predicate testpath/"ocr.pdf", :exist?
   end
 end
 """)

.travis/osx_before_install.sh

@@ -1,17 +0,0 @@
-#!/bin/bash
-# © 2017 James R. Barlow: github.com/jbarlow83
-set -euo pipefail
-set -x
-
-brew update
-
-brew install openjpeg jbig2dec libtiff     # image libraries
-brew install qpdf
-brew install ghostscript
-brew install python3
-brew install libxml2 libffi leptonica
-brew install unpaper   # optional
-brew install tesseract
-
-pip3 install --upgrade pip
-pip3 install wheel

.travis/osx_brew.sh

@@ -6,7 +6,9 @@ set -x
 pip3 install homebrew-pypi-poet
 python3 .travis/autobrew.py
 cat ocrmypdf.rb
-brew audit ocrmypdf.rb
+
+# brew audit crashes Travis
+#brew audit ocrmypdf.rb
 
 # Important: disable debug output so token is hidden
 set +x

LICENSE.rst

@@ -1,4 +1,7 @@
-Copyright (c) 2013-2016, The OCRmyPDF Authors
+Copyright (c) 2014-2017, James R. Barlow
+
+Copyright (c) 2013-2014, Julien Pfefferkorn
+Copyright (c) 2013-2017, The OCRmyPDF Authors
 
 Permission is hereby granted, free of charge, to any person obtaining a
 copy of this software and associated documentation files (the

MANIFEST.in

@@ -7,12 +7,12 @@ include dev_requirements.txt
 include .git_archival.txt
 
 # docker
-include Dockerfile
-include Dockerfile.polyglot
+include Dockerfile*
 include .dockerignore
 
 # tests
 include pytest.ini
+include .coveragerc
 recursive-include tests *.jpg
 recursive-include tests *.png
 recursive-include tests *.pdf
@@ -38,11 +38,14 @@ recursive-exclude docs/_build *
 recursive-include ocrmypdf/data *
 include *.py
 exclude tasks.py
+exclude debug-runner.py
+recursive-exclude .travis *
+exclude .travis*
+
 
 # code
 recursive-include ocrmypdf *.py
 exclude ocrmypdf/lib/_leptonica.py
-include OCRmyPDF.sh
 include docker-wrapper.sh
 
 

README.rst

@@ -4,6 +4,12 @@ OCRmyPDF
 .. image:: https://travis-ci.org/jbarlow83/OCRmyPDF.svg?branch=master
     :target: https://travis-ci.org/jbarlow83/OCRmyPDF
 
+.. image:: https://img.shields.io/pypi/v/ocrmypdf.svg   
+    :target: https://pypi.org/project/ocrmypdf/
+
+.. image:: https://img.shields.io/docker/build/jbarlow83/ocrmypdf.svg
+    :target: https://hub.docker.com/r/jbarlow83/ocrmypdf/
+
 OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to
 be searched or copy-pasted.
 
@@ -68,11 +74,10 @@ Users of Debian 9 or later or Ubuntu 16.10 or later may simply
 
    apt-get install ocrmypdf
 
-and macOS users may simply
+and macOS users with Homebrew may simply
 
 .. code-block:: bash
 
-   brew tap jbarlow83/ocrmypdf
    brew install ocrmypdf
 
 For everyone else, `see our documentation <https://ocrmypdf.readthedocs.io/en/latest/installation.html>`_ for installation steps.
@@ -116,6 +121,13 @@ If you detect an issue, please:
 -  If possible provide your input PDF file as well as the content of the
    temporary folder (using a file sharing service like Dropbox)
 
+Requirements
+------------
+
+Runs on CPython 3.6, and requires external program installations of Ghostscript, Tesseract OCR, QPDF, and Leptonica. ocrmypdf is pure Python, but uses CFFI to portably generate library bindings.
+
+Python 3.5 is fully supported, but support will end at the end of 2017, to allow f-strings in the code base.
+
 Press & Media
 -------------
 

docs/advanced.rst

@@ -33,20 +33,27 @@ If you want to adjust the amount of time spent on OCR, change ``--tesseract-time
 Overriding default tesseract
 """"""""""""""""""""""""""""
 
-OCRmyPDF checks the environment variable ``OCRMYPDF_TESSERACT`` for the full path *to the tesseract executable* first. 
+OCRmyPDF checks the environment variable ``OCRMYPDF_TESSERACT`` for the full path *to the tesseract binary* first. 
 
-For example, if you are testing tesseract 4.00 and don't wish to disturb your tesseract 3.04 installation, you can launch OCRmyPDF as follows:
+.. envvar:: OCRMYPDF_TESSERACT
+
+	Specify the location of the Tesseract binary.
+
+.. envvar:: TESSDATA_PREFIX
+
+	A Tesseract environment variable that overrides the path to Tesseract's data files.
+
+For example, if you are testing tesseract 4.00 and don't wish to disturb use an existing tesseract 3.04 installation, you can launch OCRmyPDF as follows:
 
 .. code-block:: bash
 
 	env \
 		OCRMYPDF_TESSERACT=/home/user/src/tesseract4/api/tesseract \
 		TESSDATA_PREFIX=/home/user/src/tesseract4 \
-		ocrmypdf --pdf-renderer tess4 --tesseract-oem 2 input.pdf output.pdf
+		ocrmypdf --tesseract-oem 2 input.pdf output.pdf
+
+In this example ``TESSDATA_PREFIX`` directs tesseract 4.0 to use LSTM training data. ``--tesseract-oem 1`` requests tesseract 4.0's new LSTM engine. (Tesseract 4.0 only.)
 
-* ``TESSDATA_PREFIX`` directs tesseract 4.0 to use LSTM training data. This is a tesseract environment variable.
-* ``--pdf-renderer tess4`` takes advantage of new tesseract 4.0 PDF renderer in OCRmyPDF. (Tesseract 4.0 only.)
-* ``--tesseract-oem 1`` requests tesseract 4.0's new LSTM engine. (Tesseract 4.0 only.)
 
 Overriding other support programs
 """""""""""""""""""""""""""""""""
@@ -57,7 +64,20 @@ In addition to tesseract, OCRmyPDF uses the following external binaries:
 * ``unpaper``
 * ``qpdf``
 
-In each case OCRmyPDF will check the environment variable ``OCRMYPDF_{program}`` before asking the system to find ``{program}`` on the PATH. For example, you could redirect OCRmyPDF to ``OCRMYPDF_GS`` to override Ghostscript.
+In each case OCRmyPDF will check the environment variable ``OCRMYPDF_{program}`` before asking the system to find ``{program}`` on the PATH. For example, you could redirect OCRmyPDF to ``OCRMYPDF_GS`` to override Ghostscript. The full list is below:
+
+.. envvar:: OCRMYPDF_GS
+
+	Specify the location of the Ghostscript binary.
+
+.. envvar:: OCRMYPDF_UNPAPER
+
+	Specify the location of the unpaper binary.
+
+.. envvar:: OCRMYPDF_QPDF
+
+	Specify the location of the qpdf binary.
+
 
 Changing tesseract configuration variables
 """"""""""""""""""""""""""""""""""""""""""
@@ -95,33 +115,29 @@ rendering
   Creating a new PDF from other data (such as an existing PDF).
 
 
-OCRmyPDF has three PDF renderers: ``hocr``, ``tesseract`` and ``tess4``. The renderer may be selected using ``--pdf-renderer``. The default is ``auto`` which lets OCRmyPDF select the renderer to use. Currently, ``auto`` always selects ``hocr``. 
+OCRmyPDF has three PDF renderers: ``sandwich``, ``hocr``, ``tesseract``. The renderer may be selected using ``--pdf-renderer``. The default is ``auto`` which lets OCRmyPDF select the renderer to use. Currently, ``auto`` selects ``sandwich`` for Tesseract 3.05.01, and newer, ``hocr`` for older versions of Tesseract.
+
+The ``sandwich`` renderer
+"""""""""""""""""""""""""
+
+The ``sandwich`` renderer uses Tesseract's new text-only PDF feature, which produces a PDF page that lays out the OCR in invisible text. This page is then "sandwiched" onto the original PDF page, allowing lossless application of OCR even to PDF pages that contain other vector objects.
+
+When image preprocessing features like ``--deskew`` are used, the original PDF will be rendered as a full page and the OCR layer will be placed on top.
+
+This renderer requires Tesseract 3.05.01 or newer.
 
 The ``hocr`` renderer
 """""""""""""""""""""
 
-The ``hocr`` renderer is the default because it works in most cases. In this mode the whole PDF is rasterized, the raster image is run through OCR to generate a .hocr file, which is an HTML-like file that specifies the location of all identified words.
-
-The .hocr file is then rendered as a PDF and merged with the image layer.
-
-The image layer is copied from the original PDF page if possible, avoiding potentially lossy transcoding or loss of other PDF information. If preprocessing is specified, then the image layer is a new PDF.
-
-This is the only option for tesseract 3.02 and older.
+The ``hocr`` renderer works with older versions of Tesseract. The image layer is copied from the original PDF page if possible, avoiding potentially lossy transcoding or loss of other PDF information. If preprocessing is specified, then the image layer is a new PDF.
 
+This works in all versions of Tesseract.
 
 The ``tesseract`` renderer
 """"""""""""""""""""""""""
 
-The tesseract renderer uses tesseract's capability to produce a PDF directly. In version 3, tesseract automatically combined the image layer and text, meaning that this mode *always* transcodes and loses potentially loses image quality and other PDF information.
+The ``tesseract`` renderer creates a PDF with the image and text layers precomposed, meaning that it always transcodes, loses image quality and rasterizes and vector objects. It does a better job on non-Latin text and document structure than ``hocr``.
 
-It does a much better job on non-Latin text.
+If a PDF created with this renderer using Tesseract versions older than 3.05.00 is then passed through Ghostscript's pdfwrite feature, the OCR text *may* be corrupted. The ``--output-type=pdfa`` argument will produce a warning in this situation.
 
-In a future release this will become the "tess3" renderer and ultimately will be dropped.
-
-
-The ``tess4`` renderer
-""""""""""""""""""""""
-
-The tess4 renderer uses tesseract 4.00 alpha's text-only PDF feature added in January 2017. This combines the advantages of the tesseract and hocr renderers, transcoding the image layer only if required by preprocessing options.
-
-Ghostscript PDF/A still sometimes inserts spaces between words when the tess4 renderer is used, affecting search quality.  ``--output-pdf pdf`` may be used to avoid this issue.
+*This renderer is deprecated and will be removed whenever support for older versions of Tesseract is dropped.*

docs/batch.rst

@@ -90,13 +90,83 @@ This user contributed script also provides an example of batch processing.
 API
 """
 
-OCRmyPDF is currently supported as a command line interface. Due to limitations in one of the libraries OCRmyPDF depends on, it is not yet usable as an API.
+OCRmyPDF is currently supported as a command line interface. This means that even if you are using OCRmyPDF in a Python script, you should run it in a subprocess rather importing the ocrmypdf package.
+
+The reason for this limitation is that the `ruffus <https://github.com/bunbun/ruffus/>`_ library that OCRmyPDF depends on is unfortunately not reentrant. OCRmyPDF works by defining each operation it does as a ruffus task that takes one or more files as input and generates one or more files as output. As such ruffus is fairly fundamental.
+
+(If you find individual functions implemented in OCRmyPDF useful (such as ``ocrmypdf.pdfinfo``), you can use these if you wish to.)
+
+
+Synology DiskStations
+"""""""""""""""""""""
+
+Synology DiskStations (Network Attached Storage devices) can run the Docker image of OCRmyPDF if the Synology `Docker package <https://www.synology.com/en-global/dsm/packages/Docker>`_ is installed. Attached is a script to address particular quirks of using OCRmyPDF on one of these devices.
+
+This is only possible for x86-based Synology products. Some Synology products use ARM or Power processors and do not support Docker. Further adjustments might be needed to deal with the Synology's relatively limited CPU and RAM.
+
+.. code-block:: python
+
+	#!/bin/env python3
+	# Contributed by github.com/Enantiomerie
+
+	# script needs 2 arguments
+	# 1. source dir with *.pdf - default is location of script
+	# 2. move dir where *.pdf and *_OCR.pdf are moved to
+
+	import logging
+	import os
+	import subprocess
+	import sys
+	import time
+	import shutil
+
+	script_dir = os.path.dirname(os.path.realpath(__file__))
+	timestamp = time.strftime("%Y-%m-%d-%H%M_")
+	log_file = script_dir + '/' + timestamp + 'ocrmypdf.log'
+	logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s', filename=log_file, filemode='w')
+
+	if len(sys.argv) > 1:
+	    start_dir = sys.argv[1]
+	else:
+	    start_dir = '.'
+
+	for dir_name, subdirs, file_list in os.walk(start_dir):
+	    logging.info('\n')
+	    logging.info(dir_name + '\n')
+	    os.chdir(dir_name)
+	    for filename in file_list:
+	        file_ext = os.path.splitext(filename)[1]
+	        if file_ext == '.pdf':
+	            full_path = dir_name + '/' + filename
+	            file_noext = os.path.splitext(filename)[0]
+	            timestamp_OCR = time.strftime("%Y-%m-%d-%H%M_OCR_")
+	            filename_OCR = timestamp_OCR + file_noext + '.pdf'
+	            docker_mount = dir_name + ':/home/docker'
+	# create string for pdf processing 
+	# diskstation needs a user:group docker:docker. find uid:gid of your diskstation docker:docker with id docker.
+	# use this uid:gid in -u flag
+	# rw rights for docker:docker at source dir are also necessary
+	# the script is processed as root user via chron 
+	            cmd = ['docker', 'run', '--rm', '-v', docker_mount, '-u=1030:65538', 'jbarlow83/ocrmypdf', , '--deskew' , filename, filename_OCR]
+	            logging.info(cmd)
+	            proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+	            result = proc.stdout.read()
+	            logging.info(result)
+	            full_path_OCR = dir_name + '/' + filename_OCR
+	            os.chmod(full_path_OCR, 0o666)
+	            os.chmod(full_path, 0o666)
+	            full_path_OCR_archive = sys.argv[2]
+	            full_path_archive = sys.argv[2] + '/no_ocr'
+	            shutil.move(full_path_OCR,full_path_OCR_archive)
+	            shutil.move(full_path, full_path_archive)
+	logging.info('Finished.\n')
+
 
 
 Huge batch jobs
 """""""""""""""
 
-If you have thousands of files to work with, contact the author.
+If you have thousands of files to work with, contact the author. Consulting work related to OCRmyPDF helps fund this open source project and all inquiries are appreciated.
 
 
 Hot (watched) folders

docs/cookbook.rst

@@ -11,7 +11,7 @@ ocrmypdf has built-in help.
 
 .. code-block:: bash
 
-	ocrmypdf --help
+    ocrmypdf --help
 
 
 Add an OCR layer and convert to PDF/A
@@ -19,21 +19,21 @@ Add an OCR layer and convert to PDF/A
 
 .. code-block:: bash
 
-	ocrmypdf input.pdf output.pdf
+    ocrmypdf input.pdf output.pdf
 
 Add an OCR layer and output a standard PDF
 """"""""""""""""""""""""""""""""""""""""""
 
 .. code-block:: bash
 
-	ocrmypdf --output-type pdf input.pdf output.pdf
+    ocrmypdf --output-type pdf input.pdf output.pdf
 
 Create a PDF/A with all color and grayscale images converted to JPEG
 """"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
 
 .. code-block:: bash
 
-	ocrmypdf --output-type pdfa --pdfa-image-compression jpeg input.pdf output.pdf
+    ocrmypdf --output-type pdfa --pdfa-image-compression jpeg input.pdf output.pdf
 
 Modify a file in place
 """"""""""""""""""""""
@@ -42,7 +42,7 @@ The file will only be overwritten if OCRmyPDF is successful.
 
 .. code-block:: bash
 
-	ocrmypdf myfile.pdf myfile.pdf
+    ocrmypdf myfile.pdf myfile.pdf
 
 Correct page rotation
 """""""""""""""""""""
@@ -51,7 +51,7 @@ OCR will attempt to automatic correct the rotation of each page. This can help f
 
 .. code-block:: bash
 
-	ocrmypdf --rotate-pages myfile.pdf myfile.pdf
+    ocrmypdf --rotate-pages myfile.pdf myfile.pdf
 
 You can increase (decrease) the parameter ``--rotate-pages-threshold`` to make page rotation more (less) aggressive.
 
@@ -63,8 +63,8 @@ By default OCRmyPDF assumes the document is English.
 
 .. code-block:: bash
 
-	ocrmypdf -l fre LeParisien.pdf LeParisien.pdf
-	ocrmypdf -l eng+fre Bilingual-English-French.pdf Bilingual-English-French.pdf
+    ocrmypdf -l fre LeParisien.pdf LeParisien.pdf
+    ocrmypdf -l eng+fre Bilingual-English-French.pdf Bilingual-English-French.pdf
 
 Language packs must be installed for all languages specified. See :ref:`Installing additional language packs <lang-packs>`.
 
@@ -72,19 +72,11 @@ Language packs must be installed for all languages specified. See :ref:`Installi
 Produce PDF and text file containing OCR text
 """""""""""""""""""""""""""""""""""""""""""""
 
-This produces a file named "output.pdf" and a companion text file named "output.txt". The ``pdftotext`` program from 
-`Poppler <https://poppler.freedesktop.org/>`_ is used to extract text from the finished PDF.
-
+This produces a file named "output.pdf" and a companion text file named "output.txt".
 
 .. code-block:: bash
 
-	ocrmypdf input.pdf - | tee output.pdf | pdftotext - output.txt
-
-.. note::
-
-	To get pdftotext, Debian/Ubuntu users may ``apt-get install poppler-utils`` 
-	and macOS users may ``brew install poppler`` respectively.
-
+    ocrmypdf --sidecar output.txt input.pdf output.pdf
 
 OCR images, not PDFs
 --------------------
@@ -93,7 +85,7 @@ Use a program like `img2pdf <https://gitlab.mister-muffin.de/josch/img2pdf>`_ to
 
 .. code-block:: bash
 
-	img2pdf my-images*.jpg | ocrmypdf - myfile.pdf
+    img2pdf my-images*.jpg | ocrmypdf - myfile.pdf
 
 ``img2pdf`` also has features to control the position of images on a page, if desired.
 
@@ -101,7 +93,7 @@ For convenience, OCRmyPDF can convert single images to PDFs on its own. If the r
 
 .. code-block:: bash
 
-	ocrmypdf --image-dpi 300 image.png myfile.pdf
+    ocrmypdf --image-dpi 300 image.png myfile.pdf
 
 If you have multiple images, you must use ``img2pdf`` to convert the images to PDF.
 
@@ -113,7 +105,7 @@ You can also use Tesseract 3.04+ directly to convert single page images or multi
 
 .. code-block:: bash
 
-	tesseract my-image.jpg output-prefix pdf 
+    tesseract my-image.jpg output-prefix pdf
 
 Image processing
 ----------------
@@ -132,11 +124,11 @@ OCRmyPDF perform some image processing on each page of a PDF, if desired.  The s
 
 .. note::
 
-	In many cases image processing will rasterize PDF pages as images, potentially losing quality.
+    In many cases image processing will rasterize PDF pages as images, potentially losing quality.
 
 .. warning::
 
-	``--clean-final`` and ``-remove-background`` may leave undesirable visual artifacts in some images where their algorithms have shortcomings. Files should be visually reviewed after using these options.
+    ``--clean-final`` and ``-remove-background`` may leave undesirable visual artifacts in some images where their algorithms have shortcomings. Files should be visually reviewed after using these options.
 
 
 OCR and correct document skew (crooked scan)
@@ -146,13 +138,46 @@ Deskew:
 
 .. code-block:: bash
 
-	ocrmypdf --deskew input.pdf output.pdf
+    ocrmypdf --deskew input.pdf output.pdf
 
 Image processing commands can be combined. The order in which options are given does not matter. OCRmyPDF always applies the steps of the image processing pipeline in the same order (rotate, remove background, deskew, clean).
 
 .. code-block:: bash
 
-	ocrmypdf --deskew --clean --rotate-pages input.pdf output.pdf
+    ocrmypdf --deskew --clean --rotate-pages input.pdf output.pdf
+
+
+Don't actually OCR my PDF
+"""""""""""""""""""""""""
+
+If you set ``--tesseract-timeout 0`` OCRmyPDF will apply its image processing without performing OCR, if all you want to is to apply image processing or PDF/A conversion.
+
+.. code-block:: bash
+
+    ocrmypdf --tesseract-timeout=0 --remove-background input.pdf output.pdf
+
+
+Redo OCR
+""""""""
+
+To redo OCR on a file OCRed with other OCR software or a previous version of OCRmyPDF and/or Tesseract, you may use the ``--force-ocr`` argument. Normally, OCRmyPDF does not modify files that already appear to contain OCR text.
+
+.. code-block:: bash
+
+    ocrmypdf --force-ocr input.pdf output.pdf
+
+Note that the method above will force rasterization of all pages, potentially reducing quality or losing vector content. 
+
+To ensure quality is preserved, one could extract all of the images and rebuild the PDF for a lossless transformation. This recipe does not work when PDFs contain multiple images per page, as many do in practice. It will also lose any page rotation information.
+
+.. code-block:: bash
+
+    pdfimages -all old-ocr.pdf prefix  # extract all images
+    img2pdf -o temp.pdf prefix*        # construct new PDF from the images
+    # review the new PDF to ensure it visually matches the old one
+    ocrmypdf --output-type pdf temp.pdf new-ocr.pdf
+
+``--output-type pdf`` is used here to avoid using Ghostscript which will also rasterize images.
 
 
 Improving OCR quality

docs/index.rst

@@ -9,7 +9,7 @@ OCRmyPDF documentation
 OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to
 be searched.
 
-PDFs are the best format for scanned documents.  Unfortunately, PDFs can be difficult to work with. OCRmyPDF makes it easy to apply image processing and OCR to existing PDFs.
+PDF is the best format for storing and exchanging scanned documents.  Unfortunately, PDFs can be difficult to modify. OCRmyPDF makes it easy to apply image processing and OCR to existing PDFs.
 
 .. toctree::
    :maxdepth: 1

docs/installation.rst

@@ -15,18 +15,20 @@ Users of Debian 9 ("stretch") or later or Ubuntu 16.10 or later may simply
 Installing on macOS
 -------------------
 
-A `Homebrew <https://brew.sh>`_ tap is available for macOS:
+OCRmyPDF is now a standard `Homebrew <https://brew.sh>`_ formula. To install on macOS:
 
 .. code-block:: bash
 
-    brew tap jbarlow83/ocrmypdf
     brew install ocrmypdf
 
-
-.. warning::
+.. note::
 
     Users who previously installed OCRmyPDF on macOS using ``pip install ocrmypdf`` should remove the pip version (``pip3 uninstall ocrmypdf``) before switching to the Homebrew version.
 
+.. note::
+
+    Users who previously installed OCRmyPDF from the private tap should switch to the mainline version (``brew untap jbarlow83/ocrmypdf``) and install from there.
+
 .. _Docker-install:
 
 Installing the Docker image
@@ -105,6 +107,12 @@ In this worked example, the current working directory contains an input file cal
 
 Note that ``ocrmypdf`` has its own separate ``-v VERBOSITYLEVEL`` argument to control debug verbosity. All Docker arguments should before the ``ocrmypdf`` image name and all arguments to ``ocrmypdf`` should be listed after.
 
+In some environments the permissions associated with Docker can be complex to configure. The process that executes Docker may end up not having the permissions to write the specified file system. In that case one can stream the file into and out of the Docker process and avoid all permission hassles, using ``-`` as the input and output filename:
+
+.. code-block:: bash
+
+    docker run --rm -i   ocrmypdf <other arguments to ocrmypdf> - - <input.pdf >output.pdf
+
 For convenience, a shell alias can hide the docker command:
 
 .. code-block:: bash
@@ -119,6 +127,9 @@ Or in the wonderful `fish shell <https://fishshell.com/>`_:
     alias ocrmypdf 'docker run --rm -v (pwd):/home/docker ocrmypdf'
     funcsave ocrmypdf
 
+.. note::
+
+    The ocrmypdf Docker containers are designed to be used for a single OCR job. The ``docker run --rm`` argument tells Docker to delete temporary storage associated with container when it is done executing.
 
 Manual installation on macOS
 ----------------------------
@@ -165,7 +176,13 @@ Update the homebrew pip and install Pillow:
     pip3 install --upgrade pip
     pip3 install --upgrade pillow
 
-You can then install OCRmyPDF from PyPI:
+You can then install OCRmyPDF from PyPI, for the current user:
+
+.. code-block:: bash
+
+    pip3 install --user ocrmypdf
+
+or system-wide:
 
 .. code-block:: bash
 
@@ -194,8 +211,13 @@ No package is currently available for Ubuntu 16.04, but you can install the depe
         python3-pip \
         python3-cffi
 
-If you wish install OCRmyPDF to the system Python, then install as follows (note this installs new packages
-into your system Python, which could interfere with other programs):
+If you wish install OCRmyPDF for the current user:
+
+.. code-block:: bash
+
+    pip3 install --user ocrmypdf
+
+Alternately, system-wide. Note that this may modify the system Python environment:
 
 .. code-block:: bash
 
@@ -214,15 +236,8 @@ follow these steps.
 Installing on Ubuntu 14.04 LTS
 ------------------------------
 
-Installing on Ubuntu 14.04 LTS (trusty) is more difficult than some other options, because of bugs in Python package installation and because OCRmyPDF depends on some packages newer than are available in the main distribution.
-
-Add new "apt" repositories needed for backports of Ghostscript 9.16, libav-11 (for unpaper 6.1) and Tesseract 4.00 (alpha). This will replace Ghostscript and Tesseract 3.x on your system. If you prefer to not modify your system in this matter, consider using a Docker container.
-
-.. code-block:: bash
-
-    sudo add-apt-repository ppa:vshn/ghostscript -y
-    sudo add-apt-repository ppa:heyarje/libav-11 -y
-    sudo add-apt-repository ppa:alex-p/tesseract-ocr
+Installing on Ubuntu 14.04 LTS (trusty) is more difficult than some other options,
+because it is older and does not provide ``pip``.
 
 Update apt-get:
 
@@ -239,39 +254,40 @@ Install system dependencies:
         zlib1g-dev \
         libjpeg-dev \
         libffi-dev \
-        libavformat56 libavcodec56 libavutil54 \
+        qpdf
+
+We will need backports of Ghostscript 9.16, libav-11 (for unpaper 6.1),
+Tesseract 4.00 (alpha), and Python 3.6. This will replace Ghostscript and
+Tesseract 3.x on your system. Python 3.6 will be installed alongside the system
+Python 3.
+
+If you prefer to not modify your system in this matter, consider using a Docker container.
+
+.. code-block:: bash
+
+    sudo add-apt-repository ppa:vshn/ghostscript -y
+    sudo add-apt-repository ppa:heyarje/libav-11 -y
+    sudo add-apt-repository ppa:alex-p/tesseract-ocr -y
+    sudo add-apt-repository ppa:jonathonf/python-3.6 -y
+
+    sudo apt-get update
+
+    sudo apt-get install \
+        python3.6 \
         ghostscript \
-        qpdf \
-        python3-pip \
-        python3-pil \
-        python3-pytest \
-        python3-reportlab \
-        python3-wheel \
-        python3-venv \
         tesseract-ocr \
-        tesseract-ocr-eng
+        tesseract-ocr-eng \
+        libavformat56 libavcodec56 libavutil54 \
+        wget
 
-If you wish install OCRmyPDF to the system Python, then install as follows (note this installs new packages
-into your system Python, which could interfere with other programs):
+Now we need to install ``pip`` and let it install ocrmypdf:
 
 .. code-block:: bash
 
-    sudo pip3 install ocrmypdf
+    wget -O - -o /dev/null https://bootstrap.pypa.io/get-pip.py | python3.6
+    pip3.6 install ocrmypdf
 
-If you wish to install OCRmyPDF to a virtual environment to isolate the system Python, you can
-follow these steps.  This includes a workaround `for a known, unresolved issue in Ubuntu 14.04's ensurepip
-package <http://www.thefourtheye.in/2014/12/Python-venv-problem-with-ensurepip-in-Ubuntu.html>`_:
-
-.. code-block:: bash
-
-    sudo apt-get install python3-venv
-    python3 -m venv venv-ocrmypdf --without-pip
-    source venv-ocrmypdf/bin/activate
-    wget -O - -o /dev/null https://bootstrap.pypa.io/get-pip.py | python
-    deactivate
-    python3 -m venv --system-site-packages venv-ocrmypdf
-    source venv-ocrmypdf/bin/activate
-    pip install ocrmypdf
+The ``wget`` command will download a program and run it.
 
 These installation instructions omit the optional dependency ``unpaper``, which is only available at version 0.4.2 in Ubuntu 14.04. The author could not find a backport of ``unpaper``, and created a .deb package to do the job of installing unpaper 6.1 (for x86 64-bit only):
 

docs/introduction.rst

@@ -16,7 +16,7 @@ OCRmyPDF uses `Tesseract <https://github.com/tesseract-ocr/tesseract>`_, the bes
 About PDFs
 ----------
 
-PDFs are page description files that attempts to preserve a layout exactly. They can contain `vector graphic files <http://vector-conversions.com/vectorizing/raster_vs_vector.html>`_ that can contain raster objects such as scanned images. Because PDFs can contain multiple pages (unlike many image formats) and can contain fonts and text, it is a good formats for exchanging scanned documents.
+PDFs are page description files that attempts to preserve a layout exactly. They  contain `vector graphics <http://vector-conversions.com/vectorizing/raster_vs_vector.html>`_ that can contain raster objects such as scanned images. Because PDFs can contain multiple pages (unlike many image formats) and can contain fonts and text, it is a good formats for exchanging scanned documents.
 
 .. image:: bitmap_vs_svg.svg
 
@@ -42,15 +42,13 @@ PDF/A has a few drawbacks.  Some PDF viewers include an alert that the file is a
 What OCRmyPDF does
 ------------------
 
-OCRmyPDF analyzes each page of a PDF to determine the colorspace and resolution (DPI) needed to capture all of the information on that page without losing content.  It uses `Ghostscript <http://ghostscript.com/>`_ to rasterize the page, and then performs on OCR on the rasterized image.  It is not enough to simply extract the images from each page and run OCR on them individually.  Of course one could use Ghostscript or another PDF rasterizer and then pass the image to Tesseract.  OCRmyPDF automates this process and produces a minimally changed output file that contains the same information, colorspace and resolution.
+OCRmyPDF analyzes each page of a PDF to determine the colorspace and resolution (DPI) needed to capture all of the information on that page without losing content.  It uses `Ghostscript <http://ghostscript.com/>`_ to rasterize the page, and then performs on OCR on the rasterized image to create an OCR "layer". The layer is then grafted back onto the original PDF.
 
-The Tesseract OCR engine can output 'hOCR' files, which are XML files that contain a description of the text it found on the page.  OCRmyPDF will render a new PDF that contains only the hidden text layer, and merge this with the original page.
+While one can use a program like Ghostscript or ImageMagick to get an image and put the image through Tesseract, that actually creates a new PDF and many details may be lost. OCRmyPDF can produce a minimally changed PDF as output.
 
-Alternately, OCRmyPDF can use the Tesseract OCR engine to directly output PDFs for each page, then merge them.
+OCRmyPDF also some image processing options like deskew which improve the appearance of files and quality of OCR. When these are used, the OCR layer is grafted onto the processed image instead.
 
-By default, OCRmyPDF will convert the file to a PDF/A.  This behavior can be disabled with the ``--output-type pdf`` argument.
-
-Depending on the settings selected, OCRmyPDF may "graft" the OCR layer into the existing PDF, or reconstruct a visually equivalent new PDF.
+By default, OCRmyPDF produces archival PDFs – PDF/A, which are a stricter subset of PDF features designed for long term archives. If regular PDFs are desired, this can be disabled with ``--output-type pdf``.
 
 
 Why you shouldn't do this manually

docs/languages.rst

@@ -33,6 +33,34 @@ Docker users
 
 Users of the Docker image may use the alternative :ref:`"polyglot" container <docker-polyglot>` which includes all languages.
 
+Adding individual language packs to a Docker image
+""""""""""""""""""""""""""""""""""""""""""""""""""
+
+If you wish to add a single language pack, you could do the following:
+
+* Download the desired ``.trainedata`` file from the `tessdata <https://github.com/tesseract-ocr/tessdata>`_ repository. Let's use Hebrew in this example (``heb.traineddata``)
+
+* Copy the file to ``/home/user/downloads/heb.traineddata``.
+
+* Create a new container based on the ocrmypdf-tess4 image and jump into it with a terminal:
+
+.. code-block:: bash
+
+	host$ docker run  -v /home/user/downloads:/home/docker -it --entrypoint /bin/bash ocrmypdf-tess4
+
+* Put the file where Tesseract expects it:
+
+.. code-block:: bash
+
+	docker$ cp /home/docker/heb.traineddata /usr/share/tesseract-ocr/tessdata
+
+* Note the container id, and save it as a new image (in this example, ``ocrmypdf-tess4-heb``)
+
+.. code-block:: bash
+
+    host$ docker commit <container_id> ocrmypdf-tess4-heb
+
+
 Known limitations
 -----------------
 

docs/release_notes.rst

@@ -1,7 +1,127 @@
 Release notes
 =============
 
-OCRmyPDF uses `semantic versioning <http://semver.org/>`_.
+OCRmyPDF uses `semantic versioning <http://semver.org/>`_ for its command line interface.
+
+The OCRmyPDF package itself does not contain a public API, although it is fairly stable and breaking changes are usually timed with a major release. A future release will clearly define the stable public API.
+
+v5.6.3
+------
+
+-   Suppress two debug messages that were too verbose
+
+
+v5.6.2
+------
+
+-   Development branch accidentally tagged as release. Do not use.
+
+
+v5.6.1
+------
+
+-   Fix issue #219: change how the final output file is created to avoid triggering permission errors when the output is a special file such as ``/dev/null``
+-   Fix test suite failures due to a qpdf 8.0.0 regression and Python 3.5's handling of symlink
+-   The "encrypted PDF" error message was different depending on the type of PDF encryption. Now a single clear message appears for all types of PDF encryption.
+-   ocrmypdf is now in Homebrew. Homebrew users are advised to the version of ocrmypdf in the official homebrew-core formulas rather than the private tap.
+-   Some linting
+
+
+v5.6.0
+------
+
+-   Fix issue #216: preserve "text as curves" PDFs without rasterizing file
+-   Related to the above, messages about rasterizing are more consistent
+-   For consistency versions minor releases will now get the trailing .0 they always should have had.
+
+
+v5.5
+----
+
+-   Add new argument ``--max-image-mpixels``. Pillow 5.0 now raises an exception when images may be decompression bombs. This argument can be used to override the limit Pillow sets.
+-   Fix output page cropped when using the sandwich renderer and OCR is skipped on a rotated and image-processed page
+-   A warning is now issued when old versions of Ghostscript are used in cases known to cause issues with non-Latin characters
+-   Fix a few parameter validation checks for ``-output-type pdfa-1`` and ``pdfa-2`` 
+
+
+v5.4.4
+------
+
+-   Fix issue #181: fix final merge failure for PDFs with more pages than the system file handle limit (``ulimit -n``)
+-   Fix issue #200: an uncommon syntax for formatting decimal numbers in a PDF would cause qpdf to issue a warning, which ocrmypdf treated as an error. Now this the warning is relayed.
+-   Fix an issue where intermediate PDFs would be created at version 1.3 instead of the version of the original file. It's possible but unlikely this had side effects.
+-   A warning is now issued when older versions of qpdf are used since issues like #200 cause qpdf to infinite-loop
+-   Address issue #140: if Tesseract outputs invalid UTF-8, escape it and print its message instead of aborting with a Unicode error 
+-   Adding previously unlisted setup requirement, pytest-runner
+-   Update documentation: fix an error in the example script for Synology with Docker images, improved security guidance, advised ``pip install --user``
+
+
+v5.4.3
+------
+
+-   If a subprocess fails to report its version when queried, exit cleanly with an error instead of throwing an exception
+-   Added test to confirm that the system locale is Unicode-aware and fail early if it's not
+-   Clarified some copyright information
+-   Updated pinned requirements.txt so the homebrew formula captures more recent versions
+
+
+v5.4.2
+------
+
+-   Fixed a regression from v5.4.1 that caused sidecar files to be created as empty files
+
+
+v5.4.1
+------
+
+-   Add workaround for Tesseract v4.00alpha crash when trying to obtain orientation and the latest language packs are installed
+
+
+v5.4
+----
+
+-   Change wording of a deprecation warning to improve clarity
+-   Added option to generate PDF/A-1b output if desired (``--output-type pdfa-1``); default remains PDF/A-2b generation
+-   Update documentation
+
+
+v5.3.3
+------
+
+-   Fixed missing error message that should occur when trying to force ``--pdf-renderer sandwich`` on old versions of Tesseract
+-   Update copyright information in test files
+-   Set system ``LANG`` to UTF-8 in Dockerfiles to avoid UTF-8 encoding errors
+
+
+v5.3.2
+------
+
+-   Fixed a broken test case related to language packs
+
+
+v5.3.1
+------
+
+-   Fixed wrong return code given for missing Tesseract language packs
+-   Fixed "brew audit" crashing on Travis when trying to auto-brew
+
+
+v5.3
+----
+
+-   Added ``--user-words`` and ``--user-patterns`` arguments which are forwarded to Tesseract OCR as words and regular expressions respective to use to guide OCR. Supplying a list of subject-domain words should assist Tesseract with resolving words. (#165)
+-   Using a non Latin-1 language with the "hocr" renderer now warns about possible OCR quality and recommends workarounds (#176)
+-   Output file path added to error message when that location is not writable (#175)
+-   Otherwise valid PDFs with leading whitespace at the beginning of the file are now accepted
+
+
+v5.2
+----
+
+-   When using Tesseract 3.05.01 or newer, OCRmyPDF will select the "sandwich" PDF renderer by default, unless another PDF renderer is specified with the ``--pdf-renderer`` argument. The previous behavior was to select ``--pdf-renderer=hocr``.
+-   The "tesseract" PDF renderer is now deprecated, since it can cause problems with Ghostscript on Tesseract 3.05.00
+-   The "tess4" PDF renderer has been renamed to "sandwich". "tess4" is now a deprecated alias for "sandwich".
+
 
 v5.1
 ----
@@ -12,7 +132,7 @@ v5.1
 v5.0.1
 ------
 
--   Fixed issue #169, exception due to failure to create sidecar text files on some versions of Tesseract 3.04, including the jbarlow83/ocrmypdf image
+-   Fixed issue #169, exception due to failure to create sidecar text files on some versions of Tesseract 3.04, including the jbarlow83/ocrmypdf Docker image
 
 
 v5.0
@@ -20,9 +140,9 @@ v5.0
 
 -   Backward incompatible changes
 
-    + Support for Python 3.4 dropped. Python 3.5 is now required.
-    + Support for Tesseract 3.02 and 3.03 dropped. Tesseract 3.04 or newer is required. Tesseract 4.00 (alpha) is supported.
-    + The OCRmyPDF.sh script was removed.
+     + Support for Python 3.4 dropped. Python 3.5 is now required.
+     + Support for Tesseract 3.02 and 3.03 dropped. Tesseract 3.04 or newer is required. Tesseract 4.00 (alpha) is supported.
+     + The OCRmyPDF.sh script was removed.
 
 -   Add a new feature, ``--sidecar``, which allows creating "sidecar" text files which contain the OCR results in plain text. These OCR text is more reliable than extracting text from PDFs. Closes #126.
 -   New feature: ``--pdfa-image-compression``, which allows overriding Ghostscript's lossy-or-lossless image encoding heuristic and making all images JPEG encoded or lossless encoded as desired. Fixes #163.
@@ -56,242 +176,242 @@ v4.5.4
 v4.5.3
 ------
 
--  Added a workaround for Ghostscript 9.21 and probably earlier versions would fail with the error message "VMerror -25", due to a Ghostscript bug in XMP metadata handling
--  High Unicode characters (U+10000 and up) are no longer accepted for setting metadata on the command line, as Ghostscript may not handle them correctly.
--  Fixed an issue where the ``tess4`` renderer would duplicate content onto output pages if tesseract failed or timed out
--  Fixed ``tess4`` renderer not recognized when lossless reconstruction is possible
+-   Added a workaround for Ghostscript 9.21 and probably earlier versions would fail with the error message "VMerror -25", due to a Ghostscript bug in XMP metadata handling
+-   High Unicode characters (U+10000 and up) are no longer accepted for setting metadata on the command line, as Ghostscript may not handle them correctly.
+-   Fixed an issue where the ``tess4`` renderer would duplicate content onto output pages if tesseract failed or timed out
+-   Fixed ``tess4`` renderer not recognized when lossless reconstruction is possible
 
 
 v4.5.2
 ------
 
--  Fix issue #147. ``--pdf-renderer tess4 --clean`` will produce an oversized page containing the original image in the bottom left corner, due to loss DPI information.
--  Make "using Tesseract 4.0" warning less ominous
--  Set up machinery for homebrew OCRmyPDF tap
+-   Fix issue #147. ``--pdf-renderer tess4 --clean`` will produce an oversized page containing the original image in the bottom left corner, due to loss DPI information.
+-   Make "using Tesseract 4.0" warning less ominous
+-   Set up machinery for homebrew OCRmyPDF tap
 
 
 v4.5.1
 ------
 
--  Fix issue #137, proportions of images with a non-square pixel aspect ratio would be distorted in output for ``--force-ocr`` and some other combinations of flags
+-   Fix issue #137, proportions of images with a non-square pixel aspect ratio would be distorted in output for ``--force-ocr`` and some other combinations of flags
 
 
 v4.5
 ----
 
--  Exotic PDFs containing "Form XObjects" are now supported (issue #134; PDF reference manual 8.10), and images they contain are taken into account when determining the resolution for rasterizing
--  The Tesseract 4 Docker image no longer includes all languages, because it took so long to build something would tend to fail
--  OCRmyPDF now warns about using ``--pdf-renderer tesseract`` with Tesseract 3.04 or lower due to issues with Ghostscript corrupting the OCR text in these cases
+-   Exotic PDFs containing "Form XObjects" are now supported (issue #134; PDF reference manual 8.10), and images they contain are taken into account when determining the resolution for rasterizing
+-   The Tesseract 4 Docker image no longer includes all languages, because it took so long to build something would tend to fail
+-   OCRmyPDF now warns about using ``--pdf-renderer tesseract`` with Tesseract 3.04 or lower due to issues with Ghostscript corrupting the OCR text in these cases
 
 
 v4.4.2
 ------
 
--  The Docker images (ocrmypdf, ocrmypdf-polyglot, ocrmypdf-tess4) are now based on Ubuntu 16.10 instead of Debian stretch
+-   The Docker images (ocrmypdf, ocrmypdf-polyglot, ocrmypdf-tess4) are now based on Ubuntu 16.10 instead of Debian stretch
 
-   + This makes supporting the Tesseract 4 image easier
-   + This could be a disruptive change for any Docker users who built customized these images with their own changes, and made those changes in a way that depends on Debian and not Ubuntu
+    + This makes supporting the Tesseract 4 image easier
+    + This could be a disruptive change for any Docker users who built customized these images with their own changes, and made those changes in a way that depends on Debian and not Ubuntu
 
--  OCRmyPDF now prevents running the Tesseract 4 renderer with Tesseract 3.04, which was permitted in v4.4 and v4.4.1 but will not work
+-   OCRmyPDF now prevents running the Tesseract 4 renderer with Tesseract 3.04, which was permitted in v4.4 and v4.4.1 but will not work
 
 
 v4.4.1
 ------
 
--  To prevent a `TIFF output error <https://github.com/python-pillow/Pillow/issues/2206>`_ caused by img2pdf >= 0.2.1 and Pillow <= 3.4.2, dependencies have been tightened
--  The Tesseract 4.00 simultaneous process limit was increased from 1 to 2, since it was observed that 1 lowers performance
--  Documentation improvements to describe the ``--tesseract-config`` feature 
--  Added test cases and fixed error handling for ``--tesseract-config``
--  Tweaks to setup.py to deal with issues in the v4.4 release
+-   To prevent a `TIFF output error <https://github.com/python-pillow/Pillow/issues/2206>`_ caused by img2pdf >= 0.2.1 and Pillow <= 3.4.2, dependencies have been tightened
+-   The Tesseract 4.00 simultaneous process limit was increased from 1 to 2, since it was observed that 1 lowers performance
+-   Documentation improvements to describe the ``--tesseract-config`` feature 
+-   Added test cases and fixed error handling for ``--tesseract-config``
+-   Tweaks to setup.py to deal with issues in the v4.4 release
 
 v4.4
 ----
 
--  Tesseract 4.00 is now supported on an experimental basis.
+-   Tesseract 4.00 is now supported on an experimental basis.
 
-   +  A new rendering option ``--pdf-renderer tess4`` exploits Tesseract 4's new text-only output PDF mode. See the documentation on PDF Renderers for details.
-   +  The ``--tesseract-oem`` argument allows control over the Tesseract 4 OCR engine mode (tesseract's ``--oem``). Use ``--tesseract-oem 2`` to enforce the new LSTM mode.
-   +  Fixed poor performance with Tesseract 4.00 on Linux
+    +  A new rendering option ``--pdf-renderer tess4`` exploits Tesseract 4's new text-only output PDF mode. See the documentation on PDF Renderers for details.
+    +  The ``--tesseract-oem`` argument allows control over the Tesseract 4 OCR engine mode (tesseract's ``--oem``). Use ``--tesseract-oem 2`` to enforce the new LSTM mode.
+    +  Fixed poor performance with Tesseract 4.00 on Linux
 
--  Fixed an issue that caused corruption of output to stdout in some cases
--  Removed test for Pillow JPEG and PNG support, as the minimum supported version of Pillow now enforces this
--  OCRmyPDF now tests that the intended destination file is writable before proceeding
--  The test suite now requires ``pytest-helpers-namespace`` to run (but not install)
--  Significant code reorganization to make OCRmyPDF re-entrant and improve performance. All changes should be backward compatible for the v4.x series.
+-   Fixed an issue that caused corruption of output to stdout in some cases
+-   Removed test for Pillow JPEG and PNG support, as the minimum supported version of Pillow now enforces this
+-   OCRmyPDF now tests that the intended destination file is writable before proceeding
+-   The test suite now requires ``pytest-helpers-namespace`` to run (but not install)
+-   Significant code reorganization to make OCRmyPDF re-entrant and improve performance. All changes should be backward compatible for the v4.x series.
 
-   + However, OCRmyPDF's dependency "ruffus" is not re-entrant, so no Python API is available. Scripts should continue to use the command line interface.
+    + However, OCRmyPDF's dependency "ruffus" is not re-entrant, so no Python API is available. Scripts should continue to use the command line interface.
 
 
 v4.3.5
 ------
 
--  Update documentation to confirm Python 3.6.0 compatibility. No code changes were needed, so many earlier versions are likely supported.
+-   Update documentation to confirm Python 3.6.0 compatibility. No code changes were needed, so many earlier versions are likely supported.
 
 
 v4.3.4
 ------
 
--  Fixed "decimal.InvalidOperation: quantize result has too many digits" for high DPI images
+-   Fixed "decimal.InvalidOperation: quantize result has too many digits" for high DPI images
 
 
 v4.3.3
 ------
 
--  Fixed PDF/A creation with Ghostscript 9.20 properly
--  Fixed an exception on inline stencil masks with a missing optional parameter
+-   Fixed PDF/A creation with Ghostscript 9.20 properly
+-   Fixed an exception on inline stencil masks with a missing optional parameter
 
 
 v4.3.2
 ------
 
--  Fixed a PDF/A creation issue with Ghostscript 9.20 (note: this fix did not actually work)
+-   Fixed a PDF/A creation issue with Ghostscript 9.20 (note: this fix did not actually work)
 
 
 v4.3.1
 ------
 
--  Fixed an issue where pages produced by the "hocr" renderer after a Tesseract timeout would be rotated incorrectly if the input page was rotated with a /Rotate marker
--  Fixed a file handle leak in LeptonicaErrorTrap that would cause a "too many open files" error for files around hundred pages of pages long when ``--deskew`` or ``--remove-background`` or other Leptonica based image processing features were in use, depending on the system value of ``ulimit -n``
--  Ability to specify multiple languages for multilingual documents is now advertised in documentation
--  Reduced the file sizes of some test resources
--  Cleaned up debug output
--  Tesseract caching in test cases is now more cautious about false cache hits and reproducing exact output, not that any problems were observed
+-   Fixed an issue where pages produced by the "hocr" renderer after a Tesseract timeout would be rotated incorrectly if the input page was rotated with a /Rotate marker
+-   Fixed a file handle leak in LeptonicaErrorTrap that would cause a "too many open files" error for files around hundred pages of pages long when ``--deskew`` or ``--remove-background`` or other Leptonica based image processing features were in use, depending on the system value of ``ulimit -n``
+-   Ability to specify multiple languages for multilingual documents is now advertised in documentation
+-   Reduced the file sizes of some test resources
+-   Cleaned up debug output
+-   Tesseract caching in test cases is now more cautious about false cache hits and reproducing exact output, not that any problems were observed
 
 
 v4.3
 ----
 
--  New feature ``--remove-background`` to detect and erase the background of color and grayscale images
--  Better documentation
--  Fixed an issue with PDFs that draw images when the raster stack depth is zero 
--  ocrmypdf can now redirect its output to stdout for use in a shell pipeline
+-   New feature ``--remove-background`` to detect and erase the background of color and grayscale images
+-   Better documentation
+-   Fixed an issue with PDFs that draw images when the raster stack depth is zero 
+-   ocrmypdf can now redirect its output to stdout for use in a shell pipeline
 
-   +  This does not improve performance since temporary files are still used for buffering
-   +  Some output validation is disabled in this mode
+    +  This does not improve performance since temporary files are still used for buffering
+    +  Some output validation is disabled in this mode
 
 v4.2.5
 ------
 
--  Fixed an issue (#100) with PDFs that omit the optional /BitsPerComponent parameter on images
--  Removed non-free file milk.pdf
+-   Fixed an issue (#100) with PDFs that omit the optional /BitsPerComponent parameter on images
+-   Removed non-free file milk.pdf
 
 
 v4.2.4
 ------
 
--  Fixed an error (#90) caused by PDFs that use stencil masks properly
--  Fixed handling of PDFs that try to draw images or stencil masks without properly setting up the graphics state (such images are now ignored for the purposes of calculating DPI)
+-   Fixed an error (#90) caused by PDFs that use stencil masks properly
+-   Fixed handling of PDFs that try to draw images or stencil masks without properly setting up the graphics state (such images are now ignored for the purposes of calculating DPI)
 
 v4.2.3
 ------
 
--  Fixed an issue with PDFs that store page rotation (/Rotate) in an indirect object
--  Integrated a few fixes to simplify downstream packaging (Debian)
+-   Fixed an issue with PDFs that store page rotation (/Rotate) in an indirect object
+-   Integrated a few fixes to simplify downstream packaging (Debian)
 
-   +  The test suite no longer assumes it is installed
-   +  If running Linux, skip a test that passes Unicode on the command line
+    +  The test suite no longer assumes it is installed
+    +  If running Linux, skip a test that passes Unicode on the command line
 
--  Added a test case to check explicit masks and stencil masks
--  Added a test case for indirect objects and linearized PDFs
--  Deprecated the OCRmyPDF.sh shell script
+-   Added a test case to check explicit masks and stencil masks
+-   Added a test case for indirect objects and linearized PDFs
+-   Deprecated the OCRmyPDF.sh shell script
 
 
 v4.2.2
 ------
 
--  Improvements to documentation
+-   Improvements to documentation
 
 
 v4.2.1
 ------
 
--  Fixed an issue where PDF pages that contained stencil masks would report an incorrect DPI and cause Ghostscript to abort
--  Implemented stdin streaming
+-   Fixed an issue where PDF pages that contained stencil masks would report an incorrect DPI and cause Ghostscript to abort
+-   Implemented stdin streaming
 
 
 v4.2
 ----
 
--  ocrmypdf will now try to convert single image files to PDFs if they are provided as input (#15)
+-   ocrmypdf will now try to convert single image files to PDFs if they are provided as input (#15)
 
-   +  This is a basic convenience feature. It only supports a single image and always makes the image fill the whole page.
-   +  For better control over image to PDF conversion, use ``img2pdf`` (one of ocrmypdf's dependencies)
+    +  This is a basic convenience feature. It only supports a single image and always makes the image fill the whole page.
+    +  For better control over image to PDF conversion, use ``img2pdf`` (one of ocrmypdf's dependencies)
 
--  New argument ``--output-type {pdf|pdfa}`` allows disabling Ghostscript PDF/A generation
+-   New argument ``--output-type {pdf|pdfa}`` allows disabling Ghostscript PDF/A generation
 
-   +  ``pdfa`` is the default, consistent with past behavior
-   +  ``pdf`` provides a workaround for users concerned about the increase in file size from Ghostscript forcing JBIG2 images to CCITT and transcoding JPEGs
-   +  ``pdf`` preserves as much as it can about the original file, including problems that PDF/A conversion fixes
+    +  ``pdfa`` is the default, consistent with past behavior
+    +  ``pdf`` provides a workaround for users concerned about the increase in file size from Ghostscript forcing JBIG2 images to CCITT and transcoding JPEGs
+    +  ``pdf`` preserves as much as it can about the original file, including problems that PDF/A conversion fixes
 
--  PDFs containing images with "non-square" pixel aspect ratios, such as 200x100 DPI, are now handled and converted properly (fixing a bug that caused to be cropped)
--  ``--force-ocr`` rasterizes pages even if they contain no images
+-   PDFs containing images with "non-square" pixel aspect ratios, such as 200x100 DPI, are now handled and converted properly (fixing a bug that caused to be cropped)
+-   ``--force-ocr`` rasterizes pages even if they contain no images
 
-   +  supports users who want to use OCRmyPDF to reconstruct text information in PDFs with damaged Unicode maps (copy and paste text does not match displayed text)
-   +  supports reinterpreting PDFs where text was rendered as curves for printing, and text needs to be recovered
-   +  fixes issue #82
+    +  supports users who want to use OCRmyPDF to reconstruct text information in PDFs with damaged Unicode maps (copy and paste text does not match displayed text)
+    +  supports reinterpreting PDFs where text was rendered as curves for printing, and text needs to be recovered
+    +  fixes issue #82
 
--  Fixes an issue where, with certain settings, monochrome images in PDFs would be converted to 8-bit grayscale, increasing file size (#79)
--  Support for Ubuntu 12.04 LTS "precise" has been dropped in favor of (roughly) Ubuntu 14.04 LTS "trusty" 
+-   Fixes an issue where, with certain settings, monochrome images in PDFs would be converted to 8-bit grayscale, increasing file size (#79)
+-   Support for Ubuntu 12.04 LTS "precise" has been dropped in favor of (roughly) Ubuntu 14.04 LTS "trusty" 
 
-   +  Some Ubuntu "PPAs" (backports) are needed to make it work
-      
--  Support for some older dependencies dropped
+    +  Some Ubuntu "PPAs" (backports) are needed to make it work
 
-   +  Ghostscript 9.15 or later is now required (available in Ubuntu trusty with backports)
-   +  Tesseract 3.03 or later is now required (available in Ubuntu trusty)
+-   Support for some older dependencies dropped
 
--  Ghostscript now runs in "safer" mode where possible
+    +  Ghostscript 9.15 or later is now required (available in Ubuntu trusty with backports)
+    +  Tesseract 3.03 or later is now required (available in Ubuntu trusty)
+
+-   Ghostscript now runs in "safer" mode where possible
 
 v4.1.4
 ------
 
--  Bug fix: monochrome images with an ICC profile attached were incorrectly converted to full color images if lossless reconstruction was not possible due to other settings; consequence was increased file size for these images
+-   Bug fix: monochrome images with an ICC profile attached were incorrectly converted to full color images if lossless reconstruction was not possible due to other settings; consequence was increased file size for these images
 
 
 v4.1.3
 ------
 
--  More helpful error message for PDFs with version 4 security handler
--  Update usage instructions for Windows/Docker users
--  Fix order of operations for matrix multiplication (no effect on most users)
--  Add a few leptonica wrapper functions (no effect on most users)
+-   More helpful error message for PDFs with version 4 security handler
+-   Update usage instructions for Windows/Docker users
+-   Fix order of operations for matrix multiplication (no effect on most users)
+-   Add a few leptonica wrapper functions (no effect on most users)
 
 
 v4.1.2
 ------
 
--  Replace IEC sRGB ICC profile with Debian's sRGB (from icc-profiles-free) which is more compatible with the MIT license
--  More helpful error message for an error related to certain types of malformed PDFs
+-   Replace IEC sRGB ICC profile with Debian's sRGB (from icc-profiles-free) which is more compatible with the MIT license
+-   More helpful error message for an error related to certain types of malformed PDFs
 
 
 v4.1
 ----
 
--  ``--rotate-pages`` now only rotates pages when reasonably confidence in the orientation. This behavior can be adjusted with the new argument ``--rotate-pages-threshold``
--  Fixed problems in error checking if ``unpaper`` is uninstalled or missing at run-time
--  Fixed problems with "RethrownJobError" errors during error handling that suppressed the useful error messages
+-   ``--rotate-pages`` now only rotates pages when reasonably confidence in the orientation. This behavior can be adjusted with the new argument ``--rotate-pages-threshold``
+-   Fixed problems in error checking if ``unpaper`` is uninstalled or missing at run-time
+-   Fixed problems with "RethrownJobError" errors during error handling that suppressed the useful error messages
 
 
 v4.0.7
 ------
 
--  Minor correction to Ghostscript output settings
+-   Minor correction to Ghostscript output settings
 
 
 v4.0.6
 ------
 
--  Update install instructions
--  Provide a sRGB profile instead of using Ghostscript's
+-   Update install instructions
+-   Provide a sRGB profile instead of using Ghostscript's
 
 
 v4.0.5
 ------
 
--  Remove some verbose debug messages from v4.0.4
--  Fixed temporary that wasn't being deleted
--  DPI is now calculated correctly for cropped images, along with other image transformations
--  Inline images are now checked during DPI calculation instead of rejecting the image
+-   Remove some verbose debug messages from v4.0.4
+-   Fixed temporary that wasn't being deleted
+-   DPI is now calculated correctly for cropped images, along with other image transformations
+-   Inline images are now checked during DPI calculation instead of rejecting the image
 
 v4.0.4
 ------
@@ -305,16 +425,16 @@ v4.0.3
 New features
 ^^^^^^^^^^^^
 
--  Page orientations detected are now reported in a summary comment
+-   Page orientations detected are now reported in a summary comment
 
 
 Fixes
 ^^^^^
 
--  Show stack trace if unexpected errors occur
--  Treat "too few characters" error message from Tesseract as a reason to skip that page rather than
-   abort the file
--  Docker: fix blank JPEG2000 issue by insisting on Ghostscript versions that have this fixed
+-   Show stack trace if unexpected errors occur
+-   Treat "too few characters" error message from Tesseract as a reason to skip that page rather than
+    abort the file
+-   Docker: fix blank JPEG2000 issue by insisting on Ghostscript versions that have this fixed
 
 
 v4.0.2
@@ -323,10 +443,10 @@ v4.0.2
 Fixes
 ^^^^^
 
--  Fixed compatibility with Tesseract 3.04.01 release, particularly its different way of outputting
-   orientation information
--  Improved handling of Tesseract errors and crashes
--  Fixed use of chmod on Docker that broke most test cases
+-   Fixed compatibility with Tesseract 3.04.01 release, particularly its different way of outputting
+    orientation information
+-   Improved handling of Tesseract errors and crashes
+-   Fixed use of chmod on Docker that broke most test cases
 
 
 v4.0.1
@@ -335,7 +455,7 @@ v4.0.1
 Fixes
 ^^^^^
 
--  Fixed a KeyError if tesseract fails to find page orientation information
+-   Fixed a KeyError if tesseract fails to find page orientation information
 
 
 v4.0
@@ -344,33 +464,33 @@ v4.0
 New features
 ^^^^^^^^^^^^
 
--  Automatic page rotation (``-r``) is now available. It uses ignores any prior rotation information
-   on PDFs and sets rotation based on the dominant orientation of detectable text. This feature is
-   fairly reliable but some false positives occur especially if there is not much text to work with. (#4) 
--  Deskewing is now performed using Leptonica instead of unpaper. Leptonica is faster and more reliable
-   at image deskewing than unpaper.
+-   Automatic page rotation (``-r``) is now available. It uses ignores any prior rotation information
+    on PDFs and sets rotation based on the dominant orientation of detectable text. This feature is
+    fairly reliable but some false positives occur especially if there is not much text to work with. (#4) 
+-   Deskewing is now performed using Leptonica instead of unpaper. Leptonica is faster and more reliable
+    at image deskewing than unpaper.
 
 
 Fixes
 ^^^^^
 
--  Fixed an issue where lossless reconstruction could cause some pages to be appear incorrectly
-   if the page was rotated by the user in Acrobat after being scanned (specifically if it a /Rotate tag)
--  Fixed an issue where lossless reconstruction could misalign the graphics layer with respect to
-   text layer if the page had been cropped such that its origin is not (0, 0) (#49)
+-   Fixed an issue where lossless reconstruction could cause some pages to be appear incorrectly
+    if the page was rotated by the user in Acrobat after being scanned (specifically if it a /Rotate tag)
+-   Fixed an issue where lossless reconstruction could misalign the graphics layer with respect to
+    text layer if the page had been cropped such that its origin is not (0, 0) (#49)
 
 
 Changes
 ^^^^^^^
 
--  Logging output is now much easier to read
--  ``--deskew`` is now performed by Leptonica instead of unpaper (#25)
--  libffi is now required
--  Some changes were made to the Docker and Travis build environments to support libffi
--  ``--pdf-renderer=tesseract`` now displays a warning if the Tesseract version is less than 3.04.01,
-   the planned release that will include fixes to an important OCR text rendering bug in Tesseract 3.04.00.
-   You can also manually install ./share/sharp2.ttf on top of pdf.ttf in your Tesseract tessdata folder
-   to correct the problem.
+-   Logging output is now much easier to read
+-   ``--deskew`` is now performed by Leptonica instead of unpaper (#25)
+-   libffi is now required
+-   Some changes were made to the Docker and Travis build environments to support libffi
+-   ``--pdf-renderer=tesseract`` now displays a warning if the Tesseract version is less than 3.04.01,
+    the planned release that will include fixes to an important OCR text rendering bug in Tesseract 3.04.00.
+    You can also manually install ./share/sharp2.ttf on top of pdf.ttf in your Tesseract tessdata folder
+    to correct the problem.
 
 
 v3.2.1
@@ -379,8 +499,8 @@ v3.2.1
 Changes
 ^^^^^^^
 
--  Fixed issue #47 "convert() got and unexpected keyword argument 'dpi'" by upgrading to img2pdf 0.2
--  Tweaked the Dockerfiles
+-   Fixed issue #47 "convert() got and unexpected keyword argument 'dpi'" by upgrading to img2pdf 0.2
+-   Tweaked the Dockerfiles
 
 
 v3.2
@@ -389,19 +509,19 @@ v3.2
 New features
 ^^^^^^^^^^^^
 
--  Lossless reconstruction: when possible, OCRmyPDF will inject text layers without 
-   otherwise manipulating the content and layout of a PDF page. For example, a PDF containing a mix
-   of vector and raster content would see the vector content preserved. Images may still be transcoded
-   during PDF/A conversion.  (``--deskew`` and ``--clean-final`` disable this mode, necessarily.)
--  New argument ``--tesseract-pagesegmode`` allows you to pass page segmentation arguments to Tesseract OCR.
-   This helps for two column text and other situations that confuse Tesseract.
--  Added a new "polyglot" version of the Docker image, that generates Tesseract with all languages packs installed,
-   for the polyglots among us. It is much larger.
+-   Lossless reconstruction: when possible, OCRmyPDF will inject text layers without 
+    otherwise manipulating the content and layout of a PDF page. For example, a PDF containing a mix
+    of vector and raster content would see the vector content preserved. Images may still be transcoded
+    during PDF/A conversion.  (``--deskew`` and ``--clean-final`` disable this mode, necessarily.)
+-   New argument ``--tesseract-pagesegmode`` allows you to pass page segmentation arguments to Tesseract OCR.
+    This helps for two column text and other situations that confuse Tesseract.
+-   Added a new "polyglot" version of the Docker image, that generates Tesseract with all languages packs installed,
+    for the polyglots among us. It is much larger.
 
 Changes
 ^^^^^^^
 
--  JPEG transcoding quality is now 95 instead of the default 75. Bigger file sizes for less degradation.
+-   JPEG transcoding quality is now 95 instead of the default 75. Bigger file sizes for less degradation.
 
 
 
@@ -411,7 +531,7 @@ v3.1.1
 Changes
 ^^^^^^^
 
--  Fixed bug that caused incorrect page size and DPI calculations on documents with mixed page sizes
+-   Fixed bug that caused incorrect page size and DPI calculations on documents with mixed page sizes
 
 v3.1
 ----
@@ -419,17 +539,17 @@ v3.1
 Changes
 ^^^^^^^
 
--  Default output format is now PDF/A-2b instead of PDF/A-1b
--  Python 3.5 and macOS El Capitan are now supported platforms - no changes were
-   needed to implement support
--  Improved some error messages related to missing input files
--  Fixed issue #20 - uppercase .PDF extension not accepted
--  Fixed an issue where OCRmyPDF failed to text that certain pages contained previously OCR'ed text, 
-   such as OCR text produced by Tesseract 3.04
--  Inserts /Creator tag into PDFs so that errors can be traced back to this project
--  Added new option ``--pdf-renderer=auto``, to let OCRmyPDF pick the best PDF renderer. 
-   Currently it always chooses the 'hocrtransform' renderer but that behavior may change.
--  Set up Travis CI automatic integration testing
+-   Default output format is now PDF/A-2b instead of PDF/A-1b
+-   Python 3.5 and macOS El Capitan are now supported platforms - no changes were
+    needed to implement support
+-   Improved some error messages related to missing input files
+-   Fixed issue #20 - uppercase .PDF extension not accepted
+-   Fixed an issue where OCRmyPDF failed to text that certain pages contained previously OCR'ed text, 
+    such as OCR text produced by Tesseract 3.04
+-   Inserts /Creator tag into PDFs so that errors can be traced back to this project
+-   Added new option ``--pdf-renderer=auto``, to let OCRmyPDF pick the best PDF renderer. 
+    Currently it always chooses the 'hocrtransform' renderer but that behavior may change.
+-   Set up Travis CI automatic integration testing
 
 v3.0
 ----
@@ -437,55 +557,55 @@ v3.0
 New features
 ^^^^^^^^^^^^
 
--  Easier installation with a Docker container or Python's ``pip`` package manager 
--  Eliminated many external dependencies, so it's easier to setup
--  Now installs ``ocrmypdf`` to ``/usr/local/bin`` or equivalent for system-wide
-   access and easier typing
--  Improved command line syntax and usage help (``--help``)
--  Tesseract 3.03+ PDF page rendering can be used instead for better positioning
-   of recognized text (``--pdf-renderer tesseract``)
--  PDF metadata (title, author, keywords) are now transferred to the 
-   output PDF
--  PDF metadata can also be set from the command line (``--title``, etc.)
--  Automatic repairs malformed input PDFs if possible
--  Added test cases to confirm everything is working
--  Added option to skip extremely large pages that take too long to OCR and are 
-   often not OCRable (e.g. large scanned maps or diagrams); other pages are still
-   processed (``--skip-big``)
--  Added option to kill Tesseract OCR process if it seems to be taking too long on
-   a page, while still processing other pages (``--tesseract-timeout``)
--  Less common colorspaces (CMYK, palette) are now supported by conversion to RGB
--  Multiple images on the same PDF page are now supported
+-   Easier installation with a Docker container or Python's ``pip`` package manager 
+-   Eliminated many external dependencies, so it's easier to setup
+-   Now installs ``ocrmypdf`` to ``/usr/local/bin`` or equivalent for system-wide
+    access and easier typing
+-   Improved command line syntax and usage help (``--help``)
+-   Tesseract 3.03+ PDF page rendering can be used instead for better positioning
+    of recognized text (``--pdf-renderer tesseract``)
+-   PDF metadata (title, author, keywords) are now transferred to the 
+    output PDF
+-   PDF metadata can also be set from the command line (``--title``, etc.)
+-   Automatic repairs malformed input PDFs if possible
+-   Added test cases to confirm everything is working
+-   Added option to skip extremely large pages that take too long to OCR and are 
+    often not OCRable (e.g. large scanned maps or diagrams); other pages are still
+    processed (``--skip-big``)
+-   Added option to kill Tesseract OCR process if it seems to be taking too long on
+    a page, while still processing other pages (``--tesseract-timeout``)
+-   Less common colorspaces (CMYK, palette) are now supported by conversion to RGB
+-   Multiple images on the same PDF page are now supported
 
 Changes
 ^^^^^^^
 
--  New, robust rewrite in Python 3.4+ with ruffus_ pipelines
--  Now uses Ghostscript 9.14's improved color conversion model to preserve PDF colors
--  OCR text is now rendered in the PDF as invisible text. Previous versions of OCRmyPDF
-   incorrectly rendered visible text with an image on top.
--  All "tasks" in the pipeline can be executed in parallel on any
-   available CPUs, increasing performance
--  The ``-o DPI`` argument has been phased out, in favor of ``--oversample DPI``, in
-   case we need ``-o OUTPUTFILE`` in the future
--  Removed several dependencies, so it's easier to install.  We no 
-   longer use:
-   
-   - GNU parallel_
-   - ImageMagick_
-   - Python 2.7
-   - Poppler
-   - MuPDF_ tools
-   - shell scripts
-   - Java and JHOVE_
-   - libxml2
+-   New, robust rewrite in Python 3.4+ with ruffus_ pipelines
+-   Now uses Ghostscript 9.14's improved color conversion model to preserve PDF colors
+-   OCR text is now rendered in the PDF as invisible text. Previous versions of OCRmyPDF
+    incorrectly rendered visible text with an image on top.
+-   All "tasks" in the pipeline can be executed in parallel on any
+    available CPUs, increasing performance
+-   The ``-o DPI`` argument has been phased out, in favor of ``--oversample DPI``, in
+    case we need ``-o OUTPUTFILE`` in the future
+-   Removed several dependencies, so it's easier to install.  We no 
+    longer use:
+    
+    - GNU parallel_
+    - ImageMagick_
+    - Python 2.7
+    - Poppler
+    - MuPDF_ tools
+    - shell scripts
+    - Java and JHOVE_
+    - libxml2
 
--  Some new external dependencies are required or optional, compared to v2.x:
+-   Some new external dependencies are required or optional, compared to v2.x:
 
-   - Ghostscript 9.14+
-   - qpdf_ 5.0.0+
-   - Unpaper_ 6.1 (optional)
-   - some automatically managed Python packages
+    - Ghostscript 9.14+
+    - qpdf_ 5.0.0+
+    - Unpaper_ 6.1 (optional)
+    - some automatically managed Python packages
   
 .. _ruffus: http://www.ruffus.org.uk/index.html
 .. _parallel: https://www.gnu.org/software/parallel/
@@ -498,87 +618,87 @@ Changes
 Release candidates
 ^^^^^^^^^^^^^^^^^^
 
--  rc9:
+-   rc9:
 
-   - fix issue #118: report error if ghostscript iccprofiles are missing
-   - fixed another issue related to #111: PDF rasterized to palette file
-   - add support image files with a palette
-   - don't try to validate PDF file after an exception occurs
+    - fix issue #118: report error if ghostscript iccprofiles are missing
+    - fixed another issue related to #111: PDF rasterized to palette file
+    - add support image files with a palette
+    - don't try to validate PDF file after an exception occurs
 
--  rc8:
+-   rc8:
 
-   - fix issue #111: exception thrown if PDF is missing DocumentInfo dictionary
+    - fix issue #111: exception thrown if PDF is missing DocumentInfo dictionary
 
--  rc7:
+-   rc7:
 
-   - fix error when installing direct from pip, "no such file 'requirements.txt'"
+    - fix error when installing direct from pip, "no such file 'requirements.txt'"
 
--  rc6:
+-   rc6:
 
-   - dropped libxml2 (Python lxml) since Python 3's internal XML parser is sufficient
-   - set up Docker container
-   - fix Unicode errors if recognized text contains Unicode characters and system locale is not UTF-8
+    - dropped libxml2 (Python lxml) since Python 3's internal XML parser is sufficient
+    - set up Docker container
+    - fix Unicode errors if recognized text contains Unicode characters and system locale is not UTF-8
 
--  rc5:
+-   rc5:
 
-   - dropped Java and JHOVE in favour of qpdf
-   - improved command line error output
-   - additional tests and bug fixes
-   - tested on Ubuntu 14.04 LTS
+    - dropped Java and JHOVE in favour of qpdf
+    - improved command line error output
+    - additional tests and bug fixes
+    - tested on Ubuntu 14.04 LTS
 
--  rc4:
+-   rc4:
 
-   - dropped MuPDF in favour of qpdf
-   - fixed some installer issues and errors in installation instructions
-   - improve performance: run Ghostscript with multithreaded rendering
-   - improve performance: use multiple cores by default
-   - bug fix: checking for wrong exception on process timeout 
+    - dropped MuPDF in favour of qpdf
+    - fixed some installer issues and errors in installation instructions
+    - improve performance: run Ghostscript with multithreaded rendering
+    - improve performance: use multiple cores by default
+    - bug fix: checking for wrong exception on process timeout 
 
--  rc3: skipping version number intentionally to avoid confusion with Tesseract
--  rc2: first release for public testing to test-PyPI, Github
--  rc1: testing release process
+-   rc3: skipping version number intentionally to avoid confusion with Tesseract
+-   rc2: first release for public testing to test-PyPI, Github
+-   rc1: testing release process
 
 Compatibility notes
 -------------------
 
--  ``./OCRmyPDF.sh`` script is still available for now
--  Stacking the verbosity option like ``-vvv`` is no longer supported
+-   ``./OCRmyPDF.sh`` script is still available for now
+-   Stacking the verbosity option like ``-vvv`` is no longer supported
 
--  The configuration file ``config.sh`` has been removed.  Instead, you can
-   feed a file to the arguments for common settings:
+-   The configuration file ``config.sh`` has been removed.  Instead, you can
+    feed a file to the arguments for common settings:
 
 ::
 
-   ocrmypdf input.pdf output.pdf @settings.txt
+    ocrmypdf input.pdf output.pdf @settings.txt
 
 where ``settings.txt`` contains *one argument per line*, for example:
 
 ::
 
-   -l 
-   deu 
-   --author 
-   A. Merkel 
-   --pdf-renderer 
-   tesseract
+    -l 
+    deu 
+    --author 
+    A. Merkel 
+    --pdf-renderer 
+    tesseract
 
 
 Fixes
 ^^^^^
 
--  Handling of filenames containing spaces: fixed
+-   Handling of filenames containing spaces: fixed
 
 Notes and known issues
 ^^^^^^^^^^^^^^^^^^^^^^
 
--  Some dependencies may work with lower versions than tested, so try
-   overriding dependencies if they are "in the way" to see if they work.
+-   Some dependencies may work with lower versions than tested, so try
+    overriding dependencies if they are "in the way" to see if they work.
 
--  ``--pdf-renderer tesseract`` will output files with an incorrect page size in Tesseract 3.03,
-   due to a bug in Tesseract.
+-   ``--pdf-renderer tesseract`` will output files with an incorrect page size in Tesseract 3.03,
+    due to a bug in Tesseract.
 
--  PDF files containing "inline images" are not supported and won't be for the 3.0 release. Scanned
-   images almost never contain inline images.
+-   PDF files containing "inline images" are not supported and won't be for the 3.0 release. Scanned
+    images almost never contain inline images.
 
 
 v2.2-stable (2014-09-29)

docs/security.rst

@@ -10,7 +10,7 @@ The disclaimer applies: this software has no warranties of any kind.
 PDFs may contain malware
 ------------------------
 
-PDF is a rich, complex file format. The official PDF 1.7 specification, ISO 32000:2008, is hundreds of packages long and references several annexes each of which are similar in length. PDFs can contain video, audio, JavaScript and other programming, and forms. In some cases, they can open internet connections to pre-selected URLs. All of these possible attack vectors.
+PDF is a rich, complex file format. The official PDF 1.7 specification, ISO 32000:2008, is hundreds of pages long and references several annexes each of which are similar in length. PDFs can contain video, audio, XML, JavaScript and other programming, and forms. In some cases, they can open internet connections to pre-selected URLs. All of these possible attack vectors.
 
 In short, PDFs `may contain viruses <https://security.stackexchange.com/questions/64052/can-a-pdf-file-contain-a-virus>`_.
 
@@ -21,23 +21,58 @@ How OCRmyPDF processes PDFs
 
 OCRmyPDF must open and interpret your PDF in order to insert an OCR layer. First, it runs all PDFs through `qpdf <https://github.com/qpdf/qpdf>`_, a program that repairs PDFs with syntax errors. This is done because, in the author's experience, a significant number of PDFs in the wild especially those created by scanners are not well-formed files. qpdf makes it more likely that OCRmyPDF will succeed, but offers no security guarantees. qpdf is also used to split the PDF into single page PDFs.
 
-After qpdf, OCRmyPDF examines each page using `PyPDF2 <https://github.com/mstamy2/PyPDF2>`_. This library also has no warranties or guarantees.
+After qpdf, OCRmyPDF examines each page using `PyPDF2 <https://github.com/mstamy2/PyPDF2>`_. This library also has no warranties or guarantees. OCRmyPDF works with qpdf 5.0 and up, but version 7.0 is recommended because of known security vulnerabilities in early versions.
 
-Finally, OCRmyPDF rasterizes each page of the PDF using `Ghostscript <http://ghostscript.com/>`_ in ``-dSAFER`` mode. 
+Finally, OCRmyPDF rasterizes each page of the PDF using `Ghostscript <http://ghostscript.com/>`_ in ``-dSAFER`` mode.
 
 Depending on the options specified, OCRmyPDF may graft the OCR layer into the existing PDF or it may essentially reconstruct ("re-fry") a visually identical PDF that may be quite different at the binary level. That said, OCRmyPDF is not a tool designed for sanitizing PDFs.
 
-Using OCRmyPDF online
----------------------
+Using OCRmyPDF online or as a service
+-------------------------------------
 
-OCRmyPDF is not designed to be deployed "as a service", in a setting where a user/attacker could upload a file for OCR processing online. It is not designed to be secure in this case.
+OCRmyPDF should not be deployed as a public-facing service, such as a website where a potential attacker could upload a PDF of their choice for OCR. OCRmyPDF is not designed to be secure against PDF malware. Another concern is PDFs specifically designed to be a denial of service attack: PDFs can contain recursive data structures that sometimes send parsers into infinite loops, and issue complex graphics drawing commands.
+
+Setting aside these concerns, a side effect of OCRmyPDF is it may incidentally sanitize PDFs that contain malware. It runs ``qpdf`` to repair the PDF, which could correct malformed PDF structures that are part of an attack. When PDF/A output is selected (the default), the input PDF is partially reconstructed by Ghostscript. When ``--force-ocr`` is used, all pages are rasterized and reconverted to PDF, which could remove malware in embedded images. No guarantees.
+
+OCRmyPDF should be relatively safe to use in a trusted intranet, with some considerations:
+
+Limiting CPU usage
+^^^^^^^^^^^^^^^^^^
+
+OCRmyPDF will attempt to use all available CPUs and storage, so executing ``nice ocrmypdf`` or limiting the number of jobs with the ``-j`` argument may ensure the server remains available. Another option would be run OCRmyPDF jobs inside a Docker container, a virtual machine, or a cloud instance, which can impose its own limits on CPU usage and be terminated "from orbit" if it fails to complete.
+
+Temporary storage requirements
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+OCRmyPDF will use a large amount of temporary storage for its work, proportional to the total number of pixels needed to rasterize the PDF. The raster image of a 8.5×11" color page at 300 DPI takes 25 MB uncompressed; OCRmyPDF saves its intermediates as PNG, but that still means it requires about 9 MB per intermediate based on average compression ratios. Multiple intermediates per page are also required, depending on the command line given. A rule of thumb would be to allow 100 MB of temporary storage per page in a file – meaning that a small cloud servers or small VM partitions should be provisioned with plenty of extra space, if say, a 500 page file might be sent.
+
+To check temporary storage usage on actual files, run ``ocrmypdf -k ...`` which will preserve and print the path to temporary storage when the job is done.
+
+To change where temporary files are stored, change the ``TMPDIR`` environment variable for ocrmypdf's environment. (Python's ``tempfile.gettempdir()`` returns the root directory in which temporary files will be stored.) For example, one could redirect ``TMPDIR`` to a large RAM disk to avoid wear on HDD/SSD and potentially improve performance. On Amazon Web Services, ``TMPDIR`` can be set to `empheral storage <https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/InstanceStorage.html>`_.
+
+Timeouts
+^^^^^^^^
+
+To prevent excessively long OCR jobs consider setting ``--tesseract-timeout`` and/or ``--skip-big`` arguments. ``--skip-big`` is particularly helpful if your PDFs include documents such as reports on standard page sizes with large images attached - often large images are not worth OCR'ing anyway.
+
+Commercial alternatives
+^^^^^^^^^^^^^^^^^^^^^^^
+
+The author also provides professional services that include OCR and building databases around PDFs, and is happy to provide consultation.
+
+Abbyy Cloud OCR is a viable commercial alternative with a web services API. 
 
-Abbyy Cloud OCR is a viable commercial alternative with a web services API. The author also provides professional services that include OCR and building databases around PDFs, and is happy to provide consultation.
 
 Password protection, digital signatures and certification
 ---------------------------------------------------------
 
-OCRmyPDF cannot remove password protection from a PDF. ``qpdf``, one of its dependencies, has this capability. After OCR is applied, password protection is not permitted on PDF/A documents but the file can be converted to regular PDF.
+Password protected PDFs usually have two passwords, and owner and user password. When the user password is set to empty, PDF readers will open the file automatically and marked it as "(SECURED)". While not as reliable as a digital signature, this indicates that whoever set the password approved of the file at that time. When the user password is set, the document cannot be viewed without the password. 
+
+Either way, OCRmyPDF does not remove passwords from PDFs and exits with an error on encountering them.
+
+``qpdf``, one of OCRmyPDF's dependencies, can remove passwords. If the owner and user password are set, a password is required for ``qpdf``. If only the owner password is set, then the password can be stripped, even if one does not have the owner password.
+
+After OCR is applied, password protection is not permitted on PDF/A documents but the file can be converted to regular PDF.
 
 Many programs exist which are capable of inserting an image of someone's signature. On its own, this offers no security guarantees. It is trivial to remove the signature image and apply it to other files. This practice offers no real security.
 

ocrmypdf/__init__.py

@@ -4,6 +4,7 @@ import pkg_resources
 
 PROGRAM_NAME = 'ocrmypdf'
 
-VERSION = pkg_resources.get_distribution('ocrmypdf').version
-
+# Official PEP 396
+__version__ = pkg_resources.get_distribution('ocrmypdf').version
 
+VERSION = __version__

ocrmypdf/__main__.py

@@ -1,37 +1,35 @@
 #!/usr/bin/env python3
 # © 2015-17 James R. Barlow: github.com/jbarlow83
 
-from contextlib import suppress
 from tempfile import mkdtemp
 from collections.abc import Sequence
 import sys
 import os
 import re
-import shutil
 import warnings
 import multiprocessing
 import atexit
 import textwrap
-import img2pdf
 import logging
 import argparse
 
 import PyPDF2 as pypdf
-from PIL import Image
+import PIL
 
 import ruffus.ruffus_exceptions as ruffus_exceptions
 import ruffus.cmdline as cmdline
 import ruffus.proxy_logger as proxy_logger
 
-from .pipeline import JobContext, JobContextManager, re_symlink, \
+from .pipeline import JobContext, JobContextManager, \
     cleanup_working_files, build_pipeline
 from .pdfa import file_claims_pdfa
 from .helpers import is_iterable_notstr, re_symlink, is_file_writable
-from .exec import tesseract, qpdf
+from .exec import tesseract, qpdf, ghostscript
 from . import PROGRAM_NAME, VERSION
 
-from .exceptions import *
+from .exceptions import ExitCode, ExitCodeException, MissingDependencyError
 from . import exceptions as ocrmypdf_exceptions
+from ._unicodefun import verify_python3_env
 
 warnings.simplefilter('ignore', pypdf.utils.PdfReadWarning)
 
@@ -41,6 +39,9 @@ warnings.simplefilter('ignore', pypdf.utils.PdfReadWarning)
 
 MINIMUM_TESS_VERSION = '3.04'
 
+HOCR_OK_LANGS = frozenset([
+    'eng', 'deu', 'spa', 'ita', 'por'
+])
 
 def complain(message):
     print(*textwrap.wrap(message), file=sys.stderr)
@@ -50,6 +51,11 @@ def complain(message):
 if 'IDE_PROJECT_ROOTS' in os.environ:
     os.environ['PATH'] = '/usr/local/bin:' + os.environ['PATH']
 
+# -------- 
+# Critical environment tests
+
+verify_python3_env()
+
 if tesseract.version() < MINIMUM_TESS_VERSION:
     complain(
         "Please install tesseract {0} or newer "
@@ -131,12 +137,15 @@ parser.add_argument(
     '--image-dpi', metavar='DPI', type=int,
     help="For input image instead of PDF, use this DPI instead of file's.")
 parser.add_argument(
-    '--output-type', choices=['pdfa', 'pdf'], default='pdfa',
+    '--output-type', choices=['pdfa', 'pdf', 'pdfa-1', 'pdfa-2'], 
+    default='pdfa',
     help="Choose output type. 'pdfa' creates a PDF/A-2b compliant file for "
          "long term archiving (default, recommended) but may not suitable "
          "for users who want their file altered as little as possible. 'pdfa' "
          "also has problems with full Unicode text. 'pdf' attempts to "
-         "preserve file contents as much as possible.")
+         "preserve file contents as much as possible. 'pdf-a1' creates a "
+         "PDF/A1-b file. 'pdf-a2' is equivalent to 'pdfa'."
+         )
 
 # Use null string '\0' as sentinel to indicate the user supplied no argument,
 # since that is the only invalid character for filepaths on all platforms
@@ -233,6 +242,11 @@ ocrsettings.add_argument(
 advanced = parser.add_argument_group(
     "Advanced",
     "Advanced options to control Tesseract's OCR behavior")
+advanced.add_argument(
+    '--max-image-mpixels', action='store', type=float, metavar='MPixels',
+    help="Set maximum number of pixels to unpack before treating an image as a "
+         "decompression bomb",
+    default=128.0)
 advanced.add_argument(
     '--tesseract-config', action='append', metavar='CFG', default=[],
     help="Additional Tesseract configuration files -- see documentation")
@@ -250,15 +264,18 @@ advanced.add_argument(
          "3 - default.")
     )
 advanced.add_argument(
-    '--pdf-renderer', choices=['auto', 'tesseract', 'hocr', 'tess4'], default='auto',
+    '--pdf-renderer',
+    choices=['auto', 'tesseract', 'hocr', 'tess4', 'sandwich'], default='auto',
     help="Choose OCR PDF renderer - the default option is to let OCRmyPDF "
-         "choose.  The 'tesseract' PDF renderer is more accurate and does a "
-         "better job and document structure such as recognizing columns. It "
-         "also does a better job on non-Latin languages. However, it does "
-         "not work as well when older versions of Tesseract or Ghostscript "
-         "are installed, and some combinations of arguments to do not work "
-         "with --pdf-renderer tesseract.  The 'tess4' PDF renderer is similar "
-         "to 'tesseract', requires tesseract 4, and gives superior results.")
+         "choose."
+         "auto - let OCRmyPDF choose; "
+         "sandwich - default renderer for Tesseract 3.05.01 and newer; "
+         "hocr - default renderer for older versions of Tesseract; "
+         "tesseract - gives better results for non-Latin languages and "
+         "Tesseract older than 3.05.01 but has problems with some versions "
+         " of Ghostscript; deprecated"
+         "tess4 - deprecated alias for 'sandwich'"
+    )
 advanced.add_argument(
     '--tesseract-timeout', default=180.0, type=float, metavar='SECONDS',
     help='Give up on OCR after the timeout, but copy the preprocessed page '
@@ -278,6 +295,15 @@ advanced.add_argument(
          "are applied to all pages, including those for which OCR was "
          "skipped.  Not supported for --output-type=pdf ; that setting "
          "preserves the original compression of all images.")
+advanced.add_argument(
+    '--user-words', metavar='FILE',
+    help="Specify the location of the Tesseract user words file. This is a "
+         "list of words Tesseract should consider while performing OCR in "
+         "addition to its standard language dictionaries. This can improve "
+         "OCR quality especially for specialized and technical documents.")
+advanced.add_argument(
+    '--user-patterns', metavar='FILE',
+    help="Specify the location of the Tesseract user patterns file.")
 
 debugging = parser.add_argument_group(
     "Debugging",
@@ -301,39 +327,54 @@ def check_options_languages(options, _log):
     if '+' in options.language[0]:
         options.language = options.language[0].split('+')
 
-    if not set(options.language).issubset(tesseract.languages()):
+    languages = set(options.language)
+    if not languages.issubset(tesseract.languages()):
         msg = (
             "The installed version of tesseract does not have language "
             "data for the following requested languages: \n")
-        for lang in (set(options.language) - tesseract.languages()):
+        for lang in (languages - tesseract.languages()):
             msg += lang + '\n'
-        raise argparse.ArgumentError(None, msg)
+        raise MissingDependencyError(msg)
 
 
 def check_options_output(options, log):
     if options.pdf_renderer == 'auto':
-        options.pdf_renderer = 'hocr'
+        if tesseract.has_textonly_pdf():
+            options.pdf_renderer = 'sandwich'
+        else:
+            options.pdf_renderer = 'hocr'
 
-    if options.pdf_renderer in ('tesseract', 'tess4'):
-        if tesseract.version() < '3.05':
-            log.warning(
-                "The setting --pdf-renderer=tesseract is not recommend for "
-                " use with tesseract versions less than 3.05, because it "
-                " produces OCR text that is incompatible with Ghostscript and "
-                " some other software.")
-        elif tesseract.version() == '4.00.00alpha':
-            log.warning(
-                "The setting --pdf-renderer={tesseract,tess4} is not"
-                " recommended for builds of tesseract 4.00.00alpha older than"
-                " February 2017. Make sure you are using a recent build.")
+    if options.pdf_renderer == 'sandwich' and not tesseract.has_textonly_pdf():
+        raise MissingDependencyError(
+            "The 'sandwich' renderer requires Tesseract 3.05.01 or newer; "
+            "or Tesseract 4.00 alpha newer than February 2017.")
 
-    if options.debug_rendering and options.pdf_renderer == 'tesseract':
+    if options.pdf_renderer == 'tess4':
+        log.warning("The 'tess4' PDF renderer has been renamed to 'sandwich'. "
+                    "Please use --pdf-renderer=sandwich.")
+        options.pdf_renderer = 'sandwich'
+
+    if options.pdf_renderer == 'tesseract':
+        if tesseract.version() < '3.05' and \
+                options.output_type.startswith('pdfa'):
+            log.warning(
+                "For best results use --pdf-renderer=tesseract "
+                "--output-type=pdf to disable PDF/A generation via "
+                "Ghostscript, which is known to corrupt the OCR text of "
+                "some PDFs produced your version of Tesseract.")
+        elif tesseract.has_textonly_pdf():
+            log.warning(
+                "The argument --pdf-renderer=tesseract provides support for "
+                "versions of tesseract older than your version. For best "
+                "results omit this argument and let OCRmyPDF choose the "
+                "best available renderer.")
+
+    if options.debug_rendering and options.pdf_renderer != 'hocr':
         log.info(
-            "Ignoring --debug-rendering because it is not supported with"
-            "--pdf-renderer=tesseract.")
+            "Ignoring --debug-rendering because it requires --pdf-renderer=hocr")
 
     lossless_reconstruction = False
-    if options.pdf_renderer in ('hocr', 'tess4'):
+    if options.pdf_renderer in ('hocr', 'sandwich'):
         if not any((options.deskew, options.clean_final, options.force_ocr,
                    options.remove_background)):
             lossless_reconstruction = True
@@ -379,19 +420,36 @@ def check_options_ocr_behavior(options, log):
     # if options.redo_ocr and (options.skip_text or options.force_ocr):
     #     raise argparse.ArgumentError(
     #         "Error: --redo-ocr and other OCR options are incompatible.")
+    languages = set(options.language)
+    if options.pdf_renderer == 'hocr' and \
+            not languages.issubset(HOCR_OK_LANGS):
+        msg = (
+            "The 'hocr' PDF renderer is known to cause problems with one "
+            "or more of the languages in your document. ")
 
-    if set(options.language) & {'chi_sim', 'chi_tra'} and \
-            (options.pdf_renderer == 'hocr' or options.output_type == 'pdfa'):
-        log.warning(
-            "Your settings are known to cause problems with OCR of Chinese text. "
-            "Try adding these arguments: "
-            "    ocrmypdf --pdf-renderer tesseract --output-type pdf")
+        if tesseract.has_textonly_pdf():
+            msg += (
+                "Use --pdf-renderer auto (the default) to avoid this issue.")
+        else:
+            msg += (
+                "Use --pdf-renderer tesseract --output-type pdf to avoid "
+                "this issue")
+        log.warning(msg)
+    elif ghostscript.version() < '9.20' and \
+            not languages.issubset(HOCR_OK_LANGS) \
+            and options.output_type != 'pdf':
+        msg = (
+            "The installed version of Ghostscript does not work correctly "
+            "with the OCR languages you specified. Use --output-type pdf or "
+            "upgrade to Ghostscript 9.20 or later to avoid this issue.")
+        msg += "Found Ghostscript {}".format(ghostscript.version())
+        log.warning(msg)
 
 
 def check_options_advanced(options, log):
     if tesseract.v4():
         log.info(
-            "Tesseract v4.x.alpha found. OCRmyPDF support is experimental.")
+            "Tesseract v4.x.alpha found.")
     if options.tesseract_oem and not tesseract.v4():
         log.warning(
             "--tesseract-oem requires Tesseract 4.x -- argument ignored")
@@ -400,18 +458,18 @@ def check_options_advanced(options, log):
             "--pdf-renderer tess4 requires Tesseract 4.x "
             "commit 3d9fb3b or later")
     if options.pdfa_image_compression != 'auto' and \
-            options.output_type != 'pdfa':
+            options.output_type.startswith('pdfa'):
         log.warning(
             "--pdfa-image-compression argument has no effect when "
-            "--output-type is not 'pdfa'"
+            "--output-type is not 'pdfa', 'pdfa-1', or 'pdfa-2'"
         )
 
 
 def check_options_metadata(options, log):
     import unicodedata
-    metadata = [options.title, options.author, options.keywords,
-                options.subject]
-    for s in (m for m in metadata if m):
+    docinfo = [options.title, options.author, options.keywords,
+               options.subject]
+    for s in (m for m in docinfo if m):
         for c in s:
             if unicodedata.category(c) == 'Co' or ord(c) >= 0x10000:
                 raise ValueError(
@@ -493,31 +551,47 @@ def cleanup_ruffus_error_message(msg):
 def do_ruffus_exception(ruffus_five_tuple, options, log):
     """Replace the elaborate ruffus stack trace with a user friendly
     description of the error message that occurred."""
+    exit_code = None
 
     task_name, job_name, exc_name, exc_value, exc_stack = ruffus_five_tuple
+    job_name = job_name  # unused
     if exc_name == 'builtins.SystemExit':
         match = re.search(r"\.(.+?)\)", exc_value)
         exit_code_name = match.groups()[0]
-        exit_code = getattr(ExitCode, exit_code_name, 'other_error')
-        return exit_code
+        exit_code = getattr(ExitCode, exit_code_name, 'other_error')        
     elif exc_name == 'ruffus.ruffus_exceptions.MissingInputFileError':
         log.error(cleanup_ruffus_error_message(exc_value))
-        return ExitCode.input_file
+        exit_code = ExitCode.input_file
     elif exc_name == 'builtins.TypeError':
         # Even though repair_pdf will fail, ruffus will still try
         # to call split_pages with no input files, likely due to a bug
         if task_name == 'split_pages':
             log.error("Input file '{0}' is not a valid PDF".format(
                 options.input_file))
-            return ExitCode.input_file
+            exit_code = ExitCode.input_file
     elif exc_name == 'builtins.KeyboardInterrupt':
         log.error("Interrupted by user")
-        return ExitCode.ctrl_c
+        exit_code = ExitCode.ctrl_c
     elif exc_name == 'subprocess.CalledProcessError':
         # It's up to the subprocess handler to report something useful
         msg = "Error occurred while running this command:"
         log.error(msg + '\n' + exc_value)
-        return ExitCode.child_process_error
+        exit_code = ExitCode.child_process_error
+    elif (exc_name == 'PyPDF2.utils.PdfReadError' and \
+            'not been decrypted' in exc_value) or \
+            (exc_name == 'ocrmypdf.exceptions.EncryptedPdfError'):
+        log.error(textwrap.dedent("""\
+            Input PDF is encrypted. The encryption must be removed to
+            perform OCR. 
+
+            For information about this PDF's security use
+                qpdf --show-encryption infilename
+
+            You can remove the encryption using
+                qpdf --decrypt [--password=[password]] infilename
+
+            """))
+        exit_code = ExitCode.encrypted_pdf        
     elif exc_name == 'ocrmypdf.exceptions.PdfMergeFailedError':
         log.error(textwrap.dedent("""\
             Failed to merge PDF image layer with OCR layer
@@ -528,24 +602,20 @@ def do_ruffus_exception(ruffus_five_tuple, options, log):
             Try using
                 ocrmypdf --pdf-renderer tesseract  [..other args..]
             """))
-        return ExitCode.input_file
+        exit_code = ExitCode.input_file
     elif exc_name.startswith('ocrmypdf.exceptions.'):
         base_exc_name = exc_name.replace('ocrmypdf.exceptions.', '')
         exc_class = getattr(ocrmypdf_exceptions, base_exc_name)
-        return exc_class.exit_code
-    elif exc_name == 'PyPDF2.utils.PdfReadError' and \
-            'not been decrypted' in exc_value:
-        log.error(textwrap.dedent("""\
-            Input PDF uses either an encryption algorithm or a PDF security
-            handler that is not supported by ocrmypdf.
+        exit_code = exc_class.exit_code
+    elif exc_name == 'PIL.Image.DecompressionBombError':
+        msg = cleanup_ruffus_error_message(exc_value)
+        msg += ("\nUse the --max-image-mpixels argument to set increase the "
+                "maximum number of megapixels to accept.")
+        log.error(msg)
+        exit_code = ExitCode.input_file
 
-            For information about this PDF's security use
-                qpdf --show-encryption [...input PDF...]
-
-            (Only algorithms "R = 1" and "R = 2" are supported.)
-
-            """))
-        return ExitCode.encrypted_pdf
+    if exit_code is not None:
+        return exit_code
 
     if not options.verbose:
         log.error(exc_stack)
@@ -611,6 +681,20 @@ def check_closed_streams(options):
     return True
 
 
+def log_page_orientations(pdfinfo, _log):
+    direction = {0: 'n', 90: 'e',
+                180: 's', 270: 'w'}
+    orientations = []
+    for n, page in enumerate(pdfinfo):
+        angle = page.rotation or 0
+        if angle != 0:
+            orientations.append('{0}{1}'.format(
+                n + 1,
+                direction.get(angle, '')))
+    if orientations:
+        _log.info('Page orientations detected: ' + ' '.join(orientations))
+
+
 def run_pipeline():
     options = parser.parse_args()
     options.verbose_abbreviated_path = 1
@@ -624,9 +708,23 @@ def run_pipeline():
         logging_factory, __name__, logger_args)
     _log.debug('ocrmypdf ' + VERSION)
     _log.debug('tesseract ' + tesseract.version())
+    _log.debug('qpdf ' + qpdf.version())
 
     check_options(options, _log)
 
+    PIL.Image.MAX_IMAGE_PIXELS = int(options.max_image_mpixels * 1000000)
+    if PIL.Image.MAX_IMAGE_PIXELS == 0:
+        PIL.Image.MAX_IMAGE_PIXELS = None
+
+    # Complain about qpdf version < 7.0.0
+    # Suppress the warning if in the test suite, since there are no PPAs
+    # for qpdf 7.0.0 for Ubuntu trusty (i.e. Travis)
+    if qpdf.version() < '7.0.0' and not os.environ.get('PYTEST_CURRENT_TEST'):
+        complain(
+            "You are using qpdf version {0} which has known issues including "
+            "security vulnerabilities with certain malformed PDFs. Consider "
+            "upgrading to version 7.0.0 or newer.".format(qpdf.version()))
+
     # Any changes to options will not take effect for options that are already
     # bound to function parameters in the pipeline. (For example
     # options.input_file, options.pdf_renderer are already bound.)
@@ -660,15 +758,16 @@ def run_pipeline():
                     file."""))
                 return ExitCode.bad_args
         elif not is_file_writable(options.output_file):
-            _log.error(textwrap.dedent("""\
-                Output file location is not writable."""))
+            _log.error(
+                "Output file location (" + options.output_file + ") " +
+                "is not a writable file.")
             return ExitCode.file_access_error
 
         manager = JobContextManager()
-        manager.register('JobContext', JobContext)
+        manager.register('JobContext', JobContext)  # pylint: disable=no-member
         manager.start()
 
-        context = manager.JobContext()
+        context = manager.JobContext()  # pylint: disable=no-member
         context.set_options(options)
         context.set_work_folder(work_folder)
 
@@ -699,8 +798,7 @@ def run_pipeline():
             _log.error("Unexpected ruffus exception: " + str(e))
             _log.error(repr(e))
             return ExitCode.other_error
-        else:
-            return exitcode
+        return exitcode
     except ExitCodeException as e:
         return e.exit_code
     except Exception as e:
@@ -709,8 +807,12 @@ def run_pipeline():
 
     if options.flowchart:
         _log.info("Flowchart saved to {}".format(options.flowchart))
-    elif options.output_file != '-':
-        if options.output_type == 'pdfa':
+    elif options.output_file == '-':
+        _log.info("Output sent to stdout")
+    elif os.path.samefile(options.output_file, os.devnull):
+        pass  # Say nothing when sending to dev null
+    else:
+        if options.output_type.startswith('pdfa'):
             pdfa_info = file_claims_pdfa(options.output_file)
             if pdfa_info['pass']:
                 msg = 'Output file is a {} (as expected)'
@@ -718,29 +820,17 @@ def run_pipeline():
             else:
                 msg = 'Output file is okay but is not PDF/A (seems to be {})'
                 _log.warning(msg.format(pdfa_info['conformance']))
-
                 return ExitCode.invalid_output_pdf
         if not qpdf.check(options.output_file, _log):
             _log.warning('Output file: The generated PDF is INVALID')
             return ExitCode.invalid_output_pdf
-    else:
-        _log.info("Output sent to stdout")
 
     pdfinfo = context.get_pdfinfo()
     if options.verbose:
         from pprint import pformat
         _log.debug(pformat(pdfinfo))
-    direction = {0: 'n', 90: 'e',
-                 180: 's', 270: 'w'}
-    orientations = []
-    for n, page in enumerate(pdfinfo):
-        angle = pdfinfo[n].rotation or 0
-        if angle != 0:
-            orientations.append('{0}{1}'.format(
-                n + 1,
-                direction.get(angle, '')))
-    if orientations:
-        _log.info('Page orientations detected: ' + ' '.join(orientations))
+
+    log_page_orientations(pdfinfo, _log)
 
     return ExitCode.ok
 

ocrmypdf/_unicodefun.py

@@ -0,0 +1,108 @@
+# Copyright (c) 2014, Armin Ronacher
+#
+# Copyright (c) 2017, James R Barlow
+#
+# Some rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#
+#     * Redistributions in binary form must reproduce the above
+#       copyright notice, this list of conditions and the following
+#       disclaimer in the documentation and/or other materials provided
+#       with the distribution.
+#
+#     * The names of the contributors may not be used to endorse or
+#       promote products derived from this software without specific
+#       prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import os
+import sys
+import codecs
+
+
+def verify_python3_env():
+    """Ensures that the environment is good for unicode on Python 3."""
+    try:
+        import locale
+        fs_enc = codecs.lookup(locale.getpreferredencoding()).name
+    except Exception:
+        fs_enc = 'ascii'
+    if fs_enc != 'ascii':
+        return
+
+    extra = ''
+    if os.name == 'posix':
+        import subprocess
+        rv = subprocess.Popen(['locale', '-a'], stdout=subprocess.PIPE,
+                              stderr=subprocess.PIPE).communicate()[0]
+        good_locales = set()
+        has_c_utf8 = False
+
+        # Make sure we're operating on text here.
+        if isinstance(rv, bytes):
+            rv = rv.decode('ascii', 'replace')
+
+        for line in rv.splitlines():
+            locale = line.strip()
+            if locale.lower().endswith(('.utf-8', '.utf8')):
+                good_locales.add(locale)
+                if locale.lower() in ('c.utf8', 'c.utf-8'):
+                    has_c_utf8 = True
+
+        extra += '\n\n'
+        if not good_locales:
+            extra += (
+                'Additional information: on this system no suitable UTF-8\n'
+                'locales were discovered.  This most likely requires resolving\n'
+                'by reconfiguring the locale system.'
+            )
+        elif has_c_utf8:
+            extra += (
+                'This system supports the C.UTF-8 locale which is recommended.\n'
+                'You might be able to resolve your issue by exporting the\n'
+                'following environment variables:\n\n'
+                '    export LC_ALL=C.UTF-8\n'
+                '    export LANG=C.UTF-8'
+            )
+        else:
+            extra += (
+                'This system lists a couple of UTF-8 supporting locales that\n'
+                'you can pick from.  The following suitable locales were\n'
+                'discovered: %s'
+            ) % ', '.join(sorted(good_locales))
+
+        bad_locale = None
+        for locale in os.environ.get('LC_ALL'), os.environ.get('LANG'):
+            if locale and locale.lower().endswith(('.utf-8', '.utf8')):
+                bad_locale = locale
+            if locale is not None:
+                break
+        if bad_locale is not None:
+            extra += (
+                '\nocrmypdf discovered that you exported a UTF-8 locale\n'
+                'but the locale system could not pick up from it because\n'
+                'it does not exist.  The exported locale is "%s" but it\n'
+                'is not supported'
+            ) % bad_locale
+
+    raise RuntimeError('ocrmypdf will abort further execution because Python 3 '
+                       'was configured to use ASCII as encoding for the '
+                       'environment.' + extra)

ocrmypdf/exceptions.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # © 2016 James R. Barlow: github.com/jbarlow83
 
 
@@ -20,7 +19,7 @@ class ExitCode(IntEnum):
 
 
 class ExitCodeException(Exception):
-    pass
+    exit_code = ExitCode.other_error
 
 
 class PdfMergeFailedError(ExitCodeException):

ocrmypdf/exec/__init__.py

@@ -1,11 +1,48 @@
-#!/usr/bin/env python3
 # © 2016 James R. Barlow: github.com/jbarlow83
 
 """Wrappers to manage subprocess calls"""
 
 import os
+import re
+import sys
+from subprocess import run, STDOUT, PIPE, CalledProcessError
+from ..exceptions import MissingDependencyError
 
 
 def get_program(name):
+    "Check environment variables for overrides to this program"
     envvar = 'OCRMYPDF_' + name.upper()
     return os.environ.get(envvar, name)
+
+
+def get_version(program, *, 
+        version_arg='--version', regex=r'(\d+(\.\d+)*)'):
+    "Get the version of the specified program, "
+    args_prog = [
+        get_program(program),
+        version_arg
+    ]
+    try:
+        proc = run(
+            args_prog, close_fds=True, universal_newlines=True,
+            stdout=PIPE, stderr=STDOUT, check=True)
+        output = proc.stdout
+    except CalledProcessError as e:
+        if get_program(program) == program:
+            raise MissingDependencyError(
+                "Could not find program '{}' on the PATH".format(
+                    program)) from e
+        else:
+            raise MissingDependencyError(
+                "Could not find program '{}'".format(
+                    get_program(program))) from e            
+
+    try:
+        version = re.match(regex, output.strip()).group(1)
+    except AttributeError as e:
+        raise MissingDependencyError(
+            ("The program '{}' did not report its version. "
+            "Message was:\n{}").format(program, output)
+        )
+
+    return version

ocrmypdf/exec/ghostscript.py

@@ -1,5 +1,4 @@
-#!/usr/bin/env python3
-# © 2015 James R. Barlow: github.com/jbarlow83
+# © 2017 James R. Barlow: github.com/jbarlow83
 
 from tempfile import NamedTemporaryFile
 from subprocess import run, PIPE, STDOUT, CalledProcessError
@@ -7,27 +6,15 @@ from shutil import copy
 from functools import lru_cache
 import re
 import sys
-from . import get_program
-from ..exceptions import SubprocessOutputError
 from PIL import Image
+from . import get_program, get_version
+from ..exceptions import SubprocessOutputError, MissingDependencyError
+from ..helpers import fspath
 
 
 @lru_cache(maxsize=1)
 def version():
-    args_gs = [
-        get_program('gs'),
-        '--version'
-    ]
-    try:
-        version = check_output(
-                args_gs, close_fds=True, universal_newlines=True,
-                stderr=STDOUT)
-    except CalledProcessError as e:
-        print("Could not find Ghostscript executable on system PATH.",
-              file=sys.stderr)
-        raise MissingDependencyError from e
-
-    return version.strip()
+    return get_version('gs')
 
 
 def _gs_error_reported(stream):
@@ -43,13 +30,13 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
     (xres, yres) even if those numbers are noninteger. The image's DPI will
      be overridden with the values in page_dpi.
     
-    :param input_file: 
-    :param output_file: 
+    :param input_file: pathlike
+    :param output_file: pathlike
     :param xres: resolution at which to rasterize page
     :param yres: 
     :param raster_device: 
     :param log: 
-    :param pageno: page number to rasterize
+    :param pageno: page number to rasterize (beginning at page 1)
     :param page_dpi: resolution tuple (x, y) overriding output image DPI 
     :return: 
     """
@@ -67,11 +54,11 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
             '-sDEVICE=%s' % raster_device,
             '-dFirstPage=%i' % pageno,
             '-dLastPage=%i' % pageno,
-            '-o', tmp.name,
             '-r{0}x{1}'.format(str(int_res[0]), str(int_res[1])),
-            input_file
+            '-o', tmp.name,
+            fspath(input_file)
         ]
-
+        
         p = run(args_gs, stdout=PIPE, stderr=STDOUT,
                 universal_newlines=True)
         if _gs_error_reported(p.stdout):
@@ -87,6 +74,7 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
         # if the resolution happens to be fractional, then the discrepancy
         # would change the size of the output page, especially if the DPI
         # is quite low. Resize the image to the expected size
+
         tmp.seek(0)
         with Image.open(tmp) as im:
             expected_size = round(im.size[0] / int_res[0] * res[0]), \
@@ -95,13 +83,14 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
                 log.debug(
                     "Ghostscript: resize output image {} -> {}".format(
                         im.size, expected_size))
-                im.resize(expected_size).save(output_file, dpi=page_dpi)
+                im.resize(expected_size).save(
+                    fspath(output_file), dpi=page_dpi)
             else:
-                copy(tmp.name, output_file)
+                copy(tmp.name, fspath(output_file))
 
 
 def generate_pdfa(pdf_pages, output_file, compression, log,
-                  threads=1, pdf_version='1.5'):
+                  threads=1, pdf_version='1.5', pdfa_part='2'):
     compression_args = []
     if compression == 'jpeg':
         compression_args = [
@@ -137,7 +126,7 @@ def generate_pdfa(pdf_pages, output_file, compression, log,
             "-sProcessColorModel=DeviceRGB"
         ] + compression_args + [
             "-dJPEGQ=95",
-            "-dPDFA=2",
+            "-dPDFA=" + pdfa_part,
             "-dPDFACompatibilityPolicy=1",
             "-sOutputFile=" + gs_pdf.name,
         ]
@@ -164,4 +153,4 @@ def generate_pdfa(pdf_pages, output_file, compression, log,
             copy(gs_pdf.name, output_file)
         else:
             log.error('Ghostscript PDF/A rendering failed')
-            raise SubprocessOutputError()
+            raise SubprocessOutputError()

ocrmypdf/exec/qpdf.py

@@ -1,33 +1,21 @@
-#!/usr/bin/env python3
-# © 2015 James R. Barlow: github.com/jbarlow83
+# © 2017 James R. Barlow: github.com/jbarlow83
 
-from subprocess import CalledProcessError, STDOUT, PIPE, run, check_output
+from subprocess import CalledProcessError, STDOUT, PIPE, run
 from functools import lru_cache
 import sys
 import os
 import re
+import resource
 
 from ..exceptions import InputFileError, SubprocessOutputError, \
     MissingDependencyError, EncryptedPdfError
-from . import get_program
+from . import get_program, get_version
+from ..helpers import re_symlink
 
 
 @lru_cache(maxsize=1)
 def version():
-    args_qpdf = [
-        get_program('qpdf'),
-        '--version'
-    ]
-    try:
-        p = run(args_qpdf, universal_newlines=True, stderr=STDOUT,
-                stdout=PIPE)
-    except CalledProcessError as e:
-        print("Could not find qpdf executable on system PATH.",
-              file=sys.stderr)
-        raise MissingDependencyError() from e
-
-    qpdf_version = re.match(r'qpdf version (.+)', p.stdout).group(1)
-    return qpdf_version
+    return get_version('qpdf', regex=r'qpdf version (.+)')    
 
 
 def check(input_file, log=None):
@@ -41,7 +29,8 @@ def check(input_file, log=None):
         import logging as log
 
     try:
-        check_output(args_qpdf, stderr=STDOUT, universal_newlines=True)
+        run(args_qpdf, stderr=STDOUT, stdout=PIPE, universal_newlines=True, 
+            check=True)
     except CalledProcessError as e:
         if e.returncode == 2:
             log.error("{0}: not a valid PDF, and could not repair it.".format(
@@ -70,16 +59,14 @@ def repair(input_file, output_file, log):
         get_program('qpdf'), input_file, output_file
     ]
     try:
-        check_output(args_qpdf, stderr=STDOUT, universal_newlines=True)
+        run(args_qpdf, stderr=STDOUT, stdout=PIPE, universal_newlines=True, 
+            check=True)
     except CalledProcessError as e:
         if e.returncode == 3 and e.output.find("operation succeeded"):
             log.debug('qpdf found and fixed errors: ' + e.output)
-            log.debug(e.output)
             return
 
         if _probably_encrypted(e):
-            log.error("{0}: this PDF is password-protected - password must "
-                      "be removed for OCR".format(input_file))
             raise EncryptedPdfError() from e
         elif e.returncode == 2:
             log.error("{0}: not a valid PDF, and could not repair it.".format(
@@ -95,9 +82,9 @@ def repair(input_file, output_file, log):
 
 def get_npages(input_file, log):
     try:
-        pages = check_output(
+        pages = run(
             [get_program('qpdf'), '--show-npages', input_file],
-             universal_newlines=True, close_fds=True)
+            universal_newlines=True, check=True, stdout=PIPE, stderr=STDOUT)
     except CalledProcessError as e:
         if e.returncode == 2 and e.output.find('No such file'):
             log.error(e.output)
@@ -120,18 +107,97 @@ def split_pages(input_file, work_folder, npages):
         run(args_qpdf, check=True)
 
 
-def merge(input_files, output_file, min_version=None):
+def _merge_inner(input_files, output_file, min_version=None, log=None):
     """Merge the list of input files (all filenames) into the output file.
 
     The input files may contain one or more pages.
     """
+
+    # Single page 'merges' should still be attempted to that the same error
+    # checking is applied to single page case
+
     version_arg = ['--min-version={}'.format(min_version)] \
                   if min_version else []
 
+    if log is None:
+        import logging as log
+
     args_qpdf = [
         get_program('qpdf')
     ] + version_arg + [
         input_files[0], '--pages'
     ] + input_files + ['--', output_file]
-    run(args_qpdf, check=True)
+
+    try:
+        run(args_qpdf, check=True, stderr=PIPE, universal_newlines=True)
+    except CalledProcessError as e:
+        if e.returncode == 3 and \
+                e.stderr.find("unknown token while reading object") and \
+                e.stderr.find("operation succeeded"):
+            # Only whitelist the 'unknown token' problem (decimal/string issue)
+            # qpdf issue #165
+            log.warning('qpdf found and fixed errors: ' + e.stderr)
+            return
+        raise e from e
+
+
+def merge(input_files, output_file, min_version=None, log=None, max_files=None):
+    """Merge the list of input files (all filenames) into the output file.
+
+    The input files may contain one or more pages.
+
+    """
+    # qpdf requires that every file that contributes to the output has a file 
+    # descriptor that remains open. That means, given our approach of one 
+    # intermediate PDF per, we can practically hit the number of file 
+    # descriptors. 
+
+    if max_files is None or max_files < 2:
+        # Find out how many open file descriptors we can get away with
+        ulimits = resource.getrlimit(resource.RLIMIT_NOFILE)
+        max_open_files = ulimits[0]
+        max_files = max_open_files // 2  # Conservative guess
+
+    # We'll write things alongside the output file
+    output_dir = os.path.dirname(output_file)
+
+    import random
+    import string    
+
+    def randstr():
+        return ''.join(random.sample(string.ascii_lowercase, 6))
+
+    # How many files to grab at once, merging all their contents
+    step_size = max_files
+
+    workqueue = input_files.copy()
+    counter = 1
+    next_workqueue = []
+    while len(workqueue) > 1 or len(next_workqueue) > 0:
+        # Take n files out of the queue
+        n = min(step_size, len(workqueue))
+        job = workqueue[0:n]
+        del workqueue[0:n]
+        log.debug('merging ' + repr(job))
+
+        # Merge them into 1 file, which will contain n^depth pages
+        merge_file = os.path.join(
+            output_dir, "merge-{:06d}-{}.pdf".format(counter, randstr()))
+        counter += 1
+        _merge_inner(job, merge_file, min_version=min_version, log=log)
+
+        # On the next 
+        next_workqueue.append(merge_file)
+        log.debug('next_workqueue ' + repr(next_workqueue))
+
+        # If we're out of things to do in this queue, move on to the next
+        # queue. On the counter-th pass of the workqueue we can chew through
+        # (step_size)**N pages, so on most systems the second pass finishes
+        # the job.
+        if len(workqueue) == 0:
+            workqueue = next_workqueue
+            next_workqueue = []
+
+    re_symlink(workqueue.pop(), output_file, log)
+
 

ocrmypdf/exec/tesseract.py

@@ -1,21 +1,19 @@
-#!/usr/bin/env python3
-# © 2015 James R. Barlow: github.com/jbarlow83
+# © 2017 James R. Barlow: github.com/jbarlow83
 
 import sys
 import os
 import re
 import shutil
 from functools import lru_cache
-from ..exceptions import MissingDependencyError, TesseractConfigError
-from ..helpers import page_number
-from . import get_program
 from collections import namedtuple
 from textwrap import dedent
 import PyPDF2 as pypdf
+from subprocess import PIPE, CalledProcessError, \
+    TimeoutExpired, check_output, STDOUT
 
-from subprocess import Popen, PIPE, CalledProcessError, \
-    TimeoutExpired, check_output, STDOUT, DEVNULL
-
+from ..exceptions import MissingDependencyError, TesseractConfigError
+from ..helpers import page_number
+from . import get_program, get_version
 
 OrientationConfidence = namedtuple(
     'OrientationConfidence',
@@ -41,48 +39,36 @@ HOCR_TEMPLATE = """<?xml version="1.0" encoding="UTF-8"?>
 
 @lru_cache(maxsize=1)
 def version():
-    args_tess = [
-        get_program('tesseract'),
-        '--version'
-    ]
-    try:
-        versions = check_output(
-                args_tess, close_fds=True, universal_newlines=True,
-                stderr=STDOUT)
-    except CalledProcessError as e:
-        print("Could not find Tesseract executable on system PATH.",
-              file=sys.stderr)
-        raise MissingDependencyError from e
-
-    tesseract_version = re.match(r'tesseract\s(.+)', versions).group(1)
-    return tesseract_version
+    return get_version('tesseract', regex=r'tesseract\s(.+)')
 
 
 def v4():
     "Is this Tesseract v4.0?"
-    return (version() >= '4')
+    return version() >= '4'
 
 
+@lru_cache(maxsize=1)
 def has_textonly_pdf():
-    if version() == '4.00.00alpha':
-        # textonly_pdf added during the 4.00.00alpha cycle, so we must test
-        # more carefully to see if it is present
-        args_tess = [
-            get_program('tesseract'),
-            '--print-parameters'
-        ]
-        try:
-            params = check_output(
-                    args_tess, close_fds=True, universal_newlines=True,
-                    stderr=STDOUT)
-        except CalledProcessError as e:
-            print("Could not --print-parameters from tesseract",
-                  file=sys.stderr)
-            raise MissingDependencyError from e
-        if 'textonly_pdf' in params:
-            return True
-    else:
-        return v4()
+    """Does Tesseract have textonly_pdf capability?
+    
+    Available in 3.05.01, and v4.00.00alpha since January 2017. Best to 
+    parse the parameter list
+    """
+    args_tess = [
+        get_program('tesseract'),
+        '--print-parameters'
+    ]
+    params = ''
+    try:
+        params = check_output(
+                args_tess, universal_newlines=True, stderr=STDOUT)
+    except CalledProcessError as e:
+        print("Could not --print-parameters from tesseract",
+              file=sys.stderr)
+        raise MissingDependencyError from e
+    if 'textonly_pdf' in params:
+        return True
+    return False
 
 
 def psm():
@@ -98,8 +84,7 @@ def languages():
     ]
     try:
         langs = check_output(
-                args_tess, close_fds=True, universal_newlines=True,
-                stderr=STDOUT)
+                args_tess, universal_newlines=True, stderr=STDOUT)
     except CalledProcessError as e:
         msg = dedent("""Tesseract failed to report available languages.
         Output from Tesseract:
@@ -111,12 +96,12 @@ def languages():
     return set(lang.strip() for lang in langs.splitlines()[1:])
 
 
-def tess_base_args(languages, engine_mode):
+def tess_base_args(langs, engine_mode):
     args = [
         get_program('tesseract'),
     ]
-    if languages:
-        args.extend(['-l', '+'.join(languages)])
+    if langs:
+        args.extend(['-l', '+'.join(langs)])
     if engine_mode is not None and v4():
         args.extend(['--oem', str(engine_mode)])
     return args
@@ -124,7 +109,7 @@ def tess_base_args(languages, engine_mode):
 
 def get_orientation(input_file, language: list, engine_mode, timeout: float,
                     log):
-    args_tesseract = tess_base_args(language, engine_mode) + [
+    args_tesseract = tess_base_args(['osd'], engine_mode) + [
         psm(), '0',
         input_file,
         'stdout'
@@ -132,19 +117,18 @@ def get_orientation(input_file, language: list, engine_mode, timeout: float,
 
     try:
         stdout = check_output(
-            args_tesseract, close_fds=True, stderr=STDOUT,
-            universal_newlines=True, timeout=timeout)
+            args_tesseract, stderr=STDOUT, timeout=timeout)
     except TimeoutExpired:
         return OrientationConfidence(angle=0, confidence=0.0)
     except CalledProcessError as e:
         tesseract_log_output(log, e.output, input_file)
-        if ('Too few characters. Skipping this page' in e.output or
-                'Image too large' in e.output):
+        if (b'Too few characters. Skipping this page' in e.output or
+                b'Image too large' in e.output):
             return OrientationConfidence(0, 0)
         raise e from e
     else:
         osd = {}
-        for line in stdout.splitlines():
+        for line in stdout.decode().splitlines():
             line = line.strip()
             parts = line.split(':', maxsplit=2)
             if len(parts) == 2:
@@ -169,8 +153,17 @@ def get_orientation(input_file, language: list, engine_mode, timeout: float,
 
 
 def tesseract_log_output(log, stdout, input_file):
-    lines = stdout.splitlines()
     prefix = "{0:4d}: [tesseract] ".format(page_number(input_file))
+
+    try:
+        text = stdout.decode()
+    except UnicodeDecodeError:
+        log.error(prefix + "command line output was not utf-8. " +
+            "This usually means Tesseract's language packs do not match "
+            "the installed version of Tesseract.")
+        text = stdout.decode('utf-8', 'backslashreplace')
+
+    lines = text.splitlines()
     for line in lines:
         if line.startswith("Tesseract Open Source"):
             continue
@@ -211,7 +204,8 @@ def _generate_null_hocr(output_hocr, output_sidecar, image):
 
 def generate_hocr(input_file, output_files, language: list, engine_mode,
                   tessconfig: list,
-                  timeout: float, pagesegmode: int, log):
+                  timeout: float, pagesegmode: int, user_words, user_patterns,
+                  log):
 
     output_hocr = next(o for o in output_files if o.endswith('.hocr'))
     output_sidecar = next(o for o in output_files if o.endswith('.txt'))
@@ -222,10 +216,17 @@ def generate_hocr(input_file, output_files, language: list, engine_mode,
     if pagesegmode is not None:
         args_tesseract.extend([psm(), str(pagesegmode)])
 
+    if user_words:
+        args_tesseract.extend(['--user-words', user_words])
+
+    if user_patterns:
+        args_tesseract.extend(['--user-patterns', user_patterns])
+
     # Reminder: test suite tesseract spoofers will break after any changes
     # to the number of order parameters here
     # Tesseract 3.04 requires the order here to be "hocr txt" and will fail
     # on "txt hocr"
+
     args_tesseract.extend([
         input_file,
         prefix,
@@ -235,8 +236,8 @@ def generate_hocr(input_file, output_files, language: list, engine_mode,
     try:
         log.debug(args_tesseract)
         stdout = check_output(
-            args_tesseract, close_fds=True, stderr=STDOUT,
-            universal_newlines=True, timeout=timeout)
+            args_tesseract, stderr=STDOUT,
+            timeout=timeout)
     except TimeoutExpired:
         # Generate a HOCR file with no recognized text if tesseract times out
         # Temporary workaround to hocrTransform not being able to function if
@@ -245,9 +246,9 @@ def generate_hocr(input_file, output_files, language: list, engine_mode,
         _generate_null_hocr(output_hocr, output_sidecar, input_file)
     except CalledProcessError as e:
         tesseract_log_output(log, e.output, input_file)
-        if 'read_params_file: parameter not found' in e.output:
+        if b'read_params_file: parameter not found' in e.output:
             raise TesseractConfigError() from e
-        if 'Image too large' in e.output:
+        if b'Image too large' in e.output:
             _generate_null_hocr(output_hocr, output_sidecar, input_file)
             return
 
@@ -277,13 +278,18 @@ def use_skip_page(text_only, skip_pdf, output_pdf, output_text):
     with open(output_pdf, 'wb') as out:
         pdf_out = pypdf.PdfFileWriter()
         w, h = page0.mediaBox.getWidth(), page0.mediaBox.getHeight()
+        # If skip page has a /Rotate key, replicate the rotation
+        rotation = int(page0.get('/Rotate', 0))
+        if rotation % 180 == 90:
+            w, h = h, w
         pdf_out.addBlankPage(w, h)
         pdf_out.write(out)
 
 
 def generate_pdf(*, input_image, skip_pdf, output_pdf, output_text,
                  language: list, engine_mode, text_only: bool,
-                 tessconfig: list, timeout: float, pagesegmode: int, log):
+                 tessconfig: list, timeout: float, pagesegmode: int,
+                 user_words, user_patterns, log):
     '''Use Tesseract to render a PDF.
 
     input_image -- image to analyze
@@ -306,10 +312,17 @@ def generate_pdf(*, input_image, skip_pdf, output_pdf, output_text,
     if text_only:
         args_tesseract.extend(['-c', 'textonly_pdf=1'])
 
+    if user_words:
+        args_tesseract.extend(['--user-words', user_words])
+
+    if user_patterns:
+        args_tesseract.extend(['--user-patterns', user_patterns])
+
     prefix = os.path.splitext(output_pdf)[0]  # Tesseract appends suffixes
 
-    # Reminder: test suite tesseract spoofers will break after any changes
+    # Reminder: test suite tesseract spoofers might break after any changes
     # to the number of order parameters here
+
     args_tesseract.extend([
         input_image,
         prefix,
@@ -320,8 +333,8 @@ def generate_pdf(*, input_image, skip_pdf, output_pdf, output_text,
     try:
         log.debug(args_tesseract)
         stdout = check_output(
-            args_tesseract, close_fds=True, stderr=STDOUT,
-            universal_newlines=True, timeout=timeout)
+            args_tesseract, stderr=STDOUT,
+            timeout=timeout)
         if os.path.exists(prefix + '.txt'):
             shutil.move(prefix + '.txt', output_text)
     except TimeoutExpired:
@@ -329,10 +342,10 @@ def generate_pdf(*, input_image, skip_pdf, output_pdf, output_text,
         use_skip_page(text_only, skip_pdf, output_pdf, output_text)
     except CalledProcessError as e:
         tesseract_log_output(log, e.output, input_image)
-        if 'read_params_file: parameter not found' in e.output:
+        if b'read_params_file: parameter not found' in e.output:
             raise TesseractConfigError() from e
 
-        if 'Image too large' in e.output:
+        if b'Image too large' in e.output:
             use_skip_page(text_only, skip_pdf, output_pdf, output_text)
             return
         raise e from e

ocrmypdf/exec/unpaper.py

@@ -1,27 +1,14 @@
-#!/usr/bin/env python3
 # © 2015 James R. Barlow: github.com/jbarlow83
 # unpaper documentation:
 # https://github.com/Flameeyes/unpaper/blob/master/doc/basic-concepts.md
 
-from subprocess import CalledProcessError, STDOUT, check_output, check_call
+from subprocess import CalledProcessError, STDOUT, check_output
 from tempfile import NamedTemporaryFile
 import sys
 import os
 from functools import lru_cache
 from ..exceptions import MissingDependencyError
-from . import get_program
-
-
-@lru_cache(maxsize=1)
-def version():
-    args_unpaper = [
-        get_program('unpaper'),
-        '--version'
-    ]
-    version = check_output(
-        args_unpaper, close_fds=True, universal_newlines=True,
-        stderr=STDOUT, timeout=5)
-    return version.strip()
+from . import get_program, get_version
 
 
 try:
@@ -31,6 +18,11 @@ except ImportError:
     raise
 
 
+@lru_cache(maxsize=1)
+def version():
+    return get_version('unpaper')
+
+
 def run(input_file, output_file, dpi, log, mode_args):
     args_unpaper = [
         get_program('unpaper'),

ocrmypdf/helpers.py

@@ -1,9 +1,9 @@
-#!/usr/bin/env python3
 # © 2016 James R. Barlow: github.com/jbarlow83
 
 from functools import partial
 from collections.abc import Iterable
 from contextlib import suppress, contextmanager
+from pathlib import Path
 import sys
 import os
 
@@ -32,7 +32,7 @@ def re_symlink(input_file, soft_link_name, log=None):
                 "%s exists and is not a link" % soft_link_name)
         try:
             os.unlink(soft_link_name)
-        except:
+        except OSError:
             prdebug("Can't unlink %s" % (soft_link_name))
 
     if not os.path.exists(input_file):
@@ -63,26 +63,73 @@ def is_file_writable(test_file):
     can replace it atomically. Before doing the OCR work, make sure
     the location is writable.
     """
-    if os.path.exists(test_file):
+    p = Path(test_file)
+
+    if p.is_symlink():
+        # Python 3.5 does not accept parameters for Path.resolve() and behaves
+        # as if strict=True (throws an exception on failure). Python 3.6 
+        # defaults to strict=False. This implements strict=False like behavior
+        # for Python 3.5.
+        if sys.version_info[0:2] <= (3, 5):
+            resolve = lambda: Path(os.path.realpath(str(p)))
+        else:
+            resolve = lambda: p.resolve(strict=False)
+        p = resolve()
+
+    # p.is_file() throws an exception in some cases
+    if p.exists() and p.is_file():
         return os.access(
-            test_file, os.W_OK,
+            str(p), os.W_OK,
             effective_ids=(os.access in os.supports_effective_ids))
     else:
         try:
-            fp = open(test_file, 'wb')
-        except OSError as e:
+            fp = p.open('wb')
+        except OSError:
             return False
         else:
             fp.close()
             with suppress(OSError):
-                os.unlink(test_file)
+                p.unlink()
         return True
 
 
-@contextmanager
-def universal_open(p, *args, **kwargs):
-    "Work around Python 3.5's inability to open(pathlib.Path())"
-    try:
-        yield p.open(*args, **kwargs)
-    except AttributeError:
-        yield open(p, *args, **kwargs)
+if sys.version_info[0:2] <= (3, 5):
+    def universal_open(p, *args, **kwargs):
+        "Work around Python 3.5's inability to open(pathlib.Path())"
+        try:
+            return p.open(*args, **kwargs)
+        except AttributeError:
+            return open(p, *args, **kwargs)
+
+
+    def fspath(path):
+        import pathlib
+        '''https://www.python.org/dev/peps/pep-0519/#os'''
+        if isinstance(path, (str, bytes)):
+            return path
+
+        # Work from the object's type to match method resolution of other magic
+        # methods.
+        path_type = type(path)
+        try:
+            path = path_type.__fspath__(path)
+        except AttributeError:
+            # Added for Python 3.5 support.
+            if isinstance(path, pathlib.Path):
+                return str(path)
+            elif hasattr(path_type, '__fspath__'):
+                raise
+        else:
+            if isinstance(path, (str, bytes)):
+                return path
+            else:
+                raise TypeError("expected __fspath__() to return str or bytes, "
+                                "not " + type(path).__name__)
+
+        raise TypeError(
+            "expected str, bytes, pathlib.Path or os.PathLike object, not "
+            + path_type.__name__)
+
+else:
+    universal_open = open
+    fspath = os.fspath

ocrmypdf/hocrtransform.py

@@ -1,12 +1,33 @@
 #!/usr/bin/env python3
-##############################################################################
-# Copyright (c) 2013-14: fritz-hh from Github
-#   (https://github.com/fritz-hh)
 #
-# Copyright (c) 2010: Jonathan Brinley from Github
-#   (https://github.com/jbrinley/HocrConverter)
-#   Initial version by Jonathan Brinley, jonathanbrinley@gmail.com
-##############################################################################
+# Copyright (c) 2010, Jonathan Brinley
+#   Original version from: https://github.com/jbrinley/HocrConverter
+#
+# Copyright (c) 2013-14, Julien Pfefferkorn
+#   Modifications
+#
+# Copyright (c) 2015-16, James R. Barlow
+#   Set text to transparent
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
 from reportlab.pdfgen.canvas import Canvas
 from reportlab.lib.units import inch
 from xml.etree import ElementTree

ocrmypdf/leptonica.py

@@ -14,6 +14,7 @@ from ctypes.util import find_library
 from .lib._leptonica import ffi
 from functools import lru_cache
 from enum import Enum
+from .helpers import fspath
 
 lept = ffi.dlopen(find_library('lept'))
 
@@ -208,26 +209,27 @@ class Pix:
             return 'P'
 
     @classmethod
-    def read(cls, filename):
+    def read(cls, path):
         """Load an image file into a PIX object.
 
         Leptonica can load TIFF, PNM (PBM, PGM, PPM), PNG, and JPEG.  If
         loading fails then the object will wrap a C null pointer.
         """
+        filename = fspath(path)
         with LeptonicaErrorTrap():
-            return cls(lept.pixRead(
-                filename.encode(sys.getfilesystemencoding())))
+            return cls(lept.pixRead(os.fsencode(filename)))
 
     def write_implied_format(
-            self, filename, jpeg_quality=0, jpeg_progressive=0):
+            self, path, jpeg_quality=0, jpeg_progressive=0):
         """Write pix to the filename, with the extension indicating format.
 
         jpeg_quality -- quality (iff JPEG; 1 - 100, 0 for default)
         jpeg_progressive -- (iff JPEG; 0 for baseline seq., 1 for progressive)
         """
+        filename = fspath(path)
         with LeptonicaErrorTrap():
             lept.pixWriteImpliedFormat(
-                filename.encode(sys.getfilesystemencoding()),
+                os.fsencode(filename),
                 self._pix, jpeg_quality, jpeg_progressive)
 
     def topil(self):
@@ -564,35 +566,4 @@ if __name__ == '__main__':
     parser_deskew.set_defaults(func=deskew)
 
     args = parser.parse_args()
-
-    if get_leptonica_version() != u'leptonica-1.69':
-        print("Unexpected leptonica version: %s" % getLeptonicaVersion())
-
-    args.func(args)
-
-
-def test_skew_angle():
-    from PIL import Image, ImageDraw
-    from tempfile import NamedTemporaryFile
-
-    im = Image.new(mode='1', size=(1000, 1000), color=1)
-
-    draw = ImageDraw.Draw(im)
-    for n in range(20):
-        draw.line([(50, 25 + 50*n), (950, 25 + 50*n)], width=1)
-    del draw
-
-    test_angles = [0.1 * ang for ang in range(1, 10)] + \
-                  [float(ang) for ang in range(1, 7)]
-    test_angles += [-ang for ang in test_angles]
-    test_angles = sorted(test_angles)
-
-    for rotate_angle in test_angles:
-        rotated_im = im.rotate(rotate_angle)
-        with NamedTemporaryFile(prefix='lept-skew', suffix='.png', delete=True) as tmpfile:
-            rotated_im.save(tmpfile)
-            pix = pixRead(tmpfile.name)
-            angle, confidence = pixFindSkew(pix)
-            print('{0} {1}  {2}'.format(rotate_angle, angle, confidence), file=sys.stderr)
-
-
+    args.func(args)

ocrmypdf/lib/compile_leptonica.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 # © 2017 James R. Barlow: github.com/jbarlow83
 
 from cffi import FFI
@@ -183,10 +184,39 @@ pixColorMagnitude(PIX     *pixs,
                   l_int32  bwhite,
                   l_int32  type);
 
-void
+PIX *
+pixMaskOverColorPixels(PIX     *pixs,
+                       l_int32  threshdiff,
+                       l_int32  mindist);
+
+l_int32
+pixGetAverageMaskedRGB(PIX        *pixs,
+                       PIX        *pixm,
+                       l_int32     x,
+                       l_int32     y,
+                       l_int32     factor,
+                       l_int32     type,
+                       l_float32  *prval,
+                       l_float32  *pgval,
+                       l_float32  *pbval);
+
+PIX * 
+pixGlobalNormRGB(PIX * 	pixd,
+                 PIX * 	pixs,
+                 l_int32 	rval,
+                 l_int32 	gval,
+                 l_int32 	bval,
+                 l_int32 	mapval); 
+
+PIX *
+pixInvert(PIX * pixd,
+          PIX * pixs);
+
+void                 
 boxDestroy(BOX  **pbox);
 
-void lept_free(void *ptr);
+void
+lept_free(void *ptr);  
 """)
 
 

ocrmypdf/pdfa.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # © 2015 James R. Barlow: github.com/jbarlow83
 #
 # Generate a PDFA_def.ps file for Ghostscript >= 9.14

ocrmypdf/pdfinfo.py

@@ -662,10 +662,6 @@ class PageInfo:
         else:
             return '1.5'
 
-    @property
-    def images(self):
-        return self._pageinfo['images']
-
     def __repr__(self):
         return (
             '<PageInfo '

ocrmypdf/pipeline.py

@@ -1,21 +1,12 @@
-#!/usr/bin/env python3
 # © 2016 James R. Barlow: github.com/jbarlow83
 
 from contextlib import suppress
-from tempfile import mkdtemp
-from functools import partial
+from shutil import copyfileobj
 import sys
 import os
-import re
 import shutil
-import warnings
-import multiprocessing
-import atexit
-import textwrap
 import img2pdf
-import logging
-import argparse
-
+import re
 import PyPDF2 as pypdf
 from PIL import Image
 
@@ -23,10 +14,11 @@ from ruffus import formatter, regex, Pipeline, suffix
 
 from .hocrtransform import HocrTransform
 from .pdfinfo import PdfInfo, Encoding, Colorspace
-from .pdfa import generate_pdfa_ps, file_claims_pdfa
+from .pdfa import generate_pdfa_ps
 from .helpers import re_symlink, is_iterable_notstr, page_number
 from .exec import ghostscript, tesseract, qpdf
-from .exceptions import *
+from .exceptions import PdfMergeFailedError, UnsupportedImageFormatError, \
+    DpiError, PriorOcrFoundError, InputFileError
 from . import leptonica
 from . import PROGRAM_NAME, VERSION
 
@@ -51,6 +43,8 @@ class JobContext:
 
     def __init__(self):
         self.pdfinfo = None
+        self.options = None
+        self.work_folder = None
 
     def generate_pdfinfo(self, infile):
         self.pdfinfo = PdfInfo(infile)
@@ -156,6 +150,22 @@ def triage_image_file(input_file, output_file, log, options):
         raise UnsupportedImageFormatError() from e
 
 
+def _pdf_guess_version(input_file, search_window=1024):
+    """Try to find version signature at start of file.
+    
+    Not robust enough to deal with appended files.
+
+    Returns empty string if not found, indicating file is probably not PDF.
+    """
+
+    with open(input_file, 'rb') as f:
+        signature = f.read(1024)
+    m = re.search(br'%PDF-(\d\.\d)', signature)
+    if m:
+        return m.group(1)
+    return ''
+
+
 def triage(
         input_file,
         output_file,
@@ -164,14 +174,12 @@ def triage(
 
     options = context.get_options()
     try:
-        with open(input_file, 'rb') as f:
-            signature = f.read(4)
-            if signature == b'%PDF':
-                if options.image_dpi:
-                    log.warning("Argument --image-dpi ignored because the "
-                                "input file is a PDF, not an image.")
-                re_symlink(input_file, output_file, log)
-                return
+        if _pdf_guess_version(input_file):
+            if options.image_dpi:
+                log.warning("Argument --image-dpi ignored because the "
+                            "input file is a PDF, not an image.")
+            re_symlink(input_file, output_file, log)
+            return
     except EnvironmentError as e:
         log.error(e)
         raise InputFileError() from e
@@ -189,7 +197,8 @@ def repair_pdf(
     pdfinfo = PdfInfo(output_file)
 
     if pdfinfo.has_userunit and options.output_type == 'pdfa':
-        log.error("This input file uses a PDF feature that is not supported "
+        log.error(
+            "This input file uses a PDF feature that is not supported "
             "by Ghostscript, so you cannot use --output-type=pdfa for this "
             "file. (Specifically, it uses the PDF-1.6 /UserUnit feature to "
             "support very large or small page sizes, and Ghostscript cannot "
@@ -202,7 +211,7 @@ def repair_pdf(
 
 
 def get_pageinfo(input_file, context):
-    pageno = int(os.path.basename(input_file)[0:6]) - 1
+    pageno = page_number(input_file) - 1
     pageinfo = context.get_pdfinfo()[pageno]
     return pageinfo
 
@@ -236,7 +245,30 @@ def get_canvas_square_dpi(pageinfo, options):
 def is_ocr_required(pageinfo, log, options):
     page = pageinfo.pageno + 1
     ocr_required = True
-    if not pageinfo.images:
+
+    if pageinfo.has_text:
+        msg = "{0:4d}: page already has text! – {1}"
+
+        if not options.force_ocr and not options.skip_text:
+            log.error(msg.format(page,
+                                 "aborting (use --force-ocr to force OCR)"))
+            raise PriorOcrFoundError()
+        elif options.force_ocr:
+            log.info(msg.format(page,
+                                "rasterizing text and running OCR anyway"))
+            ocr_required = True
+        elif options.skip_text:
+            log.info(msg.format(page,
+                                "skipping all processing on this page"))
+            ocr_required = False
+    elif not pageinfo.images and not options.lossless_reconstruction:
+        # We found a page with no images and no text. That means it may
+        # have vector art that the user wants to OCR. If we determined
+        # lossless reconstruction is not possible then we have to rasterize
+        # the image. So if OCR is being forced, take that to mean YES, go 
+        # ahead and rasterize. If not forced, then pretend there's no text
+        # on the page at all so we don't lose anything.
+        # This could be made smarter by explicitly searching for vector art.
         if options.force_ocr and options.oversample:
             # The user really wants to reprocess this file
             log.info(
@@ -255,23 +287,9 @@ def is_ocr_required(pageinfo, log, options):
         else:
             log.info(
                 "{0:4d}: page has no images - "
-                "skipping all processing on this page".format(page))
-            ocr_required = False
-
-    elif pageinfo.has_text:
-        msg = "{0:4d}: page already has text! – {1}"
-
-        if not options.force_ocr and not options.skip_text:
-            log.error(msg.format(page,
-                                 "aborting (use --force-ocr to force OCR)"))
-            raise PriorOcrFoundError()
-        elif options.force_ocr:
-            log.info(msg.format(page,
-                                "rasterizing text and running OCR anyway"))
-            ocr_required = True
-        elif options.skip_text:
-            log.info(msg.format(page,
-                                "skipping all processing on this page"))
+                "skipping all processing on this page to avoid losing detail. "
+                "Use --force-ocr if you wish to perform OCR on pages that "
+                "have vector content.".format(page))
             ocr_required = False
 
     if ocr_required and options.skip_big and pageinfo.images:
@@ -406,7 +424,7 @@ def orient_page(
         with open(output_file, 'wb') as out:
             writer.write(out)
 
-        pageno = int(os.path.basename(page_pdf)[0:6]) - 1
+        pageno = page_number(page_pdf) - 1
         pdfinfo = context.get_pdfinfo()
         pdfinfo[pageno].rotation = orient_conf.angle
         context.set_pdfinfo(pdfinfo)
@@ -525,6 +543,8 @@ def ocr_tesseract_hocr(
         tessconfig=options.tesseract_config,
         timeout=options.tesseract_timeout,
         pagesegmode=options.tesseract_pagesegmode,
+        user_words=options.user_words,
+        user_patterns=options.user_patterns,
         log=log
         )
 
@@ -595,7 +615,7 @@ def select_image_layer(
         # as it accurately describes the image. It would be possible to
         # resample the image at this stage back to non-square DPI to more
         # closely resemble the input, except that the hocr renderer does not
-        # understand non-square DPI. The tess4 renderer would be fine.
+        # understand non-square DPI. The sandwich renderer would be fine.
         dpi = get_page_square_dpi(pageinfo, options)
         layout_fun = img2pdf.get_fixed_dpi_layout_fun((dpi, dpi))
 
@@ -704,7 +724,7 @@ def combine_layers(
 
     if rotation != 0:
         log.info("{0:4d}: rotating image layer {1} degrees".format(
-            page_number(image), rotation, tx, ty))
+            page_number(image), rotation))
 
     try:
         page_text.mergeRotatedScaledTranslatedPage(
@@ -755,6 +775,8 @@ def ocr_tesseract_and_render_pdf(
         tessconfig=options.tesseract_config,
         timeout=options.tesseract_timeout,
         pagesegmode=options.tesseract_pagesegmode,
+        user_words=options.user_words,
+        user_patterns=options.user_patterns,
         log=log)
 
 
@@ -783,6 +805,8 @@ def ocr_tesseract_textonly_pdf(
         tessconfig=options.tesseract_config,
         timeout=options.tesseract_timeout,
         pagesegmode=options.tesseract_pagesegmode,
+        user_words=options.user_words,
+        user_patterns=options.user_patterns,
         log=log)
 
 
@@ -812,10 +836,17 @@ def get_pdfmark(base_pdf, options):
     if options.subject:
         pdfmark['/Subject'] = options.subject
 
-    pdfmark['/Creator'] = '{0} {1} / Tesseract OCR{2} {3}'.format(
-            PROGRAM_NAME, VERSION,
-            '+PDF' if options.pdf_renderer == 'tesseract' else '',
-            tesseract.version())
+    if options.pdf_renderer == 'tesseract':
+        renderer_tag = 'OCR+PDF'
+    elif options.pdf_renderer == 'sandwich':
+        renderer_tag = 'OCR-PDF'
+    else:
+        renderer_tag = 'OCR'
+
+    pdfmark['/Creator'] = '{0} {1} / Tesseract {2} {3}'.format(
+        PROGRAM_NAME, VERSION,
+        renderer_tag,
+        tesseract.version())
     return pdfmark
 
 
@@ -856,7 +887,7 @@ def merge_pages_ghostscript(
         right that way.'''
         if s.endswith('.ps'):
             return 99999999
-        key = int(os.path.basename(s)[0:6]) * 10
+        key = page_number(s) * 10
         if 'debug' in os.path.basename(s):
             key += 1
         return key
@@ -872,7 +903,8 @@ def merge_pages_ghostscript(
         output_file=output_file,
         compression=options.pdfa_image_compression,
         log=log,
-        threads=options.jobs or 1)
+        threads=options.jobs or 1,
+        pdfa_part=('1' if options.output_type == 'pdfa-1' else '2'))
 
 
 def merge_pages_qpdf(
@@ -891,7 +923,7 @@ def merge_pages_qpdf(
     def input_file_order(s):
         '''Sort order: All rendered pages followed
         by their debug page.'''
-        key = int(os.path.basename(s)[0:6]) * 10
+        key = page_number(s) * 10
         if 'debug' in os.path.basename(s):
             key += 1
         return key
@@ -906,6 +938,7 @@ def merge_pages_qpdf(
     first_page = pypdf.PdfFileReader(pdf_pages[0])
 
     writer = pypdf.PdfFileWriter()
+    writer._header = b'%PDF-' + _pdf_guess_version(pdf_pages[0])  # copy version from source
     writer.appendPagesFromReader(first_page)
     writer.addMetadata(pdfmark)
     writer_file = pdf_pages[0].replace('.pdf', '.metadata.pdf')
@@ -915,7 +948,8 @@ def merge_pages_qpdf(
     pdf_pages[0] = writer_file
 
     qpdf.merge(input_files=pdf_pages, output_file=output_file,
-               min_version=context.get_pdfinfo().min_version)
+               min_version=context.get_pdfinfo().min_version,
+               log=log)
 
 
 def merge_sidecars(
@@ -923,7 +957,6 @@ def merge_sidecars(
         output_file,
         log,
         context):
-    options = context.get_options()
     pdfinfo = context.get_pdfinfo()
 
     txt_files = [None] * len(pdfinfo)
@@ -934,21 +967,30 @@ def merge_sidecars(
             txt_files[idx] = infile
 
     def write_pages(stream):
-        for page_number, txt_file in enumerate(txt_files):
-            if page_number != 0:
+        for page_num, txt_file in enumerate(txt_files):
+            if page_num != 0:
                 stream.write('\f')  # Form feed between pages
             if txt_file:
-                with open(txt_file, 'r') as in_:
-                    stream.write(in_.read())
+                with open(txt_file, 'r', encoding="utf-8") as in_:
+                    txt = in_.read()
+                    # Tesseract v4 alpha started adding form feeds in 
+                    # commit aa6eb6b
+                    # No obvious way to detect what binaries will do this, so
+                    # for consistency just ignore its form feeds and insert our
+                    # own
+                    if txt.endswith('\f'):
+                        stream.write(txt[:-1])
+                    else:
+                        stream.write(txt)
             else:
                 stream.write('[OCR skipped on page {}]'.format(
-                        page_number + 1))
+                        page_num + 1))
 
     if output_file == '-':
         write_pages(sys.stdout)
         sys.stdout.flush()
     else:
-        with open(output_file, 'w', encoding='utf-8') as out:
+        with open(output_file, 'w', encoding="utf-8") as out:
             write_pages(out)
 
 
@@ -959,13 +1001,16 @@ def copy_final(
         context):
     input_file = next((ii for ii in input_files if ii.endswith('.pdf')))
 
-    if output_file == '-':
-        from shutil import copyfileobj
-        with open(input_file, 'rb') as input_stream:
+    with open(input_file, 'rb') as input_stream:
+        if output_file == '-':
             copyfileobj(input_stream, sys.stdout.buffer)
             sys.stdout.flush()
-    else:
-        shutil.copy(input_file, output_file)
+        else:
+            # At this point we overwrite the output_file specified by the user
+            # use copyfileobj because then we use open() to create the file and 
+            # get the appropriate umask, ownership, etc.
+            with open(output_file, 'wb') as output_stream:
+                copyfileobj(input_stream, output_stream)
 
 
 def build_pipeline(options, work_folder, log, context):
@@ -1083,7 +1128,7 @@ def build_pipeline(options, work_folder, log, context):
     task_select_image_layer.graphviz(
         fillcolor='"#00cc66"', shape='diamond')
     task_select_image_layer.active_if(
-        options.pdf_renderer == 'hocr' or options.pdf_renderer == 'tess4')
+        options.pdf_renderer == 'hocr' or options.pdf_renderer == 'sandwich')
 
     task_render_hocr_page = main_pipeline.transform(
         task_func=render_hocr_page,
@@ -1113,7 +1158,7 @@ def build_pipeline(options, work_folder, log, context):
                 os.path.join(work_folder, r'\1.text.txt')],
         extras=[log, context])
     task_ocr_tesseract_textonly_pdf.graphviz(fillcolor='"#ff69b4"')
-    task_ocr_tesseract_textonly_pdf.active_if(options.pdf_renderer == 'tess4')
+    task_ocr_tesseract_textonly_pdf.active_if(options.pdf_renderer == 'sandwich')
     if tesseract.v4():
         task_ocr_tesseract_textonly_pdf.jobs_limit(2)
 
@@ -1126,7 +1171,8 @@ def build_pipeline(options, work_folder, log, context):
         output=os.path.join(work_folder, r'\1.rendered.pdf'),
         extras=[log, context])
     task_combine_layers.graphviz(fillcolor='"#00cc66"')
-    task_combine_layers.active_if(options.pdf_renderer == 'hocr' or options.pdf_renderer == 'tess4')
+    task_combine_layers.active_if(options.pdf_renderer == 'hocr' or 
+                                  options.pdf_renderer == 'sandwich')
 
     # Tesseract OCR+PDF
     task_ocr_tesseract_and_render_pdf = main_pipeline.collate(
@@ -1148,7 +1194,7 @@ def build_pipeline(options, work_folder, log, context):
         filter=formatter(r'\.repaired\.pdf'),
         output=os.path.join(work_folder, 'pdfa.ps'),
         extras=[log, context])
-    task_generate_postscript_stub.active_if(options.output_type == 'pdfa')
+    task_generate_postscript_stub.active_if(options.output_type.startswith('pdfa'))
 
 
     # Bypass valve
@@ -1170,7 +1216,7 @@ def build_pipeline(options, work_folder, log, context):
                task_generate_postscript_stub],
         output=os.path.join(work_folder, 'merged.pdf'),
         extras=[log, context])
-    task_merge_pages_ghostscript.active_if(options.output_type == 'pdfa')
+    task_merge_pages_ghostscript.active_if(options.output_type.startswith('pdfa'))
 
     task_merge_pages_qpdf = main_pipeline.merge(
         task_func=merge_pages_qpdf,
@@ -1193,7 +1239,7 @@ def build_pipeline(options, work_folder, log, context):
     task_merge_sidecars.active_if(options.sidecar)
 
     # Finalize
-    task_copy_final = main_pipeline.merge(
+    main_pipeline.merge(
         task_func=copy_final,
         input=[task_merge_pages_ghostscript, task_merge_pages_qpdf],
         output=options.output_file,

requirements.txt

@@ -2,8 +2,8 @@
 # setup.py lists a separate set of requirements that are looser to simplify
 # installation
 ruffus == 2.6.3
-Pillow == 4.1.1
+Pillow == 5.0.0
 reportlab == 3.4.0
 PyPDF2 == 1.26.0
-img2pdf == 0.2.3
-cffi == 1.10.0
+img2pdf == 0.2.4
+cffi == 1.11.2

setup.py

@@ -185,9 +185,15 @@ if 'upload' in sys.argv[1:]:
 
 tests_require = open('test_requirements.txt').read().splitlines()
 
+
+def readme():
+    with open('README.rst') as f:
+        return f.read()
+
 setup(
     name='ocrmypdf',
     description='OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched',
+    long_description=readme(),
     url='https://github.com/jbarlow83/OCRmyPDF',
     author='James R. Barlow',
     author_email='jim@purplerock.ca',
@@ -211,9 +217,11 @@ setup(
         "Topic :: Text Processing :: Indexing",
         "Topic :: Text Processing :: Linguistic",
         ],
+    python_requires='>=3.5',
     setup_requires=[
         'setuptools_scm',       # so that version will work
-        'cffi>=1.9.1'           # to build the leptonica module
+        'cffi>=1.9.1',          # to build the leptonica module
+        'pytest-runner'         # to enable python setup.py test
     ],
     use_scm_version={'version_scheme': 'post-release'},
     cffi_modules=[

test_requirements.txt

@@ -1,4 +1,5 @@
-pytest >= 3.0
+pytest >= 3.2
 pytest-helpers-namespace
 pytest-xdist
-pytest-cov
+pytest-cov
+pytest-timeout

tests/conftest.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # © 2017 James R. Barlow: github.com/jbarlow83
 
 import sys
@@ -129,7 +128,8 @@ def run_ocrmypdf(input_file, output_file, *args, env=None):
     if env is None:
         env = os.environ
 
-    p_args = OCRMYPDF + list(args) + [str(input_file), str(output_file)]
+    p_args = OCRMYPDF + [str(arg) for arg in args] + \
+             [str(input_file), str(output_file)]
     p = Popen(
         p_args, close_fds=True, stdout=PIPE, stderr=PIPE,
         universal_newlines=True, env=env)

tests/resources/README.rst

@@ -33,7 +33,7 @@ In some cases they were converted from one image format to another without other
         - Creative Commons BY-SA 3.0
     *   - typewriter.png, 2400dpi.pdf
         - `Wikimedia: Triumph typewrtier text Linzensoep`_
-        * Creative Commons BY-SA 2.5
+        - Creative Commons BY-SA 2.5
     *   - baiona.png
         - `Wikimedia: Baionako udalerri mugakideak`_
         - Creative Commons BY-SA 4.0
@@ -57,10 +57,13 @@ under the terms of the license in LICENSE.rst.
         - test image with 200 x 100 DPI resolution
     *   - blank.pdf
         - @jbarlow83
-        - blank PDF
+        - blank PDF generated by Adobe Illustrator CC 17, containing a lot of application-specific metadata/bloat
     *   - cmyk.pdf
         - @jbarlow83
         - a CMYK image created in Photoshop
+    *   - crom.png
+        - @jbarlow83
+        - test for non-dictionary words        
     *   - enormous.pdf
         - @jbarlow83
         - very large PDF page
@@ -69,7 +72,7 @@ under the terms of the license in LICENSE.rst.
         - a linearized PDF containing some unusual indirect objects, created by an Epson printer; printout of a Wikipedia article (CC BY-SA)
     *   - formxobject.pdf
         - @jbarlow83
-        - hand-crafted exotic PDF containing an image inside a Form XObject
+        - hand-crafted PDF containing an image inside a Form XObject
     *   - francais.pdf
         - @jbarlow83
         - a page containing French accents (diacritics)  
@@ -88,14 +91,24 @@ under the terms of the license in LICENSE.rst.
     *   - overlay.pdf
         - @maxandersen
         - PDF file generated by PDFPen pro that triggered content stream parse errors
-    *   - no_conentes.pdf
+    *   - negzero.pdf
+        - @jbarlow83
+        - copy of formxobject.pdf with token that qpdf doesn't like
+    *   - no_contents.pdf
         - @jbarlow83
         - synthetic PDF with a blank page that has no /Contents entry
+    *   - trivial.pdf
+        - @jbarlow83
+        - smallest possible valid PDF-1.3 with all required fields
+    *   - vector.pdf
+        - @Catscratch
+        - a PDF with vector art and text rendered as curves with no fonts
+
 
 Assemblies
 ==========
 
-These test resources are assemblies from other previously mentioned files, released under the same license terms as their input files.
+These test resources are assemblies or derivatives from other previously mentioned files, released under the same license terms as their input files.
 
 - cardinal.pdf (four cardinal directions, baked-in rotated copies of LinnSequencer.jpg)
 - ccitt.pdf (LinnSequencer.jpg, converted to CCITT encoding)
@@ -106,7 +119,10 @@ These test resources are assemblies from other previously mentioned files, relea
 - palette.pdf (congress.jpg, converted to a 256-color palette)
 - rotated_skew.pdf (a /Rotate'd and skewed document from LinnSequencer.jpg)
 - skew.pdf (from LinnSequencer.jpg, skew simulated by adjusting the transformation matrix)
-- skew-encrypted.pdf (skew.pdf with encryption - access supported by PyPDF2)
+- skew-encrypted.pdf (skew.pdf with encryption - access supported by PyPDF2, password is "password")
+- baiona_gray.png (from baiona.png)
+- poster.pdf (from LinnSequencer.jpg)
+
 
 
 .. _`Wikimedia: LinnSequencer`: https://upload.wikimedia.org/wikipedia/en/b/b7/LinnSequencer_hardware_MIDI_sequencer_brochure_page_2_300dpi.jpg

tests/resources/trivial.pdf

@@ -0,0 +1,27 @@
+%PDF-1.3
+%¿÷¢þ
+1 0 obj
+<< /Pages 2 0 R /Type /Catalog >>
+endobj
+2 0 obj
+<< /Count 1 /Kids [ 3 0 R ] /Type /Pages >>
+endobj
+3 0 obj
+<< /Contents 4 0 R /MediaBox [ 0 0 300 300 ] /Parent 2 0 R /Resources << >> /Type /Page >>
+endobj
+4 0 obj
+<< /Length 0 >>
+stream
+endstream
+endobj
+xref
+0 5
+0000000000 65535 f 
+0000000015 00000 n 
+0000000064 00000 n 
+0000000123 00000 n 
+0000000229 00000 n 
+trailer << /Root 1 0 R /Size 5 /ID [<8088c23e0edc07ef4fbed3daa55f52ed><8088c23e0edc07ef4fbed3daa55f52ed>] >>
+startxref
+277
+%%EOF

tests/spoof/gs_feature_elision.py

@@ -22,7 +22,7 @@ not permitted in PDF/A-2, overprint mode not set"""
 def main():
     if '--version' in sys.argv:
         print('9.20')
-        print('SPOOFED: ' + os.path.basename(__filename__))
+        print('SPOOFED: ' + os.path.basename(__file__))
         sys.exit(0)
 
     gs_args = ['gs'] + sys.argv[1:]

tests/spoof/gs_pdfa_failure.py

@@ -17,7 +17,7 @@ def real_ghostscript(argv):
 def main():
     if '--version' in sys.argv:
         print('9.20')
-        print('SPOOFED: ' + os.path.basename(__filename__))
+        print('SPOOFED: ' + os.path.basename(__file__))
         sys.exit(0)
 
     # Unless some argument is calling for PDFA generation, forward to

tests/spoof/gs_raster_failure.py

@@ -16,7 +16,7 @@ def real_ghostscript(argv):
 def main():
     if '--version' in sys.argv:
         print('9.20')
-        print('SPOOFED: ' + os.path.basename(__filename__))
+        print('SPOOFED: ' + os.path.basename(__file__))
         sys.exit(0)
 
     # For any rendering calls (device == pdfwrite) call real ghostscript

tests/spoof/gs_render_failure.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# © 2016 James R. Barlow: github.com/jbarlow83
+# © 2016-18 James R. Barlow: github.com/jbarlow83
 
 import sys
 import os
@@ -16,7 +16,7 @@ def real_ghostscript(argv):
 def main():
     if '--version' in sys.argv:
         print('9.20')
-        print('SPOOFED: ' + os.path.basename(__filename__))
+        print('SPOOFED: ' + os.path.basename(__file__))
         sys.exit(0)
 
     # For any rasterize calls (device != pdfwrite) call real ghostscript

tests/spoof/qpdf_dummy_return2.py

@@ -5,6 +5,9 @@ import sys
 
 
 def main():
+    if sys.argv[-1] == '--version':
+        print('qpdf version 7.0.0')
+        sys.exit(0)
     print('qpdf dummy')
     sys.exit(2)
 

tests/spoof/tesseract_badutf8.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+# © 2017 James R. Barlow: github.com/jbarlow83
+import sys
+import img2pdf
+import PyPDF2 as pypdf
+from PIL import Image
+
+
+"""Tesseract bad utf8 spoof
+
+In 'hocr' mode or 'pdf' mode, return error code 1 and some non-Unicode
+text because tesseract seems to do that in some cases related to 
+language pack version mismatches
+
+"""
+
+
+VERSION_STRING = '''tesseract 3.05.01
+ leptonica-1.72
+  libjpeg 8d : libpng 1.6.19 : libtiff 4.0.6 : zlib 1.2.5
+SPOOFED
+'''
+
+# Japanese "Invalid UTF-8" encoded in Shift JIS
+BAD_UTF8 = b'\x96\xb3\x8c\xf8\x82\xc8UTF-8\x0a'
+
+
+def main():
+    if sys.argv[1] == '--version':
+        print(VERSION_STRING, file=sys.stderr)
+        sys.exit(0)
+    elif sys.argv[1] == '--list-langs':
+        print('List of available languages (1):\neng', file=sys.stderr)
+        sys.exit(0)
+    elif sys.argv[1] == '--print-parameters':
+        print("Some parameters", file=sys.stderr)
+        print("textonly_pdf\t1\tSome help text")
+        sys.exit(0)
+    elif sys.argv[-2] in ('hocr', 'pdf'):
+        sys.stdout.buffer.write(BAD_UTF8)
+        sys.exit(1)
+    elif sys.argv[-1] == 'stdout':
+        # input file is at sys.argv[-2] but we don't look at it
+        print("""Orientation: 0
+Orientation in degrees: 0
+Orientation confidence: 100.00
+Script: 1
+Script confidence: 100.00""", file=sys.stderr)
+    else:
+        print("Spoof doesn't understand arguments", file=sys.stderr)
+        print(sys.argv, file=sys.stderr)
+        sys.exit(1)
+
+    sys.exit(0)
+
+
+if __name__ == '__main__':
+    main()

tests/spoof/tesseract_big_image_error.py

@@ -4,9 +4,9 @@
 import sys
 
 
-VERSION_STRING = '''tesseract 3.04.00
- leptonica-1.72
-  libjpeg 8d : libpng 1.6.19 : libtiff 4.0.6 : zlib 1.2.5
+VERSION_STRING = '''tesseract 3.05.01
+ leptonica-1.74.4
+  libjpeg 9b : libpng 1.6.32 : libtiff 4.0.8 : zlib 1.2.8
 SPOOFED: return error claiming image too big
 '''
 
@@ -20,7 +20,10 @@ def main():
         print(VERSION_STRING, file=sys.stderr)
         sys.exit(0)
     elif sys.argv[1] == '--list-langs':
-        print('List of available languages (1):\neng', file=sys.stderr)
+        print('List of available languages (1):\neng\n', file=sys.stderr)
+        sys.exit(0)
+    elif sys.argv[1] == '--print-parameters':
+        print('A parameter list would go here\ntextonly_pdf 0\n', file=sys.stderr)
         sys.exit(0)
     elif sys.argv[-2] == 'hocr':
         print("Image too large: (33830, 14959)\n"

tests/spoof/tesseract_cache.py

@@ -85,6 +85,12 @@ def main():
     except ValueError:
         m.update(b'default-lang')
 
+    try:
+        textonly = sys.argv[sys.argv.index('-c') + 1]
+        m.update(textonly.encode())
+    except ValueError:
+        m.update(b'textonly_pdf=0')
+
     psm_arg = ''
     if '--psm' in sys.argv:
         psm_arg = '--psm'

tests/spoof/tesseract_crash.py

@@ -5,9 +5,9 @@ import os
 import signal
 
 
-VERSION_STRING = '''tesseract 3.04.00
- leptonica-1.72
-  libjpeg 8d : libpng 1.6.19 : libtiff 4.0.6 : zlib 1.2.5
+VERSION_STRING = '''tesseract 3.05.01
+ leptonica-1.74.4
+  libjpeg 9b : libpng 1.6.32 : libtiff 4.0.8 : zlib 1.2.8
 SPOOFED: CRASH ON OCR or -psm 0
 '''
 
@@ -29,6 +29,10 @@ def main():
     elif sys.argv[1] == '--list-langs':
         print('List of available languages (1):\neng', file=sys.stderr)
         sys.exit(0)
+    elif sys.argv[1] == '--print-parameters':
+        print('A parameter list would go here\ntextonly_pdf 0\n',
+              file=sys.stderr)
+        sys.exit(0)
     elif sys.argv[-2] == 'hocr':
         print("KABOOM! Tesseract failed for some reason", file=sys.stderr)
         sys.exit(128 + signal.SIGSEGV)

tests/spoof/tesseract_noop.py

@@ -2,6 +2,7 @@
 # © 2016 James R. Barlow: github.com/jbarlow83
 import sys
 import img2pdf
+import PyPDF2 as pypdf
 from PIL import Image
 
 
@@ -17,7 +18,7 @@ In orientation check mode, report the orientation is upright.
 """
 
 
-VERSION_STRING = '''tesseract 3.04.00
+VERSION_STRING = '''tesseract 3.05.01
  leptonica-1.72
   libjpeg 8d : libpng 1.6.19 : libtiff 4.0.6 : zlib 1.2.5
 SPOOFED
@@ -53,6 +54,10 @@ def main():
     elif sys.argv[1] == '--list-langs':
         print('List of available languages (1):\neng', file=sys.stderr)
         sys.exit(0)
+    elif sys.argv[1] == '--print-parameters':
+        print("Some parameters", file=sys.stderr)
+        print("textonly_pdf\t1\tSome help text")
+        sys.exit(0)
     elif sys.argv[-2] == 'hocr':
         inputf = sys.argv[-4]
         output = sys.argv[-3]
@@ -63,13 +68,28 @@ def main():
         with open(output + '.txt', 'w') as f:
             f.write('')
     elif sys.argv[-2] == 'pdf':
-        inputf = sys.argv[-4]
-        output = sys.argv[-3]
-        pdf_bytes = img2pdf.convert([inputf], dpi=300)
-        with open(output + '.pdf', 'wb') as f:
-            f.write(pdf_bytes)
-        with open(output + '.txt', 'w') as f:
-            f.write('')
+        if 'textonly_pdf=1' in sys.argv:
+            inputf = sys.argv[-4]
+            output = sys.argv[-3]
+            with Image.open(inputf) as im:
+                dpi = im.info['dpi']
+                pagesize = im.size[0] / dpi[0], im.size[1] / dpi[1]
+                ptsize = pagesize[0] * 72, pagesize[1] * 72
+
+            pdf_out = pypdf.PdfFileWriter()
+            pdf_out.addBlankPage(ptsize[0], ptsize[1])
+            with open(output + '.pdf', 'wb') as f:
+                pdf_out.write(f)
+            with open(output + '.txt', 'w') as f:
+                f.write('')
+        else:
+            inputf = sys.argv[-4]
+            output = sys.argv[-3]
+            pdf_bytes = img2pdf.convert([inputf], dpi=300)
+            with open(output + '.pdf', 'wb') as f:
+                f.write(pdf_bytes)
+            with open(output + '.txt', 'w') as f:
+                f.write('')
     elif sys.argv[-1] == 'stdout':
         inputf = sys.argv[-2]
         print("""Orientation: 0

tests/test_hocrtransform.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # © 2015 James R. Barlow: github.com/jbarlow83
 
 from ocrmypdf import hocrtransform

tests/test_main.py

@@ -1,25 +1,36 @@
-#!/usr/bin/env python3
-# © 2015 James R. Barlow: github.com/jbarlow83
+# © 2015-17 James R. Barlow: github.com/jbarlow83
 
 from subprocess import Popen, PIPE, check_output, check_call, DEVNULL
+from pathlib import Path
 import os
 import shutil
+import resource
 import pytest
 from ocrmypdf.pdfinfo import PdfInfo, Colorspace, Encoding
 import PyPDF2 as pypdf
 from ocrmypdf.exceptions import ExitCode
 from ocrmypdf import leptonica
 from ocrmypdf.pdfa import file_claims_pdfa
-from ocrmypdf.exec import ghostscript
+from ocrmypdf.exec import ghostscript, tesseract, qpdf
 import logging
 from math import isclose
 
+import PIL
+
+# pytest.helpers is dynamic
+# pylint: disable=no-member
+# pylint: disable=w0612
 
 check_ocrmypdf = pytest.helpers.check_ocrmypdf
 run_ocrmypdf = pytest.helpers.run_ocrmypdf
 spoof = pytest.helpers.spoof
 
 
+RENDERERS = ['hocr', 'tesseract']
+if tesseract.has_textonly_pdf():
+    RENDERERS.append('sandwich')
+
+
 @pytest.fixture
 def spoof_tesseract_crash():
     return spoof(tesseract='tesseract_crash.py')
@@ -50,6 +61,11 @@ def spoof_no_tess_gs_raster_fail():
     return spoof(tesseract='tesseract_noop.py', gs='gs_raster_failure.py')
 
 
+@pytest.fixture
+def spoof_tess_bad_utf8():
+    return spoof(tesseract='tesseract_badutf8.py')
+
+
 @pytest.fixture
 def spoof_qpdf_always_error():
     return spoof(qpdf='qpdf_dummy_return2.py')
@@ -62,7 +78,7 @@ def test_quick(spoof_tesseract_cache, resources, outpdf):
 def test_deskew(spoof_tesseract_noop, resources, outdir):
     # Run with deskew
     deskewed_pdf = check_ocrmypdf(
-        resources / 'skew.pdf', outdir / 'skew.pdf', '-d', '-v', '1',
+        resources / 'skew.pdf', outdir / 'skew.pdf', '-d',
         env=spoof_tesseract_noop)
 
     # Now render as an image again and use Leptonica to find the skew angle
@@ -72,12 +88,13 @@ def test_deskew(spoof_tesseract_noop, resources, outdir):
     deskewed_png = outdir / 'deskewed.png'
 
     ghostscript.rasterize_pdf(
-        str(deskewed_pdf),
-        str(deskewed_png),
+        deskewed_pdf,
+        deskewed_png,
         xres=150,
         yres=150,
         raster_device='pngmono',
-        log=log)
+        log=log,
+        pageno=1)
 
     from ocrmypdf.leptonica import Pix
     pix = Pix.read(str(deskewed_png))
@@ -111,13 +128,13 @@ def test_remove_background(spoof_tesseract_noop, resources, outdir):
     output_png = outdir / 'remove_bg.png'
 
     ghostscript.rasterize_pdf(
-        str(output_pdf),
-        str(output_png),
+        output_pdf,
+        output_png,
         xres=100,
         yres=100,
         raster_device='png16m',
-        log=log)
-
+        log=log,
+        pageno=1)
 
     # The output image should contain pure white and black
     im = Image.open(output_png)
@@ -128,7 +145,7 @@ def test_remove_background(spoof_tesseract_noop, resources, outdir):
 @pytest.mark.parametrize(
     "pdf",
     ['palette.pdf', 'cmyk.pdf', 'ccitt.pdf', 'jbig2.pdf', 'lichtenstein.pdf'])
-@pytest.mark.parametrize("renderer", ['hocr', 'tesseract'])
+@pytest.mark.parametrize("renderer", ['auto', 'tesseract'])
 @pytest.mark.parametrize("output_type", ['pdf', 'pdfa'])
 def test_exotic_image(spoof_tesseract_cache, pdf, renderer, output_type,
                       resources, outdir):
@@ -210,10 +227,7 @@ def test_high_unicode(spoof_tesseract_noop, resources, no_outpdf):
     assert p.returncode == ExitCode.bad_args, err
 
 
-@pytest.mark.parametrize('renderer', [
-    'hocr',
-    'tesseract',
-    ])
+@pytest.mark.parametrize('renderer', RENDERERS)
 def test_oversample(spoof_tesseract_cache, renderer, resources, outpdf):
     oversampled_pdf = check_ocrmypdf(
         resources / 'skew.pdf', outpdf, '--oversample', '350',
@@ -268,9 +282,7 @@ def check_monochrome_correlation(
             print(png)
             return
         ghostscript.rasterize_pdf(
-            str(pdf),
-            str(png),
-            xres=100, yres=100,
+            pdf, png, xres=100, yres=100,
             raster_device='pngmono', log=gslog, pageno=pageno)
 
     rasterize(reference_pdf, reference_pageno, reference_png)
@@ -303,10 +315,7 @@ def test_monochrome_correlation(resources, outdir):
     assert corr > 0.90
 
 
-@pytest.mark.parametrize('renderer', [
-    'hocr',
-    'tesseract',
-    ])
+@pytest.mark.parametrize('renderer', RENDERERS)
 def test_autorotate(spoof_tesseract_cache, renderer, resources, outdir):
     # cardinal.pdf contains four copies of an image rotated in each cardinal
     # direction - these ones are "burned in" not tagged with /Rotate
@@ -341,10 +350,7 @@ def test_autorotate_threshold(
     assert eval(correlation_test)
 
 
-@pytest.mark.parametrize('renderer', [
-    'hocr',
-    'tesseract',
-    ])
+@pytest.mark.parametrize('renderer',RENDERERS)
 def test_ocr_timeout(renderer, resources, outpdf):
     out = check_ocrmypdf(resources / 'skew.pdf', outpdf,
                          '--tesseract-timeout', '1.0')
@@ -359,7 +365,7 @@ def test_skip_big(spoof_tesseract_cache, resources, outpdf):
     assert not pdfinfo[0].has_text
 
 
-@pytest.mark.parametrize('renderer', ['hocr', 'tesseract'])
+@pytest.mark.parametrize('renderer', RENDERERS)
 @pytest.mark.parametrize('output_type', ['pdf', 'pdfa'])
 def test_maximum_options(spoof_tesseract_cache, renderer, output_type,
                          resources, outpdf):
@@ -410,9 +416,13 @@ def test_force_ocr_on_pdf_with_no_images(spoof_tesseract_crash, resources,
 @pytest.mark.skipif(
     pytest.helpers.is_macos() and pytest.helpers.running_in_travis(),
     reason="takes too long to install language packs in Travis macOS homebrew")
-def test_french(spoof_tesseract_cache, resources, outpdf):
+def test_french(spoof_tesseract_cache, resources, outdir):
+    # Produce a sidecar too - implicit test that system locale is set up
+    # properly
+    sidecar = outdir / 'francais.txt'
     p, out, err = run_ocrmypdf(
-        resources / 'francais.pdf', outpdf, '-l', 'fra',
+        resources / 'francais.pdf', outdir / 'francais.pdf', '-l', 'fra',
+        '--sidecar', sidecar,
         env=spoof_tesseract_cache)
     print(os.environ)
     assert p.returncode == ExitCode.ok, \
@@ -422,12 +432,12 @@ def test_french(spoof_tesseract_cache, resources, outpdf):
 def test_klingon(resources, outpdf):
     p, out, err = run_ocrmypdf(
         resources / 'francais.pdf', outpdf, '-l', 'klz')
-    assert p.returncode == ExitCode.bad_args
+    assert p.returncode == ExitCode.missing_dependency
 
 
 def test_missing_docinfo(spoof_tesseract_noop, resources, outpdf):
     p, out, err = run_ocrmypdf(
-        resources / 'missing_docinfo.pdf', outpdf, '-l', 'eng', '-c',
+        resources / 'missing_docinfo.pdf', outpdf, '-l', 'eng', '--skip-text',
         env=spoof_tesseract_noop)
     assert p.returncode == ExitCode.ok, err
 
@@ -474,7 +484,7 @@ def test_encrypted(resources, no_outpdf):
     p, out, err = run_ocrmypdf(
         resources / 'skew-encrypted.pdf', no_outpdf)
     assert p.returncode == ExitCode.encrypted_pdf
-    assert out.find('password')
+    assert out.find('encrypted')
 
 
 @pytest.mark.parametrize('renderer', [
@@ -489,10 +499,7 @@ def test_pagesegmode(renderer, spoof_tesseract_cache, resources, outpdf):
         '--pdf-renderer', renderer, env=spoof_tesseract_cache)
 
 
-@pytest.mark.parametrize('renderer', [
-    'hocr',
-    'tesseract',
-    ])
+@pytest.mark.parametrize('renderer', RENDERERS)
 def test_tesseract_crash(renderer, spoof_tesseract_crash,
                          resources, no_outpdf):
     p, out, err = run_ocrmypdf(
@@ -515,15 +522,14 @@ def test_tesseract_crash_autorotate(spoof_tesseract_crash,
     print(err)
 
 
-@pytest.mark.parametrize('renderer', [
-    'hocr',
-    'tesseract',
-    ])
+@pytest.mark.parametrize('renderer', RENDERERS)
 def test_tesseract_image_too_big(renderer, spoof_tesseract_big_image_error,
                                  resources, outpdf):
     check_ocrmypdf(
         resources / 'hugemono.pdf', outpdf, '-r',
-        '--pdf-renderer', renderer, env=spoof_tesseract_big_image_error)
+        '--pdf-renderer', renderer, 
+        '--max-image-mpixels', '0',
+        env=spoof_tesseract_big_image_error)
 
 
 def test_no_unpaper(resources, no_outpdf):
@@ -566,10 +572,7 @@ def test_non_square_resolution(renderer, spoof_tesseract_cache,
     assert in_pageinfo[0].yres == out_pageinfo[0].yres
 
 
-@pytest.mark.parametrize('renderer', [
-    'hocr',
-    'tesseract'
-    ])
+@pytest.mark.parametrize('renderer', RENDERERS)
 def test_convert_to_square_resolution(renderer, spoof_tesseract_cache,
                                       resources, outpdf):
     from math import isclose
@@ -770,28 +773,22 @@ language_model_penalty_non_freq_dict_word 0
 
     check_ocrmypdf(
         resources / 'ccitt.pdf', outdir / 'out.pdf',
-        '--tesseract-config', str(cfg_file))
+        '--tesseract-config', cfg_file)
 
 
-@pytest.mark.parametrize('renderer', [
-    'hocr',
-    'tesseract',
-    ])
+@pytest.mark.parametrize('renderer', RENDERERS)
 def test_tesseract_config_notfound(renderer, resources, outdir):
     cfg_file = outdir / 'nofile.cfg'
 
     p, out, err = run_ocrmypdf(
         resources / 'ccitt.pdf', outdir / 'out.pdf',
         '--pdf-renderer', renderer,
-        '--tesseract-config', str(cfg_file))
+        '--tesseract-config', cfg_file)
     assert "Can't open" in err, "No error message about missing config file"
     assert p.returncode == ExitCode.ok
 
 
-@pytest.mark.parametrize('renderer', [
-    'hocr',
-    'tesseract',
-    ])
+@pytest.mark.parametrize('renderer', RENDERERS)
 def test_tesseract_config_invalid(renderer, resources, outdir):
     cfg_file = outdir / 'test.cfg'
     with cfg_file.open('w') as f:
@@ -802,21 +799,49 @@ THIS FILE IS INVALID
     p, out, err = run_ocrmypdf(
         resources / 'ccitt.pdf', outdir / 'out.pdf',
         '--pdf-renderer', renderer,
-        '--tesseract-config', str(cfg_file))
+        '--tesseract-config', cfg_file)
     assert "parameter not found" in err, "No error message"
     assert p.returncode == ExitCode.invalid_config
 
 
+def test_user_words(resources, outdir):
+    word_list = outdir / 'wordlist.txt'
+    sidecar_before = outdir / 'sidecar_before.txt'
+    sidecar_after = outdir / 'sidecar_after.txt'
+
+    # Don't know how to make this test pass on various versions and platforms
+    # so weaken to merely testing that the argument is accepted
+    consistent = False
+
+    if consistent:
+        check_ocrmypdf(
+            resources / 'crom.png', outdir / 'out.pdf',
+            '--image-dpi', 150,
+            '--sidecar', sidecar_before
+        )
+        assert 'cromulent' not in sidecar_before.open().read()
+
+    with word_list.open('w') as f:
+        f.write('cromulent\n')  # a perfectly cromulent word
+
+    check_ocrmypdf(
+        resources / 'crom.png', outdir / 'out.pdf',
+        '--image-dpi', 150,
+        '--sidecar', sidecar_after,
+        '--user-words', word_list
+    )
+
+    if consistent:
+        assert 'cromulent' in sidecar_after.open().read()
+
+
 def test_form_xobject(spoof_tesseract_noop, resources, outpdf):
     check_ocrmypdf(resources / 'formxobject.pdf', outpdf,
                    '--force-ocr',
                    env=spoof_tesseract_noop)
 
 
-@pytest.mark.parametrize('renderer', [
-    'hocr',
-    'tesseract',
-])
+@pytest.mark.parametrize('renderer', RENDERERS)
 def test_pagesize_consistency(renderer, resources, outpdf):
     from math import isclose
 
@@ -860,6 +885,8 @@ def test_gs_raster_failure(spoof_no_tess_gs_raster_fail, resources, outpdf):
     assert p.returncode == ExitCode.child_process_error
 
 
+@pytest.mark.skipif('8.0.0' <= qpdf.version() <= '8.0.1',
+                    reason="qpdf regression")
 def test_no_contents(spoof_tesseract_noop, resources, outpdf):
     check_ocrmypdf(resources / 'no_contents.pdf', outpdf, '--force-ocr',
                    env=spoof_tesseract_noop)
@@ -970,3 +997,145 @@ def test_sidecar_pagecount(spoof_tesseract_cache, resources, outpdf):
     # formfeeds is the page count less one
     assert ocr_text.count('\f') == num_pages - 1, \
         "Sidecar page count does not match PDF page count"
+
+
+def test_sidecar_nonempty(spoof_tesseract_cache, resources, outpdf):
+    sidecar = outpdf + '.txt'
+    check_ocrmypdf(
+        resources / 'ccitt.pdf', outpdf,
+        '--sidecar', sidecar,
+        env=spoof_tesseract_cache
+    )
+
+    with open(sidecar, 'r') as f:
+        ocr_text = f.read()
+    assert 'the' in ocr_text
+
+
+def test_pdfa_1(spoof_tesseract_cache, resources, outpdf):
+    check_ocrmypdf(
+        resources / 'ccitt.pdf', outpdf,
+        '--output-type', 'pdfa-1',
+        env=spoof_tesseract_cache
+    )
+
+    pdfa_info = file_claims_pdfa(outpdf)
+    assert pdfa_info['conformance'] == 'PDF/A-1B'
+
+
+def test_bad_locale():
+    env = os.environ.copy()
+    env['LC_ALL'] = 'C'
+
+    p, out, err = run_ocrmypdf(
+        'a', 'b', env=env
+    )
+    assert out == '', "stdout not clean"
+    assert p.returncode != 0
+    assert 'configured to use ASCII as encoding' in err, "should whine"
+
+
+@pytest.mark.parametrize('renderer', RENDERERS)
+def test_bad_utf8(spoof_tess_bad_utf8, renderer, resources, no_outpdf):
+    p, out, err = run_ocrmypdf(
+        resources / 'ccitt.pdf', no_outpdf,
+        '--pdf-renderer', renderer,
+        env=spoof_tess_bad_utf8
+    )
+
+    assert out == '', "stdout not clean"
+    assert p.returncode != 0
+    assert 'not utf-8' in err, "should whine about utf-8"
+    assert '\\x96' in err, 'should repeat backslash encoded output'
+
+
+@pytest.mark.skipif(
+    not tesseract.has_textonly_pdf(), 
+    reason="issue only affects sandwich")
+def test_rotate_deskew_timeout(resources, outdir):
+    check_ocrmypdf(
+        resources / 'rotated_skew.pdf',
+        outdir / 'deskewed.pdf',
+        '--deskew',
+        '--tesseract-timeout', '0',
+        '--pdf-renderer', 'sandwich'
+    )
+
+    correlation = check_monochrome_correlation(
+        outdir,
+        reference_pdf=resources / 'ccitt.pdf',
+        reference_pageno=1,
+        test_pdf=outdir / 'deskewed.pdf',
+        test_pageno=1)
+
+    # Confirm that the page still got deskewed
+    assert correlation > 0.50
+
+
+@pytest.mark.skipif(
+    PIL.PILLOW_VERSION < '5.0.0',
+    reason="Pillow < 5.0.0 doesn't raise the exception")
+def test_decompression_bomb(resources, outpdf):
+    p, out, err = run_ocrmypdf(
+        resources / 'hugemono.pdf',
+        outpdf
+    )
+    assert 'decompression bomb' in err
+
+    p, out, err = run_ocrmypdf(
+        resources / 'hugemono.pdf',
+        outpdf,
+        '--max-image-mpixels', '2000'
+    )
+    assert p.returncode == 0
+
+
+def test_text_curves(spoof_tesseract_noop, resources, outpdf):
+    check_ocrmypdf(
+        resources / 'vector.pdf', outpdf, env=spoof_tesseract_noop)
+
+    info = PdfInfo(outpdf)
+    assert len(info.pages[0].images) == 0, "added images to the vector PDF"
+
+    check_ocrmypdf(
+        resources / 'vector.pdf', outpdf, '--force-ocr',
+        env=spoof_tesseract_noop)
+    
+    info = PdfInfo(outpdf)
+    assert len(info.pages[0].images) != 0, "force did not rasterize"
+    
+
+def test_dev_null(spoof_tesseract_noop, resources):
+    p, out, err = run_ocrmypdf(
+        resources / 'trivial.pdf',
+        os.devnull,
+        '--force-ocr',
+        env=spoof_tesseract_noop
+    )
+    assert p.returncode == 0, "could not send output to /dev/null"
+    assert len(out) == 0, "wrote to stdout"
+
+
+def test_output_is_dir(spoof_tesseract_noop, resources, outdir):
+    p, out, err = run_ocrmypdf(
+        resources / 'trivial.pdf',
+        outdir,
+        '--force-ocr',
+        env=spoof_tesseract_noop
+    )
+    assert p.returncode == ExitCode.file_access_error
+    assert 'is not a writable file' in err
+
+
+def test_output_is_symlink(spoof_tesseract_noop, resources, outdir):
+    sym = Path(outdir / 'this_is_a_symlink')
+    sym.symlink_to(outdir / 'out.pdf')
+    p, out, err = run_ocrmypdf(
+        resources / 'trivial.pdf',
+        sym,
+        '--force-ocr',
+        env=spoof_tesseract_noop
+    )
+    assert p.returncode == ExitCode.ok, err
+    assert (outdir / 'out.pdf').stat().st_size > 0, 'target file not created'
+    

tests/test_pageinfo.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # © 2015 James R. Barlow: github.com/jbarlow83
 
 from ocrmypdf import pdfinfo
@@ -122,4 +121,4 @@ def test_no_contents(resources):
 def test_oversized_page(resources):
     pdf = pdfinfo.PdfInfo(resources / 'poster.pdf')
     image = pdf[0].images[0]
-    assert image.width * image.xres > 200, "this is supposed to be oversized"
+    assert image.width * image.xres > 200, "this is supposed to be oversized"

tests/test_qpdf.py

@@ -0,0 +1,55 @@
+# © 2017 James R. Barlow: github.com/jbarlow83
+
+import logging
+import resource
+import pytest
+
+from ocrmypdf.exec import ghostscript, tesseract, qpdf
+from ocrmypdf.pdfinfo import PdfInfo
+
+
+@pytest.mark.skipif(
+    qpdf.version() < '7.0.0',
+    reason="negzero.pdf crashes earlier versions")
+def test_qpdf_negative_zero(resources, outpdf):
+    negzero = resources / 'negzero.pdf'
+    hugemono = resources / 'hugemono.pdf'
+    # raises exception on err
+    qpdf.merge([str(negzero), str(hugemono)], outpdf, log=logging.getLogger())
+    
+
+@pytest.mark.timeout(15)
+@pytest.mark.parametrize('max_files,skip', [
+    (2, 0),  # Can we merge correctly without opening more than 2 files at once?
+    (16, 0), # And does this work properly when we can one-shot it?
+    (2, 1),  # Or playing with even/odd
+    (3, 0)   # Or odd step size
+    ])
+def test_qpdf_merge_correctness(resources, outpdf, max_files, skip):
+    # All of these must be only one page long
+    inputs = [
+        '2400dpi.pdf', 'aspect.pdf', 'blank.pdf', 'ccitt.pdf', 
+        'linn.pdf', 'masks.pdf', 'poster.pdf', 'overlay.pdf',
+        'skew.pdf', 'trivial.pdf']
+    
+    input_files = [str(resources / f) for f in inputs]
+
+    qpdf.merge(
+        input_files[skip:], outpdf, log=logging.getLogger(), 
+        max_files=max_files)
+    assert len(PdfInfo(outpdf).pages) == len(input_files[skip:])
+    
+
+@pytest.mark.timeout(15)
+@pytest.mark.skipif(
+    True, 
+    reason='qpdf binary cannot open multiple files multiple times')
+def test_page_merge_ulimit(resources, outpdf):
+    # Ensure we can merge pages without opening one file descriptor per page
+    ulimits = resource.getrlimit(resource.RLIMIT_NOFILE)
+    page_count = ulimits[0]
+    print(page_count)
+    input_files = [str(resources / 'trivial.pdf')] * page_count
+
+    qpdf.merge(input_files, outpdf, log=logging.getLogger())
+

tests/test_tess3.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # © 2017 James R. Barlow: github.com/jbarlow83
 
 import pytest
@@ -11,14 +10,26 @@ pytestmark = pytest.mark.skipif(tesseract.v4(),
                                 reason="tesseract 3.x required")
 
 
-def test_textonly_pdf_on_tess3(resources, no_outpdf):
+@pytest.mark.skipif(tesseract.has_textonly_pdf(),
+                    reason="check that missing dep is reported on old tess3")
+def test_textonly_pdf_on_older_tess3(resources, no_outpdf):
     p, _, _ = pytest.helpers.run_ocrmypdf(
         resources / 'linn.pdf',
-        no_outpdf, '--pdf-renderer', 'tess4')
+        no_outpdf, '--pdf-renderer', 'sandwich')
 
     assert p.returncode == ExitCode.missing_dependency
 
 
+@pytest.mark.skipif(not tesseract.has_textonly_pdf(),
+                    reason="check that feature is exercised on new test3")
+def test_textonly_pdf_on_newer_tess3(resources, no_outpdf):
+    p, _, _ = pytest.helpers.run_ocrmypdf(
+        resources / 'linn.pdf',
+        no_outpdf, '--pdf-renderer', 'sandwich')
+
+    assert p.returncode == ExitCode.ok
+
+
 def test_oem_on_tess3(resources, no_outpdf):
     p, _, err = pytest.helpers.run_ocrmypdf(
         resources / 'aspect.pdf',

tests/test_tess4.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # © 2017 James R. Barlow: github.com/jbarlow83
 
 import pytest
@@ -8,8 +7,9 @@ from ocrmypdf import pdfinfo
 import sys
 import os
 import PyPDF2 as pypdf
+from contextlib import contextmanager
 
-
+# pylint: disable=no-member
 spoof = pytest.helpers.spoof
 
 
@@ -32,14 +32,25 @@ def ensure_tess4():
     raise EnvironmentError("Can't find Tesseract 4")
 
 
+@contextmanager
+def modified_os_environ(env):
+    old_env = os.environ.copy()
+    os.environ = env
+    yield
+    os.environ = old_env
+
+
 def tess4_available():
     """Check if a tesseract 4 binary is available, even if it's not the
     official "tesseract" on PATH
 
     """
     try:
-        ensure_tess4()
-        return True
+        # ensure_tess4 locates the tess4 binary we are going to check
+        env = ensure_tess4()
+        with modified_os_environ(env):
+            # Now jump into this environment and make sure it really is Tess4
+            return tesseract.v4() and tesseract.has_textonly_pdf()
     except EnvironmentError:
         pass
 
@@ -63,7 +74,6 @@ def test_textonly_pdf(ensure_tess4, resources, outdir):
         env=ensure_tess4)
 
 
-@pytest.mark.skipif(sys.version_info < (3, 5), reason="needs math.isclose")
 def test_pagesize_consistency_tess4(ensure_tess4, resources, outpdf):
     from math import isclose
 
@@ -117,4 +127,4 @@ def test_content_preservation(ensure_tess4, resources, outpdf):
 
     info = pdfinfo.PdfInfo(outpdf)
     page = info[0]
-    assert len(page.images) > 1, "masks were rasterized"
+    assert len(page.images) > 1, "masks were rasterized"

tests/test_userunit.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # © 2017 James R. Barlow: github.com/jbarlow83
 
 from subprocess import Popen, PIPE, check_output, check_call, DEVNULL