v4.0: release notes

Since the corrected font is provided in the Docker image, there's no
reason to show the warning.

.dockerignore

@@ -1,21 +1,31 @@
-bin/
-build/
-dist/
-include/
-lib/
-ocrmypdf.egg-info/
-staging/
-.git/
-.ruffus_history.sqlite
-MANIFEST.in
-*.sublime*
+*.ipynb
 *.pdf
-*.rst
 *.pyc
+*.rst
+*.sublime*
 */*.pyc
 */*/*.pyc
 */*/*/*.pyc
 */*/*/*/*.pyc
 */*/*/*/*/*.pyc
 */*/*/*/*/*/*.pyc
-*/*/*/*/*/*/*/*.pyc
+*/*/*/*/*/*/*/*.pyc
+.cache/
+.git/
+.ipynb_checkpoints/
+.ruffus_history.sqlite
+bin/
+build/
+dist/
+htmlcov/
+include/
+lib/
+MANIFEST.in
+ocrmypdf.egg-info/
+staging/
+tests/cache/
+tests/output/
+tests/resources/private/
+tmp/
+venv-3.4/
+venv-3.5/

.gitignore

@@ -1,23 +1,29 @@
-tmp/
-log/
+# Development environment
 *.pyc
-tests/output/
-.ruffus_history.sqlite
 *.sublime-*
-/*.pdf
+venv-*/
+pyvenv.cfg
+
+# Package building
+*.egg-info/
+.cache/
+.eggs/
 build/
 dist/
-*.egg-info/
-venv/
-venv-3.4/
-venv-3.5/
-*/test/output
-bin/
-include/
-lib/
-pip-selfcheck.json
-pyvenv.cfg
-htmlcov/
+
+# Automatically generated files
+ocrmypdf/lib/_*.py
+ocrmypdf/version.py
+
+# Code coverage
 .coverage
-.cache/
+htmlcov/
+
+# Testing
+log/
+/*.pdf
 .ipynb_checkpoints/
+tests/cache/
+tests/output/
+tests/resources/private
+tmp/

.travis.yml

@@ -1,14 +1,19 @@
 language: python
 
+cache: pip
+
 cache:
     directories:
-        - $HOME/.cache/pip
         - $HOME/.ccache
         - tarballs
+        - tests/cache
 
 python:
     - 3.4
 
+before_cache:
+    - rm -f $HOME/.cache/pip/log/debug.log
+
 before_install:
     # Ubuntu packages
     - sudo add-apt-repository ppa:evl.ms/precise -y  # for Ghostscript 9.15
@@ -16,7 +21,7 @@ before_install:
     - sudo add-apt-repository ppa:b-eltzner/qpdfview-exp -y  # for QPDF 5
     - sudo add-apt-repository ppa:itachi-san/ffmpeg -y  # for libav 11.2 (for unpaper)
     - sudo apt-get update -qq  # must go after all add-apt-repo
-    - sudo apt-get install -y ghostscript tesseract-ocr tesseract-ocr-deu tesseract-ocr-eng tesseract-ocr-fra qpdf poppler-utils gcc libavformat-dev libavcodec-dev libavutil-dev automake make pkg-config xsltproc 
+    - sudo apt-get install -y ghostscript tesseract-ocr tesseract-ocr-deu tesseract-ocr-eng tesseract-ocr-fra qpdf poppler-utils gcc libavformat-dev libavcodec-dev libavutil-dev automake make pkg-config xsltproc libffi-dev
 
     # pip
     - pip install --upgrade pip
@@ -35,7 +40,7 @@ install:
 script:
     - python setup.py clean
     - python setup.py install
-    - py.test
+    - python setup.py test
 
 os:
     - linux

Dockerfile

@@ -1,6 +1,6 @@
 # OCRmyPDF
 #
-# VERSION               3.0.0
+# VERSION               3.2
 FROM      debian:stretch
 MAINTAINER James R. Barlow <jim@purplerock.ca>
 
@@ -10,18 +10,31 @@ RUN useradd docker \
   && chown docker:docker /home/docker
 
 # Update system and install our dependencies
+# If this command takes too Docker hub's automated build will timeout,
+# so try it in portions
 RUN apt-get update && apt-get install -y --no-install-recommends \
   locales \
-  ghostscript \
-  tesseract-ocr \
-  tesseract-ocr-deu tesseract-ocr-spa tesseract-ocr-eng tesseract-ocr-fra \
-  qpdf \
-  poppler-utils \
   python3 \
   python3-pip \
   python3-venv \
   python3-reportlab \
-  python3-pil
+  python3-pil \
+  python3-wheel
+
+RUN apt-get install -y --no-install-recommends \
+  unpaper \
+  ghostscript \
+  qpdf \
+  poppler-utils
+
+RUN apt-get install -y --no-install-recommends \
+  tesseract-ocr \
+  tesseract-ocr-deu tesseract-ocr-spa tesseract-ocr-eng tesseract-ocr-fra
+
+RUN apt-get install -qy --no-install-recommends \
+  libffi-dev \
+  libpython3-dev \
+  gcc
 
 # Enforce UTF-8
 # Borrowed from https://index.docker.io/u/crosbymichael/python/ 
@@ -30,37 +43,6 @@ RUN dpkg-reconfigure locales && \
   /usr/sbin/update-locale LANG=C.UTF-8
 ENV LC_ALL C.UTF-8
 
-# Build unpaper 6.1
-RUN apt-get install -y \
-  wget \
-  gcc \
-  libavformat-dev \
-  libavcodec-dev \
-  libavutil-dev \
-  autoconf \
-  automake \
-  make \
-  pkg-config \
-  xsltproc
-
-WORKDIR /root
-RUN wget -q https://github.com/Flameeyes/unpaper/archive/unpaper-6.1.tar.gz
-RUN tar xf unpaper-6.1.tar.gz
-WORKDIR /root/unpaper-unpaper-6.1
-RUN autoreconf -i
-RUN ./configure CFLAGS="-O2 -march=native -pipe -flto"
-RUN make -j install
-WORKDIR /
-
-RUN apt-get remove -y \
-  gcc \
-  autoconf \
-  automake \
-  pkg-config \
-  xsltproc \
-  make
-RUN apt-get autoremove -y && apt-get clean -y
-RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/*
 
 # Set up a Python virtualenv and take all of the system packages, so we can
 # rely on the platform packages rather than importing GCC and compiling them
@@ -69,19 +51,34 @@ RUN pyvenv /appenv \
 
 COPY . /application/
 
+# Replace stock Tesseract 3.04.00 font with improved sharp2.ttf that resolves
+# issues in many PDF viewers.
+# Discussion is in https://github.com/tesseract-ocr/tesseract/issues/182
+COPY ./share/sharp2.ttf /usr/share/tesseract-ocr/tessdata/pdf.ttf
+RUN chmod 644 /usr/share/tesseract-ocr/tessdata/pdf.ttf
+
+# Set this here to force a docker version, allowing non-tagged versions to
+# be built
+# ENV SETUPTOOLS_SCM_PRETEND_VERSION=v3.3.0
+
 # Install application and dependencies
 # In this arrangement Pillow and reportlab will be provided by the system
 RUN . /appenv/bin/activate; \
   pip install --upgrade pip \
-  && pip install --no-cache-dir /application \
+  && pip install /application \
   && pip install --no-cache-dir -r /application/test_requirements.txt
 
+# Remove the junk
+RUN apt-get remove -qy gcc
+RUN apt-get autoremove -y && apt-get clean -y
+RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/*
+
 USER docker
 WORKDIR /home/docker
 
-ENV DEFAULT_RUFFUS_HISTORY_FILE=/tmp/.{basename}.ruffus_history.sqlite
 ENV OCRMYPDF_TEST_OUTPUT=/tmp/test-output
 ENV OCRMYPDF_IN_DOCKER=1
+ENV OCRMYPDF_SHARP_TTF=1
 
 # Must use array form of ENTRYPOINT
 # Non-array form does not append other arguments, because that is "intuitive"

Dockerfile.polyglot

@@ -0,0 +1,16 @@
+# OCRmyPDF polyglot
+#
+# VERSION               3.2
+FROM      jbarlow83/ocrmypdf:latest
+MAINTAINER James R. Barlow <jim@purplerock.ca>
+
+# Update system and install our dependencies
+USER root
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  tesseract-ocr-all 
+
+USER docker
+
+# Must use array form of ENTRYPOINT
+# Non-array form does not append other arguments, because that is "intuitive"
+ENTRYPOINT ["/application/docker-wrapper.sh"]

MANIFEST.in

@@ -1,3 +1,4 @@
 recursive-exclude tests/output *
 include requirements.txt
-include test_requirements.txt
+include test_requirements.txt
+include share/*

README.rst

@@ -9,15 +9,9 @@ Main features
 
 -  Generates a searchable
    `PDF/A <https://en.wikipedia.org/?title=PDF/A>`__ file from a regular PDF
-   only containing images
 -  Places OCR text accurately below the image to ease copy / paste
 -  Keeps the exact resolution of the original embedded images
-
-   -  or if requested oversamples the images before OCRing so as to get
-      better results
-
--  When possible, copies input images directly to output without transcoding,
-   to preserve image quality
+-  When possible, inserts OCR information as a "lossless" operation without rendering vector information
 -  Keeps file size about the same
 -  If requested deskews and/or cleans the image before performing OCR
 -  Validates input and output files
@@ -54,22 +48,22 @@ Download OCRmyPDF here: https://github.com/jbarlow83/OCRmyPDF/releases
 
 You can install it to a Python virtual environment or system-wide. 
 
-Installing the Docker container
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Installing the Docker image
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-For many users, installing the Docker container will be easier than installing all of OCRmyPDF's dependencies. For Windows, it is the only option.
+For many users, installing the Docker image will be easier than installing all of OCRmyPDF's dependencies. For Windows, it is the only option.
 
 If you have `Docker <https://docs.docker.com/>`__ installed on your system, you can install
-a Docker container of the latest release.
+a Docker image of the latest release.
 
 Follow the Docker installation instructions for your platform.  If you can run this command
 successfully, your system is ready to download and execute the image::
 
    docker run hello-world
    
-OCRmyPDF will use all available CPU cores.  By default, the VirtualBox machine instance on Windows and OS X has only a single CPU core enabled. Use the VirtualBox Manager to determine the name of your Docker container host, and then follow these optional steps to enable multiple CPUs::
+OCRmyPDF will use all available CPU cores.  By default, the VirtualBox machine instance on Windows and OS X has only a single CPU core enabled. Use the VirtualBox Manager to determine the name of your Docker engine host, and then follow these optional steps to enable multiple CPUs::
 
-   # Optional
+   # Optional step for Mac OS X users
    docker-machine stop "yourVM"
    VBoxManage modifyvm "yourVM" --cpus 2  # or whatever number of core is desired
    docker-machine start "yourVM"
@@ -83,8 +77,14 @@ the image::
 Then tag it to give a more convenient name, just ocrmypdf::
 
    docker tag jbarlow83/ocrmypdf ocrmypdf
-  
-You can then run using the command::
+
+This image contains language packs for English, French, Spanish and German. The alternative "polyglot" image provides `all available language packs <https://github.com/tesseract-ocr/tesseract/blob/master/doc/tesseract.1.asc#languages>`__::
+
+   # Alternative step: If you need all language packs
+   docker pull jbarlow83/ocrmypdf-polyglot
+   docker tag jbarlow83/ocrmypdf-polyglot ocrmypdf
+
+You can then run ocrmypdf using the command::
 
    docker run ocrmypdf --help
   
@@ -96,7 +96,8 @@ In this worked example, the current working directory contains an input file cal
 
    docker run -v "$(pwd):/home/docker"   ocrmypdf --skip-text test.pdf output.pdf
 
-Note that ``ocrmypdf`` has its own separate ``-v VERBOSITYLEVEL`` argument to control debug verbosity. All Docker arguments should before the ``ocrmypdf`` container name and all arguments to ``ocrmypdf`` should be listed after.
+Note that ``ocrmypdf`` has its own separate ``-v VERBOSITYLEVEL`` argument to control debug verbosity. All Docker arguments should before the ``ocrmypdf`` image name and all arguments to ``ocrmypdf`` should be listed after.
+
 
 Installing on Mac OS X
 ~~~~~~~~~~~~~~~~~~~~~~
@@ -119,17 +120,11 @@ Install or upgrade the required Homebrew packages, if any are missing::
    brew install leptonica
    brew install tesseract
    
-It is also recommended that install Pillow and confirm it can read and write JPEG and PNG files::
+Update the homebrew pip and install Pillow::
 
    pip3 install --upgrade pip
    pip3 install --upgrade pillow
 
-Sometimes, the Python imaging library (Pillow) can end up being compiled and installed without support for JPEG and PNG files. (Arguably, this is an unfixed bug in Pillow's installer.) To confirm that Pillow is compiled correctly and can access JPEG and PNG files, try this command::
-
-   python3 -c "from PIL import Image; im = Image.new('1', (1, 1)); im.save('test.png'); im.save('test.jpg')"
-
-If you have trouble getting Pillow to access JPEG and PNG files, `review the installation instructions <https://pillow.readthedocs.org/installation.html>`__.
-
 You can then install OCRmyPDF from PyPI::
 
    pip3 install ocrmypdf
@@ -182,6 +177,11 @@ package <http://www.thefourtheye.in/2014/12/Python-venv-problem-with-ensurepip-i
 
 Ubuntu 14.04 only installs ``unpaper`` version 0.4.2, which is not supported by OCRmyPDF because it is produces invalid output. This program is an optional dependency, and provides page deskewing and cleaning. See `Dockerfile <Dockerfile>`__ for an example of how to building unpaper 6.1 from source. If you choose to install unpaper later, OCRmyPDF will use the foremost version on the system PATH.
 
+Installing on Windows
+~~~~~~~~~~~~~~~~~~~~~
+
+Direct installation on Windows is not possible.  Install the Docker container as described above.
+
       
 Installing HEAD revision from sources
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -189,24 +189,18 @@ Installing HEAD revision from sources
 If you have ``git`` and ``python3.4`` or ``python3.5`` installed, you can install from source. When the ``pip`` installer runs,
 it will alert you if dependencies are missing.
 
-First, clone the HEAD revision::
+To install the HEAD revision from sources in the current Python 3 environment::
 
-   git clone -b master https://github.com/jbarlow83/OCRmyPDF.git
-   cd OCRmyPDF
+   pip3 install git+https://github.com/jbarlow83/OCRmyPDF.git
 
-To install the HEAD revision from sources::
+Or, to install in `development mode <https://pythonhosted.org/setuptools/setuptools.html#development-mode>`__,  allowing customization of OCRmyPDF, use the ``-e`` flag::
 
-   pip3 install .
-
-Or, to install in `development mode <https://pythonhosted.org/setuptools/setuptools.html#development-mode>`__, 
-allowing customization of OCRmyPDF, use the ``-e`` flag::
-
-   pip3 install -e .
+   pip3 install -e git+https://github.com/jbarlow83/OCRmyPDF.git
    
 On certain Linux distributions such as Ubuntu, you may need to use 
 run the install command as superuser::
 
-   sudo pip3 install [-e] .
+   sudo pip3 install [-e] git+https://github.com/jbarlow83/OCRmyPDF.git
    
 Note that this will alter your system's Python distribution. If you prefer 
 to not install as superuser, you can install the package in a Python virtual environment::
@@ -229,6 +223,18 @@ need to be installed. The script requires specific versions of the
 dependencies. Older version than the ones mentioned in the release notes
 are likely not to be compatible to OCRmyPDF.
 
+Languages
+---------
+
+OCRmyPDF uses Tesseract for OCR, and relies on its language packs. For Linux users,
+you can often find packages that provide language packs::
+
+   # Debian/Ubuntu users
+   sudo apt-get install tesseract-ocr-chi-sim
+   
+You can then pass the ``-l LANG`` argument to OCRmyPDF to give a hint as to what languages it should search for. Multiple
+languages can be requested.
+
 Support
 -------
 
@@ -241,8 +247,7 @@ In case you detect an issue, please:
 -  Append the console output of the script when running the debug mode
    (``-v 1`` option)
 -  If possible provide your input PDF file as well as the content of the
-   temporary folder (using a file sharing service like
-   www.file-upload.net)
+   temporary folder (using a file sharing service like Dropbox)
 
 Press & Media
 -------------

RELEASE_NOTES.rst

@@ -6,6 +6,81 @@ Please always read this file before installing the package
 Download software here: https://github.com/jbarlow83/OCRmyPDF/tags
 
 
+v4.0:
+=====
+
+New features
+------------
+
+-  Automatic page rotation (``-r``) is now available. It uses ignores any prior rotation information
+   on PDFs and sets rotation based on the dominant orientation of detectable text. This feature is
+   fairly reliable but some false positives occur especially if there is not much text to work with. (#4) 
+-  Deskewing is now performed using Leptonica instead of unpaper. Leptonica deskews images faster and
+   more reliably than unpaper.
+
+
+Fixes
+-----
+
+-  Fixed an issue where lossless reconstruction could cause some pages to be appear incorrectly
+   rotated in the page had a /Rotate tag applied (#50)
+-  Fixed an issue where lossless reconstruction could misalign the graphics layer with respect to
+   text layer if the page had been cropped such that its origin is not (0, 0) (#49)
+
+
+Changes
+-------
+
+-  Logging output much is now better
+-  ``--deskew`` is now performed by Leptonica instead of unpaper (#25)
+-  libffi is now required
+-  Some changes were made to the Docker and Travis build environments to support libffi
+-  ``--pdf-renderer=tesseract`` now displays a warning if the Tesseract version is less than 3.04.01,
+   the planned release that will include fixes to an important OCR text rendering bug in Tesseract 3.04.00.
+   You can also manually install ./share/sharp2.ttf on top of pdf.ttf in your Tesseract tessdata folder
+   to correct the problem.
+
+
+v3.2.1:
+=======
+
+Changes
+-------
+
+-  Fixed issue #47 "convert() got and unexpected keyword argument 'dpi'" by upgrading to img2pdf 0.2
+-  Tweaked the Dockerfiles
+
+
+v3.2:
+=====
+
+New features
+------------
+
+-  Lossless reconstruction: when possible, OCRmyPDF will inject text layers without 
+   otherwise manipulating the content and layout of a PDF page. For example, a PDF containing a mix
+   of vector and raster content would see the vector content preserved. Images may still be transcoded
+   during PDF/A conversion.  (``--deskew`` and ``--clean-final`` disable this mode, necessarily.)
+-  New argument ``--tesseract-pagesegmode`` allows you to pass page segmentation arguments to Tesseract OCR.
+   This helps for two column text and other situations that confuse Tesseract.
+-  Added a new "polyglot" version of the Docker image, that generates Tesseract with all languages packs installed,
+   for the polyglots among us. It is much larger.
+
+Changes
+-------
+
+-  JPEG transcoding quality is now 95 instead of the default 75. Bigger file sizes for less degradation.
+
+
+
+v3.1.1:
+=======
+
+Changes
+-------
+
+-  Fixed bug that caused incorrect page size and DPI calculations on documents with mixed page sizes
+
 v3.1:
 =====
 
@@ -20,7 +95,7 @@ Changes
 -  Fixed an issue where OCRmyPDF failed to text that certain pages contained previously OCR'ed text, 
    such as OCR text produced by Tesseract 3.04
 -  Inserts /Creator tag into PDFs so that errors can be traced back to this project
--  Added new option --pdf-renderer=auto, to let OCRmyPDF pick the best PDF renderer. 
+-  Added new option ``--pdf-renderer=auto``, to let OCRmyPDF pick the best PDF renderer. 
    Currently it always chooses the 'hocrtransform' renderer but that behavior may change.
 -  Set up Travis CI automatic integration testing
 

ocrmypdf/__init__.py

@@ -1,4 +1,5 @@
 from enum import IntEnum
+import os
 
 
 class ExitCode(IntEnum):
@@ -10,3 +11,12 @@ class ExitCode(IntEnum):
     file_access_error = 5
     already_done_ocr = 6
     other_error = 15
+
+
+def get_program(name):
+    envvar = 'OCRMYPDF_' + name.upper()
+    return os.environ.get(envvar, name)
+
+
+def page_number(input_file):
+    return int(os.path.basename(input_file)[0:6])

ocrmypdf/ghostscript.py

@@ -4,16 +4,20 @@
 from tempfile import NamedTemporaryFile
 from subprocess import Popen, PIPE, check_call
 from shutil import copy
+from . import get_program
 
 
-def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log):
+def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
+                  pageno=1):
     with NamedTemporaryFile(delete=True) as tmp:
         args_gs = [
-            'gs',
+            get_program('gs'),
             '-dQUIET',
             '-dBATCH',
             '-dNOPAUSE',
             '-sDEVICE=%s' % raster_device,
+            '-dFirstPage=%i' % pageno,
+            '-dLastPage=%i' % pageno,
             '-o', tmp.name,
             '-r{0}x{1}'.format(str(xres), str(yres)),
             input_file
@@ -36,14 +40,16 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log):
 def generate_pdfa(pdf_pages, output_file, threads=1):
     with NamedTemporaryFile(delete=True) as gs_pdf:
         args_gs = [
-            "gs",
+            get_program("gs"),
             "-dQUIET",
             "-dBATCH",
             "-dNOPAUSE",
             '-dNumRenderingThreads=' + str(threads),
             "-sDEVICE=pdfwrite",
+            "-dAutoRotatePages=/None",
             "-sColorConversionStrategy=/RGB",
             "-sProcessColorModel=DeviceRGB",
+            "-dJPEGQ=95",
             "-dPDFA=2",
             "-sPDFACompatibilityPolicy=2",
             "-sOutputICCProfile=srgb.icc",

ocrmypdf/hocrtransform.py

@@ -1,4 +1,4 @@
-#!/usr/local/bin/python3
+#!/usr/bin/env python3
 ##############################################################################
 # Copyright (c) 2013-14: fritz-hh from Github
 #   (https://github.com/fritz-hh)

ocrmypdf/leptonica.py

@@ -10,11 +10,15 @@
 
 from __future__ import print_function, absolute_import, division
 import argparse
-import ctypes as C
 import sys
 import os
 import logging
 from tempfile import TemporaryFile
+from ctypes.util import find_library
+from .lib._leptonica import ffi
+from functools import lru_cache
+
+lept = ffi.dlopen(find_library('lept'))
 
 logger = logging.getLogger(__name__)
 
@@ -25,67 +29,6 @@ def stderr(*objs):
     print("leptonica.py:", *objs, file=sys.stderr)
 
 
-from ctypes.util import find_library
-lept_lib = find_library('lept')
-if not lept_lib:
-    stderr("Could not find the Leptonica library")
-    sys.exit(3)
-try:
-    lept = C.cdll.LoadLibrary(lept_lib)
-except Exception:
-    stderr("Could not load the Leptonica library from %s", lept_lib)
-    sys.exit(3)
-
-
-class _PIXCOLORMAP(C.Structure):
-    """struct PixColormap from Leptonica src/pix.h
-    """
-
-    _fields_ = [
-        ("array", C.c_void_p),
-        ("depth", C.c_int32),
-        ("nalloc", C.c_int32),
-        ("n", C.c_int32)
-    ]
-
-
-class _PIX(C.Structure):
-    """struct Pix from Leptonica src/pix.h
-    """
-
-    _fields_ = [
-        ("w", C.c_uint32),
-        ("h", C.c_uint32),
-        ("d", C.c_uint32),
-        ("wpl", C.c_uint32),
-        ("refcount", C.c_uint32),
-        ("xres", C.c_int32),
-        ("yres", C.c_int32),
-        ("informat", C.c_int32),
-        ("text", C.POINTER(C.c_char)),
-        ("colormap", C.POINTER(_PIXCOLORMAP)),
-        ("data", C.POINTER(C.c_uint32))
-    ]
-
-
-PIX = C.POINTER(_PIX)
-
-lept.pixRead.argtypes = [C.c_char_p]
-lept.pixRead.restype = PIX
-lept.pixScale.argtypes = [PIX, C.c_float, C.c_float]
-lept.pixScale.restype = PIX
-lept.pixDeskew.argtypes = [PIX, C.c_int32]
-lept.pixDeskew.restype = PIX
-lept.pixFindSkew.argtypes = [PIX, C.POINTER(C.c_float), C.POINTER(C.c_float)]
-lept.pixFindSkew.restype = C.c_int32
-lept.pixWriteImpliedFormat.argtypes = [C.c_char_p, PIX, C.c_int32, C.c_int32]
-lept.pixWriteImpliedFormat.restype = C.c_int32
-lept.pixDestroy.argtypes = [C.POINTER(PIX)]
-lept.pixDestroy.restype = None
-lept.getLeptonicaVersion.argtypes = []
-lept.getLeptonicaVersion.restype = C.c_char_p
-
-
 class LeptonicaErrorTrap(object):
     """Context manager to trap errors reported by Leptonica.
 
@@ -140,103 +83,140 @@ class LeptonicaIOError(LeptonicaError):
     pass
 
 
-def pixRead(filename):
-    """Load an image file into a PIX object.
+class Pix:
+    def __init__(self, cpix):
+        self.cpix = ffi.gc(cpix, Pix._pix_destroy)
 
-    Leptonica can load TIFF, PNM (PBM, PGM, PPM), PNG, and JPEG.  If loading
-    fails then the object will wrap a C null pointer.
-
-    """
-    with LeptonicaErrorTrap():
-        return lept.pixRead(filename.encode(sys.getfilesystemencoding()))
-
-
-def pixScale(pix, scalex, scaley):
-    """Returns the pix object rescaled according to the proportions given."""
-    with LeptonicaErrorTrap():
-        return lept.pixScale(pix, scalex, scaley)
-
-
-def pixDeskew(pix, reduction_factor=0):
-    """Returns the deskewed pix object.
-
-    A clone of the original is returned when the algorithm cannot find a skew
-    angle with sufficient confidence.
-
-    reduction_factor -- amount to downsample (0 for default) when searching
-        for skew angle
-
-    """
-    with LeptonicaErrorTrap():
-        return lept.pixDeskew(pix, reduction_factor)
-
-
-def pixFindSkew(pix):
-    """Returns a tuple (deskew angle in degrees, confidence value).
-
-    Returns (None, None) if no angle is available.
-
-    """
-    with LeptonicaErrorTrap():
-        angle = C.c_float(0.0)
-        confidence = C.c_float(0.0)
-        result = lept.pixFindSkew(pix, C.byref(angle), C.byref(confidence))
-        if result == 0:
-            return (angle.value, confidence.value)
+    def __repr__(self):
+        if self.cpix:
+            s = "<leptonica.Pix image size={0}x{1} depth={2} at 0x{3:x}>"
+            return s.format(self.cpix.w, self.cpix.h, self.cpix.d,
+                            int(ffi.cast("intptr_t", self.cpix)))
         else:
-            return (None, None)
+            return "<leptonica.Pix image NULL>"
 
+    @classmethod
+    def read(cls, filename):
+        """Load an image file into a PIX object.
 
-def pixWriteImpliedFormat(filename, pix, jpeg_quality=0, jpeg_progressive=0):
-    """Write pix to the filename, with the extension indicating format.
+        Leptonica can load TIFF, PNM (PBM, PGM, PPM), PNG, and JPEG.  If
+        loading fails then the object will wrap a C null pointer.
+        """
+        with LeptonicaErrorTrap():
+            return cls(lept.pixRead(
+                filename.encode(sys.getfilesystemencoding())))
 
-    jpeg_quality -- quality (iff JPEG; 1 - 100, 0 for default)
-    jpeg_progressive -- (iff JPEG; 0 for baseline seq., 1 for progressive)
+    def write_implied_format(
+            self, filename, jpeg_quality=0, jpeg_progressive=0):
+        """Write pix to the filename, with the extension indicating format.
 
-    """
-    fileroot, extension = os.path.splitext(filename)
-    fix_pnm = False
-    if extension.lower() in ('.pbm', '.pgm', '.ppm'):
-        # Leptonica does not process handle these extensions correctly, but
-        # does handle .pnm correctly.  Add another .pnm suffix.
-        filename += '.pnm'
-        fix_pnm = True
+        jpeg_quality -- quality (iff JPEG; 1 - 100, 0 for default)
+        jpeg_progressive -- (iff JPEG; 0 for baseline seq., 1 for progressive)
+        """
+        fileroot, extension = os.path.splitext(filename)
+        fix_pnm = False
+        if extension.lower() in ('.pbm', '.pgm', '.ppm'):
+            # Leptonica does not process handle these extensions correctly, but
+            # does handle .pnm correctly.  Add another .pnm suffix.
+            filename += '.pnm'
+            fix_pnm = True
 
-    with LeptonicaErrorTrap():
-        lept.pixWriteImpliedFormat(
-            filename.encode(sys.getfilesystemencoding()),
-            pix, jpeg_quality, jpeg_progressive)
+        with LeptonicaErrorTrap():
+            lept.pixWriteImpliedFormat(
+                filename.encode(sys.getfilesystemencoding()),
+                self.cpix, jpeg_quality, jpeg_progressive)
 
-    if fix_pnm:
-        from shutil import move
-        move(filename, filename[:-4])   # Remove .pnm suffix
+        if fix_pnm:
+            from shutil import move
+            move(filename, filename[:-4])   # Remove .pnm suffix
 
+    def deskew(self, reduction_factor=0):
+        """Returns the deskewed pix object.
 
-def pixDestroy(pix):
-    """Destroy the pix object.
+        A clone of the original is returned when the algorithm cannot find a
+        skew angle with sufficient confidence.
 
-    Function signature is pixDestroy(struct Pix **), hence C.byref() to pass
-    the address of the pointer.
+        reduction_factor -- amount to downsample (0 for default) when searching
+            for skew angle
+        """
+        with LeptonicaErrorTrap():
+            return Pix(lept.pixDeskew(self.cpix, reduction_factor))
 
-    """
-    with LeptonicaErrorTrap():
-        lept.pixDestroy(C.byref(pix))
+    def scale(self, scalex, scaley):
+        "Returns the pix object rescaled according to the proportions given."
+        with LeptonicaErrorTrap():
+            return Pix(lept.pixScale(self.cpix, scalex, scaley))
+
+    def rotate180(self):
+        with LeptonicaErrorTrap():
+            return Pix(lept.pixRotate180(ffi.NULL, self.cpix))
+
+    def find_skew(self):
+        """Returns a tuple (deskew angle in degrees, confidence value).
+
+        Returns (None, None) if no angle is available.
+        """
+        with LeptonicaErrorTrap():
+            angle = ffi.new('float *', 0.0)
+            confidence = ffi.new('float *', 0.0)
+            result = lept.pixFindSkew(self.cpix, angle, confidence)
+            if result == 0:
+                return (angle[0], confidence[0])
+            else:
+                return (None, None)
+
+    @staticmethod
+    def correlation_binary(pix1, pix2):
+        if getLeptonicaVersion() < 'leptonica-1.72':
+            # Older versions of Leptonica (pre-1.72) have a buggy
+            # implementation of pixCorrelationBinary that overflows on larger
+            # images.
+            pix1_count = ffi.new('l_int32 *', 0)
+            pix2_count = ffi.new('l_int32 *', 0)
+            pixn_count = ffi.new('l_int32 *', 0)
+            tab8 = lept.makePixelSumTab8()  # Small memory leak on each call
+
+            lept.pixCountPixels(pix1.cpix, pix1_count, tab8)
+            lept.pixCountPixels(pix2.cpix, pix2_count, tab8)
+            pixn = Pix(lept.pixAnd(ffi.NULL, pix1.cpix, pix2.cpix))
+            lept.pixCountPixels(pixn.cpix, pixn_count, tab8)
+
+            # Python converts these int32s to larger units as needed
+            # to avoid overflow. Overflow happens easily here.
+            correlation = (
+                    (pixn_count[0] * pixn_count[0]) /
+                    (pix1_count[0] * pix2_count[0])
+                    )
+            return correlation
+        else:
+            correlation = ffi.new('float *', 0.0)
+            result = lept.pixCorrelationBinary(pix1.cpix, pix2.cpix,
+                                               correlation)
+            if result != 0:
+                raise LeptonicaError("Correlation failed")
+            return correlation[0]
+
+    @staticmethod
+    def _pix_destroy(pix):
+        ptr_to_pix = ffi.new('PIX **', pix)
+        lept.pixDestroy(ptr_to_pix)
+        # print('pix destroy ' + repr(pix))
 
 
+@lru_cache(maxsize=1)
 def getLeptonicaVersion():
     """Get Leptonica version string.
 
     Caveat: Leptonica expects the caller to free this memory.  We don't,
     since that would involve binding to libc to access libc.free(),
     a pointless effort to reclaim 100 bytes of memory.
-
     """
-    return lept.getLeptonicaVersion().decode()
+    return ffi.string(lept.getLeptonicaVersion()).decode()
 
 
 def deskew(infile, outfile, dpi):
     try:
-        pix_source = pixRead(infile)
+        pix_source = Pix.read(infile)
     except LeptonicaIOError:
         raise LeptonicaIOError("Failed to open file: %s" % infile)
 
@@ -244,14 +224,12 @@ def deskew(infile, outfile, dpi):
         reduction_factor = 1  # Don't downsample too much if DPI is already low
     else:
         reduction_factor = 0  # Use default
-    pix_deskewed = pixDeskew(pix_source, reduction_factor)
+    pix_deskewed = pix_source.deskew(reduction_factor)
 
     try:
-        pixWriteImpliedFormat(outfile, pix_deskewed)
+        pix_deskewed.write_implied_format(outfile)
     except LeptonicaIOError:
         raise LeptonicaIOError("Failed to open destination file: %s" % outfile)
-    pixDestroy(pix_source)
-    pixDestroy(pix_deskewed)
 
 
 if __name__ == '__main__':
@@ -325,7 +303,6 @@ def test_skew_angle():
             rotated_im.save(tmpfile)
             pix = pixRead(tmpfile.name)
             angle, confidence = pixFindSkew(pix)
-            pixDestroy(pix)
             print('{0} {1}  {2}'.format(rotate_angle, angle, confidence), file=sys.stderr)
 
 

ocrmypdf/lib/compile_leptonica.py

@@ -0,0 +1,64 @@
+from cffi import FFI
+
+ffi = FFI()
+ffi.set_source("ocrmypdf.lib._leptonica", None)
+ffi.cdef("""
+typedef signed char             l_int8;
+typedef unsigned char           l_uint8;
+typedef short                   l_int16;
+typedef unsigned short          l_uint16;
+typedef int                     l_int32;
+typedef unsigned int            l_uint32;
+typedef float                   l_float32;
+typedef double                  l_float64;
+typedef long long               l_int64;
+typedef unsigned long long      l_uint64;
+
+struct Pix
+{
+    l_uint32             w;           /* width in pixels                   */
+    l_uint32             h;           /* height in pixels                  */
+    l_uint32             d;           /* depth in bits (bpp)               */
+    l_uint32             spp;         /* number of samples per pixel       */
+    l_uint32             wpl;         /* 32-bit words/line                 */
+    l_uint32             refcount;    /* reference count (1 if no clones)  */
+    l_int32              xres;        /* image res (ppi) in x direction    */
+                                      /* (use 0 if unknown)                */
+    l_int32              yres;        /* image res (ppi) in y direction    */
+                                      /* (use 0 if unknown)                */
+    l_int32              informat;    /* input file format, IFF_*          */
+    l_int32              special;     /* special instructions for I/O, etc */
+    char                *text;        /* text string associated with pix   */
+    struct PixColormap  *colormap;    /* colormap (may be null)            */
+    l_uint32            *data;        /* the image data                    */
+};
+typedef struct Pix PIX;
+
+struct PixColormap
+{
+    void            *array;     /* colormap table (array of RGBA_QUAD)     */
+    l_int32          depth;     /* of pix (1, 2, 4 or 8 bpp)               */
+    l_int32          nalloc;    /* number of color entries allocated       */
+    l_int32          n;         /* number of color entries used            */
+};
+typedef struct PixColormap  PIXCMAP;
+""")
+
+ffi.cdef("""
+PIX * pixRead ( const char *filename );
+PIX * pixScale ( PIX *pixs, l_float32 scalex, l_float32 scaley );
+l_int32 pixFindSkew ( PIX *pixs, l_float32 *pangle, l_float32 *pconf );
+l_int32 pixWriteImpliedFormat ( const char *filename, PIX *pix, l_int32 quality, l_int32 progressive );
+void pixDestroy ( PIX **ppix );
+PIX * pixDeskew ( PIX *pixs, l_int32 redsearch );
+char * getLeptonicaVersion (  );
+l_int32 pixCorrelationBinary(PIX *pix1, PIX *pix2, l_float32 *pval);
+PIX *pixRotate180(PIX *pixd, PIX *pixs);
+
+l_int32 pixCountPixels ( PIX *pix, l_int32 *pcount, l_int32 *tab8 );
+PIX * pixAnd ( PIX *pixd, PIX *pixs1, PIX *pixs2 );
+l_int32 * makePixelSumTab8 ( void );
+""")
+
+if __name__ == '__main__':
+    ffi.compile()

ocrmypdf/main.py

@@ -2,7 +2,7 @@
 # © 2015 James R. Barlow: github.com/jbarlow83
 
 from contextlib import suppress
-from tempfile import NamedTemporaryFile, mkdtemp
+from tempfile import mkdtemp
 import sys
 import os
 import re
@@ -11,35 +11,36 @@ import warnings
 import multiprocessing
 import atexit
 import textwrap
+import img2pdf
+import logging
 
 import PyPDF2 as pypdf
 from PIL import Image
 
-from subprocess import Popen, check_call, PIPE, CalledProcessError, \
-    TimeoutExpired, check_output, STDOUT
-try:
-    from subprocess import DEVNULL
-except ImportError:
-    DEVNULL = open(os.devnull, 'wb')
-
+from functools import partial
 
 from ruffus import transform, suffix, merge, active_if, regex, jobs_limit, \
     formatter, follows, split, collate, check_if_uptodate
 import ruffus.ruffus_exceptions as ruffus_exceptions
 import ruffus.cmdline as cmdline
+import ruffus.proxy_logger as proxy_logger
 
 from .hocrtransform import HocrTransform
 from .pageinfo import pdf_get_all_pageinfo
 from .pdfa import generate_pdfa_def
 from . import ghostscript
 from . import tesseract
-from . import ExitCode
+from . import qpdf
+from . import ExitCode, page_number
+
+import pkg_resources
+
+VERSION = pkg_resources.get_distribution('ocrmypdf').version
 
 warnings.simplefilter('ignore', pypdf.utils.PdfReadWarning)
 
 
 BASEDIR = os.path.dirname(os.path.realpath(__file__))
-VERSION = '3.1'
 
 
 # -------------
@@ -59,7 +60,6 @@ if tesseract.version() < MINIMUM_TESS_VERSION:
             MINIMUM_TESS_VERSION, tesseract.version()))
     sys.exit(ExitCode.missing_dependency)
 
-
 try:
     import PIL.features
     check_codec = PIL.features.check_codec
@@ -106,7 +106,7 @@ parser = cmdline.get_argparse(
     ignored_args=[
         'touch_files_only', 'recreate_database', 'checksum_file_name',
         'key_legend_in_graph', 'draw_graph_horizontally', 'flowchart_format',
-        'forced_tasks', 'target_tasks', 'use_threads'])
+        'forced_tasks', 'target_tasks', 'use_threads', 'jobs', 'log_file'])
 
 parser.add_argument(
     'input_file',
@@ -117,10 +117,13 @@ parser.add_argument(
 parser.add_argument(
     '-l', '--language', action='append',
     help="languages of the file to be OCRed")
+parser.add_argument(
+    '-j', '--jobs', metavar='N', type=int,
+    help="Use up to N CPU cores simultaneously (default: use all)")
 
 metadata = parser.add_argument_group(
     "Metadata options",
-    "Set output PDF/A metadata (default: use input document's title)")
+    "Set output PDF/A metadata (default: use input document's metadata)")
 metadata.add_argument(
     '--title', type=str,
     help="set document title (place multiple words in quotes)")
@@ -129,15 +132,17 @@ metadata.add_argument(
     help="set document author")
 metadata.add_argument(
     '--subject', type=str,
-    help="set document")
+    help="set document subject description")
 metadata.add_argument(
     '--keywords', type=str,
     help="set document keywords")
 
-
 preprocessing = parser.add_argument_group(
     "Preprocessing options",
     "Improve OCR quality and final image")
+preprocessing.add_argument(
+    '-r', '--rotate-pages', action='store_true',
+    help="automatically rotate pages based on detected text orientation")
 preprocessing.add_argument(
     '-d', '--deskew', action='store_true',
     help="deskew each page before performing OCR")
@@ -152,32 +157,35 @@ preprocessing.add_argument(
     help="oversample images to at least the specified DPI, to improve OCR "
          "results slightly")
 
-parser.add_argument(
+ocrsettings = parser.add_argument_group(
+    "OCR options",
+    "Control how OCR is applied")
+ocrsettings.add_argument(
     '-f', '--force-ocr', action='store_true',
     help="rasterize any fonts or vector images on each page and apply OCR")
-parser.add_argument(
+ocrsettings.add_argument(
     '-s', '--skip-text', action='store_true',
     help="skip OCR on any pages that already contain text, but include the"
          " page in final output")
-parser.add_argument(
+ocrsettings.add_argument(
     '--skip-big', type=float, metavar='MPixels',
     help="skip OCR on pages larger than the specified amount of megapixels, "
          "but include skipped pages in final output")
-# parser.add_argument(
-#     '--exact-image', action='store_true',
-#     help="Use original page from PDF without re-rendering")
 
 advanced = parser.add_argument_group(
     "Advanced",
     "Advanced options for power users")
 advanced.add_argument(
-    '--tesseract-config', default=[], type=list, action='append',
+    '--tesseract-config', action='append', metavar='CFG', default=[],
     help="additional Tesseract configuration files")
+advanced.add_argument(
+    '--tesseract-pagesegmode', action='store', type=int, metavar='PSM',
+    help="set Tesseract page segmentation mode (see tesseract --help)")
 advanced.add_argument(
     '--pdf-renderer', choices=['auto', 'tesseract', 'hocr'], default='auto',
     help='choose OCR PDF renderer')
 advanced.add_argument(
-    '--tesseract-timeout', default=180.0, type=float,
+    '--tesseract-timeout', default=180.0, type=float, metavar='SECONDS',
     help='give up on OCR after the timeout, but copy the preprocessed page '
          'into the final output')
 
@@ -219,6 +227,13 @@ if not set(options.language).issubset(tesseract.languages()):
 if options.pdf_renderer == 'auto':
     options.pdf_renderer = 'hocr'
 
+if options.pdf_renderer == 'tesseract' and tesseract.version() < '3.04.01' \
+        and os.environ.get('OCRMYPDF_SHARP_TTF', '') != '1':
+    complain(
+        "WARNING: Your version of tesseract has problems with PDF output. "
+        "Some PDF viewers will fail to find searchable text.\n"
+        "--pdf-renderer=tesseract is not recommended.")
+
 if any((options.deskew, options.clean, options.clean_final)):
     try:
         from . import unpaper
@@ -245,13 +260,34 @@ if options.clean and not options.clean_final \
         "Tesseract PDF renderer cannot render --clean pages without "
         "also performing --clean-final, so --clean-final is assumed.")
 
+lossless_reconstruction = False
+if options.pdf_renderer == 'hocr':
+    if not options.deskew and not options.clean_final and not options.force_ocr:
+        lossless_reconstruction = True
+
 
 # ----------
 # Logging
 
 
-_logger, _logger_mutex = cmdline.setup_logging(__name__, options.log_file,
-                                               options.verbose)
+def logging_factory(logger_name, listargs):
+    log_file_name, verbose = listargs
+
+    root_logger = logging.getLogger(logger_name)
+    root_logger.setLevel(logging.DEBUG)
+
+    handler = logging.StreamHandler(sys.stderr)
+    formatter_ = logging.Formatter("%(levelname)7s - %(message)s")
+    handler.setFormatter(formatter_)
+    if verbose:
+        handler.setLevel(logging.DEBUG)
+    else:
+        handler.setLevel(logging.INFO)
+    root_logger.addHandler(handler)
+    return root_logger
+
+_logger, _logger_mutex = proxy_logger.make_shared_logger_and_proxy(
+    logging_factory, __name__, [None, options.verbose])
 
 
 class WrappedLogger:
@@ -350,26 +386,11 @@ def repair_pdf(
         log,
         pdfinfo,
         pdfinfo_lock):
-    args_qpdf = [
-        'qpdf', input_file, output_file
-    ]
-    try:
-        out = check_output(args_qpdf, stderr=STDOUT, universal_newlines=True)
-    except CalledProcessError as e:
-        if e.returncode == 2:
-            print("{0}: not a valid PDF, and could not repair it.".format(
-                    options.input_file))
-            print("Details:")
-            print(e.output)
-        else:
-            print(e.output)
-        sys.exit(ExitCode.input_file)
-
-    log.debug(out)
 
+    qpdf.repair(input_file, output_file, log)
     with pdfinfo_lock:
         pdfinfo.extend(pdf_get_all_pageinfo(output_file))
-        log.info(pdfinfo)
+        log.debug(pdfinfo)
 
 
 def get_pageinfo(input_file, pdfinfo, pdfinfo_lock):
@@ -387,11 +408,11 @@ def is_ocr_required(pageinfo, log):
         # or both. It seems quite unlikely that one would find meaningful text
         # from rasterizing vector content. So skip the page.
         log.info(
-            "Page {0} has no images - skipping OCR".format(page)
+            "{0:4d}: page has no images - skipping OCR".format(page)
         )
         ocr_required = False
     elif pageinfo['has_text']:
-        s = "Page {0} already has text! – {1}"
+        s = "{0:4d}: page already has text! – {1}"
 
         if not options.force_ocr and not options.skip_text:
             log.error(s.format(page,
@@ -410,9 +431,10 @@ def is_ocr_required(pageinfo, log):
         pixel_count = pageinfo['width_pixels'] * pageinfo['height_pixels']
         if pixel_count > (options.skip_big * 1000000):
             ocr_required = False
-            log.info(
-                "Page {0} is very large; skipping due to -b".format(page))
-
+            log.warning(
+                "{0:4d}: page too big, skipping OCR "
+                "({1:.1f} MPixels > {2:.1f} MPixels --skip-big)".format(
+                    page, pixel_count / 1000000, options.skip_big))
     return ocr_required
 
 
@@ -431,16 +453,14 @@ def split_pages(
         with suppress(FileNotFoundError):
             os.unlink(oo)
 
-    pages = check_output(['qpdf', '--show-npages', input_file],
-                         universal_newlines=True, close_fds=True)
+    # If no files were repaired the input will be empty
+    if not input_file:
+        log.error("{0}: file not found or invalid argument".format(
+                options.input_file))
+        sys.exit(ExitCode.input_file)
 
-    for n in range(int(pages)):
-        args_qpdf = [
-            'qpdf', input_file,
-            '--pages', input_file, '{0}'.format(n + 1), '--',
-            os.path.join(work_folder, '{0:06d}.page.pdf'.format(n + 1))
-        ]
-        check_call(args_qpdf)
+    npages = qpdf.get_npages(input_file, log)
+    qpdf.split_pages(input_file, work_folder, npages)
 
     from glob import glob
     for filename in glob(os.path.join(work_folder, '*.page.pdf')):
@@ -455,9 +475,85 @@ def split_pages(
                 os.path.basename(filename)[0:6] + alt_suffix))
 
 
+@active_if(options.rotate_pages)
 @transform(
     input=split_pages,
-    filter=suffix('.ocr.page.pdf'),
+    filter=suffix('.page.pdf'),
+    output='.preview.jpg',
+    output_dir=work_folder,
+    extras=[_log, _pdfinfo, _pdfinfo_lock])
+def rasterize_preview(
+        input_file,
+        output_file,
+        log,
+        pdfinfo,
+        pdfinfo_lock):
+    ghostscript.rasterize_pdf(
+        input_file=input_file,
+        output_file=output_file,
+        xres=200,
+        yres=200,
+        raster_device='jpeggray',
+        log=log)
+
+
+@collate(
+    input=[split_pages, rasterize_preview],
+    filter=regex(r".*/(\d{6})(\.ocr|\.skip)(?:\.page\.pdf|\.preview\.jpg)"),
+    output=os.path.join(work_folder, r'\1\2.oriented.pdf'),
+    extras=[_log, _pdfinfo, _pdfinfo_lock])
+def orient_page(
+        infiles,
+        output_file,
+        log,
+        pdfinfo,
+        pdfinfo_lock):
+
+    page_pdf = next(ii for ii in infiles if ii.endswith('.page.pdf'))
+
+    if not options.rotate_pages:
+        re_symlink(page_pdf, output_file)
+        return
+    preview = next(ii for ii in infiles if ii.endswith('.preview.jpg'))
+
+    orient_conf = tesseract.get_orientation(
+        preview,
+        language=options.language,
+        timeout=options.tesseract_timeout,
+        log=log)
+
+    direction = {
+        0: '⇧',
+        90: '⇦',
+        180: '⇩',
+        270: '⇨'
+    }
+
+    log.info(
+        '{0:4d}: page is facing {1}, confidence {2:.2f}{3}'.format(
+            page_number(preview),
+            direction.get(orient_conf.angle, '?'),
+            orient_conf.confidence,
+            ' - correcting rotation' if orient_conf.angle != 0 else '')
+    )
+
+    if orient_conf.angle == 0:
+        re_symlink(page_pdf, output_file)
+    else:
+        writer = pypdf.PdfFileWriter()
+        reader = pypdf.PdfFileReader(page_pdf)
+        page = reader.pages[0]
+
+        # Rotate opposite of orientation
+        rotated_page = page.rotateClockwise(orient_conf.angle)
+        writer.addPage(rotated_page)
+        with open(output_file, 'wb') as out:
+            writer.write(out)
+
+
+@transform(
+    input=orient_page,
+    filter=suffix('.ocr.oriented.pdf'),
     output='.page.png',
     output_dir=work_folder,
     extras=[_log, _pdfinfo, _pdfinfo_lock])
@@ -480,7 +576,7 @@ def rasterize_with_ghostscript(
                  for image in pageinfo['images']):
             device = 'pnggray'
 
-    log.debug("Rendering {0} with {1}".format(
+    log.debug("Rasterize {0} with {1}".format(
             os.path.basename(input_file), device))
     xres = max(pageinfo['xres'], options.oversample or 0)
     yres = max(pageinfo['yres'], options.oversample or 0)
@@ -507,7 +603,8 @@ def preprocess_deskew(
     pageinfo = get_pageinfo(input_file, pdfinfo, pdfinfo_lock)
     dpi = int(pageinfo['xres'])
 
-    unpaper.deskew(input_file, output_file, dpi, log)
+    from . import leptonica
+    leptonica.deskew(input_file, output_file, dpi)
 
 
 @transform(
@@ -545,63 +642,19 @@ def ocr_tesseract_hocr(
         pdfinfo,
         pdfinfo_lock):
 
-    pageinfo = get_pageinfo(input_file, pdfinfo, pdfinfo_lock)
-
-    badxml = os.path.splitext(output_file)[0] + '.badxml'
-
-    args_tesseract = [
-        'tesseract',
-        '-l', '+'.join(options.language),
-        input_file,
-        badxml,
-        'hocr'
-    ] + options.tesseract_config
-    p = Popen(args_tesseract, close_fds=True, stdout=PIPE, stderr=PIPE,
-              universal_newlines=True)
-    try:
-        stdout, stderr = p.communicate(timeout=options.tesseract_timeout)
-    except TimeoutExpired:
-        p.kill()
-        stdout, stderr = p.communicate()
-        # Generate a HOCR file with no recognized text if tesseract times out
-        # Temporary workaround to hocrTransform not being able to function if
-        # it does not have a valid hOCR file.
-        with open(output_file, 'w', encoding="utf-8") as f:
-            f.write(tesseract.HOCR_TEMPLATE.format(
-                pageinfo['width_pixels'],
-                pageinfo['height_pixels']))
-    else:
-        if stdout:
-            log.info(stdout)
-        if stderr:
-            log.error(stderr)
-
-        if p.returncode != 0:
-            raise CalledProcessError(p.returncode, args_tesseract)
-
-        if os.path.exists(badxml + '.html'):
-            # Tesseract 3.02 appends suffix ".html" on its own (.badxml.html)
-            shutil.move(badxml + '.html', badxml)
-        elif os.path.exists(badxml + '.hocr'):
-            # Tesseract 3.03 appends suffix ".hocr" on its own (.badxml.hocr)
-            shutil.move(badxml + '.hocr', badxml)
-
-        # Tesseract 3.03 inserts source filename into hocr file without
-        # escaping it, creating invalid XML and breaking the parser.
-        # As a workaround, rewrite the hocr file, replacing the filename
-        # with a space.  Don't know if Tesseract 3.02 does the same.
-
-        regex_nested_single_quotes = re.compile(
-            r"""title='image "([^"]*)";""")
-        with open(badxml, mode='r', encoding='utf-8') as f_in, \
-                open(output_file, mode='w', encoding='utf-8') as f_out:
-            for line in f_in:
-                line = regex_nested_single_quotes.sub(
-                    r"""title='image " ";""", line)
-                f_out.write(line)
+    tesseract.generate_hocr(
+        input_file=input_file,
+        output_hocr=output_file,
+        language=options.language,
+        tessconfig=options.tesseract_config,
+        timeout=options.tesseract_timeout,
+        pageinfo_getter=partial(get_pageinfo, input_file, pdfinfo,
+                                pdfinfo_lock),
+        pagesegmode=options.tesseract_pagesegmode,
+        log=log
+        )
 
 
-@active_if(options.pdf_renderer == 'hocr')
 @collate(
     input=[rasterize_with_ghostscript, preprocess_deskew, preprocess_clean],
     filter=regex(r".*/(\d{6})(?:\.page|\.pp-deskew|\.pp-clean)\.png"),
@@ -624,31 +677,70 @@ def select_image_for_pdf(
     pageinfo = get_pageinfo(image, pdfinfo, pdfinfo_lock)
     if all(image['enc'] == 'jpeg' for image in pageinfo['images']):
         # If all images were JPEGs originally, produce a JPEG as output
-        Image.open(image).save(output_file, format='JPEG')
+        im = Image.open(image)
+        dpi = im.info.get(
+            'dpi',
+            (int(pageinfo['xres']), int(pageinfo['yres']))
+        )
+        im.save(output_file, format='JPEG', dpi=dpi)
     else:
         re_symlink(image, output_file)
 
 
 @active_if(options.pdf_renderer == 'hocr')
 @collate(
-    input=[select_image_for_pdf, ocr_tesseract_hocr],
-    filter=regex(r".*/(\d{6})(?:\.image|\.hocr)"),
-    output=os.path.join(work_folder, r'\1.rendered.pdf'),
+    input=[select_image_for_pdf, orient_page],
+    filter=regex(r".*/(\d{6})(?:\.image|\.ocr\.oriented\.pdf)"),
+    output=os.path.join(work_folder, r'\1.image-layer.pdf'),
     extras=[_log, _pdfinfo, _pdfinfo_lock])
-def render_hocr_page(
+def select_image_layer(
         infiles,
         output_file,
         log,
         pdfinfo,
         pdfinfo_lock):
-    hocr = next(ii for ii in infiles if ii.endswith('.hocr'))
+
+    page_pdf = next(ii for ii in infiles if ii.endswith('.ocr.oriented.pdf'))
     image = next(ii for ii in infiles if ii.endswith('.image'))
 
-    pageinfo = get_pageinfo(image, pdfinfo, pdfinfo_lock)
+    if lossless_reconstruction:
+        log.debug("{:4d}: page eligible for lossless reconstruction".format(
+            page_number(page_pdf)))
+        re_symlink(page_pdf, output_file)
+    else:
+        pageinfo = get_pageinfo(image, pdfinfo, pdfinfo_lock)
+        dpi = round(max(pageinfo['xres'], pageinfo['yres'],
+                        options.oversample))
+        imgsize = ((img2pdf.ImgSize.dpi, dpi), (img2pdf.ImgSize.dpi, dpi))
+
+        layout_fun = img2pdf.get_layout_fun(None, imgsize, None, None, None)
+
+        with open(image, 'rb') as imfile, \
+                open(output_file, 'wb') as pdf:
+            rawdata = imfile.read()
+            pdf.write(img2pdf.convert(
+                rawdata, producer="img2pdf", with_pdfrw=False,
+                layout_fun=layout_fun))
+
+
+@active_if(options.pdf_renderer == 'hocr')
+@transform(
+    input=ocr_tesseract_hocr,
+    filter=suffix('.hocr'),
+    output='.hocr.pdf',
+    extras=[_log, _pdfinfo, _pdfinfo_lock])
+def render_hocr_page(
+        input_file,
+        output_file,
+        log,
+        pdfinfo,
+        pdfinfo_lock):
+    hocr = input_file
+    pageinfo = get_pageinfo(hocr, pdfinfo, pdfinfo_lock)
     dpi = round(max(pageinfo['xres'], pageinfo['yres'], options.oversample))
 
     hocrtransform = HocrTransform(hocr, dpi)
-    hocrtransform.to_pdf(output_file, imageFileName=image,
+    hocrtransform.to_pdf(output_file, imageFileName=None,
                          showBoundingboxes=False, invisibleText=True)
 
 
@@ -676,10 +768,77 @@ def render_hocr_debug_page(
                          showBoundingboxes=True, invisibleText=False)
 
 
+@active_if(options.pdf_renderer == 'hocr')
+@collate(
+    input=[render_hocr_page, select_image_layer],
+    filter=regex(r".*/(\d{6})(?:\.hocr\.pdf|\.image-layer\.pdf)"),
+    output=os.path.join(work_folder, r'\1.rendered.pdf'),
+    extras=[_log, _pdfinfo, _pdfinfo_lock])
+def add_text_layer(
+        infiles,
+        output_file,
+        log,
+        pdfinfo,
+        pdfinfo_lock):
+    text = next(ii for ii in infiles if ii.endswith('.hocr.pdf'))
+    image = next(ii for ii in infiles if ii.endswith('.image-layer.pdf'))
+
+    pdf_text = pypdf.PdfFileReader(open(text, "rb"))
+    pdf_image = pypdf.PdfFileReader(open(image, "rb"))
+
+    page_text = pdf_text.getPage(0)
+
+    # The text page always will be oriented up
+    # but if lossless_reconstruction, pdf_image may have a rotation applied
+    # we can't just merge the pages, because a page can only have one /Rotate
+    # tag, so the differential rotation must be corrected.
+    # Also, pdf_image may not have its mediabox nailed to (0, 0)
+    page_image = pdf_image.getPage(0)
+    rotation = page_image.get('/Rotate', 0)
+
+    # /Rotate is a clockwise rotation: 90 means page facing "east"
+    # The negative of this value is the angle that eliminates that rotation
+    rotation = -rotation % 360
+
+    x1 = page_image.mediaBox.getLowerLeft_x()
+    x2 = page_image.mediaBox.getUpperRight_x()
+    y1 = page_image.mediaBox.getLowerLeft_y()
+    y2 = page_image.mediaBox.getUpperRight_y()
+
+    # Rotation occurs about the page's (0, 0). Most pages will have the media
+    # box at (0, 0) will all content in the first quadrant but some cropped
+    # files may have an offset mediabox. We translate the page so that its
+    # bottom left corner after rotation is pinned to (0, 0) with the image
+    # in the first quadrant.
+    if rotation == 0:
+        tx, ty = -x1, -y1
+    elif rotation == 90:
+        tx, ty = y2, -x1
+    elif rotation == 180:
+        tx, ty = x2, y2
+    elif rotation == 270:
+        tx, ty = -y1, x2
+    else:
+        pass
+
+    if rotation != 0:
+        log.info("{0:4d}: rotating image layer {1} degrees".format(
+            page_number(image), rotation, tx, ty))
+
+    page_text.mergeRotatedScaledTranslatedPage(
+        page_image, rotation, 1.0, tx, ty, expand=False)
+
+    pdf_output = pypdf.PdfFileWriter()
+    pdf_output.addPage(page_text)
+
+    with open(output_file, "wb") as out:
+        pdf_output.write(out)
+
+
 @active_if(options.pdf_renderer == 'tesseract')
 @collate(
-    input=[preprocess_clean, split_pages],
-    filter=regex(r".*/(\d{6})(?:\.pp-clean\.png|\.page\.pdf)"),
+    input=[select_image_for_pdf, orient_page],
+    filter=regex(r".*/(\d{6})(?:\.image|\.ocr\.oriented\.pdf)"),
     output=os.path.join(work_folder, r'\1.rendered.pdf'),
     extras=[_log, _pdfinfo, _pdfinfo_lock])
 def tesseract_ocr_and_render_pdf(
@@ -689,40 +848,28 @@ def tesseract_ocr_and_render_pdf(
         pdfinfo,
         pdfinfo_lock):
 
-    input_image = next((ii for ii in input_files if ii.endswith('.png')), '')
+    input_image = next((ii for ii in input_files if ii.endswith('.image')), '')
     input_pdf = next((ii for ii in input_files if ii.endswith('.pdf')))
     if not input_image:
         # Skipping this page
         re_symlink(input_pdf, output_file)
         return
 
-    args_tesseract = [
-        'tesseract',
-        '-l', '+'.join(options.language),
-        input_image,
-        os.path.splitext(output_file)[0],  # Tesseract appends suffix
-        'pdf'
-    ] + options.tesseract_config
-    p = Popen(args_tesseract, close_fds=True, stdout=PIPE, stderr=PIPE,
-              universal_newlines=True)
-
-    try:
-        stdout, stderr = p.communicate(timeout=options.tesseract_timeout)
-        if stdout:
-            log.info(stdout)
-        if stderr:
-            log.error(stderr)
-    except TimeoutExpired:
-        p.kill()
-        log.info("Tesseract - page timed out")
-        re_symlink(input_pdf, output_file)
+    tesseract.generate_pdf(
+        input_image=input_image,
+        skip_pdf=input_pdf,
+        output_pdf=output_file,
+        language=options.language,
+        tessconfig=options.tesseract_config,
+        timeout=options.tesseract_timeout,
+        pagesegmode=options.tesseract_pagesegmode,
+        log=log)
 
 
 @transform(
     input=repair_pdf,
-    filter=suffix('.repaired.pdf'),
-    output='.pdfa_def.ps',
-    output_dir=work_folder,
+    filter=formatter(r'\.repaired\.pdf'),
+    output=os.path.join(work_folder, 'pdfa_def.ps'),
     extras=[_log])
 def generate_postscript_stub(
         input_file,
@@ -765,8 +912,8 @@ def generate_postscript_stub(
 
 
 @transform(
-    input=split_pages,
-    filter=suffix('.skip.page.pdf'),
+    input=orient_page,
+    filter=suffix('.skip.oriented.pdf'),
     output='.done.pdf',
     output_dir=work_folder,
     extras=[_log])
@@ -774,11 +921,15 @@ def skip_page(
         input_file,
         output_file,
         log):
+    # The purpose of this step is its filter to forward only the skipped
+    # files (.skip.oriented.pdf) while disregarding the processed ones
+    # (.ocr.oriented.pdf).  Alternative would be for merge_pages to filter
+    # pages itself if it gets multiple copies of a page.
     re_symlink(input_file, output_file, log)
 
 
 @merge(
-    input=[render_hocr_page, render_hocr_debug_page, skip_page,
+    input=[add_text_layer, render_hocr_debug_page, skip_page,
            tesseract_ocr_and_render_pdf, generate_postscript_stub],
     output=os.path.join(work_folder, 'merged.pdf'),
     extras=[_log, _pdfinfo, _pdfinfo_lock])
@@ -803,7 +954,7 @@ def merge_pages(
         return key
 
     pdf_pages = sorted(input_files, key=input_file_order)
-    log.info(pdf_pages)
+    log.debug("Final pages: " + "\n".join(pdf_pages))
     ghostscript.generate_pdfa(pdf_pages, output_file, options.jobs or 1)
 
 
@@ -824,46 +975,7 @@ def copy_final(
 def validate_pdfa(
         input_file,
         log):
-
-    args_qpdf = [
-        'qpdf',
-        '--check',
-        input_file
-    ]
-
-    try:
-        check_output(args_qpdf, stderr=STDOUT, universal_newlines=True)
-    except CalledProcessError as e:
-        if e.returncode == 2:
-            print("{0}: not a valid PDF, and could not repair it.".format(
-                    options.input_file))
-            print("Details:")
-            print(e.output)
-        elif e.returncode == 3:
-            log.info("qpdf --check returned warnings:")
-            log.info(e.output)
-        else:
-            print(e.output)
-        return False
-
-    return True
-
-
-# @active_if(ocr_required and options.exact_image)
-# @merge([render_hocr_blank_page, extract_single_page],
-#        os.path.join(work_folder, "%04i.merged.pdf") % pageno)
-# def merge_hocr_with_original_page(infiles, output_file):
-#     with open(infiles[0], 'rb') as hocr_input, \
-#             open(infiles[1], 'rb') as page_input, \
-#             open(output_file, 'wb') as output:
-#         hocr_reader = pypdf.PdfFileReader(hocr_input)
-#         page_reader = pypdf.PdfFileReader(page_input)
-#         writer = pypdf.PdfFileWriter()
-
-#         the_page = hocr_reader.getPage(0)
-#         the_page.mergePage(page_reader.getPage(0))
-#         writer.addPage(the_page)
-#         writer.write(output)
+    return qpdf.check(input_file, log)
 
 
 def available_cpu_count():
@@ -892,9 +1004,10 @@ def cleanup_ruffus_error_message(msg):
 
 
 def run_pipeline():
-    if not options.jobs or options.jobs == 1:
+    if not options.jobs:
         options.jobs = available_cpu_count()
     try:
+        options.history_file = os.path.join(work_folder, 'ruffus_history.sqlite')
         cmdline.run(options)
     except ruffus_exceptions.RethrownJobError as e:
         if options.verbose:

ocrmypdf/pageinfo.py

@@ -126,18 +126,18 @@ def _page_has_text(pdf, page):
     return False
 
 
-def _pdf_get_pageinfo(infile, page: int):
+def _pdf_get_pageinfo(infile, pageno: int):
     pageinfo = {}
-    pageinfo['pageno'] = page
+    pageinfo['pageno'] = pageno
     pageinfo['images'] = []
 
     pdf = pypdf.PdfFileReader(infile)
-    page = pdf.pages[page - 1]
+    page = pdf.pages[pageno]
 
     pageinfo['has_text'] = _page_has_text(pdf, page)
 
-    width_pt = page['/MediaBox'][2] - page['/MediaBox'][0]
-    height_pt = page['/MediaBox'][3] - page['/MediaBox'][1]
+    width_pt = page.mediaBox.getWidth()
+    height_pt = page.mediaBox.getHeight()
     pageinfo['width_inches'] = width_pt / Decimal(72.0)
     pageinfo['height_inches'] = height_pt / Decimal(72.0)
 

ocrmypdf/pdfa.py

@@ -8,6 +8,7 @@ from string import Template
 from subprocess import Popen, PIPE
 import os
 import codecs
+from . import get_program
 
 
 # This is a template written in PostScript which is needed to create PDF/A
@@ -95,7 +96,8 @@ def _get_pdfa_def(icc_profile, icc_identifier, pdfmark):
 def _get_postscript_icc_path():
     "Parse Ghostscript's help message to find where iccprofiles are stored"
 
-    p_gs = Popen(['gs', '--help'], close_fds=True, universal_newlines=True,
+    p_gs = Popen([get_program('gs'), '--help'], close_fds=True,
+                 universal_newlines=True,
                  stdout=PIPE, stderr=PIPE)
     out, _ = p_gs.communicate()
     lines = out.splitlines()

ocrmypdf/qpdf.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+# © 2015 James R. Barlow: github.com/jbarlow83
+
+from subprocess import CalledProcessError, check_output, STDOUT, check_call
+import sys
+import os
+
+from . import ExitCode, get_program
+
+
+def check(input_file, log):
+    args_qpdf = [
+        get_program('qpdf'),
+        '--check',
+        input_file
+    ]
+
+    try:
+        check_output(args_qpdf, stderr=STDOUT, universal_newlines=True)
+    except CalledProcessError as e:
+        if e.returncode == 2:
+            log.error("{0}: not a valid PDF, and could not repair it.".format(
+                    input_file))
+            log.error("Details:")
+            log.error(e.output)
+        elif e.returncode == 3:
+            log.info("qpdf --check returned warnings:")
+            log.info(e.output)
+        else:
+            log.warning(e.output)
+        return False
+    return True
+
+
+def repair(input_file, output_file, log):
+    args_qpdf = [
+        get_program('qpdf'), input_file, output_file
+    ]
+    try:
+        check_output(args_qpdf, stderr=STDOUT, universal_newlines=True)
+    except CalledProcessError as e:
+        if e.returncode == 3 and e.output.find("operation succeeded"):
+            log.debug('qpdf found and fixed errors: ' + e.output)
+            log.debug(e.output)
+            return
+
+        if e.returncode == 2 and e.output.find("invalid password"):
+            log.error("{0}: this PDF is password-protected - password must "
+                      "be removed for OCR".format(input_file))
+            sys.exit(ExitCode.input_file)
+        elif e.returncode == 2:
+            log.error("{0}: not a valid PDF, and could not repair it.".format(
+                      input_file))
+            log.error("Details: " + e.output)
+            sys.exit(ExitCode.input_file)
+        else:
+            log.error("{0}: unknown error".format(
+                      input_file))
+            log.error(e.output)
+            sys.exit(ExitCode.unknown)
+
+
+def get_npages(input_file, log):
+    try:
+        pages = check_output(
+            [get_program('qpdf'), '--show-npages', input_file],
+             universal_newlines=True, close_fds=True)
+    except CalledProcessError as e:
+        if e.returncode == 2 and e.output.find('No such file'):
+            log.error(e.output)
+            sys.exit(ExitCode.input_file)
+    return int(pages)
+
+
+def split_pages(input_file, work_folder, npages):
+    """Split multipage PDF into individual pages.
+
+    Incredibly enough, this multiple process approach is about 70 times
+    faster than using Ghostscript.
+    """
+    for n in range(int(npages)):
+        args_qpdf = [
+            get_program('qpdf'), input_file,
+            '--pages', input_file, '{0}'.format(n + 1), '--',
+            os.path.join(work_folder, '{0:06d}.page.pdf'.format(n + 1))
+        ]
+        check_call(args_qpdf)

ocrmypdf/tesseract.py

@@ -1,50 +1,25 @@
 #!/usr/bin/env python3
 # © 2015 James R. Barlow: github.com/jbarlow83
 
-from subprocess import STDOUT, CalledProcessError, check_output
 import sys
 import os
 import re
+import shutil
 from functools import lru_cache
-from . import ExitCode
+from . import ExitCode, get_program, page_number
+from collections import namedtuple
+
+from subprocess import Popen, PIPE, CalledProcessError, \
+    TimeoutExpired, check_output, STDOUT
+try:
+    from subprocess import DEVNULL
+except ImportError:
+    DEVNULL = open(os.devnull, 'wb')
 
 
-@lru_cache(maxsize=1)
-def version():
-    args_tess = [
-        'tesseract',
-        '--version'
-    ]
-    try:
-        versions = check_output(
-                args_tess, close_fds=True, universal_newlines=True,
-                stderr=STDOUT)
-    except CalledProcessError:
-        print("Could not find Tesseract executable on system PATH.")
-        sys.exit(ExitCode.missing_dependency)
-
-    tesseract_version = re.match(r'tesseract\s(.+)', versions).group(1)
-    return tesseract_version
-
-
-@lru_cache(maxsize=1)
-def languages():
-    args_tess = [
-        'tesseract',
-        '--list-langs'
-    ]
-    try:
-        langs = check_output(
-                args_tess, close_fds=True, universal_newlines=True,
-                stderr=STDOUT)
-    except CalledProcessError as e:
-        print("Tesseract failed to report available languages.")
-        print("Output from Tesseract:")
-        print("-" * 40)
-        print(e.output)
-        sys.exit(ExitCode.missing_dependency)
-    return set(lang.strip() for lang in langs.splitlines()[1:])
-
+OrientationConfidence = namedtuple(
+    'OrientationConfidence',
+    ('angle', 'confidence'))
 
 HOCR_TEMPLATE = '''<?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
@@ -67,3 +42,185 @@ HOCR_TEMPLATE = '''<?xml version="1.0" encoding="UTF-8"?>
   </div>
  </body>
 </html>'''
+
+
+@lru_cache(maxsize=1)
+def version():
+    args_tess = [
+        get_program('tesseract'),
+        '--version'
+    ]
+    try:
+        versions = check_output(
+                args_tess, close_fds=True, universal_newlines=True,
+                stderr=STDOUT)
+    except CalledProcessError:
+        print("Could not find Tesseract executable on system PATH.")
+        sys.exit(ExitCode.missing_dependency)
+
+    tesseract_version = re.match(r'tesseract\s(.+)', versions).group(1)
+    return tesseract_version
+
+
+@lru_cache(maxsize=1)
+def languages():
+    args_tess = [
+        get_program('tesseract'),
+        '--list-langs'
+    ]
+    try:
+        langs = check_output(
+                args_tess, close_fds=True, universal_newlines=True,
+                stderr=STDOUT)
+    except CalledProcessError as e:
+        print("Tesseract failed to report available languages.")
+        print("Output from Tesseract:")
+        print("-" * 40)
+        print(e.output)
+        sys.exit(ExitCode.missing_dependency)
+    return set(lang.strip() for lang in langs.splitlines()[1:])
+
+
+def get_orientation(input_file, language: list, timeout: float, log):
+    args_tesseract = [
+        get_program('tesseract'),
+        '-l', '+'.join(language),
+        '-psm', '0',
+        input_file,
+        'stdout'
+    ]
+
+    p = Popen(args_tesseract, close_fds=True, stdout=PIPE, stderr=STDOUT,
+              universal_newlines=True)
+    try:
+        stdout, _ = p.communicate(timeout=timeout)
+    except TimeoutExpired:
+        p.kill()
+        stdout, _ = p.communicate()
+        return OrientationConfidence(angle=0, confidence=0.0)
+    else:
+        osd = {}
+        for line in stdout.splitlines():
+            line = line.strip()
+            parts = line.split(':', maxsplit=2)
+            if len(parts) == 2:
+                osd[parts[0].strip()] = parts[1].strip()
+
+        oc = OrientationConfidence(
+            angle=int(osd['Orientation in degrees']),
+            confidence=float(osd['Orientation confidence']))
+        return oc
+
+
+def tesseract_log_output(log, stdout, input_file):
+    lines = stdout.splitlines()
+    prefix = "{0:4d}: [tesseract] ".format(page_number(input_file))
+    for line in lines:
+        if line.startswith("Tesseract Open Source"):
+            continue
+        elif line.startswith("Warning in pixReadMem"):
+            continue
+        elif 'diacritics' in line:
+            log.warning(prefix + "lots of diacritics - possibly poor OCR")
+        elif line.startswith('OSD: Weak margin'):
+            log.warning(prefix + "unsure about page orientation")
+        else:
+            log.info(prefix + line.strip())
+
+
+def generate_hocr(input_file, output_hocr, language: list, tessconfig: list,
+                  timeout: float, pageinfo_getter, pagesegmode: int, log):
+
+    badxml = os.path.splitext(output_hocr)[0] + '.badxml'
+
+    args_tesseract = [
+        get_program('tesseract'),
+        '-l', '+'.join(language)
+    ]
+
+    if pagesegmode is not None:
+        args_tesseract.extend(['-psm', str(pagesegmode)])
+
+    args_tesseract.extend([
+        input_file,
+        badxml,
+        'hocr'
+    ] + tessconfig)
+    p = Popen(args_tesseract, close_fds=True, stdout=PIPE, stderr=STDOUT,
+              universal_newlines=True)
+    try:
+        stdout, _ = p.communicate(timeout=timeout)
+    except TimeoutExpired:
+        p.kill()
+        stdout, _ = p.communicate()
+        # Generate a HOCR file with no recognized text if tesseract times out
+        # Temporary workaround to hocrTransform not being able to function if
+        # it does not have a valid hOCR file.
+        with open(output_hocr, 'w', encoding="utf-8") as f:
+            pageinfo = pageinfo_getter()
+            f.write(HOCR_TEMPLATE.format(
+                pageinfo['width_pixels'],
+                pageinfo['height_pixels']))
+    else:
+        tesseract_log_output(log, stdout, input_file)
+        if p.returncode != 0:
+            raise CalledProcessError(p.returncode, args_tesseract)
+
+        if os.path.exists(badxml + '.html'):
+            # Tesseract 3.02 appends suffix ".html" on its own (.badxml.html)
+            shutil.move(badxml + '.html', badxml)
+        elif os.path.exists(badxml + '.hocr'):
+            # Tesseract 3.03 appends suffix ".hocr" on its own (.badxml.hocr)
+            shutil.move(badxml + '.hocr', badxml)
+
+        # Tesseract 3.03 inserts source filename into hocr file without
+        # escaping it, creating invalid XML and breaking the parser.
+        # As a workaround, rewrite the hocr file, replacing the filename
+        # with a space.  Don't know if Tesseract 3.02 does the same.
+
+        regex_nested_single_quotes = re.compile(
+            r"""title='image "([^"]*)";""")
+        with open(badxml, mode='r', encoding='utf-8') as f_in, \
+                open(output_hocr, mode='w', encoding='utf-8') as f_out:
+            for line in f_in:
+                line = regex_nested_single_quotes.sub(
+                    r"""title='image " ";""", line)
+                f_out.write(line)
+
+
+def generate_pdf(input_image, skip_pdf, output_pdf, language: list,
+                 tessconfig: list, timeout: float, pagesegmode: int, log):
+    '''Use Tesseract to render a PDF.
+
+    input_image -- image to analyze
+    skip_pdf -- if we time out, use this file as output
+    language -- list of languages to consider
+    tessconfig -- tesseract configuration
+    timeout -- timeout (seconds)
+    log -- logger object
+    '''
+
+    args_tesseract = [
+        get_program('tesseract'),
+        '-l', '+'.join(language)
+    ]
+
+    if pagesegmode is not None:
+        args_tesseract.extend(['-psm', str(pagesegmode)])
+
+    args_tesseract.extend([
+        input_image,
+        os.path.splitext(output_pdf)[0],  # Tesseract appends suffix
+        'pdf'
+    ] + tessconfig)
+    p = Popen(args_tesseract, close_fds=True, stdout=PIPE, stderr=STDOUT,
+              universal_newlines=True)
+
+    try:
+        stdout, _ = p.communicate()
+    except TimeoutExpired:
+        p.kill()
+        log.info("Tesseract - page timed out")
+        shutil.copy(skip_pdf, output_pdf)
+    else:
+        tesseract_log_output(log, stdout, input_image)

ocrmypdf/unpaper.py

@@ -8,13 +8,13 @@ from tempfile import NamedTemporaryFile
 import sys
 import os
 from functools import lru_cache
-from . import ExitCode
+from . import ExitCode, get_program
 
 
 @lru_cache(maxsize=1)
 def version():
     args_unpaper = [
-        'unpaper',
+        get_program('unpaper'),
         '--version'
     ]
     p_unpaper = Popen(args_unpaper, close_fds=True, universal_newlines=True,
@@ -33,7 +33,7 @@ except ImportError:
 
 def run(input_file, output_file, dpi, log, mode_args):
     args_unpaper = [
-        'unpaper',
+        get_program('unpaper'),
         '-v',
         '--dpi', str(dpi)
     ] + mode_args

pipeline.svg

@@ -4,232 +4,294 @@
 <!-- Generated by graphviz version 2.38.0 (20140413.2041)
  -->
 <!-- Title: Pipeline: Pages: 1 -->
-<svg width="728pt" height="651pt"
- viewBox="0.00 0.00 728.00 650.53" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
-<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 646.53)">
+<svg width="1452pt" height="824pt"
+ viewBox="0.00 0.00 1452.00 824.08" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 820.083)">
 <title>Pipeline:</title>
-<polygon fill="white" stroke="none" points="-4,4 -4,-646.53 724,-646.53 724,4 -4,4"/>
+<polygon fill="white" stroke="none" points="-4,4 -4,-820.083 1448,-820.083 1448,4 -4,4"/>
 <g id="clust1" class="cluster"><title>clustertasks</title>
-<polygon fill="none" stroke="black" points="8,-8 8,-634.53 712,-634.53 712,-8 8,-8"/>
-<text text-anchor="middle" x="360" y="-606.53" font-family="Times,serif" font-size="30.00" fill="#ff3232">Pipeline:</text>
+<polygon fill="none" stroke="black" points="8,-8 8,-808.083 1436,-808.083 1436,-8 8,-8"/>
+<text text-anchor="middle" x="722" y="-780.083" font-family="Times,serif" font-size="30.00" fill="#ff3232">Pipeline:</text>
 </g>
 <!-- t0 -->
 <g id="node1" class="node"><title>t0</title>
-<polygon fill="#efa03b" stroke="#006000" points="481.791,-588.53 386.209,-588.53 382.209,-584.53 382.209,-552.53 477.791,-552.53 481.791,-556.53 481.791,-588.53"/>
-<polyline fill="none" stroke="#006000" points="477.791,-584.53 382.209,-584.53 "/>
-<polyline fill="none" stroke="#006000" points="477.791,-584.53 477.791,-552.53 "/>
-<polyline fill="none" stroke="#006000" points="477.791,-584.53 481.791,-588.53 "/>
-<text text-anchor="middle" x="432" y="-564.53" font-family="Times,serif" font-size="20.00" fill="#006000">repair_pdf</text>
+<polygon fill="#efa03b" stroke="black" points="1256.54,-762.083 1033.46,-762.083 1029.46,-758.083 1029.46,-726.083 1252.54,-726.083 1256.54,-730.083 1256.54,-762.083"/>
+<polyline fill="none" stroke="black" points="1252.54,-758.083 1029.46,-758.083 "/>
+<polyline fill="none" stroke="black" points="1252.54,-758.083 1252.54,-726.083 "/>
+<polyline fill="none" stroke="black" points="1252.54,-758.083 1256.54,-762.083 "/>
+<text text-anchor="middle" x="1143" y="-738.083" font-family="Times,serif" font-size="20.00">repair_pdf</text>
 </g>
 <!-- t1 -->
 <g id="node2" class="node"><title>t1</title>
-<polygon fill="#efa03b" stroke="black" points="466.782,-509.564 374,-526.497 281.218,-509.564 281.304,-482.165 466.696,-482.165 466.782,-509.564"/>
-<polygon fill="none" stroke="black" points="470.799,-512.902 374,-530.569 277.201,-512.902 277.311,-478.159 470.689,-478.159 470.799,-512.902"/>
-<text text-anchor="middle" x="374" y="-495.991" font-family="Times,serif" font-size="20.00">split_pages</text>
+<polygon fill="#efa03b" stroke="black" points="1234.11,-683.155 1030,-700.057 825.888,-683.155 826.078,-655.806 1233.92,-655.806 1234.11,-683.155"/>
+<polygon fill="none" stroke="black" points="1238.13,-686.834 1030,-704.069 821.866,-686.834 822.11,-651.808 1237.89,-651.808 1238.13,-686.834"/>
+<text text-anchor="middle" x="1030" y="-669.596" font-family="Times,serif" font-size="20.00">split_pages</text>
 </g>
 <!-- t0&#45;&gt;t1 -->
 <g id="edge1" class="edge"><title>t0&#45;&gt;t1</title>
-<path fill="none" stroke="gray" d="M417.064,-552.394C412.296,-546.925 406.86,-540.689 401.493,-534.532"/>
-<polygon fill="gray" stroke="gray" points="403.996,-532.077 394.787,-526.838 398.719,-536.676 403.996,-532.077"/>
+<path fill="none" stroke="#0044a0" d="M1113.9,-725.961C1103.58,-719.89 1091.66,-712.873 1080.09,-706.069"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1081.75,-702.982 1071.35,-700.927 1078.2,-709.015 1081.75,-702.982"/>
 </g>
-<!-- t10 -->
-<g id="node12" class="node"><title>t10</title>
-<polygon fill="#efa03b" stroke="#006000" points="704.338,-451.452 493.662,-451.452 489.662,-447.452 489.662,-415.452 700.338,-415.452 704.338,-419.452 704.338,-451.452"/>
-<polyline fill="none" stroke="#006000" points="700.338,-447.452 489.662,-447.452 "/>
-<polyline fill="none" stroke="#006000" points="700.338,-447.452 700.338,-415.452 "/>
-<polyline fill="none" stroke="#006000" points="700.338,-447.452 704.338,-451.452 "/>
-<text text-anchor="middle" x="597" y="-427.452" font-family="Times,serif" font-size="20.00" fill="#006000">generate_postscript_stub</text>
+<!-- t14 -->
+<g id="node16" class="node"><title>t14</title>
+<polygon fill="#efa03b" stroke="black" points="1428.08,-625.109 1089.92,-625.109 1085.92,-621.109 1085.92,-589.109 1424.08,-589.109 1428.08,-593.109 1428.08,-625.109"/>
+<polyline fill="none" stroke="black" points="1424.08,-621.109 1085.92,-621.109 "/>
+<polyline fill="none" stroke="black" points="1424.08,-621.109 1424.08,-589.109 "/>
+<polyline fill="none" stroke="black" points="1424.08,-621.109 1428.08,-625.109 "/>
+<text text-anchor="middle" x="1257" y="-601.109" font-family="Times,serif" font-size="20.00">generate_postscript_stub</text>
 </g>
-<!-- t0&#45;&gt;t10 -->
-<g id="edge16" class="edge"><title>t0&#45;&gt;t10</title>
-<path fill="none" stroke="gray" d="M453.048,-552.468C461.441,-545.654 471.185,-537.73 480,-530.53 510.227,-505.84 544.753,-477.467 568.421,-457.99"/>
-<polygon fill="gray" stroke="gray" points="570.818,-460.55 576.315,-451.493 566.369,-455.146 570.818,-460.55"/>
+<!-- t0&#45;&gt;t14 -->
+<g id="edge22" class="edge"><title>t0&#45;&gt;t14</title>
+<path fill="none" stroke="#0044a0" d="M1219.32,-726.004C1229.98,-720.592 1239.75,-713.466 1247,-704.083 1261.92,-684.78 1263.1,-656.385 1261.35,-635.486"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1264.8,-634.881 1260.22,-625.328 1257.84,-635.653 1264.8,-634.881"/>
 </g>
 <!-- t2 -->
 <g id="node3" class="node"><title>t2</title>
-<polygon fill="#efa03b" stroke="black" points="451.555,-451.452 228.445,-451.452 224.445,-447.452 224.445,-415.452 447.555,-415.452 451.555,-419.452 451.555,-451.452"/>
-<polyline fill="none" stroke="black" points="447.555,-447.452 224.445,-447.452 "/>
-<polyline fill="none" stroke="black" points="447.555,-447.452 447.555,-415.452 "/>
-<polyline fill="none" stroke="black" points="447.555,-447.452 451.555,-451.452 "/>
-<text text-anchor="middle" x="338" y="-427.452" font-family="Times,serif" font-size="20.00">rasterize_with_ghostscript</text>
+<polygon fill="#efa03b" stroke="black" points="1029.88,-625.109 748.117,-625.109 744.117,-621.109 744.117,-589.109 1025.88,-589.109 1029.88,-593.109 1029.88,-625.109"/>
+<polyline fill="none" stroke="black" points="1025.88,-621.109 744.117,-621.109 "/>
+<polyline fill="none" stroke="black" points="1025.88,-621.109 1025.88,-589.109 "/>
+<polyline fill="none" stroke="black" points="1025.88,-621.109 1029.88,-625.109 "/>
+<text text-anchor="middle" x="887" y="-601.109" font-family="Times,serif" font-size="20.00">rasterize_preview</text>
 </g>
 <!-- t1&#45;&gt;t2 -->
 <g id="edge2" class="edge"><title>t1&#45;&gt;t2</title>
-<path fill="none" stroke="#0044a0" d="M361.611,-478.092C358.514,-472.369 355.171,-466.19 352.003,-460.333"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="355.064,-458.636 347.227,-451.506 348.907,-461.967 355.064,-458.636"/>
-</g>
-<!-- t11 -->
-<g id="node10" class="node"><title>t11</title>
-<polygon fill="#efa03b" stroke="black" points="644.594,-393.452 551.406,-393.452 547.406,-389.452 547.406,-357.452 640.594,-357.452 644.594,-361.452 644.594,-393.452"/>
-<polyline fill="none" stroke="black" points="640.594,-389.452 547.406,-389.452 "/>
-<polyline fill="none" stroke="black" points="640.594,-389.452 640.594,-357.452 "/>
-<polyline fill="none" stroke="black" points="640.594,-389.452 644.594,-393.452 "/>
-<text text-anchor="middle" x="596" y="-369.452" font-family="Times,serif" font-size="20.00">skip_page</text>
-</g>
-<!-- t1&#45;&gt;t11 -->
-<g id="edge13" class="edge"><title>t1&#45;&gt;t11</title>
-<path fill="none" stroke="#0044a0" d="M425.139,-478.007C437.834,-470.731 450.731,-461.83 461,-451.452 473.874,-438.442 466.946,-427.178 481,-415.452 490.212,-407.766 513.965,-399.205 537.508,-392.067"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="538.592,-395.396 547.189,-389.203 536.607,-388.683 538.592,-395.396"/>
-</g>
-<!-- t9 -->
-<g id="node11" class="node"><title>t9</title>
-<polygon fill="#efa03b" stroke="black" points="272.496,-277.452 19.5039,-277.452 15.5039,-273.452 15.5039,-241.452 268.496,-241.452 272.496,-245.452 272.496,-277.452"/>
-<polyline fill="none" stroke="black" points="268.496,-273.452 15.5039,-273.452 "/>
-<polyline fill="none" stroke="black" points="268.496,-273.452 268.496,-241.452 "/>
-<polyline fill="none" stroke="black" points="268.496,-273.452 272.496,-277.452 "/>
-<text text-anchor="middle" x="144" y="-253.452" font-family="Times,serif" font-size="20.00">tesseract_ocr_and_render_pdf</text>
-</g>
-<!-- t1&#45;&gt;t9 -->
-<g id="edge15" class="edge"><title>t1&#45;&gt;t9</title>
-<path fill="none" stroke="#0044a0" d="M277.341,-486.044C254.708,-478.702 232.21,-467.759 215,-451.452 168.207,-407.114 152.056,-329.127 146.633,-287.855"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="150.084,-287.235 145.422,-277.721 143.133,-288.065 150.084,-287.235"/>
+<path fill="none" stroke="#0044a0" d="M980.788,-651.715C965.266,-644.498 948.187,-636.558 932.905,-629.452"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="934.195,-626.193 923.652,-625.15 931.244,-632.54 934.195,-626.193"/>
 </g>
 <!-- t3 -->
 <g id="node4" class="node"><title>t3</title>
-<polygon fill="#efa03b" stroke="black" points="458.999,-393.452 291.001,-393.452 287.001,-389.452 287.001,-357.452 454.999,-357.452 458.999,-361.452 458.999,-393.452"/>
-<polyline fill="none" stroke="black" points="454.999,-389.452 287.001,-389.452 "/>
-<polyline fill="none" stroke="black" points="454.999,-389.452 454.999,-357.452 "/>
-<polyline fill="none" stroke="black" points="454.999,-389.452 458.999,-393.452 "/>
-<text text-anchor="middle" x="373" y="-369.452" font-family="Times,serif" font-size="20.00">preprocess_deskew</text>
+<polygon fill="#efa03b" stroke="black" points="1128.15,-567.109 893.852,-567.109 889.852,-563.109 889.852,-531.109 1124.15,-531.109 1128.15,-535.109 1128.15,-567.109"/>
+<polyline fill="none" stroke="black" points="1124.15,-563.109 889.852,-563.109 "/>
+<polyline fill="none" stroke="black" points="1124.15,-563.109 1124.15,-531.109 "/>
+<polyline fill="none" stroke="black" points="1124.15,-563.109 1128.15,-567.109 "/>
+<text text-anchor="middle" x="1009" y="-543.109" font-family="Times,serif" font-size="20.00">orient_page</text>
+</g>
+<!-- t1&#45;&gt;t3 -->
+<g id="edge4" class="edge"><title>t1&#45;&gt;t3</title>
+<path fill="none" stroke="#0044a0" d="M1037.63,-651.508C1042.21,-633.96 1045.92,-609.455 1039,-589.109 1037.4,-584.404 1034.98,-579.822 1032.19,-575.552"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1034.85,-573.258 1026.1,-567.282 1029.21,-577.409 1034.85,-573.258"/>
 </g>
 <!-- t2&#45;&gt;t3 -->
 <g id="edge3" class="edge"><title>t2&#45;&gt;t3</title>
-<path fill="none" stroke="#0044a0" d="M348.691,-415.346C351.337,-411.112 354.23,-406.485 357.066,-401.946"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="360.042,-403.788 362.374,-393.453 354.106,-400.078 360.042,-403.788"/>
-</g>
-<!-- t6 -->
-<g id="node6" class="node"><title>t6</title>
-<polygon fill="#efa03b" stroke="black" points="664.375,-277.452 477.625,-277.452 473.625,-273.452 473.625,-241.452 660.375,-241.452 664.375,-245.452 664.375,-277.452"/>
-<polyline fill="none" stroke="black" points="660.375,-273.452 473.625,-273.452 "/>
-<polyline fill="none" stroke="black" points="660.375,-273.452 660.375,-241.452 "/>
-<polyline fill="none" stroke="black" points="660.375,-273.452 664.375,-277.452 "/>
-<text text-anchor="middle" x="569" y="-253.452" font-family="Times,serif" font-size="20.00">select_image_for_pdf</text>
-</g>
-<!-- t2&#45;&gt;t6 -->
-<g id="edge7" class="edge"><title>t2&#45;&gt;t6</title>
-<path fill="none" stroke="#0044a0" d="M423.032,-415.374C438.812,-409.925 454.543,-402.785 468,-393.452 508.201,-365.571 539.287,-316.662 555.808,-286.558"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="559.024,-287.965 560.653,-277.496 552.851,-284.664 559.024,-287.965"/>
+<path fill="none" stroke="#0044a0" d="M924.267,-589.003C936.328,-583.467 949.857,-577.257 962.502,-571.453"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="964.333,-574.463 971.961,-567.111 961.413,-568.102 964.333,-574.463"/>
 </g>
 <!-- t4 -->
 <g id="node5" class="node"><title>t4</title>
-<polygon fill="#efa03b" stroke="black" points="449.705,-335.452 300.295,-335.452 296.295,-331.452 296.295,-299.452 445.705,-299.452 449.705,-303.452 449.705,-335.452"/>
-<polyline fill="none" stroke="black" points="445.705,-331.452 296.295,-331.452 "/>
-<polyline fill="none" stroke="black" points="445.705,-331.452 445.705,-299.452 "/>
-<polyline fill="none" stroke="black" points="445.705,-331.452 449.705,-335.452 "/>
-<text text-anchor="middle" x="373" y="-311.452" font-family="Times,serif" font-size="20.00">preprocess_clean</text>
+<polygon fill="#efa03b" stroke="black" points="1027.3,-509.109 676.701,-509.109 672.701,-505.109 672.701,-473.109 1023.3,-473.109 1027.3,-477.109 1027.3,-509.109"/>
+<polyline fill="none" stroke="black" points="1023.3,-505.109 672.701,-505.109 "/>
+<polyline fill="none" stroke="black" points="1023.3,-505.109 1023.3,-473.109 "/>
+<polyline fill="none" stroke="black" points="1023.3,-505.109 1027.3,-509.109 "/>
+<text text-anchor="middle" x="850" y="-485.109" font-family="Times,serif" font-size="20.00">rasterize_with_ghostscript</text>
 </g>
 <!-- t3&#45;&gt;t4 -->
-<g id="edge4" class="edge"><title>t3&#45;&gt;t4</title>
-<path fill="none" stroke="#0044a0" d="M373,-357.346C373,-353.655 373,-349.665 373,-345.695"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="376.5,-345.453 373,-335.453 369.5,-345.453 376.5,-345.453"/>
+<g id="edge5" class="edge"><title>t3&#45;&gt;t4</title>
+<path fill="none" stroke="#0044a0" d="M960.431,-531.003C943.787,-525.141 924.999,-518.524 907.704,-512.433"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="908.867,-509.132 898.272,-509.111 906.541,-515.734 908.867,-509.132"/>
 </g>
-<!-- t3&#45;&gt;t6 -->
-<g id="edge6" class="edge"><title>t3&#45;&gt;t6</title>
-<path fill="none" stroke="#0044a0" d="M414.399,-357.412C428.785,-351.039 444.852,-343.405 459,-335.452 466.909,-331.007 506.49,-303.805 535.921,-283.431"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="538.073,-286.198 544.299,-277.625 534.086,-280.444 538.073,-286.198"/>
+<!-- t9 -->
+<g id="node11" class="node"><title>t9</title>
+<polygon fill="#efa03b" stroke="black" points="314.109,-277.109 19.8906,-277.109 15.8906,-273.109 15.8906,-241.109 310.109,-241.109 314.109,-245.109 314.109,-277.109"/>
+<polyline fill="none" stroke="black" points="310.109,-273.109 15.8906,-273.109 "/>
+<polyline fill="none" stroke="black" points="310.109,-273.109 310.109,-241.109 "/>
+<polyline fill="none" stroke="black" points="310.109,-273.109 314.109,-277.109 "/>
+<text text-anchor="middle" x="165" y="-253.109" font-family="Times,serif" font-size="20.00">select_image_layer</text>
 </g>
-<!-- t4&#45;&gt;t6 -->
-<g id="edge5" class="edge"><title>t4&#45;&gt;t6</title>
-<path fill="none" stroke="#0044a0" d="M432.605,-299.422C453.701,-293.395 477.617,-286.561 499.465,-280.319"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="500.687,-283.61 509.341,-277.497 498.764,-276.879 500.687,-283.61"/>
+<!-- t3&#45;&gt;t9 -->
+<g id="edge14" class="edge"><title>t3&#45;&gt;t9</title>
+<path fill="none" stroke="#0044a0" d="M889.675,-542.297C675.505,-530.255 248,-498.192 248,-434.109 248,-434.109 248,-434.109 248,-374.109 248,-337.597 219.83,-304.936 196.312,-284.031"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="198.437,-281.243 188.562,-277.404 193.888,-286.563 198.437,-281.243"/>
+</g>
+<!-- t15 -->
+<g id="node14" class="node"><title>t15</title>
+<polygon fill="#efa03b" stroke="black" points="1391.34,-451.109 1170.66,-451.109 1166.66,-447.109 1166.66,-415.109 1387.34,-415.109 1391.34,-419.109 1391.34,-451.109"/>
+<polyline fill="none" stroke="black" points="1387.34,-447.109 1166.66,-447.109 "/>
+<polyline fill="none" stroke="black" points="1387.34,-447.109 1387.34,-415.109 "/>
+<polyline fill="none" stroke="black" points="1387.34,-447.109 1391.34,-451.109 "/>
+<text text-anchor="middle" x="1279" y="-427.109" font-family="Times,serif" font-size="20.00">skip_page</text>
+</g>
+<!-- t3&#45;&gt;t15 -->
+<g id="edge19" class="edge"><title>t3&#45;&gt;t15</title>
+<path fill="none" stroke="#0044a0" d="M1060.32,-531.078C1078.95,-524.591 1100.07,-516.881 1119,-509.109 1159.3,-492.568 1204.11,-471.304 1236.04,-455.645"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1237.67,-458.74 1245.1,-451.181 1234.58,-452.461 1237.67,-458.74"/>
+</g>
+<!-- t13 -->
+<g id="node15" class="node"><title>t13</title>
+<polygon fill="#efa03b" stroke="black" points="1388.24,-277.109 1007.76,-277.109 1003.76,-273.109 1003.76,-241.109 1384.24,-241.109 1388.24,-245.109 1388.24,-277.109"/>
+<polyline fill="none" stroke="black" points="1384.24,-273.109 1003.76,-273.109 "/>
+<polyline fill="none" stroke="black" points="1384.24,-273.109 1384.24,-241.109 "/>
+<polyline fill="none" stroke="black" points="1384.24,-273.109 1388.24,-277.109 "/>
+<text text-anchor="middle" x="1196" y="-253.109" font-family="Times,serif" font-size="20.00">tesseract_ocr_and_render_pdf</text>
+</g>
+<!-- t3&#45;&gt;t13 -->
+<g id="edge21" class="edge"><title>t3&#45;&gt;t13</title>
+<path fill="none" stroke="#0044a0" d="M1040.79,-531.094C1071.69,-511.983 1114,-477.834 1114,-434.109 1114,-434.109 1114,-434.109 1114,-374.109 1114,-337.749 1141.83,-305.063 1165.07,-284.11"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1167.46,-286.664 1172.72,-277.466 1162.87,-281.377 1167.46,-286.664"/>
 </g>
 <!-- t5 -->
-<g id="node7" class="node"><title>t5</title>
-<polygon fill="#efa03b" stroke="black" points="455.922,-277.452 294.078,-277.452 290.078,-273.452 290.078,-241.452 451.922,-241.452 455.922,-245.452 455.922,-277.452"/>
-<polyline fill="none" stroke="black" points="451.922,-273.452 290.078,-273.452 "/>
-<polyline fill="none" stroke="black" points="451.922,-273.452 451.922,-241.452 "/>
-<polyline fill="none" stroke="black" points="451.922,-273.452 455.922,-277.452 "/>
-<text text-anchor="middle" x="373" y="-253.452" font-family="Times,serif" font-size="20.00">ocr_tesseract_hocr</text>
+<g id="node6" class="node"><title>t5</title>
+<polygon fill="#efa03b" stroke="black" points="830.742,-451.109 535.258,-451.109 531.258,-447.109 531.258,-415.109 826.742,-415.109 830.742,-419.109 830.742,-451.109"/>
+<polyline fill="none" stroke="black" points="826.742,-447.109 531.258,-447.109 "/>
+<polyline fill="none" stroke="black" points="826.742,-447.109 826.742,-415.109 "/>
+<polyline fill="none" stroke="black" points="826.742,-447.109 830.742,-451.109 "/>
+<text text-anchor="middle" x="681" y="-427.109" font-family="Times,serif" font-size="20.00">preprocess_deskew</text>
 </g>
 <!-- t4&#45;&gt;t5 -->
-<g id="edge8" class="edge"><title>t4&#45;&gt;t5</title>
-<path fill="none" stroke="#0044a0" d="M373,-299.346C373,-295.655 373,-291.665 373,-287.695"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="376.5,-287.453 373,-277.453 369.5,-287.453 376.5,-287.453"/>
+<g id="edge6" class="edge"><title>t4&#45;&gt;t5</title>
+<path fill="none" stroke="#0044a0" d="M798.376,-473.003C780.521,-467.087 760.346,-460.402 741.823,-454.264"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="742.901,-450.934 732.308,-451.111 740.699,-457.579 742.901,-450.934"/>
 </g>
-<!-- t4&#45;&gt;t9 -->
-<g id="edge14" class="edge"><title>t4&#45;&gt;t9</title>
-<path fill="none" stroke="#0044a0" d="M303.36,-299.422C278.157,-293.259 249.509,-286.253 223.521,-279.898"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="224.249,-276.473 213.704,-277.497 222.586,-283.273 224.249,-276.473"/>
+<!-- t8 -->
+<g id="node10" class="node"><title>t8</title>
+<polygon fill="#efa03b" stroke="black" points="980.119,-335.109 665.881,-335.109 661.881,-331.109 661.881,-299.109 976.119,-299.109 980.119,-303.109 980.119,-335.109"/>
+<polyline fill="none" stroke="black" points="976.119,-331.109 661.881,-331.109 "/>
+<polyline fill="none" stroke="black" points="976.119,-331.109 976.119,-299.109 "/>
+<polyline fill="none" stroke="black" points="976.119,-331.109 980.119,-335.109 "/>
+<text text-anchor="middle" x="821" y="-311.109" font-family="Times,serif" font-size="20.00">select_image_for_pdf</text>
+</g>
+<!-- t4&#45;&gt;t8 -->
+<g id="edge12" class="edge"><title>t4&#45;&gt;t8</title>
+<path fill="none" stroke="#0044a0" d="M848.03,-473.087C846.196,-457.857 843.261,-434.941 840,-415.109 836.1,-391.393 830.78,-364.592 826.761,-345.181"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="830.152,-344.296 824.68,-335.224 823.3,-345.728 830.152,-344.296"/>
+</g>
+<!-- t6 -->
+<g id="node7" class="node"><title>t6</title>
+<polygon fill="#efa03b" stroke="black" points="801.95,-393.109 524.05,-393.109 520.05,-389.109 520.05,-357.109 797.95,-357.109 801.95,-361.109 801.95,-393.109"/>
+<polyline fill="none" stroke="black" points="797.95,-389.109 520.05,-389.109 "/>
+<polyline fill="none" stroke="black" points="797.95,-389.109 797.95,-357.109 "/>
+<polyline fill="none" stroke="black" points="797.95,-389.109 801.95,-393.109 "/>
+<text text-anchor="middle" x="661" y="-369.109" font-family="Times,serif" font-size="20.00">preprocess_clean</text>
+</g>
+<!-- t5&#45;&gt;t6 -->
+<g id="edge7" class="edge"><title>t5&#45;&gt;t6</title>
+<path fill="none" stroke="#0044a0" d="M674.891,-415.003C673.495,-411.095 671.979,-406.852 670.48,-402.652"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="673.731,-401.351 667.072,-393.111 667.139,-403.705 673.731,-401.351"/>
+</g>
+<!-- t5&#45;&gt;t8 -->
+<g id="edge11" class="edge"><title>t5&#45;&gt;t8</title>
+<path fill="none" stroke="#0044a0" d="M781.944,-415.046C792.967,-409.579 803.051,-402.434 811,-393.109 822.009,-380.195 824.454,-361.215 824.174,-345.581"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="827.659,-345.2 823.617,-335.407 820.669,-345.583 827.659,-345.2"/>
 </g>
 <!-- t7 -->
 <g id="node8" class="node"><title>t7</title>
-<polygon fill="#efa03b" stroke="black" points="426.366,-219.452 269.634,-219.452 265.634,-215.452 265.634,-183.452 422.366,-183.452 426.366,-187.452 426.366,-219.452"/>
-<polyline fill="none" stroke="black" points="422.366,-215.452 265.634,-215.452 "/>
-<polyline fill="none" stroke="black" points="422.366,-215.452 422.366,-183.452 "/>
-<polyline fill="none" stroke="black" points="422.366,-215.452 426.366,-219.452 "/>
-<text text-anchor="middle" x="346" y="-195.452" font-family="Times,serif" font-size="20.00">render_hocr_page</text>
+<polygon fill="#efa03b" stroke="black" points="643.666,-335.109 354.334,-335.109 350.334,-331.109 350.334,-299.109 639.666,-299.109 643.666,-303.109 643.666,-335.109"/>
+<polyline fill="none" stroke="black" points="639.666,-331.109 350.334,-331.109 "/>
+<polyline fill="none" stroke="black" points="639.666,-331.109 639.666,-299.109 "/>
+<polyline fill="none" stroke="black" points="639.666,-331.109 643.666,-335.109 "/>
+<text text-anchor="middle" x="497" y="-311.109" font-family="Times,serif" font-size="20.00">ocr_tesseract_hocr</text>
 </g>
 <!-- t6&#45;&gt;t7 -->
-<g id="edge9" class="edge"><title>t6&#45;&gt;t7</title>
-<path fill="none" stroke="#0044a0" d="M501.184,-241.422C476.75,-235.286 448.99,-228.315 423.772,-221.982"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="424.429,-218.538 413.877,-219.497 422.724,-225.328 424.429,-218.538"/>
-</g>
-<!-- t8 -->
-<g id="node9" class="node"><title>t8</title>
-<polygon fill="#efa03b" stroke="black" points="663.742,-219.452 448.258,-219.452 444.258,-215.452 444.258,-183.452 659.742,-183.452 663.742,-187.452 663.742,-219.452"/>
-<polyline fill="none" stroke="black" points="659.742,-215.452 444.258,-215.452 "/>
-<polyline fill="none" stroke="black" points="659.742,-215.452 659.742,-183.452 "/>
-<polyline fill="none" stroke="black" points="659.742,-215.452 663.742,-219.452 "/>
-<text text-anchor="middle" x="554" y="-195.452" font-family="Times,serif" font-size="20.00">render_hocr_debug_page</text>
+<g id="edge8" class="edge"><title>t6&#45;&gt;t7</title>
+<path fill="none" stroke="#0044a0" d="M610.903,-357.003C593.736,-351.141 574.358,-344.524 556.518,-338.433"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="557.384,-335.03 546.79,-335.111 555.122,-341.654 557.384,-335.03"/>
 </g>
 <!-- t6&#45;&gt;t8 -->
-<g id="edge11" class="edge"><title>t6&#45;&gt;t8</title>
-<path fill="none" stroke="#0044a0" d="M564.418,-241.346C563.4,-237.546 562.298,-233.43 561.203,-229.345"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="564.522,-228.207 558.554,-219.453 557.761,-230.018 564.522,-228.207"/>
+<g id="edge10" class="edge"><title>t6&#45;&gt;t8</title>
+<path fill="none" stroke="#0044a0" d="M709.875,-357.003C726.624,-351.141 745.529,-344.524 762.933,-338.433"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="764.142,-341.718 772.425,-335.111 761.83,-335.111 764.142,-341.718"/>
 </g>
-<!-- t5&#45;&gt;t7 -->
-<g id="edge10" class="edge"><title>t5&#45;&gt;t7</title>
-<path fill="none" stroke="#0044a0" d="M364.752,-241.346C362.816,-237.329 360.708,-232.958 358.629,-228.645"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="361.693,-226.941 354.197,-219.453 355.387,-229.981 361.693,-226.941"/>
+<!-- t10 -->
+<g id="node9" class="node"><title>t10</title>
+<polygon fill="#efa03b" stroke="black" points="620.109,-277.109 335.891,-277.109 331.891,-273.109 331.891,-241.109 616.109,-241.109 620.109,-245.109 620.109,-277.109"/>
+<polyline fill="none" stroke="black" points="616.109,-273.109 331.891,-273.109 "/>
+<polyline fill="none" stroke="black" points="616.109,-273.109 616.109,-241.109 "/>
+<polyline fill="none" stroke="black" points="616.109,-273.109 620.109,-277.109 "/>
+<text text-anchor="middle" x="476" y="-253.109" font-family="Times,serif" font-size="20.00">render_hocr_page</text>
 </g>
-<!-- t5&#45;&gt;t8 -->
-<g id="edge12" class="edge"><title>t5&#45;&gt;t8</title>
-<path fill="none" stroke="#0044a0" d="M428.289,-241.346C447.587,-235.375 469.416,-228.622 489.404,-222.437"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="490.53,-225.753 499.049,-219.453 488.461,-219.065 490.53,-225.753"/>
+<!-- t7&#45;&gt;t10 -->
+<g id="edge9" class="edge"><title>t7&#45;&gt;t10</title>
+<path fill="none" stroke="#0044a0" d="M490.585,-299.003C489.12,-295.095 487.528,-290.852 485.953,-286.652"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="489.164,-285.245 482.376,-277.111 482.61,-287.703 489.164,-285.245"/>
+</g>
+<!-- t11 -->
+<g id="node13" class="node"><title>t11</title>
+<polygon fill="#efa03b" stroke="black" points="985.486,-277.109 642.514,-277.109 638.514,-273.109 638.514,-241.109 981.486,-241.109 985.486,-245.109 985.486,-277.109"/>
+<polyline fill="none" stroke="black" points="981.486,-273.109 638.514,-273.109 "/>
+<polyline fill="none" stroke="black" points="981.486,-273.109 981.486,-241.109 "/>
+<polyline fill="none" stroke="black" points="981.486,-273.109 985.486,-277.109 "/>
+<text text-anchor="middle" x="812" y="-253.109" font-family="Times,serif" font-size="20.00">render_hocr_debug_page</text>
+</g>
+<!-- t7&#45;&gt;t11 -->
+<g id="edge18" class="edge"><title>t7&#45;&gt;t11</title>
+<path fill="none" stroke="#0044a0" d="M592.793,-299.079C628.531,-292.726 669.303,-285.478 705.912,-278.969"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="706.887,-282.351 716.12,-277.155 705.661,-275.459 706.887,-282.351"/>
 </g>
 <!-- t12 -->
-<g id="node13" class="node"><title>t12</title>
-<polygon fill="#efa03b" stroke="black" points="473.845,-105.456 554,-78.0208 634.155,-105.456 634.08,-149.848 473.92,-149.848 473.845,-105.456"/>
-<polygon fill="none" stroke="black" points="469.836,-102.581 554,-73.7729 638.164,-102.581 638.078,-153.869 469.922,-153.869 469.836,-102.581"/>
-<text text-anchor="middle" x="554" y="-111.726" font-family="Times,serif" font-size="20.00">merge_pages</text>
-</g>
-<!-- t7&#45;&gt;t12 -->
-<g id="edge20" class="edge"><title>t7&#45;&gt;t12</title>
-<path fill="none" stroke="#0044a0" d="M389.35,-183.419C409.937,-175.33 435.398,-165.326 460.092,-155.624"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="461.479,-158.839 469.507,-151.924 458.919,-152.324 461.479,-158.839"/>
-</g>
-<!-- t8&#45;&gt;t12 -->
-<g id="edge19" class="edge"><title>t8&#45;&gt;t12</title>
-<path fill="none" stroke="#0044a0" d="M554,-183.12C554,-177.585 554,-171.177 554,-164.592"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="557.5,-164.201 554,-154.201 550.5,-164.201 557.5,-164.201"/>
-</g>
-<!-- t11&#45;&gt;t12 -->
-<g id="edge17" class="edge"><title>t11&#45;&gt;t12</title>
-<path fill="none" stroke="#0044a0" d="M615.939,-357.13C634.767,-339.333 661.745,-309.733 673,-277.452 686.754,-238.003 694.347,-219.364 673,-183.452 666.345,-172.256 656.999,-162.875 646.44,-155.047"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="648.378,-152.132 638.152,-149.36 644.417,-157.904 648.378,-152.132"/>
-</g>
-<!-- t9&#45;&gt;t12 -->
-<g id="edge18" class="edge"><title>t9&#45;&gt;t12</title>
-<path fill="none" stroke="#0044a0" d="M164.711,-241.235C186.394,-224.08 222.088,-198.224 257,-183.452 321.895,-155.994 399.893,-139.571 459.67,-130.141"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="460.504,-133.554 469.855,-128.574 459.439,-126.635 460.504,-133.554"/>
+<g id="node12" class="node"><title>t12</title>
+<polygon fill="#efa03b" stroke="black" points="607.082,-219.109 348.918,-219.109 344.918,-215.109 344.918,-183.109 603.082,-183.109 607.082,-187.109 607.082,-219.109"/>
+<polyline fill="none" stroke="black" points="603.082,-215.109 344.918,-215.109 "/>
+<polyline fill="none" stroke="black" points="603.082,-215.109 603.082,-183.109 "/>
+<polyline fill="none" stroke="black" points="603.082,-215.109 607.082,-219.109 "/>
+<text text-anchor="middle" x="476" y="-195.109" font-family="Times,serif" font-size="20.00">add_text_layer</text>
 </g>
 <!-- t10&#45;&gt;t12 -->
-<g id="edge22" class="edge"><title>t10&#45;&gt;t12</title>
-<path fill="none" stroke="gray" d="M629.379,-415.337C660.552,-396.255 703,-362.233 703,-318.452 703,-318.452 703,-318.452 703,-258.452 703,-224.066 705.581,-210.223 684,-183.452 673.976,-171.017 660.952,-160.773 647.026,-152.396"/>
-<polygon fill="gray" stroke="gray" points="648.682,-149.312 638.258,-147.419 645.227,-155.4 648.682,-149.312"/>
+<g id="edge15" class="edge"><title>t10&#45;&gt;t12</title>
+<path fill="none" stroke="#0044a0" d="M476,-241.003C476,-237.312 476,-233.322 476,-229.352"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="479.5,-229.111 476,-219.111 472.5,-229.111 479.5,-229.111"/>
 </g>
-<!-- t13 -->
-<g id="node14" class="node"><title>t13</title>
-<polygon fill="#efa03b" stroke="black" points="616.338,-52 495.662,-52 491.662,-48 491.662,-16 612.338,-16 616.338,-20 616.338,-52"/>
-<polyline fill="none" stroke="black" points="612.338,-48 491.662,-48 "/>
-<polyline fill="none" stroke="black" points="612.338,-48 612.338,-16 "/>
-<polyline fill="none" stroke="black" points="612.338,-48 616.338,-52 "/>
-<text text-anchor="middle" x="554" y="-28" font-family="Times,serif" font-size="20.00">validate_pdfa</text>
+<!-- t8&#45;&gt;t9 -->
+<g id="edge13" class="edge"><title>t8&#45;&gt;t9</title>
+<path fill="none" stroke="#0044a0" d="M661.842,-299.864C658.871,-299.606 655.921,-299.354 653,-299.109 509.812,-287.095 471.159,-289.734 324.542,-277.219"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="324.526,-273.705 314.262,-276.332 323.924,-280.679 324.526,-273.705"/>
 </g>
-<!-- t12&#45;&gt;t13 -->
-<g id="edge21" class="edge"><title>t12&#45;&gt;t13</title>
-<path fill="none" stroke="#0044a0" d="M554,-73.9482C554,-69.9654 554,-66.007 554,-62.2247"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="557.5,-62.1573 554,-52.1573 550.5,-62.1574 557.5,-62.1573"/>
+<!-- t8&#45;&gt;t11 -->
+<g id="edge17" class="edge"><title>t8&#45;&gt;t11</title>
+<path fill="none" stroke="#0044a0" d="M818.251,-299.003C817.64,-295.204 816.979,-291.087 816.322,-287.002"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="819.775,-286.429 814.732,-277.111 812.864,-287.539 819.775,-286.429"/>
+</g>
+<!-- t8&#45;&gt;t13 -->
+<g id="edge20" class="edge"><title>t8&#45;&gt;t13</title>
+<path fill="none" stroke="#0044a0" d="M935.04,-299.079C978.311,-292.618 1027.78,-285.23 1071.94,-278.636"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1072.48,-282.093 1081.86,-277.155 1071.45,-275.17 1072.48,-282.093"/>
+</g>
+<!-- t9&#45;&gt;t12 -->
+<g id="edge16" class="edge"><title>t9&#45;&gt;t12</title>
+<path fill="none" stroke="#0044a0" d="M259.577,-241.079C294.86,-234.726 335.115,-227.478 371.259,-220.969"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="372.116,-224.372 381.337,-219.155 370.875,-217.482 372.116,-224.372"/>
+</g>
+<!-- t16 -->
+<g id="node17" class="node"><title>t16</title>
+<polygon fill="#efa03b" stroke="black" points="1031.47,-105.333 1196,-78.005 1360.53,-105.333 1360.37,-149.551 1031.63,-149.551 1031.47,-105.333"/>
+<polygon fill="none" stroke="black" points="1027.46,-101.94 1196,-73.9453 1364.54,-101.94 1364.36,-153.556 1027.64,-153.556 1027.46,-101.94"/>
+<text text-anchor="middle" x="1196" y="-111.555" font-family="Times,serif" font-size="20.00">merge_pages</text>
+</g>
+<!-- t12&#45;&gt;t16 -->
+<g id="edge26" class="edge"><title>t12&#45;&gt;t16</title>
+<path fill="none" stroke="#0044a0" d="M607.222,-185.246C722.142,-172.229 890.172,-153.196 1017.34,-138.791"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1018.04,-142.235 1027.58,-137.632 1017.25,-135.279 1018.04,-142.235"/>
+</g>
+<!-- t11&#45;&gt;t16 -->
+<g id="edge27" class="edge"><title>t11&#45;&gt;t16</title>
+<path fill="none" stroke="#0044a0" d="M859.086,-240.997C916.223,-220.232 1014.29,-184.591 1089.94,-157.099"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1091.32,-160.323 1099.52,-153.618 1088.92,-153.744 1091.32,-160.323"/>
+</g>
+<!-- t15&#45;&gt;t16 -->
+<g id="edge23" class="edge"><title>t15&#45;&gt;t16</title>
+<path fill="none" stroke="#0044a0" d="M1321.98,-414.938C1359.16,-396.916 1407,-364.681 1407,-318.109 1407,-318.109 1407,-318.109 1407,-258.109 1407,-211.469 1370.69,-179.509 1328.2,-158.076"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1329.57,-154.854 1319.04,-153.66 1326.53,-161.158 1329.57,-154.854"/>
+</g>
+<!-- t13&#45;&gt;t16 -->
+<g id="edge25" class="edge"><title>t13&#45;&gt;t16</title>
+<path fill="none" stroke="#0044a0" d="M1196,-240.997C1196,-221.914 1196,-190.265 1196,-163.926"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1199.5,-163.759 1196,-153.759 1192.5,-163.759 1199.5,-163.759"/>
+</g>
+<!-- t14&#45;&gt;t16 -->
+<g id="edge24" class="edge"><title>t14&#45;&gt;t16</title>
+<path fill="none" stroke="#0044a0" d="M1326.64,-588.956C1373.58,-572.748 1427,-543.103 1427,-492.109 1427,-492.109 1427,-492.109 1427,-258.109 1427,-220.539 1416.55,-207.537 1388,-183.109 1376.82,-173.542 1364.16,-165.377 1350.82,-158.413"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1351.99,-155.087 1341.47,-153.781 1348.88,-161.358 1351.99,-155.087"/>
+</g>
+<!-- t17 -->
+<g id="node18" class="node"><title>t17</title>
+<polygon fill="#efa03b" stroke="black" points="1310.18,-52 1085.82,-52 1081.82,-48 1081.82,-16 1306.18,-16 1310.18,-20 1310.18,-52"/>
+<polyline fill="none" stroke="black" points="1306.18,-48 1081.82,-48 "/>
+<polyline fill="none" stroke="black" points="1306.18,-48 1306.18,-16 "/>
+<polyline fill="none" stroke="black" points="1306.18,-48 1310.18,-52 "/>
+<text text-anchor="middle" x="1196" y="-28" font-family="Times,serif" font-size="20.00">copy_final</text>
+</g>
+<!-- t16&#45;&gt;t17 -->
+<g id="edge28" class="edge"><title>t16&#45;&gt;t17</title>
+<path fill="none" stroke="#0044a0" d="M1196,-73.8665C1196,-69.8921 1196,-65.942 1196,-62.1676"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1199.5,-62.1213 1196,-52.1214 1192.5,-62.1214 1199.5,-62.1213"/>
 </g>
 </g>
 </svg>

pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+norecursedirs = lib 

requirements.txt

@@ -1,4 +1,6 @@
-ruffus>=2.6.3
-Pillow>=2.4.0
-reportlab>=3.1.44
-PyPDF2>=1.25.1
+ruffus==2.6.3
+Pillow==3.1.1
+reportlab==3.2.0
+PyPDF2==1.25.1
+img2pdf==0.2
+cffi==1.5.2

setup.cfg

@@ -1,4 +1,4 @@
 [bdist_wheel]
 python-tag = py34
-[pytest]
-norecursedirs = lib 
+[aliases]
+test=pytest

setup.py

@@ -1,12 +1,18 @@
 #!/usr/bin/env python3
+# -*- coding: utf-8 -*-
 # © 2015 James R. Barlow: github.com/jbarlow83
 
-from setuptools import setup
-from subprocess import Popen, STDOUT, check_output, CalledProcessError
-from string import Template
-from collections.abc import Mapping
-import re
+from __future__ import print_function, unicode_literals
+
 import sys
+if sys.version_info < (3, 4):
+    print("Python 3.4 or newer is required")
+    sys.exit(1)
+
+from setuptools import setup  # nopep8
+from subprocess import STDOUT, check_output, CalledProcessError  # nopep8
+from collections.abc import Mapping  # nopep8
+import re  # nopep8
 
 
 missing_program = '''
@@ -145,7 +151,7 @@ command = next((arg for arg in sys.argv[1:] if not arg.startswith('-')), '')
 
 
 if command.startswith('install') or \
-        command in ['check', 'test', 'nosetests', 'easy_install', 'egg_info']:
+        command in ['check', 'test', 'nosetests', 'easy_install']:
     check_external_program(
         program='tesseract',
         need_version='3.02.02',
@@ -174,12 +180,10 @@ if 'upload' in sys.argv[1:]:
     print('Use twine to upload the package - setup.py upload is insecure')
     sys.exit(1)
 
-install_requires = open('requirements.txt').read().splitlines()
 tests_require = open('test_requirements.txt').read().splitlines()
 
 setup(
     name='ocrmypdf',
-    version='3.1',  # also update: release notes, main.py
     description='OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched',
     url='https://github.com/jbarlow83/OCRmyPDF',
     author='James R. Barlow',
@@ -203,7 +207,24 @@ setup(
         "Topic :: Text Processing :: Indexing",
         "Topic :: Text Processing :: Linguistic",
         ],
-    install_requires=install_requires,
+    setup_requires=[
+        'setuptools_scm',
+        'setuptools_scm_git_archive',
+        'cffi>=1.5.0',
+        'pytest-runner'
+    ],
+    use_scm_version={'version_scheme': 'post-release'},
+    cffi_modules=[
+        'ocrmypdf/lib/compile_leptonica.py:ffi'
+    ],
+    install_requires=[
+        'ruffus>=2.6.3',
+        'Pillow>=3.0.0',
+        'reportlab>=3.1.44',
+        'PyPDF2>=1.25.1',
+        'img2pdf>=0.2.0',
+        'cffi>=1.5.0'
+    ],
     tests_require=tests_require,
     entry_points={
         'console_scripts': [

test_requirements.txt

@@ -1,2 +1 @@
-img2pdf>=0.1.5
-pytest>=2.7.2
+pytest>=2.7.2

tests/replace_ghostscript_nopdfa.py

@@ -1,72 +0,0 @@
-#!/usr/bin/env python3
-import sys
-import os
-
-'''
-For testing PDF/A generation failures, this wrapper calls ghostscript
-but filters out all PDF/A information so that a regular PDF will be
-created instead.
-
-It assumes that it is called from a staged system PATH where the first
-item on the PATH contains a file named 'gs' which is a symlink to this
-file.  It will strip out the first item on path to invoke the real
-'gs'.  That is, it expects this when called
-
-1.   tests/output/[...test name...]/bin/gs is a symlink to this file
-2.   tests/output/bin is the first item on PATH
-3.   The real executable is on the path
-
-OCRmyPDF also calls "gs --version" and "gs --help".  gs answers both
-on stdout so, this wrapper prints to stderr.
-
-'''
-
-
-def pdfa_param(arg):
-    if arg.startswith('-sPDFA'):
-        return True
-    if arg.startswith('-dPDFA'):
-        return True
-    if arg.endswith('.ps'):
-        return True
-    if 'ColorConversionStrategy' in arg:
-        return True
-    if 'ProcessColorModel' in arg:
-        return True
-    if 'OutputICCProfile' in arg:
-        return True
-    return False
-
-
-if __name__ == '__main__':
-    sys_args = sys.argv[1:]
-
-    print("Fake Ghostscript wrapper", file=sys.stderr)
-
-    if any(pdfa_param(arg) for arg in sys_args):
-        # We were asked to produce a PDF/A
-        # Filter out PDF/A arguments
-        args = [arg for arg in sys.argv[1:]
-                if not pdfa_param(arg)]
-
-        # Tell Ghostscript to create a PDF 1.3 instead so JHOVE won't
-        # think it's a PDF/A
-        indexof_pdfwrite = next(n for n, item in enumerate(args)
-                                if 'pdfwrite' in item)
-        args.insert(indexof_pdfwrite + 1, '-dCompatibilityLevel=1.3')
-        print("Rewrote arguments", file=sys.stderr)
-    else:
-        args = sys_args
-        print("Keeping arguments", file=sys.stderr)
-
-    exec_path = os.environ['PATH'].split(os.pathsep)
-    env = os.environ.copy()
-    env['PATH'] = os.pathsep.join(exec_path[1:])
-
-    print("Calling real Ghostscript with modified args and env",
-          file=sys.stderr)
-    print(args, file=sys.stderr)
-    print(env['PATH'], file=sys.stderr)
-
-    sys.stderr.flush()
-    os.execvpe('gs', ['gs'] + args, env)

tests/resources/NOTE.rst

@@ -1,34 +0,0 @@
-All test resources must come from free public domain sources for
-copyright reasons.
-
-Test files do not necessarily produce perfect (or even good) OCR
-results.
-
-+---------------------+--------------------------------------------------------------------------------+
-| File                | Source                                                                         |
-+=====================+================================================================================+
-| graph.pdf           | Wikimedia                                                                      |
-+---------------------+--------------------------------------------------------------------------------+
-| c02-22.pdf          | Project Gutenberg: https://www.gutenberg.org/files/76/76-h/images/c02-22.jpg   |
-+---------------------+--------------------------------------------------------------------------------+
-| LinnSequencer.jpg   | Wikimedia_                                                                     |
-+---------------------+--------------------------------------------------------------------------------+
-| congress.jpg        | http://www.baxleystamps.com/litho/meiji/courts_1871.jpg                        |
-+---------------------+--------------------------------------------------------------------------------+
-| blank.pdf           | Blank page from Adobe Illustrator CC 2015                                      |
-+---------------------+--------------------------------------------------------------------------------+
-| enormous.pdf        | PNG file saved to PDF using img2pdf                                            |
-+---------------------+--------------------------------------------------------------------------------+
-| invalid.pdf         | PDF file header followed by EOF marker; not valid                              |
-+---------------------+--------------------------------------------------------------------------------+
-| multipage.pdf       | several other files concatenated                                               |
-+---------------------+--------------------------------------------------------------------------------+
-| skew.pdf            | skewed version of c02-22.PDF                                                   |
-+---------------------+--------------------------------------------------------------------------------+
-| Test_Issue_28.pdf   | file with some syntax errors                                                   |
-+---------------------+--------------------------------------------------------------------------------+
-| missing_docinfo.pdf | file missing its DocumentInfo dictionary                                       |
-+---------------------+--------------------------------------------------------------------------------+ 
-
-
-.. _Wikimedia: https://upload.wikimedia.org/wikipedia/en/b/b7/LinnSequencer_hardware_MIDI_sequencer_brochure_page_2_300dpi.jpg

tests/resources/README.rst

@@ -0,0 +1,60 @@
+These test files are used in OCRmyPDF's test suite. They do not necessarily produce OCR results
+at all and are not meant as examples of OCR output. Some are even invalid PDFs that might
+crash certain PDF viewers.
+
+
+Files derived from free sources
+===============================
+
+These test resources come from free sources, under either public domain or Creative Commons licenses.
+In some cases they were converted from one image format to another without other changes.
+
++---------------------+--------------------------------------------------------------------------------+
+| File                | Source                                                                         |
++=====================+================================================================================+
+| c02-22.pdf          | `Project Gutenberg`_, Adventures of Huckleberry Finn, page 22                  |
++---------------------+--------------------------------------------------------------------------------+
+| congress.jpg        | `US Congressional Records`_                                                    |
++---------------------+--------------------------------------------------------------------------------+
+| graph.pdf           | `Wikimedia: Pandas text analysis.png`_                                         |
++---------------------+--------------------------------------------------------------------------------+
+| LinnSequencer.jpg   | `Wikimedia: LinnSequencer`_ (Creative Commons Attribution-ShareAlike 3.0)      |
++---------------------+--------------------------------------------------------------------------------+
+
+
+Files generated for this project
+================================
+
+The following test resources were crafted specifically for this project, and can be used
+under the terms of the license in LICENSE.rst.
+
+- blank.pdf (a blank PDF page)
+- cmyk.pdf (a CMYK image created in Photoshop)
+- enormous.pdf (a very lage page)
+- francais.pdf (a page containing French accented characters)
+- invalid.pdf (a PDF file header followed by EOF marker)
+- missing_docinfo.pdf (PDF file with no /DocumentInfo section)
+
+
+Assemblies
+==========
+
+These test resources are assemblies from other previously mentioned files, released under the same license terms as their input files.
+
+- cardinal.pdf (four cardinal directions, rotated copies of LinnSequencer.jpg)
+- ccitt.pdf (LinnSequencer.jpg, converted to CCITT encoding)
+- graph_ocred.pdf (from graph.pdf)
+- jbig2.pdf (congress.jpg, converted to JBIG2 encoding)
+- multipage.pdf (from several other files)
+- palette.pdf (congress.jpg, converted to a 256-color palette)
+- skew.pdf (from c02-22.pdf)
+- skew-encrypted.pdf (skew.pdf with encrypted applied)
+
+
+.. _`Wikimedia: LinnSequencer`: https://upload.wikimedia.org/wikipedia/en/b/b7/LinnSequencer_hardware_MIDI_sequencer_brochure_page_2_300dpi.jpg
+
+.. _`Project Gutenberg`: https://www.gutenberg.org/files/76/76-h/76-h.htm#c2
+
+.. _`US Congressional Records`: http://www.baxleystamps.com/litho/meiji/courts_1871.jpg
+
+.. _`Wikimedia: Pandas text analysis.png`: https://en.wikipedia.org/wiki/File:Pandas_text_analysis.png

tests/spoof/qpdf_dummy_return2.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+import sys
+
+
+def main():
+    print('qpdf dummy')
+    sys.exit(2)
+
+
+if __name__ == '__main__':
+    main()

tests/spoof/tesseract_cache.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+import sys
+import os
+import hashlib
+import shutil
+import subprocess
+
+
+CACHE_PATH = os.path.abspath(os.path.join(
+        os.path.dirname(__file__), '..', 'cache'))
+
+
+def real_tesseract():
+    tess_args = ['tesseract'] + sys.argv[1:]
+    os.execvp("tesseract", tess_args)
+    return  # Not reachable
+
+def main():
+    operation = sys.argv[-1]
+    # For anything unexpected operation, defer to real tesseract binary
+    if operation != 'hocr' and operation != 'pdf' and operation != 'stdout':
+        real_tesseract()
+        return  # Not reachable
+
+    try:
+        os.makedirs(CACHE_PATH)
+    except FileExistsError:
+        pass
+
+    m = hashlib.sha1()
+
+    version = subprocess.check_output(
+        ['tesseract', '--version'],
+        stderr=subprocess.STDOUT)
+
+    m.update(version)
+    m.update(operation.encode())
+
+    try:
+        lang = sys.argv[sys.argv.index('-l') + 1]
+        m.update(lang.encode())
+    except ValueError:
+        pass
+    try:
+        psm = sys.argv[sys.argv.index('-psm') + 1]
+        m.update(psm.encode())
+    except ValueError:
+        pass
+
+    if operation == 'stdout' and psm != '0':
+        real_tesseract()
+        return
+
+    if operation == 'stdout':
+        input_file = sys.argv[-2]
+        output_file = 'stdout'
+    else:
+        input_file = sys.argv[-3]
+        output_file = sys.argv[-2]
+
+    if operation == 'hocr':
+        output_file += '.hocr'
+    elif operation == 'pdf':
+        output_file += '.pdf'
+
+    with open(input_file, 'rb') as f:
+        m.update(f.read())
+    cache_name = os.path.join(CACHE_PATH, m.hexdigest())
+    print(cache_name)
+    if os.path.exists(cache_name):
+        # Cache hit
+        print("Tesseract cache hit", file=sys.stderr)
+        if operation != 'stdout':
+            shutil.copy(cache_name, output_file)
+
+        # Replicate output
+        with open(cache_name + '.stdout', 'r') as f:
+            print(f.read(), end='')
+        with open(cache_name + '.stderr', 'r') as f:
+            print(f.read(), end='', file=sys.stderr)
+        sys.exit(0)
+
+    # Cache miss
+    print("Tesseract cache miss", file=sys.stderr)
+
+    # Call tesseract
+    p = subprocess.Popen(
+            ['tesseract'] + sys.argv[1:],
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+            universal_newlines=True)
+    stdout, stderr = p.communicate()
+
+    with open(cache_name + '.stdout', 'w') as f:
+        f.write(stdout)
+    with open(cache_name + '.stderr', 'w') as f:
+        f.write(stderr)
+    print(stdout, end='')
+    print(stderr, end='', file=sys.stderr)
+
+    # Insert file into cache
+    if output_file != 'stdout':
+        if os.path.exists(output_file):
+            shutil.copy(output_file, cache_name)
+        else:
+            print("Could not find output file", file=sys.stderr)
+    else:
+        open(cache_name, 'w').close()
+
+
+if __name__ == '__main__':
+    main()

tests/spoof/tesseract_noop.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+import sys
+import img2pdf
+from PIL import Image
+
+
+VERSION_STRING = '''tesseract 3.04.00
+ leptonica-1.72
+  libjpeg 8d : libpng 1.6.19 : libtiff 4.0.6 : zlib 1.2.5
+SPOOFED
+'''
+
+HOCR_TEMPLATE = '''<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+  <title></title>
+  <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+  <meta name='ocr-system' content='tesseract 3.02.02' />
+  <meta name='ocr-capabilities' content='ocr_page ocr_carea ocr_par ocr_line ocrx_word'/>
+ </head>
+ <body>
+  <div class='ocr_page' id='page_1' title='image "x.tif"; bbox 0 0 {0} {1}; ppageno 0'>
+   <div class='ocr_carea' id='block_1_1' title="bbox 0 1 {0} {1}">
+    <p class='ocr_par' dir='ltr' id='par_1' title="bbox 0 1 {0} {1}">
+     <span class='ocr_line' id='line_1' title="bbox 0 1 {0} {1}"><span class='ocrx_word' id='word_1' title="bbox 0 1 {0} {1}"> </span>
+     </span>
+    </p>
+   </div>
+  </div>
+ </body>
+</html>'''
+
+
+def main():
+    if sys.argv[1] == '--version':
+        print(VERSION_STRING, file=sys.stderr)
+        sys.exit(0)
+    elif sys.argv[1] == '--list-langs':
+        print('List of available languages (1):\neng', file=sys.stderr)
+        sys.exit(0)
+    elif sys.argv[-1] == 'hocr':
+        inputf = sys.argv[-3]
+        output = sys.argv[-2]
+        with Image.open(inputf) as im, \
+                open(output + '.hocr', 'w', encoding='utf-8') as f:
+            w, h = im.size
+            f.write(HOCR_TEMPLATE.format(str(w), str(h)))
+    elif sys.argv[-1] == 'pdf':
+        inputf = sys.argv[-3]
+        output = sys.argv[-2]
+        pdf_bytes = img2pdf.convert([inputf], dpi=300)
+        with open(output + '.pdf', 'wb') as f:
+            f.write(pdf_bytes)
+    elif sys.argv[-1] == 'stdout':
+        inputf = sys.argv[-2]
+        print("""Orientation: 0
+Orientation in degrees: 0
+Orientation confidence: 100.00
+Script: 1
+Script confidence: 100.00""", file=sys.stderr)
+    else:
+        print("Spoof doesn't understand arguments", file=sys.stderr)
+        print(sys.argv, file=sys.stderr)
+        sys.exit(1)
+
+    sys.exit(0)
+
+
+if __name__ == '__main__':
+    main()

tests/test_hocrtransform.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+# © 2015 James R. Barlow: github.com/jbarlow83
+
+from ocrmypdf import hocrtransform
+from ocrmypdf.tesseract import HOCR_TEMPLATE
+from reportlab.pdfgen.canvas import Canvas
+from PIL import Image
+from tempfile import NamedTemporaryFile
+from contextlib import suppress
+import os
+import shutil
+import pytest
+import img2pdf
+import pytest
+import sys
+
+
+if sys.version_info.major < 3:
+    print("Requires Python 3.4+")
+    sys.exit(1)
+
+TESTS_ROOT = os.path.abspath(os.path.dirname(__file__))
+SPOOF_PATH = os.path.join(TESTS_ROOT, 'spoof')
+PROJECT_ROOT = os.path.dirname(TESTS_ROOT)
+OCRMYPDF = os.path.join(PROJECT_ROOT, 'OCRmyPDF.sh')
+TEST_RESOURCES = os.path.join(PROJECT_ROOT, 'tests', 'resources')
+TEST_OUTPUT = os.environ.get(
+    'OCRMYPDF_TEST_OUTPUT',
+    default=os.path.join(PROJECT_ROOT, 'tests', 'output', 'hocrtransform'))
+
+
+def setup_module():
+    with suppress(FileNotFoundError):
+        shutil.rmtree(TEST_OUTPUT)
+    with suppress(FileExistsError):
+        os.makedirs(TEST_OUTPUT)
+    with open(_make_output('blank.hocr'), 'w') as f:
+        f.write(HOCR_TEMPLATE)
+
+
+def _make_input(input_basename):
+    return os.path.join(TEST_RESOURCES, input_basename)
+
+
+def _make_output(output_basename):
+    return os.path.join(TEST_OUTPUT, output_basename)
+
+
+def test_mono_image():
+    im = Image.new('1', (8, 8), 0)
+    for n in range(8):
+        im.putpixel((n, n), 1)
+    im.save(_make_output('mono.tif'), format='TIFF')
+
+    hocr = hocrtransform.HocrTransform(_make_output('blank.hocr'), 300)
+    hocr.to_pdf(_make_output('mono.pdf'), imageFileName=_make_output('mono.tif'))
+
+
+
+
+

tests/test_main.py

@@ -2,7 +2,7 @@
 # © 2015 James R. Barlow: github.com/jbarlow83
 
 from __future__ import print_function
-from subprocess import Popen, PIPE, check_output
+from subprocess import Popen, PIPE, check_output, check_call
 import os
 import shutil
 from contextlib import suppress
@@ -11,6 +11,7 @@ import pytest
 from ocrmypdf.pageinfo import pdf_get_all_pageinfo
 import PyPDF2 as pypdf
 from ocrmypdf import ExitCode
+from ocrmypdf import leptonica
 
 
 if sys.version_info.major < 3:
@@ -18,53 +19,58 @@ if sys.version_info.major < 3:
     sys.exit(1)
 
 TESTS_ROOT = os.path.abspath(os.path.dirname(__file__))
+SPOOF_PATH = os.path.join(TESTS_ROOT, 'spoof')
 PROJECT_ROOT = os.path.dirname(TESTS_ROOT)
 OCRMYPDF = os.path.join(PROJECT_ROOT, 'OCRmyPDF.sh')
 TEST_RESOURCES = os.path.join(PROJECT_ROOT, 'tests', 'resources')
 TEST_OUTPUT = os.environ.get(
     'OCRMYPDF_TEST_OUTPUT',
-    default=os.path.join(PROJECT_ROOT, 'tests', 'output'))
-TEST_BINARY_PATH = os.path.join(TEST_OUTPUT, 'fakebin')
+    default=os.path.join(PROJECT_ROOT, 'tests', 'output', 'main'))
 
 
 def setup_module():
     with suppress(FileNotFoundError):
         shutil.rmtree(TEST_OUTPUT)
     with suppress(FileExistsError):
-        os.mkdir(TEST_OUTPUT)
+        os.makedirs(TEST_OUTPUT)
 
 
-def run_ocrmypdf_sh(input_file, output_file, *args):
+def run_ocrmypdf_sh(input_file, output_file, *args, env=None):
     sh_args = ['sh', OCRMYPDF] + list(args) + [input_file, output_file]
     sh = Popen(
         sh_args, close_fds=True, stdout=PIPE, stderr=PIPE,
-        universal_newlines=True)
+        universal_newlines=True, env=env)
     out, err = sh.communicate()
     return sh, out, err
 
 
-def _make_input(input_basename):
+def _infile(input_basename):
     return os.path.join(TEST_RESOURCES, input_basename)
 
 
-def _make_output(output_basename):
-    return os.path.join(TEST_OUTPUT, output_basename)
+def _outfile(output_basename):
+    return os.path.join(TEST_OUTPUT, os.path.basename(output_basename))
 
 
-def check_ocrmypdf(input_basename, output_basename, *args):
-    input_file = _make_input(input_basename)
-    output_file = _make_output(output_basename)
+def check_ocrmypdf(input_basename, output_basename, *args, env=None):
+    input_file = _infile(input_basename)
+    output_file = _outfile(output_basename)
 
-    sh, _, err = run_ocrmypdf_sh(input_file, output_file, *args)
-    assert sh.returncode == 0, err
+    sh, out, err = run_ocrmypdf_sh(input_file, output_file, *args, env=env)
+    if sh.returncode != 0:
+        print('stdout\n======')
+        print(out)
+        print('stderr\n======')
+        print(err)
+    assert sh.returncode == 0
     assert os.path.exists(output_file), "Output file not created"
     assert os.stat(output_file).st_size > 100, "PDF too small or empty"
     return output_file
 
 
 def run_ocrmypdf_env(input_basename, output_basename, *args, env=None):
-    input_file = _make_input(input_basename)
-    output_file = _make_output(output_basename)
+    input_file = _infile(input_basename)
+    output_file = _outfile(output_basename)
 
     if env is None:
         env = os.environ
@@ -77,13 +83,32 @@ def run_ocrmypdf_env(input_basename, output_basename, *args, env=None):
     return p, out, err
 
 
-def test_quick():
-    check_ocrmypdf('c02-22.pdf', 'test_quick.pdf')
+@pytest.fixture
+def spoof_tesseract_noop():
+    env = os.environ.copy()
+    program = os.path.join(SPOOF_PATH, 'tesseract_noop.py')
+    check_call(['chmod', "+x", program])
+    env['OCRMYPDF_TESSERACT'] = program
+    return env
 
 
-def test_deskew():
+@pytest.fixture
+def spoof_tesseract_cache():
+    env = os.environ.copy()
+    program = os.path.join(SPOOF_PATH, "tesseract_cache.py")
+    check_call(['chmod', '+x', program])
+    env['OCRMYPDF_TESSERACT'] = program
+    return env
+
+
+def test_quick(spoof_tesseract_noop):
+    check_ocrmypdf('c02-22.pdf', 'test_quick.pdf', env=spoof_tesseract_noop)
+
+
+def test_deskew(spoof_tesseract_noop):
     # Run with deskew
-    deskewed_pdf = check_ocrmypdf('skew.pdf', 'test_deskew.pdf', '-d')
+    deskewed_pdf = check_ocrmypdf(
+        'skew.pdf', 'test_deskew.pdf', '-d', '-v', '1', env=spoof_tesseract_noop)
 
     # Now render as an image again and use Leptonica to find the skew angle
     # to confirm that it was deskewed
@@ -91,7 +116,7 @@ def test_deskew():
     import logging
     log = logging.getLogger()
 
-    deskewed_png = _make_output('deskewed.png')
+    deskewed_png = _outfile('deskewed.png')
 
     rasterize_pdf(
         deskewed_pdf,
@@ -101,38 +126,42 @@ def test_deskew():
         raster_device='pngmono',
         log=log)
 
-    from ocrmypdf.leptonica import pixRead, pixDestroy, pixFindSkew
-    pix = pixRead(deskewed_png)
-    skew_angle, skew_confidence = pixFindSkew(pix)
-    pix = pixDestroy(pix)
+    from ocrmypdf.leptonica import Pix
+    pix = Pix.read(deskewed_png)
+    skew_angle, skew_confidence = pix.find_skew()
 
     print(skew_angle)
     assert -0.5 < skew_angle < 0.5, "Deskewing failed"
 
 
-def test_clean():
-    check_ocrmypdf('skew.pdf', 'test_clean.pdf', '-c')
+def test_clean(spoof_tesseract_noop):
+    check_ocrmypdf('skew.pdf', 'test_clean.pdf', '-c', env=spoof_tesseract_noop)
 
 
-def check_exotic_image(pdf, renderer):
+@pytest.mark.parametrize("pdf,renderer", [
+    ('palette.pdf', 'hocr'),
+    ('palette.pdf', 'tesseract'),
+    ('cmyk.pdf', 'hocr'),
+    ('cmyk.pdf', 'tesseract'),
+    ('ccitt.pdf', 'hocr'),
+    ('ccitt.pdf', 'tesseract'),
+    ('jbig2.pdf', 'hocr'),
+    ('jbig2.pdf', 'tesseract')
+])
+def test_exotic_image(spoof_tesseract_cache, pdf, renderer):
     check_ocrmypdf(
         pdf,
         'test_{0}_{1}.pdf'.format(pdf, renderer),
         '-dc',
-        '--pdf-renderer', renderer)
+        '-v', '1',
+        '--pdf-renderer', renderer, env=spoof_tesseract_cache)
 
 
-def test_exotic_image():
-    yield check_exotic_image, 'palette.pdf', 'hocr'
-    yield check_exotic_image, 'palette.pdf', 'tesseract'
-    yield check_exotic_image, 'cmyk.pdf', 'hocr'
-    yield check_exotic_image, 'cmyk.pdf', 'tesseract'
+def test_preserve_metadata(spoof_tesseract_noop):
+    pdf_before = pypdf.PdfFileReader(_infile('graph.pdf'))
 
-
-def test_preserve_metadata():
-    pdf_before = pypdf.PdfFileReader(_make_input('graph.pdf'))
-
-    output = check_ocrmypdf('graph.pdf', 'test_metadata_preserve.pdf')
+    output = check_ocrmypdf('graph.pdf', 'test_metadata_preserve.pdf',
+                            env=spoof_tesseract_noop)
 
     pdf_after = pypdf.PdfFileReader(output)
 
@@ -140,9 +169,9 @@ def test_preserve_metadata():
         assert pdf_before.documentInfo[key] == pdf_after.documentInfo[key]
 
 
-def test_override_metadata():
-    input_file = _make_input('c02-22.pdf')
-    output_file = _make_output('test_override_metadata.pdf')
+def test_override_metadata(spoof_tesseract_noop):
+    input_file = _infile('c02-22.pdf')
+    output_file = _outfile('test_override_metadata.pdf')
 
     german = 'Du siehst den Wald vor lauter Bäumen nicht.'
     chinese = '孔子'
@@ -152,7 +181,8 @@ def test_override_metadata():
         input_file, output_file,
         '--title', german,
         '--author', chinese,
-        '--subject', high_unicode)
+        '--subject', high_unicode,
+        env=spoof_tesseract_noop)
 
     assert p.returncode == ExitCode.ok
 
@@ -171,20 +201,20 @@ def test_override_metadata():
     assert pdfinfo.get('Keywords', '') == ''
 
 
-def check_oversample(renderer):
+@pytest.mark.parametrize('renderer', [
+    'hocr',
+    'tesseract',
+    ])
+def test_oversample(spoof_tesseract_cache, renderer):
     oversampled_pdf = check_ocrmypdf(
-        'skew.pdf', 'test_oversample_%s.pdf' % renderer, '--oversample', '300',
-        '--pdf-renderer', renderer)
+        'skew.pdf', 'test_oversample_%s.pdf' % renderer, '--oversample', '350',
+        '-f',
+        '--pdf-renderer', renderer, env=spoof_tesseract_cache)
 
     pdfinfo = pdf_get_all_pageinfo(oversampled_pdf)
 
     print(pdfinfo[0]['xres'])
-    assert abs(pdfinfo[0]['xres'] - 300) < 1
-
-
-def test_oversample():
-    yield check_oversample, 'hocr'
-    yield check_oversample, 'tesseract'
+    assert abs(pdfinfo[0]['xres'] - 350) < 1
 
 
 def test_repeat_ocr():
@@ -192,101 +222,123 @@ def test_repeat_ocr():
     assert sh.returncode != 0
 
 
-def test_force_ocr():
-    out = check_ocrmypdf('graph_ocred.pdf', 'test_force.pdf', '-f')
+def test_force_ocr(spoof_tesseract_cache):
+    out = check_ocrmypdf('graph_ocred.pdf', 'test_force.pdf', '-f',
+                         env=spoof_tesseract_cache)
     pdfinfo = pdf_get_all_pageinfo(out)
     assert pdfinfo[0]['has_text']
 
 
-def test_skip_ocr():
-    check_ocrmypdf('graph_ocred.pdf', 'test_skip.pdf', '-s')
+def test_skip_ocr(spoof_tesseract_cache):
+    check_ocrmypdf('graph_ocred.pdf', 'test_skip.pdf', '-s',
+                   env=spoof_tesseract_cache)
 
 
-def test_argsfile():
-    with open(_make_output('test_argsfile.txt'), 'w') as argsfile:
+def test_argsfile(spoof_tesseract_noop):
+    with open(_outfile('test_argsfile.txt'), 'w') as argsfile:
         print('--title', 'ArgsFile Test', '--author', 'Test Cases',
               sep='\n', end='\n', file=argsfile)
     check_ocrmypdf('graph.pdf', 'test_argsfile.pdf',
-                   '@' + _make_output('test_argsfile.txt'))
+                   '@' + _outfile('test_argsfile.txt'),
+                   env=spoof_tesseract_noop)
 
 
-def check_ocr_timeout(renderer):
+def check_monochrome_correlation(
+        reference_pdf, reference_pageno, test_pdf, test_pageno):
+
+    import ocrmypdf.ghostscript as ghostscript
+    import logging
+
+    gslog = logging.getLogger()
+
+    reference_png = _outfile('{}.ref{:04d}.png'.format(
+        reference_pdf, reference_pageno))
+    test_png = _outfile('{}.test{:04d}.png'.format(
+        test_pdf, test_pageno))
+
+    def rasterize(pdf, pageno, png):
+        if os.path.exists(png):
+            print(png)
+            return
+        ghostscript.rasterize_pdf(
+            pdf,
+            png,
+            xres=100, yres=100,
+            raster_device='pngmono', log=gslog, pageno=pageno)
+
+    rasterize(reference_pdf, reference_pageno, reference_png)
+    rasterize(test_pdf, test_pageno, test_png)
+
+    pix_ref = leptonica.Pix.read(reference_png)
+    pix_test = leptonica.Pix.read(test_png)
+
+    return leptonica.Pix.correlation_binary(pix_ref, pix_test)
+
+
+def test_monochrome_correlation():
+    # Verify leptonica: check that an incorrect rotated image has poor
+    # correlation with reference
+    corr = check_monochrome_correlation(
+        reference_pdf=_infile('cardinal.pdf'),
+        reference_pageno=1,  # north facing page
+        test_pdf=_infile('cardinal.pdf'),
+        test_pageno=3,  # south facing page
+        )
+    assert corr < 0.10
+
+
+@pytest.mark.parametrize('renderer', [
+    'hocr',
+    'tesseract',
+    ])
+def test_autorotate(spoof_tesseract_cache, renderer):
+    import ocrmypdf.ghostscript as ghostscript
+    import logging
+
+    gslog = logging.getLogger()
+
+    # cardinal.pdf contains four copies of an image rotated in each cardinal
+    # direction - these ones are "burned in" not tagged with /Rotate
+    out = check_ocrmypdf('cardinal.pdf', 'test_autorotate_%s.pdf' % renderer,
+                         '-r', '-v', '1', env=spoof_tesseract_cache)
+    for n in range(1, 4+1):
+        correlation = check_monochrome_correlation(
+            reference_pdf=_infile('cardinal.pdf'),
+            reference_pageno=1,
+            test_pdf=out,
+            test_pageno=n)
+        assert correlation > 0.80
+
+
+@pytest.mark.parametrize('renderer', [
+    'hocr',
+    'tesseract',
+    ])
+def test_ocr_timeout(renderer):
     out = check_ocrmypdf('skew.pdf', 'test_timeout_%s.pdf' % renderer,
                          '--tesseract-timeout', '1.0')
     pdfinfo = pdf_get_all_pageinfo(out)
-    assert pdfinfo[0]['has_text'] == False
+    assert not pdfinfo[0]['has_text']
 
 
-def test_ocr_timeout():
-    yield check_ocr_timeout, 'hocr'
-    yield check_ocr_timeout, 'tesseract'
-
-
-def test_skip_big():
+def test_skip_big(spoof_tesseract_cache):
     out = check_ocrmypdf('enormous.pdf', 'test_enormous.pdf',
-                         '--skip-big', '10')
+                         '--skip-big', '10', env=spoof_tesseract_cache)
     pdfinfo = pdf_get_all_pageinfo(out)
-    assert pdfinfo[0]['has_text'] == False
+    assert not pdfinfo[0]['has_text']
 
 
-def check_maximum_options(renderer):
+@pytest.mark.parametrize('renderer', [
+    'hocr',
+    'tesseract',
+    ])
+def test_maximum_options(spoof_tesseract_cache, renderer):
     check_ocrmypdf(
         'multipage.pdf', 'test_multipage%s.pdf' % renderer,
         '-d', '-c', '-i', '-g', '-f', '-k', '--oversample', '300',
         '--skip-big', '10', '--title', 'Too Many Weird Files',
-        '--author', 'py.test', '--pdf-renderer', renderer)
-
-
-def test_maximum_options():
-    yield check_maximum_options, 'hocr'
-    yield check_maximum_options, 'tesseract'
-
-
-def override_binary(binary, replacement):
-    '''Create a directory that contains a symlink named 'binary' that
-    points to replacement, another program to use in place of the
-    regular binary for testing.
-
-    override_binary('gs', 'replace_gs.py') will create an environment
-    in which "gs" will invoke replace_gs.py.
-
-    Not thread-safe with other test at the moment.
-
-    Returns the os.environ["PATH"] string under which this binary will
-    be invoked.'''
-
-    replacement_path = os.path.abspath(os.path.join(TESTS_ROOT,
-                                                    replacement))
-    subdir = os.path.splitext(os.path.basename(replacement))[0]
-    binary_path = os.path.abspath(os.path.join(TEST_BINARY_PATH,
-                                               subdir,
-                                               binary))
-    with suppress(FileExistsError):
-        os.makedirs(os.path.dirname(binary_path))
-    assert os.path.isdir(os.path.dirname(binary_path))
-    assert not os.path.lexists(binary_path)
-    print("symlink %s -> %s" % (replacement_path, binary_path))
-    os.symlink(replacement_path, binary_path)
-
-    os.chmod(replacement_path, int('755', base=8))
-
-    return os.path.dirname(binary_path) + os.pathsep + os.environ["PATH"]
-
-
-@pytest.fixture
-def break_ghostscript_pdfa():
-    return override_binary('gs', 'replace_ghostscript_nopdfa.py')
-
-
-@pytest.mark.skipif(os.environ.get('OCRMYPDF_IN_DOCKER', False),
-                    reason="Requires writable filesystem")
-def test_ghostscript_pdfa_fails(break_ghostscript_pdfa):
-    env = os.environ.copy()
-    env['PATH'] = break_ghostscript_pdfa
-
-    p, out, err = run_ocrmypdf_env(
-        'graph_ocred.pdf', 'not_a_pdfa.pdf', '-v', '1', '--skip-text', env=env)
-    assert p.returncode == ExitCode.ok, err  # no longer using JHOVE PDFA check
+        '--author', 'py.test', '--pdf-renderer', renderer,
+        env=spoof_tesseract_cache)
 
 
 def test_tesseract_missing_tessdata():
@@ -310,9 +362,9 @@ def test_blank_input_pdf():
     assert p.returncode == ExitCode.ok
 
 
-def test_french():
+def test_french(spoof_tesseract_cache):
     p, out, err = run_ocrmypdf_env(
-        'francais.pdf', 'francais.pdf', '-l', 'fra')
+        'francais.pdf', 'francais.pdf', '-l', 'fra', env=spoof_tesseract_cache)
     assert p.returncode == ExitCode.ok, \
         "This test may fail if Tesseract language packs are missing"
 
@@ -323,25 +375,27 @@ def test_klingon():
     assert p.returncode == ExitCode.bad_args
 
 
-def test_missing_docinfo():
+def test_missing_docinfo(spoof_tesseract_noop):
     p, out, err = run_ocrmypdf_env(
-        'missing_docinfo.pdf', 'missing_docinfo.pdf', '-l', 'eng', '-c')
+        'missing_docinfo.pdf', 'missing_docinfo.pdf', '-l', 'eng', '-c',
+        env=spoof_tesseract_noop)
     assert p.returncode == ExitCode.ok, err
 
 
-def test_uppercase_extension():
-    shutil.copy(_make_input("skew.pdf"), _make_input("UPPERCASE.PDF"))
+def test_uppercase_extension(spoof_tesseract_noop):
+    shutil.copy(_infile("skew.pdf"), _infile("UPPERCASE.PDF"))
     try:
-        check_ocrmypdf("UPPERCASE.PDF", "UPPERCASE_OUT.PDF")
+        check_ocrmypdf("UPPERCASE.PDF", "UPPERCASE_OUT.PDF",
+                       env=spoof_tesseract_noop)
     finally:
-        os.unlink(_make_input("UPPERCASE.PDF"))
+        os.unlink(_infile("UPPERCASE.PDF"))
 
 
 def test_input_file_not_found():
     input_file = "does not exist.pdf"
     sh, out, err = run_ocrmypdf_sh(
-        _make_input(input_file),
-        _make_output("will not happen.pdf"))
+        _infile(input_file),
+        _outfile("will not happen.pdf"))
     assert sh.returncode == ExitCode.input_file
     assert (input_file in out or input_file in err)
 
@@ -349,8 +403,39 @@ def test_input_file_not_found():
 def test_input_file_not_a_pdf():
     input_file = __file__  # Try to OCR this file
     sh, out, err = run_ocrmypdf_sh(
-        _make_input(input_file),
-        _make_output("will not happen.pdf"))
+        _infile(input_file),
+        _outfile("will not happen.pdf"))
     assert sh.returncode == ExitCode.input_file
     assert (input_file in out or input_file in err)
 
+
+def test_qpdf_repair_fails():
+    env = os.environ.copy()
+    env['OCRMYPDF_QPDF'] = os.path.abspath('./spoof/qpdf_dummy_return2.py')
+    p, out, err = run_ocrmypdf_env(
+        '-v', '1',
+        'c02-22.pdf', 'wont_be_created.pdf', env=env)
+    print(out)
+    print(err)
+    assert p.returncode == ExitCode.input_file
+
+
+def test_encrypted():
+    p, out, err = run_ocrmypdf_env('skew-encrypted.pdf', 'wont_be_created.pdf')
+    assert p.returncode == ExitCode.input_file
+    assert out.find('password')
+
+
+@pytest.mark.parametrize('renderer', [
+    'hocr',
+    'tesseract',
+    ])
+def test_pagesegmode(renderer, spoof_tesseract_cache):
+    check_ocrmypdf(
+        'skew.pdf', 'test_psm_%s.pdf' % renderer,
+        '--tesseract-pagesegmode', '7',
+        '-v', '1',
+        '--pdf-renderer', renderer, env=spoof_tesseract_cache)
+
+
+

tests/test_pageinfo.py

@@ -10,21 +10,37 @@ import os
 import shutil
 import pytest
 import img2pdf
-from pkg_resources import Requirement, resource_filename
 import pytest
+import sys
 
-req = Requirement.parse('ocrmypdf')
 
+if sys.version_info.major < 3:
+    print("Requires Python 3.4+")
+    sys.exit(1)
+
+TESTS_ROOT = os.path.abspath(os.path.dirname(__file__))
+SPOOF_PATH = os.path.join(TESTS_ROOT, 'spoof')
+PROJECT_ROOT = os.path.dirname(TESTS_ROOT)
+OCRMYPDF = os.path.join(PROJECT_ROOT, 'OCRmyPDF.sh')
+TEST_RESOURCES = os.path.join(PROJECT_ROOT, 'tests', 'resources')
 TEST_OUTPUT = os.environ.get(
     'OCRMYPDF_TEST_OUTPUT',
-    default=os.path.join(os.path.dirname(__file__), 'output'))
+    default=os.path.join(PROJECT_ROOT, 'tests', 'output', 'pageinfo'))
 
 
 def setup_module():
     with suppress(FileNotFoundError):
         shutil.rmtree(TEST_OUTPUT)
     with suppress(FileExistsError):
-        os.mkdir(TEST_OUTPUT)
+        os.makedirs(TEST_OUTPUT)
+
+
+def _make_input(input_basename):
+    return os.path.join(TEST_RESOURCES, input_basename)
+
+
+def _make_output(output_basename):
+    return os.path.join(TEST_OUTPUT, output_basename)
 
 
 def test_single_page_text():
@@ -51,13 +67,21 @@ def test_single_page_text():
 def test_single_page_image():
     filename = os.path.join(TEST_OUTPUT, 'image-mono.pdf')
 
-    with NamedTemporaryFile() as im_tmp:
+    with NamedTemporaryFile(mode='wb+', suffix='.png') as im_tmp:
         im = Image.new('1', (8, 8), 0)
         for n in range(8):
             im.putpixel((n, n), 1)
         im.save(im_tmp.name, format='PNG')
 
-        pdf_bytes = img2pdf.convert([im_tmp.name], dpi=8)
+        imgsize = ((img2pdf.ImgSize.dpi, 8), (img2pdf.ImgSize.dpi, 8))
+        layout_fun = img2pdf.get_layout_fun(None, imgsize, None, None, None)
+
+        im_tmp.seek(0)
+        im_bytes = im_tmp.read()
+        pdf_bytes = img2pdf.convert(
+                im_bytes, producer="img2pdf", with_pdfrw=False,
+                layout_fun=layout_fun)
+
         with open(filename, 'wb') as pdf:
             pdf.write(pdf_bytes)
 
@@ -100,9 +124,8 @@ def test_single_page_inline_image():
         pageinfo.pdf_get_all_pageinfo(filename)
 
 
-@pytest.mark.skipif(True, reason="need to fix packaging error")
 def test_jpeg():
-    filename = resource_filename(req, 'tests/resources/c02-22.pdf')
+    filename = _make_input('c02-22.pdf')
 
     pdfinfo = pageinfo.pdf_get_all_pageinfo(filename)