v4.0: release notes

Since the corrected font is provided in the Docker image, there's no
reason to show the warning.

.gitignore

@@ -1,9 +1,7 @@
 # Development environment
 *.pyc
 *.sublime-*
-venv-3.4/
-venv-3.5/
-venv/
+venv-*/
 pyvenv.cfg
 
 # Package building

.travis.yml

@@ -1,8 +1,9 @@
 language: python
 
+cache: pip
+
 cache:
     directories:
-        - $HOME/.cache/pip
         - $HOME/.ccache
         - tarballs
         - tests/cache
@@ -10,6 +11,9 @@ cache:
 python:
     - 3.4
 
+before_cache:
+    - rm -f $HOME/.cache/pip/log/debug.log
+
 before_install:
     # Ubuntu packages
     - sudo add-apt-repository ppa:evl.ms/precise -y  # for Ghostscript 9.15
@@ -17,7 +21,7 @@ before_install:
     - sudo add-apt-repository ppa:b-eltzner/qpdfview-exp -y  # for QPDF 5
     - sudo add-apt-repository ppa:itachi-san/ffmpeg -y  # for libav 11.2 (for unpaper)
     - sudo apt-get update -qq  # must go after all add-apt-repo
-    - sudo apt-get install -y ghostscript tesseract-ocr tesseract-ocr-deu tesseract-ocr-eng tesseract-ocr-fra qpdf poppler-utils gcc libavformat-dev libavcodec-dev libavutil-dev automake make pkg-config xsltproc 
+    - sudo apt-get install -y ghostscript tesseract-ocr tesseract-ocr-deu tesseract-ocr-eng tesseract-ocr-fra qpdf poppler-utils gcc libavformat-dev libavcodec-dev libavutil-dev automake make pkg-config xsltproc libffi-dev
 
     # pip
     - pip install --upgrade pip
@@ -36,7 +40,7 @@ install:
 script:
     - python setup.py clean
     - python setup.py install
-    - py.test
+    - python setup.py test
 
 os:
     - linux

Dockerfile

@@ -1,6 +1,6 @@
 # OCRmyPDF
 #
-# VERSION               3.0.2
+# VERSION               3.2
 FROM      debian:stretch
 MAINTAINER James R. Barlow <jim@purplerock.ca>
 
@@ -31,6 +31,11 @@ RUN apt-get install -y --no-install-recommends \
   tesseract-ocr \
   tesseract-ocr-deu tesseract-ocr-spa tesseract-ocr-eng tesseract-ocr-fra
 
+RUN apt-get install -qy --no-install-recommends \
+  libffi-dev \
+  libpython3-dev \
+  gcc
+
 # Enforce UTF-8
 # Borrowed from https://index.docker.io/u/crosbymichael/python/ 
 RUN dpkg-reconfigure locales && \
@@ -38,9 +43,6 @@ RUN dpkg-reconfigure locales && \
   /usr/sbin/update-locale LANG=C.UTF-8
 ENV LC_ALL C.UTF-8
 
-# Remove the junk
-RUN apt-get autoremove -y && apt-get clean -y
-RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/*
 
 # Set up a Python virtualenv and take all of the system packages, so we can
 # rely on the platform packages rather than importing GCC and compiling them
@@ -49,18 +51,34 @@ RUN pyvenv /appenv \
 
 COPY . /application/
 
+# Replace stock Tesseract 3.04.00 font with improved sharp2.ttf that resolves
+# issues in many PDF viewers.
+# Discussion is in https://github.com/tesseract-ocr/tesseract/issues/182
+COPY ./share/sharp2.ttf /usr/share/tesseract-ocr/tessdata/pdf.ttf
+RUN chmod 644 /usr/share/tesseract-ocr/tessdata/pdf.ttf
+
+# Set this here to force a docker version, allowing non-tagged versions to
+# be built
+# ENV SETUPTOOLS_SCM_PRETEND_VERSION=v3.3.0
+
 # Install application and dependencies
 # In this arrangement Pillow and reportlab will be provided by the system
 RUN . /appenv/bin/activate; \
   pip install --upgrade pip \
-  && pip install --no-cache-dir /application \
+  && pip install /application \
   && pip install --no-cache-dir -r /application/test_requirements.txt
 
+# Remove the junk
+RUN apt-get remove -qy gcc
+RUN apt-get autoremove -y && apt-get clean -y
+RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/*
+
 USER docker
 WORKDIR /home/docker
 
 ENV OCRMYPDF_TEST_OUTPUT=/tmp/test-output
 ENV OCRMYPDF_IN_DOCKER=1
+ENV OCRMYPDF_SHARP_TTF=1
 
 # Must use array form of ENTRYPOINT
 # Non-array form does not append other arguments, because that is "intuitive"

Dockerfile.polyglot

@@ -1,6 +1,6 @@
 # OCRmyPDF polyglot
 #
-# VERSION               3.0.2
+# VERSION               3.2
 FROM      jbarlow83/ocrmypdf:latest
 MAINTAINER James R. Barlow <jim@purplerock.ca>
 

MANIFEST.in

@@ -1,3 +1,4 @@
 recursive-exclude tests/output *
 include requirements.txt
-include test_requirements.txt
+include test_requirements.txt
+include share/*

README.rst

@@ -48,22 +48,22 @@ Download OCRmyPDF here: https://github.com/jbarlow83/OCRmyPDF/releases
 
 You can install it to a Python virtual environment or system-wide. 
 
-Installing the Docker container
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Installing the Docker image
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-For many users, installing the Docker container will be easier than installing all of OCRmyPDF's dependencies. For Windows, it is the only option.
+For many users, installing the Docker image will be easier than installing all of OCRmyPDF's dependencies. For Windows, it is the only option.
 
 If you have `Docker <https://docs.docker.com/>`__ installed on your system, you can install
-a Docker container of the latest release.
+a Docker image of the latest release.
 
 Follow the Docker installation instructions for your platform.  If you can run this command
 successfully, your system is ready to download and execute the image::
 
    docker run hello-world
    
-OCRmyPDF will use all available CPU cores.  By default, the VirtualBox machine instance on Windows and OS X has only a single CPU core enabled. Use the VirtualBox Manager to determine the name of your Docker container host, and then follow these optional steps to enable multiple CPUs::
+OCRmyPDF will use all available CPU cores.  By default, the VirtualBox machine instance on Windows and OS X has only a single CPU core enabled. Use the VirtualBox Manager to determine the name of your Docker engine host, and then follow these optional steps to enable multiple CPUs::
 
-   # Optional
+   # Optional step for Mac OS X users
    docker-machine stop "yourVM"
    VBoxManage modifyvm "yourVM" --cpus 2  # or whatever number of core is desired
    docker-machine start "yourVM"
@@ -77,8 +77,14 @@ the image::
 Then tag it to give a more convenient name, just ocrmypdf::
 
    docker tag jbarlow83/ocrmypdf ocrmypdf
-  
-You can then run using the command::
+
+This image contains language packs for English, French, Spanish and German. The alternative "polyglot" image provides `all available language packs <https://github.com/tesseract-ocr/tesseract/blob/master/doc/tesseract.1.asc#languages>`__::
+
+   # Alternative step: If you need all language packs
+   docker pull jbarlow83/ocrmypdf-polyglot
+   docker tag jbarlow83/ocrmypdf-polyglot ocrmypdf
+
+You can then run ocrmypdf using the command::
 
    docker run ocrmypdf --help
   
@@ -90,7 +96,8 @@ In this worked example, the current working directory contains an input file cal
 
    docker run -v "$(pwd):/home/docker"   ocrmypdf --skip-text test.pdf output.pdf
 
-Note that ``ocrmypdf`` has its own separate ``-v VERBOSITYLEVEL`` argument to control debug verbosity. All Docker arguments should before the ``ocrmypdf`` container name and all arguments to ``ocrmypdf`` should be listed after.
+Note that ``ocrmypdf`` has its own separate ``-v VERBOSITYLEVEL`` argument to control debug verbosity. All Docker arguments should before the ``ocrmypdf`` image name and all arguments to ``ocrmypdf`` should be listed after.
+
 
 Installing on Mac OS X
 ~~~~~~~~~~~~~~~~~~~~~~
@@ -113,17 +120,11 @@ Install or upgrade the required Homebrew packages, if any are missing::
    brew install leptonica
    brew install tesseract
    
-It is also recommended that install Pillow and confirm it can read and write JPEG and PNG files::
+Update the homebrew pip and install Pillow::
 
    pip3 install --upgrade pip
    pip3 install --upgrade pillow
 
-Sometimes, the Python imaging library (Pillow) can end up being compiled and installed without support for JPEG and PNG files. (Arguably, this is an unfixed bug in Pillow's installer.) To confirm that Pillow is compiled correctly and can access JPEG and PNG files, try this command::
-
-   python3 -c "from PIL import Image; im = Image.new('1', (1, 1)); im.save('test.png'); im.save('test.jpg')"
-
-If you have trouble getting Pillow to access JPEG and PNG files, `review the installation instructions <https://pillow.readthedocs.org/installation.html>`__.
-
 You can then install OCRmyPDF from PyPI::
 
    pip3 install ocrmypdf
@@ -188,24 +189,18 @@ Installing HEAD revision from sources
 If you have ``git`` and ``python3.4`` or ``python3.5`` installed, you can install from source. When the ``pip`` installer runs,
 it will alert you if dependencies are missing.
 
-First, clone the HEAD revision::
+To install the HEAD revision from sources in the current Python 3 environment::
 
-   git clone -b master https://github.com/jbarlow83/OCRmyPDF.git
-   cd OCRmyPDF
+   pip3 install git+https://github.com/jbarlow83/OCRmyPDF.git
 
-To install the HEAD revision from sources::
+Or, to install in `development mode <https://pythonhosted.org/setuptools/setuptools.html#development-mode>`__,  allowing customization of OCRmyPDF, use the ``-e`` flag::
 
-   pip3 install .
-
-Or, to install in `development mode <https://pythonhosted.org/setuptools/setuptools.html#development-mode>`__, 
-allowing customization of OCRmyPDF, use the ``-e`` flag::
-
-   pip3 install -e .
+   pip3 install -e git+https://github.com/jbarlow83/OCRmyPDF.git
    
 On certain Linux distributions such as Ubuntu, you may need to use 
 run the install command as superuser::
 
-   sudo pip3 install [-e] .
+   sudo pip3 install [-e] git+https://github.com/jbarlow83/OCRmyPDF.git
    
 Note that this will alter your system's Python distribution. If you prefer 
 to not install as superuser, you can install the package in a Python virtual environment::
@@ -228,6 +223,18 @@ need to be installed. The script requires specific versions of the
 dependencies. Older version than the ones mentioned in the release notes
 are likely not to be compatible to OCRmyPDF.
 
+Languages
+---------
+
+OCRmyPDF uses Tesseract for OCR, and relies on its language packs. For Linux users,
+you can often find packages that provide language packs::
+
+   # Debian/Ubuntu users
+   sudo apt-get install tesseract-ocr-chi-sim
+   
+You can then pass the ``-l LANG`` argument to OCRmyPDF to give a hint as to what languages it should search for. Multiple
+languages can be requested.
+
 Support
 -------
 

RELEASE_NOTES.rst

@@ -6,8 +6,53 @@ Please always read this file before installing the package
 Download software here: https://github.com/jbarlow83/OCRmyPDF/tags
 
 
+v4.0:
+=====
+
+New features
+------------
+
+-  Automatic page rotation (``-r``) is now available. It uses ignores any prior rotation information
+   on PDFs and sets rotation based on the dominant orientation of detectable text. This feature is
+   fairly reliable but some false positives occur especially if there is not much text to work with. (#4) 
+-  Deskewing is now performed using Leptonica instead of unpaper. Leptonica deskews images faster and
+   more reliably than unpaper.
+
+
+Fixes
+-----
+
+-  Fixed an issue where lossless reconstruction could cause some pages to be appear incorrectly
+   rotated in the page had a /Rotate tag applied (#50)
+-  Fixed an issue where lossless reconstruction could misalign the graphics layer with respect to
+   text layer if the page had been cropped such that its origin is not (0, 0) (#49)
+
+
+Changes
+-------
+
+-  Logging output much is now better
+-  ``--deskew`` is now performed by Leptonica instead of unpaper (#25)
+-  libffi is now required
+-  Some changes were made to the Docker and Travis build environments to support libffi
+-  ``--pdf-renderer=tesseract`` now displays a warning if the Tesseract version is less than 3.04.01,
+   the planned release that will include fixes to an important OCR text rendering bug in Tesseract 3.04.00.
+   You can also manually install ./share/sharp2.ttf on top of pdf.ttf in your Tesseract tessdata folder
+   to correct the problem.
+
+
+v3.2.1:
+=======
+
+Changes
+-------
+
+-  Fixed issue #47 "convert() got and unexpected keyword argument 'dpi'" by upgrading to img2pdf 0.2
+-  Tweaked the Dockerfiles
+
+
 v3.2:
-=========
+=====
 
 New features
 ------------

ocrmypdf/__init__.py

@@ -16,3 +16,7 @@ class ExitCode(IntEnum):
 def get_program(name):
     envvar = 'OCRMYPDF_' + name.upper()
     return os.environ.get(envvar, name)
+
+
+def page_number(input_file):
+    return int(os.path.basename(input_file)[0:6])

ocrmypdf/ghostscript.py

@@ -7,7 +7,8 @@ from shutil import copy
 from . import get_program
 
 
-def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log):
+def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
+                  pageno=1):
     with NamedTemporaryFile(delete=True) as tmp:
         args_gs = [
             get_program('gs'),
@@ -15,6 +16,8 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log):
             '-dBATCH',
             '-dNOPAUSE',
             '-sDEVICE=%s' % raster_device,
+            '-dFirstPage=%i' % pageno,
+            '-dLastPage=%i' % pageno,
             '-o', tmp.name,
             '-r{0}x{1}'.format(str(xres), str(yres)),
             input_file
@@ -43,6 +46,7 @@ def generate_pdfa(pdf_pages, output_file, threads=1):
             "-dNOPAUSE",
             '-dNumRenderingThreads=' + str(threads),
             "-sDEVICE=pdfwrite",
+            "-dAutoRotatePages=/None",
             "-sColorConversionStrategy=/RGB",
             "-sProcessColorModel=DeviceRGB",
             "-dJPEGQ=95",

ocrmypdf/leptonica.py

@@ -10,11 +10,15 @@
 
 from __future__ import print_function, absolute_import, division
 import argparse
-import ctypes as C
 import sys
 import os
 import logging
 from tempfile import TemporaryFile
+from ctypes.util import find_library
+from .lib._leptonica import ffi
+from functools import lru_cache
+
+lept = ffi.dlopen(find_library('lept'))
 
 logger = logging.getLogger(__name__)
 
@@ -25,67 +29,6 @@ def stderr(*objs):
     print("leptonica.py:", *objs, file=sys.stderr)
 
 
-from ctypes.util import find_library
-lept_lib = find_library('lept')
-if not lept_lib:
-    stderr("Could not find the Leptonica library")
-    sys.exit(3)
-try:
-    lept = C.cdll.LoadLibrary(lept_lib)
-except Exception:
-    stderr("Could not load the Leptonica library from %s", lept_lib)
-    sys.exit(3)
-
-
-class _PIXCOLORMAP(C.Structure):
-    """struct PixColormap from Leptonica src/pix.h
-    """
-
-    _fields_ = [
-        ("array", C.c_void_p),
-        ("depth", C.c_int32),
-        ("nalloc", C.c_int32),
-        ("n", C.c_int32)
-    ]
-
-
-class _PIX(C.Structure):
-    """struct Pix from Leptonica src/pix.h
-    """
-
-    _fields_ = [
-        ("w", C.c_uint32),
-        ("h", C.c_uint32),
-        ("d", C.c_uint32),
-        ("wpl", C.c_uint32),
-        ("refcount", C.c_uint32),
-        ("xres", C.c_int32),
-        ("yres", C.c_int32),
-        ("informat", C.c_int32),
-        ("text", C.POINTER(C.c_char)),
-        ("colormap", C.POINTER(_PIXCOLORMAP)),
-        ("data", C.POINTER(C.c_uint32))
-    ]
-
-
-PIX = C.POINTER(_PIX)
-
-lept.pixRead.argtypes = [C.c_char_p]
-lept.pixRead.restype = PIX
-lept.pixScale.argtypes = [PIX, C.c_float, C.c_float]
-lept.pixScale.restype = PIX
-lept.pixDeskew.argtypes = [PIX, C.c_int32]
-lept.pixDeskew.restype = PIX
-lept.pixFindSkew.argtypes = [PIX, C.POINTER(C.c_float), C.POINTER(C.c_float)]
-lept.pixFindSkew.restype = C.c_int32
-lept.pixWriteImpliedFormat.argtypes = [C.c_char_p, PIX, C.c_int32, C.c_int32]
-lept.pixWriteImpliedFormat.restype = C.c_int32
-lept.pixDestroy.argtypes = [C.POINTER(PIX)]
-lept.pixDestroy.restype = None
-lept.getLeptonicaVersion.argtypes = []
-lept.getLeptonicaVersion.restype = C.c_char_p
-
-
 class LeptonicaErrorTrap(object):
     """Context manager to trap errors reported by Leptonica.
 
@@ -140,103 +83,140 @@ class LeptonicaIOError(LeptonicaError):
     pass
 
 
-def pixRead(filename):
-    """Load an image file into a PIX object.
+class Pix:
+    def __init__(self, cpix):
+        self.cpix = ffi.gc(cpix, Pix._pix_destroy)
 
-    Leptonica can load TIFF, PNM (PBM, PGM, PPM), PNG, and JPEG.  If loading
-    fails then the object will wrap a C null pointer.
-
-    """
-    with LeptonicaErrorTrap():
-        return lept.pixRead(filename.encode(sys.getfilesystemencoding()))
-
-
-def pixScale(pix, scalex, scaley):
-    """Returns the pix object rescaled according to the proportions given."""
-    with LeptonicaErrorTrap():
-        return lept.pixScale(pix, scalex, scaley)
-
-
-def pixDeskew(pix, reduction_factor=0):
-    """Returns the deskewed pix object.
-
-    A clone of the original is returned when the algorithm cannot find a skew
-    angle with sufficient confidence.
-
-    reduction_factor -- amount to downsample (0 for default) when searching
-        for skew angle
-
-    """
-    with LeptonicaErrorTrap():
-        return lept.pixDeskew(pix, reduction_factor)
-
-
-def pixFindSkew(pix):
-    """Returns a tuple (deskew angle in degrees, confidence value).
-
-    Returns (None, None) if no angle is available.
-
-    """
-    with LeptonicaErrorTrap():
-        angle = C.c_float(0.0)
-        confidence = C.c_float(0.0)
-        result = lept.pixFindSkew(pix, C.byref(angle), C.byref(confidence))
-        if result == 0:
-            return (angle.value, confidence.value)
+    def __repr__(self):
+        if self.cpix:
+            s = "<leptonica.Pix image size={0}x{1} depth={2} at 0x{3:x}>"
+            return s.format(self.cpix.w, self.cpix.h, self.cpix.d,
+                            int(ffi.cast("intptr_t", self.cpix)))
         else:
-            return (None, None)
+            return "<leptonica.Pix image NULL>"
 
+    @classmethod
+    def read(cls, filename):
+        """Load an image file into a PIX object.
 
-def pixWriteImpliedFormat(filename, pix, jpeg_quality=0, jpeg_progressive=0):
-    """Write pix to the filename, with the extension indicating format.
+        Leptonica can load TIFF, PNM (PBM, PGM, PPM), PNG, and JPEG.  If
+        loading fails then the object will wrap a C null pointer.
+        """
+        with LeptonicaErrorTrap():
+            return cls(lept.pixRead(
+                filename.encode(sys.getfilesystemencoding())))
 
-    jpeg_quality -- quality (iff JPEG; 1 - 100, 0 for default)
-    jpeg_progressive -- (iff JPEG; 0 for baseline seq., 1 for progressive)
+    def write_implied_format(
+            self, filename, jpeg_quality=0, jpeg_progressive=0):
+        """Write pix to the filename, with the extension indicating format.
 
-    """
-    fileroot, extension = os.path.splitext(filename)
-    fix_pnm = False
-    if extension.lower() in ('.pbm', '.pgm', '.ppm'):
-        # Leptonica does not process handle these extensions correctly, but
-        # does handle .pnm correctly.  Add another .pnm suffix.
-        filename += '.pnm'
-        fix_pnm = True
+        jpeg_quality -- quality (iff JPEG; 1 - 100, 0 for default)
+        jpeg_progressive -- (iff JPEG; 0 for baseline seq., 1 for progressive)
+        """
+        fileroot, extension = os.path.splitext(filename)
+        fix_pnm = False
+        if extension.lower() in ('.pbm', '.pgm', '.ppm'):
+            # Leptonica does not process handle these extensions correctly, but
+            # does handle .pnm correctly.  Add another .pnm suffix.
+            filename += '.pnm'
+            fix_pnm = True
 
-    with LeptonicaErrorTrap():
-        lept.pixWriteImpliedFormat(
-            filename.encode(sys.getfilesystemencoding()),
-            pix, jpeg_quality, jpeg_progressive)
+        with LeptonicaErrorTrap():
+            lept.pixWriteImpliedFormat(
+                filename.encode(sys.getfilesystemencoding()),
+                self.cpix, jpeg_quality, jpeg_progressive)
 
-    if fix_pnm:
-        from shutil import move
-        move(filename, filename[:-4])   # Remove .pnm suffix
+        if fix_pnm:
+            from shutil import move
+            move(filename, filename[:-4])   # Remove .pnm suffix
 
+    def deskew(self, reduction_factor=0):
+        """Returns the deskewed pix object.
 
-def pixDestroy(pix):
-    """Destroy the pix object.
+        A clone of the original is returned when the algorithm cannot find a
+        skew angle with sufficient confidence.
 
-    Function signature is pixDestroy(struct Pix **), hence C.byref() to pass
-    the address of the pointer.
+        reduction_factor -- amount to downsample (0 for default) when searching
+            for skew angle
+        """
+        with LeptonicaErrorTrap():
+            return Pix(lept.pixDeskew(self.cpix, reduction_factor))
 
-    """
-    with LeptonicaErrorTrap():
-        lept.pixDestroy(C.byref(pix))
+    def scale(self, scalex, scaley):
+        "Returns the pix object rescaled according to the proportions given."
+        with LeptonicaErrorTrap():
+            return Pix(lept.pixScale(self.cpix, scalex, scaley))
+
+    def rotate180(self):
+        with LeptonicaErrorTrap():
+            return Pix(lept.pixRotate180(ffi.NULL, self.cpix))
+
+    def find_skew(self):
+        """Returns a tuple (deskew angle in degrees, confidence value).
+
+        Returns (None, None) if no angle is available.
+        """
+        with LeptonicaErrorTrap():
+            angle = ffi.new('float *', 0.0)
+            confidence = ffi.new('float *', 0.0)
+            result = lept.pixFindSkew(self.cpix, angle, confidence)
+            if result == 0:
+                return (angle[0], confidence[0])
+            else:
+                return (None, None)
+
+    @staticmethod
+    def correlation_binary(pix1, pix2):
+        if getLeptonicaVersion() < 'leptonica-1.72':
+            # Older versions of Leptonica (pre-1.72) have a buggy
+            # implementation of pixCorrelationBinary that overflows on larger
+            # images.
+            pix1_count = ffi.new('l_int32 *', 0)
+            pix2_count = ffi.new('l_int32 *', 0)
+            pixn_count = ffi.new('l_int32 *', 0)
+            tab8 = lept.makePixelSumTab8()  # Small memory leak on each call
+
+            lept.pixCountPixels(pix1.cpix, pix1_count, tab8)
+            lept.pixCountPixels(pix2.cpix, pix2_count, tab8)
+            pixn = Pix(lept.pixAnd(ffi.NULL, pix1.cpix, pix2.cpix))
+            lept.pixCountPixels(pixn.cpix, pixn_count, tab8)
+
+            # Python converts these int32s to larger units as needed
+            # to avoid overflow. Overflow happens easily here.
+            correlation = (
+                    (pixn_count[0] * pixn_count[0]) /
+                    (pix1_count[0] * pix2_count[0])
+                    )
+            return correlation
+        else:
+            correlation = ffi.new('float *', 0.0)
+            result = lept.pixCorrelationBinary(pix1.cpix, pix2.cpix,
+                                               correlation)
+            if result != 0:
+                raise LeptonicaError("Correlation failed")
+            return correlation[0]
+
+    @staticmethod
+    def _pix_destroy(pix):
+        ptr_to_pix = ffi.new('PIX **', pix)
+        lept.pixDestroy(ptr_to_pix)
+        # print('pix destroy ' + repr(pix))
 
 
+@lru_cache(maxsize=1)
 def getLeptonicaVersion():
     """Get Leptonica version string.
 
     Caveat: Leptonica expects the caller to free this memory.  We don't,
     since that would involve binding to libc to access libc.free(),
     a pointless effort to reclaim 100 bytes of memory.
-
     """
-    return lept.getLeptonicaVersion().decode()
+    return ffi.string(lept.getLeptonicaVersion()).decode()
 
 
 def deskew(infile, outfile, dpi):
     try:
-        pix_source = pixRead(infile)
+        pix_source = Pix.read(infile)
     except LeptonicaIOError:
         raise LeptonicaIOError("Failed to open file: %s" % infile)
 
@@ -244,14 +224,12 @@ def deskew(infile, outfile, dpi):
         reduction_factor = 1  # Don't downsample too much if DPI is already low
     else:
         reduction_factor = 0  # Use default
-    pix_deskewed = pixDeskew(pix_source, reduction_factor)
+    pix_deskewed = pix_source.deskew(reduction_factor)
 
     try:
-        pixWriteImpliedFormat(outfile, pix_deskewed)
+        pix_deskewed.write_implied_format(outfile)
     except LeptonicaIOError:
         raise LeptonicaIOError("Failed to open destination file: %s" % outfile)
-    pixDestroy(pix_source)
-    pixDestroy(pix_deskewed)
 
 
 if __name__ == '__main__':
@@ -325,7 +303,6 @@ def test_skew_angle():
             rotated_im.save(tmpfile)
             pix = pixRead(tmpfile.name)
             angle, confidence = pixFindSkew(pix)
-            pixDestroy(pix)
             print('{0} {1}  {2}'.format(rotate_angle, angle, confidence), file=sys.stderr)
 
 

ocrmypdf/lib/compile_leptonica.py

@@ -0,0 +1,64 @@
+from cffi import FFI
+
+ffi = FFI()
+ffi.set_source("ocrmypdf.lib._leptonica", None)
+ffi.cdef("""
+typedef signed char             l_int8;
+typedef unsigned char           l_uint8;
+typedef short                   l_int16;
+typedef unsigned short          l_uint16;
+typedef int                     l_int32;
+typedef unsigned int            l_uint32;
+typedef float                   l_float32;
+typedef double                  l_float64;
+typedef long long               l_int64;
+typedef unsigned long long      l_uint64;
+
+struct Pix
+{
+    l_uint32             w;           /* width in pixels                   */
+    l_uint32             h;           /* height in pixels                  */
+    l_uint32             d;           /* depth in bits (bpp)               */
+    l_uint32             spp;         /* number of samples per pixel       */
+    l_uint32             wpl;         /* 32-bit words/line                 */
+    l_uint32             refcount;    /* reference count (1 if no clones)  */
+    l_int32              xres;        /* image res (ppi) in x direction    */
+                                      /* (use 0 if unknown)                */
+    l_int32              yres;        /* image res (ppi) in y direction    */
+                                      /* (use 0 if unknown)                */
+    l_int32              informat;    /* input file format, IFF_*          */
+    l_int32              special;     /* special instructions for I/O, etc */
+    char                *text;        /* text string associated with pix   */
+    struct PixColormap  *colormap;    /* colormap (may be null)            */
+    l_uint32            *data;        /* the image data                    */
+};
+typedef struct Pix PIX;
+
+struct PixColormap
+{
+    void            *array;     /* colormap table (array of RGBA_QUAD)     */
+    l_int32          depth;     /* of pix (1, 2, 4 or 8 bpp)               */
+    l_int32          nalloc;    /* number of color entries allocated       */
+    l_int32          n;         /* number of color entries used            */
+};
+typedef struct PixColormap  PIXCMAP;
+""")
+
+ffi.cdef("""
+PIX * pixRead ( const char *filename );
+PIX * pixScale ( PIX *pixs, l_float32 scalex, l_float32 scaley );
+l_int32 pixFindSkew ( PIX *pixs, l_float32 *pangle, l_float32 *pconf );
+l_int32 pixWriteImpliedFormat ( const char *filename, PIX *pix, l_int32 quality, l_int32 progressive );
+void pixDestroy ( PIX **ppix );
+PIX * pixDeskew ( PIX *pixs, l_int32 redsearch );
+char * getLeptonicaVersion (  );
+l_int32 pixCorrelationBinary(PIX *pix1, PIX *pix2, l_float32 *pval);
+PIX *pixRotate180(PIX *pixd, PIX *pixs);
+
+l_int32 pixCountPixels ( PIX *pix, l_int32 *pcount, l_int32 *tab8 );
+PIX * pixAnd ( PIX *pixd, PIX *pixs1, PIX *pixs2 );
+l_int32 * makePixelSumTab8 ( void );
+""")
+
+if __name__ == '__main__':
+    ffi.compile()

ocrmypdf/main.py

@@ -12,6 +12,7 @@ import multiprocessing
 import atexit
 import textwrap
 import img2pdf
+import logging
 
 import PyPDF2 as pypdf
 from PIL import Image
@@ -22,6 +23,7 @@ from ruffus import transform, suffix, merge, active_if, regex, jobs_limit, \
     formatter, follows, split, collate, check_if_uptodate
 import ruffus.ruffus_exceptions as ruffus_exceptions
 import ruffus.cmdline as cmdline
+import ruffus.proxy_logger as proxy_logger
 
 from .hocrtransform import HocrTransform
 from .pageinfo import pdf_get_all_pageinfo
@@ -29,7 +31,7 @@ from .pdfa import generate_pdfa_def
 from . import ghostscript
 from . import tesseract
 from . import qpdf
-from . import ExitCode
+from . import ExitCode, page_number
 
 import pkg_resources
 
@@ -58,7 +60,6 @@ if tesseract.version() < MINIMUM_TESS_VERSION:
             MINIMUM_TESS_VERSION, tesseract.version()))
     sys.exit(ExitCode.missing_dependency)
 
-
 try:
     import PIL.features
     check_codec = PIL.features.check_codec
@@ -105,7 +106,7 @@ parser = cmdline.get_argparse(
     ignored_args=[
         'touch_files_only', 'recreate_database', 'checksum_file_name',
         'key_legend_in_graph', 'draw_graph_horizontally', 'flowchart_format',
-        'forced_tasks', 'target_tasks', 'use_threads', 'jobs'])
+        'forced_tasks', 'target_tasks', 'use_threads', 'jobs', 'log_file'])
 
 parser.add_argument(
     'input_file',
@@ -122,7 +123,7 @@ parser.add_argument(
 
 metadata = parser.add_argument_group(
     "Metadata options",
-    "Set output PDF/A metadata (default: use input document's title)")
+    "Set output PDF/A metadata (default: use input document's metadata)")
 metadata.add_argument(
     '--title', type=str,
     help="set document title (place multiple words in quotes)")
@@ -131,15 +132,17 @@ metadata.add_argument(
     help="set document author")
 metadata.add_argument(
     '--subject', type=str,
-    help="set document")
+    help="set document subject description")
 metadata.add_argument(
     '--keywords', type=str,
     help="set document keywords")
 
-
 preprocessing = parser.add_argument_group(
     "Preprocessing options",
     "Improve OCR quality and final image")
+preprocessing.add_argument(
+    '-r', '--rotate-pages', action='store_true',
+    help="automatically rotate pages based on detected text orientation")
 preprocessing.add_argument(
     '-d', '--deskew', action='store_true',
     help="deskew each page before performing OCR")
@@ -154,20 +157,20 @@ preprocessing.add_argument(
     help="oversample images to at least the specified DPI, to improve OCR "
          "results slightly")
 
-parser.add_argument(
+ocrsettings = parser.add_argument_group(
+    "OCR options",
+    "Control how OCR is applied")
+ocrsettings.add_argument(
     '-f', '--force-ocr', action='store_true',
     help="rasterize any fonts or vector images on each page and apply OCR")
-parser.add_argument(
+ocrsettings.add_argument(
     '-s', '--skip-text', action='store_true',
     help="skip OCR on any pages that already contain text, but include the"
          " page in final output")
-parser.add_argument(
+ocrsettings.add_argument(
     '--skip-big', type=float, metavar='MPixels',
     help="skip OCR on pages larger than the specified amount of megapixels, "
          "but include skipped pages in final output")
-# parser.add_argument(
-#     '--exact-image', action='store_true',
-#     help="Use original page from PDF without re-rendering")
 
 advanced = parser.add_argument_group(
     "Advanced",
@@ -224,6 +227,13 @@ if not set(options.language).issubset(tesseract.languages()):
 if options.pdf_renderer == 'auto':
     options.pdf_renderer = 'hocr'
 
+if options.pdf_renderer == 'tesseract' and tesseract.version() < '3.04.01' \
+        and os.environ.get('OCRMYPDF_SHARP_TTF', '') != '1':
+    complain(
+        "WARNING: Your version of tesseract has problems with PDF output. "
+        "Some PDF viewers will fail to find searchable text.\n"
+        "--pdf-renderer=tesseract is not recommended.")
+
 if any((options.deskew, options.clean, options.clean_final)):
     try:
         from . import unpaper
@@ -255,12 +265,29 @@ if options.pdf_renderer == 'hocr':
     if not options.deskew and not options.clean_final and not options.force_ocr:
         lossless_reconstruction = True
 
+
 # ----------
 # Logging
 
 
-_logger, _logger_mutex = cmdline.setup_logging(__name__, options.log_file,
-                                               options.verbose)
+def logging_factory(logger_name, listargs):
+    log_file_name, verbose = listargs
+
+    root_logger = logging.getLogger(logger_name)
+    root_logger.setLevel(logging.DEBUG)
+
+    handler = logging.StreamHandler(sys.stderr)
+    formatter_ = logging.Formatter("%(levelname)7s - %(message)s")
+    handler.setFormatter(formatter_)
+    if verbose:
+        handler.setLevel(logging.DEBUG)
+    else:
+        handler.setLevel(logging.INFO)
+    root_logger.addHandler(handler)
+    return root_logger
+
+_logger, _logger_mutex = proxy_logger.make_shared_logger_and_proxy(
+    logging_factory, __name__, [None, options.verbose])
 
 
 class WrappedLogger:
@@ -363,7 +390,7 @@ def repair_pdf(
     qpdf.repair(input_file, output_file, log)
     with pdfinfo_lock:
         pdfinfo.extend(pdf_get_all_pageinfo(output_file))
-        log.info(pdfinfo)
+        log.debug(pdfinfo)
 
 
 def get_pageinfo(input_file, pdfinfo, pdfinfo_lock):
@@ -381,11 +408,11 @@ def is_ocr_required(pageinfo, log):
         # or both. It seems quite unlikely that one would find meaningful text
         # from rasterizing vector content. So skip the page.
         log.info(
-            "Page {0} has no images - skipping OCR".format(page)
+            "{0:4d}: page has no images - skipping OCR".format(page)
         )
         ocr_required = False
     elif pageinfo['has_text']:
-        s = "Page {0} already has text! – {1}"
+        s = "{0:4d}: page already has text! – {1}"
 
         if not options.force_ocr and not options.skip_text:
             log.error(s.format(page,
@@ -404,9 +431,10 @@ def is_ocr_required(pageinfo, log):
         pixel_count = pageinfo['width_pixels'] * pageinfo['height_pixels']
         if pixel_count > (options.skip_big * 1000000):
             ocr_required = False
-            log.info(
-                "Page {0} is very large; skipping due to -b".format(page))
-
+            log.warning(
+                "{0:4d}: page too big, skipping OCR "
+                "({1:.1f} MPixels > {2:.1f} MPixels --skip-big)".format(
+                    page, pixel_count / 1000000, options.skip_big))
     return ocr_required
 
 
@@ -425,7 +453,13 @@ def split_pages(
         with suppress(FileNotFoundError):
             os.unlink(oo)
 
-    npages = qpdf.get_npages(input_file)
+    # If no files were repaired the input will be empty
+    if not input_file:
+        log.error("{0}: file not found or invalid argument".format(
+                options.input_file))
+        sys.exit(ExitCode.input_file)
+
+    npages = qpdf.get_npages(input_file, log)
     qpdf.split_pages(input_file, work_folder, npages)
 
     from glob import glob
@@ -441,9 +475,85 @@ def split_pages(
                 os.path.basename(filename)[0:6] + alt_suffix))
 
 
+@active_if(options.rotate_pages)
 @transform(
     input=split_pages,
-    filter=suffix('.ocr.page.pdf'),
+    filter=suffix('.page.pdf'),
+    output='.preview.jpg',
+    output_dir=work_folder,
+    extras=[_log, _pdfinfo, _pdfinfo_lock])
+def rasterize_preview(
+        input_file,
+        output_file,
+        log,
+        pdfinfo,
+        pdfinfo_lock):
+    ghostscript.rasterize_pdf(
+        input_file=input_file,
+        output_file=output_file,
+        xres=200,
+        yres=200,
+        raster_device='jpeggray',
+        log=log)
+
+
+@collate(
+    input=[split_pages, rasterize_preview],
+    filter=regex(r".*/(\d{6})(\.ocr|\.skip)(?:\.page\.pdf|\.preview\.jpg)"),
+    output=os.path.join(work_folder, r'\1\2.oriented.pdf'),
+    extras=[_log, _pdfinfo, _pdfinfo_lock])
+def orient_page(
+        infiles,
+        output_file,
+        log,
+        pdfinfo,
+        pdfinfo_lock):
+
+    page_pdf = next(ii for ii in infiles if ii.endswith('.page.pdf'))
+
+    if not options.rotate_pages:
+        re_symlink(page_pdf, output_file)
+        return
+    preview = next(ii for ii in infiles if ii.endswith('.preview.jpg'))
+
+    orient_conf = tesseract.get_orientation(
+        preview,
+        language=options.language,
+        timeout=options.tesseract_timeout,
+        log=log)
+
+    direction = {
+        0: '⇧',
+        90: '⇦',
+        180: '⇩',
+        270: '⇨'
+    }
+
+    log.info(
+        '{0:4d}: page is facing {1}, confidence {2:.2f}{3}'.format(
+            page_number(preview),
+            direction.get(orient_conf.angle, '?'),
+            orient_conf.confidence,
+            ' - correcting rotation' if orient_conf.angle != 0 else '')
+    )
+
+    if orient_conf.angle == 0:
+        re_symlink(page_pdf, output_file)
+    else:
+        writer = pypdf.PdfFileWriter()
+        reader = pypdf.PdfFileReader(page_pdf)
+        page = reader.pages[0]
+
+        # Rotate opposite of orientation
+        rotated_page = page.rotateClockwise(orient_conf.angle)
+        writer.addPage(rotated_page)
+        with open(output_file, 'wb') as out:
+            writer.write(out)
+
+
+@transform(
+    input=orient_page,
+    filter=suffix('.ocr.oriented.pdf'),
     output='.page.png',
     output_dir=work_folder,
     extras=[_log, _pdfinfo, _pdfinfo_lock])
@@ -466,7 +576,7 @@ def rasterize_with_ghostscript(
                  for image in pageinfo['images']):
             device = 'pnggray'
 
-    log.debug("Rendering {0} with {1}".format(
+    log.debug("Rasterize {0} with {1}".format(
             os.path.basename(input_file), device))
     xres = max(pageinfo['xres'], options.oversample or 0)
     yres = max(pageinfo['yres'], options.oversample or 0)
@@ -493,7 +603,8 @@ def preprocess_deskew(
     pageinfo = get_pageinfo(input_file, pdfinfo, pdfinfo_lock)
     dpi = int(pageinfo['xres'])
 
-    unpaper.deskew(input_file, output_file, dpi, log)
+    from . import leptonica
+    leptonica.deskew(input_file, output_file, dpi)
 
 
 @transform(
@@ -544,7 +655,6 @@ def ocr_tesseract_hocr(
         )
 
 
-@active_if(options.pdf_renderer == 'hocr')
 @collate(
     input=[rasterize_with_ghostscript, preprocess_deskew, preprocess_clean],
     filter=regex(r".*/(\d{6})(?:\.page|\.pp-deskew|\.pp-clean)\.png"),
@@ -567,15 +677,20 @@ def select_image_for_pdf(
     pageinfo = get_pageinfo(image, pdfinfo, pdfinfo_lock)
     if all(image['enc'] == 'jpeg' for image in pageinfo['images']):
         # If all images were JPEGs originally, produce a JPEG as output
-        Image.open(image).save(output_file, format='JPEG')
+        im = Image.open(image)
+        dpi = im.info.get(
+            'dpi',
+            (int(pageinfo['xres']), int(pageinfo['yres']))
+        )
+        im.save(output_file, format='JPEG', dpi=dpi)
     else:
         re_symlink(image, output_file)
 
 
 @active_if(options.pdf_renderer == 'hocr')
 @collate(
-    input=[select_image_for_pdf, split_pages],
-    filter=regex(r".*/(\d{6})(?:\.image|\.ocr\.page\.pdf)"),
+    input=[select_image_for_pdf, orient_page],
+    filter=regex(r".*/(\d{6})(?:\.image|\.ocr\.oriented\.pdf)"),
     output=os.path.join(work_folder, r'\1.image-layer.pdf'),
     extras=[_log, _pdfinfo, _pdfinfo_lock])
 def select_image_layer(
@@ -585,16 +700,27 @@ def select_image_layer(
         pdfinfo,
         pdfinfo_lock):
 
-    page_pdf = next(ii for ii in infiles if ii.endswith('.page.pdf'))
+    page_pdf = next(ii for ii in infiles if ii.endswith('.ocr.oriented.pdf'))
     image = next(ii for ii in infiles if ii.endswith('.image'))
 
     if lossless_reconstruction:
+        log.debug("{:4d}: page eligible for lossless reconstruction".format(
+            page_number(page_pdf)))
         re_symlink(page_pdf, output_file)
     else:
         pageinfo = get_pageinfo(image, pdfinfo, pdfinfo_lock)
-        dpi = round(max(pageinfo['xres'], pageinfo['yres'], options.oversample))
-        with open(output_file, 'wb') as pdf:
-            img2pdf.convert([image], dpi=dpi, outputstream=pdf)
+        dpi = round(max(pageinfo['xres'], pageinfo['yres'],
+                        options.oversample))
+        imgsize = ((img2pdf.ImgSize.dpi, dpi), (img2pdf.ImgSize.dpi, dpi))
+
+        layout_fun = img2pdf.get_layout_fun(None, imgsize, None, None, None)
+
+        with open(image, 'rb') as imfile, \
+                open(output_file, 'wb') as pdf:
+            rawdata = imfile.read()
+            pdf.write(img2pdf.convert(
+                rawdata, producer="img2pdf", with_pdfrw=False,
+                layout_fun=layout_fun))
 
 
 @active_if(options.pdf_renderer == 'hocr')
@@ -657,15 +783,53 @@ def add_text_layer(
     text = next(ii for ii in infiles if ii.endswith('.hocr.pdf'))
     image = next(ii for ii in infiles if ii.endswith('.image-layer.pdf'))
 
-    pdf_output = pypdf.PdfFileWriter()
-
     pdf_text = pypdf.PdfFileReader(open(text, "rb"))
     pdf_image = pypdf.PdfFileReader(open(image, "rb"))
 
-    page = pdf_text.getPage(0)
-    page.mergePage(pdf_image.getPage(0))
+    page_text = pdf_text.getPage(0)
 
-    pdf_output.addPage(page)
+    # The text page always will be oriented up
+    # but if lossless_reconstruction, pdf_image may have a rotation applied
+    # we can't just merge the pages, because a page can only have one /Rotate
+    # tag, so the differential rotation must be corrected.
+    # Also, pdf_image may not have its mediabox nailed to (0, 0)
+    page_image = pdf_image.getPage(0)
+    rotation = page_image.get('/Rotate', 0)
+
+    # /Rotate is a clockwise rotation: 90 means page facing "east"
+    # The negative of this value is the angle that eliminates that rotation
+    rotation = -rotation % 360
+
+    x1 = page_image.mediaBox.getLowerLeft_x()
+    x2 = page_image.mediaBox.getUpperRight_x()
+    y1 = page_image.mediaBox.getLowerLeft_y()
+    y2 = page_image.mediaBox.getUpperRight_y()
+
+    # Rotation occurs about the page's (0, 0). Most pages will have the media
+    # box at (0, 0) will all content in the first quadrant but some cropped
+    # files may have an offset mediabox. We translate the page so that its
+    # bottom left corner after rotation is pinned to (0, 0) with the image
+    # in the first quadrant.
+    if rotation == 0:
+        tx, ty = -x1, -y1
+    elif rotation == 90:
+        tx, ty = y2, -x1
+    elif rotation == 180:
+        tx, ty = x2, y2
+    elif rotation == 270:
+        tx, ty = -y1, x2
+    else:
+        pass
+
+    if rotation != 0:
+        log.info("{0:4d}: rotating image layer {1} degrees".format(
+            page_number(image), rotation, tx, ty))
+
+    page_text.mergeRotatedScaledTranslatedPage(
+        page_image, rotation, 1.0, tx, ty, expand=False)
+
+    pdf_output = pypdf.PdfFileWriter()
+    pdf_output.addPage(page_text)
 
     with open(output_file, "wb") as out:
         pdf_output.write(out)
@@ -673,8 +837,8 @@ def add_text_layer(
 
 @active_if(options.pdf_renderer == 'tesseract')
 @collate(
-    input=[preprocess_clean, split_pages],
-    filter=regex(r".*/(\d{6})(?:\.pp-clean\.png|\.page\.pdf)"),
+    input=[select_image_for_pdf, orient_page],
+    filter=regex(r".*/(\d{6})(?:\.image|\.ocr\.oriented\.pdf)"),
     output=os.path.join(work_folder, r'\1.rendered.pdf'),
     extras=[_log, _pdfinfo, _pdfinfo_lock])
 def tesseract_ocr_and_render_pdf(
@@ -684,7 +848,7 @@ def tesseract_ocr_and_render_pdf(
         pdfinfo,
         pdfinfo_lock):
 
-    input_image = next((ii for ii in input_files if ii.endswith('.png')), '')
+    input_image = next((ii for ii in input_files if ii.endswith('.image')), '')
     input_pdf = next((ii for ii in input_files if ii.endswith('.pdf')))
     if not input_image:
         # Skipping this page
@@ -748,8 +912,8 @@ def generate_postscript_stub(
 
 
 @transform(
-    input=split_pages,
-    filter=suffix('.skip.page.pdf'),
+    input=orient_page,
+    filter=suffix('.skip.oriented.pdf'),
     output='.done.pdf',
     output_dir=work_folder,
     extras=[_log])
@@ -757,6 +921,10 @@ def skip_page(
         input_file,
         output_file,
         log):
+    # The purpose of this step is its filter to forward only the skipped
+    # files (.skip.oriented.pdf) while disregarding the processed ones
+    # (.ocr.oriented.pdf).  Alternative would be for merge_pages to filter
+    # pages itself if it gets multiple copies of a page.
     re_symlink(input_file, output_file, log)
 
 
@@ -786,7 +954,7 @@ def merge_pages(
         return key
 
     pdf_pages = sorted(input_files, key=input_file_order)
-    log.info(pdf_pages)
+    log.debug("Final pages: " + "\n".join(pdf_pages))
     ghostscript.generate_pdfa(pdf_pages, output_file, options.jobs or 1)
 
 

ocrmypdf/pageinfo.py

@@ -136,8 +136,8 @@ def _pdf_get_pageinfo(infile, pageno: int):
 
     pageinfo['has_text'] = _page_has_text(pdf, page)
 
-    width_pt = page['/MediaBox'][2] - page['/MediaBox'][0]
-    height_pt = page['/MediaBox'][3] - page['/MediaBox'][1]
+    width_pt = page.mediaBox.getWidth()
+    height_pt = page.mediaBox.getHeight()
     pageinfo['width_inches'] = width_pt / Decimal(72.0)
     pageinfo['height_inches'] = height_pt / Decimal(72.0)
 

ocrmypdf/qpdf.py

@@ -19,15 +19,15 @@ def check(input_file, log):
         check_output(args_qpdf, stderr=STDOUT, universal_newlines=True)
     except CalledProcessError as e:
         if e.returncode == 2:
-            print("{0}: not a valid PDF, and could not repair it.".format(
+            log.error("{0}: not a valid PDF, and could not repair it.".format(
                     input_file))
-            print("Details:")
-            print(e.output)
+            log.error("Details:")
+            log.error(e.output)
         elif e.returncode == 3:
             log.info("qpdf --check returned warnings:")
             log.info(e.output)
         else:
-            print(e.output)
+            log.warning(e.output)
         return False
     return True
 
@@ -40,32 +40,35 @@ def repair(input_file, output_file, log):
         check_output(args_qpdf, stderr=STDOUT, universal_newlines=True)
     except CalledProcessError as e:
         if e.returncode == 3 and e.output.find("operation succeeded"):
-            log.debug('qpdf found and fixed errors:')
+            log.debug('qpdf found and fixed errors: ' + e.output)
             log.debug(e.output)
-            print(e.output)
             return
 
         if e.returncode == 2 and e.output.find("invalid password"):
-            print("{0}: this PDF is password-protected - password must "
-                  "be removed for OCR".format(input_file))
+            log.error("{0}: this PDF is password-protected - password must "
+                      "be removed for OCR".format(input_file))
             sys.exit(ExitCode.input_file)
         elif e.returncode == 2:
-            print("{0}: not a valid PDF, and could not repair it.".format(
-                    input_file))
-            print("Details:")
-            print(e.output)
+            log.error("{0}: not a valid PDF, and could not repair it.".format(
+                      input_file))
+            log.error("Details: " + e.output)
             sys.exit(ExitCode.input_file)
         else:
-            print("{0}: unknown error".format(
-                    input_file))
-            print(e.output)
+            log.error("{0}: unknown error".format(
+                      input_file))
+            log.error(e.output)
             sys.exit(ExitCode.unknown)
 
 
-def get_npages(input_file):
-    pages = check_output(
-        [get_program('qpdf'), '--show-npages', input_file],
-         universal_newlines=True, close_fds=True)
+def get_npages(input_file, log):
+    try:
+        pages = check_output(
+            [get_program('qpdf'), '--show-npages', input_file],
+             universal_newlines=True, close_fds=True)
+    except CalledProcessError as e:
+        if e.returncode == 2 and e.output.find('No such file'):
+            log.error(e.output)
+            sys.exit(ExitCode.input_file)
     return int(pages)
 
 

ocrmypdf/tesseract.py

@@ -6,7 +6,8 @@ import os
 import re
 import shutil
 from functools import lru_cache
-from . import ExitCode, get_program
+from . import ExitCode, get_program, page_number
+from collections import namedtuple
 
 from subprocess import Popen, PIPE, CalledProcessError, \
     TimeoutExpired, check_output, STDOUT
@@ -16,6 +17,10 @@ except ImportError:
     DEVNULL = open(os.devnull, 'wb')
 
 
+OrientationConfidence = namedtuple(
+    'OrientationConfidence',
+    ('angle', 'confidence'))
+
 HOCR_TEMPLATE = '''<?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
@@ -76,6 +81,53 @@ def languages():
     return set(lang.strip() for lang in langs.splitlines()[1:])
 
 
+def get_orientation(input_file, language: list, timeout: float, log):
+    args_tesseract = [
+        get_program('tesseract'),
+        '-l', '+'.join(language),
+        '-psm', '0',
+        input_file,
+        'stdout'
+    ]
+
+    p = Popen(args_tesseract, close_fds=True, stdout=PIPE, stderr=STDOUT,
+              universal_newlines=True)
+    try:
+        stdout, _ = p.communicate(timeout=timeout)
+    except TimeoutExpired:
+        p.kill()
+        stdout, _ = p.communicate()
+        return OrientationConfidence(angle=0, confidence=0.0)
+    else:
+        osd = {}
+        for line in stdout.splitlines():
+            line = line.strip()
+            parts = line.split(':', maxsplit=2)
+            if len(parts) == 2:
+                osd[parts[0].strip()] = parts[1].strip()
+
+        oc = OrientationConfidence(
+            angle=int(osd['Orientation in degrees']),
+            confidence=float(osd['Orientation confidence']))
+        return oc
+
+
+def tesseract_log_output(log, stdout, input_file):
+    lines = stdout.splitlines()
+    prefix = "{0:4d}: [tesseract] ".format(page_number(input_file))
+    for line in lines:
+        if line.startswith("Tesseract Open Source"):
+            continue
+        elif line.startswith("Warning in pixReadMem"):
+            continue
+        elif 'diacritics' in line:
+            log.warning(prefix + "lots of diacritics - possibly poor OCR")
+        elif line.startswith('OSD: Weak margin'):
+            log.warning(prefix + "unsure about page orientation")
+        else:
+            log.info(prefix + line.strip())
+
+
 def generate_hocr(input_file, output_hocr, language: list, tessconfig: list,
                   timeout: float, pageinfo_getter, pagesegmode: int, log):
 
@@ -94,13 +146,13 @@ def generate_hocr(input_file, output_hocr, language: list, tessconfig: list,
         badxml,
         'hocr'
     ] + tessconfig)
-    p = Popen(args_tesseract, close_fds=True, stdout=PIPE, stderr=PIPE,
+    p = Popen(args_tesseract, close_fds=True, stdout=PIPE, stderr=STDOUT,
               universal_newlines=True)
     try:
-        stdout, stderr = p.communicate(timeout=timeout)
+        stdout, _ = p.communicate(timeout=timeout)
     except TimeoutExpired:
         p.kill()
-        stdout, stderr = p.communicate()
+        stdout, _ = p.communicate()
         # Generate a HOCR file with no recognized text if tesseract times out
         # Temporary workaround to hocrTransform not being able to function if
         # it does not have a valid hOCR file.
@@ -110,11 +162,7 @@ def generate_hocr(input_file, output_hocr, language: list, tessconfig: list,
                 pageinfo['width_pixels'],
                 pageinfo['height_pixels']))
     else:
-        if stdout:
-            log.info(stdout)
-        if stderr:
-            log.error(stderr)
-
+        tesseract_log_output(log, stdout, input_file)
         if p.returncode != 0:
             raise CalledProcessError(p.returncode, args_tesseract)
 
@@ -165,17 +213,14 @@ def generate_pdf(input_image, skip_pdf, output_pdf, language: list,
         os.path.splitext(output_pdf)[0],  # Tesseract appends suffix
         'pdf'
     ] + tessconfig)
-    p = Popen(args_tesseract, close_fds=True, stdout=PIPE, stderr=PIPE,
+    p = Popen(args_tesseract, close_fds=True, stdout=PIPE, stderr=STDOUT,
               universal_newlines=True)
 
     try:
-        stdout, stderr = p.communicate(timeout=timeout)
-        if stdout:
-            log.info(stdout)
-        if stderr:
-            log.error(stderr)
+        stdout, _ = p.communicate()
     except TimeoutExpired:
         p.kill()
         log.info("Tesseract - page timed out")
         shutil.copy(skip_pdf, output_pdf)
-
+    else:
+        tesseract_log_output(log, stdout, input_image)

pipeline.svg

@@ -4,263 +4,294 @@
 <!-- Generated by graphviz version 2.38.0 (20140413.2041)
  -->
 <!-- Title: Pipeline: Pages: 1 -->
-<svg width="1132pt" height="708pt"
- viewBox="0.00 0.00 1132.00 708.08" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
-<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 704.083)">
+<svg width="1452pt" height="824pt"
+ viewBox="0.00 0.00 1452.00 824.08" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 820.083)">
 <title>Pipeline:</title>
-<polygon fill="white" stroke="none" points="-4,4 -4,-704.083 1128,-704.083 1128,4 -4,4"/>
+<polygon fill="white" stroke="none" points="-4,4 -4,-820.083 1448,-820.083 1448,4 -4,4"/>
 <g id="clust1" class="cluster"><title>clustertasks</title>
-<polygon fill="none" stroke="black" points="8,-8 8,-692.083 1116,-692.083 1116,-8 8,-8"/>
-<text text-anchor="middle" x="562" y="-664.083" font-family="Times,serif" font-size="30.00" fill="#ff3232">Pipeline:</text>
+<polygon fill="none" stroke="black" points="8,-8 8,-808.083 1436,-808.083 1436,-8 8,-8"/>
+<text text-anchor="middle" x="722" y="-780.083" font-family="Times,serif" font-size="30.00" fill="#ff3232">Pipeline:</text>
 </g>
 <!-- t0 -->
 <g id="node1" class="node"><title>t0</title>
-<polygon fill="#efa03b" stroke="black" points="936.535,-646.083 713.465,-646.083 709.465,-642.083 709.465,-610.083 932.535,-610.083 936.535,-614.083 936.535,-646.083"/>
-<polyline fill="none" stroke="black" points="932.535,-642.083 709.465,-642.083 "/>
-<polyline fill="none" stroke="black" points="932.535,-642.083 932.535,-610.083 "/>
-<polyline fill="none" stroke="black" points="932.535,-642.083 936.535,-646.083 "/>
-<text text-anchor="middle" x="823" y="-622.083" font-family="Times,serif" font-size="20.00">repair_pdf</text>
+<polygon fill="#efa03b" stroke="black" points="1256.54,-762.083 1033.46,-762.083 1029.46,-758.083 1029.46,-726.083 1252.54,-726.083 1256.54,-730.083 1256.54,-762.083"/>
+<polyline fill="none" stroke="black" points="1252.54,-758.083 1029.46,-758.083 "/>
+<polyline fill="none" stroke="black" points="1252.54,-758.083 1252.54,-726.083 "/>
+<polyline fill="none" stroke="black" points="1252.54,-758.083 1256.54,-762.083 "/>
+<text text-anchor="middle" x="1143" y="-738.083" font-family="Times,serif" font-size="20.00">repair_pdf</text>
 </g>
 <!-- t1 -->
 <g id="node2" class="node"><title>t1</title>
-<polygon fill="#efa03b" stroke="black" points="914.112,-567.155 710,-584.057 505.888,-567.155 506.078,-539.806 913.922,-539.806 914.112,-567.155"/>
-<polygon fill="none" stroke="black" points="918.134,-570.834 710,-588.069 501.866,-570.834 502.11,-535.808 917.89,-535.808 918.134,-570.834"/>
-<text text-anchor="middle" x="710" y="-553.596" font-family="Times,serif" font-size="20.00">split_pages</text>
+<polygon fill="#efa03b" stroke="black" points="1234.11,-683.155 1030,-700.057 825.888,-683.155 826.078,-655.806 1233.92,-655.806 1234.11,-683.155"/>
+<polygon fill="none" stroke="black" points="1238.13,-686.834 1030,-704.069 821.866,-686.834 822.11,-651.808 1237.89,-651.808 1238.13,-686.834"/>
+<text text-anchor="middle" x="1030" y="-669.596" font-family="Times,serif" font-size="20.00">split_pages</text>
 </g>
 <!-- t0&#45;&gt;t1 -->
 <g id="edge1" class="edge"><title>t0&#45;&gt;t1</title>
-<path fill="none" stroke="#0044a0" d="M793.9,-609.961C783.582,-603.89 771.656,-596.873 760.092,-590.069"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="761.747,-586.982 751.353,-584.927 758.197,-593.015 761.747,-586.982"/>
+<path fill="none" stroke="#0044a0" d="M1113.9,-725.961C1103.58,-719.89 1091.66,-712.873 1080.09,-706.069"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1081.75,-702.982 1071.35,-700.927 1078.2,-709.015 1081.75,-702.982"/>
 </g>
-<!-- t12 -->
-<g id="node14" class="node"><title>t12</title>
-<polygon fill="#efa03b" stroke="black" points="1108.08,-509.109 769.918,-509.109 765.918,-505.109 765.918,-473.109 1104.08,-473.109 1108.08,-477.109 1108.08,-509.109"/>
-<polyline fill="none" stroke="black" points="1104.08,-505.109 765.918,-505.109 "/>
-<polyline fill="none" stroke="black" points="1104.08,-505.109 1104.08,-473.109 "/>
-<polyline fill="none" stroke="black" points="1104.08,-505.109 1108.08,-509.109 "/>
-<text text-anchor="middle" x="937" y="-485.109" font-family="Times,serif" font-size="20.00">generate_postscript_stub</text>
+<!-- t14 -->
+<g id="node16" class="node"><title>t14</title>
+<polygon fill="#efa03b" stroke="black" points="1428.08,-625.109 1089.92,-625.109 1085.92,-621.109 1085.92,-589.109 1424.08,-589.109 1428.08,-593.109 1428.08,-625.109"/>
+<polyline fill="none" stroke="black" points="1424.08,-621.109 1085.92,-621.109 "/>
+<polyline fill="none" stroke="black" points="1424.08,-621.109 1424.08,-589.109 "/>
+<polyline fill="none" stroke="black" points="1424.08,-621.109 1428.08,-625.109 "/>
+<text text-anchor="middle" x="1257" y="-601.109" font-family="Times,serif" font-size="20.00">generate_postscript_stub</text>
 </g>
-<!-- t0&#45;&gt;t12 -->
-<g id="edge19" class="edge"><title>t0&#45;&gt;t12</title>
-<path fill="none" stroke="#0044a0" d="M899.32,-610.004C909.979,-604.592 919.748,-597.466 927,-588.083 941.916,-568.78 943.099,-540.385 941.345,-519.486"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="944.8,-518.881 940.218,-509.328 937.843,-519.653 944.8,-518.881"/>
+<!-- t0&#45;&gt;t14 -->
+<g id="edge22" class="edge"><title>t0&#45;&gt;t14</title>
+<path fill="none" stroke="#0044a0" d="M1219.32,-726.004C1229.98,-720.592 1239.75,-713.466 1247,-704.083 1261.92,-684.78 1263.1,-656.385 1261.35,-635.486"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1264.8,-634.881 1260.22,-625.328 1257.84,-635.653 1264.8,-634.881"/>
 </g>
 <!-- t2 -->
 <g id="node3" class="node"><title>t2</title>
-<polygon fill="#efa03b" stroke="black" points="592.299,-509.109 241.701,-509.109 237.701,-505.109 237.701,-473.109 588.299,-473.109 592.299,-477.109 592.299,-509.109"/>
-<polyline fill="none" stroke="black" points="588.299,-505.109 237.701,-505.109 "/>
-<polyline fill="none" stroke="black" points="588.299,-505.109 588.299,-473.109 "/>
-<polyline fill="none" stroke="black" points="588.299,-505.109 592.299,-509.109 "/>
-<text text-anchor="middle" x="415" y="-485.109" font-family="Times,serif" font-size="20.00">rasterize_with_ghostscript</text>
+<polygon fill="#efa03b" stroke="black" points="1029.88,-625.109 748.117,-625.109 744.117,-621.109 744.117,-589.109 1025.88,-589.109 1029.88,-593.109 1029.88,-625.109"/>
+<polyline fill="none" stroke="black" points="1025.88,-621.109 744.117,-621.109 "/>
+<polyline fill="none" stroke="black" points="1025.88,-621.109 1025.88,-589.109 "/>
+<polyline fill="none" stroke="black" points="1025.88,-621.109 1029.88,-625.109 "/>
+<text text-anchor="middle" x="887" y="-601.109" font-family="Times,serif" font-size="20.00">rasterize_preview</text>
 </g>
 <!-- t1&#45;&gt;t2 -->
 <g id="edge2" class="edge"><title>t1&#45;&gt;t2</title>
-<path fill="none" stroke="#0044a0" d="M608.89,-535.808C573.672,-527.87 534.524,-519.048 500.679,-511.42"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="501.345,-507.982 490.82,-509.198 499.806,-514.811 501.345,-507.982"/>
+<path fill="none" stroke="#0044a0" d="M980.788,-651.715C965.266,-644.498 948.187,-636.558 932.905,-629.452"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="934.195,-626.193 923.652,-625.15 931.244,-632.54 934.195,-626.193"/>
 </g>
-<!-- t7 -->
-<g id="node9" class="node"><title>t7</title>
+<!-- t3 -->
+<g id="node4" class="node"><title>t3</title>
+<polygon fill="#efa03b" stroke="black" points="1128.15,-567.109 893.852,-567.109 889.852,-563.109 889.852,-531.109 1124.15,-531.109 1128.15,-535.109 1128.15,-567.109"/>
+<polyline fill="none" stroke="black" points="1124.15,-563.109 889.852,-563.109 "/>
+<polyline fill="none" stroke="black" points="1124.15,-563.109 1124.15,-531.109 "/>
+<polyline fill="none" stroke="black" points="1124.15,-563.109 1128.15,-567.109 "/>
+<text text-anchor="middle" x="1009" y="-543.109" font-family="Times,serif" font-size="20.00">orient_page</text>
+</g>
+<!-- t1&#45;&gt;t3 -->
+<g id="edge4" class="edge"><title>t1&#45;&gt;t3</title>
+<path fill="none" stroke="#0044a0" d="M1037.63,-651.508C1042.21,-633.96 1045.92,-609.455 1039,-589.109 1037.4,-584.404 1034.98,-579.822 1032.19,-575.552"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1034.85,-573.258 1026.1,-567.282 1029.21,-577.409 1034.85,-573.258"/>
+</g>
+<!-- t2&#45;&gt;t3 -->
+<g id="edge3" class="edge"><title>t2&#45;&gt;t3</title>
+<path fill="none" stroke="#0044a0" d="M924.267,-589.003C936.328,-583.467 949.857,-577.257 962.502,-571.453"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="964.333,-574.463 971.961,-567.111 961.413,-568.102 964.333,-574.463"/>
+</g>
+<!-- t4 -->
+<g id="node5" class="node"><title>t4</title>
+<polygon fill="#efa03b" stroke="black" points="1027.3,-509.109 676.701,-509.109 672.701,-505.109 672.701,-473.109 1023.3,-473.109 1027.3,-477.109 1027.3,-509.109"/>
+<polyline fill="none" stroke="black" points="1023.3,-505.109 672.701,-505.109 "/>
+<polyline fill="none" stroke="black" points="1023.3,-505.109 1023.3,-473.109 "/>
+<polyline fill="none" stroke="black" points="1023.3,-505.109 1027.3,-509.109 "/>
+<text text-anchor="middle" x="850" y="-485.109" font-family="Times,serif" font-size="20.00">rasterize_with_ghostscript</text>
+</g>
+<!-- t3&#45;&gt;t4 -->
+<g id="edge5" class="edge"><title>t3&#45;&gt;t4</title>
+<path fill="none" stroke="#0044a0" d="M960.431,-531.003C943.787,-525.141 924.999,-518.524 907.704,-512.433"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="908.867,-509.132 898.272,-509.111 906.541,-515.734 908.867,-509.132"/>
+</g>
+<!-- t9 -->
+<g id="node11" class="node"><title>t9</title>
 <polygon fill="#efa03b" stroke="black" points="314.109,-277.109 19.8906,-277.109 15.8906,-273.109 15.8906,-241.109 310.109,-241.109 314.109,-245.109 314.109,-277.109"/>
 <polyline fill="none" stroke="black" points="310.109,-273.109 15.8906,-273.109 "/>
 <polyline fill="none" stroke="black" points="310.109,-273.109 310.109,-241.109 "/>
 <polyline fill="none" stroke="black" points="310.109,-273.109 314.109,-277.109 "/>
 <text text-anchor="middle" x="165" y="-253.109" font-family="Times,serif" font-size="20.00">select_image_layer</text>
 </g>
-<!-- t1&#45;&gt;t7 -->
-<g id="edge11" class="edge"><title>t1&#45;&gt;t7</title>
-<path fill="none" stroke="#0044a0" d="M501.985,-548.028C416.651,-540.897 317.351,-528.985 229,-509.109 131.492,-487.174 17,-534.054 17,-434.109 17,-434.109 17,-434.109 17,-374.109 17,-340.481 4.97908,-324.525 27,-299.109 32.8004,-292.415 39.6539,-286.828 47.1559,-282.17"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="49.1993,-285.038 56.2361,-277.118 45.7959,-278.921 49.1993,-285.038"/>
+<!-- t3&#45;&gt;t9 -->
+<g id="edge14" class="edge"><title>t3&#45;&gt;t9</title>
+<path fill="none" stroke="#0044a0" d="M889.675,-542.297C675.505,-530.255 248,-498.192 248,-434.109 248,-434.109 248,-434.109 248,-374.109 248,-337.597 219.83,-304.936 196.312,-284.031"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="198.437,-281.243 188.562,-277.404 193.888,-286.563 198.437,-281.243"/>
+</g>
+<!-- t15 -->
+<g id="node14" class="node"><title>t15</title>
+<polygon fill="#efa03b" stroke="black" points="1391.34,-451.109 1170.66,-451.109 1166.66,-447.109 1166.66,-415.109 1387.34,-415.109 1391.34,-419.109 1391.34,-451.109"/>
+<polyline fill="none" stroke="black" points="1387.34,-447.109 1166.66,-447.109 "/>
+<polyline fill="none" stroke="black" points="1387.34,-447.109 1387.34,-415.109 "/>
+<polyline fill="none" stroke="black" points="1387.34,-447.109 1391.34,-451.109 "/>
+<text text-anchor="middle" x="1279" y="-427.109" font-family="Times,serif" font-size="20.00">skip_page</text>
+</g>
+<!-- t3&#45;&gt;t15 -->
+<g id="edge19" class="edge"><title>t3&#45;&gt;t15</title>
+<path fill="none" stroke="#0044a0" d="M1060.32,-531.078C1078.95,-524.591 1100.07,-516.881 1119,-509.109 1159.3,-492.568 1204.11,-471.304 1236.04,-455.645"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1237.67,-458.74 1245.1,-451.181 1234.58,-452.461 1237.67,-458.74"/>
 </g>
 <!-- t13 -->
-<g id="node12" class="node"><title>t13</title>
-<polygon fill="#efa03b" stroke="black" points="1029.34,-451.109 808.662,-451.109 804.662,-447.109 804.662,-415.109 1025.34,-415.109 1029.34,-419.109 1029.34,-451.109"/>
-<polyline fill="none" stroke="black" points="1025.34,-447.109 804.662,-447.109 "/>
-<polyline fill="none" stroke="black" points="1025.34,-447.109 1025.34,-415.109 "/>
-<polyline fill="none" stroke="black" points="1025.34,-447.109 1029.34,-451.109 "/>
-<text text-anchor="middle" x="917" y="-427.109" font-family="Times,serif" font-size="20.00">skip_page</text>
+<g id="node15" class="node"><title>t13</title>
+<polygon fill="#efa03b" stroke="black" points="1388.24,-277.109 1007.76,-277.109 1003.76,-273.109 1003.76,-241.109 1384.24,-241.109 1388.24,-245.109 1388.24,-277.109"/>
+<polyline fill="none" stroke="black" points="1384.24,-273.109 1003.76,-273.109 "/>
+<polyline fill="none" stroke="black" points="1384.24,-273.109 1384.24,-241.109 "/>
+<polyline fill="none" stroke="black" points="1384.24,-273.109 1388.24,-277.109 "/>
+<text text-anchor="middle" x="1196" y="-253.109" font-family="Times,serif" font-size="20.00">tesseract_ocr_and_render_pdf</text>
 </g>
-<!-- t1&#45;&gt;t13 -->
-<g id="edge16" class="edge"><title>t1&#45;&gt;t13</title>
-<path fill="none" stroke="#0044a0" d="M716.849,-535.484C723.825,-515.965 736.562,-488.763 757,-473.109 768.342,-464.422 781.352,-457.641 794.947,-452.352"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="796.223,-455.612 804.44,-448.923 793.845,-449.029 796.223,-455.612"/>
-</g>
-<!-- t11 -->
-<g id="node13" class="node"><title>t11</title>
-<polygon fill="#efa03b" stroke="black" points="1068.24,-335.109 687.76,-335.109 683.76,-331.109 683.76,-299.109 1064.24,-299.109 1068.24,-303.109 1068.24,-335.109"/>
-<polyline fill="none" stroke="black" points="1064.24,-331.109 683.76,-331.109 "/>
-<polyline fill="none" stroke="black" points="1064.24,-331.109 1064.24,-299.109 "/>
-<polyline fill="none" stroke="black" points="1064.24,-331.109 1068.24,-335.109 "/>
-<text text-anchor="middle" x="876" y="-311.109" font-family="Times,serif" font-size="20.00">tesseract_ocr_and_render_pdf</text>
-</g>
-<!-- t1&#45;&gt;t11 -->
-<g id="edge18" class="edge"><title>t1&#45;&gt;t11</title>
-<path fill="none" stroke="#0044a0" d="M710.717,-535.657C712.098,-517.744 715.879,-492.693 726,-473.109 754.651,-417.673 809.56,-368.731 844.403,-341.318"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="846.607,-344.038 852.372,-335.148 842.322,-338.503 846.607,-344.038"/>
-</g>
-<!-- t3 -->
-<g id="node4" class="node"><title>t3</title>
-<polygon fill="#efa03b" stroke="black" points="564.742,-451.109 269.258,-451.109 265.258,-447.109 265.258,-415.109 560.742,-415.109 564.742,-419.109 564.742,-451.109"/>
-<polyline fill="none" stroke="black" points="560.742,-447.109 265.258,-447.109 "/>
-<polyline fill="none" stroke="black" points="560.742,-447.109 560.742,-415.109 "/>
-<polyline fill="none" stroke="black" points="560.742,-447.109 564.742,-451.109 "/>
-<text text-anchor="middle" x="415" y="-427.109" font-family="Times,serif" font-size="20.00">preprocess_deskew</text>
-</g>
-<!-- t2&#45;&gt;t3 -->
-<g id="edge3" class="edge"><title>t2&#45;&gt;t3</title>
-<path fill="none" stroke="#0044a0" d="M415,-473.003C415,-469.312 415,-465.322 415,-461.352"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="418.5,-461.111 415,-451.111 411.5,-461.111 418.5,-461.111"/>
-</g>
-<!-- t6 -->
-<g id="node8" class="node"><title>t6</title>
-<polygon fill="#efa03b" stroke="black" points="354.119,-335.109 39.8808,-335.109 35.8808,-331.109 35.8808,-299.109 350.119,-299.109 354.119,-303.109 354.119,-335.109"/>
-<polyline fill="none" stroke="black" points="350.119,-331.109 35.8808,-331.109 "/>
-<polyline fill="none" stroke="black" points="350.119,-331.109 350.119,-299.109 "/>
-<polyline fill="none" stroke="black" points="350.119,-331.109 354.119,-335.109 "/>
-<text text-anchor="middle" x="195" y="-311.109" font-family="Times,serif" font-size="20.00">select_image_for_pdf</text>
-</g>
-<!-- t2&#45;&gt;t6 -->
-<g id="edge9" class="edge"><title>t2&#45;&gt;t6</title>
-<path fill="none" stroke="#0044a0" d="M294.315,-473.06C280.461,-467.585 267.283,-460.434 256,-451.109 223.245,-424.039 207.251,-375.559 200.099,-345.207"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="203.486,-344.31 197.925,-335.292 196.648,-345.809 203.486,-344.31"/>
-</g>
-<!-- t4 -->
-<g id="node5" class="node"><title>t4</title>
-<polygon fill="#efa03b" stroke="black" points="587.95,-393.109 310.05,-393.109 306.05,-389.109 306.05,-357.109 583.95,-357.109 587.95,-361.109 587.95,-393.109"/>
-<polyline fill="none" stroke="black" points="583.95,-389.109 306.05,-389.109 "/>
-<polyline fill="none" stroke="black" points="583.95,-389.109 583.95,-357.109 "/>
-<polyline fill="none" stroke="black" points="583.95,-389.109 587.95,-393.109 "/>
-<text text-anchor="middle" x="447" y="-369.109" font-family="Times,serif" font-size="20.00">preprocess_clean</text>
-</g>
-<!-- t3&#45;&gt;t4 -->
-<g id="edge4" class="edge"><title>t3&#45;&gt;t4</title>
-<path fill="none" stroke="#0044a0" d="M424.775,-415.003C427.132,-410.878 429.703,-406.379 432.232,-401.952"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="435.362,-403.53 437.285,-393.111 429.285,-400.057 435.362,-403.53"/>
-</g>
-<!-- t3&#45;&gt;t6 -->
-<g id="edge8" class="edge"><title>t3&#45;&gt;t6</title>
-<path fill="none" stroke="#0044a0" d="M351.041,-415.02C333.086,-409.151 313.871,-401.82 297,-393.109 269.828,-379.08 242.065,-358.149 222.377,-341.96"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="224.378,-339.071 214.46,-335.347 219.891,-344.444 224.378,-339.071"/>
+<!-- t3&#45;&gt;t13 -->
+<g id="edge21" class="edge"><title>t3&#45;&gt;t13</title>
+<path fill="none" stroke="#0044a0" d="M1040.79,-531.094C1071.69,-511.983 1114,-477.834 1114,-434.109 1114,-434.109 1114,-434.109 1114,-374.109 1114,-337.749 1141.83,-305.063 1165.07,-284.11"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1167.46,-286.664 1172.72,-277.466 1162.87,-281.377 1167.46,-286.664"/>
 </g>
 <!-- t5 -->
 <g id="node6" class="node"><title>t5</title>
-<polygon fill="#efa03b" stroke="black" points="665.666,-335.109 376.334,-335.109 372.334,-331.109 372.334,-299.109 661.666,-299.109 665.666,-303.109 665.666,-335.109"/>
-<polyline fill="none" stroke="black" points="661.666,-331.109 372.334,-331.109 "/>
-<polyline fill="none" stroke="black" points="661.666,-331.109 661.666,-299.109 "/>
-<polyline fill="none" stroke="black" points="661.666,-331.109 665.666,-335.109 "/>
-<text text-anchor="middle" x="519" y="-311.109" font-family="Times,serif" font-size="20.00">ocr_tesseract_hocr</text>
+<polygon fill="#efa03b" stroke="black" points="830.742,-451.109 535.258,-451.109 531.258,-447.109 531.258,-415.109 826.742,-415.109 830.742,-419.109 830.742,-451.109"/>
+<polyline fill="none" stroke="black" points="826.742,-447.109 531.258,-447.109 "/>
+<polyline fill="none" stroke="black" points="826.742,-447.109 826.742,-415.109 "/>
+<polyline fill="none" stroke="black" points="826.742,-447.109 830.742,-451.109 "/>
+<text text-anchor="middle" x="681" y="-427.109" font-family="Times,serif" font-size="20.00">preprocess_deskew</text>
 </g>
 <!-- t4&#45;&gt;t5 -->
-<g id="edge5" class="edge"><title>t4&#45;&gt;t5</title>
-<path fill="none" stroke="#0044a0" d="M468.994,-357.003C475.274,-352.118 482.229,-346.709 488.905,-341.516"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="491.396,-344.013 497.141,-335.111 487.099,-338.487 491.396,-344.013"/>
-</g>
-<!-- t4&#45;&gt;t6 -->
-<g id="edge7" class="edge"><title>t4&#45;&gt;t6</title>
-<path fill="none" stroke="#0044a0" d="M370.365,-357.079C342.386,-350.862 310.55,-343.787 281.752,-337.388"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="282.225,-333.907 271.704,-335.155 280.707,-340.741 282.225,-333.907"/>
-</g>
-<!-- t4&#45;&gt;t11 -->
-<g id="edge17" class="edge"><title>t4&#45;&gt;t11</title>
-<path fill="none" stroke="#0044a0" d="M577.461,-357.079C627.38,-350.563 684.512,-343.106 735.342,-336.47"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="735.957,-339.92 745.42,-335.155 735.051,-332.979 735.957,-339.92"/>
+<g id="edge6" class="edge"><title>t4&#45;&gt;t5</title>
+<path fill="none" stroke="#0044a0" d="M798.376,-473.003C780.521,-467.087 760.346,-460.402 741.823,-454.264"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="742.901,-450.934 732.308,-451.111 740.699,-457.579 742.901,-450.934"/>
 </g>
 <!-- t8 -->
-<g id="node7" class="node"><title>t8</title>
-<polygon fill="#efa03b" stroke="black" points="986.109,-277.109 701.891,-277.109 697.891,-273.109 697.891,-241.109 982.109,-241.109 986.109,-245.109 986.109,-277.109"/>
-<polyline fill="none" stroke="black" points="982.109,-273.109 697.891,-273.109 "/>
-<polyline fill="none" stroke="black" points="982.109,-273.109 982.109,-241.109 "/>
-<polyline fill="none" stroke="black" points="982.109,-273.109 986.109,-277.109 "/>
-<text text-anchor="middle" x="842" y="-253.109" font-family="Times,serif" font-size="20.00">render_hocr_page</text>
+<g id="node10" class="node"><title>t8</title>
+<polygon fill="#efa03b" stroke="black" points="980.119,-335.109 665.881,-335.109 661.881,-331.109 661.881,-299.109 976.119,-299.109 980.119,-303.109 980.119,-335.109"/>
+<polyline fill="none" stroke="black" points="976.119,-331.109 661.881,-331.109 "/>
+<polyline fill="none" stroke="black" points="976.119,-331.109 976.119,-299.109 "/>
+<polyline fill="none" stroke="black" points="976.119,-331.109 980.119,-335.109 "/>
+<text text-anchor="middle" x="821" y="-311.109" font-family="Times,serif" font-size="20.00">select_image_for_pdf</text>
+</g>
+<!-- t4&#45;&gt;t8 -->
+<g id="edge12" class="edge"><title>t4&#45;&gt;t8</title>
+<path fill="none" stroke="#0044a0" d="M848.03,-473.087C846.196,-457.857 843.261,-434.941 840,-415.109 836.1,-391.393 830.78,-364.592 826.761,-345.181"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="830.152,-344.296 824.68,-335.224 823.3,-345.728 830.152,-344.296"/>
+</g>
+<!-- t6 -->
+<g id="node7" class="node"><title>t6</title>
+<polygon fill="#efa03b" stroke="black" points="801.95,-393.109 524.05,-393.109 520.05,-389.109 520.05,-357.109 797.95,-357.109 801.95,-361.109 801.95,-393.109"/>
+<polyline fill="none" stroke="black" points="797.95,-389.109 520.05,-389.109 "/>
+<polyline fill="none" stroke="black" points="797.95,-389.109 797.95,-357.109 "/>
+<polyline fill="none" stroke="black" points="797.95,-389.109 801.95,-393.109 "/>
+<text text-anchor="middle" x="661" y="-369.109" font-family="Times,serif" font-size="20.00">preprocess_clean</text>
+</g>
+<!-- t5&#45;&gt;t6 -->
+<g id="edge7" class="edge"><title>t5&#45;&gt;t6</title>
+<path fill="none" stroke="#0044a0" d="M674.891,-415.003C673.495,-411.095 671.979,-406.852 670.48,-402.652"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="673.731,-401.351 667.072,-393.111 667.139,-403.705 673.731,-401.351"/>
 </g>
 <!-- t5&#45;&gt;t8 -->
-<g id="edge6" class="edge"><title>t5&#45;&gt;t8</title>
-<path fill="none" stroke="#0044a0" d="M617.226,-299.079C654.028,-292.699 696.036,-285.416 733.699,-278.886"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="734.429,-282.312 743.685,-277.155 733.234,-275.415 734.429,-282.312"/>
+<g id="edge11" class="edge"><title>t5&#45;&gt;t8</title>
+<path fill="none" stroke="#0044a0" d="M781.944,-415.046C792.967,-409.579 803.051,-402.434 811,-393.109 822.009,-380.195 824.454,-361.215 824.174,-345.581"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="827.659,-345.2 823.617,-335.407 820.669,-345.583 827.659,-345.2"/>
 </g>
-<!-- t9 -->
-<g id="node11" class="node"><title>t9</title>
-<polygon fill="#efa03b" stroke="black" points="679.486,-277.109 336.514,-277.109 332.514,-273.109 332.514,-241.109 675.486,-241.109 679.486,-245.109 679.486,-277.109"/>
-<polyline fill="none" stroke="black" points="675.486,-273.109 332.514,-273.109 "/>
-<polyline fill="none" stroke="black" points="675.486,-273.109 675.486,-241.109 "/>
-<polyline fill="none" stroke="black" points="675.486,-273.109 679.486,-277.109 "/>
-<text text-anchor="middle" x="506" y="-253.109" font-family="Times,serif" font-size="20.00">render_hocr_debug_page</text>
-</g>
-<!-- t5&#45;&gt;t9 -->
-<g id="edge15" class="edge"><title>t5&#45;&gt;t9</title>
-<path fill="none" stroke="#0044a0" d="M515.029,-299.003C514.147,-295.204 513.191,-291.087 512.243,-287.002"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="515.617,-286.06 509.947,-277.111 508.799,-287.643 515.617,-286.06"/>
-</g>
-<!-- t10 -->
-<g id="node10" class="node"><title>t10</title>
-<polygon fill="#efa03b" stroke="black" points="973.082,-219.109 714.918,-219.109 710.918,-215.109 710.918,-183.109 969.082,-183.109 973.082,-187.109 973.082,-219.109"/>
-<polyline fill="none" stroke="black" points="969.082,-215.109 710.918,-215.109 "/>
-<polyline fill="none" stroke="black" points="969.082,-215.109 969.082,-183.109 "/>
-<polyline fill="none" stroke="black" points="969.082,-215.109 973.082,-219.109 "/>
-<text text-anchor="middle" x="842" y="-195.109" font-family="Times,serif" font-size="20.00">add_text_layer</text>
-</g>
-<!-- t8&#45;&gt;t10 -->
-<g id="edge12" class="edge"><title>t8&#45;&gt;t10</title>
-<path fill="none" stroke="#0044a0" d="M842,-241.003C842,-237.312 842,-233.322 842,-229.352"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="845.5,-229.111 842,-219.111 838.5,-229.111 845.5,-229.111"/>
+<!-- t7 -->
+<g id="node8" class="node"><title>t7</title>
+<polygon fill="#efa03b" stroke="black" points="643.666,-335.109 354.334,-335.109 350.334,-331.109 350.334,-299.109 639.666,-299.109 643.666,-303.109 643.666,-335.109"/>
+<polyline fill="none" stroke="black" points="639.666,-331.109 350.334,-331.109 "/>
+<polyline fill="none" stroke="black" points="639.666,-331.109 639.666,-299.109 "/>
+<polyline fill="none" stroke="black" points="639.666,-331.109 643.666,-335.109 "/>
+<text text-anchor="middle" x="497" y="-311.109" font-family="Times,serif" font-size="20.00">ocr_tesseract_hocr</text>
 </g>
 <!-- t6&#45;&gt;t7 -->
-<g id="edge10" class="edge"><title>t6&#45;&gt;t7</title>
-<path fill="none" stroke="#0044a0" d="M185.836,-299.003C183.626,-294.878 181.216,-290.379 178.845,-285.952"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="181.915,-284.273 174.108,-277.111 175.745,-287.578 181.915,-284.273"/>
+<g id="edge8" class="edge"><title>t6&#45;&gt;t7</title>
+<path fill="none" stroke="#0044a0" d="M610.903,-357.003C593.736,-351.141 574.358,-344.524 556.518,-338.433"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="557.384,-335.03 546.79,-335.111 555.122,-341.654 557.384,-335.03"/>
 </g>
-<!-- t6&#45;&gt;t9 -->
-<g id="edge14" class="edge"><title>t6&#45;&gt;t9</title>
-<path fill="none" stroke="#0044a0" d="M289.577,-299.079C324.86,-292.726 365.115,-285.478 401.259,-278.969"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="402.116,-282.372 411.337,-277.155 400.875,-275.482 402.116,-282.372"/>
+<!-- t6&#45;&gt;t8 -->
+<g id="edge10" class="edge"><title>t6&#45;&gt;t8</title>
+<path fill="none" stroke="#0044a0" d="M709.875,-357.003C726.624,-351.141 745.529,-344.524 762.933,-338.433"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="764.142,-341.718 772.425,-335.111 761.83,-335.111 764.142,-341.718"/>
+</g>
+<!-- t10 -->
+<g id="node9" class="node"><title>t10</title>
+<polygon fill="#efa03b" stroke="black" points="620.109,-277.109 335.891,-277.109 331.891,-273.109 331.891,-241.109 616.109,-241.109 620.109,-245.109 620.109,-277.109"/>
+<polyline fill="none" stroke="black" points="616.109,-273.109 331.891,-273.109 "/>
+<polyline fill="none" stroke="black" points="616.109,-273.109 616.109,-241.109 "/>
+<polyline fill="none" stroke="black" points="616.109,-273.109 620.109,-277.109 "/>
+<text text-anchor="middle" x="476" y="-253.109" font-family="Times,serif" font-size="20.00">render_hocr_page</text>
 </g>
 <!-- t7&#45;&gt;t10 -->
-<g id="edge13" class="edge"><title>t7&#45;&gt;t10</title>
-<path fill="none" stroke="#0044a0" d="M314.366,-242.002C317.606,-241.697 320.821,-241.399 324,-241.109 451.228,-229.521 596.262,-218.816 700.44,-211.565"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="700.976,-215.036 710.71,-210.852 700.491,-208.053 700.976,-215.036"/>
+<g id="edge9" class="edge"><title>t7&#45;&gt;t10</title>
+<path fill="none" stroke="#0044a0" d="M490.585,-299.003C489.12,-295.095 487.528,-290.852 485.953,-286.652"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="489.164,-285.245 482.376,-277.111 482.61,-287.703 489.164,-285.245"/>
 </g>
-<!-- t14 -->
-<g id="node15" class="node"><title>t14</title>
-<polygon fill="#efa03b" stroke="black" points="774.472,-105.333 939,-78.005 1103.53,-105.333 1103.37,-149.551 774.625,-149.551 774.472,-105.333"/>
-<polygon fill="none" stroke="black" points="770.46,-101.94 939,-73.9453 1107.54,-101.94 1107.36,-153.556 770.639,-153.556 770.46,-101.94"/>
-<text text-anchor="middle" x="939" y="-111.555" font-family="Times,serif" font-size="20.00">merge_pages</text>
+<!-- t11 -->
+<g id="node13" class="node"><title>t11</title>
+<polygon fill="#efa03b" stroke="black" points="985.486,-277.109 642.514,-277.109 638.514,-273.109 638.514,-241.109 981.486,-241.109 985.486,-245.109 985.486,-277.109"/>
+<polyline fill="none" stroke="black" points="981.486,-273.109 638.514,-273.109 "/>
+<polyline fill="none" stroke="black" points="981.486,-273.109 981.486,-241.109 "/>
+<polyline fill="none" stroke="black" points="981.486,-273.109 985.486,-277.109 "/>
+<text text-anchor="middle" x="812" y="-253.109" font-family="Times,serif" font-size="20.00">render_hocr_debug_page</text>
 </g>
-<!-- t10&#45;&gt;t14 -->
-<g id="edge23" class="edge"><title>t10&#45;&gt;t14</title>
-<path fill="none" stroke="#0044a0" d="M862.571,-182.814C870.479,-176.165 879.888,-168.254 889.356,-160.293"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="891.89,-162.736 897.292,-153.622 887.385,-157.378 891.89,-162.736"/>
+<!-- t7&#45;&gt;t11 -->
+<g id="edge18" class="edge"><title>t7&#45;&gt;t11</title>
+<path fill="none" stroke="#0044a0" d="M592.793,-299.079C628.531,-292.726 669.303,-285.478 705.912,-278.969"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="706.887,-282.351 716.12,-277.155 705.661,-275.459 706.887,-282.351"/>
 </g>
-<!-- t9&#45;&gt;t14 -->
-<g id="edge24" class="edge"><title>t9&#45;&gt;t14</title>
-<path fill="none" stroke="#0044a0" d="M546.87,-241.103C586.226,-225.054 647.626,-200.869 702,-183.109 730.59,-173.771 761.415,-164.679 791.074,-156.405"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="792.38,-159.675 801.082,-153.632 790.511,-152.93 792.38,-159.675"/>
+<!-- t12 -->
+<g id="node12" class="node"><title>t12</title>
+<polygon fill="#efa03b" stroke="black" points="607.082,-219.109 348.918,-219.109 344.918,-215.109 344.918,-183.109 603.082,-183.109 607.082,-187.109 607.082,-219.109"/>
+<polyline fill="none" stroke="black" points="603.082,-215.109 344.918,-215.109 "/>
+<polyline fill="none" stroke="black" points="603.082,-215.109 603.082,-183.109 "/>
+<polyline fill="none" stroke="black" points="603.082,-215.109 607.082,-219.109 "/>
+<text text-anchor="middle" x="476" y="-195.109" font-family="Times,serif" font-size="20.00">add_text_layer</text>
 </g>
-<!-- t13&#45;&gt;t14 -->
-<g id="edge20" class="edge"><title>t13&#45;&gt;t14</title>
-<path fill="none" stroke="#0044a0" d="M986.636,-414.956C1033.58,-398.748 1087,-369.103 1087,-318.109 1087,-318.109 1087,-318.109 1087,-258.109 1087,-215.948 1054.81,-182.761 1020.46,-159.346"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="1021.98,-156.158 1011.7,-153.604 1018.15,-162.013 1021.98,-156.158"/>
+<!-- t10&#45;&gt;t12 -->
+<g id="edge15" class="edge"><title>t10&#45;&gt;t12</title>
+<path fill="none" stroke="#0044a0" d="M476,-241.003C476,-237.312 476,-233.322 476,-229.352"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="479.5,-229.111 476,-219.111 472.5,-229.111 479.5,-229.111"/>
 </g>
-<!-- t11&#45;&gt;t14 -->
-<g id="edge22" class="edge"><title>t11&#45;&gt;t14</title>
-<path fill="none" stroke="#0044a0" d="M969.056,-299.033C979.177,-293.579 988.239,-286.44 995,-277.109 1020.03,-242.568 998.243,-195.803 974.929,-162.021"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="977.661,-159.825 968.996,-153.728 971.968,-163.897 977.661,-159.825"/>
+<!-- t8&#45;&gt;t9 -->
+<g id="edge13" class="edge"><title>t8&#45;&gt;t9</title>
+<path fill="none" stroke="#0044a0" d="M661.842,-299.864C658.871,-299.606 655.921,-299.354 653,-299.109 509.812,-287.095 471.159,-289.734 324.542,-277.219"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="324.526,-273.705 314.262,-276.332 323.924,-280.679 324.526,-273.705"/>
 </g>
-<!-- t12&#45;&gt;t14 -->
-<g id="edge21" class="edge"><title>t12&#45;&gt;t14</title>
-<path fill="none" stroke="#0044a0" d="M1006.64,-472.956C1053.58,-456.748 1107,-427.103 1107,-376.109 1107,-376.109 1107,-376.109 1107,-258.109 1107,-220.742 1095.53,-209.426 1069,-183.109 1059.8,-173.988 1049.06,-165.927 1037.76,-158.869"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="1039.41,-155.777 1029.03,-153.665 1035.83,-161.791 1039.41,-155.777"/>
+<!-- t8&#45;&gt;t11 -->
+<g id="edge17" class="edge"><title>t8&#45;&gt;t11</title>
+<path fill="none" stroke="#0044a0" d="M818.251,-299.003C817.64,-295.204 816.979,-291.087 816.322,-287.002"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="819.775,-286.429 814.732,-277.111 812.864,-287.539 819.775,-286.429"/>
 </g>
-<!-- t15 -->
-<g id="node16" class="node"><title>t15</title>
-<polygon fill="#efa03b" stroke="black" points="1053.18,-52 828.822,-52 824.822,-48 824.822,-16 1049.18,-16 1053.18,-20 1053.18,-52"/>
-<polyline fill="none" stroke="black" points="1049.18,-48 824.822,-48 "/>
-<polyline fill="none" stroke="black" points="1049.18,-48 1049.18,-16 "/>
-<polyline fill="none" stroke="black" points="1049.18,-48 1053.18,-52 "/>
-<text text-anchor="middle" x="939" y="-28" font-family="Times,serif" font-size="20.00">copy_final</text>
+<!-- t8&#45;&gt;t13 -->
+<g id="edge20" class="edge"><title>t8&#45;&gt;t13</title>
+<path fill="none" stroke="#0044a0" d="M935.04,-299.079C978.311,-292.618 1027.78,-285.23 1071.94,-278.636"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1072.48,-282.093 1081.86,-277.155 1071.45,-275.17 1072.48,-282.093"/>
 </g>
-<!-- t14&#45;&gt;t15 -->
-<g id="edge25" class="edge"><title>t14&#45;&gt;t15</title>
-<path fill="none" stroke="#0044a0" d="M939,-73.8665C939,-69.8921 939,-65.942 939,-62.1676"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="942.5,-62.1213 939,-52.1214 935.5,-62.1214 942.5,-62.1213"/>
+<!-- t9&#45;&gt;t12 -->
+<g id="edge16" class="edge"><title>t9&#45;&gt;t12</title>
+<path fill="none" stroke="#0044a0" d="M259.577,-241.079C294.86,-234.726 335.115,-227.478 371.259,-220.969"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="372.116,-224.372 381.337,-219.155 370.875,-217.482 372.116,-224.372"/>
+</g>
+<!-- t16 -->
+<g id="node17" class="node"><title>t16</title>
+<polygon fill="#efa03b" stroke="black" points="1031.47,-105.333 1196,-78.005 1360.53,-105.333 1360.37,-149.551 1031.63,-149.551 1031.47,-105.333"/>
+<polygon fill="none" stroke="black" points="1027.46,-101.94 1196,-73.9453 1364.54,-101.94 1364.36,-153.556 1027.64,-153.556 1027.46,-101.94"/>
+<text text-anchor="middle" x="1196" y="-111.555" font-family="Times,serif" font-size="20.00">merge_pages</text>
+</g>
+<!-- t12&#45;&gt;t16 -->
+<g id="edge26" class="edge"><title>t12&#45;&gt;t16</title>
+<path fill="none" stroke="#0044a0" d="M607.222,-185.246C722.142,-172.229 890.172,-153.196 1017.34,-138.791"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1018.04,-142.235 1027.58,-137.632 1017.25,-135.279 1018.04,-142.235"/>
+</g>
+<!-- t11&#45;&gt;t16 -->
+<g id="edge27" class="edge"><title>t11&#45;&gt;t16</title>
+<path fill="none" stroke="#0044a0" d="M859.086,-240.997C916.223,-220.232 1014.29,-184.591 1089.94,-157.099"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1091.32,-160.323 1099.52,-153.618 1088.92,-153.744 1091.32,-160.323"/>
+</g>
+<!-- t15&#45;&gt;t16 -->
+<g id="edge23" class="edge"><title>t15&#45;&gt;t16</title>
+<path fill="none" stroke="#0044a0" d="M1321.98,-414.938C1359.16,-396.916 1407,-364.681 1407,-318.109 1407,-318.109 1407,-318.109 1407,-258.109 1407,-211.469 1370.69,-179.509 1328.2,-158.076"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1329.57,-154.854 1319.04,-153.66 1326.53,-161.158 1329.57,-154.854"/>
+</g>
+<!-- t13&#45;&gt;t16 -->
+<g id="edge25" class="edge"><title>t13&#45;&gt;t16</title>
+<path fill="none" stroke="#0044a0" d="M1196,-240.997C1196,-221.914 1196,-190.265 1196,-163.926"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1199.5,-163.759 1196,-153.759 1192.5,-163.759 1199.5,-163.759"/>
+</g>
+<!-- t14&#45;&gt;t16 -->
+<g id="edge24" class="edge"><title>t14&#45;&gt;t16</title>
+<path fill="none" stroke="#0044a0" d="M1326.64,-588.956C1373.58,-572.748 1427,-543.103 1427,-492.109 1427,-492.109 1427,-492.109 1427,-258.109 1427,-220.539 1416.55,-207.537 1388,-183.109 1376.82,-173.542 1364.16,-165.377 1350.82,-158.413"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1351.99,-155.087 1341.47,-153.781 1348.88,-161.358 1351.99,-155.087"/>
+</g>
+<!-- t17 -->
+<g id="node18" class="node"><title>t17</title>
+<polygon fill="#efa03b" stroke="black" points="1310.18,-52 1085.82,-52 1081.82,-48 1081.82,-16 1306.18,-16 1310.18,-20 1310.18,-52"/>
+<polyline fill="none" stroke="black" points="1306.18,-48 1081.82,-48 "/>
+<polyline fill="none" stroke="black" points="1306.18,-48 1306.18,-16 "/>
+<polyline fill="none" stroke="black" points="1306.18,-48 1310.18,-52 "/>
+<text text-anchor="middle" x="1196" y="-28" font-family="Times,serif" font-size="20.00">copy_final</text>
+</g>
+<!-- t16&#45;&gt;t17 -->
+<g id="edge28" class="edge"><title>t16&#45;&gt;t17</title>
+<path fill="none" stroke="#0044a0" d="M1196,-73.8665C1196,-69.8921 1196,-65.942 1196,-62.1676"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1199.5,-62.1213 1196,-52.1214 1192.5,-62.1214 1199.5,-62.1213"/>
 </g>
 </g>
 </svg>

pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+norecursedirs = lib 

requirements.txt

@@ -1,5 +1,6 @@
-ruffus>=2.6.3
-Pillow>=2.4.0
-reportlab>=3.1.44
-PyPDF2>=1.25.1
-git+https://github.com/jbarlow83/img2pdf.git@e9bcce0afc3720752ca53a991db93f911a1df709#egg=img2pdf-0.1.5.dev
+ruffus==2.6.3
+Pillow==3.1.1
+reportlab==3.2.0
+PyPDF2==1.25.1
+img2pdf==0.2
+cffi==1.5.2

setup.cfg

@@ -1,4 +1,4 @@
 [bdist_wheel]
 python-tag = py34
-[pytest]
-norecursedirs = lib 
+[aliases]
+test=pytest

setup.py

@@ -1,18 +1,20 @@
 #!/usr/bin/env python3
+# -*- coding: utf-8 -*-
 # © 2015 James R. Barlow: github.com/jbarlow83
 
 from __future__ import print_function, unicode_literals
-from setuptools import setup
-from subprocess import STDOUT, check_output, CalledProcessError
-from collections.abc import Mapping
-import re
+
 import sys
-
-
 if sys.version_info < (3, 4):
     print("Python 3.4 or newer is required")
     sys.exit(1)
 
+from setuptools import setup  # nopep8
+from subprocess import STDOUT, check_output, CalledProcessError  # nopep8
+from collections.abc import Mapping  # nopep8
+import re  # nopep8
+
+
 missing_program = '''
 The program '{program}' could not be executed or was not found on your
 system PATH.
@@ -149,7 +151,7 @@ command = next((arg for arg in sys.argv[1:] if not arg.startswith('-')), '')
 
 
 if command.startswith('install') or \
-        command in ['check', 'test', 'nosetests', 'easy_install', 'egg_info']:
+        command in ['check', 'test', 'nosetests', 'easy_install']:
     check_external_program(
         program='tesseract',
         need_version='3.02.02',
@@ -206,15 +208,22 @@ setup(
         "Topic :: Text Processing :: Linguistic",
         ],
     setup_requires=[
-        'setuptools_scm'
+        'setuptools_scm',
+        'setuptools_scm_git_archive',
+        'cffi>=1.5.0',
+        'pytest-runner'
     ],
     use_scm_version={'version_scheme': 'post-release'},
+    cffi_modules=[
+        'ocrmypdf/lib/compile_leptonica.py:ffi'
+    ],
     install_requires=[
-        'ruffus',
-        'Pillow',
-        'reportlab',
-        'PyPDF2',
-        'img2pdf'
+        'ruffus>=2.6.3',
+        'Pillow>=3.0.0',
+        'reportlab>=3.1.44',
+        'PyPDF2>=1.25.1',
+        'img2pdf>=0.2.0',
+        'cffi>=1.5.0'
     ],
     tests_require=tests_require,
     entry_points={

test_requirements.txt

@@ -1 +1 @@
-pytest>=2.7.2
+pytest>=2.7.2

tests/resources/NOTE.rst

@@ -1,34 +0,0 @@
-All test resources must come from free public domain sources for
-copyright reasons.
-
-Test files do not necessarily produce perfect (or even good) OCR
-results.
-
-+---------------------+--------------------------------------------------------------------------------+
-| File                | Source                                                                         |
-+=====================+================================================================================+
-| graph.pdf           | Wikimedia                                                                      |
-+---------------------+--------------------------------------------------------------------------------+
-| c02-22.pdf          | Project Gutenberg: https://www.gutenberg.org/files/76/76-h/images/c02-22.jpg   |
-+---------------------+--------------------------------------------------------------------------------+
-| LinnSequencer.jpg   | Wikimedia_                                                                     |
-+---------------------+--------------------------------------------------------------------------------+
-| congress.jpg        | http://www.baxleystamps.com/litho/meiji/courts_1871.jpg                        |
-+---------------------+--------------------------------------------------------------------------------+
-| blank.pdf           | Blank page from Adobe Illustrator CC 2015                                      |
-+---------------------+--------------------------------------------------------------------------------+
-| enormous.pdf        | PNG file saved to PDF using img2pdf                                            |
-+---------------------+--------------------------------------------------------------------------------+
-| invalid.pdf         | PDF file header followed by EOF marker; not valid                              |
-+---------------------+--------------------------------------------------------------------------------+
-| multipage.pdf       | several other files concatenated                                               |
-+---------------------+--------------------------------------------------------------------------------+
-| skew.pdf            | skewed version of c02-22.PDF                                                   |
-+---------------------+--------------------------------------------------------------------------------+
-| Test_Issue_28.pdf   | file with some syntax errors                                                   |
-+---------------------+--------------------------------------------------------------------------------+
-| missing_docinfo.pdf | file missing its DocumentInfo dictionary                                       |
-+---------------------+--------------------------------------------------------------------------------+ 
-
-
-.. _Wikimedia: https://upload.wikimedia.org/wikipedia/en/b/b7/LinnSequencer_hardware_MIDI_sequencer_brochure_page_2_300dpi.jpg

tests/resources/README.rst

@@ -0,0 +1,60 @@
+These test files are used in OCRmyPDF's test suite. They do not necessarily produce OCR results
+at all and are not meant as examples of OCR output. Some are even invalid PDFs that might
+crash certain PDF viewers.
+
+
+Files derived from free sources
+===============================
+
+These test resources come from free sources, under either public domain or Creative Commons licenses.
+In some cases they were converted from one image format to another without other changes.
+
++---------------------+--------------------------------------------------------------------------------+
+| File                | Source                                                                         |
++=====================+================================================================================+
+| c02-22.pdf          | `Project Gutenberg`_, Adventures of Huckleberry Finn, page 22                  |
++---------------------+--------------------------------------------------------------------------------+
+| congress.jpg        | `US Congressional Records`_                                                    |
++---------------------+--------------------------------------------------------------------------------+
+| graph.pdf           | `Wikimedia: Pandas text analysis.png`_                                         |
++---------------------+--------------------------------------------------------------------------------+
+| LinnSequencer.jpg   | `Wikimedia: LinnSequencer`_ (Creative Commons Attribution-ShareAlike 3.0)      |
++---------------------+--------------------------------------------------------------------------------+
+
+
+Files generated for this project
+================================
+
+The following test resources were crafted specifically for this project, and can be used
+under the terms of the license in LICENSE.rst.
+
+- blank.pdf (a blank PDF page)
+- cmyk.pdf (a CMYK image created in Photoshop)
+- enormous.pdf (a very lage page)
+- francais.pdf (a page containing French accented characters)
+- invalid.pdf (a PDF file header followed by EOF marker)
+- missing_docinfo.pdf (PDF file with no /DocumentInfo section)
+
+
+Assemblies
+==========
+
+These test resources are assemblies from other previously mentioned files, released under the same license terms as their input files.
+
+- cardinal.pdf (four cardinal directions, rotated copies of LinnSequencer.jpg)
+- ccitt.pdf (LinnSequencer.jpg, converted to CCITT encoding)
+- graph_ocred.pdf (from graph.pdf)
+- jbig2.pdf (congress.jpg, converted to JBIG2 encoding)
+- multipage.pdf (from several other files)
+- palette.pdf (congress.jpg, converted to a 256-color palette)
+- skew.pdf (from c02-22.pdf)
+- skew-encrypted.pdf (skew.pdf with encrypted applied)
+
+
+.. _`Wikimedia: LinnSequencer`: https://upload.wikimedia.org/wikipedia/en/b/b7/LinnSequencer_hardware_MIDI_sequencer_brochure_page_2_300dpi.jpg
+
+.. _`Project Gutenberg`: https://www.gutenberg.org/files/76/76-h/76-h.htm#c2
+
+.. _`US Congressional Records`: http://www.baxleystamps.com/litho/meiji/courts_1871.jpg
+
+.. _`Wikimedia: Pandas text analysis.png`: https://en.wikipedia.org/wiki/File:Pandas_text_analysis.png

tests/spoof/tesseract_cache.py

@@ -10,12 +10,16 @@ CACHE_PATH = os.path.abspath(os.path.join(
         os.path.dirname(__file__), '..', 'cache'))
 
 
+def real_tesseract():
+    tess_args = ['tesseract'] + sys.argv[1:]
+    os.execvp("tesseract", tess_args)
+    return  # Not reachable
+
 def main():
     operation = sys.argv[-1]
-    # For anything except a hocr or pdf, defer to real tesseract
-    if operation != 'hocr' and operation != 'pdf':
-        tess_args = ['tesseract'] + sys.argv[1:]
-        os.execvp("tesseract", tess_args)
+    # For anything unexpected operation, defer to real tesseract binary
+    if operation != 'hocr' and operation != 'pdf' and operation != 'stdout':
+        real_tesseract()
         return  # Not reachable
 
     try:
@@ -43,8 +47,16 @@ def main():
     except ValueError:
         pass
 
-    input_file = sys.argv[-3]
-    output_file = sys.argv[-2]
+    if operation == 'stdout' and psm != '0':
+        real_tesseract()
+        return
+
+    if operation == 'stdout':
+        input_file = sys.argv[-2]
+        output_file = 'stdout'
+    else:
+        input_file = sys.argv[-3]
+        output_file = sys.argv[-2]
 
     if operation == 'hocr':
         output_file += '.hocr'
@@ -53,25 +65,46 @@ def main():
 
     with open(input_file, 'rb') as f:
         m.update(f.read())
-
     cache_name = os.path.join(CACHE_PATH, m.hexdigest())
+    print(cache_name)
     if os.path.exists(cache_name):
         # Cache hit
         print("Tesseract cache hit", file=sys.stderr)
-        shutil.copy(cache_name, output_file)
+        if operation != 'stdout':
+            shutil.copy(cache_name, output_file)
+
+        # Replicate output
+        with open(cache_name + '.stdout', 'r') as f:
+            print(f.read(), end='')
+        with open(cache_name + '.stderr', 'r') as f:
+            print(f.read(), end='', file=sys.stderr)
         sys.exit(0)
 
     # Cache miss
     print("Tesseract cache miss", file=sys.stderr)
 
     # Call tesseract
-    subprocess.check_call(['tesseract'] + sys.argv[1:])
+    p = subprocess.Popen(
+            ['tesseract'] + sys.argv[1:],
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+            universal_newlines=True)
+    stdout, stderr = p.communicate()
+
+    with open(cache_name + '.stdout', 'w') as f:
+        f.write(stdout)
+    with open(cache_name + '.stderr', 'w') as f:
+        f.write(stderr)
+    print(stdout, end='')
+    print(stderr, end='', file=sys.stderr)
 
     # Insert file into cache
-    if os.path.exists(output_file):
-        shutil.copy(output_file, cache_name)
+    if output_file != 'stdout':
+        if os.path.exists(output_file):
+            shutil.copy(output_file, cache_name)
+        else:
+            print("Could not find output file", file=sys.stderr)
     else:
-        print("Could not find output file", file=sys.stderr)
+        open(cache_name, 'w').close()
 
 
 if __name__ == '__main__':

tests/spoof/tesseract_noop.py

@@ -53,6 +53,13 @@ def main():
         pdf_bytes = img2pdf.convert([inputf], dpi=300)
         with open(output + '.pdf', 'wb') as f:
             f.write(pdf_bytes)
+    elif sys.argv[-1] == 'stdout':
+        inputf = sys.argv[-2]
+        print("""Orientation: 0
+Orientation in degrees: 0
+Orientation confidence: 100.00
+Script: 1
+Script confidence: 100.00""", file=sys.stderr)
     else:
         print("Spoof doesn't understand arguments", file=sys.stderr)
         print(sys.argv, file=sys.stderr)

tests/test_main.py

@@ -11,6 +11,7 @@ import pytest
 from ocrmypdf.pageinfo import pdf_get_all_pageinfo
 import PyPDF2 as pypdf
 from ocrmypdf import ExitCode
+from ocrmypdf import leptonica
 
 
 if sys.version_info.major < 3:
@@ -43,28 +44,33 @@ def run_ocrmypdf_sh(input_file, output_file, *args, env=None):
     return sh, out, err
 
 
-def _make_input(input_basename):
+def _infile(input_basename):
     return os.path.join(TEST_RESOURCES, input_basename)
 
 
-def _make_output(output_basename):
-    return os.path.join(TEST_OUTPUT, output_basename)
+def _outfile(output_basename):
+    return os.path.join(TEST_OUTPUT, os.path.basename(output_basename))
 
 
 def check_ocrmypdf(input_basename, output_basename, *args, env=None):
-    input_file = _make_input(input_basename)
-    output_file = _make_output(output_basename)
+    input_file = _infile(input_basename)
+    output_file = _outfile(output_basename)
 
     sh, out, err = run_ocrmypdf_sh(input_file, output_file, *args, env=env)
-    assert sh.returncode == 0, dict(stdout=out, stderr=err)
+    if sh.returncode != 0:
+        print('stdout\n======')
+        print(out)
+        print('stderr\n======')
+        print(err)
+    assert sh.returncode == 0
     assert os.path.exists(output_file), "Output file not created"
     assert os.stat(output_file).st_size > 100, "PDF too small or empty"
     return output_file
 
 
 def run_ocrmypdf_env(input_basename, output_basename, *args, env=None):
-    input_file = _make_input(input_basename)
-    output_file = _make_output(output_basename)
+    input_file = _infile(input_basename)
+    output_file = _outfile(output_basename)
 
     if env is None:
         env = os.environ
@@ -102,7 +108,7 @@ def test_quick(spoof_tesseract_noop):
 def test_deskew(spoof_tesseract_noop):
     # Run with deskew
     deskewed_pdf = check_ocrmypdf(
-        'skew.pdf', 'test_deskew.pdf', '-d', env=spoof_tesseract_noop)
+        'skew.pdf', 'test_deskew.pdf', '-d', '-v', '1', env=spoof_tesseract_noop)
 
     # Now render as an image again and use Leptonica to find the skew angle
     # to confirm that it was deskewed
@@ -110,7 +116,7 @@ def test_deskew(spoof_tesseract_noop):
     import logging
     log = logging.getLogger()
 
-    deskewed_png = _make_output('deskewed.png')
+    deskewed_png = _outfile('deskewed.png')
 
     rasterize_pdf(
         deskewed_pdf,
@@ -120,10 +126,9 @@ def test_deskew(spoof_tesseract_noop):
         raster_device='pngmono',
         log=log)
 
-    from ocrmypdf.leptonica import pixRead, pixDestroy, pixFindSkew
-    pix = pixRead(deskewed_png)
-    skew_angle, skew_confidence = pixFindSkew(pix)
-    pix = pixDestroy(pix)
+    from ocrmypdf.leptonica import Pix
+    pix = Pix.read(deskewed_png)
+    skew_angle, skew_confidence = pix.find_skew()
 
     print(skew_angle)
     assert -0.5 < skew_angle < 0.5, "Deskewing failed"
@@ -153,7 +158,7 @@ def test_exotic_image(spoof_tesseract_cache, pdf, renderer):
 
 
 def test_preserve_metadata(spoof_tesseract_noop):
-    pdf_before = pypdf.PdfFileReader(_make_input('graph.pdf'))
+    pdf_before = pypdf.PdfFileReader(_infile('graph.pdf'))
 
     output = check_ocrmypdf('graph.pdf', 'test_metadata_preserve.pdf',
                             env=spoof_tesseract_noop)
@@ -165,8 +170,8 @@ def test_preserve_metadata(spoof_tesseract_noop):
 
 
 def test_override_metadata(spoof_tesseract_noop):
-    input_file = _make_input('c02-22.pdf')
-    output_file = _make_output('test_override_metadata.pdf')
+    input_file = _infile('c02-22.pdf')
+    output_file = _outfile('test_override_metadata.pdf')
 
     german = 'Du siehst den Wald vor lauter Bäumen nicht.'
     chinese = '孔子'
@@ -202,14 +207,14 @@ def test_override_metadata(spoof_tesseract_noop):
     ])
 def test_oversample(spoof_tesseract_cache, renderer):
     oversampled_pdf = check_ocrmypdf(
-        'skew.pdf', 'test_oversample_%s.pdf' % renderer, '--oversample', '300',
+        'skew.pdf', 'test_oversample_%s.pdf' % renderer, '--oversample', '350',
         '-f',
         '--pdf-renderer', renderer, env=spoof_tesseract_cache)
 
     pdfinfo = pdf_get_all_pageinfo(oversampled_pdf)
 
     print(pdfinfo[0]['xres'])
-    assert abs(pdfinfo[0]['xres'] - 300) < 1
+    assert abs(pdfinfo[0]['xres'] - 350) < 1
 
 
 def test_repeat_ocr():
@@ -230,14 +235,81 @@ def test_skip_ocr(spoof_tesseract_cache):
 
 
 def test_argsfile(spoof_tesseract_noop):
-    with open(_make_output('test_argsfile.txt'), 'w') as argsfile:
+    with open(_outfile('test_argsfile.txt'), 'w') as argsfile:
         print('--title', 'ArgsFile Test', '--author', 'Test Cases',
               sep='\n', end='\n', file=argsfile)
     check_ocrmypdf('graph.pdf', 'test_argsfile.pdf',
-                   '@' + _make_output('test_argsfile.txt'),
+                   '@' + _outfile('test_argsfile.txt'),
                    env=spoof_tesseract_noop)
 
 
+def check_monochrome_correlation(
+        reference_pdf, reference_pageno, test_pdf, test_pageno):
+
+    import ocrmypdf.ghostscript as ghostscript
+    import logging
+
+    gslog = logging.getLogger()
+
+    reference_png = _outfile('{}.ref{:04d}.png'.format(
+        reference_pdf, reference_pageno))
+    test_png = _outfile('{}.test{:04d}.png'.format(
+        test_pdf, test_pageno))
+
+    def rasterize(pdf, pageno, png):
+        if os.path.exists(png):
+            print(png)
+            return
+        ghostscript.rasterize_pdf(
+            pdf,
+            png,
+            xres=100, yres=100,
+            raster_device='pngmono', log=gslog, pageno=pageno)
+
+    rasterize(reference_pdf, reference_pageno, reference_png)
+    rasterize(test_pdf, test_pageno, test_png)
+
+    pix_ref = leptonica.Pix.read(reference_png)
+    pix_test = leptonica.Pix.read(test_png)
+
+    return leptonica.Pix.correlation_binary(pix_ref, pix_test)
+
+
+def test_monochrome_correlation():
+    # Verify leptonica: check that an incorrect rotated image has poor
+    # correlation with reference
+    corr = check_monochrome_correlation(
+        reference_pdf=_infile('cardinal.pdf'),
+        reference_pageno=1,  # north facing page
+        test_pdf=_infile('cardinal.pdf'),
+        test_pageno=3,  # south facing page
+        )
+    assert corr < 0.10
+
+
+@pytest.mark.parametrize('renderer', [
+    'hocr',
+    'tesseract',
+    ])
+def test_autorotate(spoof_tesseract_cache, renderer):
+    import ocrmypdf.ghostscript as ghostscript
+    import logging
+
+    gslog = logging.getLogger()
+
+    # cardinal.pdf contains four copies of an image rotated in each cardinal
+    # direction - these ones are "burned in" not tagged with /Rotate
+    out = check_ocrmypdf('cardinal.pdf', 'test_autorotate_%s.pdf' % renderer,
+                         '-r', '-v', '1', env=spoof_tesseract_cache)
+    for n in range(1, 4+1):
+        correlation = check_monochrome_correlation(
+            reference_pdf=_infile('cardinal.pdf'),
+            reference_pageno=1,
+            test_pdf=out,
+            test_pageno=n)
+        assert correlation > 0.80
+
+
 @pytest.mark.parametrize('renderer', [
     'hocr',
     'tesseract',
@@ -246,14 +318,14 @@ def test_ocr_timeout(renderer):
     out = check_ocrmypdf('skew.pdf', 'test_timeout_%s.pdf' % renderer,
                          '--tesseract-timeout', '1.0')
     pdfinfo = pdf_get_all_pageinfo(out)
-    assert pdfinfo[0]['has_text'] == False
+    assert not pdfinfo[0]['has_text']
 
 
 def test_skip_big(spoof_tesseract_cache):
     out = check_ocrmypdf('enormous.pdf', 'test_enormous.pdf',
                          '--skip-big', '10', env=spoof_tesseract_cache)
     pdfinfo = pdf_get_all_pageinfo(out)
-    assert pdfinfo[0]['has_text'] == False
+    assert not pdfinfo[0]['has_text']
 
 
 @pytest.mark.parametrize('renderer', [
@@ -311,19 +383,19 @@ def test_missing_docinfo(spoof_tesseract_noop):
 
 
 def test_uppercase_extension(spoof_tesseract_noop):
-    shutil.copy(_make_input("skew.pdf"), _make_input("UPPERCASE.PDF"))
+    shutil.copy(_infile("skew.pdf"), _infile("UPPERCASE.PDF"))
     try:
         check_ocrmypdf("UPPERCASE.PDF", "UPPERCASE_OUT.PDF",
                        env=spoof_tesseract_noop)
     finally:
-        os.unlink(_make_input("UPPERCASE.PDF"))
+        os.unlink(_infile("UPPERCASE.PDF"))
 
 
 def test_input_file_not_found():
     input_file = "does not exist.pdf"
     sh, out, err = run_ocrmypdf_sh(
-        _make_input(input_file),
-        _make_output("will not happen.pdf"))
+        _infile(input_file),
+        _outfile("will not happen.pdf"))
     assert sh.returncode == ExitCode.input_file
     assert (input_file in out or input_file in err)
 
@@ -331,8 +403,8 @@ def test_input_file_not_found():
 def test_input_file_not_a_pdf():
     input_file = __file__  # Try to OCR this file
     sh, out, err = run_ocrmypdf_sh(
-        _make_input(input_file),
-        _make_output("will not happen.pdf"))
+        _infile(input_file),
+        _outfile("will not happen.pdf"))
     assert sh.returncode == ExitCode.input_file
     assert (input_file in out or input_file in err)
 

tests/test_pageinfo.py

@@ -67,13 +67,21 @@ def test_single_page_text():
 def test_single_page_image():
     filename = os.path.join(TEST_OUTPUT, 'image-mono.pdf')
 
-    with NamedTemporaryFile() as im_tmp:
+    with NamedTemporaryFile(mode='wb+', suffix='.png') as im_tmp:
         im = Image.new('1', (8, 8), 0)
         for n in range(8):
             im.putpixel((n, n), 1)
         im.save(im_tmp.name, format='PNG')
 
-        pdf_bytes = img2pdf.convert([im_tmp.name], dpi=8)
+        imgsize = ((img2pdf.ImgSize.dpi, 8), (img2pdf.ImgSize.dpi, 8))
+        layout_fun = img2pdf.get_layout_fun(None, imgsize, None, None, None)
+
+        im_tmp.seek(0)
+        im_bytes = im_tmp.read()
+        pdf_bytes = img2pdf.convert(
+                im_bytes, producer="img2pdf", with_pdfrw=False,
+                layout_fun=layout_fun)
+
         with open(filename, 'wb') as pdf:
             pdf.write(pdf_bytes)