Merge branch 'release/v3.2-rc1'

2016-01-19 16:58:37 -08:00
parent 12bc58b5b6 52e8aa434f
commit bacbcba58a
28 changed files with 991 additions and 626 deletions
@@ -1,21 +1,31 @@
-bin/
-build/
-dist/
-include/
-lib/
-ocrmypdf.egg-info/
-staging/
-.git/
-.ruffus_history.sqlite
-MANIFEST.in
-*.sublime*
+*.ipynb
 *.pdf
-*.rst
 *.pyc
+*.rst
+*.sublime*
 */*.pyc
 */*/*.pyc
 */*/*/*.pyc
 */*/*/*/*.pyc
 */*/*/*/*/*.pyc
 */*/*/*/*/*/*.pyc
-*/*/*/*/*/*/*/*.pyc
+*/*/*/*/*/*/*/*.pyc
+.cache/
+.git/
+.ipynb_checkpoints/
+.ruffus_history.sqlite
+bin/
+build/
+dist/
+htmlcov/
+include/
+lib/
+MANIFEST.in
+ocrmypdf.egg-info/
+staging/
+tests/cache/
+tests/output/
+tests/resources/private/
+tmp/
+venv-3.4/
+venv-3.5/
@@ -21,3 +21,6 @@ htmlcov/
 .coverage
 .cache/
 .ipynb_checkpoints/
+tests/cache/
+tests/resources/private
+ocrmypdf/version.py
@@ -5,6 +5,7 @@ cache:
        - $HOME/.cache/pip
        - $HOME/.ccache
        - tarballs
+        - tests/cache

 python:
    - 3.4
@@ -1,6 +1,6 @@
 # OCRmyPDF
 #
-# VERSION               3.0.0
+# VERSION               3.0.2
 FROM      debian:stretch
 MAINTAINER James R. Barlow <jim@purplerock.ca>

@@ -21,7 +21,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
  python3-pip \
  python3-venv \
  python3-reportlab \
-  python3-pil
+  python3-pil \
+  python3-wheel \
+  unpaper

 # Enforce UTF-8
 # Borrowed from https://index.docker.io/u/crosbymichael/python/ 
@@ -30,35 +32,7 @@ RUN dpkg-reconfigure locales && \
  /usr/sbin/update-locale LANG=C.UTF-8
 ENV LC_ALL C.UTF-8

-# Build unpaper 6.1
-RUN apt-get install -y \
-  wget \
-  gcc \
-  libavformat-dev \
-  libavcodec-dev \
-  libavutil-dev \
-  autoconf \
-  automake \
-  make \
-  pkg-config \
-  xsltproc
-
-WORKDIR /root
-RUN wget -q https://github.com/Flameeyes/unpaper/archive/unpaper-6.1.tar.gz
-RUN tar xf unpaper-6.1.tar.gz
-WORKDIR /root/unpaper-unpaper-6.1
-RUN autoreconf -i
-RUN ./configure CFLAGS="-O2 -march=native -pipe -flto"
-RUN make -j install
-WORKDIR /
-
-RUN apt-get remove -y \
-  gcc \
-  autoconf \
-  automake \
-  pkg-config \
-  xsltproc \
-  make
+# Remove the junk
 RUN apt-get autoremove -y && apt-get clean -y
 RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/*

@@ -79,7 +53,6 @@ RUN . /appenv/bin/activate; \
 USER docker
 WORKDIR /home/docker

-ENV DEFAULT_RUFFUS_HISTORY_FILE=/tmp/.{basename}.ruffus_history.sqlite
 ENV OCRMYPDF_TEST_OUTPUT=/tmp/test-output
 ENV OCRMYPDF_IN_DOCKER=1

@@ -0,0 +1,13 @@
+# OCRmyPDF polyglot
+#
+# VERSION               3.0.2
+FROM      jbarlow83/ocrmypdf:latest
+MAINTAINER James R. Barlow <jim@purplerock.ca>
+
+# Update system and install our dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  tesseract-ocr-all 
+
+# Must use array form of ENTRYPOINT
+# Non-array form does not append other arguments, because that is "intuitive"
+ENTRYPOINT ["/application/docker-wrapper.sh"]
@@ -9,15 +9,14 @@ Main features

 -  Generates a searchable
   `PDF/A <https://en.wikipedia.org/?title=PDF/A>`__ file from a regular PDF
-   only containing images
 -  Places OCR text accurately below the image to ease copy / paste
 -  Keeps the exact resolution of the original embedded images

   -  or if requested oversamples the images before OCRing so as to get
      better results

-  When possible, copies input images directly to output without transcoding,
-   to preserve image quality
+-  When possible, inserts OCR information as a "lossless" operation without transcoding
+   images or rendering vector information
 -  Keeps file size about the same
 -  If requested deskews and/or cleans the image before performing OCR
 -  Validates input and output files
@@ -182,6 +181,11 @@ package <http://www.thefourtheye.in/2014/12/Python-venv-problem-with-ensurepip-i

 Ubuntu 14.04 only installs ``unpaper`` version 0.4.2, which is not supported by OCRmyPDF because it is produces invalid output. This program is an optional dependency, and provides page deskewing and cleaning. See `Dockerfile <Dockerfile>`__ for an example of how to building unpaper 6.1 from source. If you choose to install unpaper later, OCRmyPDF will use the foremost version on the system PATH.

+Installing on Windows
+~~~~~~~~~~~~~~~~~~~~~
+
+Direct installation on Windows is not possible.  Install the Docker container as described above.
+
      
 Installing HEAD revision from sources
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -241,8 +245,7 @@ In case you detect an issue, please:
 -  Append the console output of the script when running the debug mode
   (``-v 1`` option)
 -  If possible provide your input PDF file as well as the content of the
-   temporary folder (using a file sharing service like
-   www.file-upload.net)
+   temporary folder (using a file sharing service like Dropbox)

 Press & Media
 -------------
@@ -6,14 +6,26 @@ Please always read this file before installing the package
 Download software here: https://github.com/jbarlow83/OCRmyPDF/tags


+v3.2-rc1:
+=========
+
+New features
+------------
+
+-  Lossless reconstruction: when possible, OCRmyPDF will inject text layers without transcoding
+   images or otherwise manipulating the PDF.
+-  New argument --tesseract-pagesegmode allows you to pass page segmentation arguments to Tesseract OCR.
+   This helps for two column text and other situations that confuse Tesseract.
+
+
 v3.1.1:
+=======

 Changes
 -------

 -  Fixed bug that caused incorrect page size and DPI calculations on documents with mixed page sizes

-
 v3.1:
 =====

@@ -1,4 +1,5 @@
 from enum import IntEnum
+import os


 class ExitCode(IntEnum):
@@ -10,3 +11,8 @@ class ExitCode(IntEnum):
    file_access_error = 5
    already_done_ocr = 6
    other_error = 15
+
+
+def get_program(name):
+    envvar = 'OCRMYPDF_' + name.upper()
+    return os.environ.get(envvar, name)
@@ -4,12 +4,13 @@
 from tempfile import NamedTemporaryFile
 from subprocess import Popen, PIPE, check_call
 from shutil import copy
+from . import get_program


 def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log):
    with NamedTemporaryFile(delete=True) as tmp:
        args_gs = [
-            'gs',
+            get_program('gs'),
            '-dQUIET',
            '-dBATCH',
            '-dNOPAUSE',
@@ -36,7 +37,7 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log):
 def generate_pdfa(pdf_pages, output_file, threads=1):
    with NamedTemporaryFile(delete=True) as gs_pdf:
        args_gs = [
-            "gs",
+            get_program("gs"),
            "-dQUIET",
            "-dBATCH",
            "-dNOPAUSE",
@@ -2,7 +2,7 @@
 # © 2015 James R. Barlow: github.com/jbarlow83

 from contextlib import suppress
-from tempfile import NamedTemporaryFile, mkdtemp
+from tempfile import mkdtemp
 import sys
 import os
 import re
@@ -11,17 +11,12 @@ import warnings
 import multiprocessing
 import atexit
 import textwrap
+import img2pdf

 import PyPDF2 as pypdf
 from PIL import Image

-from subprocess import Popen, check_call, PIPE, CalledProcessError, \
-    TimeoutExpired, check_output, STDOUT
-try:
-    from subprocess import DEVNULL
-except ImportError:
-    DEVNULL = open(os.devnull, 'wb')
-
+from functools import partial

 from ruffus import transform, suffix, merge, active_if, regex, jobs_limit, \
    formatter, follows, split, collate, check_if_uptodate
@@ -33,13 +28,17 @@ from .pageinfo import pdf_get_all_pageinfo
 from .pdfa import generate_pdfa_def
 from . import ghostscript
 from . import tesseract
+from . import qpdf
 from . import ExitCode

+import pkg_resources
+
+VERSION = pkg_resources.get_distribution('ocrmypdf').version
+
 warnings.simplefilter('ignore', pypdf.utils.PdfReadWarning)


 BASEDIR = os.path.dirname(os.path.realpath(__file__))
-VERSION = '3.1.1'


 # -------------
@@ -106,7 +105,7 @@ parser = cmdline.get_argparse(
    ignored_args=[
        'touch_files_only', 'recreate_database', 'checksum_file_name',
        'key_legend_in_graph', 'draw_graph_horizontally', 'flowchart_format',
-        'forced_tasks', 'target_tasks', 'use_threads'])
+        'forced_tasks', 'target_tasks', 'use_threads', 'jobs'])

 parser.add_argument(
    'input_file',
@@ -117,6 +116,9 @@ parser.add_argument(
 parser.add_argument(
    '-l', '--language', action='append',
    help="languages of the file to be OCRed")
+parser.add_argument(
+    '-j', '--jobs', metavar='N', type=int,
+    help="Use up to N CPU cores simultaneously (default: use all)")

 metadata = parser.add_argument_group(
    "Metadata options",
@@ -171,13 +173,16 @@ advanced = parser.add_argument_group(
    "Advanced",
    "Advanced options for power users")
 advanced.add_argument(
-    '--tesseract-config', default=[], type=list, action='append',
+    '--tesseract-config', action='append', metavar='CFG', default=[],
    help="additional Tesseract configuration files")
+advanced.add_argument(
+    '--tesseract-pagesegmode', action='store', type=int, metavar='PSM',
+    help="set Tesseract page segmentation mode (see tesseract --help)")
 advanced.add_argument(
    '--pdf-renderer', choices=['auto', 'tesseract', 'hocr'], default='auto',
    help='choose OCR PDF renderer')
 advanced.add_argument(
-    '--tesseract-timeout', default=180.0, type=float,
+    '--tesseract-timeout', default=180.0, type=float, metavar='SECONDS',
    help='give up on OCR after the timeout, but copy the preprocessed page '
         'into the final output')

@@ -245,6 +250,10 @@ if options.clean and not options.clean_final \
        "Tesseract PDF renderer cannot render --clean pages without "
        "also performing --clean-final, so --clean-final is assumed.")

+lossless_reconstruction = False
+if options.pdf_renderer == 'hocr':
+    if not options.deskew and not options.clean_final and not options.force_ocr:
+        lossless_reconstruction = True

 # ----------
 # Logging
@@ -350,29 +359,8 @@ def repair_pdf(
        log,
        pdfinfo,
        pdfinfo_lock):
-    args_qpdf = [
-        'qpdf', input_file, output_file
-    ]
-    try:
-        out = check_output(args_qpdf, stderr=STDOUT, universal_newlines=True)
-    except CalledProcessError as e:
-        exit_with_error = True
-        if e.returncode == 2:
-            print("{0}: not a valid PDF, and could not repair it.".format(
-                    options.input_file))
-            print("Details:")
-            print(e.output)
-        elif e.returncode == 3 and e.output.find("operation succeeded"):
-            exit_with_error = False
-            out = e.output
-            print(e.output)
-        else:
-            print(e.output)
-        if exit_with_error:
-            sys.exit(ExitCode.input_file)
-
-    log.debug(out)

+    qpdf.repair(input_file, output_file, log)
    with pdfinfo_lock:
        pdfinfo.extend(pdf_get_all_pageinfo(output_file))
        log.info(pdfinfo)
@@ -437,16 +425,8 @@ def split_pages(
        with suppress(FileNotFoundError):
            os.unlink(oo)

-    pages = check_output(['qpdf', '--show-npages', input_file],
-                         universal_newlines=True, close_fds=True)
-
-    for n in range(int(pages)):
-        args_qpdf = [
-            'qpdf', input_file,
-            '--pages', input_file, '{0}'.format(n + 1), '--',
-            os.path.join(work_folder, '{0:06d}.page.pdf'.format(n + 1))
-        ]
-        check_call(args_qpdf)
+    npages = qpdf.get_npages(input_file)
+    qpdf.split_pages(input_file, work_folder, npages)

    from glob import glob
    for filename in glob(os.path.join(work_folder, '*.page.pdf')):
@@ -551,60 +531,17 @@ def ocr_tesseract_hocr(
        pdfinfo,
        pdfinfo_lock):

-    pageinfo = get_pageinfo(input_file, pdfinfo, pdfinfo_lock)
-
-    badxml = os.path.splitext(output_file)[0] + '.badxml'
-
-    args_tesseract = [
-        'tesseract',
-        '-l', '+'.join(options.language),
-        input_file,
-        badxml,
-        'hocr'
-    ] + options.tesseract_config
-    p = Popen(args_tesseract, close_fds=True, stdout=PIPE, stderr=PIPE,
-              universal_newlines=True)
-    try:
-        stdout, stderr = p.communicate(timeout=options.tesseract_timeout)
-    except TimeoutExpired:
-        p.kill()
-        stdout, stderr = p.communicate()
-        # Generate a HOCR file with no recognized text if tesseract times out
-        # Temporary workaround to hocrTransform not being able to function if
-        # it does not have a valid hOCR file.
-        with open(output_file, 'w', encoding="utf-8") as f:
-            f.write(tesseract.HOCR_TEMPLATE.format(
-                pageinfo['width_pixels'],
-                pageinfo['height_pixels']))
-    else:
-        if stdout:
-            log.info(stdout)
-        if stderr:
-            log.error(stderr)
-
-        if p.returncode != 0:
-            raise CalledProcessError(p.returncode, args_tesseract)
-
-        if os.path.exists(badxml + '.html'):
-            # Tesseract 3.02 appends suffix ".html" on its own (.badxml.html)
-            shutil.move(badxml + '.html', badxml)
-        elif os.path.exists(badxml + '.hocr'):
-            # Tesseract 3.03 appends suffix ".hocr" on its own (.badxml.hocr)
-            shutil.move(badxml + '.hocr', badxml)
-
-        # Tesseract 3.03 inserts source filename into hocr file without
-        # escaping it, creating invalid XML and breaking the parser.
-        # As a workaround, rewrite the hocr file, replacing the filename
-        # with a space.  Don't know if Tesseract 3.02 does the same.
-
-        regex_nested_single_quotes = re.compile(
-            r"""title='image "([^"]*)";""")
-        with open(badxml, mode='r', encoding='utf-8') as f_in, \
-                open(output_file, mode='w', encoding='utf-8') as f_out:
-            for line in f_in:
-                line = regex_nested_single_quotes.sub(
-                    r"""title='image " ";""", line)
-                f_out.write(line)
+    tesseract.generate_hocr(
+        input_file=input_file,
+        output_hocr=output_file,
+        language=options.language,
+        tessconfig=options.tesseract_config,
+        timeout=options.tesseract_timeout,
+        pageinfo_getter=partial(get_pageinfo, input_file, pdfinfo,
+                                pdfinfo_lock),
+        pagesegmode=options.tesseract_pagesegmode,
+        log=log
+        )


@active_if(options.pdf_renderer == 'hocr')
@@ -637,24 +574,47 @@ def select_image_for_pdf(

@active_if(options.pdf_renderer == 'hocr')
@collate(
-    input=[select_image_for_pdf, ocr_tesseract_hocr],
-    filter=regex(r".*/(\d{6})(?:\.image|\.hocr)"),
-    output=os.path.join(work_folder, r'\1.rendered.pdf'),
+    input=[select_image_for_pdf, split_pages],
+    filter=regex(r".*/(\d{6})(?:\.image|\.ocr\.page\.pdf)"),
+    output=os.path.join(work_folder, r'\1.image-layer.pdf'),
    extras=[_log, _pdfinfo, _pdfinfo_lock])
-def render_hocr_page(
+def select_image_layer(
        infiles,
        output_file,
        log,
        pdfinfo,
        pdfinfo_lock):
-    hocr = next(ii for ii in infiles if ii.endswith('.hocr'))
+
+    page_pdf = next(ii for ii in infiles if ii.endswith('.page.pdf'))
    image = next(ii for ii in infiles if ii.endswith('.image'))

-    pageinfo = get_pageinfo(image, pdfinfo, pdfinfo_lock)
+    if lossless_reconstruction:
+        re_symlink(page_pdf, output_file)
+    else:
+        pageinfo = get_pageinfo(image, pdfinfo, pdfinfo_lock)
+        dpi = round(max(pageinfo['xres'], pageinfo['yres'], options.oversample))
+        with open(output_file, 'wb') as pdf:
+            img2pdf.convert([image], dpi=dpi, outputstream=pdf)
+
+
+@active_if(options.pdf_renderer == 'hocr')
+@transform(
+    input=ocr_tesseract_hocr,
+    filter=suffix('.hocr'),
+    output='.hocr.pdf',
+    extras=[_log, _pdfinfo, _pdfinfo_lock])
+def render_hocr_page(
+        input_file,
+        output_file,
+        log,
+        pdfinfo,
+        pdfinfo_lock):
+    hocr = input_file
+    pageinfo = get_pageinfo(hocr, pdfinfo, pdfinfo_lock)
    dpi = round(max(pageinfo['xres'], pageinfo['yres'], options.oversample))

    hocrtransform = HocrTransform(hocr, dpi)
-    hocrtransform.to_pdf(output_file, imageFileName=image,
+    hocrtransform.to_pdf(output_file, imageFileName=None,
                         showBoundingboxes=False, invisibleText=True)


@@ -682,6 +642,35 @@ def render_hocr_debug_page(
                         showBoundingboxes=True, invisibleText=False)


+@active_if(options.pdf_renderer == 'hocr')
+@collate(
+    input=[render_hocr_page, select_image_layer],
+    filter=regex(r".*/(\d{6})(?:\.hocr\.pdf|\.image-layer\.pdf)"),
+    output=os.path.join(work_folder, r'\1.rendered.pdf'),
+    extras=[_log, _pdfinfo, _pdfinfo_lock])
+def add_text_layer(
+        infiles,
+        output_file,
+        log,
+        pdfinfo,
+        pdfinfo_lock):
+    text = next(ii for ii in infiles if ii.endswith('.hocr.pdf'))
+    image = next(ii for ii in infiles if ii.endswith('.image-layer.pdf'))
+
+    pdf_output = pypdf.PdfFileWriter()
+
+    pdf_text = pypdf.PdfFileReader(open(text, "rb"))
+    pdf_image = pypdf.PdfFileReader(open(image, "rb"))
+
+    page = pdf_text.getPage(0)
+    page.mergePage(pdf_image.getPage(0))
+
+    pdf_output.addPage(page)
+
+    with open(output_file, "wb") as out:
+        pdf_output.write(out)
+
+
@active_if(options.pdf_renderer == 'tesseract')
@collate(
    input=[preprocess_clean, split_pages],
@@ -702,33 +691,21 @@ def tesseract_ocr_and_render_pdf(
        re_symlink(input_pdf, output_file)
        return

-    args_tesseract = [
-        'tesseract',
-        '-l', '+'.join(options.language),
-        input_image,
-        os.path.splitext(output_file)[0],  # Tesseract appends suffix
-        'pdf'
-    ] + options.tesseract_config
-    p = Popen(args_tesseract, close_fds=True, stdout=PIPE, stderr=PIPE,
-              universal_newlines=True)
-
-    try:
-        stdout, stderr = p.communicate(timeout=options.tesseract_timeout)
-        if stdout:
-            log.info(stdout)
-        if stderr:
-            log.error(stderr)
-    except TimeoutExpired:
-        p.kill()
-        log.info("Tesseract - page timed out")
-        re_symlink(input_pdf, output_file)
+    tesseract.generate_pdf(
+        input_image=input_image,
+        skip_pdf=input_pdf,
+        output_pdf=output_file,
+        language=options.language,
+        tessconfig=options.tesseract_config,
+        timeout=options.tesseract_timeout,
+        pagesegmode=options.tesseract_pagesegmode,
+        log=log)


@transform(
    input=repair_pdf,
-    filter=suffix('.repaired.pdf'),
-    output='.pdfa_def.ps',
-    output_dir=work_folder,
+    filter=formatter(r'\.repaired\.pdf'),
+    output=os.path.join(work_folder, 'pdfa_def.ps'),
    extras=[_log])
 def generate_postscript_stub(
        input_file,
@@ -784,7 +761,7 @@ def skip_page(


@merge(
-    input=[render_hocr_page, render_hocr_debug_page, skip_page,
+    input=[add_text_layer, render_hocr_debug_page, skip_page,
           tesseract_ocr_and_render_pdf, generate_postscript_stub],
    output=os.path.join(work_folder, 'merged.pdf'),
    extras=[_log, _pdfinfo, _pdfinfo_lock])
@@ -830,46 +807,7 @@ def copy_final(
 def validate_pdfa(
        input_file,
        log):
-
-    args_qpdf = [
-        'qpdf',
-        '--check',
-        input_file
-    ]
-
-    try:
-        check_output(args_qpdf, stderr=STDOUT, universal_newlines=True)
-    except CalledProcessError as e:
-        if e.returncode == 2:
-            print("{0}: not a valid PDF, and could not repair it.".format(
-                    options.input_file))
-            print("Details:")
-            print(e.output)
-        elif e.returncode == 3:
-            log.info("qpdf --check returned warnings:")
-            log.info(e.output)
-        else:
-            print(e.output)
-        return False
-
-    return True
-
-
-# @active_if(ocr_required and options.exact_image)
-# @merge([render_hocr_blank_page, extract_single_page],
-#        os.path.join(work_folder, "%04i.merged.pdf") % pageno)
-# def merge_hocr_with_original_page(infiles, output_file):
-#     with open(infiles[0], 'rb') as hocr_input, \
-#             open(infiles[1], 'rb') as page_input, \
-#             open(output_file, 'wb') as output:
-#         hocr_reader = pypdf.PdfFileReader(hocr_input)
-#         page_reader = pypdf.PdfFileReader(page_input)
-#         writer = pypdf.PdfFileWriter()
-
-#         the_page = hocr_reader.getPage(0)
-#         the_page.mergePage(page_reader.getPage(0))
-#         writer.addPage(the_page)
-#         writer.write(output)
+    return qpdf.check(input_file, log)


 def available_cpu_count():
@@ -898,9 +836,10 @@ def cleanup_ruffus_error_message(msg):


 def run_pipeline():
-    if not options.jobs or options.jobs == 1:
+    if not options.jobs:
        options.jobs = available_cpu_count()
    try:
+        options.history_file = os.path.join(work_folder, 'ruffus_history.sqlite')
        cmdline.run(options)
    except ruffus_exceptions.RethrownJobError as e:
        if options.verbose:
@@ -8,6 +8,7 @@ from string import Template
 from subprocess import Popen, PIPE
 import os
 import codecs
+from . import get_program


 # This is a template written in PostScript which is needed to create PDF/A
@@ -95,7 +96,8 @@ def _get_pdfa_def(icc_profile, icc_identifier, pdfmark):
 def _get_postscript_icc_path():
    "Parse Ghostscript's help message to find where iccprofiles are stored"

-    p_gs = Popen(['gs', '--help'], close_fds=True, universal_newlines=True,
+    p_gs = Popen([get_program('gs'), '--help'], close_fds=True,
+                 universal_newlines=True,
                 stdout=PIPE, stderr=PIPE)
    out, _ = p_gs.communicate()
    lines = out.splitlines()
@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+# © 2015 James R. Barlow: github.com/jbarlow83
+
+from subprocess import CalledProcessError, check_output, STDOUT, check_call
+import sys
+import os
+
+from . import ExitCode, get_program
+
+
+def check(input_file, log):
+    args_qpdf = [
+        get_program('qpdf'),
+        '--check',
+        input_file
+    ]
+
+    try:
+        check_output(args_qpdf, stderr=STDOUT, universal_newlines=True)
+    except CalledProcessError as e:
+        if e.returncode == 2:
+            print("{0}: not a valid PDF, and could not repair it.".format(
+                    input_file))
+            print("Details:")
+            print(e.output)
+        elif e.returncode == 3:
+            log.info("qpdf --check returned warnings:")
+            log.info(e.output)
+        else:
+            print(e.output)
+        return False
+    return True
+
+
+def repair(input_file, output_file, log):
+    args_qpdf = [
+        get_program('qpdf'), input_file, output_file
+    ]
+    try:
+        check_output(args_qpdf, stderr=STDOUT, universal_newlines=True)
+    except CalledProcessError as e:
+        if e.returncode == 3 and e.output.find("operation succeeded"):
+            log.debug('qpdf found and fixed errors:')
+            log.debug(e.output)
+            print(e.output)
+            return
+
+        if e.returncode == 2 and e.output.find("invalid password"):
+            print("{0}: this PDF is password-protected - password must "
+                  "be removed for OCR".format(input_file))
+            sys.exit(ExitCode.input_file)
+        elif e.returncode == 2:
+            print("{0}: not a valid PDF, and could not repair it.".format(
+                    input_file))
+            print("Details:")
+            print(e.output)
+            sys.exit(ExitCode.input_file)
+        else:
+            print("{0}: unknown error".format(
+                    input_file))
+            print(e.output)
+            sys.exit(ExitCode.unknown)
+
+
+def get_npages(input_file):
+    pages = check_output(
+        [get_program('qpdf'), '--show-npages', input_file],
+         universal_newlines=True, close_fds=True)
+    return int(pages)
+
+
+def split_pages(input_file, work_folder, npages):
+    """Split multipage PDF into individual pages.
+
+    Incredibly enough, this multiple process approach is about 70 times
+    faster than using Ghostscript.
+    """
+    for n in range(int(npages)):
+        args_qpdf = [
+            get_program('qpdf'), input_file,
+            '--pages', input_file, '{0}'.format(n + 1), '--',
+            os.path.join(work_folder, '{0:06d}.page.pdf'.format(n + 1))
+        ]
+        check_call(args_qpdf)
@@ -1,49 +1,19 @@
 #!/usr/bin/env python3
 # © 2015 James R. Barlow: github.com/jbarlow83

-from subprocess import STDOUT, CalledProcessError, check_output
 import sys
 import os
 import re
+import shutil
 from functools import lru_cache
-from . import ExitCode
+from . import ExitCode, get_program

-
-@lru_cache(maxsize=1)
-def version():
-    args_tess = [
-        'tesseract',
-        '--version'
-    ]
-    try:
-        versions = check_output(
-                args_tess, close_fds=True, universal_newlines=True,
-                stderr=STDOUT)
-    except CalledProcessError:
-        print("Could not find Tesseract executable on system PATH.")
-        sys.exit(ExitCode.missing_dependency)
-
-    tesseract_version = re.match(r'tesseract\s(.+)', versions).group(1)
-    return tesseract_version
-
-
-@lru_cache(maxsize=1)
-def languages():
-    args_tess = [
-        'tesseract',
-        '--list-langs'
-    ]
-    try:
-        langs = check_output(
-                args_tess, close_fds=True, universal_newlines=True,
-                stderr=STDOUT)
-    except CalledProcessError as e:
-        print("Tesseract failed to report available languages.")
-        print("Output from Tesseract:")
-        print("-" * 40)
-        print(e.output)
-        sys.exit(ExitCode.missing_dependency)
-    return set(lang.strip() for lang in langs.splitlines()[1:])
+from subprocess import Popen, PIPE, CalledProcessError, \
+    TimeoutExpired, check_output, STDOUT
+try:
+    from subprocess import DEVNULL
+except ImportError:
+    DEVNULL = open(os.devnull, 'wb')


 HOCR_TEMPLATE = '''<?xml version="1.0" encoding="UTF-8"?>
@@ -67,3 +37,146 @@ HOCR_TEMPLATE = '''<?xml version="1.0" encoding="UTF-8"?>
  </div>
 </body>
 </html>'''
+
+
+@lru_cache(maxsize=1)
+def version():
+    args_tess = [
+        get_program('tesseract'),
+        '--version'
+    ]
+    try:
+        versions = check_output(
+                args_tess, close_fds=True, universal_newlines=True,
+                stderr=STDOUT)
+    except CalledProcessError:
+        print("Could not find Tesseract executable on system PATH.")
+        sys.exit(ExitCode.missing_dependency)
+
+    tesseract_version = re.match(r'tesseract\s(.+)', versions).group(1)
+    return tesseract_version
+
+
+@lru_cache(maxsize=1)
+def languages():
+    args_tess = [
+        get_program('tesseract'),
+        '--list-langs'
+    ]
+    try:
+        langs = check_output(
+                args_tess, close_fds=True, universal_newlines=True,
+                stderr=STDOUT)
+    except CalledProcessError as e:
+        print("Tesseract failed to report available languages.")
+        print("Output from Tesseract:")
+        print("-" * 40)
+        print(e.output)
+        sys.exit(ExitCode.missing_dependency)
+    return set(lang.strip() for lang in langs.splitlines()[1:])
+
+
+def generate_hocr(input_file, output_hocr, language: list, tessconfig: list,
+                  timeout: float, pageinfo_getter, pagesegmode: int, log):
+
+    badxml = os.path.splitext(output_hocr)[0] + '.badxml'
+
+    args_tesseract = [
+        get_program('tesseract'),
+        '-l', '+'.join(language)
+    ]
+
+    if pagesegmode is not None:
+        args_tesseract.extend(['-psm', str(pagesegmode)])
+
+    args_tesseract.extend([
+        input_file,
+        badxml,
+        'hocr'
+    ] + tessconfig)
+    print(args_tesseract)
+    p = Popen(args_tesseract, close_fds=True, stdout=PIPE, stderr=PIPE,
+              universal_newlines=True)
+    try:
+        stdout, stderr = p.communicate(timeout=timeout)
+    except TimeoutExpired:
+        p.kill()
+        stdout, stderr = p.communicate()
+        # Generate a HOCR file with no recognized text if tesseract times out
+        # Temporary workaround to hocrTransform not being able to function if
+        # it does not have a valid hOCR file.
+        with open(output_hocr, 'w', encoding="utf-8") as f:
+            pageinfo = pageinfo_getter()
+            f.write(HOCR_TEMPLATE.format(
+                pageinfo['width_pixels'],
+                pageinfo['height_pixels']))
+    else:
+        if stdout:
+            log.info(stdout)
+        if stderr:
+            log.error(stderr)
+
+        if p.returncode != 0:
+            raise CalledProcessError(p.returncode, args_tesseract)
+
+        if os.path.exists(badxml + '.html'):
+            # Tesseract 3.02 appends suffix ".html" on its own (.badxml.html)
+            shutil.move(badxml + '.html', badxml)
+        elif os.path.exists(badxml + '.hocr'):
+            # Tesseract 3.03 appends suffix ".hocr" on its own (.badxml.hocr)
+            shutil.move(badxml + '.hocr', badxml)
+
+        # Tesseract 3.03 inserts source filename into hocr file without
+        # escaping it, creating invalid XML and breaking the parser.
+        # As a workaround, rewrite the hocr file, replacing the filename
+        # with a space.  Don't know if Tesseract 3.02 does the same.
+
+        regex_nested_single_quotes = re.compile(
+            r"""title='image "([^"]*)";""")
+        with open(badxml, mode='r', encoding='utf-8') as f_in, \
+                open(output_hocr, mode='w', encoding='utf-8') as f_out:
+            for line in f_in:
+                line = regex_nested_single_quotes.sub(
+                    r"""title='image " ";""", line)
+                f_out.write(line)
+
+
+def generate_pdf(input_image, skip_pdf, output_pdf, language: list,
+                 tessconfig: list, timeout: float, pagesegmode: int, log):
+    '''Use Tesseract to render a PDF.
+
+    input_image -- image to analyze
+    skip_pdf -- if we time out, use this file as output
+    language -- list of languages to consider
+    tessconfig -- tesseract configuration
+    timeout -- timeout (seconds)
+    log -- logger object
+    '''
+
+    args_tesseract = [
+        get_program('tesseract'),
+        '-l', '+'.join(language)
+    ]
+
+    if pagesegmode is not None:
+        args_tesseract.extend(['-psm', str(pagesegmode)])
+
+    args_tesseract.extend([
+        input_image,
+        os.path.splitext(output_pdf)[0],  # Tesseract appends suffix
+        'pdf'
+    ] + tessconfig)
+    p = Popen(args_tesseract, close_fds=True, stdout=PIPE, stderr=PIPE,
+              universal_newlines=True)
+
+    try:
+        stdout, stderr = p.communicate(timeout=timeout)
+        if stdout:
+            log.info(stdout)
+        if stderr:
+            log.error(stderr)
+    except TimeoutExpired:
+        p.kill()
+        log.info("Tesseract - page timed out")
+        shutil.copy(skip_pdf, output_pdf)
+
@@ -8,13 +8,13 @@ from tempfile import NamedTemporaryFile
 import sys
 import os
 from functools import lru_cache
-from . import ExitCode
+from . import ExitCode, get_program


@lru_cache(maxsize=1)
 def version():
    args_unpaper = [
-        'unpaper',
+        get_program('unpaper'),
        '--version'
    ]
    p_unpaper = Popen(args_unpaper, close_fds=True, universal_newlines=True,
@@ -33,7 +33,7 @@ except ImportError:

 def run(input_file, output_file, dpi, log, mode_args):
    args_unpaper = [
-        'unpaper',
+        get_program('unpaper'),
        '-v',
        '--dpi', str(dpi)
    ] + mode_args
@@ -4,232 +4,263 @@
 <!-- Generated by graphviz version 2.38.0 (20140413.2041)
 -->
 <!-- Title: Pipeline: Pages: 1 -->
-<svg width="728pt" height="651pt"
- viewBox="0.00 0.00 728.00 650.53" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
-<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 646.53)">
+<svg width="1132pt" height="708pt"
+ viewBox="0.00 0.00 1132.00 708.08" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 704.083)">
 <title>Pipeline:</title>
-<polygon fill="white" stroke="none" points="-4,4 -4,-646.53 724,-646.53 724,4 -4,4"/>
+<polygon fill="white" stroke="none" points="-4,4 -4,-704.083 1128,-704.083 1128,4 -4,4"/>
 <g id="clust1" class="cluster"><title>clustertasks</title>
-<polygon fill="none" stroke="black" points="8,-8 8,-634.53 712,-634.53 712,-8 8,-8"/>
-<text text-anchor="middle" x="360" y="-606.53" font-family="Times,serif" font-size="30.00" fill="#ff3232">Pipeline:</text>
+<polygon fill="none" stroke="black" points="8,-8 8,-692.083 1116,-692.083 1116,-8 8,-8"/>
+<text text-anchor="middle" x="562" y="-664.083" font-family="Times,serif" font-size="30.00" fill="#ff3232">Pipeline:</text>
 </g>
 <!-- t0 -->
 <g id="node1" class="node"><title>t0</title>
-<polygon fill="#efa03b" stroke="#006000" points="481.791,-588.53 386.209,-588.53 382.209,-584.53 382.209,-552.53 477.791,-552.53 481.791,-556.53 481.791,-588.53"/>
-<polyline fill="none" stroke="#006000" points="477.791,-584.53 382.209,-584.53 "/>
-<polyline fill="none" stroke="#006000" points="477.791,-584.53 477.791,-552.53 "/>
-<polyline fill="none" stroke="#006000" points="477.791,-584.53 481.791,-588.53 "/>
-<text text-anchor="middle" x="432" y="-564.53" font-family="Times,serif" font-size="20.00" fill="#006000">repair_pdf</text>
+<polygon fill="#efa03b" stroke="black" points="936.535,-646.083 713.465,-646.083 709.465,-642.083 709.465,-610.083 932.535,-610.083 936.535,-614.083 936.535,-646.083"/>
+<polyline fill="none" stroke="black" points="932.535,-642.083 709.465,-642.083 "/>
+<polyline fill="none" stroke="black" points="932.535,-642.083 932.535,-610.083 "/>
+<polyline fill="none" stroke="black" points="932.535,-642.083 936.535,-646.083 "/>
+<text text-anchor="middle" x="823" y="-622.083" font-family="Times,serif" font-size="20.00">repair_pdf</text>
 </g>
 <!-- t1 -->
 <g id="node2" class="node"><title>t1</title>
-<polygon fill="#efa03b" stroke="black" points="466.782,-509.564 374,-526.497 281.218,-509.564 281.304,-482.165 466.696,-482.165 466.782,-509.564"/>
-<polygon fill="none" stroke="black" points="470.799,-512.902 374,-530.569 277.201,-512.902 277.311,-478.159 470.689,-478.159 470.799,-512.902"/>
-<text text-anchor="middle" x="374" y="-495.991" font-family="Times,serif" font-size="20.00">split_pages</text>
+<polygon fill="#efa03b" stroke="black" points="914.112,-567.155 710,-584.057 505.888,-567.155 506.078,-539.806 913.922,-539.806 914.112,-567.155"/>
+<polygon fill="none" stroke="black" points="918.134,-570.834 710,-588.069 501.866,-570.834 502.11,-535.808 917.89,-535.808 918.134,-570.834"/>
+<text text-anchor="middle" x="710" y="-553.596" font-family="Times,serif" font-size="20.00">split_pages</text>
 </g>
 <!-- t0&#45;&gt;t1 -->
 <g id="edge1" class="edge"><title>t0&#45;&gt;t1</title>
-<path fill="none" stroke="gray" d="M417.064,-552.394C412.296,-546.925 406.86,-540.689 401.493,-534.532"/>
-<polygon fill="gray" stroke="gray" points="403.996,-532.077 394.787,-526.838 398.719,-536.676 403.996,-532.077"/>
+<path fill="none" stroke="#0044a0" d="M793.9,-609.961C783.582,-603.89 771.656,-596.873 760.092,-590.069"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="761.747,-586.982 751.353,-584.927 758.197,-593.015 761.747,-586.982"/>
 </g>
-<!-- t10 -->
-<g id="node12" class="node"><title>t10</title>
-<polygon fill="#efa03b" stroke="#006000" points="704.338,-451.452 493.662,-451.452 489.662,-447.452 489.662,-415.452 700.338,-415.452 704.338,-419.452 704.338,-451.452"/>
-<polyline fill="none" stroke="#006000" points="700.338,-447.452 489.662,-447.452 "/>
-<polyline fill="none" stroke="#006000" points="700.338,-447.452 700.338,-415.452 "/>
-<polyline fill="none" stroke="#006000" points="700.338,-447.452 704.338,-451.452 "/>
-<text text-anchor="middle" x="597" y="-427.452" font-family="Times,serif" font-size="20.00" fill="#006000">generate_postscript_stub</text>
+<!-- t12 -->
+<g id="node14" class="node"><title>t12</title>
+<polygon fill="#efa03b" stroke="black" points="1108.08,-509.109 769.918,-509.109 765.918,-505.109 765.918,-473.109 1104.08,-473.109 1108.08,-477.109 1108.08,-509.109"/>
+<polyline fill="none" stroke="black" points="1104.08,-505.109 765.918,-505.109 "/>
+<polyline fill="none" stroke="black" points="1104.08,-505.109 1104.08,-473.109 "/>
+<polyline fill="none" stroke="black" points="1104.08,-505.109 1108.08,-509.109 "/>
+<text text-anchor="middle" x="937" y="-485.109" font-family="Times,serif" font-size="20.00">generate_postscript_stub</text>
 </g>
-<!-- t0&#45;&gt;t10 -->
-<g id="edge16" class="edge"><title>t0&#45;&gt;t10</title>
-<path fill="none" stroke="gray" d="M453.048,-552.468C461.441,-545.654 471.185,-537.73 480,-530.53 510.227,-505.84 544.753,-477.467 568.421,-457.99"/>
-<polygon fill="gray" stroke="gray" points="570.818,-460.55 576.315,-451.493 566.369,-455.146 570.818,-460.55"/>
+<!-- t0&#45;&gt;t12 -->
+<g id="edge19" class="edge"><title>t0&#45;&gt;t12</title>
+<path fill="none" stroke="#0044a0" d="M899.32,-610.004C909.979,-604.592 919.748,-597.466 927,-588.083 941.916,-568.78 943.099,-540.385 941.345,-519.486"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="944.8,-518.881 940.218,-509.328 937.843,-519.653 944.8,-518.881"/>
 </g>
 <!-- t2 -->
 <g id="node3" class="node"><title>t2</title>
-<polygon fill="#efa03b" stroke="black" points="451.555,-451.452 228.445,-451.452 224.445,-447.452 224.445,-415.452 447.555,-415.452 451.555,-419.452 451.555,-451.452"/>
-<polyline fill="none" stroke="black" points="447.555,-447.452 224.445,-447.452 "/>
-<polyline fill="none" stroke="black" points="447.555,-447.452 447.555,-415.452 "/>
-<polyline fill="none" stroke="black" points="447.555,-447.452 451.555,-451.452 "/>
-<text text-anchor="middle" x="338" y="-427.452" font-family="Times,serif" font-size="20.00">rasterize_with_ghostscript</text>
+<polygon fill="#efa03b" stroke="black" points="592.299,-509.109 241.701,-509.109 237.701,-505.109 237.701,-473.109 588.299,-473.109 592.299,-477.109 592.299,-509.109"/>
+<polyline fill="none" stroke="black" points="588.299,-505.109 237.701,-505.109 "/>
+<polyline fill="none" stroke="black" points="588.299,-505.109 588.299,-473.109 "/>
+<polyline fill="none" stroke="black" points="588.299,-505.109 592.299,-509.109 "/>
+<text text-anchor="middle" x="415" y="-485.109" font-family="Times,serif" font-size="20.00">rasterize_with_ghostscript</text>
 </g>
 <!-- t1&#45;&gt;t2 -->
 <g id="edge2" class="edge"><title>t1&#45;&gt;t2</title>
-<path fill="none" stroke="#0044a0" d="M361.611,-478.092C358.514,-472.369 355.171,-466.19 352.003,-460.333"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="355.064,-458.636 347.227,-451.506 348.907,-461.967 355.064,-458.636"/>
+<path fill="none" stroke="#0044a0" d="M608.89,-535.808C573.672,-527.87 534.524,-519.048 500.679,-511.42"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="501.345,-507.982 490.82,-509.198 499.806,-514.811 501.345,-507.982"/>
+</g>
+<!-- t7 -->
+<g id="node9" class="node"><title>t7</title>
+<polygon fill="#efa03b" stroke="black" points="314.109,-277.109 19.8906,-277.109 15.8906,-273.109 15.8906,-241.109 310.109,-241.109 314.109,-245.109 314.109,-277.109"/>
+<polyline fill="none" stroke="black" points="310.109,-273.109 15.8906,-273.109 "/>
+<polyline fill="none" stroke="black" points="310.109,-273.109 310.109,-241.109 "/>
+<polyline fill="none" stroke="black" points="310.109,-273.109 314.109,-277.109 "/>
+<text text-anchor="middle" x="165" y="-253.109" font-family="Times,serif" font-size="20.00">select_image_layer</text>
+</g>
+<!-- t1&#45;&gt;t7 -->
+<g id="edge11" class="edge"><title>t1&#45;&gt;t7</title>
+<path fill="none" stroke="#0044a0" d="M501.985,-548.028C416.651,-540.897 317.351,-528.985 229,-509.109 131.492,-487.174 17,-534.054 17,-434.109 17,-434.109 17,-434.109 17,-374.109 17,-340.481 4.97908,-324.525 27,-299.109 32.8004,-292.415 39.6539,-286.828 47.1559,-282.17"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="49.1993,-285.038 56.2361,-277.118 45.7959,-278.921 49.1993,-285.038"/>
+</g>
+<!-- t13 -->
+<g id="node12" class="node"><title>t13</title>
+<polygon fill="#efa03b" stroke="black" points="1029.34,-451.109 808.662,-451.109 804.662,-447.109 804.662,-415.109 1025.34,-415.109 1029.34,-419.109 1029.34,-451.109"/>
+<polyline fill="none" stroke="black" points="1025.34,-447.109 804.662,-447.109 "/>
+<polyline fill="none" stroke="black" points="1025.34,-447.109 1025.34,-415.109 "/>
+<polyline fill="none" stroke="black" points="1025.34,-447.109 1029.34,-451.109 "/>
+<text text-anchor="middle" x="917" y="-427.109" font-family="Times,serif" font-size="20.00">skip_page</text>
+</g>
+<!-- t1&#45;&gt;t13 -->
+<g id="edge16" class="edge"><title>t1&#45;&gt;t13</title>
+<path fill="none" stroke="#0044a0" d="M716.849,-535.484C723.825,-515.965 736.562,-488.763 757,-473.109 768.342,-464.422 781.352,-457.641 794.947,-452.352"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="796.223,-455.612 804.44,-448.923 793.845,-449.029 796.223,-455.612"/>
 </g>
 <!-- t11 -->
-<g id="node10" class="node"><title>t11</title>
-<polygon fill="#efa03b" stroke="black" points="644.594,-393.452 551.406,-393.452 547.406,-389.452 547.406,-357.452 640.594,-357.452 644.594,-361.452 644.594,-393.452"/>
-<polyline fill="none" stroke="black" points="640.594,-389.452 547.406,-389.452 "/>
-<polyline fill="none" stroke="black" points="640.594,-389.452 640.594,-357.452 "/>
-<polyline fill="none" stroke="black" points="640.594,-389.452 644.594,-393.452 "/>
-<text text-anchor="middle" x="596" y="-369.452" font-family="Times,serif" font-size="20.00">skip_page</text>
+<g id="node13" class="node"><title>t11</title>
+<polygon fill="#efa03b" stroke="black" points="1068.24,-335.109 687.76,-335.109 683.76,-331.109 683.76,-299.109 1064.24,-299.109 1068.24,-303.109 1068.24,-335.109"/>
+<polyline fill="none" stroke="black" points="1064.24,-331.109 683.76,-331.109 "/>
+<polyline fill="none" stroke="black" points="1064.24,-331.109 1064.24,-299.109 "/>
+<polyline fill="none" stroke="black" points="1064.24,-331.109 1068.24,-335.109 "/>
+<text text-anchor="middle" x="876" y="-311.109" font-family="Times,serif" font-size="20.00">tesseract_ocr_and_render_pdf</text>
 </g>
 <!-- t1&#45;&gt;t11 -->
-<g id="edge13" class="edge"><title>t1&#45;&gt;t11</title>
-<path fill="none" stroke="#0044a0" d="M425.139,-478.007C437.834,-470.731 450.731,-461.83 461,-451.452 473.874,-438.442 466.946,-427.178 481,-415.452 490.212,-407.766 513.965,-399.205 537.508,-392.067"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="538.592,-395.396 547.189,-389.203 536.607,-388.683 538.592,-395.396"/>
-</g>
-<!-- t9 -->
-<g id="node11" class="node"><title>t9</title>
-<polygon fill="#efa03b" stroke="black" points="272.496,-277.452 19.5039,-277.452 15.5039,-273.452 15.5039,-241.452 268.496,-241.452 272.496,-245.452 272.496,-277.452"/>
-<polyline fill="none" stroke="black" points="268.496,-273.452 15.5039,-273.452 "/>
-<polyline fill="none" stroke="black" points="268.496,-273.452 268.496,-241.452 "/>
-<polyline fill="none" stroke="black" points="268.496,-273.452 272.496,-277.452 "/>
-<text text-anchor="middle" x="144" y="-253.452" font-family="Times,serif" font-size="20.00">tesseract_ocr_and_render_pdf</text>
-</g>
-<!-- t1&#45;&gt;t9 -->
-<g id="edge15" class="edge"><title>t1&#45;&gt;t9</title>
-<path fill="none" stroke="#0044a0" d="M277.341,-486.044C254.708,-478.702 232.21,-467.759 215,-451.452 168.207,-407.114 152.056,-329.127 146.633,-287.855"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="150.084,-287.235 145.422,-277.721 143.133,-288.065 150.084,-287.235"/>
+<g id="edge18" class="edge"><title>t1&#45;&gt;t11</title>
+<path fill="none" stroke="#0044a0" d="M710.717,-535.657C712.098,-517.744 715.879,-492.693 726,-473.109 754.651,-417.673 809.56,-368.731 844.403,-341.318"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="846.607,-344.038 852.372,-335.148 842.322,-338.503 846.607,-344.038"/>
 </g>
 <!-- t3 -->
 <g id="node4" class="node"><title>t3</title>
-<polygon fill="#efa03b" stroke="black" points="458.999,-393.452 291.001,-393.452 287.001,-389.452 287.001,-357.452 454.999,-357.452 458.999,-361.452 458.999,-393.452"/>
-<polyline fill="none" stroke="black" points="454.999,-389.452 287.001,-389.452 "/>
-<polyline fill="none" stroke="black" points="454.999,-389.452 454.999,-357.452 "/>
-<polyline fill="none" stroke="black" points="454.999,-389.452 458.999,-393.452 "/>
-<text text-anchor="middle" x="373" y="-369.452" font-family="Times,serif" font-size="20.00">preprocess_deskew</text>
+<polygon fill="#efa03b" stroke="black" points="564.742,-451.109 269.258,-451.109 265.258,-447.109 265.258,-415.109 560.742,-415.109 564.742,-419.109 564.742,-451.109"/>
+<polyline fill="none" stroke="black" points="560.742,-447.109 265.258,-447.109 "/>
+<polyline fill="none" stroke="black" points="560.742,-447.109 560.742,-415.109 "/>
+<polyline fill="none" stroke="black" points="560.742,-447.109 564.742,-451.109 "/>
+<text text-anchor="middle" x="415" y="-427.109" font-family="Times,serif" font-size="20.00">preprocess_deskew</text>
 </g>
 <!-- t2&#45;&gt;t3 -->
 <g id="edge3" class="edge"><title>t2&#45;&gt;t3</title>
-<path fill="none" stroke="#0044a0" d="M348.691,-415.346C351.337,-411.112 354.23,-406.485 357.066,-401.946"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="360.042,-403.788 362.374,-393.453 354.106,-400.078 360.042,-403.788"/>
+<path fill="none" stroke="#0044a0" d="M415,-473.003C415,-469.312 415,-465.322 415,-461.352"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="418.5,-461.111 415,-451.111 411.5,-461.111 418.5,-461.111"/>
 </g>
 <!-- t6 -->
-<g id="node6" class="node"><title>t6</title>
-<polygon fill="#efa03b" stroke="black" points="664.375,-277.452 477.625,-277.452 473.625,-273.452 473.625,-241.452 660.375,-241.452 664.375,-245.452 664.375,-277.452"/>
-<polyline fill="none" stroke="black" points="660.375,-273.452 473.625,-273.452 "/>
-<polyline fill="none" stroke="black" points="660.375,-273.452 660.375,-241.452 "/>
-<polyline fill="none" stroke="black" points="660.375,-273.452 664.375,-277.452 "/>
-<text text-anchor="middle" x="569" y="-253.452" font-family="Times,serif" font-size="20.00">select_image_for_pdf</text>
+<g id="node8" class="node"><title>t6</title>
+<polygon fill="#efa03b" stroke="black" points="354.119,-335.109 39.8808,-335.109 35.8808,-331.109 35.8808,-299.109 350.119,-299.109 354.119,-303.109 354.119,-335.109"/>
+<polyline fill="none" stroke="black" points="350.119,-331.109 35.8808,-331.109 "/>
+<polyline fill="none" stroke="black" points="350.119,-331.109 350.119,-299.109 "/>
+<polyline fill="none" stroke="black" points="350.119,-331.109 354.119,-335.109 "/>
+<text text-anchor="middle" x="195" y="-311.109" font-family="Times,serif" font-size="20.00">select_image_for_pdf</text>
 </g>
 <!-- t2&#45;&gt;t6 -->
-<g id="edge7" class="edge"><title>t2&#45;&gt;t6</title>
-<path fill="none" stroke="#0044a0" d="M423.032,-415.374C438.812,-409.925 454.543,-402.785 468,-393.452 508.201,-365.571 539.287,-316.662 555.808,-286.558"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="559.024,-287.965 560.653,-277.496 552.851,-284.664 559.024,-287.965"/>
+<g id="edge9" class="edge"><title>t2&#45;&gt;t6</title>
+<path fill="none" stroke="#0044a0" d="M294.315,-473.06C280.461,-467.585 267.283,-460.434 256,-451.109 223.245,-424.039 207.251,-375.559 200.099,-345.207"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="203.486,-344.31 197.925,-335.292 196.648,-345.809 203.486,-344.31"/>
 </g>
 <!-- t4 -->
 <g id="node5" class="node"><title>t4</title>
-<polygon fill="#efa03b" stroke="black" points="449.705,-335.452 300.295,-335.452 296.295,-331.452 296.295,-299.452 445.705,-299.452 449.705,-303.452 449.705,-335.452"/>
-<polyline fill="none" stroke="black" points="445.705,-331.452 296.295,-331.452 "/>
-<polyline fill="none" stroke="black" points="445.705,-331.452 445.705,-299.452 "/>
-<polyline fill="none" stroke="black" points="445.705,-331.452 449.705,-335.452 "/>
-<text text-anchor="middle" x="373" y="-311.452" font-family="Times,serif" font-size="20.00">preprocess_clean</text>
+<polygon fill="#efa03b" stroke="black" points="587.95,-393.109 310.05,-393.109 306.05,-389.109 306.05,-357.109 583.95,-357.109 587.95,-361.109 587.95,-393.109"/>
+<polyline fill="none" stroke="black" points="583.95,-389.109 306.05,-389.109 "/>
+<polyline fill="none" stroke="black" points="583.95,-389.109 583.95,-357.109 "/>
+<polyline fill="none" stroke="black" points="583.95,-389.109 587.95,-393.109 "/>
+<text text-anchor="middle" x="447" y="-369.109" font-family="Times,serif" font-size="20.00">preprocess_clean</text>
 </g>
 <!-- t3&#45;&gt;t4 -->
 <g id="edge4" class="edge"><title>t3&#45;&gt;t4</title>
-<path fill="none" stroke="#0044a0" d="M373,-357.346C373,-353.655 373,-349.665 373,-345.695"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="376.5,-345.453 373,-335.453 369.5,-345.453 376.5,-345.453"/>
+<path fill="none" stroke="#0044a0" d="M424.775,-415.003C427.132,-410.878 429.703,-406.379 432.232,-401.952"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="435.362,-403.53 437.285,-393.111 429.285,-400.057 435.362,-403.53"/>
 </g>
 <!-- t3&#45;&gt;t6 -->
-<g id="edge6" class="edge"><title>t3&#45;&gt;t6</title>
-<path fill="none" stroke="#0044a0" d="M414.399,-357.412C428.785,-351.039 444.852,-343.405 459,-335.452 466.909,-331.007 506.49,-303.805 535.921,-283.431"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="538.073,-286.198 544.299,-277.625 534.086,-280.444 538.073,-286.198"/>
-</g>
-<!-- t4&#45;&gt;t6 -->
-<g id="edge5" class="edge"><title>t4&#45;&gt;t6</title>
-<path fill="none" stroke="#0044a0" d="M432.605,-299.422C453.701,-293.395 477.617,-286.561 499.465,-280.319"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="500.687,-283.61 509.341,-277.497 498.764,-276.879 500.687,-283.61"/>
+<g id="edge8" class="edge"><title>t3&#45;&gt;t6</title>
+<path fill="none" stroke="#0044a0" d="M351.041,-415.02C333.086,-409.151 313.871,-401.82 297,-393.109 269.828,-379.08 242.065,-358.149 222.377,-341.96"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="224.378,-339.071 214.46,-335.347 219.891,-344.444 224.378,-339.071"/>
 </g>
 <!-- t5 -->
-<g id="node7" class="node"><title>t5</title>
-<polygon fill="#efa03b" stroke="black" points="455.922,-277.452 294.078,-277.452 290.078,-273.452 290.078,-241.452 451.922,-241.452 455.922,-245.452 455.922,-277.452"/>
-<polyline fill="none" stroke="black" points="451.922,-273.452 290.078,-273.452 "/>
-<polyline fill="none" stroke="black" points="451.922,-273.452 451.922,-241.452 "/>
-<polyline fill="none" stroke="black" points="451.922,-273.452 455.922,-277.452 "/>
-<text text-anchor="middle" x="373" y="-253.452" font-family="Times,serif" font-size="20.00">ocr_tesseract_hocr</text>
+<g id="node6" class="node"><title>t5</title>
+<polygon fill="#efa03b" stroke="black" points="665.666,-335.109 376.334,-335.109 372.334,-331.109 372.334,-299.109 661.666,-299.109 665.666,-303.109 665.666,-335.109"/>
+<polyline fill="none" stroke="black" points="661.666,-331.109 372.334,-331.109 "/>
+<polyline fill="none" stroke="black" points="661.666,-331.109 661.666,-299.109 "/>
+<polyline fill="none" stroke="black" points="661.666,-331.109 665.666,-335.109 "/>
+<text text-anchor="middle" x="519" y="-311.109" font-family="Times,serif" font-size="20.00">ocr_tesseract_hocr</text>
 </g>
 <!-- t4&#45;&gt;t5 -->
-<g id="edge8" class="edge"><title>t4&#45;&gt;t5</title>
-<path fill="none" stroke="#0044a0" d="M373,-299.346C373,-295.655 373,-291.665 373,-287.695"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="376.5,-287.453 373,-277.453 369.5,-287.453 376.5,-287.453"/>
+<g id="edge5" class="edge"><title>t4&#45;&gt;t5</title>
+<path fill="none" stroke="#0044a0" d="M468.994,-357.003C475.274,-352.118 482.229,-346.709 488.905,-341.516"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="491.396,-344.013 497.141,-335.111 487.099,-338.487 491.396,-344.013"/>
 </g>
-<!-- t4&#45;&gt;t9 -->
-<g id="edge14" class="edge"><title>t4&#45;&gt;t9</title>
-<path fill="none" stroke="#0044a0" d="M303.36,-299.422C278.157,-293.259 249.509,-286.253 223.521,-279.898"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="224.249,-276.473 213.704,-277.497 222.586,-283.273 224.249,-276.473"/>
+<!-- t4&#45;&gt;t6 -->
+<g id="edge7" class="edge"><title>t4&#45;&gt;t6</title>
+<path fill="none" stroke="#0044a0" d="M370.365,-357.079C342.386,-350.862 310.55,-343.787 281.752,-337.388"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="282.225,-333.907 271.704,-335.155 280.707,-340.741 282.225,-333.907"/>
 </g>
-<!-- t7 -->
-<g id="node8" class="node"><title>t7</title>
-<polygon fill="#efa03b" stroke="black" points="426.366,-219.452 269.634,-219.452 265.634,-215.452 265.634,-183.452 422.366,-183.452 426.366,-187.452 426.366,-219.452"/>
-<polyline fill="none" stroke="black" points="422.366,-215.452 265.634,-215.452 "/>
-<polyline fill="none" stroke="black" points="422.366,-215.452 422.366,-183.452 "/>
-<polyline fill="none" stroke="black" points="422.366,-215.452 426.366,-219.452 "/>
-<text text-anchor="middle" x="346" y="-195.452" font-family="Times,serif" font-size="20.00">render_hocr_page</text>
-</g>
-<!-- t6&#45;&gt;t7 -->
-<g id="edge9" class="edge"><title>t6&#45;&gt;t7</title>
-<path fill="none" stroke="#0044a0" d="M501.184,-241.422C476.75,-235.286 448.99,-228.315 423.772,-221.982"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="424.429,-218.538 413.877,-219.497 422.724,-225.328 424.429,-218.538"/>
+<!-- t4&#45;&gt;t11 -->
+<g id="edge17" class="edge"><title>t4&#45;&gt;t11</title>
+<path fill="none" stroke="#0044a0" d="M577.461,-357.079C627.38,-350.563 684.512,-343.106 735.342,-336.47"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="735.957,-339.92 745.42,-335.155 735.051,-332.979 735.957,-339.92"/>
 </g>
 <!-- t8 -->
-<g id="node9" class="node"><title>t8</title>
-<polygon fill="#efa03b" stroke="black" points="663.742,-219.452 448.258,-219.452 444.258,-215.452 444.258,-183.452 659.742,-183.452 663.742,-187.452 663.742,-219.452"/>
-<polyline fill="none" stroke="black" points="659.742,-215.452 444.258,-215.452 "/>
-<polyline fill="none" stroke="black" points="659.742,-215.452 659.742,-183.452 "/>
-<polyline fill="none" stroke="black" points="659.742,-215.452 663.742,-219.452 "/>
-<text text-anchor="middle" x="554" y="-195.452" font-family="Times,serif" font-size="20.00">render_hocr_debug_page</text>
-</g>
-<!-- t6&#45;&gt;t8 -->
-<g id="edge11" class="edge"><title>t6&#45;&gt;t8</title>
-<path fill="none" stroke="#0044a0" d="M564.418,-241.346C563.4,-237.546 562.298,-233.43 561.203,-229.345"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="564.522,-228.207 558.554,-219.453 557.761,-230.018 564.522,-228.207"/>
-</g>
-<!-- t5&#45;&gt;t7 -->
-<g id="edge10" class="edge"><title>t5&#45;&gt;t7</title>
-<path fill="none" stroke="#0044a0" d="M364.752,-241.346C362.816,-237.329 360.708,-232.958 358.629,-228.645"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="361.693,-226.941 354.197,-219.453 355.387,-229.981 361.693,-226.941"/>
+<g id="node7" class="node"><title>t8</title>
+<polygon fill="#efa03b" stroke="black" points="986.109,-277.109 701.891,-277.109 697.891,-273.109 697.891,-241.109 982.109,-241.109 986.109,-245.109 986.109,-277.109"/>
+<polyline fill="none" stroke="black" points="982.109,-273.109 697.891,-273.109 "/>
+<polyline fill="none" stroke="black" points="982.109,-273.109 982.109,-241.109 "/>
+<polyline fill="none" stroke="black" points="982.109,-273.109 986.109,-277.109 "/>
+<text text-anchor="middle" x="842" y="-253.109" font-family="Times,serif" font-size="20.00">render_hocr_page</text>
 </g>
 <!-- t5&#45;&gt;t8 -->
-<g id="edge12" class="edge"><title>t5&#45;&gt;t8</title>
-<path fill="none" stroke="#0044a0" d="M428.289,-241.346C447.587,-235.375 469.416,-228.622 489.404,-222.437"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="490.53,-225.753 499.049,-219.453 488.461,-219.065 490.53,-225.753"/>
+<g id="edge6" class="edge"><title>t5&#45;&gt;t8</title>
+<path fill="none" stroke="#0044a0" d="M617.226,-299.079C654.028,-292.699 696.036,-285.416 733.699,-278.886"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="734.429,-282.312 743.685,-277.155 733.234,-275.415 734.429,-282.312"/>
 </g>
-<!-- t12 -->
-<g id="node13" class="node"><title>t12</title>
-<polygon fill="#efa03b" stroke="black" points="473.845,-105.456 554,-78.0208 634.155,-105.456 634.08,-149.848 473.92,-149.848 473.845,-105.456"/>
-<polygon fill="none" stroke="black" points="469.836,-102.581 554,-73.7729 638.164,-102.581 638.078,-153.869 469.922,-153.869 469.836,-102.581"/>
-<text text-anchor="middle" x="554" y="-111.726" font-family="Times,serif" font-size="20.00">merge_pages</text>
+<!-- t9 -->
+<g id="node11" class="node"><title>t9</title>
+<polygon fill="#efa03b" stroke="black" points="679.486,-277.109 336.514,-277.109 332.514,-273.109 332.514,-241.109 675.486,-241.109 679.486,-245.109 679.486,-277.109"/>
+<polyline fill="none" stroke="black" points="675.486,-273.109 332.514,-273.109 "/>
+<polyline fill="none" stroke="black" points="675.486,-273.109 675.486,-241.109 "/>
+<polyline fill="none" stroke="black" points="675.486,-273.109 679.486,-277.109 "/>
+<text text-anchor="middle" x="506" y="-253.109" font-family="Times,serif" font-size="20.00">render_hocr_debug_page</text>
 </g>
-<!-- t7&#45;&gt;t12 -->
-<g id="edge20" class="edge"><title>t7&#45;&gt;t12</title>
-<path fill="none" stroke="#0044a0" d="M389.35,-183.419C409.937,-175.33 435.398,-165.326 460.092,-155.624"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="461.479,-158.839 469.507,-151.924 458.919,-152.324 461.479,-158.839"/>
+<!-- t5&#45;&gt;t9 -->
+<g id="edge15" class="edge"><title>t5&#45;&gt;t9</title>
+<path fill="none" stroke="#0044a0" d="M515.029,-299.003C514.147,-295.204 513.191,-291.087 512.243,-287.002"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="515.617,-286.06 509.947,-277.111 508.799,-287.643 515.617,-286.06"/>
 </g>
-<!-- t8&#45;&gt;t12 -->
-<g id="edge19" class="edge"><title>t8&#45;&gt;t12</title>
-<path fill="none" stroke="#0044a0" d="M554,-183.12C554,-177.585 554,-171.177 554,-164.592"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="557.5,-164.201 554,-154.201 550.5,-164.201 557.5,-164.201"/>
+<!-- t10 -->
+<g id="node10" class="node"><title>t10</title>
+<polygon fill="#efa03b" stroke="black" points="973.082,-219.109 714.918,-219.109 710.918,-215.109 710.918,-183.109 969.082,-183.109 973.082,-187.109 973.082,-219.109"/>
+<polyline fill="none" stroke="black" points="969.082,-215.109 710.918,-215.109 "/>
+<polyline fill="none" stroke="black" points="969.082,-215.109 969.082,-183.109 "/>
+<polyline fill="none" stroke="black" points="969.082,-215.109 973.082,-219.109 "/>
+<text text-anchor="middle" x="842" y="-195.109" font-family="Times,serif" font-size="20.00">add_text_layer</text>
 </g>
-<!-- t11&#45;&gt;t12 -->
-<g id="edge17" class="edge"><title>t11&#45;&gt;t12</title>
-<path fill="none" stroke="#0044a0" d="M615.939,-357.13C634.767,-339.333 661.745,-309.733 673,-277.452 686.754,-238.003 694.347,-219.364 673,-183.452 666.345,-172.256 656.999,-162.875 646.44,-155.047"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="648.378,-152.132 638.152,-149.36 644.417,-157.904 648.378,-152.132"/>
+<!-- t8&#45;&gt;t10 -->
+<g id="edge12" class="edge"><title>t8&#45;&gt;t10</title>
+<path fill="none" stroke="#0044a0" d="M842,-241.003C842,-237.312 842,-233.322 842,-229.352"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="845.5,-229.111 842,-219.111 838.5,-229.111 845.5,-229.111"/>
 </g>
-<!-- t9&#45;&gt;t12 -->
-<g id="edge18" class="edge"><title>t9&#45;&gt;t12</title>
-<path fill="none" stroke="#0044a0" d="M164.711,-241.235C186.394,-224.08 222.088,-198.224 257,-183.452 321.895,-155.994 399.893,-139.571 459.67,-130.141"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="460.504,-133.554 469.855,-128.574 459.439,-126.635 460.504,-133.554"/>
+<!-- t6&#45;&gt;t7 -->
+<g id="edge10" class="edge"><title>t6&#45;&gt;t7</title>
+<path fill="none" stroke="#0044a0" d="M185.836,-299.003C183.626,-294.878 181.216,-290.379 178.845,-285.952"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="181.915,-284.273 174.108,-277.111 175.745,-287.578 181.915,-284.273"/>
 </g>
-<!-- t10&#45;&gt;t12 -->
-<g id="edge22" class="edge"><title>t10&#45;&gt;t12</title>
-<path fill="none" stroke="gray" d="M629.379,-415.337C660.552,-396.255 703,-362.233 703,-318.452 703,-318.452 703,-318.452 703,-258.452 703,-224.066 705.581,-210.223 684,-183.452 673.976,-171.017 660.952,-160.773 647.026,-152.396"/>
-<polygon fill="gray" stroke="gray" points="648.682,-149.312 638.258,-147.419 645.227,-155.4 648.682,-149.312"/>
+<!-- t6&#45;&gt;t9 -->
+<g id="edge14" class="edge"><title>t6&#45;&gt;t9</title>
+<path fill="none" stroke="#0044a0" d="M289.577,-299.079C324.86,-292.726 365.115,-285.478 401.259,-278.969"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="402.116,-282.372 411.337,-277.155 400.875,-275.482 402.116,-282.372"/>
 </g>
-<!-- t13 -->
-<g id="node14" class="node"><title>t13</title>
-<polygon fill="#efa03b" stroke="black" points="616.338,-52 495.662,-52 491.662,-48 491.662,-16 612.338,-16 616.338,-20 616.338,-52"/>
-<polyline fill="none" stroke="black" points="612.338,-48 491.662,-48 "/>
-<polyline fill="none" stroke="black" points="612.338,-48 612.338,-16 "/>
-<polyline fill="none" stroke="black" points="612.338,-48 616.338,-52 "/>
-<text text-anchor="middle" x="554" y="-28" font-family="Times,serif" font-size="20.00">validate_pdfa</text>
+<!-- t7&#45;&gt;t10 -->
+<g id="edge13" class="edge"><title>t7&#45;&gt;t10</title>
+<path fill="none" stroke="#0044a0" d="M314.366,-242.002C317.606,-241.697 320.821,-241.399 324,-241.109 451.228,-229.521 596.262,-218.816 700.44,-211.565"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="700.976,-215.036 710.71,-210.852 700.491,-208.053 700.976,-215.036"/>
 </g>
-<!-- t12&#45;&gt;t13 -->
-<g id="edge21" class="edge"><title>t12&#45;&gt;t13</title>
-<path fill="none" stroke="#0044a0" d="M554,-73.9482C554,-69.9654 554,-66.007 554,-62.2247"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="557.5,-62.1573 554,-52.1573 550.5,-62.1574 557.5,-62.1573"/>
+<!-- t14 -->
+<g id="node15" class="node"><title>t14</title>
+<polygon fill="#efa03b" stroke="black" points="774.472,-105.333 939,-78.005 1103.53,-105.333 1103.37,-149.551 774.625,-149.551 774.472,-105.333"/>
+<polygon fill="none" stroke="black" points="770.46,-101.94 939,-73.9453 1107.54,-101.94 1107.36,-153.556 770.639,-153.556 770.46,-101.94"/>
+<text text-anchor="middle" x="939" y="-111.555" font-family="Times,serif" font-size="20.00">merge_pages</text>
+</g>
+<!-- t10&#45;&gt;t14 -->
+<g id="edge23" class="edge"><title>t10&#45;&gt;t14</title>
+<path fill="none" stroke="#0044a0" d="M862.571,-182.814C870.479,-176.165 879.888,-168.254 889.356,-160.293"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="891.89,-162.736 897.292,-153.622 887.385,-157.378 891.89,-162.736"/>
+</g>
+<!-- t9&#45;&gt;t14 -->
+<g id="edge24" class="edge"><title>t9&#45;&gt;t14</title>
+<path fill="none" stroke="#0044a0" d="M546.87,-241.103C586.226,-225.054 647.626,-200.869 702,-183.109 730.59,-173.771 761.415,-164.679 791.074,-156.405"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="792.38,-159.675 801.082,-153.632 790.511,-152.93 792.38,-159.675"/>
+</g>
+<!-- t13&#45;&gt;t14 -->
+<g id="edge20" class="edge"><title>t13&#45;&gt;t14</title>
+<path fill="none" stroke="#0044a0" d="M986.636,-414.956C1033.58,-398.748 1087,-369.103 1087,-318.109 1087,-318.109 1087,-318.109 1087,-258.109 1087,-215.948 1054.81,-182.761 1020.46,-159.346"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1021.98,-156.158 1011.7,-153.604 1018.15,-162.013 1021.98,-156.158"/>
+</g>
+<!-- t11&#45;&gt;t14 -->
+<g id="edge22" class="edge"><title>t11&#45;&gt;t14</title>
+<path fill="none" stroke="#0044a0" d="M969.056,-299.033C979.177,-293.579 988.239,-286.44 995,-277.109 1020.03,-242.568 998.243,-195.803 974.929,-162.021"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="977.661,-159.825 968.996,-153.728 971.968,-163.897 977.661,-159.825"/>
+</g>
+<!-- t12&#45;&gt;t14 -->
+<g id="edge21" class="edge"><title>t12&#45;&gt;t14</title>
+<path fill="none" stroke="#0044a0" d="M1006.64,-472.956C1053.58,-456.748 1107,-427.103 1107,-376.109 1107,-376.109 1107,-376.109 1107,-258.109 1107,-220.742 1095.53,-209.426 1069,-183.109 1059.8,-173.988 1049.06,-165.927 1037.76,-158.869"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1039.41,-155.777 1029.03,-153.665 1035.83,-161.791 1039.41,-155.777"/>
+</g>
+<!-- t15 -->
+<g id="node16" class="node"><title>t15</title>
+<polygon fill="#efa03b" stroke="black" points="1053.18,-52 828.822,-52 824.822,-48 824.822,-16 1049.18,-16 1053.18,-20 1053.18,-52"/>
+<polyline fill="none" stroke="black" points="1049.18,-48 824.822,-48 "/>
+<polyline fill="none" stroke="black" points="1049.18,-48 1049.18,-16 "/>
+<polyline fill="none" stroke="black" points="1049.18,-48 1053.18,-52 "/>
+<text text-anchor="middle" x="939" y="-28" font-family="Times,serif" font-size="20.00">copy_final</text>
+</g>
+<!-- t14&#45;&gt;t15 -->
+<g id="edge25" class="edge"><title>t14&#45;&gt;t15</title>
+<path fill="none" stroke="#0044a0" d="M939,-73.8665C939,-69.8921 939,-65.942 939,-62.1676"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="942.5,-62.1213 939,-52.1214 935.5,-62.1214 942.5,-62.1213"/>
 </g>
 </g>
 </svg>
@@ -1,4 +1,5 @@
 ruffus>=2.6.3
 Pillow>=2.4.0
 reportlab>=3.1.44
-PyPDF2>=1.25.1
+PyPDF2>=1.25.1
+git+https://github.com/jbarlow83/img2pdf.git@e9bcce0afc3720752ca53a991db93f911a1df709#egg=img2pdf-0.1.5.dev
@@ -3,8 +3,7 @@

 from __future__ import print_function, unicode_literals
 from setuptools import setup
-from subprocess import Popen, STDOUT, check_output, CalledProcessError
-from string import Template
+from subprocess import STDOUT, check_output, CalledProcessError
 from collections.abc import Mapping
 import re
 import sys
@@ -179,12 +178,10 @@ if 'upload' in sys.argv[1:]:
    print('Use twine to upload the package - setup.py upload is insecure')
    sys.exit(1)

-install_requires = open('requirements.txt').read().splitlines()
 tests_require = open('test_requirements.txt').read().splitlines()

 setup(
    name='ocrmypdf',
-    version='3.1.1',  # also update: release notes, main.py
    description='OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched',
    url='https://github.com/jbarlow83/OCRmyPDF',
    author='James R. Barlow',
@@ -208,7 +205,17 @@ setup(
        "Topic :: Text Processing :: Indexing",
        "Topic :: Text Processing :: Linguistic",
        ],
-    install_requires=install_requires,
+    setup_requires=[
+        'setuptools_scm'
+    ],
+    use_scm_version={'version_scheme': 'post-release'},
+    install_requires=[
+        'ruffus',
+        'Pillow',
+        'reportlab',
+        'PyPDF2',
+        'img2pdf'
+    ],
    tests_require=tests_require,
    entry_points={
        'console_scripts': [
@@ -1,2 +1 @@
-img2pdf>=0.1.5
 pytest>=2.7.2
@@ -1,72 +0,0 @@
-#!/usr/bin/env python3
-import sys
-import os
-
-'''
-For testing PDF/A generation failures, this wrapper calls ghostscript
-but filters out all PDF/A information so that a regular PDF will be
-created instead.
-
-It assumes that it is called from a staged system PATH where the first
-item on the PATH contains a file named 'gs' which is a symlink to this
-file.  It will strip out the first item on path to invoke the real
-'gs'.  That is, it expects this when called
-
-1.   tests/output/[...test name...]/bin/gs is a symlink to this file
-2.   tests/output/bin is the first item on PATH
-3.   The real executable is on the path
-
-OCRmyPDF also calls "gs --version" and "gs --help".  gs answers both
-on stdout so, this wrapper prints to stderr.
-
-'''
-
-
-def pdfa_param(arg):
-    if arg.startswith('-sPDFA'):
-        return True
-    if arg.startswith('-dPDFA'):
-        return True
-    if arg.endswith('.ps'):
-        return True
-    if 'ColorConversionStrategy' in arg:
-        return True
-    if 'ProcessColorModel' in arg:
-        return True
-    if 'OutputICCProfile' in arg:
-        return True
-    return False
-
-
-if __name__ == '__main__':
-    sys_args = sys.argv[1:]
-
-    print("Fake Ghostscript wrapper", file=sys.stderr)
-
-    if any(pdfa_param(arg) for arg in sys_args):
-        # We were asked to produce a PDF/A
-        # Filter out PDF/A arguments
-        args = [arg for arg in sys.argv[1:]
-                if not pdfa_param(arg)]
-
-        # Tell Ghostscript to create a PDF 1.3 instead so JHOVE won't
-        # think it's a PDF/A
-        indexof_pdfwrite = next(n for n, item in enumerate(args)
-                                if 'pdfwrite' in item)
-        args.insert(indexof_pdfwrite + 1, '-dCompatibilityLevel=1.3')
-        print("Rewrote arguments", file=sys.stderr)
-    else:
-        args = sys_args
-        print("Keeping arguments", file=sys.stderr)
-
-    exec_path = os.environ['PATH'].split(os.pathsep)
-    env = os.environ.copy()
-    env['PATH'] = os.pathsep.join(exec_path[1:])
-
-    print("Calling real Ghostscript with modified args and env",
-          file=sys.stderr)
-    print(args, file=sys.stderr)
-    print(env['PATH'], file=sys.stderr)
-
-    sys.stderr.flush()
-    os.execvpe('gs', ['gs'] + args, env)
@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+import sys
+
+
+def main():
+    print('qpdf dummy')
+    sys.exit(2)
+
+
+if __name__ == '__main__':
+    main()
@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+import sys
+import os
+import hashlib
+import shutil
+import subprocess
+
+
+CACHE_PATH = os.path.abspath(os.path.join(
+        os.path.dirname(__file__), '..', 'cache'))
+
+
+def main():
+    operation = sys.argv[-1]
+    # For anything except a hocr or pdf, defer to real tesseract
+    if operation != 'hocr' and operation != 'pdf':
+        tess_args = ['tesseract'] + sys.argv[1:]
+        os.execvp("tesseract", tess_args)
+        return  # Not reachable
+
+    try:
+        os.makedirs(CACHE_PATH)
+    except FileExistsError:
+        pass
+
+    m = hashlib.sha1()
+
+    version = subprocess.check_output(
+        ['tesseract', '--version'],
+        stderr=subprocess.STDOUT)
+
+    m.update(version)
+    m.update(operation.encode())
+
+    try:
+        lang = sys.argv[sys.argv.index('-l') + 1]
+        m.update(lang.encode())
+    except ValueError:
+        pass
+    try:
+        psm = sys.argv[sys.argv.index('-psm') + 1]
+        m.update(psm.encode())
+    except ValueError:
+        pass
+
+    input_file = sys.argv[-3]
+    output_file = sys.argv[-2]
+
+    if operation == 'hocr':
+        output_file += '.hocr'
+    elif operation == 'pdf':
+        output_file += '.pdf'
+
+    with open(input_file, 'rb') as f:
+        m.update(f.read())
+
+    cache_name = os.path.join(CACHE_PATH, m.hexdigest())
+    if os.path.exists(cache_name):
+        # Cache hit
+        print("Tesseract cache hit", file=sys.stderr)
+        shutil.copy(cache_name, output_file)
+        sys.exit(0)
+
+    # Cache miss
+    print("Tesseract cache miss", file=sys.stderr)
+
+    # Call tesseract
+    subprocess.check_call(['tesseract'] + sys.argv[1:])
+
+    # Insert file into cache
+    if os.path.exists(output_file):
+        shutil.copy(output_file, cache_name)
+    else:
+        print("Could not find output file", file=sys.stderr)
+
+
+if __name__ == '__main__':
+    main()
@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+import sys
+import img2pdf
+
+
+VERSION_STRING = '''tesseract 3.04.00
+ leptonica-1.72
+  libjpeg 8d : libpng 1.6.19 : libtiff 4.0.6 : zlib 1.2.5
+SPOOFED
+'''
+
+HOCR_TEMPLATE = '''<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+  <title></title>
+  <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+  <meta name='ocr-system' content='tesseract 3.02.02' />
+  <meta name='ocr-capabilities' content='ocr_page ocr_carea ocr_par ocr_line ocrx_word'/>
+ </head>
+ <body>
+  <div class='ocr_page' id='page_1' title='image "x.tif"; bbox 0 0 {0} {1}; ppageno 0'>
+   <div class='ocr_carea' id='block_1_1' title="bbox 0 1 {0} {1}">
+    <p class='ocr_par' dir='ltr' id='par_1' title="bbox 0 1 {0} {1}">
+     <span class='ocr_line' id='line_1' title="bbox 0 1 {0} {1}"><span class='ocrx_word' id='word_1' title="bbox 0 1 {0} {1}"> </span>
+     </span>
+    </p>
+   </div>
+  </div>
+ </body>
+</html>'''
+
+
+def main():
+    if sys.argv[1] == '--version':
+        print(VERSION_STRING, file=sys.stderr)
+        sys.exit(0)
+    elif sys.argv[1] == '--list-langs':
+        print('List of available languages (1):\neng', file=sys.stderr)
+        sys.exit(0)
+    elif sys.argv[-1] == 'hocr':
+        output = sys.argv[-2]
+        with open(output + '.hocr', 'w', encoding='utf-8') as f:
+            f.write(HOCR_TEMPLATE.format('1000', '1000'))
+    elif sys.argv[-1] == 'pdf':
+        inputf = sys.argv[-3]
+        output = sys.argv[-2]
+        pdf_bytes = img2pdf.convert([inputf], dpi=300)
+        with open(output + '.pdf', 'wb') as f:
+            f.write(pdf_bytes)
+    else:
+        print("Spoof doesn't understand arguments", file=sys.stderr)
+        print(sys.argv, file=sys.stderr)
+        sys.exit(1)
+
+    sys.exit(0)
+
+
+if __name__ == '__main__':
+    main()
@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+# © 2015 James R. Barlow: github.com/jbarlow83
+
+from ocrmypdf import hocrtransform
+from ocrmypdf.tesseract import HOCR_TEMPLATE
+from reportlab.pdfgen.canvas import Canvas
+from PIL import Image
+from tempfile import NamedTemporaryFile
+from contextlib import suppress
+import os
+import shutil
+import pytest
+import img2pdf
+import pytest
+import sys
+
+
+if sys.version_info.major < 3:
+    print("Requires Python 3.4+")
+    sys.exit(1)
+
+TESTS_ROOT = os.path.abspath(os.path.dirname(__file__))
+SPOOF_PATH = os.path.join(TESTS_ROOT, 'spoof')
+PROJECT_ROOT = os.path.dirname(TESTS_ROOT)
+OCRMYPDF = os.path.join(PROJECT_ROOT, 'OCRmyPDF.sh')
+TEST_RESOURCES = os.path.join(PROJECT_ROOT, 'tests', 'resources')
+TEST_OUTPUT = os.environ.get(
+    'OCRMYPDF_TEST_OUTPUT',
+    default=os.path.join(PROJECT_ROOT, 'tests', 'output', 'hocrtransform'))
+
+
+def setup_module():
+    with suppress(FileNotFoundError):
+        shutil.rmtree(TEST_OUTPUT)
+    with suppress(FileExistsError):
+        os.makedirs(TEST_OUTPUT)
+    with open(_make_output('blank.hocr'), 'w') as f:
+        f.write(HOCR_TEMPLATE)
+
+
+def _make_input(input_basename):
+    return os.path.join(TEST_RESOURCES, input_basename)
+
+
+def _make_output(output_basename):
+    return os.path.join(TEST_OUTPUT, output_basename)
+
+
+def test_mono_image():
+    im = Image.new('1', (8, 8), 0)
+    for n in range(8):
+        im.putpixel((n, n), 1)
+    im.save(_make_output('mono.tif'), format='TIFF')
+
+    hocr = hocrtransform.HocrTransform(_make_output('blank.hocr'), 300)
+    hocr.to_pdf(_make_output('mono.pdf'), imageFileName=_make_output('mono.tif'))
+
+
+
+
+
@@ -2,7 +2,7 @@
 # © 2015 James R. Barlow: github.com/jbarlow83

 from __future__ import print_function
-from subprocess import Popen, PIPE, check_output
+from subprocess import Popen, PIPE, check_output, check_call
 import os
 import shutil
 from contextlib import suppress
@@ -18,27 +18,27 @@ if sys.version_info.major < 3:
    sys.exit(1)

 TESTS_ROOT = os.path.abspath(os.path.dirname(__file__))
+SPOOF_PATH = os.path.join(TESTS_ROOT, 'spoof')
 PROJECT_ROOT = os.path.dirname(TESTS_ROOT)
 OCRMYPDF = os.path.join(PROJECT_ROOT, 'OCRmyPDF.sh')
 TEST_RESOURCES = os.path.join(PROJECT_ROOT, 'tests', 'resources')
 TEST_OUTPUT = os.environ.get(
    'OCRMYPDF_TEST_OUTPUT',
-    default=os.path.join(PROJECT_ROOT, 'tests', 'output'))
-TEST_BINARY_PATH = os.path.join(TEST_OUTPUT, 'fakebin')
+    default=os.path.join(PROJECT_ROOT, 'tests', 'output', 'main'))


 def setup_module():
    with suppress(FileNotFoundError):
        shutil.rmtree(TEST_OUTPUT)
    with suppress(FileExistsError):
-        os.mkdir(TEST_OUTPUT)
+        os.makedirs(TEST_OUTPUT)


-def run_ocrmypdf_sh(input_file, output_file, *args):
+def run_ocrmypdf_sh(input_file, output_file, *args, env=None):
    sh_args = ['sh', OCRMYPDF] + list(args) + [input_file, output_file]
    sh = Popen(
        sh_args, close_fds=True, stdout=PIPE, stderr=PIPE,
-        universal_newlines=True)
+        universal_newlines=True, env=env)
    out, err = sh.communicate()
    return sh, out, err

@@ -51,12 +51,12 @@ def _make_output(output_basename):
    return os.path.join(TEST_OUTPUT, output_basename)


-def check_ocrmypdf(input_basename, output_basename, *args):
+def check_ocrmypdf(input_basename, output_basename, *args, env=None):
    input_file = _make_input(input_basename)
    output_file = _make_output(output_basename)

-    sh, _, err = run_ocrmypdf_sh(input_file, output_file, *args)
-    assert sh.returncode == 0, err
+    sh, out, err = run_ocrmypdf_sh(input_file, output_file, *args, env=env)
+    assert sh.returncode == 0, dict(stdout=out, stderr=err)
    assert os.path.exists(output_file), "Output file not created"
    assert os.stat(output_file).st_size > 100, "PDF too small or empty"
    return output_file
@@ -77,13 +77,32 @@ def run_ocrmypdf_env(input_basename, output_basename, *args, env=None):
    return p, out, err


-def test_quick():
-    check_ocrmypdf('c02-22.pdf', 'test_quick.pdf')
+@pytest.fixture
+def spoof_tesseract_noop():
+    env = os.environ.copy()
+    program = os.path.join(SPOOF_PATH, 'tesseract_noop.py')
+    check_call(['chmod', "+x", program])
+    env['OCRMYPDF_TESSERACT'] = program
+    return env


-def test_deskew():
+@pytest.fixture
+def spoof_tesseract_cache():
+    env = os.environ.copy()
+    program = os.path.join(SPOOF_PATH, "tesseract_cache.py")
+    check_call(['chmod', '+x', program])
+    env['OCRMYPDF_TESSERACT'] = program
+    return env
+
+
+def test_quick(spoof_tesseract_noop):
+    check_ocrmypdf('c02-22.pdf', 'test_quick.pdf', env=spoof_tesseract_noop)
+
+
+def test_deskew(spoof_tesseract_noop):
    # Run with deskew
-    deskewed_pdf = check_ocrmypdf('skew.pdf', 'test_deskew.pdf', '-d')
+    deskewed_pdf = check_ocrmypdf(
+        'skew.pdf', 'test_deskew.pdf', '-d', env=spoof_tesseract_noop)

    # Now render as an image again and use Leptonica to find the skew angle
    # to confirm that it was deskewed
@@ -110,29 +129,34 @@ def test_deskew():
    assert -0.5 < skew_angle < 0.5, "Deskewing failed"


-def test_clean():
-    check_ocrmypdf('skew.pdf', 'test_clean.pdf', '-c')
+def test_clean(spoof_tesseract_noop):
+    check_ocrmypdf('skew.pdf', 'test_clean.pdf', '-c', env=spoof_tesseract_noop)


-def check_exotic_image(pdf, renderer):
+@pytest.mark.parametrize("pdf,renderer", [
+    ('palette.pdf', 'hocr'),
+    ('palette.pdf', 'tesseract'),
+    ('cmyk.pdf', 'hocr'),
+    ('cmyk.pdf', 'tesseract'),
+    ('ccitt.pdf', 'hocr'),
+    ('ccitt.pdf', 'tesseract'),
+    ('jbig2.pdf', 'hocr'),
+    ('jbig2.pdf', 'tesseract')
+])
+def test_exotic_image(spoof_tesseract_cache, pdf, renderer):
    check_ocrmypdf(
        pdf,
        'test_{0}_{1}.pdf'.format(pdf, renderer),
        '-dc',
-        '--pdf-renderer', renderer)
+        '-v', '1',
+        '--pdf-renderer', renderer, env=spoof_tesseract_cache)


-def test_exotic_image():
-    yield check_exotic_image, 'palette.pdf', 'hocr'
-    yield check_exotic_image, 'palette.pdf', 'tesseract'
-    yield check_exotic_image, 'cmyk.pdf', 'hocr'
-    yield check_exotic_image, 'cmyk.pdf', 'tesseract'
-
-
-def test_preserve_metadata():
+def test_preserve_metadata(spoof_tesseract_noop):
    pdf_before = pypdf.PdfFileReader(_make_input('graph.pdf'))

-    output = check_ocrmypdf('graph.pdf', 'test_metadata_preserve.pdf')
+    output = check_ocrmypdf('graph.pdf', 'test_metadata_preserve.pdf',
+                            env=spoof_tesseract_noop)

    pdf_after = pypdf.PdfFileReader(output)

@@ -140,7 +164,7 @@ def test_preserve_metadata():
        assert pdf_before.documentInfo[key] == pdf_after.documentInfo[key]


-def test_override_metadata():
+def test_override_metadata(spoof_tesseract_noop):
    input_file = _make_input('c02-22.pdf')
    output_file = _make_output('test_override_metadata.pdf')

@@ -152,7 +176,8 @@ def test_override_metadata():
        input_file, output_file,
        '--title', german,
        '--author', chinese,
-        '--subject', high_unicode)
+        '--subject', high_unicode,
+        env=spoof_tesseract_noop)

    assert p.returncode == ExitCode.ok

@@ -171,10 +196,15 @@ def test_override_metadata():
    assert pdfinfo.get('Keywords', '') == ''


-def check_oversample(renderer):
+@pytest.mark.parametrize('renderer', [
+    'hocr',
+    'tesseract',
+    ])
+def test_oversample(spoof_tesseract_cache, renderer):
    oversampled_pdf = check_ocrmypdf(
        'skew.pdf', 'test_oversample_%s.pdf' % renderer, '--oversample', '300',
-        '--pdf-renderer', renderer)
+        '-f',
+        '--pdf-renderer', renderer, env=spoof_tesseract_cache)

    pdfinfo = pdf_get_all_pageinfo(oversampled_pdf)

@@ -182,111 +212,61 @@ def check_oversample(renderer):
    assert abs(pdfinfo[0]['xres'] - 300) < 1


-def test_oversample():
-    yield check_oversample, 'hocr'
-    yield check_oversample, 'tesseract'
-
-
 def test_repeat_ocr():
    sh, _, _ = run_ocrmypdf_sh('graph_ocred.pdf', 'wontwork.pdf')
    assert sh.returncode != 0


-def test_force_ocr():
-    out = check_ocrmypdf('graph_ocred.pdf', 'test_force.pdf', '-f')
+def test_force_ocr(spoof_tesseract_cache):
+    out = check_ocrmypdf('graph_ocred.pdf', 'test_force.pdf', '-f',
+                         env=spoof_tesseract_cache)
    pdfinfo = pdf_get_all_pageinfo(out)
    assert pdfinfo[0]['has_text']


-def test_skip_ocr():
-    check_ocrmypdf('graph_ocred.pdf', 'test_skip.pdf', '-s')
+def test_skip_ocr(spoof_tesseract_cache):
+    check_ocrmypdf('graph_ocred.pdf', 'test_skip.pdf', '-s',
+                   env=spoof_tesseract_cache)


-def test_argsfile():
+def test_argsfile(spoof_tesseract_noop):
    with open(_make_output('test_argsfile.txt'), 'w') as argsfile:
        print('--title', 'ArgsFile Test', '--author', 'Test Cases',
              sep='\n', end='\n', file=argsfile)
    check_ocrmypdf('graph.pdf', 'test_argsfile.pdf',
-                   '@' + _make_output('test_argsfile.txt'))
+                   '@' + _make_output('test_argsfile.txt'),
+                   env=spoof_tesseract_noop)


-def check_ocr_timeout(renderer):
+@pytest.mark.parametrize('renderer', [
+    'hocr',
+    'tesseract',
+    ])
+def test_ocr_timeout(renderer):
    out = check_ocrmypdf('skew.pdf', 'test_timeout_%s.pdf' % renderer,
                         '--tesseract-timeout', '1.0')
    pdfinfo = pdf_get_all_pageinfo(out)
    assert pdfinfo[0]['has_text'] == False


-def test_ocr_timeout():
-    yield check_ocr_timeout, 'hocr'
-    yield check_ocr_timeout, 'tesseract'
-
-
-def test_skip_big():
+def test_skip_big(spoof_tesseract_cache):
    out = check_ocrmypdf('enormous.pdf', 'test_enormous.pdf',
-                         '--skip-big', '10')
+                         '--skip-big', '10', env=spoof_tesseract_cache)
    pdfinfo = pdf_get_all_pageinfo(out)
    assert pdfinfo[0]['has_text'] == False


-def check_maximum_options(renderer):
+@pytest.mark.parametrize('renderer', [
+    'hocr',
+    'tesseract',
+    ])
+def test_maximum_options(spoof_tesseract_cache, renderer):
    check_ocrmypdf(
        'multipage.pdf', 'test_multipage%s.pdf' % renderer,
        '-d', '-c', '-i', '-g', '-f', '-k', '--oversample', '300',
        '--skip-big', '10', '--title', 'Too Many Weird Files',
-        '--author', 'py.test', '--pdf-renderer', renderer)
-
-
-def test_maximum_options():
-    yield check_maximum_options, 'hocr'
-    yield check_maximum_options, 'tesseract'
-
-
-def override_binary(binary, replacement):
-    '''Create a directory that contains a symlink named 'binary' that
-    points to replacement, another program to use in place of the
-    regular binary for testing.
-
-    override_binary('gs', 'replace_gs.py') will create an environment
-    in which "gs" will invoke replace_gs.py.
-
-    Not thread-safe with other test at the moment.
-
-    Returns the os.environ["PATH"] string under which this binary will
-    be invoked.'''
-
-    replacement_path = os.path.abspath(os.path.join(TESTS_ROOT,
-                                                    replacement))
-    subdir = os.path.splitext(os.path.basename(replacement))[0]
-    binary_path = os.path.abspath(os.path.join(TEST_BINARY_PATH,
-                                               subdir,
-                                               binary))
-    with suppress(FileExistsError):
-        os.makedirs(os.path.dirname(binary_path))
-    assert os.path.isdir(os.path.dirname(binary_path))
-    assert not os.path.lexists(binary_path)
-    print("symlink %s -> %s" % (replacement_path, binary_path))
-    os.symlink(replacement_path, binary_path)
-
-    os.chmod(replacement_path, int('755', base=8))
-
-    return os.path.dirname(binary_path) + os.pathsep + os.environ["PATH"]
-
-
-@pytest.fixture
-def break_ghostscript_pdfa():
-    return override_binary('gs', 'replace_ghostscript_nopdfa.py')
-
-
-@pytest.mark.skipif(os.environ.get('OCRMYPDF_IN_DOCKER', False),
-                    reason="Requires writable filesystem")
-def test_ghostscript_pdfa_fails(break_ghostscript_pdfa):
-    env = os.environ.copy()
-    env['PATH'] = break_ghostscript_pdfa
-
-    p, out, err = run_ocrmypdf_env(
-        'graph_ocred.pdf', 'not_a_pdfa.pdf', '-v', '1', '--skip-text', env=env)
-    assert p.returncode == ExitCode.ok, err  # no longer using JHOVE PDFA check
+        '--author', 'py.test', '--pdf-renderer', renderer,
+        env=spoof_tesseract_cache)


 def test_tesseract_missing_tessdata():
@@ -310,9 +290,9 @@ def test_blank_input_pdf():
    assert p.returncode == ExitCode.ok


-def test_french():
+def test_french(spoof_tesseract_cache):
    p, out, err = run_ocrmypdf_env(
-        'francais.pdf', 'francais.pdf', '-l', 'fra')
+        'francais.pdf', 'francais.pdf', '-l', 'fra', env=spoof_tesseract_cache)
    assert p.returncode == ExitCode.ok, \
        "This test may fail if Tesseract language packs are missing"

@@ -323,16 +303,18 @@ def test_klingon():
    assert p.returncode == ExitCode.bad_args


-def test_missing_docinfo():
+def test_missing_docinfo(spoof_tesseract_noop):
    p, out, err = run_ocrmypdf_env(
-        'missing_docinfo.pdf', 'missing_docinfo.pdf', '-l', 'eng', '-c')
+        'missing_docinfo.pdf', 'missing_docinfo.pdf', '-l', 'eng', '-c',
+        env=spoof_tesseract_noop)
    assert p.returncode == ExitCode.ok, err


-def test_uppercase_extension():
+def test_uppercase_extension(spoof_tesseract_noop):
    shutil.copy(_make_input("skew.pdf"), _make_input("UPPERCASE.PDF"))
    try:
-        check_ocrmypdf("UPPERCASE.PDF", "UPPERCASE_OUT.PDF")
+        check_ocrmypdf("UPPERCASE.PDF", "UPPERCASE_OUT.PDF",
+                       env=spoof_tesseract_noop)
    finally:
        os.unlink(_make_input("UPPERCASE.PDF"))

@@ -354,3 +336,34 @@ def test_input_file_not_a_pdf():
    assert sh.returncode == ExitCode.input_file
    assert (input_file in out or input_file in err)

+
+def test_qpdf_repair_fails():
+    env = os.environ.copy()
+    env['OCRMYPDF_QPDF'] = os.path.abspath('./spoof/qpdf_dummy_return2.py')
+    p, out, err = run_ocrmypdf_env(
+        '-v', '1',
+        'c02-22.pdf', 'wont_be_created.pdf', env=env)
+    print(out)
+    print(err)
+    assert p.returncode == ExitCode.input_file
+
+
+def test_encrypted():
+    p, out, err = run_ocrmypdf_env('skew-encrypted.pdf', 'wont_be_created.pdf')
+    assert p.returncode == ExitCode.input_file
+    assert out.find('password')
+
+
+@pytest.mark.parametrize('renderer', [
+    'hocr',
+    'tesseract',
+    ])
+def test_pagesegmode(renderer, spoof_tesseract_cache):
+    check_ocrmypdf(
+        'skew.pdf', 'test_psm_%s.pdf' % renderer,
+        '--tesseract-pagesegmode', '7',
+        '-v', '1',
+        '--pdf-renderer', renderer, env=spoof_tesseract_cache)
+
+
+
@@ -10,21 +10,37 @@ import os
 import shutil
 import pytest
 import img2pdf
-from pkg_resources import Requirement, resource_filename
 import pytest
+import sys

-req = Requirement.parse('ocrmypdf')

+if sys.version_info.major < 3:
+    print("Requires Python 3.4+")
+    sys.exit(1)
+
+TESTS_ROOT = os.path.abspath(os.path.dirname(__file__))
+SPOOF_PATH = os.path.join(TESTS_ROOT, 'spoof')
+PROJECT_ROOT = os.path.dirname(TESTS_ROOT)
+OCRMYPDF = os.path.join(PROJECT_ROOT, 'OCRmyPDF.sh')
+TEST_RESOURCES = os.path.join(PROJECT_ROOT, 'tests', 'resources')
 TEST_OUTPUT = os.environ.get(
    'OCRMYPDF_TEST_OUTPUT',
-    default=os.path.join(os.path.dirname(__file__), 'output'))
+    default=os.path.join(PROJECT_ROOT, 'tests', 'output', 'pageinfo'))


 def setup_module():
    with suppress(FileNotFoundError):
        shutil.rmtree(TEST_OUTPUT)
    with suppress(FileExistsError):
-        os.mkdir(TEST_OUTPUT)
+        os.makedirs(TEST_OUTPUT)
+
+
+def _make_input(input_basename):
+    return os.path.join(TEST_RESOURCES, input_basename)
+
+
+def _make_output(output_basename):
+    return os.path.join(TEST_OUTPUT, output_basename)


 def test_single_page_text():
@@ -100,9 +116,8 @@ def test_single_page_inline_image():
        pageinfo.pdf_get_all_pageinfo(filename)


-@pytest.mark.skipif(True, reason="need to fix packaging error")
 def test_jpeg():
-    filename = resource_filename(req, 'tests/resources/c02-22.pdf')
+    filename = _make_input('c02-22.pdf')

    pdfinfo = pageinfo.pdf_get_all_pageinfo(filename)