v4.0: release notes

Since the corrected font is provided in the Docker image, there's no
reason to show the warning.

.dockerignore

@@ -0,0 +1,31 @@
+*.ipynb
+*.pdf
+*.pyc
+*.rst
+*.sublime*
+*/*.pyc
+*/*/*.pyc
+*/*/*/*.pyc
+*/*/*/*/*.pyc
+*/*/*/*/*/*.pyc
+*/*/*/*/*/*/*.pyc
+*/*/*/*/*/*/*/*.pyc
+.cache/
+.git/
+.ipynb_checkpoints/
+.ruffus_history.sqlite
+bin/
+build/
+dist/
+htmlcov/
+include/
+lib/
+MANIFEST.in
+ocrmypdf.egg-info/
+staging/
+tests/cache/
+tests/output/
+tests/resources/private/
+tmp/
+venv-3.4/
+venv-3.5/

.gitignore

@@ -1,17 +1,29 @@
-tmp/
-log/
+# Development environment
 *.pyc
-tests/output/
-.ruffus_history.sqlite
 *.sublime-*
-/*.pdf
+venv-*/
+pyvenv.cfg
+
+# Package building
+*.egg-info/
+.cache/
+.eggs/
 build/
 dist/
-*.egg-info/
-venv/
-*/test/output
-bin/
-include/
-lib/
-pip-selfcheck.json
-pyvenv.cfg
+
+# Automatically generated files
+ocrmypdf/lib/_*.py
+ocrmypdf/version.py
+
+# Code coverage
+.coverage
+htmlcov/
+
+# Testing
+log/
+/*.pdf
+.ipynb_checkpoints/
+tests/cache/
+tests/output/
+tests/resources/private
+tmp/

.travis.yml

@@ -0,0 +1,46 @@
+language: python
+
+cache: pip
+
+cache:
+    directories:
+        - $HOME/.ccache
+        - tarballs
+        - tests/cache
+
+python:
+    - 3.4
+
+before_cache:
+    - rm -f $HOME/.cache/pip/log/debug.log
+
+before_install:
+    # Ubuntu packages
+    - sudo add-apt-repository ppa:evl.ms/precise -y  # for Ghostscript 9.15
+    - sudo add-apt-repository ppa:lyrasis/precise-backports -y  # for Tesseract 3.03
+    - sudo add-apt-repository ppa:b-eltzner/qpdfview-exp -y  # for QPDF 5
+    - sudo add-apt-repository ppa:itachi-san/ffmpeg -y  # for libav 11.2 (for unpaper)
+    - sudo apt-get update -qq  # must go after all add-apt-repo
+    - sudo apt-get install -y ghostscript tesseract-ocr tesseract-ocr-deu tesseract-ocr-eng tesseract-ocr-fra qpdf poppler-utils gcc libavformat-dev libavcodec-dev libavutil-dev automake make pkg-config xsltproc libffi-dev
+
+    # pip
+    - pip install --upgrade pip
+
+    # Download, make and install unpaper (using ccache)
+    - mkdir -p tarballs
+    - "[ -f tarballs/unpaper-6.1.tar.xz ] || wget -q https://www.flameeyes.eu/files/unpaper-6.1.tar.xz -O tarballs/unpaper-6.1.tar.xz"
+    - tar -xvf tarballs/unpaper-6.1.tar.xz
+    - export PATH="/usr/lib/ccache:$PATH"
+    - pushd unpaper-6.1 && ./configure --prefix=/usr && make -j && sudo make install && popd
+
+install:
+    - pip install -r requirements.txt
+    - pip install -r test_requirements.txt
+
+script:
+    - python setup.py clean
+    - python setup.py install
+    - python setup.py test
+
+os:
+    - linux

Dockerfile

@@ -0,0 +1,85 @@
+# OCRmyPDF
+#
+# VERSION               3.2
+FROM      debian:stretch
+MAINTAINER James R. Barlow <jim@purplerock.ca>
+
+# Add unprivileged user
+RUN useradd docker \
+  && mkdir /home/docker \
+  && chown docker:docker /home/docker
+
+# Update system and install our dependencies
+# If this command takes too Docker hub's automated build will timeout,
+# so try it in portions
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  locales \
+  python3 \
+  python3-pip \
+  python3-venv \
+  python3-reportlab \
+  python3-pil \
+  python3-wheel
+
+RUN apt-get install -y --no-install-recommends \
+  unpaper \
+  ghostscript \
+  qpdf \
+  poppler-utils
+
+RUN apt-get install -y --no-install-recommends \
+  tesseract-ocr \
+  tesseract-ocr-deu tesseract-ocr-spa tesseract-ocr-eng tesseract-ocr-fra
+
+RUN apt-get install -qy --no-install-recommends \
+  libffi-dev \
+  libpython3-dev \
+  gcc
+
+# Enforce UTF-8
+# Borrowed from https://index.docker.io/u/crosbymichael/python/ 
+RUN dpkg-reconfigure locales && \
+  locale-gen C.UTF-8 && \
+  /usr/sbin/update-locale LANG=C.UTF-8
+ENV LC_ALL C.UTF-8
+
+
+# Set up a Python virtualenv and take all of the system packages, so we can
+# rely on the platform packages rather than importing GCC and compiling them
+RUN pyvenv /appenv \
+  && pyvenv --system-site-packages /appenv
+
+COPY . /application/
+
+# Replace stock Tesseract 3.04.00 font with improved sharp2.ttf that resolves
+# issues in many PDF viewers.
+# Discussion is in https://github.com/tesseract-ocr/tesseract/issues/182
+COPY ./share/sharp2.ttf /usr/share/tesseract-ocr/tessdata/pdf.ttf
+RUN chmod 644 /usr/share/tesseract-ocr/tessdata/pdf.ttf
+
+# Set this here to force a docker version, allowing non-tagged versions to
+# be built
+# ENV SETUPTOOLS_SCM_PRETEND_VERSION=v3.3.0
+
+# Install application and dependencies
+# In this arrangement Pillow and reportlab will be provided by the system
+RUN . /appenv/bin/activate; \
+  pip install --upgrade pip \
+  && pip install /application \
+  && pip install --no-cache-dir -r /application/test_requirements.txt
+
+# Remove the junk
+RUN apt-get remove -qy gcc
+RUN apt-get autoremove -y && apt-get clean -y
+RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/*
+
+USER docker
+WORKDIR /home/docker
+
+ENV OCRMYPDF_TEST_OUTPUT=/tmp/test-output
+ENV OCRMYPDF_IN_DOCKER=1
+ENV OCRMYPDF_SHARP_TTF=1
+
+# Must use array form of ENTRYPOINT
+# Non-array form does not append other arguments, because that is "intuitive"
+ENTRYPOINT ["/application/docker-wrapper.sh"]

Dockerfile.polyglot

@@ -0,0 +1,16 @@
+# OCRmyPDF polyglot
+#
+# VERSION               3.2
+FROM      jbarlow83/ocrmypdf:latest
+MAINTAINER James R. Barlow <jim@purplerock.ca>
+
+# Update system and install our dependencies
+USER root
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  tesseract-ocr-all 
+
+USER docker
+
+# Must use array form of ENTRYPOINT
+# Non-array form does not append other arguments, because that is "intuitive"
+ENTRYPOINT ["/application/docker-wrapper.sh"]

MANIFEST.in

@@ -1,4 +1,4 @@
-recursive-include ocrmypdf/jhove/bin *.jar
-recursive-include ocrmypdf/jhove/lib *.jar
-recursive-include ocrmypdf/jhove/conf *.conf
 recursive-exclude tests/output *
+include requirements.txt
+include test_requirements.txt
+include share/*

README.rst

@@ -8,23 +8,21 @@ Main features
 -------------
 
 -  Generates a searchable
-   `PDF/A <https://en.wikipedia.org/?title=PDF/A>`__ file from a PDF
-   file containing only images
--  Places OCRed text accurately below the image to ease copy / paste
+   `PDF/A <https://en.wikipedia.org/?title=PDF/A>`__ file from a regular PDF
+-  Places OCR text accurately below the image to ease copy / paste
 -  Keeps the exact resolution of the original embedded images
-
-   -  or if requested oversamples the images before OCRing so as to get
-      better results
-
+-  When possible, inserts OCR information as a "lossless" operation without rendering vector information
+-  Keeps file size about the same
 -  If requested deskews and/or cleans the image before performing OCR
--  Validates the generated file against the PDF/A-1b specification using
-   `JHOVE <http://jhove.sourceforge.net/>`__
+-  Validates input and output files
 -  Provides debug mode to enable easy verification of the OCR results
--  Processes several pages in parallel when more than one CPU core is
+-  Processes pages in parallel when more than one CPU core is
    available
--  Uses Tesseract OCR engine
+-  Uses `Tesseract OCR <https://github.com/tesseract-ocr/tesseract>`__ engine
+-  Supports the `39 languages <https://code.google.com/p/tesseract-ocr/downloads/list>`__ recognized by Tesseract
+-  Battle-tested on thousands of PDFs, a test suite and continuous integration
 
-For details: please consult the `release notes <RELEASE_NOTES.rst>`__
+For details: please consult the `release notes <RELEASE_NOTES.rst>`__.
 
 Motivation
 ----------
@@ -43,35 +41,178 @@ Linux/UNIX: I found many, but none of them were really satisfying.
 ... so I decided to develop my own tool (using various existing scripts
 as an inspiration)
 
-Install
--------
+Installation
+------------
 
-Download OCRmyPDF here: https://github.com/fritz-hh/OCRmyPDF/releases
+Download OCRmyPDF here: https://github.com/jbarlow83/OCRmyPDF/releases
 
-You can install it within a virtual environment or system-wide. 
+You can install it to a Python virtual environment or system-wide. 
 
-To install the HEAD revision from sources in development mode::
+Installing the Docker image
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-   git clone -b master https://github.com/fritz-hh/OCRmyPDF.git
-   cd OCRmyPDF
-   pip3 install -e .
+For many users, installing the Docker image will be easier than installing all of OCRmyPDF's dependencies. For Windows, it is the only option.
+
+If you have `Docker <https://docs.docker.com/>`__ installed on your system, you can install
+a Docker image of the latest release.
+
+Follow the Docker installation instructions for your platform.  If you can run this command
+successfully, your system is ready to download and execute the image::
+
+   docker run hello-world
    
-On certain Linux/UNIX platforms such as Ubuntu, you may need to use 
+OCRmyPDF will use all available CPU cores.  By default, the VirtualBox machine instance on Windows and OS X has only a single CPU core enabled. Use the VirtualBox Manager to determine the name of your Docker engine host, and then follow these optional steps to enable multiple CPUs::
+
+   # Optional step for Mac OS X users
+   docker-machine stop "yourVM"
+   VBoxManage modifyvm "yourVM" --cpus 2  # or whatever number of core is desired
+   docker-machine start "yourVM"
+   eval $(docker-machine env "yourVM")
+
+Assuming you have a Docker engine running somewhere, you can run these commands to download
+the image::
+
+   docker pull jbarlow83/ocrmypdf
+
+Then tag it to give a more convenient name, just ocrmypdf::
+
+   docker tag jbarlow83/ocrmypdf ocrmypdf
+
+This image contains language packs for English, French, Spanish and German. The alternative "polyglot" image provides `all available language packs <https://github.com/tesseract-ocr/tesseract/blob/master/doc/tesseract.1.asc#languages>`__::
+
+   # Alternative step: If you need all language packs
+   docker pull jbarlow83/ocrmypdf-polyglot
+   docker tag jbarlow83/ocrmypdf-polyglot ocrmypdf
+
+You can then run ocrmypdf using the command::
+
+   docker run ocrmypdf --help
+  
+To execute the OCRmyPDF on a local file, you must `provide a writable volume to the Docker image <https://docs.docker.com/userguide/dockervolumes/>`__, such as this in this template::
+
+   docker run -v "$(pwd):/home/docker" <other docker arguments>   ocrmypdf <your arguments to ocrmypdf>
+
+In this worked example, the current working directory contains an input file called ``test.pdf`` and the output will go to ``output.pdf``:: 
+
+   docker run -v "$(pwd):/home/docker"   ocrmypdf --skip-text test.pdf output.pdf
+
+Note that ``ocrmypdf`` has its own separate ``-v VERBOSITYLEVEL`` argument to control debug verbosity. All Docker arguments should before the ``ocrmypdf`` image name and all arguments to ``ocrmypdf`` should be listed after.
+
+
+Installing on Mac OS X
+~~~~~~~~~~~~~~~~~~~~~~
+
+These instructions probably work on all Mac OS X versions later than 10.7 (Lion). OCRmyPDF is known to work on Yosemite and El Capitan, and regularly tested on El Capitan.
+
+If it's not already present, `install Homebrew <http://brew.sh/>`__.
+
+Update Homebrew::
+
+   brew update
+   
+Install or upgrade the required Homebrew packages, if any are missing::
+
+   brew install libpng openjpeg jbig2dec     # image libraries
+   brew install qpdf
+   brew install ghostscript
+   brew install python3
+   brew install libxml2
+   brew install leptonica
+   brew install tesseract
+   
+Update the homebrew pip and install Pillow::
+
+   pip3 install --upgrade pip
+   pip3 install --upgrade pillow
+
+You can then install OCRmyPDF from PyPI::
+
+   pip3 install ocrmypdf
+
+The command line program should now be available::
+
+   ocrmypdf --help
+
+Installing on Ubuntu 14.04 LTS
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Installing on Ubuntu 14.04 LTS (trusty) is more difficult than other options, because of certain bugs in Python package installation.
+
+Update apt-get::
+
+   sudo apt-get update
+   sudo apt-get upgrade
+   
+Install system dependencies::
+
+   sudo apt-get install \
+      zlib1g-dev \
+      libjpeg-dev \
+      ghostscript \
+      tesseract-ocr \
+      qpdf \
+      unpaper \
+      python3-pip \
+      python3-pil \
+      python3-pytest \
+      python3-reportlab
+
+If you wish install OCRmyPDF to the system Python, then install as follows (note this installs new packages
+into your system Python, which could interfere with other programs)::
+
+   sudo pip3 install ocrmypdf
+   
+If you wish to install OCRmyPDF to a virtual environment to isolate system Python from modified, you can
+follow these steps.  This includes a workaround `for a known, unresolved issue in Ubuntu 14.04's ensurepip
+package <http://www.thefourtheye.in/2014/12/Python-venv-problem-with-ensurepip-in-Ubuntu.html>`__::
+
+   sudo apt-get install python3-venv
+   python3 -m venv venv-ocrmypdf --without-pip
+   source venv-ocrmypdf/bin/activate
+   wget -O - -o /dev/null https://bootstrap.pypa.io/get-pip.py | python
+   deactivate
+   pyvenv --system-site-packages venv-ocrmypdf
+   source venv-ocrmypdf/bin/activate
+   pip install ocrmypdf
+
+Ubuntu 14.04 only installs ``unpaper`` version 0.4.2, which is not supported by OCRmyPDF because it is produces invalid output. This program is an optional dependency, and provides page deskewing and cleaning. See `Dockerfile <Dockerfile>`__ for an example of how to building unpaper 6.1 from source. If you choose to install unpaper later, OCRmyPDF will use the foremost version on the system PATH.
+
+Installing on Windows
+~~~~~~~~~~~~~~~~~~~~~
+
+Direct installation on Windows is not possible.  Install the Docker container as described above.
+
+      
+Installing HEAD revision from sources
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you have ``git`` and ``python3.4`` or ``python3.5`` installed, you can install from source. When the ``pip`` installer runs,
+it will alert you if dependencies are missing.
+
+To install the HEAD revision from sources in the current Python 3 environment::
+
+   pip3 install git+https://github.com/jbarlow83/OCRmyPDF.git
+
+Or, to install in `development mode <https://pythonhosted.org/setuptools/setuptools.html#development-mode>`__,  allowing customization of OCRmyPDF, use the ``-e`` flag::
+
+   pip3 install -e git+https://github.com/jbarlow83/OCRmyPDF.git
+   
+On certain Linux distributions such as Ubuntu, you may need to use 
 run the install command as superuser::
 
-   sudo pip3 install -e .
+   sudo pip3 install [-e] git+https://github.com/jbarlow83/OCRmyPDF.git
    
 Note that this will alter your system's Python distribution. If you prefer 
 to not install as superuser, you can install the package in a Python virtual environment::
 
-   git clone -b master https://github.com/fritz-hh/OCRmyPDF.git
+   git clone -b master https://github.com/jbarlow83/OCRmyPDF.git
    pyvenv venv
    source venv/bin/activate
    cd OCRmyPDF
-   pip3 install -e .
+   pip3 install .
 
-If your platform does not have ``pip3``, make sure that Python 3.4+ and the `pip` 
-package are installed.
+However, ``ocrmypdf`` will only be accessible on the system PATH after
+you activate the virtual environment.
 
 To run the program::
    
@@ -82,6 +223,18 @@ need to be installed. The script requires specific versions of the
 dependencies. Older version than the ones mentioned in the release notes
 are likely not to be compatible to OCRmyPDF.
 
+Languages
+---------
+
+OCRmyPDF uses Tesseract for OCR, and relies on its language packs. For Linux users,
+you can often find packages that provide language packs::
+
+   # Debian/Ubuntu users
+   sudo apt-get install tesseract-ocr-chi-sim
+   
+You can then pass the ``-l LANG`` argument to OCRmyPDF to give a hint as to what languages it should search for. Multiple
+languages can be requested.
+
 Support
 -------
 
@@ -89,13 +242,12 @@ In case you detect an issue, please:
 
 -  Check if your issue is already known
 -  If no problem report exists on github, please create one here:
-   https://github.com/fritz-hh/OCRmyPDF/issues
+   https://github.com/jbarlow83/OCRmyPDF/issues
 -  Describe your problem thoroughly
 -  Append the console output of the script when running the debug mode
-   (-v 1 option)
+   (``-v 1`` option)
 -  If possible provide your input PDF file as well as the content of the
-   temporary folder (using a file sharing service like
-   www.file-upload.net)
+   temporary folder (using a file sharing service like Dropbox)
 
 Press & Media
 -------------

RELEASE_NOTES.rst

@@ -3,7 +3,101 @@ RELEASE NOTES
 
 Please always read this file before installing the package
 
-Download software here: https://github.com/fritz-hh/OCRmyPDF/tags
+Download software here: https://github.com/jbarlow83/OCRmyPDF/tags
+
+
+v4.0:
+=====
+
+New features
+------------
+
+-  Automatic page rotation (``-r``) is now available. It uses ignores any prior rotation information
+   on PDFs and sets rotation based on the dominant orientation of detectable text. This feature is
+   fairly reliable but some false positives occur especially if there is not much text to work with. (#4) 
+-  Deskewing is now performed using Leptonica instead of unpaper. Leptonica deskews images faster and
+   more reliably than unpaper.
+
+
+Fixes
+-----
+
+-  Fixed an issue where lossless reconstruction could cause some pages to be appear incorrectly
+   rotated in the page had a /Rotate tag applied (#50)
+-  Fixed an issue where lossless reconstruction could misalign the graphics layer with respect to
+   text layer if the page had been cropped such that its origin is not (0, 0) (#49)
+
+
+Changes
+-------
+
+-  Logging output much is now better
+-  ``--deskew`` is now performed by Leptonica instead of unpaper (#25)
+-  libffi is now required
+-  Some changes were made to the Docker and Travis build environments to support libffi
+-  ``--pdf-renderer=tesseract`` now displays a warning if the Tesseract version is less than 3.04.01,
+   the planned release that will include fixes to an important OCR text rendering bug in Tesseract 3.04.00.
+   You can also manually install ./share/sharp2.ttf on top of pdf.ttf in your Tesseract tessdata folder
+   to correct the problem.
+
+
+v3.2.1:
+=======
+
+Changes
+-------
+
+-  Fixed issue #47 "convert() got and unexpected keyword argument 'dpi'" by upgrading to img2pdf 0.2
+-  Tweaked the Dockerfiles
+
+
+v3.2:
+=====
+
+New features
+------------
+
+-  Lossless reconstruction: when possible, OCRmyPDF will inject text layers without 
+   otherwise manipulating the content and layout of a PDF page. For example, a PDF containing a mix
+   of vector and raster content would see the vector content preserved. Images may still be transcoded
+   during PDF/A conversion.  (``--deskew`` and ``--clean-final`` disable this mode, necessarily.)
+-  New argument ``--tesseract-pagesegmode`` allows you to pass page segmentation arguments to Tesseract OCR.
+   This helps for two column text and other situations that confuse Tesseract.
+-  Added a new "polyglot" version of the Docker image, that generates Tesseract with all languages packs installed,
+   for the polyglots among us. It is much larger.
+
+Changes
+-------
+
+-  JPEG transcoding quality is now 95 instead of the default 75. Bigger file sizes for less degradation.
+
+
+
+v3.1.1:
+=======
+
+Changes
+-------
+
+-  Fixed bug that caused incorrect page size and DPI calculations on documents with mixed page sizes
+
+v3.1:
+=====
+
+Changes
+-------
+
+-  Default output format is now PDF/A-2b instead of PDF/A-1b
+-  Python 3.5 and OS X El Capitan are now supported platforms - no changes were
+   needed to implement support
+-  Improved some error messages related to missing input files
+-  Fixed issue #20 - uppercase .PDF extension not accepted
+-  Fixed an issue where OCRmyPDF failed to text that certain pages contained previously OCR'ed text, 
+   such as OCR text produced by Tesseract 3.04
+-  Inserts /Creator tag into PDFs so that errors can be traced back to this project
+-  Added new option ``--pdf-renderer=auto``, to let OCRmyPDF pick the best PDF renderer. 
+   Currently it always chooses the 'hocrtransform' renderer but that behavior may change.
+-  Set up Travis CI automatic integration testing
 
 v3.0:
 =====
@@ -11,27 +105,33 @@ v3.0:
 New features
 ------------
 
--  Easier installation with Python's package manager 
+-  Easier installation with a Docker container or Python's ``pip`` package manager 
+-  Eliminated many external dependencies, so it's easier to setup
 -  Now installs ``ocrmypdf`` to ``/usr/local/bin`` or equivalent for system-wide
-   access
--  Tesseract 3.03 PDF page can be used instead for better positioning
-   of recognized text (``--pdf-renderer tesseract``)
+   access and easier typing
 -  Improved command line syntax and usage help (``--help``)
+-  Tesseract 3.03+ PDF page rendering can be used instead for better positioning
+   of recognized text (``--pdf-renderer tesseract``)
 -  PDF metadata (title, author, keywords) are now transferred to the 
    output PDF
 -  PDF metadata can also be set from the command line (``--title``, etc.)
+-  Automatic repairs malformed input PDFs if possible
 -  Added test cases to confirm everything is working
 -  Added option to skip extremely large pages that take too long to OCR and are 
    often not OCRable (e.g. large scanned maps or diagrams); other pages are still
    processed (``--skip-big``)
 -  Added option to kill Tesseract OCR process if it seems to be taking too long on
    a page, while still processing other pages (``--tesseract-timeout``)
+-  Less common colorspaces (CMYK, palette) are now supported by conversion to RGB
+-  Multiple images on the same PDF page are now supported
 
 Changes
 -------
 
 -  New, robust rewrite in Python 3.4+ with ruffus_ pipelines
--  Now uses Ghostscript 9.14's improved color conversion model
+-  Now uses Ghostscript 9.14's improved color conversion model to preserve PDF colors
+-  OCR text is now rendered in the PDF as invisible text. Previous versions of OCRmyPDF
+   incorrectly rendered visible text with an image on top.
 -  All "tasks" in the pipeline can be executed in parallel on any
    available CPUs, increasing performance
 -  The ``-o DPI`` argument has been phased out, in favor of ``--oversample DPI``, in
@@ -45,6 +145,8 @@ Changes
    - Poppler
    - MuPDF_ tools
    - shell scripts
+   - Java and JHOVE_
+   - libxml2
 
 -  Some new external dependencies are required or optional, compared to v2.x:
 
@@ -59,11 +161,39 @@ Changes
 .. _MuPDF: http://mupdf.com/docs/
 .. _qpdf: http://qpdf.sourceforge.net/
 .. _Unpaper: https://github.com/Flameeyes/unpaper
-
+.. _JHOVE: http://jhove.sourceforge.net/
 
 Release candidates
 ------------------
 
+-  rc9:
+
+   - fix issue #118: report error if ghostscript iccprofiles are missing
+   - fixed another issue related to #111: PDF rasterized to palette file
+   - add support image files with a palette
+   - don't try to validate PDF file after an exception occurs
+
+-  rc8:
+
+   - fix issue #111: exception thrown if PDF is missing DocumentInfo dictionary
+
+-  rc7:
+
+   - fix error when installing direct from pip, "no such file 'requirements.txt'"
+
+-  rc6:
+
+   - dropped libxml2 (Python lxml) since Python 3's internal XML parser is sufficient
+   - set up Docker container
+   - fix Unicode errors if recognized text contains Unicode characters and system locale is not UTF-8
+
+-  rc5:
+
+   - dropped Java and JHOVE in favour of qpdf
+   - improved command line error output
+   - additional tests and bug fixes
+   - tested on Ubuntu 14.04 LTS
+
 -  rc4:
 
    - dropped MuPDF in favour of qpdf
@@ -106,12 +236,18 @@ Fixes
 
 -  Handling of filenames containing spaces: fixed
 
-Notes
------
+Notes and known issues
+----------------------
 
 -  Some dependencies may work with lower versions than tested, so try
    overriding dependencies if they are "in the way" to see if they work.
 
+-  ``--pdf-renderer tesseract`` will output files with an incorrect page size in Tesseract 3.03,
+   due to a bug in Tesseract.
+
+-  PDF files containing "inline images" are not supported and won't be for the 3.0 release. Scanned
+   images almost never contain inline images.
+
 
 v2.2-stable (2014-09-29):
 =========================

docker-wrapper.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+. /appenv/bin/activate
+cd /home/docker
+exec ocrmypdf "$@"

ocrmypdf/__init__.py

@@ -0,0 +1,22 @@
+from enum import IntEnum
+import os
+
+
+class ExitCode(IntEnum):
+    ok = 0
+    bad_args = 1
+    input_file = 2
+    missing_dependency = 3
+    invalid_output_pdfa = 4
+    file_access_error = 5
+    already_done_ocr = 6
+    other_error = 15
+
+
+def get_program(name):
+    envvar = 'OCRMYPDF_' + name.upper()
+    return os.environ.get(envvar, name)
+
+
+def page_number(input_file):
+    return int(os.path.basename(input_file)[0:6])

ocrmypdf/ghostscript.py

@@ -4,14 +4,20 @@
 from tempfile import NamedTemporaryFile
 from subprocess import Popen, PIPE, check_call
 from shutil import copy
+from . import get_program
 
 
-def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log):
+def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
+                  pageno=1):
     with NamedTemporaryFile(delete=True) as tmp:
         args_gs = [
-            'gs',
-            '-dBATCH', '-dNOPAUSE',
+            get_program('gs'),
+            '-dQUIET',
+            '-dBATCH',
+            '-dNOPAUSE',
             '-sDEVICE=%s' % raster_device,
+            '-dFirstPage=%i' % pageno,
+            '-dLastPage=%i' % pageno,
             '-o', tmp.name,
             '-r{0}x{1}'.format(str(xres), str(yres)),
             input_file
@@ -34,15 +40,17 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log):
 def generate_pdfa(pdf_pages, output_file, threads=1):
     with NamedTemporaryFile(delete=True) as gs_pdf:
         args_gs = [
-            "gs",
+            get_program("gs"),
             "-dQUIET",
             "-dBATCH",
             "-dNOPAUSE",
             '-dNumRenderingThreads=' + str(threads),
             "-sDEVICE=pdfwrite",
+            "-dAutoRotatePages=/None",
             "-sColorConversionStrategy=/RGB",
             "-sProcessColorModel=DeviceRGB",
-            "-dPDFA",
+            "-dJPEGQ=95",
+            "-dPDFA=2",
             "-sPDFACompatibilityPolicy=2",
             "-sOutputICCProfile=srgb.icc",
             "-sOutputFile=" + gs_pdf.name,

ocrmypdf/hocrtransform.py

@@ -1,4 +1,4 @@
-#!/usr/local/bin/python3
+#!/usr/bin/env python3
 ##############################################################################
 # Copyright (c) 2013-14: fritz-hh from Github
 #   (https://github.com/fritz-hh)
@@ -9,7 +9,7 @@
 ##############################################################################
 from reportlab.pdfgen.canvas import Canvas
 from reportlab.lib.units import inch
-from lxml import etree as ElementTree
+from xml.etree import ElementTree
 from PIL import Image
 from collections import namedtuple
 import re
@@ -35,8 +35,7 @@ class HocrTransform():
         self.dpi = dpi
         self.boxPattern = re.compile(r'bbox((\s+\d+){4})')
 
-        self.hocr = ElementTree.ElementTree()
-        self.hocr.parse(hocrFileName)
+        self.hocr = ElementTree.parse(hocrFileName)
 
         # if the hOCR file has a namespace, ElementTree requires its use to
         # find elements

ocrmypdf/jhove/COPYING

@@ -1,502 +0,0 @@
-		  GNU LESSER GENERAL PUBLIC LICENSE
-		       Version 2.1, February 1999
-
- Copyright (C) 1991, 1999 Free Software Foundation, Inc.
-     59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
-[This is the first released version of the Lesser GPL.  It also counts
- as the successor of the GNU Library Public License, version 2, hence
- the version number 2.1.]
-
-			    Preamble
-
-  The licenses for most software are designed to take away your
-freedom to share and change it.  By contrast, the GNU General Public
-Licenses are intended to guarantee your freedom to share and change
-free software--to make sure the software is free for all its users.
-
-  This license, the Lesser General Public License, applies to some
-specially designated software packages--typically libraries--of the
-Free Software Foundation and other authors who decide to use it.  You
-can use it too, but we suggest you first think carefully about whether
-this license or the ordinary General Public License is the better
-strategy to use in any particular case, based on the explanations below.
-
-  When we speak of free software, we are referring to freedom of use,
-not price.  Our General Public Licenses are designed to make sure that
-you have the freedom to distribute copies of free software (and charge
-for this service if you wish); that you receive source code or can get
-it if you want it; that you can change the software and use pieces of
-it in new free programs; and that you are informed that you can do
-these things.
-
-  To protect your rights, we need to make restrictions that forbid
-distributors to deny you these rights or to ask you to surrender these
-rights.  These restrictions translate to certain responsibilities for
-you if you distribute copies of the library or if you modify it.
-
-  For example, if you distribute copies of the library, whether gratis
-or for a fee, you must give the recipients all the rights that we gave
-you.  You must make sure that they, too, receive or can get the source
-code.  If you link other code with the library, you must provide
-complete object files to the recipients, so that they can relink them
-with the library after making changes to the library and recompiling
-it.  And you must show them these terms so they know their rights.
-
-  We protect your rights with a two-step method: (1) we copyright the
-library, and (2) we offer you this license, which gives you legal
-permission to copy, distribute and/or modify the library.
-
-  To protect each distributor, we want to make it very clear that
-there is no warranty for the free library.  Also, if the library is
-modified by someone else and passed on, the recipients should know
-that what they have is not the original version, so that the original
-author's reputation will not be affected by problems that might be
-introduced by others.
-
-  Finally, software patents pose a constant threat to the existence of
-any free program.  We wish to make sure that a company cannot
-effectively restrict the users of a free program by obtaining a
-restrictive license from a patent holder.  Therefore, we insist that
-any patent license obtained for a version of the library must be
-consistent with the full freedom of use specified in this license.
-
-  Most GNU software, including some libraries, is covered by the
-ordinary GNU General Public License.  This license, the GNU Lesser
-General Public License, applies to certain designated libraries, and
-is quite different from the ordinary General Public License.  We use
-this license for certain libraries in order to permit linking those
-libraries into non-free programs.
-
-  When a program is linked with a library, whether statically or using
-a shared library, the combination of the two is legally speaking a
-combined work, a derivative of the original library.  The ordinary
-General Public License therefore permits such linking only if the
-entire combination fits its criteria of freedom.  The Lesser General
-Public License permits more lax criteria for linking other code with
-the library.
-
-  We call this license the "Lesser" General Public License because it
-does Less to protect the user's freedom than the ordinary General
-Public License.  It also provides other free software developers Less
-of an advantage over competing non-free programs.  These disadvantages
-are the reason we use the ordinary General Public License for many
-libraries.  However, the Lesser license provides advantages in certain
-special circumstances.
-
-  For example, on rare occasions, there may be a special need to
-encourage the widest possible use of a certain library, so that it becomes
-a de-facto standard.  To achieve this, non-free programs must be
-allowed to use the library.  A more frequent case is that a free
-library does the same job as widely used non-free libraries.  In this
-case, there is little to gain by limiting the free library to free
-software only, so we use the Lesser General Public License.
-
-  In other cases, permission to use a particular library in non-free
-programs enables a greater number of people to use a large body of
-free software.  For example, permission to use the GNU C Library in
-non-free programs enables many more people to use the whole GNU
-operating system, as well as its variant, the GNU/Linux operating
-system.
-
-  Although the Lesser General Public License is Less protective of the
-users' freedom, it does ensure that the user of a program that is
-linked with the Library has the freedom and the wherewithal to run
-that program using a modified version of the Library.
-
-  The precise terms and conditions for copying, distribution and
-modification follow.  Pay close attention to the difference between a
-"work based on the library" and a "work that uses the library".  The
-former contains code derived from the library, whereas the latter must
-be combined with the library in order to run.
-
-		  GNU LESSER GENERAL PUBLIC LICENSE
-   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
-
-  0. This License Agreement applies to any software library or other
-program which contains a notice placed by the copyright holder or
-other authorized party saying it may be distributed under the terms of
-this Lesser General Public License (also called "this License").
-Each licensee is addressed as "you".
-
-  A "library" means a collection of software functions and/or data
-prepared so as to be conveniently linked with application programs
-(which use some of those functions and data) to form executables.
-
-  The "Library", below, refers to any such software library or work
-which has been distributed under these terms.  A "work based on the
-Library" means either the Library or any derivative work under
-copyright law: that is to say, a work containing the Library or a
-portion of it, either verbatim or with modifications and/or translated
-straightforwardly into another language.  (Hereinafter, translation is
-included without limitation in the term "modification".)
-
-  "Source code" for a work means the preferred form of the work for
-making modifications to it.  For a library, complete source code means
-all the source code for all modules it contains, plus any associated
-interface definition files, plus the scripts used to control compilation
-and installation of the library.
-
-  Activities other than copying, distribution and modification are not
-covered by this License; they are outside its scope.  The act of
-running a program using the Library is not restricted, and output from
-such a program is covered only if its contents constitute a work based
-on the Library (independent of the use of the Library in a tool for
-writing it).  Whether that is true depends on what the Library does
-and what the program that uses the Library does.
-  
-  1. You may copy and distribute verbatim copies of the Library's
-complete source code as you receive it, in any medium, provided that
-you conspicuously and appropriately publish on each copy an
-appropriate copyright notice and disclaimer of warranty; keep intact
-all the notices that refer to this License and to the absence of any
-warranty; and distribute a copy of this License along with the
-Library.
-
-  You may charge a fee for the physical act of transferring a copy,
-and you may at your option offer warranty protection in exchange for a
-fee.
-
-  2. You may modify your copy or copies of the Library or any portion
-of it, thus forming a work based on the Library, and copy and
-distribute such modifications or work under the terms of Section 1
-above, provided that you also meet all of these conditions:
-
-    a) The modified work must itself be a software library.
-
-    b) You must cause the files modified to carry prominent notices
-    stating that you changed the files and the date of any change.
-
-    c) You must cause the whole of the work to be licensed at no
-    charge to all third parties under the terms of this License.
-
-    d) If a facility in the modified Library refers to a function or a
-    table of data to be supplied by an application program that uses
-    the facility, other than as an argument passed when the facility
-    is invoked, then you must make a good faith effort to ensure that,
-    in the event an application does not supply such function or
-    table, the facility still operates, and performs whatever part of
-    its purpose remains meaningful.
-
-    (For example, a function in a library to compute square roots has
-    a purpose that is entirely well-defined independent of the
-    application.  Therefore, Subsection 2d requires that any
-    application-supplied function or table used by this function must
-    be optional: if the application does not supply it, the square
-    root function must still compute square roots.)
-
-These requirements apply to the modified work as a whole.  If
-identifiable sections of that work are not derived from the Library,
-and can be reasonably considered independent and separate works in
-themselves, then this License, and its terms, do not apply to those
-sections when you distribute them as separate works.  But when you
-distribute the same sections as part of a whole which is a work based
-on the Library, the distribution of the whole must be on the terms of
-this License, whose permissions for other licensees extend to the
-entire whole, and thus to each and every part regardless of who wrote
-it.
-
-Thus, it is not the intent of this section to claim rights or contest
-your rights to work written entirely by you; rather, the intent is to
-exercise the right to control the distribution of derivative or
-collective works based on the Library.
-
-In addition, mere aggregation of another work not based on the Library
-with the Library (or with a work based on the Library) on a volume of
-a storage or distribution medium does not bring the other work under
-the scope of this License.
-
-  3. You may opt to apply the terms of the ordinary GNU General Public
-License instead of this License to a given copy of the Library.  To do
-this, you must alter all the notices that refer to this License, so
-that they refer to the ordinary GNU General Public License, version 2,
-instead of to this License.  (If a newer version than version 2 of the
-ordinary GNU General Public License has appeared, then you can specify
-that version instead if you wish.)  Do not make any other change in
-these notices.
-
-  Once this change is made in a given copy, it is irreversible for
-that copy, so the ordinary GNU General Public License applies to all
-subsequent copies and derivative works made from that copy.
-
-  This option is useful when you wish to copy part of the code of
-the Library into a program that is not a library.
-
-  4. You may copy and distribute the Library (or a portion or
-derivative of it, under Section 2) in object code or executable form
-under the terms of Sections 1 and 2 above provided that you accompany
-it with the complete corresponding machine-readable source code, which
-must be distributed under the terms of Sections 1 and 2 above on a
-medium customarily used for software interchange.
-
-  If distribution of object code is made by offering access to copy
-from a designated place, then offering equivalent access to copy the
-source code from the same place satisfies the requirement to
-distribute the source code, even though third parties are not
-compelled to copy the source along with the object code.
-
-  5. A program that contains no derivative of any portion of the
-Library, but is designed to work with the Library by being compiled or
-linked with it, is called a "work that uses the Library".  Such a
-work, in isolation, is not a derivative work of the Library, and
-therefore falls outside the scope of this License.
-
-  However, linking a "work that uses the Library" with the Library
-creates an executable that is a derivative of the Library (because it
-contains portions of the Library), rather than a "work that uses the
-library".  The executable is therefore covered by this License.
-Section 6 states terms for distribution of such executables.
-
-  When a "work that uses the Library" uses material from a header file
-that is part of the Library, the object code for the work may be a
-derivative work of the Library even though the source code is not.
-Whether this is true is especially significant if the work can be
-linked without the Library, or if the work is itself a library.  The
-threshold for this to be true is not precisely defined by law.
-
-  If such an object file uses only numerical parameters, data
-structure layouts and accessors, and small macros and small inline
-functions (ten lines or less in length), then the use of the object
-file is unrestricted, regardless of whether it is legally a derivative
-work.  (Executables containing this object code plus portions of the
-Library will still fall under Section 6.)
-
-  Otherwise, if the work is a derivative of the Library, you may
-distribute the object code for the work under the terms of Section 6.
-Any executables containing that work also fall under Section 6,
-whether or not they are linked directly with the Library itself.
-
-  6. As an exception to the Sections above, you may also combine or
-link a "work that uses the Library" with the Library to produce a
-work containing portions of the Library, and distribute that work
-under terms of your choice, provided that the terms permit
-modification of the work for the customer's own use and reverse
-engineering for debugging such modifications.
-
-  You must give prominent notice with each copy of the work that the
-Library is used in it and that the Library and its use are covered by
-this License.  You must supply a copy of this License.  If the work
-during execution displays copyright notices, you must include the
-copyright notice for the Library among them, as well as a reference
-directing the user to the copy of this License.  Also, you must do one
-of these things:
-
-    a) Accompany the work with the complete corresponding
-    machine-readable source code for the Library including whatever
-    changes were used in the work (which must be distributed under
-    Sections 1 and 2 above); and, if the work is an executable linked
-    with the Library, with the complete machine-readable "work that
-    uses the Library", as object code and/or source code, so that the
-    user can modify the Library and then relink to produce a modified
-    executable containing the modified Library.  (It is understood
-    that the user who changes the contents of definitions files in the
-    Library will not necessarily be able to recompile the application
-    to use the modified definitions.)
-
-    b) Use a suitable shared library mechanism for linking with the
-    Library.  A suitable mechanism is one that (1) uses at run time a
-    copy of the library already present on the user's computer system,
-    rather than copying library functions into the executable, and (2)
-    will operate properly with a modified version of the library, if
-    the user installs one, as long as the modified version is
-    interface-compatible with the version that the work was made with.
-
-    c) Accompany the work with a written offer, valid for at
-    least three years, to give the same user the materials
-    specified in Subsection 6a, above, for a charge no more
-    than the cost of performing this distribution.
-
-    d) If distribution of the work is made by offering access to copy
-    from a designated place, offer equivalent access to copy the above
-    specified materials from the same place.
-
-    e) Verify that the user has already received a copy of these
-    materials or that you have already sent this user a copy.
-
-  For an executable, the required form of the "work that uses the
-Library" must include any data and utility programs needed for
-reproducing the executable from it.  However, as a special exception,
-the materials to be distributed need not include anything that is
-normally distributed (in either source or binary form) with the major
-components (compiler, kernel, and so on) of the operating system on
-which the executable runs, unless that component itself accompanies
-the executable.
-
-  It may happen that this requirement contradicts the license
-restrictions of other proprietary libraries that do not normally
-accompany the operating system.  Such a contradiction means you cannot
-use both them and the Library together in an executable that you
-distribute.
-
-  7. You may place library facilities that are a work based on the
-Library side-by-side in a single library together with other library
-facilities not covered by this License, and distribute such a combined
-library, provided that the separate distribution of the work based on
-the Library and of the other library facilities is otherwise
-permitted, and provided that you do these two things:
-
-    a) Accompany the combined library with a copy of the same work
-    based on the Library, uncombined with any other library
-    facilities.  This must be distributed under the terms of the
-    Sections above.
-
-    b) Give prominent notice with the combined library of the fact
-    that part of it is a work based on the Library, and explaining
-    where to find the accompanying uncombined form of the same work.
-
-  8. You may not copy, modify, sublicense, link with, or distribute
-the Library except as expressly provided under this License.  Any
-attempt otherwise to copy, modify, sublicense, link with, or
-distribute the Library is void, and will automatically terminate your
-rights under this License.  However, parties who have received copies,
-or rights, from you under this License will not have their licenses
-terminated so long as such parties remain in full compliance.
-
-  9. You are not required to accept this License, since you have not
-signed it.  However, nothing else grants you permission to modify or
-distribute the Library or its derivative works.  These actions are
-prohibited by law if you do not accept this License.  Therefore, by
-modifying or distributing the Library (or any work based on the
-Library), you indicate your acceptance of this License to do so, and
-all its terms and conditions for copying, distributing or modifying
-the Library or works based on it.
-
-  10. Each time you redistribute the Library (or any work based on the
-Library), the recipient automatically receives a license from the
-original licensor to copy, distribute, link with or modify the Library
-subject to these terms and conditions.  You may not impose any further
-restrictions on the recipients' exercise of the rights granted herein.
-You are not responsible for enforcing compliance by third parties with
-this License.
-
-  11. If, as a consequence of a court judgment or allegation of patent
-infringement or for any other reason (not limited to patent issues),
-conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License.  If you cannot
-distribute so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you
-may not distribute the Library at all.  For example, if a patent
-license would not permit royalty-free redistribution of the Library by
-all those who receive copies directly or indirectly through you, then
-the only way you could satisfy both it and this License would be to
-refrain entirely from distribution of the Library.
-
-If any portion of this section is held invalid or unenforceable under any
-particular circumstance, the balance of the section is intended to apply,
-and the section as a whole is intended to apply in other circumstances.
-
-It is not the purpose of this section to induce you to infringe any
-patents or other property right claims or to contest validity of any
-such claims; this section has the sole purpose of protecting the
-integrity of the free software distribution system which is
-implemented by public license practices.  Many people have made
-generous contributions to the wide range of software distributed
-through that system in reliance on consistent application of that
-system; it is up to the author/donor to decide if he or she is willing
-to distribute software through any other system and a licensee cannot
-impose that choice.
-
-This section is intended to make thoroughly clear what is believed to
-be a consequence of the rest of this License.
-
-  12. If the distribution and/or use of the Library is restricted in
-certain countries either by patents or by copyrighted interfaces, the
-original copyright holder who places the Library under this License may add
-an explicit geographical distribution limitation excluding those countries,
-so that distribution is permitted only in or among countries not thus
-excluded.  In such case, this License incorporates the limitation as if
-written in the body of this License.
-
-  13. The Free Software Foundation may publish revised and/or new
-versions of the Lesser General Public License from time to time.
-Such new versions will be similar in spirit to the present version,
-but may differ in detail to address new problems or concerns.
-
-Each version is given a distinguishing version number.  If the Library
-specifies a version number of this License which applies to it and
-"any later version", you have the option of following the terms and
-conditions either of that version or of any later version published by
-the Free Software Foundation.  If the Library does not specify a
-license version number, you may choose any version ever published by
-the Free Software Foundation.
-
-  14. If you wish to incorporate parts of the Library into other free
-programs whose distribution conditions are incompatible with these,
-write to the author to ask for permission.  For software which is
-copyrighted by the Free Software Foundation, write to the Free
-Software Foundation; we sometimes make exceptions for this.  Our
-decision will be guided by the two goals of preserving the free status
-of all derivatives of our free software and of promoting the sharing
-and reuse of software generally.
-
-			    NO WARRANTY
-
-  15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
-WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
-EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR
-OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY
-KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
-LIBRARY IS WITH YOU.  SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME
-THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
-
-  16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
-WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
-AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU
-FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
-CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
-LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
-RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
-FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
-SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
-DAMAGES.
-
-		     END OF TERMS AND CONDITIONS
-
-           How to Apply These Terms to Your New Libraries
-
-  If you develop a new library, and you want it to be of the greatest
-possible use to the public, we recommend making it free software that
-everyone can redistribute and change.  You can do so by permitting
-redistribution under these terms (or, alternatively, under the terms of the
-ordinary General Public License).
-
-  To apply these terms, attach the following notices to the library.  It is
-safest to attach them to the start of each source file to most effectively
-convey the exclusion of warranty; and each file should have at least the
-"copyright" line and a pointer to where the full notice is found.
-
-    <one line to give the library's name and a brief idea of what it does.>
-    Copyright (C) <year>  <name of author>
-
-    This library is free software; you can redistribute it and/or
-    modify it under the terms of the GNU Lesser General Public
-    License as published by the Free Software Foundation; either
-    version 2.1 of the License, or (at your option) any later version.
-
-    This library is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-    Lesser General Public License for more details.
-
-    You should have received a copy of the GNU Lesser General Public
-    License along with this library; if not, write to the Free Software
-    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-
-Also add information on how to contact you by electronic and paper mail.
-
-You should also get your employer (if you work as a programmer) or your
-school, if any, to sign a "copyright disclaimer" for the library, if
-necessary.  Here is a sample; alter the names:
-
-  Yoyodyne, Inc., hereby disclaims all copyright interest in the
-  library `Frob' (a library for tweaking knobs) written by James Random Hacker.
-
-  <signature of Ty Coon>, 1 April 1990
-  Ty Coon, President of Vice
-
-That's all there is to it!

ocrmypdf/jhove/LICENSE

@@ -1,17 +0,0 @@
-JHOVE - JSTOR/Harvard Object Validation Environment
-Copyright 2003-2008 by JSTOR and the President and Fellows of Harvard College
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU Lessor General Public License as
-published by the Free Software Foundation; either version 2.1 of the
-License, or (at your option) any later version.
-
-This program is distributed in the hope that it will be useful, but
-WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-Lesser General Public License for more details.
-
-You should have received a copy of the GNU Lesser General Public
-License along with this program; if not, write to the Free Software
-Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
-USA

ocrmypdf/jhove/README

@@ -1,227 +0,0 @@
-JHOVE - JSTOR/Harvard Object Validation Environment
-Copyright 2003-2012 by JSTOR and the President and Fellows of Harvard College
-JHOVE is made available under the GNU Lesser General Public License (LGPL;
-see the file LICENSE for details)
-
-Rev. 1.11, 2013-09-29
-
-JHOVE (the JSTOR/Harvard Object Validation Environment, pronounced "jhove")
-is an extensible software framework for performing format identification,
-validation, and characterization of digital objects.
-
-o Format identification is the process of determining the format to which a
-  digital object conforms: "I have a digital object; what format is it?"
-o Format validation is the process of determining the level of compliance of a
-  digital object to the specification for its purported format: "I have an
-  object purportedly of format F; is it?"
-o Format characterization is the process of determing the format-specific
-  significant properties of an object of a given format: "I have an object of
-  format F; what are its salient properties?"
-
-These actions are frequently necessary during routine operation of digital
-repositories and for digital preservation activities.
-
-The output from JHOVE is controlled by output handlers. JHOVE uses an
-extensible plug-in architecture; it can be configured at the time of its
-invocation to include whatever specific format modules and output handlers
-that are desired. The initial release of JHOVE includes modules for
-arbitrary byte streams, ASCII and UTF-8 encoded text, AIFF and WAVE audio,
-GIF, JPEG, JPEG 2000, TIFF, and PDF; and text and XML output handlers.
-
-The JHOVE project is a collaboration of JSTOR and the Harvard University
-Library.  Development of JHOVE was funded in part by the Andrew W. Mellon
-Foundation.  JHOVE is made available under the GNU Lesser General Public
-License (LGPL; see the file LICENSE for details).
-
-JHOVE is currently being maintained by indpendent developers.
-
-REQUIREMENTS
-
-1. Java J2SE 1.5
-(JHOVE was originally implemented using the Sun J2SE SDK 1.4.1 and has
-been tested to work with 1.5)
-
-2. If you would like to compile the JHOVE source code, then
-Apache Ant, a Java-based build tool <http://ant.apache.org/> is necessary.
-Note that the JAVA_HOME environment variable must be set appropriately for
-Ant to work properly.
-(JHOVE was implemented and tested using Ant 1.5.1.)
-
-DISTRIBUTION
-
-The JHOVE distribution package includes:
-
-    jhove/                                 # JHOVE home directory
-          COPYING                          # GNU Lesser General Public License
-          LICENSE                          # JHOVE license information
-          README
-          RELEASENOTES                     # JHOVE release notes
-          bin/
-                    jhove.jar              # JHOVE API package
-                    jhove-handler.jar      # Standard output handler package
-                    jhove-module.jar       # Standard module package
-                    JhoveApp.jar           # JHOVE command line application
-                    JhoveView.jar          # JHOVE with Swing GUI front-end
-          build.xml                        # Ant configuration file
-          classes/
-                    build.xml              # Ant configuration file
-                    edu/ ...               # JHOVE API packages
-                    ADump.*                # AIFF dump utility class
-                    GDump.*                # GIF dump utility class
-                    Jhove.*                # JHOVE main class
-                    JDump.*                # JPEG dump utility class
-                    J2Dump.*               # JPEG 2000 dump utility class
-                    PDump.*                # PDF dump utility class
-                    TDump.*                # TIFF dump utility class
-                    UserHome.*             # user.home property utility class
-                    WDump.*                # WAVE dump utility class
-          conf/
-                   jhove.conf              # JHOVE configuration file
-                   jhove.xsd               # JHOVE output schema
-                   jhoveConfig.xsd         # JHOVE configuration file schema
-          doc/
-                   *.html                  # API documentation
-                   ...
-          examples/                        # Sample files
-                   ascii/ ...
-                   gif/   ...
-                   jpeg/  ...
-                   jpeg2000/ ...
-                   pdf/   ...
-                   tiff/  ...
-                   utf-8/ ...
-          adump*                           # AIFF dump Bourne shell driver
-          adump.bat*                       # AIFF dump DOS shell driver script
-          gdump*                           # GIF dump Bourne shell driver
-          gdump.bat*                       # GIF dump DOS shell driver script
-          jdump*                           # JPEG dump Bourne shell driver
-          jdump.bat*                       # JPEG dump DOS shell driver script
-          j2dump*                          # JPEG 2000 dump Bourne shell driver
-          j2dump.bat*                      # JPEG 2000 dump DOS shell driver
-          jhove.tmpl*                      # Template for JHOVE Bourne shell driver script
-          jhove_bat.tmpl*                  # Template for JHOVE DOS shell driver script
-          pdump*                           # PDF dump Bourne shell driver
-          pdump.bat*                       # PDF dump DOS shell driver script
-          tdump*                           # TIFF dump Bourne shell driver
-          tdump.bat*                       # TIFF dump DOS shell driver script
-          userhome*                        # user.home Bourne shell driver
-          userhome.bat*                    # user.home DOS shell driver script
-          wdump*                           # WAVE dump Bourne shell driver
-          wdump.bat*                       # WAVE dump DOS shell driver script
-
-INSTALLATION
-
-Edit the configuration file, jhove/conf/jhove.conf, and set the absolute
-pathname of the JHOVE home directory and the temporary directory (in which
-temporary files are created):
-
-    <jhoveHome>jhove-home-directory</jhoveHome>
-    <tempDirectory>temporary-directory</tempDirectory>
-
-The JHOVE home directory is the top-most directory in the distribution TAR
-or ZIP file.  On Unix systems, "/var/tmp" is an appropriate temporary
-directory; on Windows, "C:\Temp".  For example, if the distribution TAR
-file is disaggregated on a Unix system in the directory "/users/stephen/
-projects", then the configuration file should read:
-
-    <jhoveHome>/users/stephen/projects/jhove</jhoveHome>
-    <tempDirectory>/var/tmp</jhoveHome>
-
-In the JHOVE home directory, copy the JHOVE Bourne shell driver script
-template, "jhove.tmpl", to "jhove" (or the equivalent Windows shell 
-script, "jhove_bat.tmpl" to "jhove.bat"), and set the
-JHOVE home directory, Java home directory, and Java interpreter:
-
-    JHOVE_HOME=jhove-home-directory
-    JAVA_HOME=java-home-directory
-    JAVA=java-interpreter
-
-The JAVA_HOME property should provide the absolute pathname of the Java
-runtime or SDK installation; JAVA should provide the absolute pathname of the
-Java interpreter.  For example:
-
-    JHOVE_HOME=/users/stephen/projects/jhove
-    JAVA_HOME=/usr/local/j2re1.4.1_02
-    JAVA=$JAVA_HOME/bin/java
-
-In the DOS shell driver script, jhove.bat, the equivalent three
-variables are:
-
-    SET JHOVE_HOME=jhove-home-directory
-    SET JAVA_HOME=java-home-directory
-    SET JAVA=%JAVA_HOME%\bin\java
-
-For example:
-
-    SET JHOVE_HOME="C:\Program Files\jhove"
-    SET JAVA_HOME="C:\Program Files\java\j2re1.4.1_02"
-    SET JAVA=%JAVA_HOME%\bin\java
-
-The quotation marks are necessary because of the embedded space characters.
-On Windows platforms it may also be necessary to add the Java bin subdirectory
-to the System PATH environment variable:
-
-    PATH=C:\Program Files\java\j2re1.4.1_02\bin;...
-
-(For information on setting a Windows environment variable, consult your local
-documentation or system administrator.)
-
-USAGE
-
-    java Jhove [-c config] [-m module] [-h handler] [-e encoding] [-H handler]
-               [-o output] [-x saxclass] [-t tempdir] [-b bufsize]
-               [-l loglevel] [[-krs] dir-file-or-uri [...]]
-
-where -c config   Configuration file pathname
-      -m module   Module name
-      -h handler  Output handler name (defaults to TEXT)
-      -e encoding Character encoding used by output handler (defaults to UTF-8)
-      -H handler  About handler name
-      -o output   Output file pathname (defaults to standard output)
-      -x saxclass SAX parser class (defaults to J2SE default)
-      -t tempdir  Temporary directory in which to create temporary files
-      -b bufsize  Buffer size for buffered I/O (defaults to J2SE 1.4 default)
-      -l loglevel Logging level
-      -k          Calculate CRC32, MD5, and SHA-1 checksums
-      -r          Display raw data flags, not textual equivalents
-      -s          Format identification based on internal signatures only
-      dir-file-or-uri Directory or file pathname or URI of formated content
-                      stream
-
-All named modules and output handlers must be found on the Java CLASSPATH at
-the time of invocation.  The JHOVE driver script, jhove/jhove, automatically
-sets the CLASSPATH and invokes the Jhove main class:
-
-    jhove [-c config] [-m module] [-h handler] [-e encoding] [-H handler]
-          [-o output] [-x saxclass] [-t tempdir] [-b bufsize] [-l loglevel]
-          [[-krs] dir-file-or-uri [...]]
-
-The following additional programs are available, primarily for testing
-and debugging purposes.  They display a minimally processed, human-readable
-version of the contents of AIFF, GIF, JPEG, JPEG 2000, PDF, TIFF, and WAVE
-files:
-
-    java ADump  aiff-file
-    java GDump  gif-file
-    java JDump  jpeg-file
-    java J2Dump jpeg2000-file
-    java PDump  pdf-file
-    java TDump  tiff-file
-    java WDump  wave-file
-
-For convenience, the following driver scripts are also available:
-
-    adump  aiff-file
-    gdump  gif-file
-    jdump  jpeg-file
-    j2dump jpeg2000-file
-    pdump  pdf-file
-    tdump  tiff-file
-    wdump  wave-file
-
-The JHOVE Swing-based GUI interface can be invoked from a command shell from
-the jhove/bin sub-directory:
-
-    java -jar JhoveView.jar -c <configFile>
-
-where <configFile> is the pathname of the JHOVE configuration file.

ocrmypdf/jhove/bin/README

@@ -1,25 +0,0 @@
-JHOVE - JSTOR/Harvard Object Validation Environment
-Copyright 2003 by JSTOR and the President and Fellows of Harvard College
-JHOVE is made available under the GNU General Public License (see the file
-LICENSE for details)
-
-Rev. 2003-11-25
-
-The following jar files are meant to be used for embedding JHOVE functionality
-into new applications or systems.
-
-  jhove.jar          Contains the JHOVE API interfaces and classes
-  jhove-module.jar   Contains the standard JHOVE modules ()
-  jhove-handler.jar  Contains the standard JHOVE output handlers (TEXT and XML)
-
-The following jar file is meant to be used with the stand-alone JHOVE
-application using a command-line interface.  It contains the main Jhove class
-and the contents of jhove.jar, jhove-module.jar, and jhove-handler.jar.
-
-  JhoveApp.jar
-
-The following jar file is meant to be used with the stand-alone JHOVE
-application using a Swing GUI interface.  It contains the main JhoveView class
-and the contents of jhove.jar, jhove-module.jar, and jhove-handler.jar.
-
-  JhoveView.jar

ocrmypdf/jhove/build.xml

@@ -1,78 +0,0 @@
-<project name="Jhove" default="dist" basedir=".">
-  <description>Project build file
-    Jhove - JSTOR/Harvard Object Validation Environment
-    Version 1.0 2004-09-10
-    Copyright 2004 by JSTOR and the President and Fellows of Harvard College
-  </description>
-
-  <!-- ant (or ant dist) Build everything
-       ant debug         Build everything with debug enabled
-       ant clean         Delete backup files
-       ant cleanclass    Delete backup and class files
-       ant cleandist     Delete backup, class, and jar files
-       ant javadoc       Build javadocs
-    -->
-
-  <!-- set global properties for this build -->
-  <property name="bin"     location="bin"/>
-  <property name="classes" location="classes"/>
-  <property name="doc"     location="doc"/>
-
-  <target name="dist" description="Create distribution">
-    <ant dir="${classes}" inheritAll="false">
-      <property name="dbg" value="off"/>
-    </ant>
-  	<chmod file="jhove" perm="ugo+x"/>
-  	<chmod file="${bin}/JhoveView.jar" perm="ugo+x"/>
-  </target>
-
-  <target name="debug" description="Create distribution with debug enabled">
-    <ant dir="${classes}" inheritAll="false">
-      <property name="dbg" value="on"/>
-    </ant>
-  </target>
-
-  <target name="view" description="Create JhoveView application">
-    <ant dir="${classes}" target="view" inheritAll="false">
-      <property name="dbs" value="on"/>
-    </ant>
-  </target>
-
-  <target name="clean" description="Delete backup files">
-    <ant dir="${classes}" target="main-clean" inheritAll="false"/>
-  </target>
-
-  <target name="cleanclass" depends="clean">
-    <ant dir="${classes}" target="main-cleanclass" inheritAll="false"/>
-  </target>
-
-  <target name="cleandist" depends="cleanclass">
-    <delete file="${bin}/jhove.jar"/>
-    <delete file="${bin}/jhove-handler.jar"/>
-    <delete file="${bin}/jhove-module.jar"/>
-    <delete file="${bin}/JhoveApp.jar"/>
-    <delete file="${bin}/JhoveView.jar"/>
-  </target>
-
-  <target name="javadoc">
-    <javadoc sourcepath="${classes}" destdir="${doc}"
-             windowtitle="JHOVE Documentation"
-             Overview="${classes}/overview.html">
-      <package name="edu.harvard.hul.ois.jhove"/>
-      <package name="edu.harvard.hul.ois.jhove.handler"/>
-      <package name="edu.harvard.hul.ois.jhove.handler.audit"/>
-      <package name="edu.harvard.hul.ois.jhove.module"/>
-      <package name="edu.harvard.hul.ois.jhove.module.aiff"/>
-      <package name="edu.harvard.hul.ois.jhove.module.gif"/>
-      <package name="edu.harvard.hul.ois.jhove.module.html"/>
-      <package name="edu.harvard.hul.ois.jhove.module.iff"/>
-      <package name="edu.harvard.hul.ois.jhove.module.jpeg"/>
-      <package name="edu.harvard.hul.ois.jhove.module.jpeg2000"/>
-      <package name="edu.harvard.hul.ois.jhove.module.pdf"/>
-      <package name="edu.harvard.hul.ois.jhove.module.tiff"/>
-      <package name="edu.harvard.hul.ois.jhove.module.wave"/>
-      <package name="edu.harvard.hul.ois.jhove.module.xml"/>
-      <package name="edu.harvard.hul.ois.jhove.viewer"/>
-    </javadoc>
-  </target>
-</project>

ocrmypdf/jhove/conf/README

@@ -1,63 +0,0 @@
-JHOVE - JSTOR/Harvard Object Validation Environment
-Copyright 2003-2007 by JSTOR and the President and Fellows of Harvard College
-JHOVE is made available under the GNU General Public License (see the file
-LICENSE for details)
-
-Rev. 2007-08-30
-
-Edit the configuration file, jhove.conf, and set the JHOVE home
-directory:
-
-    <jhoveHome>jhove-home-directory</jhoveHome>
-
-and temporary directory:
-
-    <tempDirectory>temporary-directory</tempDirectory>
-
-On most Unix systems, a reasonable temporary directory is "/var/tmp";
-on Windows, "C:\temp".
-
-The optional
-
-    <bufferSize>buffer-size</bufferSize>
-
-element defines the buffer size used for buffer I/O operations.
-
-The optional
-
-    <mixVersion>1.0</mixVersion>
-
-element specifies that the XML output handler should conform to the
-MIX 1.0 schema.  The default behavior is for handler output to conform
-to the MIX 0.2 schema.
-
-The optional
-
-    <sigBytes>n</sigBytes>
-
-element specifies that JHOVE modules will look for format signatures
-in the first <n> bytes of the file.  The default value is 1024.
-
-All class names must be fully qualified with their package name:
-
-    <module>
-      <class>fully-package-qualified-class-name</class>
-      <init>optional-initialization-argument</init>
-      <param>optional-invocation-argument</param>
-    </module>
-
-The optional <init> argument is passed to the module once at the time
-its class is instantiated.  See module-specific documentation for a
-description of any initialization options.
-
-The optional <param> argument is passed to the module every time it is
-invoked.  See module-specific documentation for a description of any
-invocation options.
-
-The order in which format modules are defined is important; when
-performing a format identification operation, JHOVE will search for a
-matching module in the order in which the modules are defined in the
-configuration file. In general, the modules for more generic formats
-should come later in the list. For example, the standard module ASCII
-should be defined before the UTF-8 module, since all ASCII objects
-are, by definition, UTF-8 objects, but not vice versa.

ocrmypdf/jhove/conf/jhove-byteoffset=true.conf

@@ -1,47 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<jhoveConfig version="1.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xmlns="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig"
- xsi:schemaLocation="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig
-                     http://hul.harvard.edu/ois/xml/xsd/jhove/1.6/jhoveConfig.xsd">
- <jhoveHome>/users/stephen/projects/jhove</jhoveHome>
- <defaultEncoding>utf-8</defaultEncoding>
- <tempDirectory>/var/tmp</tempDirectory>
- <bufferSize>131072</bufferSize>
- <mixVersion>1.0</mixVersion>
- <sigBytes>1024</sigBytes>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.AiffModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.WaveModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.PdfModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.Jpeg2000Module</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.JpegModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.GifModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.TiffModule</class>
-  <param>byteoffset=true</param>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.XmlModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.HtmlModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.AsciiModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.Utf8Module</class>
- </module>
-</jhoveConfig>

ocrmypdf/jhove/conf/jhove-withTextMD.conf

@@ -1,51 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<jhoveConfig version="1.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xmlns="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig"
- xsi:schemaLocation="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig
-                     http://hul.harvard.edu/ois/xml/xsd/jhove/1.6/jhoveConfig.xsd">
- <jhoveHome>/users/stephen/projects/jhove</jhoveHome>
- <defaultEncoding>utf-8</defaultEncoding>
- <tempDirectory>/var/tmp</tempDirectory>
- <bufferSize>131072</bufferSize>
- <mixVersion>1.0</mixVersion>
- <sigBytes>1024</sigBytes>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.AiffModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.WaveModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.PdfModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.Jpeg2000Module</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.JpegModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.GifModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.TiffModule</class>
-  <param>byteoffset=true</param>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.XmlModule</class>
-  <param>withTextMD=true</param>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.HtmlModule</class>
-  <param>withTextMD=true</param>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.AsciiModule</class>
-  <param>withTextMD=true</param>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.Utf8Module</class>
-  <param>withTextMD=true</param>
- </module>
-</jhoveConfig>

ocrmypdf/jhove/conf/jhove.conf

@@ -1,45 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<jhoveConfig version="1.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xmlns="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig"
- xsi:schemaLocation="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig
-                     http://hul.harvard.edu/ois/xml/xsd/jhove/1.6/jhoveConfig.xsd">
- <jhoveHome>./jhove/</jhoveHome>
- <defaultEncoding>utf-8</defaultEncoding>
- <tempDirectory>/var/tmp</tempDirectory>
- <bufferSize>131072</bufferSize>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.AiffModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.WaveModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.PdfModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.Jpeg2000Module</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.JpegModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.GifModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.TiffModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.XmlModule</class>
-  <param>schema=http://www.example.com/schema;/home/schemas/exampleschema.xsd</param>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.HtmlModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.AsciiModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.Utf8Module</class>
- </module>
-</jhoveConfig>

ocrmypdf/jhove/configure.pl

@@ -1,88 +0,0 @@
-#!/usr/bin/perl
-
-########################################################################
-# Jhove - JSTOR/Harvard Object Validation Environment
-# Copyright 2004 by JSTOR and the President and Fellows of Harvard College
-#
-# A Perl script for plugging local path information into the
-# various script files of JHOVE, as well as conf/jhove.conf.
-#
-# This is configured only for Unix (including OS X).
-#
-#    Usage: configure.pl jhove_home_directory [java_home_directory [java_runtime_directory]]
-#
-#    If invoked with no arguments, it will output a usage message.
-#
-########################################################################
-use File::Copy;
-
-sub mung {
-    my $f = $_[0];
-    my $bak = $f . "~";
-    #If there is no backup file, copy the file to the
-    #backup.  Otherwise work from the backup.
-    if (!(-e $bak)) {
-        rename ($f, $bak);
-    }
-    open (INFILE, $bak);
-    open (OUTFILE, ">" . $f);
-
-    #Walks through each line of file, making substitutions.
-    #Remember that the JAVA_HOME and JAVA arguments are optional.
-    while (<INFILE>) {
-        s/^JHOVE_HOME=.*/JHOVE_HOME=$ARGV[0]/;
-        if ($narg >= 2) {
-            s/^JAVA_HOME=.*/JAVA_HOME=$ARGV[1]/;
-        }
-        if ($narg >= 3) {
-            s/^JAVA=.*/JAVA=$ARGV[2]/;
-        }
-        print OUTFILE;
-    }
-    close (INFILE);
-    close (OUTFILE);
-    if (-e $f) {
-        print ("Fixed " . $f . "\n");
-    }
-}
-
-
-$narg = $#ARGV + 1;
-if ($narg <= 0) {
-    print "Usage: configure.pl jhove_home_directory [java_home_directory [java_runtime_directory]]\n";
-    exit;
-}
-print "JHOVE_HOME will be set to " . $ARGV[0] . "\n";
-if ($narg >= 2) {
-    print "JAVA_HOME will be set to " . $ARGV[1] . "\n";
-}
-if ($narg >= 3) {
-    print "JAVA will be set to " . $ARGV[2] . "\n";
-}
-mung ("jhove");
-mung ("adump");
-mung ("gdump");
-mung ("jdump");
-mung ("j2dump");
-mung ("pdump");
-mung ("tdump");
-mung ("wdump");
-
-#Fix up the config file.  We assume that the <jhoveHome>
-#element is all on one line.
-if (!(-e "conf/jhove.conf~")) {
-    rename ("conf/jhove.conf", "conf/jhove.conf~");
-}
-open (INFILE, "conf/jhove.conf~");
-open (OUTFILE, ">conf/jhove.conf");
-while (<INFILE>) {
-    s!<jhoveHome>.*</jhoveHome>!<jhoveHome>$ARGV[0]</jhoveHome>!;
-    print OUTFILE;
-}
-close (INFILE);
-close (OUTFILE);
-if (-e "conf/jhove.conf") {
-    print "Fixed conf/jhove.conf\n";
-}
-exit;
-

ocrmypdf/jhove/gdump

@@ -1,37 +0,0 @@
-#!/bin/sh
-
-########################################################################
-# gdump - JSTOR/Harvard Object Validation Environment
-# Copyright 2004-2005 by the President and Fellows of Harvard College
-# JHOVE is made available under the GNU General Public License (see the
-# file LICENSE for details)
-#
-# Driver script for the GIF dump utility
-#
-# Usage: gdump file
-#
-# where  file  is a GIF file
-#
-# Configuration constants:
-
-JHOVE_HOME=/users/stephen/projects/jhove
-
-JAVA_HOME=/usr/java      # Java JRE directory
-JAVA=$JAVA_HOME/bin/java # Java interpreter
-
-EXTRA_JARS=              # Extra .jar files to add to CLASSPATH
-
-# NOTE: Nothing below this line should be edited
-########################################################################
-
-CP=${JHOVE_HOME}/bin/JhoveApp.jar:${EXTRA_JARS}
-
-# Retrieve a copy of all command line arguments to pass to the application.
-
-ARGS=""
-for ARG do
-    ARGS="$ARGS $ARG"
-done
-
-# Set the CLASSPATH and invoke the Java loader.
-${JAVA} -classpath $CP GDump $ARGS

ocrmypdf/jhove/j2dump

@@ -1,37 +0,0 @@
-#!/bin/sh
-
-########################################################################
-# j2dump - JSTOR/Harvard Object Validation Environment
-# Copyright 2004-2005 by the President and Fellows of Harvard College
-# JHOVE is made available under the GNU General Public License (see the
-# file LICENSE for details)
-#
-# Driver script for the JPEG 2000 dump utility
-#
-# Usage: j2dump file
-#
-# where  file  is a JPEG file
-#
-# Configuration constants:
-
-JHOVE_HOME=/users/stephen/projects/jhove
-
-JAVA_HOME=/usr/java      # Java JRE directory
-JAVA=$JAVA_HOME/bin/java # Java interpreter
-
-EXTRA_JARS=              # Extra .jar files to add to CLASSPATH
-
-# NOTE: Nothing below this line should be edited
-########################################################################
-
-CP=${JHOVE_HOME}/bin/JhoveApp.jar:${EXTRA_JARS}
-
-# Retrieve a copy of all command line arguments to pass to the application.
-
-ARGS=""
-for ARG do
-    ARGS="$ARGS $ARG"
-done
-
-# Set the CLASSPATH and invoke the Java loader.
-${JAVA} -classpath $CP J2Dump $ARGS

ocrmypdf/jhove/jdump

@@ -1,37 +0,0 @@
-#!/bin/sh
-
-########################################################################
-# jdump - JSTOR/Harvard Object Validation Environment
-# Copyright 2004-2005 by the President and Fellows of Harvard College
-# JHOVE is made available under the GNU General Public License (see the
-# file LICENSE for details)
-#
-# Driver script for the JPEG dump utility
-#
-# Usage: jdump file
-#
-# where  file  is a JPEG file
-#
-# Configuration constants:
-
-JHOVE_HOME=/users/stephen/projects/jhove
-
-JAVA_HOME=/usr/java      # Java JRE directory
-JAVA=$JAVA_HOME/bin/java # Java interpreter
-
-EXTRA_JARS=              # Extra .jar files to add to CLASSPATH
-
-# NOTE: Nothing below this line should be edited
-########################################################################
-
-CP=${JHOVE_HOME}/bin/JhoveApp.jar:${EXTRA_JARS}
-
-# Retrieve a copy of all command line arguments to pass to the application.
-
-ARGS=""
-for ARG do
-    ARGS="$ARGS $ARG"
-done
-
-# Set the CLASSPATH and invoke the Java loader.
-${JAVA} -classpath $CP JDump $ARGS

ocrmypdf/jhove/jhove

@@ -1,58 +0,0 @@
-#!/bin/sh
-
-########################################################################
-# JHOVE - JSTOR/Harvard Object Validation Environment
-# Copyright 2003-2005 by JSTOR and the President and Fellows of Harvard College
-# JHOVE is made available under the GNU General Public License (see the
-# file LICENSE for details)
-#
-# Usage: jhove [-c config] [-m module] [-h handler] [-e encoding] [-H handler]
-#              [-o output] [-x saxclass] [-t tempdir] [-b bufsize]
-#              [-l loglevel] [[-krs] dir-file-or-uri [...]]
-#
-# where -c config   Configuration file pathname
-#       -m module   Module name
-#       -h handler  Output handler name (defaults to TEXT)
-#       -e encoding Character encoding of output handler (defaults to UTF-8)
-#       -H handler  About handler name
-#       -o output   Output file pathname (defaults to standard output)
-#       -x saxclass SAX parser class (defaults to J2SE 1.4 default)
-#       -t tempdir  Temporary directory in which to create temporary files
-#       -b bufsize  Buffer size for buffered I/O (defaults to J2SE 1.4 default)
-#       -k          Calculate CRC32, MD5, and SHA-1 checksums
-#       -r          Display raw data flags, not textual equivalents
-#       -s          Format identification based on internal signatures only
-#       dir-file-or-uri Directory, file pathname or URI of formatted content
-#
-# CHANGE for JHOVE 1.8:
-# You no longer have to figure out where JAVA_HOME is; that's the
-# operating system's job. If the OS tells you it can't find Java, 
-# adjust your shell's path or revert to the old way (commented out).
-# Configuration constants:
-
-#JHOVE_HOME=/users/gary/dev/jhove
-JHOVE_HOME=[fill in path to jhove directory]
-
-JAVA_HOME=/usr/java      # Java JRE directory -- change to your local java home
-JAVA=$JAVA_HOME/bin/java # Java interpreter -- usually won't need change
-
-#XTRA_JARS=/users/stephen/xercesImpl.jar
-EXTRA_JARS=              # Extra .jar files to add to CLASSPATH
-
-# NOTE: Nothing below this line should be edited
-########################################################################
-
-CP=${JHOVE_HOME}/bin/JhoveApp.jar:${EXTRA_JARS}
-
-# Retrieve a copy of all command line arguments to pass to the application.
-
-ARGS=""
-for ARG do
-    ARGS="$ARGS $ARG"
-done
-
-# Set the CLASSPATH and invoke the Java loader.
-#{JAVA} -classpath $CP Jhove $ARGS -x org.apache.xerces.parsers.SAXParser
-#${JAVA} -classpath $CP Jhove $ARGS
-# New way, doesn't require you to use JAVA_HOME.
-java -classpath $CP Jhove $ARGS

ocrmypdf/jhove/jhove.tmpl

@@ -1,54 +0,0 @@
-#!/bin/sh
-
-########################################################################
-# JHOVE - JSTOR/Harvard Object Validation Environment
-# Copyright 2003-2004 by JSTOR and the President and Fellows of Harvard College
-# JHOVE is made available under the GNU General Public License (see the
-# file LICENSE for details)
-#
-# Copy jhove.tmpl to jhove, and replace the value of JHOVE_HOME with
-# the path to your jhove directory.
-#
-# Usage: jhove [-c config] [-m module [-p param]] [-h handler [-P param]]
-#              [-e encoding] [-H handler] [-o output] [-x saxclass]
-#              [-t tempdir] [-b bufsize] [[-krs] dir-file-or-uri [...]]
-#
-# where -c config   Configuration file pathname
-#       -m module   Module name
-#       -p param    Module-specific parameter
-#       -h handler  Output handler name (defaults to TEXT)
-#       -P param    Handler-specific parameter
-#       -o output   Output file pathname (defaults to standard output)
-#       -x saxclass SAX parser class (defaults to J2SE 1.4 default)
-#       -t tempdir  Temporary directory in which to create temporary files
-#       -b bufsize  Buffer size for buffered I/O (defaults to J2SE 1.4 default)
-#       -k          Calculate CRC32, MD5, and SHA-1 checksums
-#       -r          Display raw data flags, not textual equivalents
-#       -s          Format identification based on internal signatures only
-#       dir-file-or-uri Directory, file pathname or URI of formatted content
-#
-# Configuration constants:
-
-JHOVE_HOME=[your directory path]/jhove
-
-JAVA_HOME=/usr/java
-JAVA=/usr/bin/java
-
-#XTRA_JARS=/users/stephen/xercesImpl.jar
-EXTRA_JARS=              # Extra .jar files to add to CLASSPATH
-
-# NOTE: Nothing below this line should be edited
-########################################################################
-
-CP=${JHOVE_HOME}/bin/JhoveApp.jar:${EXTRA_JARS}
-
-# Retrieve a copy of all command line arguments to pass to the application.
-
-ARGS=""
-for ARG do
-    ARGS="$ARGS $ARG"
-done
-
-# Set the CLASSPATH and invoke the Java loader.
-#{JAVA} -classpath $CP Jhove $ARGS -x org.apache.xerces.parsers.SAXParser
-${JAVA} -classpath $CP Jhove $ARGS

ocrmypdf/jhove/jhove_bat.tmpl

@@ -1,60 +0,0 @@
-@ECHO OFF
-REM JHOVE - JSTOR/Harvard Object Validation Environment
-REM Copyright 2003-2005 by JSTOR and the President and Fellows of Harvard College
-REM JHOVE is made available under the GNU General Public License (see the
-REM file LICENSE for details)
-REM
-REM Usage: jhove [-c config] [-m module] [-h handler] [-e encoding]
-REM              [-H handler] [-o output] [-x saxclass] [-t tempdir]
-REM              [-b bufsize] [-l loglevel] [[-krs] dir-file-or-uri [...]]
-REM
-REM For Windows systems, copy jhove_bat.tmpl to jhove.bat and change
-REM the value of JHOVE_HOME to the path to your jhove directory.
-REM
-REM where -c config   Configuration file pathname
-REM       -m module   Module name
-REM       -h handler  Output handler name (defaults to TEXT)
-REM       -e encoding Character encoding of output handler (defaults to UTF-8)
-REM       -H handler  About handler name
-REM       -o output   Output file pathname (defaults to standard output)
-REM       -x saxclass SAX parser class (defaults to J2SE 1.4 default)
-REM       -t tempdir  Temporary directory in which to create temporary files
-REM       -b bufsize  Buffer size for buffered I/O (defaults to J2SE default)
-REM       -l loglevel Logging level
-REM       -k          Calculate CRC32, MD5, and SHA-1 checksums
-REM       -r          Display raw data flags, not textual equivalents
-REM       -s          Format identification based on internal signatures only
-REM       dir-file-or-uri Directory, file pathname, or URI of formatted content
-REM
-REM Configuration constants:
-REM JHOVE_HOME Jhove installation directory
-REM JAVA_HOME  Java JRE directory
-REM JAVA       Java interpreter
-REM EXTRA_JARS Extra jar files to add to CLASSPATH
-
-REM SET JHOVE_HOME="C:\Program Files\jhove"
-SET JHOVE_HOME="[your directory path]\jhove"
-
-SET EXTRA_JARS=
-
-REM NOTE: Nothing below this line should be edited
-REM #########################################################################
-
-
-SET CP=%JHOVE_HOME%\bin\JhoveApp.jar
-IF "%EXTRA_JARS%"=="" GOTO FI
-  SET CP=%CP%:%EXTRA_JARS
-:FI
-
-REM Retrieve a copy of all command line arguments to pass to the application
-
-SET ARGS=
-:WHILE
-IF "%1"=="" GOTO LOOP
-  SET ARGS=%ARGS% %1
-  SHIFT
-  GOTO WHILE
-:LOOP
-
-REM Set the CLASSPATH and invoke the Java loader
-JAVA -classpath %CP% Jhove %ARGS%

ocrmypdf/jhove/md5.pl

@@ -1,14 +0,0 @@
-#!/usr/bin/perl -w
-
-# Generate MD5 checksum
-
-use Digest::MD5;
-
-die "usage: md5.pl file\n" if $#ARGV < 0;
-
-open (FILE, "<$ARGV[0]") or die "can't open file\"$ARGV[0]\"!\n";
-$ctx = Digest::MD5->new->addfile (*FILE);
-close (FILE);
-$digest = $ctx->hexdigest;
-
-print "$digest\n";

ocrmypdf/jhove/packagejhove.sh

@@ -1,44 +0,0 @@
-#!/bin/sh
-#DO NOT RUN THIS ON A DEVELOPEMENT DIRECTORY, ONLY ON A 
-#CHECKED-OUT COPY TO BE PACKAGED!
-
-if [ "$1" = "" ]; then
- echo "Usage: packagejhove.sh [version]"
- echo "e.g., packagejhove.sh 1_8"
- exit 1
-fi
-
-echo "This script will prepare your directory for uploading."
-echo "DO NOT RUN IT unless you're a developer and know what"
-echo "you are doing. "
-
-echo "Start in the top-level directory of the JHOVE checkout."
-echo "Run ant and ant javadoc and do any necessary testing and"
-echo "committing before running this script."
-echo 
-echo
-
-echo "To continue, enter the secret phrase."
-read OATH
-if [ "$OATH" != "I solemnly swear that I am up to no good" ]; then
- exit 1
-fi
-
-cd ..
-cat >>CVSS << EOF
-CVS
-.cvsignore
-EOF
-tar cvfX jhove-$1.tar CVSS jhove
-gzip jhove-$1.tar
-cp -r jhove jhove-zip
-cd jhove-zip
-find . \( -name CVS -o -name .cvsignore \) -exec rm -r {} \;	
-cd ..
-mv jhove jhove-ok
-mv jhove-zip jhove
-zip -r jhove-$1.zip jhove
-rm -r jhove
-mv jhove-ok jhove
-jhove/md5.pl jhove-$1.tar.gz >jhove-$1.tar.gz.md5
-jhove/md5.pl jhove-$1.zip >jhove-$1.zip.md5 

ocrmypdf/jhove/pdump

@@ -1,37 +0,0 @@
-#!/bin/sh
-
-########################################################################
-# pdump - JSTOR/Harvard Object Validation Environment
-# Copyright 2003-2005 by JSTOR and the President and Fellows of Harvard College
-# JHOVE is made available under the GNU General Public License (see the
-# file LICENSE for details)
-#
-# Driver script for the PDF dump utility
-#
-# Usage: pdump file
-#
-# where  file  is a PDF file
-#
-# Configuration constants:
-
-JHOVE_HOME=/users/stephen/projects/jhove
-
-JAVA_HOME=/usr/java      # Java JRE directory
-JAVA=$JAVA_HOME/bin/java # Java interpreter
-
-EXTRA_JARS=              # Extra .jar files to add to CLASSPATH
-
-# NOTE: Nothing below this line should be edited
-########################################################################
-
-CP=${JHOVE_HOME}/bin/JhoveApp.jar:${EXTRA_JARS}
-
-# Retrieve a copy of all command line arguments to pass to the application.
-
-ARGS=""
-for ARG do
-    ARGS="$ARGS $ARG"
-done
-
-# Set the CLASSPATH and invoke the Java loader.
-${JAVA} -classpath $CP PDump $ARGS

ocrmypdf/jhove/userhome

@@ -1,24 +0,0 @@
-#!/bin/sh
-
-########################################################################
-# userhome - JSTOR/Harvard Object Validation Environment
-# Copyright 2004-2006 by the President and Fellows of Harvard College
-# JHOVE is made available under the GNU General Public License (see the
-# file LICENSE for details)
-#
-# Driver script to display the default Java user.home property
-#
-# Usage: userhome
-#
-# Configuration constants:
-
-JHOVE_HOME=/users/stephen/projects/jhove
-
-JAVA_HOME=/usr/java      # Java JRE directory
-JAVA=$JAVA_HOME/bin/java # Java interpreter
-
-# NOTE: Nothing below this line should be edited
-########################################################################
-
-# Set the CLASSPATH and invoke the Java loader.
-${JAVA} -classpath ${JHOVE_HOME}/classes UserHome

ocrmypdf/leptonica.py

@@ -10,11 +10,15 @@
 
 from __future__ import print_function, absolute_import, division
 import argparse
-import ctypes as C
 import sys
 import os
 import logging
 from tempfile import TemporaryFile
+from ctypes.util import find_library
+from .lib._leptonica import ffi
+from functools import lru_cache
+
+lept = ffi.dlopen(find_library('lept'))
 
 logger = logging.getLogger(__name__)
 
@@ -25,67 +29,6 @@ def stderr(*objs):
     print("leptonica.py:", *objs, file=sys.stderr)
 
 
-from ctypes.util import find_library
-lept_lib = find_library('lept')
-if not lept_lib:
-    stderr("Could not find the Leptonica library")
-    sys.exit(3)
-try:
-    lept = C.cdll.LoadLibrary(lept_lib)
-except Exception:
-    stderr("Could not load the Leptonica library from %s", lept_lib)
-    sys.exit(3)
-
-
-class _PIXCOLORMAP(C.Structure):
-    """struct PixColormap from Leptonica src/pix.h
-    """
-
-    _fields_ = [
-        ("array", C.c_void_p),
-        ("depth", C.c_int32),
-        ("nalloc", C.c_int32),
-        ("n", C.c_int32)
-    ]
-
-
-class _PIX(C.Structure):
-    """struct Pix from Leptonica src/pix.h
-    """
-
-    _fields_ = [
-        ("w", C.c_uint32),
-        ("h", C.c_uint32),
-        ("d", C.c_uint32),
-        ("wpl", C.c_uint32),
-        ("refcount", C.c_uint32),
-        ("xres", C.c_int32),
-        ("yres", C.c_int32),
-        ("informat", C.c_int32),
-        ("text", C.POINTER(C.c_char)),
-        ("colormap", C.POINTER(_PIXCOLORMAP)),
-        ("data", C.POINTER(C.c_uint32))
-    ]
-
-
-PIX = C.POINTER(_PIX)
-
-lept.pixRead.argtypes = [C.c_char_p]
-lept.pixRead.restype = PIX
-lept.pixScale.argtypes = [PIX, C.c_float, C.c_float]
-lept.pixScale.restype = PIX
-lept.pixDeskew.argtypes = [PIX, C.c_int32]
-lept.pixDeskew.restype = PIX
-lept.pixFindSkew.argtypes = [PIX, C.POINTER(C.c_float), C.POINTER(C.c_float)]
-lept.pixFindSkew.restype = C.c_int32
-lept.pixWriteImpliedFormat.argtypes = [C.c_char_p, PIX, C.c_int32, C.c_int32]
-lept.pixWriteImpliedFormat.restype = C.c_int32
-lept.pixDestroy.argtypes = [C.POINTER(PIX)]
-lept.pixDestroy.restype = None
-lept.getLeptonicaVersion.argtypes = []
-lept.getLeptonicaVersion.restype = C.c_char_p
-
-
 class LeptonicaErrorTrap(object):
     """Context manager to trap errors reported by Leptonica.
 
@@ -140,103 +83,140 @@ class LeptonicaIOError(LeptonicaError):
     pass
 
 
-def pixRead(filename):
-    """Load an image file into a PIX object.
+class Pix:
+    def __init__(self, cpix):
+        self.cpix = ffi.gc(cpix, Pix._pix_destroy)
 
-    Leptonica can load TIFF, PNM (PBM, PGM, PPM), PNG, and JPEG.  If loading
-    fails then the object will wrap a C null pointer.
-
-    """
-    with LeptonicaErrorTrap():
-        return lept.pixRead(filename.encode(sys.getfilesystemencoding()))
-
-
-def pixScale(pix, scalex, scaley):
-    """Returns the pix object rescaled according to the proportions given."""
-    with LeptonicaErrorTrap():
-        return lept.pixScale(pix, scalex, scaley)
-
-
-def pixDeskew(pix, reduction_factor=0):
-    """Returns the deskewed pix object.
-
-    A clone of the original is returned when the algorithm cannot find a skew
-    angle with sufficient confidence.
-
-    reduction_factor -- amount to downsample (0 for default) when searching
-        for skew angle
-
-    """
-    with LeptonicaErrorTrap():
-        return lept.pixDeskew(pix, reduction_factor)
-
-
-def pixFindSkew(pix):
-    """Returns a tuple (deskew angle in degrees, confidence value).
-
-    Returns (None, None) if no angle is available.
-
-    """
-    with LeptonicaErrorTrap():
-        angle = C.c_float(0.0)
-        confidence = C.c_float(0.0)
-        result = lept.pixFindSkew(pix, C.byref(angle), C.byref(confidence))
-        if result == 0:
-            return (angle.value, confidence.value)
+    def __repr__(self):
+        if self.cpix:
+            s = "<leptonica.Pix image size={0}x{1} depth={2} at 0x{3:x}>"
+            return s.format(self.cpix.w, self.cpix.h, self.cpix.d,
+                            int(ffi.cast("intptr_t", self.cpix)))
         else:
-            return (None, None)
+            return "<leptonica.Pix image NULL>"
 
+    @classmethod
+    def read(cls, filename):
+        """Load an image file into a PIX object.
 
-def pixWriteImpliedFormat(filename, pix, jpeg_quality=0, jpeg_progressive=0):
-    """Write pix to the filename, with the extension indicating format.
+        Leptonica can load TIFF, PNM (PBM, PGM, PPM), PNG, and JPEG.  If
+        loading fails then the object will wrap a C null pointer.
+        """
+        with LeptonicaErrorTrap():
+            return cls(lept.pixRead(
+                filename.encode(sys.getfilesystemencoding())))
 
-    jpeg_quality -- quality (iff JPEG; 1 - 100, 0 for default)
-    jpeg_progressive -- (iff JPEG; 0 for baseline seq., 1 for progressive)
+    def write_implied_format(
+            self, filename, jpeg_quality=0, jpeg_progressive=0):
+        """Write pix to the filename, with the extension indicating format.
 
-    """
-    fileroot, extension = os.path.splitext(filename)
-    fix_pnm = False
-    if extension.lower() in ('.pbm', '.pgm', '.ppm'):
-        # Leptonica does not process handle these extensions correctly, but
-        # does handle .pnm correctly.  Add another .pnm suffix.
-        filename += '.pnm'
-        fix_pnm = True
+        jpeg_quality -- quality (iff JPEG; 1 - 100, 0 for default)
+        jpeg_progressive -- (iff JPEG; 0 for baseline seq., 1 for progressive)
+        """
+        fileroot, extension = os.path.splitext(filename)
+        fix_pnm = False
+        if extension.lower() in ('.pbm', '.pgm', '.ppm'):
+            # Leptonica does not process handle these extensions correctly, but
+            # does handle .pnm correctly.  Add another .pnm suffix.
+            filename += '.pnm'
+            fix_pnm = True
 
-    with LeptonicaErrorTrap():
-        lept.pixWriteImpliedFormat(
-            filename.encode(sys.getfilesystemencoding()),
-            pix, jpeg_quality, jpeg_progressive)
+        with LeptonicaErrorTrap():
+            lept.pixWriteImpliedFormat(
+                filename.encode(sys.getfilesystemencoding()),
+                self.cpix, jpeg_quality, jpeg_progressive)
 
-    if fix_pnm:
-        from shutil import move
-        move(filename, filename[:-4])   # Remove .pnm suffix
+        if fix_pnm:
+            from shutil import move
+            move(filename, filename[:-4])   # Remove .pnm suffix
 
+    def deskew(self, reduction_factor=0):
+        """Returns the deskewed pix object.
 
-def pixDestroy(pix):
-    """Destroy the pix object.
+        A clone of the original is returned when the algorithm cannot find a
+        skew angle with sufficient confidence.
 
-    Function signature is pixDestroy(struct Pix **), hence C.byref() to pass
-    the address of the pointer.
+        reduction_factor -- amount to downsample (0 for default) when searching
+            for skew angle
+        """
+        with LeptonicaErrorTrap():
+            return Pix(lept.pixDeskew(self.cpix, reduction_factor))
 
-    """
-    with LeptonicaErrorTrap():
-        lept.pixDestroy(C.byref(pix))
+    def scale(self, scalex, scaley):
+        "Returns the pix object rescaled according to the proportions given."
+        with LeptonicaErrorTrap():
+            return Pix(lept.pixScale(self.cpix, scalex, scaley))
+
+    def rotate180(self):
+        with LeptonicaErrorTrap():
+            return Pix(lept.pixRotate180(ffi.NULL, self.cpix))
+
+    def find_skew(self):
+        """Returns a tuple (deskew angle in degrees, confidence value).
+
+        Returns (None, None) if no angle is available.
+        """
+        with LeptonicaErrorTrap():
+            angle = ffi.new('float *', 0.0)
+            confidence = ffi.new('float *', 0.0)
+            result = lept.pixFindSkew(self.cpix, angle, confidence)
+            if result == 0:
+                return (angle[0], confidence[0])
+            else:
+                return (None, None)
+
+    @staticmethod
+    def correlation_binary(pix1, pix2):
+        if getLeptonicaVersion() < 'leptonica-1.72':
+            # Older versions of Leptonica (pre-1.72) have a buggy
+            # implementation of pixCorrelationBinary that overflows on larger
+            # images.
+            pix1_count = ffi.new('l_int32 *', 0)
+            pix2_count = ffi.new('l_int32 *', 0)
+            pixn_count = ffi.new('l_int32 *', 0)
+            tab8 = lept.makePixelSumTab8()  # Small memory leak on each call
+
+            lept.pixCountPixels(pix1.cpix, pix1_count, tab8)
+            lept.pixCountPixels(pix2.cpix, pix2_count, tab8)
+            pixn = Pix(lept.pixAnd(ffi.NULL, pix1.cpix, pix2.cpix))
+            lept.pixCountPixels(pixn.cpix, pixn_count, tab8)
+
+            # Python converts these int32s to larger units as needed
+            # to avoid overflow. Overflow happens easily here.
+            correlation = (
+                    (pixn_count[0] * pixn_count[0]) /
+                    (pix1_count[0] * pix2_count[0])
+                    )
+            return correlation
+        else:
+            correlation = ffi.new('float *', 0.0)
+            result = lept.pixCorrelationBinary(pix1.cpix, pix2.cpix,
+                                               correlation)
+            if result != 0:
+                raise LeptonicaError("Correlation failed")
+            return correlation[0]
+
+    @staticmethod
+    def _pix_destroy(pix):
+        ptr_to_pix = ffi.new('PIX **', pix)
+        lept.pixDestroy(ptr_to_pix)
+        # print('pix destroy ' + repr(pix))
 
 
+@lru_cache(maxsize=1)
 def getLeptonicaVersion():
     """Get Leptonica version string.
 
     Caveat: Leptonica expects the caller to free this memory.  We don't,
     since that would involve binding to libc to access libc.free(),
     a pointless effort to reclaim 100 bytes of memory.
-
     """
-    return lept.getLeptonicaVersion().decode()
+    return ffi.string(lept.getLeptonicaVersion()).decode()
 
 
 def deskew(infile, outfile, dpi):
     try:
-        pix_source = pixRead(infile)
+        pix_source = Pix.read(infile)
     except LeptonicaIOError:
         raise LeptonicaIOError("Failed to open file: %s" % infile)
 
@@ -244,14 +224,12 @@ def deskew(infile, outfile, dpi):
         reduction_factor = 1  # Don't downsample too much if DPI is already low
     else:
         reduction_factor = 0  # Use default
-    pix_deskewed = pixDeskew(pix_source, reduction_factor)
+    pix_deskewed = pix_source.deskew(reduction_factor)
 
     try:
-        pixWriteImpliedFormat(outfile, pix_deskewed)
+        pix_deskewed.write_implied_format(outfile)
     except LeptonicaIOError:
         raise LeptonicaIOError("Failed to open destination file: %s" % outfile)
-    pixDestroy(pix_source)
-    pixDestroy(pix_deskewed)
 
 
 if __name__ == '__main__':
@@ -325,7 +303,6 @@ def test_skew_angle():
             rotated_im.save(tmpfile)
             pix = pixRead(tmpfile.name)
             angle, confidence = pixFindSkew(pix)
-            pixDestroy(pix)
             print('{0} {1}  {2}'.format(rotate_angle, angle, confidence), file=sys.stderr)
 
 

ocrmypdf/lib/compile_leptonica.py

@@ -0,0 +1,64 @@
+from cffi import FFI
+
+ffi = FFI()
+ffi.set_source("ocrmypdf.lib._leptonica", None)
+ffi.cdef("""
+typedef signed char             l_int8;
+typedef unsigned char           l_uint8;
+typedef short                   l_int16;
+typedef unsigned short          l_uint16;
+typedef int                     l_int32;
+typedef unsigned int            l_uint32;
+typedef float                   l_float32;
+typedef double                  l_float64;
+typedef long long               l_int64;
+typedef unsigned long long      l_uint64;
+
+struct Pix
+{
+    l_uint32             w;           /* width in pixels                   */
+    l_uint32             h;           /* height in pixels                  */
+    l_uint32             d;           /* depth in bits (bpp)               */
+    l_uint32             spp;         /* number of samples per pixel       */
+    l_uint32             wpl;         /* 32-bit words/line                 */
+    l_uint32             refcount;    /* reference count (1 if no clones)  */
+    l_int32              xres;        /* image res (ppi) in x direction    */
+                                      /* (use 0 if unknown)                */
+    l_int32              yres;        /* image res (ppi) in y direction    */
+                                      /* (use 0 if unknown)                */
+    l_int32              informat;    /* input file format, IFF_*          */
+    l_int32              special;     /* special instructions for I/O, etc */
+    char                *text;        /* text string associated with pix   */
+    struct PixColormap  *colormap;    /* colormap (may be null)            */
+    l_uint32            *data;        /* the image data                    */
+};
+typedef struct Pix PIX;
+
+struct PixColormap
+{
+    void            *array;     /* colormap table (array of RGBA_QUAD)     */
+    l_int32          depth;     /* of pix (1, 2, 4 or 8 bpp)               */
+    l_int32          nalloc;    /* number of color entries allocated       */
+    l_int32          n;         /* number of color entries used            */
+};
+typedef struct PixColormap  PIXCMAP;
+""")
+
+ffi.cdef("""
+PIX * pixRead ( const char *filename );
+PIX * pixScale ( PIX *pixs, l_float32 scalex, l_float32 scaley );
+l_int32 pixFindSkew ( PIX *pixs, l_float32 *pangle, l_float32 *pconf );
+l_int32 pixWriteImpliedFormat ( const char *filename, PIX *pix, l_int32 quality, l_int32 progressive );
+void pixDestroy ( PIX **ppix );
+PIX * pixDeskew ( PIX *pixs, l_int32 redsearch );
+char * getLeptonicaVersion (  );
+l_int32 pixCorrelationBinary(PIX *pix1, PIX *pix2, l_float32 *pval);
+PIX *pixRotate180(PIX *pixd, PIX *pixs);
+
+l_int32 pixCountPixels ( PIX *pix, l_int32 *pcount, l_int32 *tab8 );
+PIX * pixAnd ( PIX *pixd, PIX *pixs1, PIX *pixs2 );
+l_int32 * makePixelSumTab8 ( void );
+""")
+
+if __name__ == '__main__':
+    ffi.compile()

ocrmypdf/main.py

@@ -2,54 +2,46 @@
 # © 2015 James R. Barlow: github.com/jbarlow83
 
 from contextlib import suppress
-from tempfile import NamedTemporaryFile, mkdtemp
+from tempfile import mkdtemp
 import sys
 import os
-import fileinput
 import re
 import shutil
 import warnings
 import multiprocessing
 import atexit
 import textwrap
+import img2pdf
+import logging
 
 import PyPDF2 as pypdf
 from PIL import Image
 
-from subprocess import Popen, check_call, PIPE, CalledProcessError, \
-    TimeoutExpired, check_output
-try:
-    from subprocess import DEVNULL
-except ImportError:
-    DEVNULL = open(os.devnull, 'wb')
-
+from functools import partial
 
 from ruffus import transform, suffix, merge, active_if, regex, jobs_limit, \
     formatter, follows, split, collate, check_if_uptodate
+import ruffus.ruffus_exceptions as ruffus_exceptions
 import ruffus.cmdline as cmdline
+import ruffus.proxy_logger as proxy_logger
 
 from .hocrtransform import HocrTransform
 from .pageinfo import pdf_get_all_pageinfo
 from .pdfa import generate_pdfa_def
 from . import ghostscript
 from . import tesseract
+from . import qpdf
+from . import ExitCode, page_number
 
+import pkg_resources
+
+VERSION = pkg_resources.get_distribution('ocrmypdf').version
 
 warnings.simplefilter('ignore', pypdf.utils.PdfReadWarning)
 
 
 BASEDIR = os.path.dirname(os.path.realpath(__file__))
-JHOVE_PATH = os.path.realpath(os.path.join(BASEDIR, 'jhove'))
-JHOVE_JAR = os.path.join(JHOVE_PATH, 'bin', 'JhoveApp.jar')
-JHOVE_CFG = os.path.join(JHOVE_PATH, 'conf', 'jhove.conf')
 
-EXIT_BAD_ARGS = 1
-EXIT_BAD_INPUT_FILE = 2
-EXIT_MISSING_DEPENDENCY = 3
-EXIT_INVALID_OUTPUT_PDFA = 4
-EXIT_FILE_ACCESS_ERROR = 5
-EXIT_ALREADY_DONE_OCR = 6
-EXIT_OTHER_ERROR = 15
 
 # -------------
 # External dependencies
@@ -58,7 +50,7 @@ MINIMUM_TESS_VERSION = '3.02.02'
 
 
 def complain(message):
-    print(textwrap.wrap(message), file=sys.stderr)
+    print(*textwrap.wrap(message), file=sys.stderr)
 
 
 if tesseract.version() < MINIMUM_TESS_VERSION:
@@ -66,7 +58,41 @@ if tesseract.version() < MINIMUM_TESS_VERSION:
         "Please install tesseract {0} or newer "
         "(currently installed version is {1})".format(
             MINIMUM_TESS_VERSION, tesseract.version()))
-    sys.exit(EXIT_MISSING_DEPENDENCY)
+    sys.exit(ExitCode.missing_dependency)
+
+try:
+    import PIL.features
+    check_codec = PIL.features.check_codec
+except (ImportError, AttributeError):
+    def check_codec(codec_name):
+        if codec_name == 'jpg':
+            return 'jpeg_encoder' in dir(Image.core)
+        elif codec_name == 'zlib':
+            return 'zip_encoder' in dir(Image.core)
+        raise NotImplementedError(codec_name)
+
+
+def check_pil_encoder(codec_name, friendly_name):
+    try:
+        if check_codec(codec_name):
+            return
+    except Exception:
+        pass
+    complain(
+        "ERROR: Your version of the Python imaging library (Pillow) was "
+        "compiled without support for " + friendly_name + " encoding/decoding."
+        "\n"
+        "You will need to uninstall Pillow and reinstall it with PNG and JPEG "
+        "support (libjpeg and zlib)."
+        "\n"
+        "See installation instructions for your platform here:\n"
+        "    https://pillow.readthedocs.org/installation.html"
+    )
+    sys.exit(ExitCode.missing_dependency)
+
+
+check_pil_encoder('jpg', 'JPEG')
+check_pil_encoder('zlib', 'PNG')
 
 
 # -------------
@@ -75,12 +101,12 @@ if tesseract.version() < MINIMUM_TESS_VERSION:
 parser = cmdline.get_argparse(
     prog="ocrmypdf",
     description="Generate searchable PDF file from an image-only PDF file.",
-    version='3.0rc4',
+    version=VERSION,
     fromfile_prefix_chars='@',
     ignored_args=[
         'touch_files_only', 'recreate_database', 'checksum_file_name',
         'key_legend_in_graph', 'draw_graph_horizontally', 'flowchart_format',
-        'forced_tasks', 'target_tasks'])
+        'forced_tasks', 'target_tasks', 'use_threads', 'jobs', 'log_file'])
 
 parser.add_argument(
     'input_file',
@@ -91,10 +117,13 @@ parser.add_argument(
 parser.add_argument(
     '-l', '--language', action='append',
     help="languages of the file to be OCRed")
+parser.add_argument(
+    '-j', '--jobs', metavar='N', type=int,
+    help="Use up to N CPU cores simultaneously (default: use all)")
 
 metadata = parser.add_argument_group(
     "Metadata options",
-    "Set output PDF/A metadata (default: use input document's title)")
+    "Set output PDF/A metadata (default: use input document's metadata)")
 metadata.add_argument(
     '--title', type=str,
     help="set document title (place multiple words in quotes)")
@@ -103,15 +132,17 @@ metadata.add_argument(
     help="set document author")
 metadata.add_argument(
     '--subject', type=str,
-    help="set document")
+    help="set document subject description")
 metadata.add_argument(
     '--keywords', type=str,
     help="set document keywords")
 
-
 preprocessing = parser.add_argument_group(
     "Preprocessing options",
     "Improve OCR quality and final image")
+preprocessing.add_argument(
+    '-r', '--rotate-pages', action='store_true',
+    help="automatically rotate pages based on detected text orientation")
 preprocessing.add_argument(
     '-d', '--deskew', action='store_true',
     help="deskew each page before performing OCR")
@@ -126,32 +157,35 @@ preprocessing.add_argument(
     help="oversample images to at least the specified DPI, to improve OCR "
          "results slightly")
 
-parser.add_argument(
+ocrsettings = parser.add_argument_group(
+    "OCR options",
+    "Control how OCR is applied")
+ocrsettings.add_argument(
     '-f', '--force-ocr', action='store_true',
     help="rasterize any fonts or vector images on each page and apply OCR")
-parser.add_argument(
+ocrsettings.add_argument(
     '-s', '--skip-text', action='store_true',
     help="skip OCR on any pages that already contain text, but include the"
          " page in final output")
-parser.add_argument(
+ocrsettings.add_argument(
     '--skip-big', type=float, metavar='MPixels',
     help="skip OCR on pages larger than the specified amount of megapixels, "
          "but include skipped pages in final output")
-# parser.add_argument(
-#     '--exact-image', action='store_true',
-#     help="Use original page from PDF without re-rendering")
 
 advanced = parser.add_argument_group(
     "Advanced",
     "Advanced options for power users")
 advanced.add_argument(
-    '--tesseract-config', default=[], type=list, action='append',
+    '--tesseract-config', action='append', metavar='CFG', default=[],
     help="additional Tesseract configuration files")
 advanced.add_argument(
-    '--pdf-renderer', choices=['tesseract', 'hocr'], default='hocr',
+    '--tesseract-pagesegmode', action='store', type=int, metavar='PSM',
+    help="set Tesseract page segmentation mode (see tesseract --help)")
+advanced.add_argument(
+    '--pdf-renderer', choices=['auto', 'tesseract', 'hocr'], default='auto',
     help='choose OCR PDF renderer')
 advanced.add_argument(
-    '--tesseract-timeout', default=180.0, type=float,
+    '--tesseract-timeout', default=180.0, type=float, metavar='SECONDS',
     help='give up on OCR after the timeout, but copy the preprocessed page '
          'into the final output')
 
@@ -183,13 +217,22 @@ if not set(options.language).issubset(tesseract.languages()):
         "The installed version of tesseract does not have language "
         "data for the following requested languages: ")
     for lang in (set(options.language) - tesseract.languages()):
-        complain(lang, file=sys.stderr)
-    sys.exit(EXIT_BAD_ARGS)
+        complain(lang)
+    sys.exit(ExitCode.bad_args)
 
 
 # ----------
 # Arguments
 
+if options.pdf_renderer == 'auto':
+    options.pdf_renderer = 'hocr'
+
+if options.pdf_renderer == 'tesseract' and tesseract.version() < '3.04.01' \
+        and os.environ.get('OCRMYPDF_SHARP_TTF', '') != '1':
+    complain(
+        "WARNING: Your version of tesseract has problems with PDF output. "
+        "Some PDF viewers will fail to find searchable text.\n"
+        "--pdf-renderer=tesseract is not recommended.")
 
 if any((options.deskew, options.clean, options.clean_final)):
     try:
@@ -197,7 +240,7 @@ if any((options.deskew, options.clean, options.clean_final)):
     except ImportError:
         complain(
             "Install the 'unpaper' program to use --deskew or --clean.")
-        sys.exit(EXIT_BAD_ARGS)
+        sys.exit(ExitCode.bad_args)
 else:
     unpaper = None
 
@@ -209,7 +252,7 @@ if options.debug_rendering and options.pdf_renderer == 'tesseract':
 if options.force_ocr and options.skip_text:
     complain(
         "Error: --force-ocr and --skip-text are mutually incompatible.")
-    sys.exit(EXIT_BAD_ARGS)
+    sys.exit(ExitCode.bad_args)
 
 if options.clean and not options.clean_final \
         and options.pdf_renderer == 'tesseract':
@@ -217,13 +260,34 @@ if options.clean and not options.clean_final \
         "Tesseract PDF renderer cannot render --clean pages without "
         "also performing --clean-final, so --clean-final is assumed.")
 
+lossless_reconstruction = False
+if options.pdf_renderer == 'hocr':
+    if not options.deskew and not options.clean_final and not options.force_ocr:
+        lossless_reconstruction = True
+
 
 # ----------
 # Logging
 
 
-_logger, _logger_mutex = cmdline.setup_logging(__name__, options.log_file,
-                                               options.verbose)
+def logging_factory(logger_name, listargs):
+    log_file_name, verbose = listargs
+
+    root_logger = logging.getLogger(logger_name)
+    root_logger.setLevel(logging.DEBUG)
+
+    handler = logging.StreamHandler(sys.stderr)
+    formatter_ = logging.Formatter("%(levelname)7s - %(message)s")
+    handler.setFormatter(formatter_)
+    if verbose:
+        handler.setLevel(logging.DEBUG)
+    else:
+        handler.setLevel(logging.INFO)
+    root_logger.addHandler(handler)
+    return root_logger
+
+_logger, _logger_mutex = proxy_logger.make_shared_logger_and_proxy(
+    logging_factory, __name__, [None, options.verbose])
 
 
 class WrappedLogger:
@@ -313,9 +377,8 @@ def cleanup_working_files(*args):
 
 @transform(
     input=options.input_file,
-    filter=suffix('.pdf'),
-    output='.repaired.pdf',
-    output_dir=work_folder,
+    filter=formatter('(?i)\.pdf'),
+    output=work_folder + '{basename[0]}.repaired.pdf',
     extras=[_log, _pdfinfo, _pdfinfo_lock])
 def repair_pdf(
         input_file,
@@ -323,14 +386,11 @@ def repair_pdf(
         log,
         pdfinfo,
         pdfinfo_lock):
-    args_qpdf = [
-        'qpdf', input_file, output_file
-    ]
-    check_call(args_qpdf)
 
+    qpdf.repair(input_file, output_file, log)
     with pdfinfo_lock:
         pdfinfo.extend(pdf_get_all_pageinfo(output_file))
-        log.info(pdfinfo)
+        log.debug(pdfinfo)
 
 
 def get_pageinfo(input_file, pdfinfo, pdfinfo_lock):
@@ -348,16 +408,16 @@ def is_ocr_required(pageinfo, log):
         # or both. It seems quite unlikely that one would find meaningful text
         # from rasterizing vector content. So skip the page.
         log.info(
-            "Page {0} has no images - skipping OCR".format(page)
+            "{0:4d}: page has no images - skipping OCR".format(page)
         )
         ocr_required = False
     elif pageinfo['has_text']:
-        s = "Page {0} already has text! – {1}"
+        s = "{0:4d}: page already has text! – {1}"
 
         if not options.force_ocr and not options.skip_text:
             log.error(s.format(page,
                                "aborting (use --force-ocr to force OCR)"))
-            sys.exit(EXIT_ALREADY_DONE_OCR)
+            sys.exit(ExitCode.already_done_ocr)
         elif options.force_ocr:
             log.info(s.format(page,
                               "rasterizing text and running OCR anyway"))
@@ -371,9 +431,10 @@ def is_ocr_required(pageinfo, log):
         pixel_count = pageinfo['width_pixels'] * pageinfo['height_pixels']
         if pixel_count > (options.skip_big * 1000000):
             ocr_required = False
-            log.info(
-                "Page {0} is very large; skipping due to -b".format(page))
-
+            log.warning(
+                "{0:4d}: page too big, skipping OCR "
+                "({1:.1f} MPixels > {2:.1f} MPixels --skip-big)".format(
+                    page, pixel_count / 1000000, options.skip_big))
     return ocr_required
 
 
@@ -392,16 +453,14 @@ def split_pages(
         with suppress(FileNotFoundError):
             os.unlink(oo)
 
-    pages = check_output(['qpdf', '--show-npages', input_file],
-                         universal_newlines=True, close_fds=True)
+    # If no files were repaired the input will be empty
+    if not input_file:
+        log.error("{0}: file not found or invalid argument".format(
+                options.input_file))
+        sys.exit(ExitCode.input_file)
 
-    for n in range(int(pages)):
-        args_qpdf = [
-            'qpdf', input_file,
-            '--pages', input_file, '{0}'.format(n + 1), '--',
-            os.path.join(work_folder, '{0:06d}.page.pdf'.format(n + 1))
-        ]
-        check_call(args_qpdf)
+    npages = qpdf.get_npages(input_file, log)
+    qpdf.split_pages(input_file, work_folder, npages)
 
     from glob import glob
     for filename in glob(os.path.join(work_folder, '*.page.pdf')):
@@ -416,9 +475,85 @@ def split_pages(
                 os.path.basename(filename)[0:6] + alt_suffix))
 
 
+@active_if(options.rotate_pages)
 @transform(
     input=split_pages,
-    filter=suffix('.ocr.page.pdf'),
+    filter=suffix('.page.pdf'),
+    output='.preview.jpg',
+    output_dir=work_folder,
+    extras=[_log, _pdfinfo, _pdfinfo_lock])
+def rasterize_preview(
+        input_file,
+        output_file,
+        log,
+        pdfinfo,
+        pdfinfo_lock):
+    ghostscript.rasterize_pdf(
+        input_file=input_file,
+        output_file=output_file,
+        xres=200,
+        yres=200,
+        raster_device='jpeggray',
+        log=log)
+
+
+@collate(
+    input=[split_pages, rasterize_preview],
+    filter=regex(r".*/(\d{6})(\.ocr|\.skip)(?:\.page\.pdf|\.preview\.jpg)"),
+    output=os.path.join(work_folder, r'\1\2.oriented.pdf'),
+    extras=[_log, _pdfinfo, _pdfinfo_lock])
+def orient_page(
+        infiles,
+        output_file,
+        log,
+        pdfinfo,
+        pdfinfo_lock):
+
+    page_pdf = next(ii for ii in infiles if ii.endswith('.page.pdf'))
+
+    if not options.rotate_pages:
+        re_symlink(page_pdf, output_file)
+        return
+    preview = next(ii for ii in infiles if ii.endswith('.preview.jpg'))
+
+    orient_conf = tesseract.get_orientation(
+        preview,
+        language=options.language,
+        timeout=options.tesseract_timeout,
+        log=log)
+
+    direction = {
+        0: '⇧',
+        90: '⇦',
+        180: '⇩',
+        270: '⇨'
+    }
+
+    log.info(
+        '{0:4d}: page is facing {1}, confidence {2:.2f}{3}'.format(
+            page_number(preview),
+            direction.get(orient_conf.angle, '?'),
+            orient_conf.confidence,
+            ' - correcting rotation' if orient_conf.angle != 0 else '')
+    )
+
+    if orient_conf.angle == 0:
+        re_symlink(page_pdf, output_file)
+    else:
+        writer = pypdf.PdfFileWriter()
+        reader = pypdf.PdfFileReader(page_pdf)
+        page = reader.pages[0]
+
+        # Rotate opposite of orientation
+        rotated_page = page.rotateClockwise(orient_conf.angle)
+        writer.addPage(rotated_page)
+        with open(output_file, 'wb') as out:
+            writer.write(out)
+
+
+@transform(
+    input=orient_page,
+    filter=suffix('.ocr.oriented.pdf'),
     output='.page.png',
     output_dir=work_folder,
     extras=[_log, _pdfinfo, _pdfinfo_lock])
@@ -428,17 +563,21 @@ def rasterize_with_ghostscript(
         log,
         pdfinfo,
         pdfinfo_lock):
-
     pageinfo = get_pageinfo(input_file, pdfinfo, pdfinfo_lock)
 
     device = 'png16m'  # 24-bit
     if all(image['comp'] == 1 for image in pageinfo['images']):
         if all(image['bpc'] == 1 for image in pageinfo['images']):
             device = 'pngmono'
-        elif not any(image['color'] == 'color'
-                     for image in pageinfo['images']):
+        elif all(image['bpc'] > 1 and image['color'] == 'index'
+                 for image in pageinfo['images']):
+            device = 'png256'
+        elif all(image['bpc'] > 1 and image['color'] == 'gray'
+                 for image in pageinfo['images']):
             device = 'pnggray'
 
+    log.debug("Rasterize {0} with {1}".format(
+            os.path.basename(input_file), device))
     xres = max(pageinfo['xres'], options.oversample or 0)
     yres = max(pageinfo['yres'], options.oversample or 0)
 
@@ -464,7 +603,8 @@ def preprocess_deskew(
     pageinfo = get_pageinfo(input_file, pdfinfo, pdfinfo_lock)
     dpi = int(pageinfo['xres'])
 
-    unpaper.deskew(input_file, output_file, dpi, log)
+    from . import leptonica
+    leptonica.deskew(input_file, output_file, dpi)
 
 
 @transform(
@@ -502,59 +642,19 @@ def ocr_tesseract_hocr(
         pdfinfo,
         pdfinfo_lock):
 
-    pageinfo = get_pageinfo(input_file, pdfinfo, pdfinfo_lock)
-
-    args_tesseract = [
-        'tesseract',
-        '-l', '+'.join(options.language),
-        input_file,
-        output_file,
-        'hocr'
-    ] + options.tesseract_config
-    p = Popen(args_tesseract, close_fds=True, stdout=PIPE, stderr=PIPE,
-              universal_newlines=True)
-    try:
-        stdout, stderr = p.communicate(timeout=options.tesseract_timeout)
-    except TimeoutExpired:
-        p.kill()
-        stdout, stderr = p.communicate()
-        # Generate a HOCR file with no recognized text if tesseract times out
-        # Temporary workaround to hocrTransform not being able to function if
-        # it does not have a valid hOCR file.
-        with open(output_file, 'w', encoding="utf-8") as f:
-            f.write(tesseract.HOCR_TEMPLATE.format(
-                pageinfo['width_pixels'],
-                pageinfo['height_pixels']))
-    else:
-        if stdout:
-            log.info(stdout)
-        if stderr:
-            log.error(stderr)
-
-        if p.returncode != 0:
-            raise CalledProcessError(p.returncode, args_tesseract)
-
-        if os.path.exists(output_file + '.html'):
-            # Tesseract 3.02 appends suffix ".html" on its own (.hocr.html)
-            shutil.move(output_file + '.html', output_file)
-        elif os.path.exists(output_file + '.hocr'):
-            # Tesseract 3.03 appends suffix ".hocr" on its own (.hocr.hocr)
-            shutil.move(output_file + '.hocr', output_file)
-
-        # Tesseract 3.03 inserts source filename into hocr file without
-        # escaping it, creating invalid XML and breaking the parser.
-        # As a workaround, rewrite the hocr file, replacing the filename
-        # with a space.
-        regex_nested_single_quotes = re.compile(
-            r"""title='image "([^"]*)";""")
-        with fileinput.input(files=(output_file,), inplace=True) as f:
-            for line in f:
-                line = regex_nested_single_quotes.sub(
-                    r"""title='image " ";""", line)
-                print(line, end='')  # fileinput.input redirects stdout
+    tesseract.generate_hocr(
+        input_file=input_file,
+        output_hocr=output_file,
+        language=options.language,
+        tessconfig=options.tesseract_config,
+        timeout=options.tesseract_timeout,
+        pageinfo_getter=partial(get_pageinfo, input_file, pdfinfo,
+                                pdfinfo_lock),
+        pagesegmode=options.tesseract_pagesegmode,
+        log=log
+        )
 
 
-@active_if(options.pdf_renderer == 'hocr')
 @collate(
     input=[rasterize_with_ghostscript, preprocess_deskew, preprocess_clean],
     filter=regex(r".*/(\d{6})(?:\.page|\.pp-deskew|\.pp-clean)\.png"),
@@ -577,31 +677,70 @@ def select_image_for_pdf(
     pageinfo = get_pageinfo(image, pdfinfo, pdfinfo_lock)
     if all(image['enc'] == 'jpeg' for image in pageinfo['images']):
         # If all images were JPEGs originally, produce a JPEG as output
-        Image.open(image).save(output_file, format='JPEG')
+        im = Image.open(image)
+        dpi = im.info.get(
+            'dpi',
+            (int(pageinfo['xres']), int(pageinfo['yres']))
+        )
+        im.save(output_file, format='JPEG', dpi=dpi)
     else:
         re_symlink(image, output_file)
 
 
 @active_if(options.pdf_renderer == 'hocr')
 @collate(
-    input=[select_image_for_pdf, ocr_tesseract_hocr],
-    filter=regex(r".*/(\d{6})(?:\.image|\.hocr)"),
-    output=os.path.join(work_folder, r'\1.rendered.pdf'),
+    input=[select_image_for_pdf, orient_page],
+    filter=regex(r".*/(\d{6})(?:\.image|\.ocr\.oriented\.pdf)"),
+    output=os.path.join(work_folder, r'\1.image-layer.pdf'),
     extras=[_log, _pdfinfo, _pdfinfo_lock])
-def render_hocr_page(
+def select_image_layer(
         infiles,
         output_file,
         log,
         pdfinfo,
         pdfinfo_lock):
-    hocr = next(ii for ii in infiles if ii.endswith('.hocr'))
+
+    page_pdf = next(ii for ii in infiles if ii.endswith('.ocr.oriented.pdf'))
     image = next(ii for ii in infiles if ii.endswith('.image'))
 
-    pageinfo = get_pageinfo(image, pdfinfo, pdfinfo_lock)
+    if lossless_reconstruction:
+        log.debug("{:4d}: page eligible for lossless reconstruction".format(
+            page_number(page_pdf)))
+        re_symlink(page_pdf, output_file)
+    else:
+        pageinfo = get_pageinfo(image, pdfinfo, pdfinfo_lock)
+        dpi = round(max(pageinfo['xres'], pageinfo['yres'],
+                        options.oversample))
+        imgsize = ((img2pdf.ImgSize.dpi, dpi), (img2pdf.ImgSize.dpi, dpi))
+
+        layout_fun = img2pdf.get_layout_fun(None, imgsize, None, None, None)
+
+        with open(image, 'rb') as imfile, \
+                open(output_file, 'wb') as pdf:
+            rawdata = imfile.read()
+            pdf.write(img2pdf.convert(
+                rawdata, producer="img2pdf", with_pdfrw=False,
+                layout_fun=layout_fun))
+
+
+@active_if(options.pdf_renderer == 'hocr')
+@transform(
+    input=ocr_tesseract_hocr,
+    filter=suffix('.hocr'),
+    output='.hocr.pdf',
+    extras=[_log, _pdfinfo, _pdfinfo_lock])
+def render_hocr_page(
+        input_file,
+        output_file,
+        log,
+        pdfinfo,
+        pdfinfo_lock):
+    hocr = input_file
+    pageinfo = get_pageinfo(hocr, pdfinfo, pdfinfo_lock)
     dpi = round(max(pageinfo['xres'], pageinfo['yres'], options.oversample))
 
     hocrtransform = HocrTransform(hocr, dpi)
-    hocrtransform.to_pdf(output_file, imageFileName=image,
+    hocrtransform.to_pdf(output_file, imageFileName=None,
                          showBoundingboxes=False, invisibleText=True)
 
 
@@ -629,10 +768,77 @@ def render_hocr_debug_page(
                          showBoundingboxes=True, invisibleText=False)
 
 
+@active_if(options.pdf_renderer == 'hocr')
+@collate(
+    input=[render_hocr_page, select_image_layer],
+    filter=regex(r".*/(\d{6})(?:\.hocr\.pdf|\.image-layer\.pdf)"),
+    output=os.path.join(work_folder, r'\1.rendered.pdf'),
+    extras=[_log, _pdfinfo, _pdfinfo_lock])
+def add_text_layer(
+        infiles,
+        output_file,
+        log,
+        pdfinfo,
+        pdfinfo_lock):
+    text = next(ii for ii in infiles if ii.endswith('.hocr.pdf'))
+    image = next(ii for ii in infiles if ii.endswith('.image-layer.pdf'))
+
+    pdf_text = pypdf.PdfFileReader(open(text, "rb"))
+    pdf_image = pypdf.PdfFileReader(open(image, "rb"))
+
+    page_text = pdf_text.getPage(0)
+
+    # The text page always will be oriented up
+    # but if lossless_reconstruction, pdf_image may have a rotation applied
+    # we can't just merge the pages, because a page can only have one /Rotate
+    # tag, so the differential rotation must be corrected.
+    # Also, pdf_image may not have its mediabox nailed to (0, 0)
+    page_image = pdf_image.getPage(0)
+    rotation = page_image.get('/Rotate', 0)
+
+    # /Rotate is a clockwise rotation: 90 means page facing "east"
+    # The negative of this value is the angle that eliminates that rotation
+    rotation = -rotation % 360
+
+    x1 = page_image.mediaBox.getLowerLeft_x()
+    x2 = page_image.mediaBox.getUpperRight_x()
+    y1 = page_image.mediaBox.getLowerLeft_y()
+    y2 = page_image.mediaBox.getUpperRight_y()
+
+    # Rotation occurs about the page's (0, 0). Most pages will have the media
+    # box at (0, 0) will all content in the first quadrant but some cropped
+    # files may have an offset mediabox. We translate the page so that its
+    # bottom left corner after rotation is pinned to (0, 0) with the image
+    # in the first quadrant.
+    if rotation == 0:
+        tx, ty = -x1, -y1
+    elif rotation == 90:
+        tx, ty = y2, -x1
+    elif rotation == 180:
+        tx, ty = x2, y2
+    elif rotation == 270:
+        tx, ty = -y1, x2
+    else:
+        pass
+
+    if rotation != 0:
+        log.info("{0:4d}: rotating image layer {1} degrees".format(
+            page_number(image), rotation, tx, ty))
+
+    page_text.mergeRotatedScaledTranslatedPage(
+        page_image, rotation, 1.0, tx, ty, expand=False)
+
+    pdf_output = pypdf.PdfFileWriter()
+    pdf_output.addPage(page_text)
+
+    with open(output_file, "wb") as out:
+        pdf_output.write(out)
+
+
 @active_if(options.pdf_renderer == 'tesseract')
 @collate(
-    input=[preprocess_clean, split_pages],
-    filter=regex(r".*/(\d{6})(?:\.pp-clean\.png|\.page\.pdf)"),
+    input=[select_image_for_pdf, orient_page],
+    filter=regex(r".*/(\d{6})(?:\.image|\.ocr\.oriented\.pdf)"),
     output=os.path.join(work_folder, r'\1.rendered.pdf'),
     extras=[_log, _pdfinfo, _pdfinfo_lock])
 def tesseract_ocr_and_render_pdf(
@@ -642,40 +848,28 @@ def tesseract_ocr_and_render_pdf(
         pdfinfo,
         pdfinfo_lock):
 
-    input_image = next((ii for ii in input_files if ii.endswith('.png')), '')
+    input_image = next((ii for ii in input_files if ii.endswith('.image')), '')
     input_pdf = next((ii for ii in input_files if ii.endswith('.pdf')))
     if not input_image:
         # Skipping this page
         re_symlink(input_pdf, output_file)
         return
 
-    args_tesseract = [
-        'tesseract',
-        '-l', '+'.join(options.language),
-        input_image,
-        os.path.splitext(output_file)[0],  # Tesseract appends suffix
-        'pdf'
-    ] + options.tesseract_config
-    p = Popen(args_tesseract, close_fds=True, stdout=PIPE, stderr=PIPE,
-              universal_newlines=True)
-
-    try:
-        stdout, stderr = p.communicate(timeout=options.tesseract_timeout)
-        if stdout:
-            log.info(stdout)
-        if stderr:
-            log.error(stderr)
-    except TimeoutExpired:
-        p.kill()
-        log.info("Tesseract - page timed out")
-        re_symlink(input_pdf, output_file)
+    tesseract.generate_pdf(
+        input_image=input_image,
+        skip_pdf=input_pdf,
+        output_pdf=output_file,
+        language=options.language,
+        tessconfig=options.tesseract_config,
+        timeout=options.tesseract_timeout,
+        pagesegmode=options.tesseract_pagesegmode,
+        log=log)
 
 
 @transform(
     input=repair_pdf,
-    filter=suffix('.repaired.pdf'),
-    output='.pdfa_def.ps',
-    output_dir=work_folder,
+    filter=formatter(r'\.repaired\.pdf'),
+    output=os.path.join(work_folder, 'pdfa_def.ps'),
     extras=[_log])
 def generate_postscript_stub(
         input_file,
@@ -685,11 +879,13 @@ def generate_postscript_stub(
     pdf = pypdf.PdfFileReader(input_file)
 
     def from_document_info(key):
-        # pdf.documentInfo.get() DOES NOT work as expected
+        # pdf.documentInfo.get() DOES NOT behave as expected for a dict-like
+        # object, so call with precautions.  TypeError may occur if the PDF
+        # is missing the optional document info section.
         try:
             s = pdf.documentInfo[key]
             return str(s)
-        except KeyError:
+        except (KeyError, TypeError):
             return ''
 
     pdfmark = {
@@ -707,12 +903,17 @@ def generate_postscript_stub(
     if options.subject:
         pdfmark['subject'] = options.subject
 
+    pdfmark['creator'] = '{0} {1} / Tesseract OCR{2} {3}'.format(
+            parser.prog, VERSION,
+            '+PDF' if options.pdf_renderer == 'tesseract' else '',
+            tesseract.version())
+
     generate_pdfa_def(output_file, pdfmark)
 
 
 @transform(
-    input=split_pages,
-    filter=suffix('.skip.page.pdf'),
+    input=orient_page,
+    filter=suffix('.skip.oriented.pdf'),
     output='.done.pdf',
     output_dir=work_folder,
     extras=[_log])
@@ -720,11 +921,15 @@ def skip_page(
         input_file,
         output_file,
         log):
+    # The purpose of this step is its filter to forward only the skipped
+    # files (.skip.oriented.pdf) while disregarding the processed ones
+    # (.ocr.oriented.pdf).  Alternative would be for merge_pages to filter
+    # pages itself if it gets multiple copies of a page.
     re_symlink(input_file, output_file, log)
 
 
 @merge(
-    input=[render_hocr_page, render_hocr_debug_page, skip_page,
+    input=[add_text_layer, render_hocr_debug_page, skip_page,
            tesseract_ocr_and_render_pdf, generate_postscript_stub],
     output=os.path.join(work_folder, 'merged.pdf'),
     extras=[_log, _pdfinfo, _pdfinfo_lock])
@@ -749,7 +954,7 @@ def merge_pages(
         return key
 
     pdf_pages = sorted(input_files, key=input_file_order)
-    log.info(pdf_pages)
+    log.debug("Final pages: " + "\n".join(pdf_pages))
     ghostscript.generate_pdfa(pdf_pages, output_file, options.jobs or 1)
 
 
@@ -758,70 +963,19 @@ def merge_pages(
     filter=formatter(),
     output=options.output_file,
     extras=[_log, _pdfinfo, _pdfinfo_lock])
-def validate_pdfa(
+def copy_final(
         input_file,
         output_file,
         log,
         pdfinfo,
         pdfinfo_lock):
-
-    args_jhove = [
-        'java',
-        '-jar', JHOVE_JAR,
-        '-c', JHOVE_CFG,
-        '-m', 'PDF-hul',
-        input_file
-    ]
-    p_jhove = Popen(args_jhove, close_fds=True, universal_newlines=True,
-                    stdout=PIPE, stderr=DEVNULL)
-    stdout, _ = p_jhove.communicate()
-
-    log.debug(stdout)
-    if p_jhove.returncode != 0:
-        log.error(stdout)
-        raise RuntimeError(
-            "Unexpected error while checking compliance to PDF/A file.")
-
-    pdf_is_valid = True
-    if re.search(r'ErrorMessage', stdout,
-                 re.IGNORECASE | re.MULTILINE):
-        pdf_is_valid = False
-    if re.search(r'^\s+Status.*not valid', stdout,
-                 re.IGNORECASE | re.MULTILINE):
-        pdf_is_valid = False
-    if re.search(r'^\s+Status.*Not well-formed', stdout,
-                 re.IGNORECASE | re.MULTILINE):
-        pdf_is_valid = False
-
-    pdf_is_pdfa = False
-    if re.search(r'^\s+Profile:.*PDF/A-1', stdout,
-                 re.IGNORECASE | re.MULTILINE):
-        pdf_is_pdfa = True
-
-    if not pdf_is_valid:
-        log.warning('Output file: The generated PDF/A file is INVALID')
-    elif pdf_is_valid and not pdf_is_pdfa:
-        log.warning('Output file: Generated file is a VALID PDF but not PDF/A')
-    elif pdf_is_valid and pdf_is_pdfa:
-        log.info('Output file: The generated PDF/A file is VALID')
     shutil.copy(input_file, output_file)
 
 
-# @active_if(ocr_required and options.exact_image)
-# @merge([render_hocr_blank_page, extract_single_page],
-#        os.path.join(work_folder, "%04i.merged.pdf") % pageno)
-# def merge_hocr_with_original_page(infiles, output_file):
-#     with open(infiles[0], 'rb') as hocr_input, \
-#             open(infiles[1], 'rb') as page_input, \
-#             open(output_file, 'wb') as output:
-#         hocr_reader = pypdf.PdfFileReader(hocr_input)
-#         page_reader = pypdf.PdfFileReader(page_input)
-#         writer = pypdf.PdfFileWriter()
-
-#         the_page = hocr_reader.getPage(0)
-#         the_page.mergePage(page_reader.getPage(0))
-#         writer.addPage(the_page)
-#         writer.write(output)
+def validate_pdfa(
+        input_file,
+        log):
+    return qpdf.check(input_file, log)
 
 
 def available_cpu_count():
@@ -842,11 +996,51 @@ def available_cpu_count():
     return 1
 
 
+def cleanup_ruffus_error_message(msg):
+    msg = re.sub(r'\s+', r' ', msg, re.MULTILINE)
+    msg = re.sub(r"\((.+?)\)", r'\1', msg)
+    msg = msg.strip()
+    return msg
+
+
 def run_pipeline():
-    if not options.jobs or options.jobs == 1:
+    if not options.jobs:
         options.jobs = available_cpu_count()
-    cmdline.run(options)
+    try:
+        options.history_file = os.path.join(work_folder, 'ruffus_history.sqlite')
+        cmdline.run(options)
+    except ruffus_exceptions.RethrownJobError as e:
+        if options.verbose:
+            print(e)
+
+        # Yuck. Hunt through the ruffus exception to find out what the
+        # return code is supposed to be.
+        for exc in e.args:
+            task_name, job_name, exc_name, exc_value, exc_stack = exc
+            if exc_name == 'builtins.SystemExit':
+                match = re.search(r"\.(.+?)\)", exc_value)
+                exit_code_name = match.groups()[0]
+                exit_code = getattr(ExitCode, exit_code_name, 'other_error')
+                return exit_code
+            elif exc_name == 'ruffus.ruffus_exceptions.MissingInputFileError':
+                print(cleanup_ruffus_error_message(exc_value))
+                return ExitCode.input_file
+            elif exc_name == 'builtins.TypeError':
+                # Even though repair_pdf will fail, ruffus will still try
+                # to call split_pages with no input files, likely due to a bug
+                if task_name == 'split_pages':
+                    print("Input file '{0}' is not a valid PDF".format(
+                        options.input_file))
+                    return ExitCode.input_file
+
+        return ExitCode.other_error
+
+    if not validate_pdfa(options.output_file, _log):
+        _log.warning('Output file: The generated PDF/A file is INVALID')
+        return ExitCode.invalid_output_pdfa
+
+    return ExitCode.ok
 
 
 if __name__ == '__main__':
-    run_pipeline()
+    sys.exit(run_pipeline())

ocrmypdf/pageinfo.py

@@ -34,6 +34,7 @@ FRIENDLY_COMP = {
     'rgb': 3,
     'cmyk': 4,
     'lab': 3,
+    'index': 1
 }
 
 
@@ -41,8 +42,14 @@ def _page_has_inline_images(page):
     # PDF always uses \r\n for separator regardless of platform
     # Really basic heuristic that might trigger the odd false positive
     # This is only finds the first image and is not quite spec compliant
-    contents = page.getContents()
-    data = contents.getData()
+    try:
+        contents = page.getContents()
+        data = contents.getData()
+    except AttributeError:
+        # If we can't access the contents or data (empty page?) then there
+        # are no inline images
+        return False
+
     begin_image, image_data, end_image = False, False, False
     for data in re.split(b'\s+', data):
         if data == b'BI':
@@ -53,6 +60,7 @@ def _page_has_inline_images(page):
             end_image = True
         if all((begin_image, image_data, end_image)):
             return True
+    return False
 
 
 def _find_page_images(page, pageinfo):
@@ -96,19 +104,40 @@ def _find_page_images(page, pageinfo):
         yield image
 
 
-def _pdf_get_pageinfo(infile, page: int):
+def _page_has_text(pdf, page):
+    # Simple test
+    text = page.extractText()
+    if text.strip() != '':
+        return True
+
+    # More nuanced test to deal with quirks of Tesseract PDF generation
+    # Check if there's a Glyphless font
+    try:
+        font = page['/Resources']['/Font']
+    except KeyError:
+        pass
+    else:
+        font_objects = list(font.keys())
+        for font_object in font_objects:
+            basefont = font[font_object]['/BaseFont']
+            if basefont.endswith('GlyphLessFont'):
+                return True
+
+    return False
+
+
+def _pdf_get_pageinfo(infile, pageno: int):
     pageinfo = {}
-    pageinfo['pageno'] = page
+    pageinfo['pageno'] = pageno
     pageinfo['images'] = []
 
     pdf = pypdf.PdfFileReader(infile)
-    page = pdf.pages[page - 1]
+    page = pdf.pages[pageno]
 
-    text = page.extractText()
-    pageinfo['has_text'] = (text.strip() != '')
+    pageinfo['has_text'] = _page_has_text(pdf, page)
 
-    width_pt = page['/MediaBox'][2] - page['/MediaBox'][0]
-    height_pt = page['/MediaBox'][3] - page['/MediaBox'][1]
+    width_pt = page.mediaBox.getWidth()
+    height_pt = page.mediaBox.getHeight()
     pageinfo['width_inches'] = width_pt / Decimal(72.0)
     pageinfo['height_inches'] = height_pt / Decimal(72.0)
 

ocrmypdf/pdfa.py

@@ -8,6 +8,7 @@ from string import Template
 from subprocess import Popen, PIPE
 import os
 import codecs
+from . import get_program
 
 
 # This is a template written in PostScript which is needed to create PDF/A
@@ -27,6 +28,7 @@ def
   /Author <$author>
   /Subject <$subject>
   /Keywords <$keywords>
+  /Creator <$creator>
   /DOCINFO pdfmark
 
 % Define an ICC profile :
@@ -86,6 +88,7 @@ def _get_pdfa_def(icc_profile, icc_identifier, pdfmark):
                           title=pdfmark_utf16.get('title', ''),
                           author=pdfmark_utf16.get('author', ''),
                           subject=pdfmark_utf16.get('subject', ''),
+                          creator=pdfmark_utf16.get('creator', ''),
                           keywords=pdfmark_utf16.get('keywords', ''))
     return result
 
@@ -93,7 +96,8 @@ def _get_pdfa_def(icc_profile, icc_identifier, pdfmark):
 def _get_postscript_icc_path():
     "Parse Ghostscript's help message to find where iccprofiles are stored"
 
-    p_gs = Popen(['gs', '--help'], close_fds=True, universal_newlines=True,
+    p_gs = Popen([get_program('gs'), '--help'], close_fds=True,
+                 universal_newlines=True,
                  stdout=PIPE, stderr=PIPE)
     out, _ = p_gs.communicate()
     lines = out.splitlines()
@@ -115,6 +119,8 @@ def _get_postscript_icc_path():
         if os.path.exists(path):
             return path
 
+    raise FileNotFoundError("Could not find Ghostscript's iccprofiles")
+
 
 def generate_pdfa_def(target_filename, pdfmark, icc='sRGB'):
     if icc == 'sRGB':

ocrmypdf/qpdf.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+# © 2015 James R. Barlow: github.com/jbarlow83
+
+from subprocess import CalledProcessError, check_output, STDOUT, check_call
+import sys
+import os
+
+from . import ExitCode, get_program
+
+
+def check(input_file, log):
+    args_qpdf = [
+        get_program('qpdf'),
+        '--check',
+        input_file
+    ]
+
+    try:
+        check_output(args_qpdf, stderr=STDOUT, universal_newlines=True)
+    except CalledProcessError as e:
+        if e.returncode == 2:
+            log.error("{0}: not a valid PDF, and could not repair it.".format(
+                    input_file))
+            log.error("Details:")
+            log.error(e.output)
+        elif e.returncode == 3:
+            log.info("qpdf --check returned warnings:")
+            log.info(e.output)
+        else:
+            log.warning(e.output)
+        return False
+    return True
+
+
+def repair(input_file, output_file, log):
+    args_qpdf = [
+        get_program('qpdf'), input_file, output_file
+    ]
+    try:
+        check_output(args_qpdf, stderr=STDOUT, universal_newlines=True)
+    except CalledProcessError as e:
+        if e.returncode == 3 and e.output.find("operation succeeded"):
+            log.debug('qpdf found and fixed errors: ' + e.output)
+            log.debug(e.output)
+            return
+
+        if e.returncode == 2 and e.output.find("invalid password"):
+            log.error("{0}: this PDF is password-protected - password must "
+                      "be removed for OCR".format(input_file))
+            sys.exit(ExitCode.input_file)
+        elif e.returncode == 2:
+            log.error("{0}: not a valid PDF, and could not repair it.".format(
+                      input_file))
+            log.error("Details: " + e.output)
+            sys.exit(ExitCode.input_file)
+        else:
+            log.error("{0}: unknown error".format(
+                      input_file))
+            log.error(e.output)
+            sys.exit(ExitCode.unknown)
+
+
+def get_npages(input_file, log):
+    try:
+        pages = check_output(
+            [get_program('qpdf'), '--show-npages', input_file],
+             universal_newlines=True, close_fds=True)
+    except CalledProcessError as e:
+        if e.returncode == 2 and e.output.find('No such file'):
+            log.error(e.output)
+            sys.exit(ExitCode.input_file)
+    return int(pages)
+
+
+def split_pages(input_file, work_folder, npages):
+    """Split multipage PDF into individual pages.
+
+    Incredibly enough, this multiple process approach is about 70 times
+    faster than using Ghostscript.
+    """
+    for n in range(int(npages)):
+        args_qpdf = [
+            get_program('qpdf'), input_file,
+            '--pages', input_file, '{0}'.format(n + 1), '--',
+            os.path.join(work_folder, '{0:06d}.page.pdf'.format(n + 1))
+        ]
+        check_call(args_qpdf)

ocrmypdf/tesseract.py

@@ -1,42 +1,25 @@
 #!/usr/bin/env python3
 # © 2015 James R. Barlow: github.com/jbarlow83
 
-from subprocess import STDOUT, CalledProcessError, check_output
 import sys
 import os
 import re
+import shutil
 from functools import lru_cache
+from . import ExitCode, get_program, page_number
+from collections import namedtuple
+
+from subprocess import Popen, PIPE, CalledProcessError, \
+    TimeoutExpired, check_output, STDOUT
+try:
+    from subprocess import DEVNULL
+except ImportError:
+    DEVNULL = open(os.devnull, 'wb')
 
 
-@lru_cache(maxsize=1)
-def version():
-    args_tess = [
-        'tesseract',
-        '--version'
-    ]
-    try:
-        versions = check_output(
-                args_tess, close_fds=True, universal_newlines=True,
-                stderr=STDOUT)
-    except CalledProcessError:
-        print("Could not find Tesseract executable on system PATH.")
-        sys.exit(1)
-
-    tesseract_version = re.match(r'tesseract\s(.+)', versions).group(1)
-    return tesseract_version
-
-
-@lru_cache(maxsize=1)
-def languages():
-    args_tess = [
-        'tesseract',
-        '--list-langs'
-    ]
-    langs = check_output(
-            args_tess, close_fds=True, universal_newlines=True,
-            stderr=STDOUT)
-    return set(lang.strip() for lang in langs.splitlines()[1:])
-
+OrientationConfidence = namedtuple(
+    'OrientationConfidence',
+    ('angle', 'confidence'))
 
 HOCR_TEMPLATE = '''<?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
@@ -59,3 +42,185 @@ HOCR_TEMPLATE = '''<?xml version="1.0" encoding="UTF-8"?>
   </div>
  </body>
 </html>'''
+
+
+@lru_cache(maxsize=1)
+def version():
+    args_tess = [
+        get_program('tesseract'),
+        '--version'
+    ]
+    try:
+        versions = check_output(
+                args_tess, close_fds=True, universal_newlines=True,
+                stderr=STDOUT)
+    except CalledProcessError:
+        print("Could not find Tesseract executable on system PATH.")
+        sys.exit(ExitCode.missing_dependency)
+
+    tesseract_version = re.match(r'tesseract\s(.+)', versions).group(1)
+    return tesseract_version
+
+
+@lru_cache(maxsize=1)
+def languages():
+    args_tess = [
+        get_program('tesseract'),
+        '--list-langs'
+    ]
+    try:
+        langs = check_output(
+                args_tess, close_fds=True, universal_newlines=True,
+                stderr=STDOUT)
+    except CalledProcessError as e:
+        print("Tesseract failed to report available languages.")
+        print("Output from Tesseract:")
+        print("-" * 40)
+        print(e.output)
+        sys.exit(ExitCode.missing_dependency)
+    return set(lang.strip() for lang in langs.splitlines()[1:])
+
+
+def get_orientation(input_file, language: list, timeout: float, log):
+    args_tesseract = [
+        get_program('tesseract'),
+        '-l', '+'.join(language),
+        '-psm', '0',
+        input_file,
+        'stdout'
+    ]
+
+    p = Popen(args_tesseract, close_fds=True, stdout=PIPE, stderr=STDOUT,
+              universal_newlines=True)
+    try:
+        stdout, _ = p.communicate(timeout=timeout)
+    except TimeoutExpired:
+        p.kill()
+        stdout, _ = p.communicate()
+        return OrientationConfidence(angle=0, confidence=0.0)
+    else:
+        osd = {}
+        for line in stdout.splitlines():
+            line = line.strip()
+            parts = line.split(':', maxsplit=2)
+            if len(parts) == 2:
+                osd[parts[0].strip()] = parts[1].strip()
+
+        oc = OrientationConfidence(
+            angle=int(osd['Orientation in degrees']),
+            confidence=float(osd['Orientation confidence']))
+        return oc
+
+
+def tesseract_log_output(log, stdout, input_file):
+    lines = stdout.splitlines()
+    prefix = "{0:4d}: [tesseract] ".format(page_number(input_file))
+    for line in lines:
+        if line.startswith("Tesseract Open Source"):
+            continue
+        elif line.startswith("Warning in pixReadMem"):
+            continue
+        elif 'diacritics' in line:
+            log.warning(prefix + "lots of diacritics - possibly poor OCR")
+        elif line.startswith('OSD: Weak margin'):
+            log.warning(prefix + "unsure about page orientation")
+        else:
+            log.info(prefix + line.strip())
+
+
+def generate_hocr(input_file, output_hocr, language: list, tessconfig: list,
+                  timeout: float, pageinfo_getter, pagesegmode: int, log):
+
+    badxml = os.path.splitext(output_hocr)[0] + '.badxml'
+
+    args_tesseract = [
+        get_program('tesseract'),
+        '-l', '+'.join(language)
+    ]
+
+    if pagesegmode is not None:
+        args_tesseract.extend(['-psm', str(pagesegmode)])
+
+    args_tesseract.extend([
+        input_file,
+        badxml,
+        'hocr'
+    ] + tessconfig)
+    p = Popen(args_tesseract, close_fds=True, stdout=PIPE, stderr=STDOUT,
+              universal_newlines=True)
+    try:
+        stdout, _ = p.communicate(timeout=timeout)
+    except TimeoutExpired:
+        p.kill()
+        stdout, _ = p.communicate()
+        # Generate a HOCR file with no recognized text if tesseract times out
+        # Temporary workaround to hocrTransform not being able to function if
+        # it does not have a valid hOCR file.
+        with open(output_hocr, 'w', encoding="utf-8") as f:
+            pageinfo = pageinfo_getter()
+            f.write(HOCR_TEMPLATE.format(
+                pageinfo['width_pixels'],
+                pageinfo['height_pixels']))
+    else:
+        tesseract_log_output(log, stdout, input_file)
+        if p.returncode != 0:
+            raise CalledProcessError(p.returncode, args_tesseract)
+
+        if os.path.exists(badxml + '.html'):
+            # Tesseract 3.02 appends suffix ".html" on its own (.badxml.html)
+            shutil.move(badxml + '.html', badxml)
+        elif os.path.exists(badxml + '.hocr'):
+            # Tesseract 3.03 appends suffix ".hocr" on its own (.badxml.hocr)
+            shutil.move(badxml + '.hocr', badxml)
+
+        # Tesseract 3.03 inserts source filename into hocr file without
+        # escaping it, creating invalid XML and breaking the parser.
+        # As a workaround, rewrite the hocr file, replacing the filename
+        # with a space.  Don't know if Tesseract 3.02 does the same.
+
+        regex_nested_single_quotes = re.compile(
+            r"""title='image "([^"]*)";""")
+        with open(badxml, mode='r', encoding='utf-8') as f_in, \
+                open(output_hocr, mode='w', encoding='utf-8') as f_out:
+            for line in f_in:
+                line = regex_nested_single_quotes.sub(
+                    r"""title='image " ";""", line)
+                f_out.write(line)
+
+
+def generate_pdf(input_image, skip_pdf, output_pdf, language: list,
+                 tessconfig: list, timeout: float, pagesegmode: int, log):
+    '''Use Tesseract to render a PDF.
+
+    input_image -- image to analyze
+    skip_pdf -- if we time out, use this file as output
+    language -- list of languages to consider
+    tessconfig -- tesseract configuration
+    timeout -- timeout (seconds)
+    log -- logger object
+    '''
+
+    args_tesseract = [
+        get_program('tesseract'),
+        '-l', '+'.join(language)
+    ]
+
+    if pagesegmode is not None:
+        args_tesseract.extend(['-psm', str(pagesegmode)])
+
+    args_tesseract.extend([
+        input_image,
+        os.path.splitext(output_pdf)[0],  # Tesseract appends suffix
+        'pdf'
+    ] + tessconfig)
+    p = Popen(args_tesseract, close_fds=True, stdout=PIPE, stderr=STDOUT,
+              universal_newlines=True)
+
+    try:
+        stdout, _ = p.communicate()
+    except TimeoutExpired:
+        p.kill()
+        log.info("Tesseract - page timed out")
+        shutil.copy(skip_pdf, output_pdf)
+    else:
+        tesseract_log_output(log, stdout, input_image)

ocrmypdf/unpaper.py

@@ -8,12 +8,13 @@ from tempfile import NamedTemporaryFile
 import sys
 import os
 from functools import lru_cache
+from . import ExitCode, get_program
 
 
 @lru_cache(maxsize=1)
 def version():
     args_unpaper = [
-        'unpaper',
+        get_program('unpaper'),
         '--version'
     ]
     p_unpaper = Popen(args_unpaper, close_fds=True, universal_newlines=True,
@@ -32,16 +33,33 @@ except ImportError:
 
 def run(input_file, output_file, dpi, log, mode_args):
     args_unpaper = [
-        'unpaper',
+        get_program('unpaper'),
         '-v',
         '--dpi', str(dpi)
     ] + mode_args
 
     SUFFIXES = {'1': '.pbm', 'L': '.pgm', 'RGB': '.ppm'}
-    suffix = ''
 
     im = Image.open(input_file)
-    suffix = SUFFIXES[im.mode]
+    if im.mode not in SUFFIXES.keys():
+        log.info("Converting image to other colorspace")
+        try:
+            if im.mode == 'P' and len(im.getcolors()) == 2:
+                im = im.convert(mode='1')
+            else:
+                im = im.convert(mode='RGB')
+        except IOError:
+            log.error(
+                    "Could not convert image with type " + im.mode)
+            sys.exit(ExitCode.missing_dependency)
+
+    try:
+        suffix = SUFFIXES[im.mode]
+    except KeyError:
+        log.error(
+                "Failed to convert image to a supported format.")
+        sys.exit(ExitCode.missing_dependency)
+
     with NamedTemporaryFile(suffix=suffix) as input_pnm, \
             NamedTemporaryFile(suffix=suffix, mode="r+b") as output_pnm:
         im.save(input_pnm, format='PPM')

pipeline.svg

@@ -4,232 +4,294 @@
 <!-- Generated by graphviz version 2.38.0 (20140413.2041)
  -->
 <!-- Title: Pipeline: Pages: 1 -->
-<svg width="728pt" height="651pt"
- viewBox="0.00 0.00 728.00 650.53" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
-<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 646.53)">
+<svg width="1452pt" height="824pt"
+ viewBox="0.00 0.00 1452.00 824.08" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 820.083)">
 <title>Pipeline:</title>
-<polygon fill="white" stroke="none" points="-4,4 -4,-646.53 724,-646.53 724,4 -4,4"/>
+<polygon fill="white" stroke="none" points="-4,4 -4,-820.083 1448,-820.083 1448,4 -4,4"/>
 <g id="clust1" class="cluster"><title>clustertasks</title>
-<polygon fill="none" stroke="black" points="8,-8 8,-634.53 712,-634.53 712,-8 8,-8"/>
-<text text-anchor="middle" x="360" y="-606.53" font-family="Times,serif" font-size="30.00" fill="#ff3232">Pipeline:</text>
+<polygon fill="none" stroke="black" points="8,-8 8,-808.083 1436,-808.083 1436,-8 8,-8"/>
+<text text-anchor="middle" x="722" y="-780.083" font-family="Times,serif" font-size="30.00" fill="#ff3232">Pipeline:</text>
 </g>
 <!-- t0 -->
 <g id="node1" class="node"><title>t0</title>
-<polygon fill="#efa03b" stroke="#006000" points="481.791,-588.53 386.209,-588.53 382.209,-584.53 382.209,-552.53 477.791,-552.53 481.791,-556.53 481.791,-588.53"/>
-<polyline fill="none" stroke="#006000" points="477.791,-584.53 382.209,-584.53 "/>
-<polyline fill="none" stroke="#006000" points="477.791,-584.53 477.791,-552.53 "/>
-<polyline fill="none" stroke="#006000" points="477.791,-584.53 481.791,-588.53 "/>
-<text text-anchor="middle" x="432" y="-564.53" font-family="Times,serif" font-size="20.00" fill="#006000">repair_pdf</text>
+<polygon fill="#efa03b" stroke="black" points="1256.54,-762.083 1033.46,-762.083 1029.46,-758.083 1029.46,-726.083 1252.54,-726.083 1256.54,-730.083 1256.54,-762.083"/>
+<polyline fill="none" stroke="black" points="1252.54,-758.083 1029.46,-758.083 "/>
+<polyline fill="none" stroke="black" points="1252.54,-758.083 1252.54,-726.083 "/>
+<polyline fill="none" stroke="black" points="1252.54,-758.083 1256.54,-762.083 "/>
+<text text-anchor="middle" x="1143" y="-738.083" font-family="Times,serif" font-size="20.00">repair_pdf</text>
 </g>
 <!-- t1 -->
 <g id="node2" class="node"><title>t1</title>
-<polygon fill="#efa03b" stroke="black" points="466.782,-509.564 374,-526.497 281.218,-509.564 281.304,-482.165 466.696,-482.165 466.782,-509.564"/>
-<polygon fill="none" stroke="black" points="470.799,-512.902 374,-530.569 277.201,-512.902 277.311,-478.159 470.689,-478.159 470.799,-512.902"/>
-<text text-anchor="middle" x="374" y="-495.991" font-family="Times,serif" font-size="20.00">split_pages</text>
+<polygon fill="#efa03b" stroke="black" points="1234.11,-683.155 1030,-700.057 825.888,-683.155 826.078,-655.806 1233.92,-655.806 1234.11,-683.155"/>
+<polygon fill="none" stroke="black" points="1238.13,-686.834 1030,-704.069 821.866,-686.834 822.11,-651.808 1237.89,-651.808 1238.13,-686.834"/>
+<text text-anchor="middle" x="1030" y="-669.596" font-family="Times,serif" font-size="20.00">split_pages</text>
 </g>
 <!-- t0&#45;&gt;t1 -->
 <g id="edge1" class="edge"><title>t0&#45;&gt;t1</title>
-<path fill="none" stroke="gray" d="M417.064,-552.394C412.296,-546.925 406.86,-540.689 401.493,-534.532"/>
-<polygon fill="gray" stroke="gray" points="403.996,-532.077 394.787,-526.838 398.719,-536.676 403.996,-532.077"/>
+<path fill="none" stroke="#0044a0" d="M1113.9,-725.961C1103.58,-719.89 1091.66,-712.873 1080.09,-706.069"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1081.75,-702.982 1071.35,-700.927 1078.2,-709.015 1081.75,-702.982"/>
 </g>
-<!-- t10 -->
-<g id="node12" class="node"><title>t10</title>
-<polygon fill="#efa03b" stroke="#006000" points="704.338,-451.452 493.662,-451.452 489.662,-447.452 489.662,-415.452 700.338,-415.452 704.338,-419.452 704.338,-451.452"/>
-<polyline fill="none" stroke="#006000" points="700.338,-447.452 489.662,-447.452 "/>
-<polyline fill="none" stroke="#006000" points="700.338,-447.452 700.338,-415.452 "/>
-<polyline fill="none" stroke="#006000" points="700.338,-447.452 704.338,-451.452 "/>
-<text text-anchor="middle" x="597" y="-427.452" font-family="Times,serif" font-size="20.00" fill="#006000">generate_postscript_stub</text>
+<!-- t14 -->
+<g id="node16" class="node"><title>t14</title>
+<polygon fill="#efa03b" stroke="black" points="1428.08,-625.109 1089.92,-625.109 1085.92,-621.109 1085.92,-589.109 1424.08,-589.109 1428.08,-593.109 1428.08,-625.109"/>
+<polyline fill="none" stroke="black" points="1424.08,-621.109 1085.92,-621.109 "/>
+<polyline fill="none" stroke="black" points="1424.08,-621.109 1424.08,-589.109 "/>
+<polyline fill="none" stroke="black" points="1424.08,-621.109 1428.08,-625.109 "/>
+<text text-anchor="middle" x="1257" y="-601.109" font-family="Times,serif" font-size="20.00">generate_postscript_stub</text>
 </g>
-<!-- t0&#45;&gt;t10 -->
-<g id="edge16" class="edge"><title>t0&#45;&gt;t10</title>
-<path fill="none" stroke="gray" d="M453.048,-552.468C461.441,-545.654 471.185,-537.73 480,-530.53 510.227,-505.84 544.753,-477.467 568.421,-457.99"/>
-<polygon fill="gray" stroke="gray" points="570.818,-460.55 576.315,-451.493 566.369,-455.146 570.818,-460.55"/>
+<!-- t0&#45;&gt;t14 -->
+<g id="edge22" class="edge"><title>t0&#45;&gt;t14</title>
+<path fill="none" stroke="#0044a0" d="M1219.32,-726.004C1229.98,-720.592 1239.75,-713.466 1247,-704.083 1261.92,-684.78 1263.1,-656.385 1261.35,-635.486"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1264.8,-634.881 1260.22,-625.328 1257.84,-635.653 1264.8,-634.881"/>
 </g>
 <!-- t2 -->
 <g id="node3" class="node"><title>t2</title>
-<polygon fill="#efa03b" stroke="black" points="451.555,-451.452 228.445,-451.452 224.445,-447.452 224.445,-415.452 447.555,-415.452 451.555,-419.452 451.555,-451.452"/>
-<polyline fill="none" stroke="black" points="447.555,-447.452 224.445,-447.452 "/>
-<polyline fill="none" stroke="black" points="447.555,-447.452 447.555,-415.452 "/>
-<polyline fill="none" stroke="black" points="447.555,-447.452 451.555,-451.452 "/>
-<text text-anchor="middle" x="338" y="-427.452" font-family="Times,serif" font-size="20.00">rasterize_with_ghostscript</text>
+<polygon fill="#efa03b" stroke="black" points="1029.88,-625.109 748.117,-625.109 744.117,-621.109 744.117,-589.109 1025.88,-589.109 1029.88,-593.109 1029.88,-625.109"/>
+<polyline fill="none" stroke="black" points="1025.88,-621.109 744.117,-621.109 "/>
+<polyline fill="none" stroke="black" points="1025.88,-621.109 1025.88,-589.109 "/>
+<polyline fill="none" stroke="black" points="1025.88,-621.109 1029.88,-625.109 "/>
+<text text-anchor="middle" x="887" y="-601.109" font-family="Times,serif" font-size="20.00">rasterize_preview</text>
 </g>
 <!-- t1&#45;&gt;t2 -->
 <g id="edge2" class="edge"><title>t1&#45;&gt;t2</title>
-<path fill="none" stroke="#0044a0" d="M361.611,-478.092C358.514,-472.369 355.171,-466.19 352.003,-460.333"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="355.064,-458.636 347.227,-451.506 348.907,-461.967 355.064,-458.636"/>
-</g>
-<!-- t11 -->
-<g id="node10" class="node"><title>t11</title>
-<polygon fill="#efa03b" stroke="black" points="644.594,-393.452 551.406,-393.452 547.406,-389.452 547.406,-357.452 640.594,-357.452 644.594,-361.452 644.594,-393.452"/>
-<polyline fill="none" stroke="black" points="640.594,-389.452 547.406,-389.452 "/>
-<polyline fill="none" stroke="black" points="640.594,-389.452 640.594,-357.452 "/>
-<polyline fill="none" stroke="black" points="640.594,-389.452 644.594,-393.452 "/>
-<text text-anchor="middle" x="596" y="-369.452" font-family="Times,serif" font-size="20.00">skip_page</text>
-</g>
-<!-- t1&#45;&gt;t11 -->
-<g id="edge13" class="edge"><title>t1&#45;&gt;t11</title>
-<path fill="none" stroke="#0044a0" d="M425.139,-478.007C437.834,-470.731 450.731,-461.83 461,-451.452 473.874,-438.442 466.946,-427.178 481,-415.452 490.212,-407.766 513.965,-399.205 537.508,-392.067"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="538.592,-395.396 547.189,-389.203 536.607,-388.683 538.592,-395.396"/>
-</g>
-<!-- t9 -->
-<g id="node11" class="node"><title>t9</title>
-<polygon fill="#efa03b" stroke="black" points="272.496,-277.452 19.5039,-277.452 15.5039,-273.452 15.5039,-241.452 268.496,-241.452 272.496,-245.452 272.496,-277.452"/>
-<polyline fill="none" stroke="black" points="268.496,-273.452 15.5039,-273.452 "/>
-<polyline fill="none" stroke="black" points="268.496,-273.452 268.496,-241.452 "/>
-<polyline fill="none" stroke="black" points="268.496,-273.452 272.496,-277.452 "/>
-<text text-anchor="middle" x="144" y="-253.452" font-family="Times,serif" font-size="20.00">tesseract_ocr_and_render_pdf</text>
-</g>
-<!-- t1&#45;&gt;t9 -->
-<g id="edge15" class="edge"><title>t1&#45;&gt;t9</title>
-<path fill="none" stroke="#0044a0" d="M277.341,-486.044C254.708,-478.702 232.21,-467.759 215,-451.452 168.207,-407.114 152.056,-329.127 146.633,-287.855"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="150.084,-287.235 145.422,-277.721 143.133,-288.065 150.084,-287.235"/>
+<path fill="none" stroke="#0044a0" d="M980.788,-651.715C965.266,-644.498 948.187,-636.558 932.905,-629.452"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="934.195,-626.193 923.652,-625.15 931.244,-632.54 934.195,-626.193"/>
 </g>
 <!-- t3 -->
 <g id="node4" class="node"><title>t3</title>
-<polygon fill="#efa03b" stroke="black" points="458.999,-393.452 291.001,-393.452 287.001,-389.452 287.001,-357.452 454.999,-357.452 458.999,-361.452 458.999,-393.452"/>
-<polyline fill="none" stroke="black" points="454.999,-389.452 287.001,-389.452 "/>
-<polyline fill="none" stroke="black" points="454.999,-389.452 454.999,-357.452 "/>
-<polyline fill="none" stroke="black" points="454.999,-389.452 458.999,-393.452 "/>
-<text text-anchor="middle" x="373" y="-369.452" font-family="Times,serif" font-size="20.00">preprocess_deskew</text>
+<polygon fill="#efa03b" stroke="black" points="1128.15,-567.109 893.852,-567.109 889.852,-563.109 889.852,-531.109 1124.15,-531.109 1128.15,-535.109 1128.15,-567.109"/>
+<polyline fill="none" stroke="black" points="1124.15,-563.109 889.852,-563.109 "/>
+<polyline fill="none" stroke="black" points="1124.15,-563.109 1124.15,-531.109 "/>
+<polyline fill="none" stroke="black" points="1124.15,-563.109 1128.15,-567.109 "/>
+<text text-anchor="middle" x="1009" y="-543.109" font-family="Times,serif" font-size="20.00">orient_page</text>
+</g>
+<!-- t1&#45;&gt;t3 -->
+<g id="edge4" class="edge"><title>t1&#45;&gt;t3</title>
+<path fill="none" stroke="#0044a0" d="M1037.63,-651.508C1042.21,-633.96 1045.92,-609.455 1039,-589.109 1037.4,-584.404 1034.98,-579.822 1032.19,-575.552"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1034.85,-573.258 1026.1,-567.282 1029.21,-577.409 1034.85,-573.258"/>
 </g>
 <!-- t2&#45;&gt;t3 -->
 <g id="edge3" class="edge"><title>t2&#45;&gt;t3</title>
-<path fill="none" stroke="#0044a0" d="M348.691,-415.346C351.337,-411.112 354.23,-406.485 357.066,-401.946"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="360.042,-403.788 362.374,-393.453 354.106,-400.078 360.042,-403.788"/>
-</g>
-<!-- t6 -->
-<g id="node6" class="node"><title>t6</title>
-<polygon fill="#efa03b" stroke="black" points="664.375,-277.452 477.625,-277.452 473.625,-273.452 473.625,-241.452 660.375,-241.452 664.375,-245.452 664.375,-277.452"/>
-<polyline fill="none" stroke="black" points="660.375,-273.452 473.625,-273.452 "/>
-<polyline fill="none" stroke="black" points="660.375,-273.452 660.375,-241.452 "/>
-<polyline fill="none" stroke="black" points="660.375,-273.452 664.375,-277.452 "/>
-<text text-anchor="middle" x="569" y="-253.452" font-family="Times,serif" font-size="20.00">select_image_for_pdf</text>
-</g>
-<!-- t2&#45;&gt;t6 -->
-<g id="edge7" class="edge"><title>t2&#45;&gt;t6</title>
-<path fill="none" stroke="#0044a0" d="M423.032,-415.374C438.812,-409.925 454.543,-402.785 468,-393.452 508.201,-365.571 539.287,-316.662 555.808,-286.558"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="559.024,-287.965 560.653,-277.496 552.851,-284.664 559.024,-287.965"/>
+<path fill="none" stroke="#0044a0" d="M924.267,-589.003C936.328,-583.467 949.857,-577.257 962.502,-571.453"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="964.333,-574.463 971.961,-567.111 961.413,-568.102 964.333,-574.463"/>
 </g>
 <!-- t4 -->
 <g id="node5" class="node"><title>t4</title>
-<polygon fill="#efa03b" stroke="black" points="449.705,-335.452 300.295,-335.452 296.295,-331.452 296.295,-299.452 445.705,-299.452 449.705,-303.452 449.705,-335.452"/>
-<polyline fill="none" stroke="black" points="445.705,-331.452 296.295,-331.452 "/>
-<polyline fill="none" stroke="black" points="445.705,-331.452 445.705,-299.452 "/>
-<polyline fill="none" stroke="black" points="445.705,-331.452 449.705,-335.452 "/>
-<text text-anchor="middle" x="373" y="-311.452" font-family="Times,serif" font-size="20.00">preprocess_clean</text>
+<polygon fill="#efa03b" stroke="black" points="1027.3,-509.109 676.701,-509.109 672.701,-505.109 672.701,-473.109 1023.3,-473.109 1027.3,-477.109 1027.3,-509.109"/>
+<polyline fill="none" stroke="black" points="1023.3,-505.109 672.701,-505.109 "/>
+<polyline fill="none" stroke="black" points="1023.3,-505.109 1023.3,-473.109 "/>
+<polyline fill="none" stroke="black" points="1023.3,-505.109 1027.3,-509.109 "/>
+<text text-anchor="middle" x="850" y="-485.109" font-family="Times,serif" font-size="20.00">rasterize_with_ghostscript</text>
 </g>
 <!-- t3&#45;&gt;t4 -->
-<g id="edge4" class="edge"><title>t3&#45;&gt;t4</title>
-<path fill="none" stroke="#0044a0" d="M373,-357.346C373,-353.655 373,-349.665 373,-345.695"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="376.5,-345.453 373,-335.453 369.5,-345.453 376.5,-345.453"/>
+<g id="edge5" class="edge"><title>t3&#45;&gt;t4</title>
+<path fill="none" stroke="#0044a0" d="M960.431,-531.003C943.787,-525.141 924.999,-518.524 907.704,-512.433"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="908.867,-509.132 898.272,-509.111 906.541,-515.734 908.867,-509.132"/>
 </g>
-<!-- t3&#45;&gt;t6 -->
-<g id="edge6" class="edge"><title>t3&#45;&gt;t6</title>
-<path fill="none" stroke="#0044a0" d="M414.399,-357.412C428.785,-351.039 444.852,-343.405 459,-335.452 466.909,-331.007 506.49,-303.805 535.921,-283.431"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="538.073,-286.198 544.299,-277.625 534.086,-280.444 538.073,-286.198"/>
+<!-- t9 -->
+<g id="node11" class="node"><title>t9</title>
+<polygon fill="#efa03b" stroke="black" points="314.109,-277.109 19.8906,-277.109 15.8906,-273.109 15.8906,-241.109 310.109,-241.109 314.109,-245.109 314.109,-277.109"/>
+<polyline fill="none" stroke="black" points="310.109,-273.109 15.8906,-273.109 "/>
+<polyline fill="none" stroke="black" points="310.109,-273.109 310.109,-241.109 "/>
+<polyline fill="none" stroke="black" points="310.109,-273.109 314.109,-277.109 "/>
+<text text-anchor="middle" x="165" y="-253.109" font-family="Times,serif" font-size="20.00">select_image_layer</text>
 </g>
-<!-- t4&#45;&gt;t6 -->
-<g id="edge5" class="edge"><title>t4&#45;&gt;t6</title>
-<path fill="none" stroke="#0044a0" d="M432.605,-299.422C453.701,-293.395 477.617,-286.561 499.465,-280.319"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="500.687,-283.61 509.341,-277.497 498.764,-276.879 500.687,-283.61"/>
+<!-- t3&#45;&gt;t9 -->
+<g id="edge14" class="edge"><title>t3&#45;&gt;t9</title>
+<path fill="none" stroke="#0044a0" d="M889.675,-542.297C675.505,-530.255 248,-498.192 248,-434.109 248,-434.109 248,-434.109 248,-374.109 248,-337.597 219.83,-304.936 196.312,-284.031"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="198.437,-281.243 188.562,-277.404 193.888,-286.563 198.437,-281.243"/>
+</g>
+<!-- t15 -->
+<g id="node14" class="node"><title>t15</title>
+<polygon fill="#efa03b" stroke="black" points="1391.34,-451.109 1170.66,-451.109 1166.66,-447.109 1166.66,-415.109 1387.34,-415.109 1391.34,-419.109 1391.34,-451.109"/>
+<polyline fill="none" stroke="black" points="1387.34,-447.109 1166.66,-447.109 "/>
+<polyline fill="none" stroke="black" points="1387.34,-447.109 1387.34,-415.109 "/>
+<polyline fill="none" stroke="black" points="1387.34,-447.109 1391.34,-451.109 "/>
+<text text-anchor="middle" x="1279" y="-427.109" font-family="Times,serif" font-size="20.00">skip_page</text>
+</g>
+<!-- t3&#45;&gt;t15 -->
+<g id="edge19" class="edge"><title>t3&#45;&gt;t15</title>
+<path fill="none" stroke="#0044a0" d="M1060.32,-531.078C1078.95,-524.591 1100.07,-516.881 1119,-509.109 1159.3,-492.568 1204.11,-471.304 1236.04,-455.645"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1237.67,-458.74 1245.1,-451.181 1234.58,-452.461 1237.67,-458.74"/>
+</g>
+<!-- t13 -->
+<g id="node15" class="node"><title>t13</title>
+<polygon fill="#efa03b" stroke="black" points="1388.24,-277.109 1007.76,-277.109 1003.76,-273.109 1003.76,-241.109 1384.24,-241.109 1388.24,-245.109 1388.24,-277.109"/>
+<polyline fill="none" stroke="black" points="1384.24,-273.109 1003.76,-273.109 "/>
+<polyline fill="none" stroke="black" points="1384.24,-273.109 1384.24,-241.109 "/>
+<polyline fill="none" stroke="black" points="1384.24,-273.109 1388.24,-277.109 "/>
+<text text-anchor="middle" x="1196" y="-253.109" font-family="Times,serif" font-size="20.00">tesseract_ocr_and_render_pdf</text>
+</g>
+<!-- t3&#45;&gt;t13 -->
+<g id="edge21" class="edge"><title>t3&#45;&gt;t13</title>
+<path fill="none" stroke="#0044a0" d="M1040.79,-531.094C1071.69,-511.983 1114,-477.834 1114,-434.109 1114,-434.109 1114,-434.109 1114,-374.109 1114,-337.749 1141.83,-305.063 1165.07,-284.11"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1167.46,-286.664 1172.72,-277.466 1162.87,-281.377 1167.46,-286.664"/>
 </g>
 <!-- t5 -->
-<g id="node7" class="node"><title>t5</title>
-<polygon fill="#efa03b" stroke="black" points="455.922,-277.452 294.078,-277.452 290.078,-273.452 290.078,-241.452 451.922,-241.452 455.922,-245.452 455.922,-277.452"/>
-<polyline fill="none" stroke="black" points="451.922,-273.452 290.078,-273.452 "/>
-<polyline fill="none" stroke="black" points="451.922,-273.452 451.922,-241.452 "/>
-<polyline fill="none" stroke="black" points="451.922,-273.452 455.922,-277.452 "/>
-<text text-anchor="middle" x="373" y="-253.452" font-family="Times,serif" font-size="20.00">ocr_tesseract_hocr</text>
+<g id="node6" class="node"><title>t5</title>
+<polygon fill="#efa03b" stroke="black" points="830.742,-451.109 535.258,-451.109 531.258,-447.109 531.258,-415.109 826.742,-415.109 830.742,-419.109 830.742,-451.109"/>
+<polyline fill="none" stroke="black" points="826.742,-447.109 531.258,-447.109 "/>
+<polyline fill="none" stroke="black" points="826.742,-447.109 826.742,-415.109 "/>
+<polyline fill="none" stroke="black" points="826.742,-447.109 830.742,-451.109 "/>
+<text text-anchor="middle" x="681" y="-427.109" font-family="Times,serif" font-size="20.00">preprocess_deskew</text>
 </g>
 <!-- t4&#45;&gt;t5 -->
-<g id="edge8" class="edge"><title>t4&#45;&gt;t5</title>
-<path fill="none" stroke="#0044a0" d="M373,-299.346C373,-295.655 373,-291.665 373,-287.695"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="376.5,-287.453 373,-277.453 369.5,-287.453 376.5,-287.453"/>
+<g id="edge6" class="edge"><title>t4&#45;&gt;t5</title>
+<path fill="none" stroke="#0044a0" d="M798.376,-473.003C780.521,-467.087 760.346,-460.402 741.823,-454.264"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="742.901,-450.934 732.308,-451.111 740.699,-457.579 742.901,-450.934"/>
 </g>
-<!-- t4&#45;&gt;t9 -->
-<g id="edge14" class="edge"><title>t4&#45;&gt;t9</title>
-<path fill="none" stroke="#0044a0" d="M303.36,-299.422C278.157,-293.259 249.509,-286.253 223.521,-279.898"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="224.249,-276.473 213.704,-277.497 222.586,-283.273 224.249,-276.473"/>
+<!-- t8 -->
+<g id="node10" class="node"><title>t8</title>
+<polygon fill="#efa03b" stroke="black" points="980.119,-335.109 665.881,-335.109 661.881,-331.109 661.881,-299.109 976.119,-299.109 980.119,-303.109 980.119,-335.109"/>
+<polyline fill="none" stroke="black" points="976.119,-331.109 661.881,-331.109 "/>
+<polyline fill="none" stroke="black" points="976.119,-331.109 976.119,-299.109 "/>
+<polyline fill="none" stroke="black" points="976.119,-331.109 980.119,-335.109 "/>
+<text text-anchor="middle" x="821" y="-311.109" font-family="Times,serif" font-size="20.00">select_image_for_pdf</text>
+</g>
+<!-- t4&#45;&gt;t8 -->
+<g id="edge12" class="edge"><title>t4&#45;&gt;t8</title>
+<path fill="none" stroke="#0044a0" d="M848.03,-473.087C846.196,-457.857 843.261,-434.941 840,-415.109 836.1,-391.393 830.78,-364.592 826.761,-345.181"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="830.152,-344.296 824.68,-335.224 823.3,-345.728 830.152,-344.296"/>
+</g>
+<!-- t6 -->
+<g id="node7" class="node"><title>t6</title>
+<polygon fill="#efa03b" stroke="black" points="801.95,-393.109 524.05,-393.109 520.05,-389.109 520.05,-357.109 797.95,-357.109 801.95,-361.109 801.95,-393.109"/>
+<polyline fill="none" stroke="black" points="797.95,-389.109 520.05,-389.109 "/>
+<polyline fill="none" stroke="black" points="797.95,-389.109 797.95,-357.109 "/>
+<polyline fill="none" stroke="black" points="797.95,-389.109 801.95,-393.109 "/>
+<text text-anchor="middle" x="661" y="-369.109" font-family="Times,serif" font-size="20.00">preprocess_clean</text>
+</g>
+<!-- t5&#45;&gt;t6 -->
+<g id="edge7" class="edge"><title>t5&#45;&gt;t6</title>
+<path fill="none" stroke="#0044a0" d="M674.891,-415.003C673.495,-411.095 671.979,-406.852 670.48,-402.652"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="673.731,-401.351 667.072,-393.111 667.139,-403.705 673.731,-401.351"/>
+</g>
+<!-- t5&#45;&gt;t8 -->
+<g id="edge11" class="edge"><title>t5&#45;&gt;t8</title>
+<path fill="none" stroke="#0044a0" d="M781.944,-415.046C792.967,-409.579 803.051,-402.434 811,-393.109 822.009,-380.195 824.454,-361.215 824.174,-345.581"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="827.659,-345.2 823.617,-335.407 820.669,-345.583 827.659,-345.2"/>
 </g>
 <!-- t7 -->
 <g id="node8" class="node"><title>t7</title>
-<polygon fill="#efa03b" stroke="black" points="426.366,-219.452 269.634,-219.452 265.634,-215.452 265.634,-183.452 422.366,-183.452 426.366,-187.452 426.366,-219.452"/>
-<polyline fill="none" stroke="black" points="422.366,-215.452 265.634,-215.452 "/>
-<polyline fill="none" stroke="black" points="422.366,-215.452 422.366,-183.452 "/>
-<polyline fill="none" stroke="black" points="422.366,-215.452 426.366,-219.452 "/>
-<text text-anchor="middle" x="346" y="-195.452" font-family="Times,serif" font-size="20.00">render_hocr_page</text>
+<polygon fill="#efa03b" stroke="black" points="643.666,-335.109 354.334,-335.109 350.334,-331.109 350.334,-299.109 639.666,-299.109 643.666,-303.109 643.666,-335.109"/>
+<polyline fill="none" stroke="black" points="639.666,-331.109 350.334,-331.109 "/>
+<polyline fill="none" stroke="black" points="639.666,-331.109 639.666,-299.109 "/>
+<polyline fill="none" stroke="black" points="639.666,-331.109 643.666,-335.109 "/>
+<text text-anchor="middle" x="497" y="-311.109" font-family="Times,serif" font-size="20.00">ocr_tesseract_hocr</text>
 </g>
 <!-- t6&#45;&gt;t7 -->
-<g id="edge9" class="edge"><title>t6&#45;&gt;t7</title>
-<path fill="none" stroke="#0044a0" d="M501.184,-241.422C476.75,-235.286 448.99,-228.315 423.772,-221.982"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="424.429,-218.538 413.877,-219.497 422.724,-225.328 424.429,-218.538"/>
-</g>
-<!-- t8 -->
-<g id="node9" class="node"><title>t8</title>
-<polygon fill="#efa03b" stroke="black" points="663.742,-219.452 448.258,-219.452 444.258,-215.452 444.258,-183.452 659.742,-183.452 663.742,-187.452 663.742,-219.452"/>
-<polyline fill="none" stroke="black" points="659.742,-215.452 444.258,-215.452 "/>
-<polyline fill="none" stroke="black" points="659.742,-215.452 659.742,-183.452 "/>
-<polyline fill="none" stroke="black" points="659.742,-215.452 663.742,-219.452 "/>
-<text text-anchor="middle" x="554" y="-195.452" font-family="Times,serif" font-size="20.00">render_hocr_debug_page</text>
+<g id="edge8" class="edge"><title>t6&#45;&gt;t7</title>
+<path fill="none" stroke="#0044a0" d="M610.903,-357.003C593.736,-351.141 574.358,-344.524 556.518,-338.433"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="557.384,-335.03 546.79,-335.111 555.122,-341.654 557.384,-335.03"/>
 </g>
 <!-- t6&#45;&gt;t8 -->
-<g id="edge11" class="edge"><title>t6&#45;&gt;t8</title>
-<path fill="none" stroke="#0044a0" d="M564.418,-241.346C563.4,-237.546 562.298,-233.43 561.203,-229.345"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="564.522,-228.207 558.554,-219.453 557.761,-230.018 564.522,-228.207"/>
+<g id="edge10" class="edge"><title>t6&#45;&gt;t8</title>
+<path fill="none" stroke="#0044a0" d="M709.875,-357.003C726.624,-351.141 745.529,-344.524 762.933,-338.433"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="764.142,-341.718 772.425,-335.111 761.83,-335.111 764.142,-341.718"/>
 </g>
-<!-- t5&#45;&gt;t7 -->
-<g id="edge10" class="edge"><title>t5&#45;&gt;t7</title>
-<path fill="none" stroke="#0044a0" d="M364.752,-241.346C362.816,-237.329 360.708,-232.958 358.629,-228.645"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="361.693,-226.941 354.197,-219.453 355.387,-229.981 361.693,-226.941"/>
+<!-- t10 -->
+<g id="node9" class="node"><title>t10</title>
+<polygon fill="#efa03b" stroke="black" points="620.109,-277.109 335.891,-277.109 331.891,-273.109 331.891,-241.109 616.109,-241.109 620.109,-245.109 620.109,-277.109"/>
+<polyline fill="none" stroke="black" points="616.109,-273.109 331.891,-273.109 "/>
+<polyline fill="none" stroke="black" points="616.109,-273.109 616.109,-241.109 "/>
+<polyline fill="none" stroke="black" points="616.109,-273.109 620.109,-277.109 "/>
+<text text-anchor="middle" x="476" y="-253.109" font-family="Times,serif" font-size="20.00">render_hocr_page</text>
 </g>
-<!-- t5&#45;&gt;t8 -->
-<g id="edge12" class="edge"><title>t5&#45;&gt;t8</title>
-<path fill="none" stroke="#0044a0" d="M428.289,-241.346C447.587,-235.375 469.416,-228.622 489.404,-222.437"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="490.53,-225.753 499.049,-219.453 488.461,-219.065 490.53,-225.753"/>
+<!-- t7&#45;&gt;t10 -->
+<g id="edge9" class="edge"><title>t7&#45;&gt;t10</title>
+<path fill="none" stroke="#0044a0" d="M490.585,-299.003C489.12,-295.095 487.528,-290.852 485.953,-286.652"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="489.164,-285.245 482.376,-277.111 482.61,-287.703 489.164,-285.245"/>
+</g>
+<!-- t11 -->
+<g id="node13" class="node"><title>t11</title>
+<polygon fill="#efa03b" stroke="black" points="985.486,-277.109 642.514,-277.109 638.514,-273.109 638.514,-241.109 981.486,-241.109 985.486,-245.109 985.486,-277.109"/>
+<polyline fill="none" stroke="black" points="981.486,-273.109 638.514,-273.109 "/>
+<polyline fill="none" stroke="black" points="981.486,-273.109 981.486,-241.109 "/>
+<polyline fill="none" stroke="black" points="981.486,-273.109 985.486,-277.109 "/>
+<text text-anchor="middle" x="812" y="-253.109" font-family="Times,serif" font-size="20.00">render_hocr_debug_page</text>
+</g>
+<!-- t7&#45;&gt;t11 -->
+<g id="edge18" class="edge"><title>t7&#45;&gt;t11</title>
+<path fill="none" stroke="#0044a0" d="M592.793,-299.079C628.531,-292.726 669.303,-285.478 705.912,-278.969"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="706.887,-282.351 716.12,-277.155 705.661,-275.459 706.887,-282.351"/>
 </g>
 <!-- t12 -->
-<g id="node13" class="node"><title>t12</title>
-<polygon fill="#efa03b" stroke="black" points="473.845,-105.456 554,-78.0208 634.155,-105.456 634.08,-149.848 473.92,-149.848 473.845,-105.456"/>
-<polygon fill="none" stroke="black" points="469.836,-102.581 554,-73.7729 638.164,-102.581 638.078,-153.869 469.922,-153.869 469.836,-102.581"/>
-<text text-anchor="middle" x="554" y="-111.726" font-family="Times,serif" font-size="20.00">merge_pages</text>
-</g>
-<!-- t7&#45;&gt;t12 -->
-<g id="edge20" class="edge"><title>t7&#45;&gt;t12</title>
-<path fill="none" stroke="#0044a0" d="M389.35,-183.419C409.937,-175.33 435.398,-165.326 460.092,-155.624"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="461.479,-158.839 469.507,-151.924 458.919,-152.324 461.479,-158.839"/>
-</g>
-<!-- t8&#45;&gt;t12 -->
-<g id="edge19" class="edge"><title>t8&#45;&gt;t12</title>
-<path fill="none" stroke="#0044a0" d="M554,-183.12C554,-177.585 554,-171.177 554,-164.592"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="557.5,-164.201 554,-154.201 550.5,-164.201 557.5,-164.201"/>
-</g>
-<!-- t11&#45;&gt;t12 -->
-<g id="edge17" class="edge"><title>t11&#45;&gt;t12</title>
-<path fill="none" stroke="#0044a0" d="M615.939,-357.13C634.767,-339.333 661.745,-309.733 673,-277.452 686.754,-238.003 694.347,-219.364 673,-183.452 666.345,-172.256 656.999,-162.875 646.44,-155.047"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="648.378,-152.132 638.152,-149.36 644.417,-157.904 648.378,-152.132"/>
-</g>
-<!-- t9&#45;&gt;t12 -->
-<g id="edge18" class="edge"><title>t9&#45;&gt;t12</title>
-<path fill="none" stroke="#0044a0" d="M164.711,-241.235C186.394,-224.08 222.088,-198.224 257,-183.452 321.895,-155.994 399.893,-139.571 459.67,-130.141"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="460.504,-133.554 469.855,-128.574 459.439,-126.635 460.504,-133.554"/>
+<g id="node12" class="node"><title>t12</title>
+<polygon fill="#efa03b" stroke="black" points="607.082,-219.109 348.918,-219.109 344.918,-215.109 344.918,-183.109 603.082,-183.109 607.082,-187.109 607.082,-219.109"/>
+<polyline fill="none" stroke="black" points="603.082,-215.109 344.918,-215.109 "/>
+<polyline fill="none" stroke="black" points="603.082,-215.109 603.082,-183.109 "/>
+<polyline fill="none" stroke="black" points="603.082,-215.109 607.082,-219.109 "/>
+<text text-anchor="middle" x="476" y="-195.109" font-family="Times,serif" font-size="20.00">add_text_layer</text>
 </g>
 <!-- t10&#45;&gt;t12 -->
-<g id="edge22" class="edge"><title>t10&#45;&gt;t12</title>
-<path fill="none" stroke="gray" d="M629.379,-415.337C660.552,-396.255 703,-362.233 703,-318.452 703,-318.452 703,-318.452 703,-258.452 703,-224.066 705.581,-210.223 684,-183.452 673.976,-171.017 660.952,-160.773 647.026,-152.396"/>
-<polygon fill="gray" stroke="gray" points="648.682,-149.312 638.258,-147.419 645.227,-155.4 648.682,-149.312"/>
+<g id="edge15" class="edge"><title>t10&#45;&gt;t12</title>
+<path fill="none" stroke="#0044a0" d="M476,-241.003C476,-237.312 476,-233.322 476,-229.352"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="479.5,-229.111 476,-219.111 472.5,-229.111 479.5,-229.111"/>
 </g>
-<!-- t13 -->
-<g id="node14" class="node"><title>t13</title>
-<polygon fill="#efa03b" stroke="black" points="616.338,-52 495.662,-52 491.662,-48 491.662,-16 612.338,-16 616.338,-20 616.338,-52"/>
-<polyline fill="none" stroke="black" points="612.338,-48 491.662,-48 "/>
-<polyline fill="none" stroke="black" points="612.338,-48 612.338,-16 "/>
-<polyline fill="none" stroke="black" points="612.338,-48 616.338,-52 "/>
-<text text-anchor="middle" x="554" y="-28" font-family="Times,serif" font-size="20.00">validate_pdfa</text>
+<!-- t8&#45;&gt;t9 -->
+<g id="edge13" class="edge"><title>t8&#45;&gt;t9</title>
+<path fill="none" stroke="#0044a0" d="M661.842,-299.864C658.871,-299.606 655.921,-299.354 653,-299.109 509.812,-287.095 471.159,-289.734 324.542,-277.219"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="324.526,-273.705 314.262,-276.332 323.924,-280.679 324.526,-273.705"/>
 </g>
-<!-- t12&#45;&gt;t13 -->
-<g id="edge21" class="edge"><title>t12&#45;&gt;t13</title>
-<path fill="none" stroke="#0044a0" d="M554,-73.9482C554,-69.9654 554,-66.007 554,-62.2247"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="557.5,-62.1573 554,-52.1573 550.5,-62.1574 557.5,-62.1573"/>
+<!-- t8&#45;&gt;t11 -->
+<g id="edge17" class="edge"><title>t8&#45;&gt;t11</title>
+<path fill="none" stroke="#0044a0" d="M818.251,-299.003C817.64,-295.204 816.979,-291.087 816.322,-287.002"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="819.775,-286.429 814.732,-277.111 812.864,-287.539 819.775,-286.429"/>
+</g>
+<!-- t8&#45;&gt;t13 -->
+<g id="edge20" class="edge"><title>t8&#45;&gt;t13</title>
+<path fill="none" stroke="#0044a0" d="M935.04,-299.079C978.311,-292.618 1027.78,-285.23 1071.94,-278.636"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1072.48,-282.093 1081.86,-277.155 1071.45,-275.17 1072.48,-282.093"/>
+</g>
+<!-- t9&#45;&gt;t12 -->
+<g id="edge16" class="edge"><title>t9&#45;&gt;t12</title>
+<path fill="none" stroke="#0044a0" d="M259.577,-241.079C294.86,-234.726 335.115,-227.478 371.259,-220.969"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="372.116,-224.372 381.337,-219.155 370.875,-217.482 372.116,-224.372"/>
+</g>
+<!-- t16 -->
+<g id="node17" class="node"><title>t16</title>
+<polygon fill="#efa03b" stroke="black" points="1031.47,-105.333 1196,-78.005 1360.53,-105.333 1360.37,-149.551 1031.63,-149.551 1031.47,-105.333"/>
+<polygon fill="none" stroke="black" points="1027.46,-101.94 1196,-73.9453 1364.54,-101.94 1364.36,-153.556 1027.64,-153.556 1027.46,-101.94"/>
+<text text-anchor="middle" x="1196" y="-111.555" font-family="Times,serif" font-size="20.00">merge_pages</text>
+</g>
+<!-- t12&#45;&gt;t16 -->
+<g id="edge26" class="edge"><title>t12&#45;&gt;t16</title>
+<path fill="none" stroke="#0044a0" d="M607.222,-185.246C722.142,-172.229 890.172,-153.196 1017.34,-138.791"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1018.04,-142.235 1027.58,-137.632 1017.25,-135.279 1018.04,-142.235"/>
+</g>
+<!-- t11&#45;&gt;t16 -->
+<g id="edge27" class="edge"><title>t11&#45;&gt;t16</title>
+<path fill="none" stroke="#0044a0" d="M859.086,-240.997C916.223,-220.232 1014.29,-184.591 1089.94,-157.099"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1091.32,-160.323 1099.52,-153.618 1088.92,-153.744 1091.32,-160.323"/>
+</g>
+<!-- t15&#45;&gt;t16 -->
+<g id="edge23" class="edge"><title>t15&#45;&gt;t16</title>
+<path fill="none" stroke="#0044a0" d="M1321.98,-414.938C1359.16,-396.916 1407,-364.681 1407,-318.109 1407,-318.109 1407,-318.109 1407,-258.109 1407,-211.469 1370.69,-179.509 1328.2,-158.076"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1329.57,-154.854 1319.04,-153.66 1326.53,-161.158 1329.57,-154.854"/>
+</g>
+<!-- t13&#45;&gt;t16 -->
+<g id="edge25" class="edge"><title>t13&#45;&gt;t16</title>
+<path fill="none" stroke="#0044a0" d="M1196,-240.997C1196,-221.914 1196,-190.265 1196,-163.926"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1199.5,-163.759 1196,-153.759 1192.5,-163.759 1199.5,-163.759"/>
+</g>
+<!-- t14&#45;&gt;t16 -->
+<g id="edge24" class="edge"><title>t14&#45;&gt;t16</title>
+<path fill="none" stroke="#0044a0" d="M1326.64,-588.956C1373.58,-572.748 1427,-543.103 1427,-492.109 1427,-492.109 1427,-492.109 1427,-258.109 1427,-220.539 1416.55,-207.537 1388,-183.109 1376.82,-173.542 1364.16,-165.377 1350.82,-158.413"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1351.99,-155.087 1341.47,-153.781 1348.88,-161.358 1351.99,-155.087"/>
+</g>
+<!-- t17 -->
+<g id="node18" class="node"><title>t17</title>
+<polygon fill="#efa03b" stroke="black" points="1310.18,-52 1085.82,-52 1081.82,-48 1081.82,-16 1306.18,-16 1310.18,-20 1310.18,-52"/>
+<polyline fill="none" stroke="black" points="1306.18,-48 1081.82,-48 "/>
+<polyline fill="none" stroke="black" points="1306.18,-48 1306.18,-16 "/>
+<polyline fill="none" stroke="black" points="1306.18,-48 1310.18,-52 "/>
+<text text-anchor="middle" x="1196" y="-28" font-family="Times,serif" font-size="20.00">copy_final</text>
+</g>
+<!-- t16&#45;&gt;t17 -->
+<g id="edge28" class="edge"><title>t16&#45;&gt;t17</title>
+<path fill="none" stroke="#0044a0" d="M1196,-73.8665C1196,-69.8921 1196,-65.942 1196,-62.1676"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1199.5,-62.1213 1196,-52.1214 1192.5,-62.1214 1199.5,-62.1213"/>
 </g>
 </g>
 </svg>

pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+norecursedirs = lib 

requirements.txt

@@ -0,0 +1,6 @@
+ruffus==2.6.3
+Pillow==3.1.1
+reportlab==3.2.0
+PyPDF2==1.25.1
+img2pdf==0.2
+cffi==1.5.2

setup.cfg

@@ -1,2 +1,4 @@
 [bdist_wheel]
-python-tag = py34
+python-tag = py34
+[aliases]
+test=pytest

setup.py

@@ -1,11 +1,18 @@
 #!/usr/bin/env python3
+# -*- coding: utf-8 -*-
 # © 2015 James R. Barlow: github.com/jbarlow83
 
-from setuptools import setup
-from subprocess import Popen, STDOUT, check_output, CalledProcessError
-from string import Template
-import re
+from __future__ import print_function, unicode_literals
+
 import sys
+if sys.version_info < (3, 4):
+    print("Python 3.4 or newer is required")
+    sys.exit(1)
+
+from setuptools import setup  # nopep8
+from subprocess import STDOUT, check_output, CalledProcessError  # nopep8
+from collections.abc import Mapping  # nopep8
+import re  # nopep8
 
 
 missing_program = '''
@@ -48,21 +55,30 @@ commands:
     sudo apt-get install {package}
 
 On RPM-based systems (Red Hat, Fedora), search for instructions on
-installing the RPM for {package}.
+installing the RPM for {program}.
 '''
 
 
-def _error_trailer(program, package, optional, **kwargs):
-    if program == 'java':
-        return  # You're fucked
+def get_platform():
+    if sys.platform.startswith('freebsd'):
+        return 'freebsd'
+    elif sys.platform.startswith('linux'):
+        return 'linux'
+    return sys.platform
 
+
+def _error_trailer(program, package, optional, **kwargs):
     if optional:
         print(okay_its_optional.format(**locals()), file=sys.stderr)
     else:
         print(not_okay_its_required.format(**locals()), file=sys.stderr)
-    if sys.platform.startswith('darwin'):
+
+    if isinstance(package, Mapping):
+        package = package[get_platform()]
+
+    if get_platform() == 'darwin':
         print(osx_install_advice.format(**locals()), file=sys.stderr)
-    elif sys.platform.startswith('linux'):
+    elif get_platform() == 'linux':
         print(linux_install_advice.format(**locals()), file=sys.stderr)
 
 
@@ -130,14 +146,16 @@ def check_external_program(
     print('Found {program} {found_version}'.format(
             program=program, found_version=found_version))
 
+
 command = next((arg for arg in sys.argv[1:] if not arg.startswith('-')), '')
 
+
 if command.startswith('install') or \
-        command in ['check', 'test', 'nosetests', 'easy_install', 'egg_info']:
+        command in ['check', 'test', 'nosetests', 'easy_install']:
     check_external_program(
         program='tesseract',
         need_version='3.02.02',
-        package='tesseract'
+        package={'darwin': 'tesseract', 'linux': 'tesseract-ocr'}
     )
     check_external_program(
         program='gs',
@@ -150,12 +168,6 @@ if command.startswith('install') or \
         package='unpaper',
         optional=True
     )
-    check_external_program(
-        program='java',
-        need_version='1.5.0',
-        package='Java Runtime Environment',
-        version_check_args=['-version']
-    )
     check_external_program(
         program='qpdf',
         need_version='5.0.0',
@@ -163,19 +175,25 @@ if command.startswith('install') or \
         version_check_args=['--version']
     )
 
+
+if 'upload' in sys.argv[1:]:
+    print('Use twine to upload the package - setup.py upload is insecure')
+    sys.exit(1)
+
+tests_require = open('test_requirements.txt').read().splitlines()
+
 setup(
     name='ocrmypdf',
-    version='3.0rc4',  # also update: release notes, main.py
     description='OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched',
-    url='https://github.com/fritz-hh/OCRmyPDF',
-    author='James. R. Barlow',
+    url='https://github.com/jbarlow83/OCRmyPDF',
+    author='James R. Barlow',
     author_email='jim@purplerock.ca',
     license='Public Domain',
     packages=['ocrmypdf'],
     keywords=['PDF', 'OCR', 'optical character recognition', 'PDF/A', 'scanning'],
     classifiers=[
         "Programming Language :: Python :: 3",
-        "Development Status :: 4 - Beta",
+        "Development Status :: 5 - Production/Stable",
         "Environment :: Console",
         "Intended Audience :: End Users/Desktop",
         "Intended Audience :: Science/Research",
@@ -189,26 +207,29 @@ setup(
         "Topic :: Text Processing :: Indexing",
         "Topic :: Text Processing :: Linguistic",
         ],
+    setup_requires=[
+        'setuptools_scm',
+        'setuptools_scm_git_archive',
+        'cffi>=1.5.0',
+        'pytest-runner'
+    ],
+    use_scm_version={'version_scheme': 'post-release'},
+    cffi_modules=[
+        'ocrmypdf/lib/compile_leptonica.py:ffi'
+    ],
     install_requires=[
         'ruffus>=2.6.3',
-        'Pillow>=2.7.0',
-        'lxml>=3.4.2',
+        'Pillow>=3.0.0',
         'reportlab>=3.1.44',
-        'PyPDF2>=1.25.1'
-    ],
-    test_requires=[
-        'img2pdf>=0.1.5',
-        'pytest>=2.7.2'
+        'PyPDF2>=1.25.1',
+        'img2pdf>=0.2.0',
+        'cffi>=1.5.0'
     ],
+    tests_require=tests_require,
     entry_points={
         'console_scripts': [
             'ocrmypdf = ocrmypdf.main:run_pipeline'
         ],
     },
-    eager_resources=[
-        'ocrmypdf/jhove/bin/*.jar',
-        'ocrmypdf/jhove/conf/*.conf',
-        'ocrmypdf/jhove/lib/*.jar'
-    ],
     include_package_data=True,
     zip_safe=False)

test_requirements.txt

@@ -0,0 +1 @@
+pytest>=2.7.2

tests/resources/NOTE.rst

@@ -1,17 +0,0 @@
-All test resources must come from free public domain sources for
-copyright reasons.
-
-Test files do not necessarily produce perfect (or even good) OCR
-results.
-
-+-------------------+--------------------------------------------------------------------------------+
-| File              | Source                                                                         |
-+===================+================================================================================+
-| graph.pdf         | Wikimedia                                                                      |
-+-------------------+--------------------------------------------------------------------------------+
-| c02-22.pdf        | Project Gutenberg: https://www.gutenberg.org/files/76/76-h/images/c02-22.jpg   |
-+-------------------+--------------------------------------------------------------------------------+
-| LinnSequencer.jpg | Wikimedia: https://upload.wikimedia.org/wikipedia/en/b/b7/LinnSequencer_hardware_MIDI_sequencer_brochure_page_2_300dpi.jpg   |
-+-------------------+--------------------------------------------------------------------------------+
-| congress.jpg      | http://www.baxleystamps.com/litho/meiji/courts_1871.jpg                        |
-+-------------------+--------------------------------------------------------------------------------+

tests/resources/README.rst

@@ -0,0 +1,60 @@
+These test files are used in OCRmyPDF's test suite. They do not necessarily produce OCR results
+at all and are not meant as examples of OCR output. Some are even invalid PDFs that might
+crash certain PDF viewers.
+
+
+Files derived from free sources
+===============================
+
+These test resources come from free sources, under either public domain or Creative Commons licenses.
+In some cases they were converted from one image format to another without other changes.
+
++---------------------+--------------------------------------------------------------------------------+
+| File                | Source                                                                         |
++=====================+================================================================================+
+| c02-22.pdf          | `Project Gutenberg`_, Adventures of Huckleberry Finn, page 22                  |
++---------------------+--------------------------------------------------------------------------------+
+| congress.jpg        | `US Congressional Records`_                                                    |
++---------------------+--------------------------------------------------------------------------------+
+| graph.pdf           | `Wikimedia: Pandas text analysis.png`_                                         |
++---------------------+--------------------------------------------------------------------------------+
+| LinnSequencer.jpg   | `Wikimedia: LinnSequencer`_ (Creative Commons Attribution-ShareAlike 3.0)      |
++---------------------+--------------------------------------------------------------------------------+
+
+
+Files generated for this project
+================================
+
+The following test resources were crafted specifically for this project, and can be used
+under the terms of the license in LICENSE.rst.
+
+- blank.pdf (a blank PDF page)
+- cmyk.pdf (a CMYK image created in Photoshop)
+- enormous.pdf (a very lage page)
+- francais.pdf (a page containing French accented characters)
+- invalid.pdf (a PDF file header followed by EOF marker)
+- missing_docinfo.pdf (PDF file with no /DocumentInfo section)
+
+
+Assemblies
+==========
+
+These test resources are assemblies from other previously mentioned files, released under the same license terms as their input files.
+
+- cardinal.pdf (four cardinal directions, rotated copies of LinnSequencer.jpg)
+- ccitt.pdf (LinnSequencer.jpg, converted to CCITT encoding)
+- graph_ocred.pdf (from graph.pdf)
+- jbig2.pdf (congress.jpg, converted to JBIG2 encoding)
+- multipage.pdf (from several other files)
+- palette.pdf (congress.jpg, converted to a 256-color palette)
+- skew.pdf (from c02-22.pdf)
+- skew-encrypted.pdf (skew.pdf with encrypted applied)
+
+
+.. _`Wikimedia: LinnSequencer`: https://upload.wikimedia.org/wikipedia/en/b/b7/LinnSequencer_hardware_MIDI_sequencer_brochure_page_2_300dpi.jpg
+
+.. _`Project Gutenberg`: https://www.gutenberg.org/files/76/76-h/76-h.htm#c2
+
+.. _`US Congressional Records`: http://www.baxleystamps.com/litho/meiji/courts_1871.jpg
+
+.. _`Wikimedia: Pandas text analysis.png`: https://en.wikipedia.org/wiki/File:Pandas_text_analysis.png

tests/resources/invalid.pdf

@@ -0,0 +1,3 @@
+%PDF-1.3
+This is not a valid PDF file
+%%EOF

tests/resources/missing_docinfo.pdf

@@ -0,0 +1,41 @@
+%PDF-1.3
+%¿÷¢þ
+1 0 obj
+<< /Outlines 2 0 R /PageMode /UseNone /Pages 3 0 R /Type /Catalog >>
+endobj
+2 0 obj
+<< /Count 0 /Type /Outlines >>
+endobj
+3 0 obj
+<< /Count 1 /Kids [ 4 0 R ] /Type /Pages >>
+endobj
+4 0 obj
+<< /Contents 5 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 3 0 R /Resources << /Font 6 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] >> /Rotate 0 /Trans << >> /Type /Page >>
+endobj
+5 0 obj
+<< /Length 104 /Filter /FlateDecode >>
+stream
+xœ-Œ1
+ƒ@{O1e´ˆûEI¢ Xþè⢉»[¨÷Wƒ<˜bž`ÎÉŸÖÃ[ÉA
+Ô!å³D;jMN/w*Æðªêyè4¯¸y™ú•É?Ñî~[\dX¢ý¥èÍ®’™…endstream
+endobj
+6 0 obj
+<< /F1 7 0 R >>
+endobj
+7 0 obj
+<< /BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font >>
+endobj
+xref
+0 8
+0000000000 65535 f 
+0000000015 00000 n 
+0000000099 00000 n 
+0000000145 00000 n 
+0000000204 00000 n 
+0000000403 00000 n 
+0000000578 00000 n 
+0000000609 00000 n 
+trailer << /Root 1 0 R /Size 8 /ID [<85bf6c3c41ed927eb43bdfdfff4c0cb5><0304c5f87e589cb7b06f6f52dc6e6707>] >>
+startxref
+716
+%%EOF

tests/spoof/qpdf_dummy_return2.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+import sys
+
+
+def main():
+    print('qpdf dummy')
+    sys.exit(2)
+
+
+if __name__ == '__main__':
+    main()

tests/spoof/tesseract_cache.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+import sys
+import os
+import hashlib
+import shutil
+import subprocess
+
+
+CACHE_PATH = os.path.abspath(os.path.join(
+        os.path.dirname(__file__), '..', 'cache'))
+
+
+def real_tesseract():
+    tess_args = ['tesseract'] + sys.argv[1:]
+    os.execvp("tesseract", tess_args)
+    return  # Not reachable
+
+def main():
+    operation = sys.argv[-1]
+    # For anything unexpected operation, defer to real tesseract binary
+    if operation != 'hocr' and operation != 'pdf' and operation != 'stdout':
+        real_tesseract()
+        return  # Not reachable
+
+    try:
+        os.makedirs(CACHE_PATH)
+    except FileExistsError:
+        pass
+
+    m = hashlib.sha1()
+
+    version = subprocess.check_output(
+        ['tesseract', '--version'],
+        stderr=subprocess.STDOUT)
+
+    m.update(version)
+    m.update(operation.encode())
+
+    try:
+        lang = sys.argv[sys.argv.index('-l') + 1]
+        m.update(lang.encode())
+    except ValueError:
+        pass
+    try:
+        psm = sys.argv[sys.argv.index('-psm') + 1]
+        m.update(psm.encode())
+    except ValueError:
+        pass
+
+    if operation == 'stdout' and psm != '0':
+        real_tesseract()
+        return
+
+    if operation == 'stdout':
+        input_file = sys.argv[-2]
+        output_file = 'stdout'
+    else:
+        input_file = sys.argv[-3]
+        output_file = sys.argv[-2]
+
+    if operation == 'hocr':
+        output_file += '.hocr'
+    elif operation == 'pdf':
+        output_file += '.pdf'
+
+    with open(input_file, 'rb') as f:
+        m.update(f.read())
+    cache_name = os.path.join(CACHE_PATH, m.hexdigest())
+    print(cache_name)
+    if os.path.exists(cache_name):
+        # Cache hit
+        print("Tesseract cache hit", file=sys.stderr)
+        if operation != 'stdout':
+            shutil.copy(cache_name, output_file)
+
+        # Replicate output
+        with open(cache_name + '.stdout', 'r') as f:
+            print(f.read(), end='')
+        with open(cache_name + '.stderr', 'r') as f:
+            print(f.read(), end='', file=sys.stderr)
+        sys.exit(0)
+
+    # Cache miss
+    print("Tesseract cache miss", file=sys.stderr)
+
+    # Call tesseract
+    p = subprocess.Popen(
+            ['tesseract'] + sys.argv[1:],
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+            universal_newlines=True)
+    stdout, stderr = p.communicate()
+
+    with open(cache_name + '.stdout', 'w') as f:
+        f.write(stdout)
+    with open(cache_name + '.stderr', 'w') as f:
+        f.write(stderr)
+    print(stdout, end='')
+    print(stderr, end='', file=sys.stderr)
+
+    # Insert file into cache
+    if output_file != 'stdout':
+        if os.path.exists(output_file):
+            shutil.copy(output_file, cache_name)
+        else:
+            print("Could not find output file", file=sys.stderr)
+    else:
+        open(cache_name, 'w').close()
+
+
+if __name__ == '__main__':
+    main()

tests/spoof/tesseract_noop.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+import sys
+import img2pdf
+from PIL import Image
+
+
+VERSION_STRING = '''tesseract 3.04.00
+ leptonica-1.72
+  libjpeg 8d : libpng 1.6.19 : libtiff 4.0.6 : zlib 1.2.5
+SPOOFED
+'''
+
+HOCR_TEMPLATE = '''<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+  <title></title>
+  <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+  <meta name='ocr-system' content='tesseract 3.02.02' />
+  <meta name='ocr-capabilities' content='ocr_page ocr_carea ocr_par ocr_line ocrx_word'/>
+ </head>
+ <body>
+  <div class='ocr_page' id='page_1' title='image "x.tif"; bbox 0 0 {0} {1}; ppageno 0'>
+   <div class='ocr_carea' id='block_1_1' title="bbox 0 1 {0} {1}">
+    <p class='ocr_par' dir='ltr' id='par_1' title="bbox 0 1 {0} {1}">
+     <span class='ocr_line' id='line_1' title="bbox 0 1 {0} {1}"><span class='ocrx_word' id='word_1' title="bbox 0 1 {0} {1}"> </span>
+     </span>
+    </p>
+   </div>
+  </div>
+ </body>
+</html>'''
+
+
+def main():
+    if sys.argv[1] == '--version':
+        print(VERSION_STRING, file=sys.stderr)
+        sys.exit(0)
+    elif sys.argv[1] == '--list-langs':
+        print('List of available languages (1):\neng', file=sys.stderr)
+        sys.exit(0)
+    elif sys.argv[-1] == 'hocr':
+        inputf = sys.argv[-3]
+        output = sys.argv[-2]
+        with Image.open(inputf) as im, \
+                open(output + '.hocr', 'w', encoding='utf-8') as f:
+            w, h = im.size
+            f.write(HOCR_TEMPLATE.format(str(w), str(h)))
+    elif sys.argv[-1] == 'pdf':
+        inputf = sys.argv[-3]
+        output = sys.argv[-2]
+        pdf_bytes = img2pdf.convert([inputf], dpi=300)
+        with open(output + '.pdf', 'wb') as f:
+            f.write(pdf_bytes)
+    elif sys.argv[-1] == 'stdout':
+        inputf = sys.argv[-2]
+        print("""Orientation: 0
+Orientation in degrees: 0
+Orientation confidence: 100.00
+Script: 1
+Script confidence: 100.00""", file=sys.stderr)
+    else:
+        print("Spoof doesn't understand arguments", file=sys.stderr)
+        print(sys.argv, file=sys.stderr)
+        sys.exit(1)
+
+    sys.exit(0)
+
+
+if __name__ == '__main__':
+    main()

tests/test_hocrtransform.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+# © 2015 James R. Barlow: github.com/jbarlow83
+
+from ocrmypdf import hocrtransform
+from ocrmypdf.tesseract import HOCR_TEMPLATE
+from reportlab.pdfgen.canvas import Canvas
+from PIL import Image
+from tempfile import NamedTemporaryFile
+from contextlib import suppress
+import os
+import shutil
+import pytest
+import img2pdf
+import pytest
+import sys
+
+
+if sys.version_info.major < 3:
+    print("Requires Python 3.4+")
+    sys.exit(1)
+
+TESTS_ROOT = os.path.abspath(os.path.dirname(__file__))
+SPOOF_PATH = os.path.join(TESTS_ROOT, 'spoof')
+PROJECT_ROOT = os.path.dirname(TESTS_ROOT)
+OCRMYPDF = os.path.join(PROJECT_ROOT, 'OCRmyPDF.sh')
+TEST_RESOURCES = os.path.join(PROJECT_ROOT, 'tests', 'resources')
+TEST_OUTPUT = os.environ.get(
+    'OCRMYPDF_TEST_OUTPUT',
+    default=os.path.join(PROJECT_ROOT, 'tests', 'output', 'hocrtransform'))
+
+
+def setup_module():
+    with suppress(FileNotFoundError):
+        shutil.rmtree(TEST_OUTPUT)
+    with suppress(FileExistsError):
+        os.makedirs(TEST_OUTPUT)
+    with open(_make_output('blank.hocr'), 'w') as f:
+        f.write(HOCR_TEMPLATE)
+
+
+def _make_input(input_basename):
+    return os.path.join(TEST_RESOURCES, input_basename)
+
+
+def _make_output(output_basename):
+    return os.path.join(TEST_OUTPUT, output_basename)
+
+
+def test_mono_image():
+    im = Image.new('1', (8, 8), 0)
+    for n in range(8):
+        im.putpixel((n, n), 1)
+    im.save(_make_output('mono.tif'), format='TIFF')
+
+    hocr = hocrtransform.HocrTransform(_make_output('blank.hocr'), 300)
+    hocr.to_pdf(_make_output('mono.pdf'), imageFileName=_make_output('mono.tif'))
+
+
+
+
+

tests/test_main.py

@@ -2,15 +2,16 @@
 # © 2015 James R. Barlow: github.com/jbarlow83
 
 from __future__ import print_function
-from subprocess import Popen, PIPE, check_output
+from subprocess import Popen, PIPE, check_output, check_call
 import os
 import shutil
 from contextlib import suppress
 import sys
-from unittest.mock import patch, create_autospec
 import pytest
 from ocrmypdf.pageinfo import pdf_get_all_pageinfo
 import PyPDF2 as pypdf
+from ocrmypdf import ExitCode
+from ocrmypdf import leptonica
 
 
 if sys.version_info.major < 3:
@@ -18,54 +19,96 @@ if sys.version_info.major < 3:
     sys.exit(1)
 
 TESTS_ROOT = os.path.abspath(os.path.dirname(__file__))
+SPOOF_PATH = os.path.join(TESTS_ROOT, 'spoof')
 PROJECT_ROOT = os.path.dirname(TESTS_ROOT)
 OCRMYPDF = os.path.join(PROJECT_ROOT, 'OCRmyPDF.sh')
 TEST_RESOURCES = os.path.join(PROJECT_ROOT, 'tests', 'resources')
-TEST_OUTPUT = os.path.join(PROJECT_ROOT, 'tests', 'output')
+TEST_OUTPUT = os.environ.get(
+    'OCRMYPDF_TEST_OUTPUT',
+    default=os.path.join(PROJECT_ROOT, 'tests', 'output', 'main'))
 
 
 def setup_module():
     with suppress(FileNotFoundError):
         shutil.rmtree(TEST_OUTPUT)
     with suppress(FileExistsError):
-        os.mkdir(TEST_OUTPUT)
+        os.makedirs(TEST_OUTPUT)
 
 
-def run_ocrmypdf_sh(input_file, output_file, *args):
+def run_ocrmypdf_sh(input_file, output_file, *args, env=None):
     sh_args = ['sh', OCRMYPDF] + list(args) + [input_file, output_file]
     sh = Popen(
         sh_args, close_fds=True, stdout=PIPE, stderr=PIPE,
-        universal_newlines=True)
+        universal_newlines=True, env=env)
     out, err = sh.communicate()
     return sh, out, err
 
 
-def _make_input(input_basename):
+def _infile(input_basename):
     return os.path.join(TEST_RESOURCES, input_basename)
 
 
-def _make_output(output_basename):
-    return os.path.join(TEST_OUTPUT, output_basename)
+def _outfile(output_basename):
+    return os.path.join(TEST_OUTPUT, os.path.basename(output_basename))
 
 
-def check_ocrmypdf(input_basename, output_basename, *args):
-    input_file = _make_input(input_basename)
-    output_file = _make_output(output_basename)
+def check_ocrmypdf(input_basename, output_basename, *args, env=None):
+    input_file = _infile(input_basename)
+    output_file = _outfile(output_basename)
 
-    sh, _, err = run_ocrmypdf_sh(input_file, output_file, *args)
-    assert sh.returncode == 0, err
+    sh, out, err = run_ocrmypdf_sh(input_file, output_file, *args, env=env)
+    if sh.returncode != 0:
+        print('stdout\n======')
+        print(out)
+        print('stderr\n======')
+        print(err)
+    assert sh.returncode == 0
     assert os.path.exists(output_file), "Output file not created"
     assert os.stat(output_file).st_size > 100, "PDF too small or empty"
     return output_file
 
 
-def test_quick():
-    check_ocrmypdf('c02-22.pdf', 'test_quick.pdf')
+def run_ocrmypdf_env(input_basename, output_basename, *args, env=None):
+    input_file = _infile(input_basename)
+    output_file = _outfile(output_basename)
+
+    if env is None:
+        env = os.environ
+
+    p_args = ['ocrmypdf'] + list(args) + [input_file, output_file]
+    p = Popen(
+        p_args, close_fds=True, stdout=PIPE, stderr=PIPE,
+        universal_newlines=True, env=env)
+    out, err = p.communicate()
+    return p, out, err
 
 
-def test_deskew():
+@pytest.fixture
+def spoof_tesseract_noop():
+    env = os.environ.copy()
+    program = os.path.join(SPOOF_PATH, 'tesseract_noop.py')
+    check_call(['chmod', "+x", program])
+    env['OCRMYPDF_TESSERACT'] = program
+    return env
+
+
+@pytest.fixture
+def spoof_tesseract_cache():
+    env = os.environ.copy()
+    program = os.path.join(SPOOF_PATH, "tesseract_cache.py")
+    check_call(['chmod', '+x', program])
+    env['OCRMYPDF_TESSERACT'] = program
+    return env
+
+
+def test_quick(spoof_tesseract_noop):
+    check_ocrmypdf('c02-22.pdf', 'test_quick.pdf', env=spoof_tesseract_noop)
+
+
+def test_deskew(spoof_tesseract_noop):
     # Run with deskew
-    deskewed_pdf = check_ocrmypdf('skew.pdf', 'test_deskew.pdf', '-d')
+    deskewed_pdf = check_ocrmypdf(
+        'skew.pdf', 'test_deskew.pdf', '-d', '-v', '1', env=spoof_tesseract_noop)
 
     # Now render as an image again and use Leptonica to find the skew angle
     # to confirm that it was deskewed
@@ -73,7 +116,7 @@ def test_deskew():
     import logging
     log = logging.getLogger()
 
-    deskewed_png = _make_output('deskewed.png')
+    deskewed_png = _outfile('deskewed.png')
 
     rasterize_pdf(
         deskewed_pdf,
@@ -83,23 +126,42 @@ def test_deskew():
         raster_device='pngmono',
         log=log)
 
-    from ocrmypdf.leptonica import pixRead, pixDestroy, pixFindSkew
-    pix = pixRead(deskewed_png)
-    skew_angle, skew_confidence = pixFindSkew(pix)
-    pix = pixDestroy(pix)
+    from ocrmypdf.leptonica import Pix
+    pix = Pix.read(deskewed_png)
+    skew_angle, skew_confidence = pix.find_skew()
 
     print(skew_angle)
     assert -0.5 < skew_angle < 0.5, "Deskewing failed"
 
 
-def test_clean():
-    check_ocrmypdf('skew.pdf', 'test_clean.pdf', '-c')
+def test_clean(spoof_tesseract_noop):
+    check_ocrmypdf('skew.pdf', 'test_clean.pdf', '-c', env=spoof_tesseract_noop)
 
 
-def test_preserve_metadata():
-    pdf_before = pypdf.PdfFileReader(_make_input('graph.pdf'))
+@pytest.mark.parametrize("pdf,renderer", [
+    ('palette.pdf', 'hocr'),
+    ('palette.pdf', 'tesseract'),
+    ('cmyk.pdf', 'hocr'),
+    ('cmyk.pdf', 'tesseract'),
+    ('ccitt.pdf', 'hocr'),
+    ('ccitt.pdf', 'tesseract'),
+    ('jbig2.pdf', 'hocr'),
+    ('jbig2.pdf', 'tesseract')
+])
+def test_exotic_image(spoof_tesseract_cache, pdf, renderer):
+    check_ocrmypdf(
+        pdf,
+        'test_{0}_{1}.pdf'.format(pdf, renderer),
+        '-dc',
+        '-v', '1',
+        '--pdf-renderer', renderer, env=spoof_tesseract_cache)
 
-    output = check_ocrmypdf('graph.pdf', 'test_metadata_preserve.pdf')
+
+def test_preserve_metadata(spoof_tesseract_noop):
+    pdf_before = pypdf.PdfFileReader(_infile('graph.pdf'))
+
+    output = check_ocrmypdf('graph.pdf', 'test_metadata_preserve.pdf',
+                            env=spoof_tesseract_noop)
 
     pdf_after = pypdf.PdfFileReader(output)
 
@@ -107,16 +169,24 @@ def test_preserve_metadata():
         assert pdf_before.documentInfo[key] == pdf_after.documentInfo[key]
 
 
-def test_override_metadata():
+def test_override_metadata(spoof_tesseract_noop):
+    input_file = _infile('c02-22.pdf')
+    output_file = _outfile('test_override_metadata.pdf')
+
     german = 'Du siehst den Wald vor lauter Bäumen nicht.'
     chinese = '孔子'
     high_unicode = 'U+1030C is: 𐌌'
 
-    pdf = check_ocrmypdf(
-        'c02-22.pdf', 'test_metadata.pdf',
+    p, out, err = run_ocrmypdf_env(
+        input_file, output_file,
         '--title', german,
         '--author', chinese,
-        '--subject', high_unicode)
+        '--subject', high_unicode,
+        env=spoof_tesseract_noop)
+
+    assert p.returncode == ExitCode.ok
+
+    pdf = output_file
 
     out_pdfinfo = check_output(['pdfinfo', pdf], universal_newlines=True)
     lines_pdfinfo = out_pdfinfo.splitlines()
@@ -131,20 +201,20 @@ def test_override_metadata():
     assert pdfinfo.get('Keywords', '') == ''
 
 
-def check_oversample(renderer):
+@pytest.mark.parametrize('renderer', [
+    'hocr',
+    'tesseract',
+    ])
+def test_oversample(spoof_tesseract_cache, renderer):
     oversampled_pdf = check_ocrmypdf(
-        'skew.pdf', 'test_oversample_%s.pdf' % renderer, '--oversample', '300',
-        '--pdf-renderer', renderer)
+        'skew.pdf', 'test_oversample_%s.pdf' % renderer, '--oversample', '350',
+        '-f',
+        '--pdf-renderer', renderer, env=spoof_tesseract_cache)
 
     pdfinfo = pdf_get_all_pageinfo(oversampled_pdf)
 
     print(pdfinfo[0]['xres'])
-    assert abs(pdfinfo[0]['xres'] - 300) < 1
-
-
-def test_oversample():
-    yield check_oversample, 'hocr'
-    yield check_oversample, 'tesseract'
+    assert abs(pdfinfo[0]['xres'] - 350) < 1
 
 
 def test_repeat_ocr():
@@ -152,51 +222,220 @@ def test_repeat_ocr():
     assert sh.returncode != 0
 
 
-def test_force_ocr():
-    out = check_ocrmypdf('graph_ocred.pdf', 'test_force.pdf', '-f')
+def test_force_ocr(spoof_tesseract_cache):
+    out = check_ocrmypdf('graph_ocred.pdf', 'test_force.pdf', '-f',
+                         env=spoof_tesseract_cache)
     pdfinfo = pdf_get_all_pageinfo(out)
     assert pdfinfo[0]['has_text']
 
 
-def test_skip_ocr():
-    check_ocrmypdf('graph_ocred.pdf', 'test_skip.pdf', '-s')
+def test_skip_ocr(spoof_tesseract_cache):
+    check_ocrmypdf('graph_ocred.pdf', 'test_skip.pdf', '-s',
+                   env=spoof_tesseract_cache)
 
 
-def test_argsfile():
-    with open(_make_output('test_argsfile.txt'), 'w') as argsfile:
+def test_argsfile(spoof_tesseract_noop):
+    with open(_outfile('test_argsfile.txt'), 'w') as argsfile:
         print('--title', 'ArgsFile Test', '--author', 'Test Cases',
               sep='\n', end='\n', file=argsfile)
     check_ocrmypdf('graph.pdf', 'test_argsfile.pdf',
-                   '@' + _make_output('test_argsfile.txt'))
+                   '@' + _outfile('test_argsfile.txt'),
+                   env=spoof_tesseract_noop)
 
 
-def check_ocr_timeout(renderer):
+def check_monochrome_correlation(
+        reference_pdf, reference_pageno, test_pdf, test_pageno):
+
+    import ocrmypdf.ghostscript as ghostscript
+    import logging
+
+    gslog = logging.getLogger()
+
+    reference_png = _outfile('{}.ref{:04d}.png'.format(
+        reference_pdf, reference_pageno))
+    test_png = _outfile('{}.test{:04d}.png'.format(
+        test_pdf, test_pageno))
+
+    def rasterize(pdf, pageno, png):
+        if os.path.exists(png):
+            print(png)
+            return
+        ghostscript.rasterize_pdf(
+            pdf,
+            png,
+            xres=100, yres=100,
+            raster_device='pngmono', log=gslog, pageno=pageno)
+
+    rasterize(reference_pdf, reference_pageno, reference_png)
+    rasterize(test_pdf, test_pageno, test_png)
+
+    pix_ref = leptonica.Pix.read(reference_png)
+    pix_test = leptonica.Pix.read(test_png)
+
+    return leptonica.Pix.correlation_binary(pix_ref, pix_test)
+
+
+def test_monochrome_correlation():
+    # Verify leptonica: check that an incorrect rotated image has poor
+    # correlation with reference
+    corr = check_monochrome_correlation(
+        reference_pdf=_infile('cardinal.pdf'),
+        reference_pageno=1,  # north facing page
+        test_pdf=_infile('cardinal.pdf'),
+        test_pageno=3,  # south facing page
+        )
+    assert corr < 0.10
+
+
+@pytest.mark.parametrize('renderer', [
+    'hocr',
+    'tesseract',
+    ])
+def test_autorotate(spoof_tesseract_cache, renderer):
+    import ocrmypdf.ghostscript as ghostscript
+    import logging
+
+    gslog = logging.getLogger()
+
+    # cardinal.pdf contains four copies of an image rotated in each cardinal
+    # direction - these ones are "burned in" not tagged with /Rotate
+    out = check_ocrmypdf('cardinal.pdf', 'test_autorotate_%s.pdf' % renderer,
+                         '-r', '-v', '1', env=spoof_tesseract_cache)
+    for n in range(1, 4+1):
+        correlation = check_monochrome_correlation(
+            reference_pdf=_infile('cardinal.pdf'),
+            reference_pageno=1,
+            test_pdf=out,
+            test_pageno=n)
+        assert correlation > 0.80
+
+
+@pytest.mark.parametrize('renderer', [
+    'hocr',
+    'tesseract',
+    ])
+def test_ocr_timeout(renderer):
     out = check_ocrmypdf('skew.pdf', 'test_timeout_%s.pdf' % renderer,
                          '--tesseract-timeout', '1.0')
     pdfinfo = pdf_get_all_pageinfo(out)
-    assert pdfinfo[0]['has_text'] == False
+    assert not pdfinfo[0]['has_text']
 
 
-def test_ocr_timeout():
-    yield check_ocr_timeout, 'hocr'
-    yield check_ocr_timeout, 'tesseract'
-
-
-def test_skip_big():
+def test_skip_big(spoof_tesseract_cache):
     out = check_ocrmypdf('enormous.pdf', 'test_enormous.pdf',
-                         '--skip-big', '10')
+                         '--skip-big', '10', env=spoof_tesseract_cache)
     pdfinfo = pdf_get_all_pageinfo(out)
-    assert pdfinfo[0]['has_text'] == False
+    assert not pdfinfo[0]['has_text']
 
 
-def check_maximum_options(renderer):
+@pytest.mark.parametrize('renderer', [
+    'hocr',
+    'tesseract',
+    ])
+def test_maximum_options(spoof_tesseract_cache, renderer):
     check_ocrmypdf(
         'multipage.pdf', 'test_multipage%s.pdf' % renderer,
         '-d', '-c', '-i', '-g', '-f', '-k', '--oversample', '300',
         '--skip-big', '10', '--title', 'Too Many Weird Files',
-        '--author', 'py.test', '--pdf-renderer', renderer)
+        '--author', 'py.test', '--pdf-renderer', renderer,
+        env=spoof_tesseract_cache)
+
+
+def test_tesseract_missing_tessdata():
+    env = os.environ.copy()
+    env['TESSDATA_PREFIX'] = '/tmp'
+
+    p, _, err = run_ocrmypdf_env(
+        'graph_ocred.pdf', 'not_a_pdfa.pdf', '-v', '1', '--skip-text', env=env)
+    assert p.returncode == ExitCode.missing_dependency, err
+
+
+def test_invalid_input_pdf():
+    p, out, err = run_ocrmypdf_env(
+        'invalid.pdf', 'wont_be_created.pdf')
+    assert p.returncode == ExitCode.input_file, err
+
+
+def test_blank_input_pdf():
+    p, out, err = run_ocrmypdf_env(
+        'blank.pdf', 'still_blank.pdf')
+    assert p.returncode == ExitCode.ok
+
+
+def test_french(spoof_tesseract_cache):
+    p, out, err = run_ocrmypdf_env(
+        'francais.pdf', 'francais.pdf', '-l', 'fra', env=spoof_tesseract_cache)
+    assert p.returncode == ExitCode.ok, \
+        "This test may fail if Tesseract language packs are missing"
+
+
+def test_klingon():
+    p, out, err = run_ocrmypdf_env(
+        'francais.pdf', 'francais.pdf', '-l', 'klz')
+    assert p.returncode == ExitCode.bad_args
+
+
+def test_missing_docinfo(spoof_tesseract_noop):
+    p, out, err = run_ocrmypdf_env(
+        'missing_docinfo.pdf', 'missing_docinfo.pdf', '-l', 'eng', '-c',
+        env=spoof_tesseract_noop)
+    assert p.returncode == ExitCode.ok, err
+
+
+def test_uppercase_extension(spoof_tesseract_noop):
+    shutil.copy(_infile("skew.pdf"), _infile("UPPERCASE.PDF"))
+    try:
+        check_ocrmypdf("UPPERCASE.PDF", "UPPERCASE_OUT.PDF",
+                       env=spoof_tesseract_noop)
+    finally:
+        os.unlink(_infile("UPPERCASE.PDF"))
+
+
+def test_input_file_not_found():
+    input_file = "does not exist.pdf"
+    sh, out, err = run_ocrmypdf_sh(
+        _infile(input_file),
+        _outfile("will not happen.pdf"))
+    assert sh.returncode == ExitCode.input_file
+    assert (input_file in out or input_file in err)
+
+
+def test_input_file_not_a_pdf():
+    input_file = __file__  # Try to OCR this file
+    sh, out, err = run_ocrmypdf_sh(
+        _infile(input_file),
+        _outfile("will not happen.pdf"))
+    assert sh.returncode == ExitCode.input_file
+    assert (input_file in out or input_file in err)
+
+
+def test_qpdf_repair_fails():
+    env = os.environ.copy()
+    env['OCRMYPDF_QPDF'] = os.path.abspath('./spoof/qpdf_dummy_return2.py')
+    p, out, err = run_ocrmypdf_env(
+        '-v', '1',
+        'c02-22.pdf', 'wont_be_created.pdf', env=env)
+    print(out)
+    print(err)
+    assert p.returncode == ExitCode.input_file
+
+
+def test_encrypted():
+    p, out, err = run_ocrmypdf_env('skew-encrypted.pdf', 'wont_be_created.pdf')
+    assert p.returncode == ExitCode.input_file
+    assert out.find('password')
+
+
+@pytest.mark.parametrize('renderer', [
+    'hocr',
+    'tesseract',
+    ])
+def test_pagesegmode(renderer, spoof_tesseract_cache):
+    check_ocrmypdf(
+        'skew.pdf', 'test_psm_%s.pdf' % renderer,
+        '--tesseract-pagesegmode', '7',
+        '-v', '1',
+        '--pdf-renderer', renderer, env=spoof_tesseract_cache)
+
 
 
-def test_maximum_options():
-    yield check_maximum_options, 'hocr'
-    yield check_maximum_options, 'tesseract'

tests/test_pageinfo.py

@@ -7,22 +7,40 @@ from PIL import Image
 from tempfile import NamedTemporaryFile
 from contextlib import suppress
 import os
-import sys
 import shutil
 import pytest
 import img2pdf
-from pkg_resources import Requirement, resource_filename
+import pytest
+import sys
 
-req = Requirement.parse('ocrmypdf')
 
-TEST_OUTPUT = os.path.join(os.path.dirname(__file__), 'output')
+if sys.version_info.major < 3:
+    print("Requires Python 3.4+")
+    sys.exit(1)
+
+TESTS_ROOT = os.path.abspath(os.path.dirname(__file__))
+SPOOF_PATH = os.path.join(TESTS_ROOT, 'spoof')
+PROJECT_ROOT = os.path.dirname(TESTS_ROOT)
+OCRMYPDF = os.path.join(PROJECT_ROOT, 'OCRmyPDF.sh')
+TEST_RESOURCES = os.path.join(PROJECT_ROOT, 'tests', 'resources')
+TEST_OUTPUT = os.environ.get(
+    'OCRMYPDF_TEST_OUTPUT',
+    default=os.path.join(PROJECT_ROOT, 'tests', 'output', 'pageinfo'))
 
 
 def setup_module():
     with suppress(FileNotFoundError):
         shutil.rmtree(TEST_OUTPUT)
     with suppress(FileExistsError):
-        os.mkdir(TEST_OUTPUT)
+        os.makedirs(TEST_OUTPUT)
+
+
+def _make_input(input_basename):
+    return os.path.join(TEST_RESOURCES, input_basename)
+
+
+def _make_output(output_basename):
+    return os.path.join(TEST_OUTPUT, output_basename)
 
 
 def test_single_page_text():
@@ -49,13 +67,21 @@ def test_single_page_text():
 def test_single_page_image():
     filename = os.path.join(TEST_OUTPUT, 'image-mono.pdf')
 
-    with NamedTemporaryFile() as im_tmp:
+    with NamedTemporaryFile(mode='wb+', suffix='.png') as im_tmp:
         im = Image.new('1', (8, 8), 0)
         for n in range(8):
             im.putpixel((n, n), 1)
         im.save(im_tmp.name, format='PNG')
 
-        pdf_bytes = img2pdf.convert([im_tmp.name], dpi=8)
+        imgsize = ((img2pdf.ImgSize.dpi, 8), (img2pdf.ImgSize.dpi, 8))
+        layout_fun = img2pdf.get_layout_fun(None, imgsize, None, None, None)
+
+        im_tmp.seek(0)
+        im_bytes = im_tmp.read()
+        pdf_bytes = img2pdf.convert(
+                im_bytes, producer="img2pdf", with_pdfrw=False,
+                layout_fun=layout_fun)
+
         with open(filename, 'wb') as pdf:
             pdf.write(pdf_bytes)
 
@@ -99,7 +125,7 @@ def test_single_page_inline_image():
 
 
 def test_jpeg():
-    filename = resource_filename(req, 'tests/resources/c02-22.pdf')
+    filename = _make_input('c02-22.pdf')
 
     pdfinfo = pageinfo.pdf_get_all_pageinfo(filename)