v4.1.3 release notes

Issue #73. The order of operations happens to not matter for scaling
but does matter for translation. We only need scaling to find the DPI,
so the error was not noticed. Mainly useful to other uses of this
library.

.dockerignore

@@ -0,0 +1,31 @@
+*.ipynb
+*.pdf
+*.pyc
+*.rst
+*.sublime*
+*/*.pyc
+*/*/*.pyc
+*/*/*/*.pyc
+*/*/*/*/*.pyc
+*/*/*/*/*/*.pyc
+*/*/*/*/*/*/*.pyc
+*/*/*/*/*/*/*/*.pyc
+.cache/
+.git/
+.ipynb_checkpoints/
+.ruffus_history.sqlite
+bin/
+build/
+dist/
+htmlcov/
+include/
+lib/
+MANIFEST.in
+ocrmypdf.egg-info/
+staging/
+tests/cache/
+tests/output/
+tests/resources/private/
+tmp/
+venv-3.4/
+venv-3.5/

.git_archival.txt

@@ -0,0 +1 @@
+ref-names: $Format:%D$

.gitattributes

@@ -5,4 +5,6 @@
 # (binary is a macro for -text -diff)
 *.jar	binary
 *.pdf	binary
-*.PDF	binary
+*.PDF	binary
+
+.git_archival.txt  export-subst

.gitignore

@@ -1,17 +1,30 @@
-tmp/
-log/
+# Development environment
 *.pyc
-tests/output/
-.ruffus_history.sqlite
 *.sublime-*
-/*.pdf
+venv-*/
+pyvenv.cfg
+
+# Package building
+*.egg-info/
+.cache/
+.eggs/
 build/
 dist/
-*.egg-info/
-venv/
-*/test/output
-bin/
-include/
-lib/
-pip-selfcheck.json
-pyvenv.cfg
+wheelhouse/
+
+# Automatically generated files
+ocrmypdf/lib/_*.py
+ocrmypdf/version.py
+
+# Code coverage
+.coverage
+htmlcov/
+
+# Testing
+log/
+/*.pdf
+.ipynb_checkpoints/
+tests/cache/
+tests/output/
+tests/resources/private
+tmp/

.travis.yml

@@ -0,0 +1,44 @@
+language: python
+
+cache: pip
+
+cache:
+    directories:
+        - $HOME/.ccache
+        - packages
+        - tests/cache
+
+python:
+    - 3.4
+
+before_cache:
+    - rm -f $HOME/.cache/pip/log/debug.log
+
+before_install:
+    # Ubuntu packages
+    - sudo add-apt-repository ppa:evl.ms/precise -y  # for Ghostscript 9.15
+    - sudo add-apt-repository ppa:lyrasis/precise-backports -y  # for Tesseract 3.03
+    - sudo add-apt-repository ppa:b-eltzner/qpdfview-exp -y  # for QPDF 5
+    - sudo add-apt-repository ppa:itachi-san/ffmpeg -y  # for libav 11.2 (for unpaper)
+    - sudo apt-get update -qq  # must go after all add-apt-repo
+    - sudo apt-get install -y ghostscript tesseract-ocr tesseract-ocr-deu tesseract-ocr-eng tesseract-ocr-fra qpdf poppler-utils libavformat56 libavcodec56 libavutil54 libffi-dev
+
+    # pip
+    - pip install --upgrade pip
+
+    # Download, make and install unpaper (using ccache)
+    - mkdir -p packages
+    - "[ -f packages/unpaper_6.1-1.deb ] || wget -q https://dl.dropboxusercontent.com/u/28971240/unpaper_6.1-1.deb -O packages/unpaper_6.1-1.deb"
+    - sudo dpkg -i packages/unpaper_6.1-1.deb
+
+install:
+    - pip install -r requirements.txt
+    - pip install -r test_requirements.txt
+
+script:
+    - python setup.py clean
+    - python setup.py install
+    - python setup.py test
+
+os:
+    - linux

Dockerfile

@@ -0,0 +1,91 @@
+# OCRmyPDF
+#
+# VERSION               3.2
+FROM      debian:stretch
+MAINTAINER James R. Barlow <jim@purplerock.ca>
+
+# Add unprivileged user
+RUN useradd docker \
+  && mkdir /home/docker \
+  && chown docker:docker /home/docker
+
+# Update system and install our dependencies
+# If this command takes too Docker hub's automated build will timeout,
+# so try it in portions
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  locales \
+  python3 \
+  python3-pip \
+  python3-venv \
+  python3-reportlab \
+  python3-pil \
+  python3-wheel
+
+RUN apt-get install -y --no-install-recommends \
+  unpaper \
+  qpdf \
+  poppler-utils \
+  tesseract-ocr \
+  tesseract-ocr-deu tesseract-ocr-spa tesseract-ocr-eng tesseract-ocr-fra
+
+RUN apt-get install -qy --no-install-recommends \
+  libffi-dev \
+  libpython3-dev \
+  gcc
+
+# Install Ghostscript from Debian sid to work around JPEG 2000 issue in
+# Debian stretch libgs9 or gs 9.16~dfsg-2.1
+
+COPY ./share/etc-apt-sources.list /etc/apt/sources.list
+
+RUN apt-get update && apt-get install -y ghostscript/sid
+
+
+# Enforce UTF-8
+# Borrowed from https://index.docker.io/u/crosbymichael/python/ 
+RUN dpkg-reconfigure locales && \
+  locale-gen C.UTF-8 && \
+  /usr/sbin/update-locale LANG=C.UTF-8
+ENV LC_ALL C.UTF-8
+
+
+# Set up a Python virtualenv and take all of the system packages, so we can
+# rely on the platform packages rather than importing GCC and compiling them
+RUN pyvenv /appenv \
+  && pyvenv --system-site-packages /appenv
+
+COPY . /application/
+
+# Replace stock Tesseract 3.04.00 font with improved sharp2.ttf that resolves
+# issues in many PDF viewers.
+# Discussion is in https://github.com/tesseract-ocr/tesseract/issues/182
+COPY ./share/sharp2.ttf /usr/share/tesseract-ocr/tessdata/pdf.ttf
+RUN chmod 644 /usr/share/tesseract-ocr/tessdata/pdf.ttf
+
+# Set this here to force a docker version, allowing non-tagged versions to
+# be built
+# ENV SETUPTOOLS_SCM_PRETEND_VERSION=v3.3.0
+
+# Install application and dependencies
+# In this arrangement Pillow and reportlab will be provided by the system
+# Even though ocrmypdf is locally present, pull from PyPI because
+# Dockerhub and setuptools_scm clash
+RUN . /appenv/bin/activate; \
+  pip install --upgrade pip \
+  && pip install ocrmypdf \
+  && pip install --no-cache-dir -r /application/test_requirements.txt
+
+# Remove the junk
+RUN apt-get remove -qy gcc
+RUN apt-get autoremove -y && apt-get clean -y
+RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/*
+
+USER docker
+WORKDIR /home/docker
+
+ENV OCRMYPDF_TEST_OUTPUT=/tmp/test-output
+ENV OCRMYPDF_SHARP_TTF=1
+
+# Must use array form of ENTRYPOINT
+# Non-array form does not append other arguments, because that is "intuitive"
+ENTRYPOINT ["/application/docker-wrapper.sh"]

Dockerfile.polyglot

@@ -0,0 +1,16 @@
+# OCRmyPDF polyglot
+#
+# VERSION               3.2
+FROM      jbarlow83/ocrmypdf:latest
+MAINTAINER James R. Barlow <jim@purplerock.ca>
+
+# Update system and install our dependencies
+USER root
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  tesseract-ocr-all 
+
+USER docker
+
+# Must use array form of ENTRYPOINT
+# Non-array form does not append other arguments, because that is "intuitive"
+ENTRYPOINT ["/application/docker-wrapper.sh"]

MANIFEST.in

@@ -1,4 +1,4 @@
-recursive-include ocrmypdf/jhove/bin *.jar
-recursive-include ocrmypdf/jhove/lib *.jar
-recursive-include ocrmypdf/jhove/conf *.conf
 recursive-exclude tests/output *
+include requirements.txt
+include test_requirements.txt
+include share/*

README.rst

@@ -8,23 +8,21 @@ Main features
 -------------
 
 -  Generates a searchable
-   `PDF/A <https://en.wikipedia.org/?title=PDF/A>`__ file from a PDF
-   file containing only images
--  Places OCRed text accurately below the image to ease copy / paste
+   `PDF/A <https://en.wikipedia.org/?title=PDF/A>`__ file from a regular PDF
+-  Places OCR text accurately below the image to ease copy / paste
 -  Keeps the exact resolution of the original embedded images
-
-   -  or if requested oversamples the images before OCRing so as to get
-      better results
-
+-  When possible, inserts OCR information as a "lossless" operation without rendering vector information
+-  Keeps file size about the same
 -  If requested deskews and/or cleans the image before performing OCR
--  Validates the generated file against the PDF/A-1b specification using
-   `JHOVE <http://jhove.sourceforge.net/>`__
+-  Validates input and output files
 -  Provides debug mode to enable easy verification of the OCR results
--  Processes several pages in parallel when more than one CPU core is
+-  Processes pages in parallel when more than one CPU core is
    available
--  Uses Tesseract OCR engine
+-  Uses `Tesseract OCR <https://github.com/tesseract-ocr/tesseract>`__ engine
+-  Supports the `39 languages <https://code.google.com/p/tesseract-ocr/downloads/list>`__ recognized by Tesseract
+-  Battle-tested on thousands of PDFs, a test suite and continuous integration
 
-For details: please consult the `release notes <RELEASE_NOTES.rst>`__
+For details: please consult the `release notes <RELEASE_NOTES.rst>`__.
 
 Motivation
 ----------
@@ -43,35 +41,191 @@ Linux/UNIX: I found many, but none of them were really satisfying.
 ... so I decided to develop my own tool (using various existing scripts
 as an inspiration)
 
-Install
--------
+Installation
+------------
 
-Download OCRmyPDF here: https://github.com/fritz-hh/OCRmyPDF/releases
+Download OCRmyPDF here: https://github.com/jbarlow83/OCRmyPDF/releases
 
-You can install it within a virtual environment or system-wide. 
+You can install it to a Python virtual environment or system-wide. 
 
-To install the HEAD revision from sources in development mode::
+Debian and Ubuntu
+~~~~~~~~~~~~~~~~~
 
-   git clone -b master https://github.com/fritz-hh/OCRmyPDF.git
-   cd OCRmyPDF
-   pip3 install -e .
+Users of Debian 9 or later or Ubuntu 16.10 or later may simply
+``apt-get install ocrmypdf``.
+
+Installing the Docker image
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+For many users, installing the Docker image will be easier than installing all of OCRmyPDF's dependencies. For Windows, it is the only option.
+
+If you have `Docker <https://docs.docker.com/>`__ installed on your system, you can install
+a Docker image of the latest release.
+
+Follow the Docker installation instructions for your platform.  If you can run this command
+successfully, your system is ready to download and execute the image::
+
+   docker run hello-world
    
-On certain Linux/UNIX platforms such as Ubuntu, you may need to use 
+OCRmyPDF will use all available CPU cores.  By default, the VirtualBox machine instance on Windows and OS X has only a single CPU core enabled. Use the VirtualBox Manager to determine the name of your Docker engine host, and then follow these optional steps to enable multiple CPUs::
+
+   # Optional step for Mac OS X users
+   docker-machine stop "yourVM"
+   VBoxManage modifyvm "yourVM" --cpus 2  # or whatever number of core is desired
+   docker-machine start "yourVM"
+   eval $(docker-machine env "yourVM")
+
+Assuming you have a Docker engine running somewhere, you can run these commands to download
+the image::
+
+   docker pull jbarlow83/ocrmypdf
+
+Then tag it to give a more convenient name, just ocrmypdf::
+
+   docker tag jbarlow83/ocrmypdf ocrmypdf
+
+This image contains language packs for English, French, Spanish and German. The alternative "polyglot" image provides `all available language packs <https://github.com/tesseract-ocr/tesseract/blob/master/doc/tesseract.1.asc#languages>`__::
+
+   # Alternative step: If you need all language packs
+   docker pull jbarlow83/ocrmypdf-polyglot
+   docker tag jbarlow83/ocrmypdf-polyglot ocrmypdf
+
+You can then run ocrmypdf using the command::
+
+   docker run ocrmypdf --help
+  
+To execute the OCRmyPDF on a local file, you must `provide a writable volume to the Docker image <https://docs.docker.com/userguide/dockervolumes/>`__, such as this in this template::
+
+   docker run -v "$(pwd):/home/docker" <other docker arguments>   ocrmypdf <your arguments to ocrmypdf>
+
+In this worked example, the current working directory contains an input file called ``test.pdf`` and the output will go to ``output.pdf``:: 
+
+   docker run -v "$(pwd):/home/docker"   ocrmypdf --skip-text test.pdf output.pdf
+
+Note that ``ocrmypdf`` has its own separate ``-v VERBOSITYLEVEL`` argument to control debug verbosity. All Docker arguments should before the ``ocrmypdf`` image name and all arguments to ``ocrmypdf`` should be listed after.
+
+
+Installing on Mac OS X
+~~~~~~~~~~~~~~~~~~~~~~
+
+These instructions probably work on all Mac OS X versions later than 10.7 (Lion). OCRmyPDF is known to work on Yosemite and El Capitan, and regularly tested on El Capitan.
+
+If it's not already present, `install Homebrew <http://brew.sh/>`__.
+
+Update Homebrew::
+
+   brew update
+   
+Install or upgrade the required Homebrew packages, if any are missing::
+
+   brew install libpng openjpeg jbig2dec     # image libraries
+   brew install qpdf
+   brew install ghostscript
+   brew install python3
+   brew install libxml2 libffi leptonica
+   brew install unpaper    # optional
+   brew install tesseract
+   
+Update the homebrew pip and install Pillow::
+
+   pip3 install --upgrade pip
+   pip3 install --upgrade pillow
+
+You can then install OCRmyPDF from PyPI::
+
+   pip3 install ocrmypdf
+
+The command line program should now be available::
+
+   ocrmypdf --help
+
+Installing on Ubuntu 14.04 LTS
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Installing on Ubuntu 14.04 LTS (trusty) is more difficult than other options, because of certain bugs in Python package installation.
+
+Update apt-get::
+
+   sudo apt-get update
+   sudo apt-get upgrade
+   
+Install system dependencies::
+
+   sudo apt-get install \
+      zlib1g-dev \
+      libjpeg-dev \
+      libffi-dev \
+      ghostscript \
+      tesseract-ocr \
+      qpdf \
+      unpaper \
+      python3-pip \
+      python3-pil \
+      python3-pytest \
+      python3-reportlab
+
+If you wish install OCRmyPDF to the system Python, then install as follows (note this installs new packages
+into your system Python, which could interfere with other programs)::
+
+   sudo pip3 install ocrmypdf
+   
+If you wish to install OCRmyPDF to a virtual environment to isolate system Python from modified, you can
+follow these steps.  This includes a workaround `for a known, unresolved issue in Ubuntu 14.04's ensurepip
+package <http://www.thefourtheye.in/2014/12/Python-venv-problem-with-ensurepip-in-Ubuntu.html>`__::
+
+   sudo apt-get install python3-venv
+   python3 -m venv venv-ocrmypdf --without-pip
+   source venv-ocrmypdf/bin/activate
+   wget -O - -o /dev/null https://bootstrap.pypa.io/get-pip.py | python
+   deactivate
+   pyvenv --system-site-packages venv-ocrmypdf
+   source venv-ocrmypdf/bin/activate
+   pip install ocrmypdf
+
+Ubuntu 14.04 only installs ``unpaper`` version 0.4.2, which is not supported by OCRmyPDF because it is produces invalid output. This program is an optional dependency, and provides page deskewing and cleaning. See `Dockerfile <Dockerfile>`__ for an example of how to building unpaper 6.1 from source. If you choose to install unpaper later, OCRmyPDF will use the foremost version on the system PATH.
+
+Installing on Windows
+~~~~~~~~~~~~~~~~~~~~~
+
+Direct installation on Windows is not possible.  Install the Docker container as described above.  Ensure that your command prompt can run the docker "hello world" container.
+
+The command line syntax to run ocrmypdf from a command prompt will resemble::
+
+   docker run -v /c/Users/sampleuser:/home/docker ocrmypdf --skip-text test.pdf output.pdf
+
+where /c/Users/sampleuser is a Unix representation of the Windows path C:\Users\sampleuser, assuming a user named "sampleuser" is running ocrmypdf on a file in their home directory, and the files "test.pdf" and "output.pdf" are in the sampleuser folder. The Windows user must have read and write permissions.
+
+      
+Installing HEAD revision from sources
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you have ``git`` and ``python3.4`` or ``python3.5`` installed, you can install from source. When the ``pip`` installer runs,
+it will alert you if dependencies are missing.
+
+To install the HEAD revision from sources in the current Python 3 environment::
+
+   pip3 install git+https://github.com/jbarlow83/OCRmyPDF.git
+
+Or, to install in `development mode <https://pythonhosted.org/setuptools/setuptools.html#development-mode>`__,  allowing customization of OCRmyPDF, use the ``-e`` flag::
+
+   pip3 install -e git+https://github.com/jbarlow83/OCRmyPDF.git
+   
+On certain Linux distributions such as Ubuntu, you may need to use 
 run the install command as superuser::
 
-   sudo pip3 install -e .
+   sudo pip3 install [-e] git+https://github.com/jbarlow83/OCRmyPDF.git
    
 Note that this will alter your system's Python distribution. If you prefer 
 to not install as superuser, you can install the package in a Python virtual environment::
 
-   git clone -b master https://github.com/fritz-hh/OCRmyPDF.git
+   git clone -b master https://github.com/jbarlow83/OCRmyPDF.git
    pyvenv venv
    source venv/bin/activate
    cd OCRmyPDF
-   pip3 install -e .
+   pip3 install .
 
-If your platform does not have ``pip3``, make sure that Python 3.4+ and the `pip` 
-package are installed.
+However, ``ocrmypdf`` will only be accessible on the system PATH after
+you activate the virtual environment.
 
 To run the program::
    
@@ -82,6 +236,18 @@ need to be installed. The script requires specific versions of the
 dependencies. Older version than the ones mentioned in the release notes
 are likely not to be compatible to OCRmyPDF.
 
+Languages
+---------
+
+OCRmyPDF uses Tesseract for OCR, and relies on its language packs. For Linux users,
+you can often find packages that provide language packs::
+
+   # Debian/Ubuntu users
+   sudo apt-get install tesseract-ocr-chi-sim
+   
+You can then pass the ``-l LANG`` argument to OCRmyPDF to give a hint as to what languages it should search for. Multiple
+languages can be requested.
+
 Support
 -------
 
@@ -89,22 +255,21 @@ In case you detect an issue, please:
 
 -  Check if your issue is already known
 -  If no problem report exists on github, please create one here:
-   https://github.com/fritz-hh/OCRmyPDF/issues
+   https://github.com/jbarlow83/OCRmyPDF/issues
 -  Describe your problem thoroughly
 -  Append the console output of the script when running the debug mode
-   (-v 1 option)
+   (``-v 1`` option)
 -  If possible provide your input PDF file as well as the content of the
-   temporary folder (using a file sharing service like
-   www.file-upload.net)
+   temporary folder (using a file sharing service like Dropbox)
 
 Press & Media
 -------------
 
--  `c't 1-2014, page 59 <http://www.heise.de/ct/inhalt/2014/1/58/>`__:
+-  `c't 1-2014, page 59 <http://heise.de/-2279695>`__:
    Detailed presentation of OCRmyPDF v1.0 in the leading German IT
    magazine c't
 -  `heise Open Source, 09/2014: Texterkennung mit
-   OCRmyPDF <http://www.heise.de/-2356670>`__
+   OCRmyPDF <http://heise.de/-2356670>`__
 
 Disclaimer
 ----------

RELEASE_NOTES.rst

@@ -3,7 +3,194 @@ RELEASE NOTES
 
 Please always read this file before installing the package
 
-Download software here: https://github.com/fritz-hh/OCRmyPDF/tags
+Download software here: https://github.com/jbarlow83/OCRmyPDF/tags
+
+
+v4.1.3:
+=======
+
+-  More helpful error message for PDFs with version 4 security handler
+-  Update usage instructions for Windows/Docker users
+-  Fix order of operations for matrix multiplication (no effect on most users)
+-  Add a few leptonica wrapper functions (no effect on most users)
+
+
+v4.1.2:
+=======
+
+-  Replace IEC sRGB ICC profile with Debian's sRGB (from icc-profiles-free) which is more compatible with the MIT license
+-  More helpful error message for an error related to certain types of malformed PDFs
+
+
+v4.1:
+=====
+
+-  ``--rotate-pages`` now only rotates pages when reasonably confidence in the orientation. This behavior can be adjusted with the new argument ``--rotate-pages-threshold``
+-  Fixed problems in error checking if ``unpaper`` is uninstalled or missing at run-time
+-  Fixed problems with "RethrownJobError" errors during error handling that suppressed the useful error messages
+
+
+v4.0.7:
+=======
+
+-  Minor correction to Ghostscript output settings
+
+
+v4.0.6:
+=======
+
+-  Update install instructions
+-  Provide a sRGB profile instead of using Ghostscript's
+
+
+v4.0.5:
+=======
+
+Fixes
+-----
+
+-  Remove some verbose debug messages from v4.0.4
+-  Fixed temporary that wasn't being deleted
+-  DPI is now calculated correctly for cropped images, along with other image transformations
+-  Inline images are now checked during DPI calculation instead of rejecting the image
+
+v4.0.4:
+=======
+
+Released with verbose debug message turned on. Do not use. Skip to v4.0.5.
+
+
+v4.0.3:
+=======
+
+New features
+------------
+
+-  Page orientations detected are now reported in a summary comment
+
+
+Fixes
+-----
+
+-  Show stack trace if unexpect errors occur
+-  Treat "too few characters" error message from Tesseract as a reason to skip that page rather than
+   abort the file
+-  Docker: fix blank JPEG2000 issue by insisting on Ghostscript versions that have this fixed
+
+
+v4.0.2:
+=======
+
+Fixes
+-----
+
+-  Fixed compatibility with Tesseract 3.04.01 release, particularly its different way of outputting
+   orientation information
+-  Improved handling of Tesseract errors and crashes
+-  Fixed use of chmod on Docker that broke most test cases
+
+
+v4.0.1:
+=======
+
+Fixes
+-----
+
+-  Fixed a KeyError if tesseract fails to find page orientation information
+
+
+v4.0:
+=====
+
+New features
+------------
+
+-  Automatic page rotation (``-r``) is now available. It uses ignores any prior rotation information
+   on PDFs and sets rotation based on the dominant orientation of detectable text. This feature is
+   fairly reliable but some false positives occur especially if there is not much text to work with. (#4) 
+-  Deskewing is now performed using Leptonica instead of unpaper. Leptonica is faster and more reliable
+   at image deskewing than unpaper.
+
+
+Fixes
+-----
+
+-  Fixed an issue where lossless reconstruction could cause some pages to be appear incorrectly
+   if the page was rotated by the user in Acrobat after being scanned (specifically if it a /Rotate tag)
+-  Fixed an issue where lossless reconstruction could misalign the graphics layer with respect to
+   text layer if the page had been cropped such that its origin is not (0, 0) (#49)
+
+
+Changes
+-------
+
+-  Logging output is now much easier to read
+-  ``--deskew`` is now performed by Leptonica instead of unpaper (#25)
+-  libffi is now required
+-  Some changes were made to the Docker and Travis build environments to support libffi
+-  ``--pdf-renderer=tesseract`` now displays a warning if the Tesseract version is less than 3.04.01,
+   the planned release that will include fixes to an important OCR text rendering bug in Tesseract 3.04.00.
+   You can also manually install ./share/sharp2.ttf on top of pdf.ttf in your Tesseract tessdata folder
+   to correct the problem.
+
+
+v3.2.1:
+=======
+
+Changes
+-------
+
+-  Fixed issue #47 "convert() got and unexpected keyword argument 'dpi'" by upgrading to img2pdf 0.2
+-  Tweaked the Dockerfiles
+
+
+v3.2:
+=====
+
+New features
+------------
+
+-  Lossless reconstruction: when possible, OCRmyPDF will inject text layers without 
+   otherwise manipulating the content and layout of a PDF page. For example, a PDF containing a mix
+   of vector and raster content would see the vector content preserved. Images may still be transcoded
+   during PDF/A conversion.  (``--deskew`` and ``--clean-final`` disable this mode, necessarily.)
+-  New argument ``--tesseract-pagesegmode`` allows you to pass page segmentation arguments to Tesseract OCR.
+   This helps for two column text and other situations that confuse Tesseract.
+-  Added a new "polyglot" version of the Docker image, that generates Tesseract with all languages packs installed,
+   for the polyglots among us. It is much larger.
+
+Changes
+-------
+
+-  JPEG transcoding quality is now 95 instead of the default 75. Bigger file sizes for less degradation.
+
+
+
+v3.1.1:
+=======
+
+Changes
+-------
+
+-  Fixed bug that caused incorrect page size and DPI calculations on documents with mixed page sizes
+
+v3.1:
+=====
+
+Changes
+-------
+
+-  Default output format is now PDF/A-2b instead of PDF/A-1b
+-  Python 3.5 and OS X El Capitan are now supported platforms - no changes were
+   needed to implement support
+-  Improved some error messages related to missing input files
+-  Fixed issue #20 - uppercase .PDF extension not accepted
+-  Fixed an issue where OCRmyPDF failed to text that certain pages contained previously OCR'ed text, 
+   such as OCR text produced by Tesseract 3.04
+-  Inserts /Creator tag into PDFs so that errors can be traced back to this project
+-  Added new option ``--pdf-renderer=auto``, to let OCRmyPDF pick the best PDF renderer. 
+   Currently it always chooses the 'hocrtransform' renderer but that behavior may change.
+-  Set up Travis CI automatic integration testing
 
 v3.0:
 =====
@@ -11,27 +198,33 @@ v3.0:
 New features
 ------------
 
--  Easier installation with Python's package manager 
+-  Easier installation with a Docker container or Python's ``pip`` package manager 
+-  Eliminated many external dependencies, so it's easier to setup
 -  Now installs ``ocrmypdf`` to ``/usr/local/bin`` or equivalent for system-wide
-   access
--  Tesseract 3.03 PDF page can be used instead for better positioning
-   of recognized text (``--pdf-renderer tesseract``)
+   access and easier typing
 -  Improved command line syntax and usage help (``--help``)
+-  Tesseract 3.03+ PDF page rendering can be used instead for better positioning
+   of recognized text (``--pdf-renderer tesseract``)
 -  PDF metadata (title, author, keywords) are now transferred to the 
    output PDF
 -  PDF metadata can also be set from the command line (``--title``, etc.)
+-  Automatic repairs malformed input PDFs if possible
 -  Added test cases to confirm everything is working
 -  Added option to skip extremely large pages that take too long to OCR and are 
    often not OCRable (e.g. large scanned maps or diagrams); other pages are still
    processed (``--skip-big``)
 -  Added option to kill Tesseract OCR process if it seems to be taking too long on
    a page, while still processing other pages (``--tesseract-timeout``)
+-  Less common colorspaces (CMYK, palette) are now supported by conversion to RGB
+-  Multiple images on the same PDF page are now supported
 
 Changes
 -------
 
 -  New, robust rewrite in Python 3.4+ with ruffus_ pipelines
--  Now uses Ghostscript 9.14's improved color conversion model
+-  Now uses Ghostscript 9.14's improved color conversion model to preserve PDF colors
+-  OCR text is now rendered in the PDF as invisible text. Previous versions of OCRmyPDF
+   incorrectly rendered visible text with an image on top.
 -  All "tasks" in the pipeline can be executed in parallel on any
    available CPUs, increasing performance
 -  The ``-o DPI`` argument has been phased out, in favor of ``--oversample DPI``, in
@@ -45,6 +238,8 @@ Changes
    - Poppler
    - MuPDF_ tools
    - shell scripts
+   - Java and JHOVE_
+   - libxml2
 
 -  Some new external dependencies are required or optional, compared to v2.x:
 
@@ -59,11 +254,39 @@ Changes
 .. _MuPDF: http://mupdf.com/docs/
 .. _qpdf: http://qpdf.sourceforge.net/
 .. _Unpaper: https://github.com/Flameeyes/unpaper
-
+.. _JHOVE: http://jhove.sourceforge.net/
 
 Release candidates
 ------------------
 
+-  rc9:
+
+   - fix issue #118: report error if ghostscript iccprofiles are missing
+   - fixed another issue related to #111: PDF rasterized to palette file
+   - add support image files with a palette
+   - don't try to validate PDF file after an exception occurs
+
+-  rc8:
+
+   - fix issue #111: exception thrown if PDF is missing DocumentInfo dictionary
+
+-  rc7:
+
+   - fix error when installing direct from pip, "no such file 'requirements.txt'"
+
+-  rc6:
+
+   - dropped libxml2 (Python lxml) since Python 3's internal XML parser is sufficient
+   - set up Docker container
+   - fix Unicode errors if recognized text contains Unicode characters and system locale is not UTF-8
+
+-  rc5:
+
+   - dropped Java and JHOVE in favour of qpdf
+   - improved command line error output
+   - additional tests and bug fixes
+   - tested on Ubuntu 14.04 LTS
+
 -  rc4:
 
    - dropped MuPDF in favour of qpdf
@@ -106,12 +329,18 @@ Fixes
 
 -  Handling of filenames containing spaces: fixed
 
-Notes
------
+Notes and known issues
+----------------------
 
 -  Some dependencies may work with lower versions than tested, so try
    overriding dependencies if they are "in the way" to see if they work.
 
+-  ``--pdf-renderer tesseract`` will output files with an incorrect page size in Tesseract 3.03,
+   due to a bug in Tesseract.
+
+-  PDF files containing "inline images" are not supported and won't be for the 3.0 release. Scanned
+   images almost never contain inline images.
+
 
 v2.2-stable (2014-09-29):
 =========================

docker-wrapper.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+. /appenv/bin/activate
+cd /home/docker
+exec ocrmypdf "$@"

ocrmypdf/__init__.py

@@ -0,0 +1,24 @@
+from enum import IntEnum
+import os
+
+
+class ExitCode(IntEnum):
+    ok = 0
+    bad_args = 1
+    input_file = 2
+    missing_dependency = 3
+    invalid_output_pdfa = 4
+    file_access_error = 5
+    already_done_ocr = 6
+    child_process_error = 7
+    encrypted_pdf = 8
+    other_error = 15
+
+
+def get_program(name):
+    envvar = 'OCRMYPDF_' + name.upper()
+    return os.environ.get(envvar, name)
+
+
+def page_number(input_file):
+    return int(os.path.basename(input_file)[0:6])

ocrmypdf/ghostscript.py

@@ -4,14 +4,21 @@
 from tempfile import NamedTemporaryFile
 from subprocess import Popen, PIPE, check_call
 from shutil import copy
+from . import get_program
+from .pdfa import SRGB_ICC_PROFILE
 
 
-def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log):
+def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
+                  pageno=1):
     with NamedTemporaryFile(delete=True) as tmp:
         args_gs = [
-            'gs',
-            '-dBATCH', '-dNOPAUSE',
+            get_program('gs'),
+            '-dQUIET',
+            '-dBATCH',
+            '-dNOPAUSE',
             '-sDEVICE=%s' % raster_device,
+            '-dFirstPage=%i' % pageno,
+            '-dLastPage=%i' % pageno,
             '-o', tmp.name,
             '-r{0}x{1}'.format(str(xres), str(yres)),
             input_file
@@ -34,17 +41,18 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log):
 def generate_pdfa(pdf_pages, output_file, threads=1):
     with NamedTemporaryFile(delete=True) as gs_pdf:
         args_gs = [
-            "gs",
+            get_program("gs"),
             "-dQUIET",
             "-dBATCH",
             "-dNOPAUSE",
             '-dNumRenderingThreads=' + str(threads),
             "-sDEVICE=pdfwrite",
+            "-dAutoRotatePages=/None",
             "-sColorConversionStrategy=/RGB",
             "-sProcessColorModel=DeviceRGB",
-            "-dPDFA",
+            "-dJPEGQ=95",
+            "-dPDFA=2",
             "-sPDFACompatibilityPolicy=2",
-            "-sOutputICCProfile=srgb.icc",
             "-sOutputFile=" + gs_pdf.name,
         ]
         args_gs.extend(pdf_pages)

ocrmypdf/hocrtransform.py

@@ -1,4 +1,4 @@
-#!/usr/local/bin/python3
+#!/usr/bin/env python3
 ##############################################################################
 # Copyright (c) 2013-14: fritz-hh from Github
 #   (https://github.com/fritz-hh)
@@ -9,7 +9,7 @@
 ##############################################################################
 from reportlab.pdfgen.canvas import Canvas
 from reportlab.lib.units import inch
-from lxml import etree as ElementTree
+from xml.etree import ElementTree
 from PIL import Image
 from collections import namedtuple
 import re
@@ -35,8 +35,7 @@ class HocrTransform():
         self.dpi = dpi
         self.boxPattern = re.compile(r'bbox((\s+\d+){4})')
 
-        self.hocr = ElementTree.ElementTree()
-        self.hocr.parse(hocrFileName)
+        self.hocr = ElementTree.parse(hocrFileName)
 
         # if the hOCR file has a namespace, ElementTree requires its use to
         # find elements

ocrmypdf/jhove/COPYING

@@ -1,502 +0,0 @@
-		  GNU LESSER GENERAL PUBLIC LICENSE
-		       Version 2.1, February 1999
-
- Copyright (C) 1991, 1999 Free Software Foundation, Inc.
-     59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
-[This is the first released version of the Lesser GPL.  It also counts
- as the successor of the GNU Library Public License, version 2, hence
- the version number 2.1.]
-
-			    Preamble
-
-  The licenses for most software are designed to take away your
-freedom to share and change it.  By contrast, the GNU General Public
-Licenses are intended to guarantee your freedom to share and change
-free software--to make sure the software is free for all its users.
-
-  This license, the Lesser General Public License, applies to some
-specially designated software packages--typically libraries--of the
-Free Software Foundation and other authors who decide to use it.  You
-can use it too, but we suggest you first think carefully about whether
-this license or the ordinary General Public License is the better
-strategy to use in any particular case, based on the explanations below.
-
-  When we speak of free software, we are referring to freedom of use,
-not price.  Our General Public Licenses are designed to make sure that
-you have the freedom to distribute copies of free software (and charge
-for this service if you wish); that you receive source code or can get
-it if you want it; that you can change the software and use pieces of
-it in new free programs; and that you are informed that you can do
-these things.
-
-  To protect your rights, we need to make restrictions that forbid
-distributors to deny you these rights or to ask you to surrender these
-rights.  These restrictions translate to certain responsibilities for
-you if you distribute copies of the library or if you modify it.
-
-  For example, if you distribute copies of the library, whether gratis
-or for a fee, you must give the recipients all the rights that we gave
-you.  You must make sure that they, too, receive or can get the source
-code.  If you link other code with the library, you must provide
-complete object files to the recipients, so that they can relink them
-with the library after making changes to the library and recompiling
-it.  And you must show them these terms so they know their rights.
-
-  We protect your rights with a two-step method: (1) we copyright the
-library, and (2) we offer you this license, which gives you legal
-permission to copy, distribute and/or modify the library.
-
-  To protect each distributor, we want to make it very clear that
-there is no warranty for the free library.  Also, if the library is
-modified by someone else and passed on, the recipients should know
-that what they have is not the original version, so that the original
-author's reputation will not be affected by problems that might be
-introduced by others.
-
-  Finally, software patents pose a constant threat to the existence of
-any free program.  We wish to make sure that a company cannot
-effectively restrict the users of a free program by obtaining a
-restrictive license from a patent holder.  Therefore, we insist that
-any patent license obtained for a version of the library must be
-consistent with the full freedom of use specified in this license.
-
-  Most GNU software, including some libraries, is covered by the
-ordinary GNU General Public License.  This license, the GNU Lesser
-General Public License, applies to certain designated libraries, and
-is quite different from the ordinary General Public License.  We use
-this license for certain libraries in order to permit linking those
-libraries into non-free programs.
-
-  When a program is linked with a library, whether statically or using
-a shared library, the combination of the two is legally speaking a
-combined work, a derivative of the original library.  The ordinary
-General Public License therefore permits such linking only if the
-entire combination fits its criteria of freedom.  The Lesser General
-Public License permits more lax criteria for linking other code with
-the library.
-
-  We call this license the "Lesser" General Public License because it
-does Less to protect the user's freedom than the ordinary General
-Public License.  It also provides other free software developers Less
-of an advantage over competing non-free programs.  These disadvantages
-are the reason we use the ordinary General Public License for many
-libraries.  However, the Lesser license provides advantages in certain
-special circumstances.
-
-  For example, on rare occasions, there may be a special need to
-encourage the widest possible use of a certain library, so that it becomes
-a de-facto standard.  To achieve this, non-free programs must be
-allowed to use the library.  A more frequent case is that a free
-library does the same job as widely used non-free libraries.  In this
-case, there is little to gain by limiting the free library to free
-software only, so we use the Lesser General Public License.
-
-  In other cases, permission to use a particular library in non-free
-programs enables a greater number of people to use a large body of
-free software.  For example, permission to use the GNU C Library in
-non-free programs enables many more people to use the whole GNU
-operating system, as well as its variant, the GNU/Linux operating
-system.
-
-  Although the Lesser General Public License is Less protective of the
-users' freedom, it does ensure that the user of a program that is
-linked with the Library has the freedom and the wherewithal to run
-that program using a modified version of the Library.
-
-  The precise terms and conditions for copying, distribution and
-modification follow.  Pay close attention to the difference between a
-"work based on the library" and a "work that uses the library".  The
-former contains code derived from the library, whereas the latter must
-be combined with the library in order to run.
-
-		  GNU LESSER GENERAL PUBLIC LICENSE
-   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
-
-  0. This License Agreement applies to any software library or other
-program which contains a notice placed by the copyright holder or
-other authorized party saying it may be distributed under the terms of
-this Lesser General Public License (also called "this License").
-Each licensee is addressed as "you".
-
-  A "library" means a collection of software functions and/or data
-prepared so as to be conveniently linked with application programs
-(which use some of those functions and data) to form executables.
-
-  The "Library", below, refers to any such software library or work
-which has been distributed under these terms.  A "work based on the
-Library" means either the Library or any derivative work under
-copyright law: that is to say, a work containing the Library or a
-portion of it, either verbatim or with modifications and/or translated
-straightforwardly into another language.  (Hereinafter, translation is
-included without limitation in the term "modification".)
-
-  "Source code" for a work means the preferred form of the work for
-making modifications to it.  For a library, complete source code means
-all the source code for all modules it contains, plus any associated
-interface definition files, plus the scripts used to control compilation
-and installation of the library.
-
-  Activities other than copying, distribution and modification are not
-covered by this License; they are outside its scope.  The act of
-running a program using the Library is not restricted, and output from
-such a program is covered only if its contents constitute a work based
-on the Library (independent of the use of the Library in a tool for
-writing it).  Whether that is true depends on what the Library does
-and what the program that uses the Library does.
-  
-  1. You may copy and distribute verbatim copies of the Library's
-complete source code as you receive it, in any medium, provided that
-you conspicuously and appropriately publish on each copy an
-appropriate copyright notice and disclaimer of warranty; keep intact
-all the notices that refer to this License and to the absence of any
-warranty; and distribute a copy of this License along with the
-Library.
-
-  You may charge a fee for the physical act of transferring a copy,
-and you may at your option offer warranty protection in exchange for a
-fee.
-
-  2. You may modify your copy or copies of the Library or any portion
-of it, thus forming a work based on the Library, and copy and
-distribute such modifications or work under the terms of Section 1
-above, provided that you also meet all of these conditions:
-
-    a) The modified work must itself be a software library.
-
-    b) You must cause the files modified to carry prominent notices
-    stating that you changed the files and the date of any change.
-
-    c) You must cause the whole of the work to be licensed at no
-    charge to all third parties under the terms of this License.
-
-    d) If a facility in the modified Library refers to a function or a
-    table of data to be supplied by an application program that uses
-    the facility, other than as an argument passed when the facility
-    is invoked, then you must make a good faith effort to ensure that,
-    in the event an application does not supply such function or
-    table, the facility still operates, and performs whatever part of
-    its purpose remains meaningful.
-
-    (For example, a function in a library to compute square roots has
-    a purpose that is entirely well-defined independent of the
-    application.  Therefore, Subsection 2d requires that any
-    application-supplied function or table used by this function must
-    be optional: if the application does not supply it, the square
-    root function must still compute square roots.)
-
-These requirements apply to the modified work as a whole.  If
-identifiable sections of that work are not derived from the Library,
-and can be reasonably considered independent and separate works in
-themselves, then this License, and its terms, do not apply to those
-sections when you distribute them as separate works.  But when you
-distribute the same sections as part of a whole which is a work based
-on the Library, the distribution of the whole must be on the terms of
-this License, whose permissions for other licensees extend to the
-entire whole, and thus to each and every part regardless of who wrote
-it.
-
-Thus, it is not the intent of this section to claim rights or contest
-your rights to work written entirely by you; rather, the intent is to
-exercise the right to control the distribution of derivative or
-collective works based on the Library.
-
-In addition, mere aggregation of another work not based on the Library
-with the Library (or with a work based on the Library) on a volume of
-a storage or distribution medium does not bring the other work under
-the scope of this License.
-
-  3. You may opt to apply the terms of the ordinary GNU General Public
-License instead of this License to a given copy of the Library.  To do
-this, you must alter all the notices that refer to this License, so
-that they refer to the ordinary GNU General Public License, version 2,
-instead of to this License.  (If a newer version than version 2 of the
-ordinary GNU General Public License has appeared, then you can specify
-that version instead if you wish.)  Do not make any other change in
-these notices.
-
-  Once this change is made in a given copy, it is irreversible for
-that copy, so the ordinary GNU General Public License applies to all
-subsequent copies and derivative works made from that copy.
-
-  This option is useful when you wish to copy part of the code of
-the Library into a program that is not a library.
-
-  4. You may copy and distribute the Library (or a portion or
-derivative of it, under Section 2) in object code or executable form
-under the terms of Sections 1 and 2 above provided that you accompany
-it with the complete corresponding machine-readable source code, which
-must be distributed under the terms of Sections 1 and 2 above on a
-medium customarily used for software interchange.
-
-  If distribution of object code is made by offering access to copy
-from a designated place, then offering equivalent access to copy the
-source code from the same place satisfies the requirement to
-distribute the source code, even though third parties are not
-compelled to copy the source along with the object code.
-
-  5. A program that contains no derivative of any portion of the
-Library, but is designed to work with the Library by being compiled or
-linked with it, is called a "work that uses the Library".  Such a
-work, in isolation, is not a derivative work of the Library, and
-therefore falls outside the scope of this License.
-
-  However, linking a "work that uses the Library" with the Library
-creates an executable that is a derivative of the Library (because it
-contains portions of the Library), rather than a "work that uses the
-library".  The executable is therefore covered by this License.
-Section 6 states terms for distribution of such executables.
-
-  When a "work that uses the Library" uses material from a header file
-that is part of the Library, the object code for the work may be a
-derivative work of the Library even though the source code is not.
-Whether this is true is especially significant if the work can be
-linked without the Library, or if the work is itself a library.  The
-threshold for this to be true is not precisely defined by law.
-
-  If such an object file uses only numerical parameters, data
-structure layouts and accessors, and small macros and small inline
-functions (ten lines or less in length), then the use of the object
-file is unrestricted, regardless of whether it is legally a derivative
-work.  (Executables containing this object code plus portions of the
-Library will still fall under Section 6.)
-
-  Otherwise, if the work is a derivative of the Library, you may
-distribute the object code for the work under the terms of Section 6.
-Any executables containing that work also fall under Section 6,
-whether or not they are linked directly with the Library itself.
-
-  6. As an exception to the Sections above, you may also combine or
-link a "work that uses the Library" with the Library to produce a
-work containing portions of the Library, and distribute that work
-under terms of your choice, provided that the terms permit
-modification of the work for the customer's own use and reverse
-engineering for debugging such modifications.
-
-  You must give prominent notice with each copy of the work that the
-Library is used in it and that the Library and its use are covered by
-this License.  You must supply a copy of this License.  If the work
-during execution displays copyright notices, you must include the
-copyright notice for the Library among them, as well as a reference
-directing the user to the copy of this License.  Also, you must do one
-of these things:
-
-    a) Accompany the work with the complete corresponding
-    machine-readable source code for the Library including whatever
-    changes were used in the work (which must be distributed under
-    Sections 1 and 2 above); and, if the work is an executable linked
-    with the Library, with the complete machine-readable "work that
-    uses the Library", as object code and/or source code, so that the
-    user can modify the Library and then relink to produce a modified
-    executable containing the modified Library.  (It is understood
-    that the user who changes the contents of definitions files in the
-    Library will not necessarily be able to recompile the application
-    to use the modified definitions.)
-
-    b) Use a suitable shared library mechanism for linking with the
-    Library.  A suitable mechanism is one that (1) uses at run time a
-    copy of the library already present on the user's computer system,
-    rather than copying library functions into the executable, and (2)
-    will operate properly with a modified version of the library, if
-    the user installs one, as long as the modified version is
-    interface-compatible with the version that the work was made with.
-
-    c) Accompany the work with a written offer, valid for at
-    least three years, to give the same user the materials
-    specified in Subsection 6a, above, for a charge no more
-    than the cost of performing this distribution.
-
-    d) If distribution of the work is made by offering access to copy
-    from a designated place, offer equivalent access to copy the above
-    specified materials from the same place.
-
-    e) Verify that the user has already received a copy of these
-    materials or that you have already sent this user a copy.
-
-  For an executable, the required form of the "work that uses the
-Library" must include any data and utility programs needed for
-reproducing the executable from it.  However, as a special exception,
-the materials to be distributed need not include anything that is
-normally distributed (in either source or binary form) with the major
-components (compiler, kernel, and so on) of the operating system on
-which the executable runs, unless that component itself accompanies
-the executable.
-
-  It may happen that this requirement contradicts the license
-restrictions of other proprietary libraries that do not normally
-accompany the operating system.  Such a contradiction means you cannot
-use both them and the Library together in an executable that you
-distribute.
-
-  7. You may place library facilities that are a work based on the
-Library side-by-side in a single library together with other library
-facilities not covered by this License, and distribute such a combined
-library, provided that the separate distribution of the work based on
-the Library and of the other library facilities is otherwise
-permitted, and provided that you do these two things:
-
-    a) Accompany the combined library with a copy of the same work
-    based on the Library, uncombined with any other library
-    facilities.  This must be distributed under the terms of the
-    Sections above.
-
-    b) Give prominent notice with the combined library of the fact
-    that part of it is a work based on the Library, and explaining
-    where to find the accompanying uncombined form of the same work.
-
-  8. You may not copy, modify, sublicense, link with, or distribute
-the Library except as expressly provided under this License.  Any
-attempt otherwise to copy, modify, sublicense, link with, or
-distribute the Library is void, and will automatically terminate your
-rights under this License.  However, parties who have received copies,
-or rights, from you under this License will not have their licenses
-terminated so long as such parties remain in full compliance.
-
-  9. You are not required to accept this License, since you have not
-signed it.  However, nothing else grants you permission to modify or
-distribute the Library or its derivative works.  These actions are
-prohibited by law if you do not accept this License.  Therefore, by
-modifying or distributing the Library (or any work based on the
-Library), you indicate your acceptance of this License to do so, and
-all its terms and conditions for copying, distributing or modifying
-the Library or works based on it.
-
-  10. Each time you redistribute the Library (or any work based on the
-Library), the recipient automatically receives a license from the
-original licensor to copy, distribute, link with or modify the Library
-subject to these terms and conditions.  You may not impose any further
-restrictions on the recipients' exercise of the rights granted herein.
-You are not responsible for enforcing compliance by third parties with
-this License.
-
-  11. If, as a consequence of a court judgment or allegation of patent
-infringement or for any other reason (not limited to patent issues),
-conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License.  If you cannot
-distribute so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you
-may not distribute the Library at all.  For example, if a patent
-license would not permit royalty-free redistribution of the Library by
-all those who receive copies directly or indirectly through you, then
-the only way you could satisfy both it and this License would be to
-refrain entirely from distribution of the Library.
-
-If any portion of this section is held invalid or unenforceable under any
-particular circumstance, the balance of the section is intended to apply,
-and the section as a whole is intended to apply in other circumstances.
-
-It is not the purpose of this section to induce you to infringe any
-patents or other property right claims or to contest validity of any
-such claims; this section has the sole purpose of protecting the
-integrity of the free software distribution system which is
-implemented by public license practices.  Many people have made
-generous contributions to the wide range of software distributed
-through that system in reliance on consistent application of that
-system; it is up to the author/donor to decide if he or she is willing
-to distribute software through any other system and a licensee cannot
-impose that choice.
-
-This section is intended to make thoroughly clear what is believed to
-be a consequence of the rest of this License.
-
-  12. If the distribution and/or use of the Library is restricted in
-certain countries either by patents or by copyrighted interfaces, the
-original copyright holder who places the Library under this License may add
-an explicit geographical distribution limitation excluding those countries,
-so that distribution is permitted only in or among countries not thus
-excluded.  In such case, this License incorporates the limitation as if
-written in the body of this License.
-
-  13. The Free Software Foundation may publish revised and/or new
-versions of the Lesser General Public License from time to time.
-Such new versions will be similar in spirit to the present version,
-but may differ in detail to address new problems or concerns.
-
-Each version is given a distinguishing version number.  If the Library
-specifies a version number of this License which applies to it and
-"any later version", you have the option of following the terms and
-conditions either of that version or of any later version published by
-the Free Software Foundation.  If the Library does not specify a
-license version number, you may choose any version ever published by
-the Free Software Foundation.
-
-  14. If you wish to incorporate parts of the Library into other free
-programs whose distribution conditions are incompatible with these,
-write to the author to ask for permission.  For software which is
-copyrighted by the Free Software Foundation, write to the Free
-Software Foundation; we sometimes make exceptions for this.  Our
-decision will be guided by the two goals of preserving the free status
-of all derivatives of our free software and of promoting the sharing
-and reuse of software generally.
-
-			    NO WARRANTY
-
-  15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
-WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
-EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR
-OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY
-KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
-PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
-LIBRARY IS WITH YOU.  SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME
-THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
-
-  16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
-WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
-AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU
-FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
-CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
-LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
-RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
-FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
-SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
-DAMAGES.
-
-		     END OF TERMS AND CONDITIONS
-
-           How to Apply These Terms to Your New Libraries
-
-  If you develop a new library, and you want it to be of the greatest
-possible use to the public, we recommend making it free software that
-everyone can redistribute and change.  You can do so by permitting
-redistribution under these terms (or, alternatively, under the terms of the
-ordinary General Public License).
-
-  To apply these terms, attach the following notices to the library.  It is
-safest to attach them to the start of each source file to most effectively
-convey the exclusion of warranty; and each file should have at least the
-"copyright" line and a pointer to where the full notice is found.
-
-    <one line to give the library's name and a brief idea of what it does.>
-    Copyright (C) <year>  <name of author>
-
-    This library is free software; you can redistribute it and/or
-    modify it under the terms of the GNU Lesser General Public
-    License as published by the Free Software Foundation; either
-    version 2.1 of the License, or (at your option) any later version.
-
-    This library is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-    Lesser General Public License for more details.
-
-    You should have received a copy of the GNU Lesser General Public
-    License along with this library; if not, write to the Free Software
-    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-
-Also add information on how to contact you by electronic and paper mail.
-
-You should also get your employer (if you work as a programmer) or your
-school, if any, to sign a "copyright disclaimer" for the library, if
-necessary.  Here is a sample; alter the names:
-
-  Yoyodyne, Inc., hereby disclaims all copyright interest in the
-  library `Frob' (a library for tweaking knobs) written by James Random Hacker.
-
-  <signature of Ty Coon>, 1 April 1990
-  Ty Coon, President of Vice
-
-That's all there is to it!

ocrmypdf/jhove/LICENSE

@@ -1,17 +0,0 @@
-JHOVE - JSTOR/Harvard Object Validation Environment
-Copyright 2003-2008 by JSTOR and the President and Fellows of Harvard College
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU Lessor General Public License as
-published by the Free Software Foundation; either version 2.1 of the
-License, or (at your option) any later version.
-
-This program is distributed in the hope that it will be useful, but
-WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-Lesser General Public License for more details.
-
-You should have received a copy of the GNU Lesser General Public
-License along with this program; if not, write to the Free Software
-Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
-USA

ocrmypdf/jhove/README

@@ -1,227 +0,0 @@
-JHOVE - JSTOR/Harvard Object Validation Environment
-Copyright 2003-2012 by JSTOR and the President and Fellows of Harvard College
-JHOVE is made available under the GNU Lesser General Public License (LGPL;
-see the file LICENSE for details)
-
-Rev. 1.11, 2013-09-29
-
-JHOVE (the JSTOR/Harvard Object Validation Environment, pronounced "jhove")
-is an extensible software framework for performing format identification,
-validation, and characterization of digital objects.
-
-o Format identification is the process of determining the format to which a
-  digital object conforms: "I have a digital object; what format is it?"
-o Format validation is the process of determining the level of compliance of a
-  digital object to the specification for its purported format: "I have an
-  object purportedly of format F; is it?"
-o Format characterization is the process of determing the format-specific
-  significant properties of an object of a given format: "I have an object of
-  format F; what are its salient properties?"
-
-These actions are frequently necessary during routine operation of digital
-repositories and for digital preservation activities.
-
-The output from JHOVE is controlled by output handlers. JHOVE uses an
-extensible plug-in architecture; it can be configured at the time of its
-invocation to include whatever specific format modules and output handlers
-that are desired. The initial release of JHOVE includes modules for
-arbitrary byte streams, ASCII and UTF-8 encoded text, AIFF and WAVE audio,
-GIF, JPEG, JPEG 2000, TIFF, and PDF; and text and XML output handlers.
-
-The JHOVE project is a collaboration of JSTOR and the Harvard University
-Library.  Development of JHOVE was funded in part by the Andrew W. Mellon
-Foundation.  JHOVE is made available under the GNU Lesser General Public
-License (LGPL; see the file LICENSE for details).
-
-JHOVE is currently being maintained by indpendent developers.
-
-REQUIREMENTS
-
-1. Java J2SE 1.5
-(JHOVE was originally implemented using the Sun J2SE SDK 1.4.1 and has
-been tested to work with 1.5)
-
-2. If you would like to compile the JHOVE source code, then
-Apache Ant, a Java-based build tool <http://ant.apache.org/> is necessary.
-Note that the JAVA_HOME environment variable must be set appropriately for
-Ant to work properly.
-(JHOVE was implemented and tested using Ant 1.5.1.)
-
-DISTRIBUTION
-
-The JHOVE distribution package includes:
-
-    jhove/                                 # JHOVE home directory
-          COPYING                          # GNU Lesser General Public License
-          LICENSE                          # JHOVE license information
-          README
-          RELEASENOTES                     # JHOVE release notes
-          bin/
-                    jhove.jar              # JHOVE API package
-                    jhove-handler.jar      # Standard output handler package
-                    jhove-module.jar       # Standard module package
-                    JhoveApp.jar           # JHOVE command line application
-                    JhoveView.jar          # JHOVE with Swing GUI front-end
-          build.xml                        # Ant configuration file
-          classes/
-                    build.xml              # Ant configuration file
-                    edu/ ...               # JHOVE API packages
-                    ADump.*                # AIFF dump utility class
-                    GDump.*                # GIF dump utility class
-                    Jhove.*                # JHOVE main class
-                    JDump.*                # JPEG dump utility class
-                    J2Dump.*               # JPEG 2000 dump utility class
-                    PDump.*                # PDF dump utility class
-                    TDump.*                # TIFF dump utility class
-                    UserHome.*             # user.home property utility class
-                    WDump.*                # WAVE dump utility class
-          conf/
-                   jhove.conf              # JHOVE configuration file
-                   jhove.xsd               # JHOVE output schema
-                   jhoveConfig.xsd         # JHOVE configuration file schema
-          doc/
-                   *.html                  # API documentation
-                   ...
-          examples/                        # Sample files
-                   ascii/ ...
-                   gif/   ...
-                   jpeg/  ...
-                   jpeg2000/ ...
-                   pdf/   ...
-                   tiff/  ...
-                   utf-8/ ...
-          adump*                           # AIFF dump Bourne shell driver
-          adump.bat*                       # AIFF dump DOS shell driver script
-          gdump*                           # GIF dump Bourne shell driver
-          gdump.bat*                       # GIF dump DOS shell driver script
-          jdump*                           # JPEG dump Bourne shell driver
-          jdump.bat*                       # JPEG dump DOS shell driver script
-          j2dump*                          # JPEG 2000 dump Bourne shell driver
-          j2dump.bat*                      # JPEG 2000 dump DOS shell driver
-          jhove.tmpl*                      # Template for JHOVE Bourne shell driver script
-          jhove_bat.tmpl*                  # Template for JHOVE DOS shell driver script
-          pdump*                           # PDF dump Bourne shell driver
-          pdump.bat*                       # PDF dump DOS shell driver script
-          tdump*                           # TIFF dump Bourne shell driver
-          tdump.bat*                       # TIFF dump DOS shell driver script
-          userhome*                        # user.home Bourne shell driver
-          userhome.bat*                    # user.home DOS shell driver script
-          wdump*                           # WAVE dump Bourne shell driver
-          wdump.bat*                       # WAVE dump DOS shell driver script
-
-INSTALLATION
-
-Edit the configuration file, jhove/conf/jhove.conf, and set the absolute
-pathname of the JHOVE home directory and the temporary directory (in which
-temporary files are created):
-
-    <jhoveHome>jhove-home-directory</jhoveHome>
-    <tempDirectory>temporary-directory</tempDirectory>
-
-The JHOVE home directory is the top-most directory in the distribution TAR
-or ZIP file.  On Unix systems, "/var/tmp" is an appropriate temporary
-directory; on Windows, "C:\Temp".  For example, if the distribution TAR
-file is disaggregated on a Unix system in the directory "/users/stephen/
-projects", then the configuration file should read:
-
-    <jhoveHome>/users/stephen/projects/jhove</jhoveHome>
-    <tempDirectory>/var/tmp</jhoveHome>
-
-In the JHOVE home directory, copy the JHOVE Bourne shell driver script
-template, "jhove.tmpl", to "jhove" (or the equivalent Windows shell 
-script, "jhove_bat.tmpl" to "jhove.bat"), and set the
-JHOVE home directory, Java home directory, and Java interpreter:
-
-    JHOVE_HOME=jhove-home-directory
-    JAVA_HOME=java-home-directory
-    JAVA=java-interpreter
-
-The JAVA_HOME property should provide the absolute pathname of the Java
-runtime or SDK installation; JAVA should provide the absolute pathname of the
-Java interpreter.  For example:
-
-    JHOVE_HOME=/users/stephen/projects/jhove
-    JAVA_HOME=/usr/local/j2re1.4.1_02
-    JAVA=$JAVA_HOME/bin/java
-
-In the DOS shell driver script, jhove.bat, the equivalent three
-variables are:
-
-    SET JHOVE_HOME=jhove-home-directory
-    SET JAVA_HOME=java-home-directory
-    SET JAVA=%JAVA_HOME%\bin\java
-
-For example:
-
-    SET JHOVE_HOME="C:\Program Files\jhove"
-    SET JAVA_HOME="C:\Program Files\java\j2re1.4.1_02"
-    SET JAVA=%JAVA_HOME%\bin\java
-
-The quotation marks are necessary because of the embedded space characters.
-On Windows platforms it may also be necessary to add the Java bin subdirectory
-to the System PATH environment variable:
-
-    PATH=C:\Program Files\java\j2re1.4.1_02\bin;...
-
-(For information on setting a Windows environment variable, consult your local
-documentation or system administrator.)
-
-USAGE
-
-    java Jhove [-c config] [-m module] [-h handler] [-e encoding] [-H handler]
-               [-o output] [-x saxclass] [-t tempdir] [-b bufsize]
-               [-l loglevel] [[-krs] dir-file-or-uri [...]]
-
-where -c config   Configuration file pathname
-      -m module   Module name
-      -h handler  Output handler name (defaults to TEXT)
-      -e encoding Character encoding used by output handler (defaults to UTF-8)
-      -H handler  About handler name
-      -o output   Output file pathname (defaults to standard output)
-      -x saxclass SAX parser class (defaults to J2SE default)
-      -t tempdir  Temporary directory in which to create temporary files
-      -b bufsize  Buffer size for buffered I/O (defaults to J2SE 1.4 default)
-      -l loglevel Logging level
-      -k          Calculate CRC32, MD5, and SHA-1 checksums
-      -r          Display raw data flags, not textual equivalents
-      -s          Format identification based on internal signatures only
-      dir-file-or-uri Directory or file pathname or URI of formated content
-                      stream
-
-All named modules and output handlers must be found on the Java CLASSPATH at
-the time of invocation.  The JHOVE driver script, jhove/jhove, automatically
-sets the CLASSPATH and invokes the Jhove main class:
-
-    jhove [-c config] [-m module] [-h handler] [-e encoding] [-H handler]
-          [-o output] [-x saxclass] [-t tempdir] [-b bufsize] [-l loglevel]
-          [[-krs] dir-file-or-uri [...]]
-
-The following additional programs are available, primarily for testing
-and debugging purposes.  They display a minimally processed, human-readable
-version of the contents of AIFF, GIF, JPEG, JPEG 2000, PDF, TIFF, and WAVE
-files:
-
-    java ADump  aiff-file
-    java GDump  gif-file
-    java JDump  jpeg-file
-    java J2Dump jpeg2000-file
-    java PDump  pdf-file
-    java TDump  tiff-file
-    java WDump  wave-file
-
-For convenience, the following driver scripts are also available:
-
-    adump  aiff-file
-    gdump  gif-file
-    jdump  jpeg-file
-    j2dump jpeg2000-file
-    pdump  pdf-file
-    tdump  tiff-file
-    wdump  wave-file
-
-The JHOVE Swing-based GUI interface can be invoked from a command shell from
-the jhove/bin sub-directory:
-
-    java -jar JhoveView.jar -c <configFile>
-
-where <configFile> is the pathname of the JHOVE configuration file.

ocrmypdf/jhove/bin/README

@@ -1,25 +0,0 @@
-JHOVE - JSTOR/Harvard Object Validation Environment
-Copyright 2003 by JSTOR and the President and Fellows of Harvard College
-JHOVE is made available under the GNU General Public License (see the file
-LICENSE for details)
-
-Rev. 2003-11-25
-
-The following jar files are meant to be used for embedding JHOVE functionality
-into new applications or systems.
-
-  jhove.jar          Contains the JHOVE API interfaces and classes
-  jhove-module.jar   Contains the standard JHOVE modules ()
-  jhove-handler.jar  Contains the standard JHOVE output handlers (TEXT and XML)
-
-The following jar file is meant to be used with the stand-alone JHOVE
-application using a command-line interface.  It contains the main Jhove class
-and the contents of jhove.jar, jhove-module.jar, and jhove-handler.jar.
-
-  JhoveApp.jar
-
-The following jar file is meant to be used with the stand-alone JHOVE
-application using a Swing GUI interface.  It contains the main JhoveView class
-and the contents of jhove.jar, jhove-module.jar, and jhove-handler.jar.
-
-  JhoveView.jar

ocrmypdf/jhove/build.xml

@@ -1,78 +0,0 @@
-<project name="Jhove" default="dist" basedir=".">
-  <description>Project build file
-    Jhove - JSTOR/Harvard Object Validation Environment
-    Version 1.0 2004-09-10
-    Copyright 2004 by JSTOR and the President and Fellows of Harvard College
-  </description>
-
-  <!-- ant (or ant dist) Build everything
-       ant debug         Build everything with debug enabled
-       ant clean         Delete backup files
-       ant cleanclass    Delete backup and class files
-       ant cleandist     Delete backup, class, and jar files
-       ant javadoc       Build javadocs
-    -->
-
-  <!-- set global properties for this build -->
-  <property name="bin"     location="bin"/>
-  <property name="classes" location="classes"/>
-  <property name="doc"     location="doc"/>
-
-  <target name="dist" description="Create distribution">
-    <ant dir="${classes}" inheritAll="false">
-      <property name="dbg" value="off"/>
-    </ant>
-  	<chmod file="jhove" perm="ugo+x"/>
-  	<chmod file="${bin}/JhoveView.jar" perm="ugo+x"/>
-  </target>
-
-  <target name="debug" description="Create distribution with debug enabled">
-    <ant dir="${classes}" inheritAll="false">
-      <property name="dbg" value="on"/>
-    </ant>
-  </target>
-
-  <target name="view" description="Create JhoveView application">
-    <ant dir="${classes}" target="view" inheritAll="false">
-      <property name="dbs" value="on"/>
-    </ant>
-  </target>
-
-  <target name="clean" description="Delete backup files">
-    <ant dir="${classes}" target="main-clean" inheritAll="false"/>
-  </target>
-
-  <target name="cleanclass" depends="clean">
-    <ant dir="${classes}" target="main-cleanclass" inheritAll="false"/>
-  </target>
-
-  <target name="cleandist" depends="cleanclass">
-    <delete file="${bin}/jhove.jar"/>
-    <delete file="${bin}/jhove-handler.jar"/>
-    <delete file="${bin}/jhove-module.jar"/>
-    <delete file="${bin}/JhoveApp.jar"/>
-    <delete file="${bin}/JhoveView.jar"/>
-  </target>
-
-  <target name="javadoc">
-    <javadoc sourcepath="${classes}" destdir="${doc}"
-             windowtitle="JHOVE Documentation"
-             Overview="${classes}/overview.html">
-      <package name="edu.harvard.hul.ois.jhove"/>
-      <package name="edu.harvard.hul.ois.jhove.handler"/>
-      <package name="edu.harvard.hul.ois.jhove.handler.audit"/>
-      <package name="edu.harvard.hul.ois.jhove.module"/>
-      <package name="edu.harvard.hul.ois.jhove.module.aiff"/>
-      <package name="edu.harvard.hul.ois.jhove.module.gif"/>
-      <package name="edu.harvard.hul.ois.jhove.module.html"/>
-      <package name="edu.harvard.hul.ois.jhove.module.iff"/>
-      <package name="edu.harvard.hul.ois.jhove.module.jpeg"/>
-      <package name="edu.harvard.hul.ois.jhove.module.jpeg2000"/>
-      <package name="edu.harvard.hul.ois.jhove.module.pdf"/>
-      <package name="edu.harvard.hul.ois.jhove.module.tiff"/>
-      <package name="edu.harvard.hul.ois.jhove.module.wave"/>
-      <package name="edu.harvard.hul.ois.jhove.module.xml"/>
-      <package name="edu.harvard.hul.ois.jhove.viewer"/>
-    </javadoc>
-  </target>
-</project>

ocrmypdf/jhove/conf/README

@@ -1,63 +0,0 @@
-JHOVE - JSTOR/Harvard Object Validation Environment
-Copyright 2003-2007 by JSTOR and the President and Fellows of Harvard College
-JHOVE is made available under the GNU General Public License (see the file
-LICENSE for details)
-
-Rev. 2007-08-30
-
-Edit the configuration file, jhove.conf, and set the JHOVE home
-directory:
-
-    <jhoveHome>jhove-home-directory</jhoveHome>
-
-and temporary directory:
-
-    <tempDirectory>temporary-directory</tempDirectory>
-
-On most Unix systems, a reasonable temporary directory is "/var/tmp";
-on Windows, "C:\temp".
-
-The optional
-
-    <bufferSize>buffer-size</bufferSize>
-
-element defines the buffer size used for buffer I/O operations.
-
-The optional
-
-    <mixVersion>1.0</mixVersion>
-
-element specifies that the XML output handler should conform to the
-MIX 1.0 schema.  The default behavior is for handler output to conform
-to the MIX 0.2 schema.
-
-The optional
-
-    <sigBytes>n</sigBytes>
-
-element specifies that JHOVE modules will look for format signatures
-in the first <n> bytes of the file.  The default value is 1024.
-
-All class names must be fully qualified with their package name:
-
-    <module>
-      <class>fully-package-qualified-class-name</class>
-      <init>optional-initialization-argument</init>
-      <param>optional-invocation-argument</param>
-    </module>
-
-The optional <init> argument is passed to the module once at the time
-its class is instantiated.  See module-specific documentation for a
-description of any initialization options.
-
-The optional <param> argument is passed to the module every time it is
-invoked.  See module-specific documentation for a description of any
-invocation options.
-
-The order in which format modules are defined is important; when
-performing a format identification operation, JHOVE will search for a
-matching module in the order in which the modules are defined in the
-configuration file. In general, the modules for more generic formats
-should come later in the list. For example, the standard module ASCII
-should be defined before the UTF-8 module, since all ASCII objects
-are, by definition, UTF-8 objects, but not vice versa.

ocrmypdf/jhove/conf/jhove-byteoffset=true.conf

@@ -1,47 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<jhoveConfig version="1.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xmlns="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig"
- xsi:schemaLocation="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig
-                     http://hul.harvard.edu/ois/xml/xsd/jhove/1.6/jhoveConfig.xsd">
- <jhoveHome>/users/stephen/projects/jhove</jhoveHome>
- <defaultEncoding>utf-8</defaultEncoding>
- <tempDirectory>/var/tmp</tempDirectory>
- <bufferSize>131072</bufferSize>
- <mixVersion>1.0</mixVersion>
- <sigBytes>1024</sigBytes>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.AiffModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.WaveModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.PdfModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.Jpeg2000Module</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.JpegModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.GifModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.TiffModule</class>
-  <param>byteoffset=true</param>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.XmlModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.HtmlModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.AsciiModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.Utf8Module</class>
- </module>
-</jhoveConfig>

ocrmypdf/jhove/conf/jhove-withTextMD.conf

@@ -1,51 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<jhoveConfig version="1.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xmlns="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig"
- xsi:schemaLocation="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig
-                     http://hul.harvard.edu/ois/xml/xsd/jhove/1.6/jhoveConfig.xsd">
- <jhoveHome>/users/stephen/projects/jhove</jhoveHome>
- <defaultEncoding>utf-8</defaultEncoding>
- <tempDirectory>/var/tmp</tempDirectory>
- <bufferSize>131072</bufferSize>
- <mixVersion>1.0</mixVersion>
- <sigBytes>1024</sigBytes>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.AiffModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.WaveModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.PdfModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.Jpeg2000Module</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.JpegModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.GifModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.TiffModule</class>
-  <param>byteoffset=true</param>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.XmlModule</class>
-  <param>withTextMD=true</param>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.HtmlModule</class>
-  <param>withTextMD=true</param>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.AsciiModule</class>
-  <param>withTextMD=true</param>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.Utf8Module</class>
-  <param>withTextMD=true</param>
- </module>
-</jhoveConfig>

ocrmypdf/jhove/conf/jhove.conf

@@ -1,45 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<jhoveConfig version="1.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xmlns="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig"
- xsi:schemaLocation="http://hul.harvard.edu/ois/xml/ns/jhove/jhoveConfig
-                     http://hul.harvard.edu/ois/xml/xsd/jhove/1.6/jhoveConfig.xsd">
- <jhoveHome>./jhove/</jhoveHome>
- <defaultEncoding>utf-8</defaultEncoding>
- <tempDirectory>/var/tmp</tempDirectory>
- <bufferSize>131072</bufferSize>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.AiffModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.WaveModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.PdfModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.Jpeg2000Module</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.JpegModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.GifModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.TiffModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.XmlModule</class>
-  <param>schema=http://www.example.com/schema;/home/schemas/exampleschema.xsd</param>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.HtmlModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.AsciiModule</class>
- </module>
- <module>
-  <class>edu.harvard.hul.ois.jhove.module.Utf8Module</class>
- </module>
-</jhoveConfig>

ocrmypdf/jhove/configure.pl

@@ -1,88 +0,0 @@
-#!/usr/bin/perl
-
-########################################################################
-# Jhove - JSTOR/Harvard Object Validation Environment
-# Copyright 2004 by JSTOR and the President and Fellows of Harvard College
-#
-# A Perl script for plugging local path information into the
-# various script files of JHOVE, as well as conf/jhove.conf.
-#
-# This is configured only for Unix (including OS X).
-#
-#    Usage: configure.pl jhove_home_directory [java_home_directory [java_runtime_directory]]
-#
-#    If invoked with no arguments, it will output a usage message.
-#
-########################################################################
-use File::Copy;
-
-sub mung {
-    my $f = $_[0];
-    my $bak = $f . "~";
-    #If there is no backup file, copy the file to the
-    #backup.  Otherwise work from the backup.
-    if (!(-e $bak)) {
-        rename ($f, $bak);
-    }
-    open (INFILE, $bak);
-    open (OUTFILE, ">" . $f);
-
-    #Walks through each line of file, making substitutions.
-    #Remember that the JAVA_HOME and JAVA arguments are optional.
-    while (<INFILE>) {
-        s/^JHOVE_HOME=.*/JHOVE_HOME=$ARGV[0]/;
-        if ($narg >= 2) {
-            s/^JAVA_HOME=.*/JAVA_HOME=$ARGV[1]/;
-        }
-        if ($narg >= 3) {
-            s/^JAVA=.*/JAVA=$ARGV[2]/;
-        }
-        print OUTFILE;
-    }
-    close (INFILE);
-    close (OUTFILE);
-    if (-e $f) {
-        print ("Fixed " . $f . "\n");
-    }
-}
-
-
-$narg = $#ARGV + 1;
-if ($narg <= 0) {
-    print "Usage: configure.pl jhove_home_directory [java_home_directory [java_runtime_directory]]\n";
-    exit;
-}
-print "JHOVE_HOME will be set to " . $ARGV[0] . "\n";
-if ($narg >= 2) {
-    print "JAVA_HOME will be set to " . $ARGV[1] . "\n";
-}
-if ($narg >= 3) {
-    print "JAVA will be set to " . $ARGV[2] . "\n";
-}
-mung ("jhove");
-mung ("adump");
-mung ("gdump");
-mung ("jdump");
-mung ("j2dump");
-mung ("pdump");
-mung ("tdump");
-mung ("wdump");
-
-#Fix up the config file.  We assume that the <jhoveHome>
-#element is all on one line.
-if (!(-e "conf/jhove.conf~")) {
-    rename ("conf/jhove.conf", "conf/jhove.conf~");
-}
-open (INFILE, "conf/jhove.conf~");
-open (OUTFILE, ">conf/jhove.conf");
-while (<INFILE>) {
-    s!<jhoveHome>.*</jhoveHome>!<jhoveHome>$ARGV[0]</jhoveHome>!;
-    print OUTFILE;
-}
-close (INFILE);
-close (OUTFILE);
-if (-e "conf/jhove.conf") {
-    print "Fixed conf/jhove.conf\n";
-}
-exit;
-

ocrmypdf/jhove/gdump

@@ -1,37 +0,0 @@
-#!/bin/sh
-
-########################################################################
-# gdump - JSTOR/Harvard Object Validation Environment
-# Copyright 2004-2005 by the President and Fellows of Harvard College
-# JHOVE is made available under the GNU General Public License (see the
-# file LICENSE for details)
-#
-# Driver script for the GIF dump utility
-#
-# Usage: gdump file
-#
-# where  file  is a GIF file
-#
-# Configuration constants:
-
-JHOVE_HOME=/users/stephen/projects/jhove
-
-JAVA_HOME=/usr/java      # Java JRE directory
-JAVA=$JAVA_HOME/bin/java # Java interpreter
-
-EXTRA_JARS=              # Extra .jar files to add to CLASSPATH
-
-# NOTE: Nothing below this line should be edited
-########################################################################
-
-CP=${JHOVE_HOME}/bin/JhoveApp.jar:${EXTRA_JARS}
-
-# Retrieve a copy of all command line arguments to pass to the application.
-
-ARGS=""
-for ARG do
-    ARGS="$ARGS $ARG"
-done
-
-# Set the CLASSPATH and invoke the Java loader.
-${JAVA} -classpath $CP GDump $ARGS

ocrmypdf/jhove/j2dump

@@ -1,37 +0,0 @@
-#!/bin/sh
-
-########################################################################
-# j2dump - JSTOR/Harvard Object Validation Environment
-# Copyright 2004-2005 by the President and Fellows of Harvard College
-# JHOVE is made available under the GNU General Public License (see the
-# file LICENSE for details)
-#
-# Driver script for the JPEG 2000 dump utility
-#
-# Usage: j2dump file
-#
-# where  file  is a JPEG file
-#
-# Configuration constants:
-
-JHOVE_HOME=/users/stephen/projects/jhove
-
-JAVA_HOME=/usr/java      # Java JRE directory
-JAVA=$JAVA_HOME/bin/java # Java interpreter
-
-EXTRA_JARS=              # Extra .jar files to add to CLASSPATH
-
-# NOTE: Nothing below this line should be edited
-########################################################################
-
-CP=${JHOVE_HOME}/bin/JhoveApp.jar:${EXTRA_JARS}
-
-# Retrieve a copy of all command line arguments to pass to the application.
-
-ARGS=""
-for ARG do
-    ARGS="$ARGS $ARG"
-done
-
-# Set the CLASSPATH and invoke the Java loader.
-${JAVA} -classpath $CP J2Dump $ARGS

ocrmypdf/jhove/jdump

@@ -1,37 +0,0 @@
-#!/bin/sh
-
-########################################################################
-# jdump - JSTOR/Harvard Object Validation Environment
-# Copyright 2004-2005 by the President and Fellows of Harvard College
-# JHOVE is made available under the GNU General Public License (see the
-# file LICENSE for details)
-#
-# Driver script for the JPEG dump utility
-#
-# Usage: jdump file
-#
-# where  file  is a JPEG file
-#
-# Configuration constants:
-
-JHOVE_HOME=/users/stephen/projects/jhove
-
-JAVA_HOME=/usr/java      # Java JRE directory
-JAVA=$JAVA_HOME/bin/java # Java interpreter
-
-EXTRA_JARS=              # Extra .jar files to add to CLASSPATH
-
-# NOTE: Nothing below this line should be edited
-########################################################################
-
-CP=${JHOVE_HOME}/bin/JhoveApp.jar:${EXTRA_JARS}
-
-# Retrieve a copy of all command line arguments to pass to the application.
-
-ARGS=""
-for ARG do
-    ARGS="$ARGS $ARG"
-done
-
-# Set the CLASSPATH and invoke the Java loader.
-${JAVA} -classpath $CP JDump $ARGS

ocrmypdf/jhove/jhove

@@ -1,58 +0,0 @@
-#!/bin/sh
-
-########################################################################
-# JHOVE - JSTOR/Harvard Object Validation Environment
-# Copyright 2003-2005 by JSTOR and the President and Fellows of Harvard College
-# JHOVE is made available under the GNU General Public License (see the
-# file LICENSE for details)
-#
-# Usage: jhove [-c config] [-m module] [-h handler] [-e encoding] [-H handler]
-#              [-o output] [-x saxclass] [-t tempdir] [-b bufsize]
-#              [-l loglevel] [[-krs] dir-file-or-uri [...]]
-#
-# where -c config   Configuration file pathname
-#       -m module   Module name
-#       -h handler  Output handler name (defaults to TEXT)
-#       -e encoding Character encoding of output handler (defaults to UTF-8)
-#       -H handler  About handler name
-#       -o output   Output file pathname (defaults to standard output)
-#       -x saxclass SAX parser class (defaults to J2SE 1.4 default)
-#       -t tempdir  Temporary directory in which to create temporary files
-#       -b bufsize  Buffer size for buffered I/O (defaults to J2SE 1.4 default)
-#       -k          Calculate CRC32, MD5, and SHA-1 checksums
-#       -r          Display raw data flags, not textual equivalents
-#       -s          Format identification based on internal signatures only
-#       dir-file-or-uri Directory, file pathname or URI of formatted content
-#
-# CHANGE for JHOVE 1.8:
-# You no longer have to figure out where JAVA_HOME is; that's the
-# operating system's job. If the OS tells you it can't find Java, 
-# adjust your shell's path or revert to the old way (commented out).
-# Configuration constants:
-
-#JHOVE_HOME=/users/gary/dev/jhove
-JHOVE_HOME=[fill in path to jhove directory]
-
-JAVA_HOME=/usr/java      # Java JRE directory -- change to your local java home
-JAVA=$JAVA_HOME/bin/java # Java interpreter -- usually won't need change
-
-#XTRA_JARS=/users/stephen/xercesImpl.jar
-EXTRA_JARS=              # Extra .jar files to add to CLASSPATH
-
-# NOTE: Nothing below this line should be edited
-########################################################################
-
-CP=${JHOVE_HOME}/bin/JhoveApp.jar:${EXTRA_JARS}
-
-# Retrieve a copy of all command line arguments to pass to the application.
-
-ARGS=""
-for ARG do
-    ARGS="$ARGS $ARG"
-done
-
-# Set the CLASSPATH and invoke the Java loader.
-#{JAVA} -classpath $CP Jhove $ARGS -x org.apache.xerces.parsers.SAXParser
-#${JAVA} -classpath $CP Jhove $ARGS
-# New way, doesn't require you to use JAVA_HOME.
-java -classpath $CP Jhove $ARGS

ocrmypdf/jhove/jhove.tmpl

@@ -1,54 +0,0 @@
-#!/bin/sh
-
-########################################################################
-# JHOVE - JSTOR/Harvard Object Validation Environment
-# Copyright 2003-2004 by JSTOR and the President and Fellows of Harvard College
-# JHOVE is made available under the GNU General Public License (see the
-# file LICENSE for details)
-#
-# Copy jhove.tmpl to jhove, and replace the value of JHOVE_HOME with
-# the path to your jhove directory.
-#
-# Usage: jhove [-c config] [-m module [-p param]] [-h handler [-P param]]
-#              [-e encoding] [-H handler] [-o output] [-x saxclass]
-#              [-t tempdir] [-b bufsize] [[-krs] dir-file-or-uri [...]]
-#
-# where -c config   Configuration file pathname
-#       -m module   Module name
-#       -p param    Module-specific parameter
-#       -h handler  Output handler name (defaults to TEXT)
-#       -P param    Handler-specific parameter
-#       -o output   Output file pathname (defaults to standard output)
-#       -x saxclass SAX parser class (defaults to J2SE 1.4 default)
-#       -t tempdir  Temporary directory in which to create temporary files
-#       -b bufsize  Buffer size for buffered I/O (defaults to J2SE 1.4 default)
-#       -k          Calculate CRC32, MD5, and SHA-1 checksums
-#       -r          Display raw data flags, not textual equivalents
-#       -s          Format identification based on internal signatures only
-#       dir-file-or-uri Directory, file pathname or URI of formatted content
-#
-# Configuration constants:
-
-JHOVE_HOME=[your directory path]/jhove
-
-JAVA_HOME=/usr/java
-JAVA=/usr/bin/java
-
-#XTRA_JARS=/users/stephen/xercesImpl.jar
-EXTRA_JARS=              # Extra .jar files to add to CLASSPATH
-
-# NOTE: Nothing below this line should be edited
-########################################################################
-
-CP=${JHOVE_HOME}/bin/JhoveApp.jar:${EXTRA_JARS}
-
-# Retrieve a copy of all command line arguments to pass to the application.
-
-ARGS=""
-for ARG do
-    ARGS="$ARGS $ARG"
-done
-
-# Set the CLASSPATH and invoke the Java loader.
-#{JAVA} -classpath $CP Jhove $ARGS -x org.apache.xerces.parsers.SAXParser
-${JAVA} -classpath $CP Jhove $ARGS

ocrmypdf/jhove/jhove_bat.tmpl

@@ -1,60 +0,0 @@
-@ECHO OFF
-REM JHOVE - JSTOR/Harvard Object Validation Environment
-REM Copyright 2003-2005 by JSTOR and the President and Fellows of Harvard College
-REM JHOVE is made available under the GNU General Public License (see the
-REM file LICENSE for details)
-REM
-REM Usage: jhove [-c config] [-m module] [-h handler] [-e encoding]
-REM              [-H handler] [-o output] [-x saxclass] [-t tempdir]
-REM              [-b bufsize] [-l loglevel] [[-krs] dir-file-or-uri [...]]
-REM
-REM For Windows systems, copy jhove_bat.tmpl to jhove.bat and change
-REM the value of JHOVE_HOME to the path to your jhove directory.
-REM
-REM where -c config   Configuration file pathname
-REM       -m module   Module name
-REM       -h handler  Output handler name (defaults to TEXT)
-REM       -e encoding Character encoding of output handler (defaults to UTF-8)
-REM       -H handler  About handler name
-REM       -o output   Output file pathname (defaults to standard output)
-REM       -x saxclass SAX parser class (defaults to J2SE 1.4 default)
-REM       -t tempdir  Temporary directory in which to create temporary files
-REM       -b bufsize  Buffer size for buffered I/O (defaults to J2SE default)
-REM       -l loglevel Logging level
-REM       -k          Calculate CRC32, MD5, and SHA-1 checksums
-REM       -r          Display raw data flags, not textual equivalents
-REM       -s          Format identification based on internal signatures only
-REM       dir-file-or-uri Directory, file pathname, or URI of formatted content
-REM
-REM Configuration constants:
-REM JHOVE_HOME Jhove installation directory
-REM JAVA_HOME  Java JRE directory
-REM JAVA       Java interpreter
-REM EXTRA_JARS Extra jar files to add to CLASSPATH
-
-REM SET JHOVE_HOME="C:\Program Files\jhove"
-SET JHOVE_HOME="[your directory path]\jhove"
-
-SET EXTRA_JARS=
-
-REM NOTE: Nothing below this line should be edited
-REM #########################################################################
-
-
-SET CP=%JHOVE_HOME%\bin\JhoveApp.jar
-IF "%EXTRA_JARS%"=="" GOTO FI
-  SET CP=%CP%:%EXTRA_JARS
-:FI
-
-REM Retrieve a copy of all command line arguments to pass to the application
-
-SET ARGS=
-:WHILE
-IF "%1"=="" GOTO LOOP
-  SET ARGS=%ARGS% %1
-  SHIFT
-  GOTO WHILE
-:LOOP
-
-REM Set the CLASSPATH and invoke the Java loader
-JAVA -classpath %CP% Jhove %ARGS%

ocrmypdf/jhove/md5.pl

@@ -1,14 +0,0 @@
-#!/usr/bin/perl -w
-
-# Generate MD5 checksum
-
-use Digest::MD5;
-
-die "usage: md5.pl file\n" if $#ARGV < 0;
-
-open (FILE, "<$ARGV[0]") or die "can't open file\"$ARGV[0]\"!\n";
-$ctx = Digest::MD5->new->addfile (*FILE);
-close (FILE);
-$digest = $ctx->hexdigest;
-
-print "$digest\n";

ocrmypdf/jhove/packagejhove.sh

@@ -1,44 +0,0 @@
-#!/bin/sh
-#DO NOT RUN THIS ON A DEVELOPEMENT DIRECTORY, ONLY ON A 
-#CHECKED-OUT COPY TO BE PACKAGED!
-
-if [ "$1" = "" ]; then
- echo "Usage: packagejhove.sh [version]"
- echo "e.g., packagejhove.sh 1_8"
- exit 1
-fi
-
-echo "This script will prepare your directory for uploading."
-echo "DO NOT RUN IT unless you're a developer and know what"
-echo "you are doing. "
-
-echo "Start in the top-level directory of the JHOVE checkout."
-echo "Run ant and ant javadoc and do any necessary testing and"
-echo "committing before running this script."
-echo 
-echo
-
-echo "To continue, enter the secret phrase."
-read OATH
-if [ "$OATH" != "I solemnly swear that I am up to no good" ]; then
- exit 1
-fi
-
-cd ..
-cat >>CVSS << EOF
-CVS
-.cvsignore
-EOF
-tar cvfX jhove-$1.tar CVSS jhove
-gzip jhove-$1.tar
-cp -r jhove jhove-zip
-cd jhove-zip
-find . \( -name CVS -o -name .cvsignore \) -exec rm -r {} \;	
-cd ..
-mv jhove jhove-ok
-mv jhove-zip jhove
-zip -r jhove-$1.zip jhove
-rm -r jhove
-mv jhove-ok jhove
-jhove/md5.pl jhove-$1.tar.gz >jhove-$1.tar.gz.md5
-jhove/md5.pl jhove-$1.zip >jhove-$1.zip.md5 

ocrmypdf/jhove/pdump

@@ -1,37 +0,0 @@
-#!/bin/sh
-
-########################################################################
-# pdump - JSTOR/Harvard Object Validation Environment
-# Copyright 2003-2005 by JSTOR and the President and Fellows of Harvard College
-# JHOVE is made available under the GNU General Public License (see the
-# file LICENSE for details)
-#
-# Driver script for the PDF dump utility
-#
-# Usage: pdump file
-#
-# where  file  is a PDF file
-#
-# Configuration constants:
-
-JHOVE_HOME=/users/stephen/projects/jhove
-
-JAVA_HOME=/usr/java      # Java JRE directory
-JAVA=$JAVA_HOME/bin/java # Java interpreter
-
-EXTRA_JARS=              # Extra .jar files to add to CLASSPATH
-
-# NOTE: Nothing below this line should be edited
-########################################################################
-
-CP=${JHOVE_HOME}/bin/JhoveApp.jar:${EXTRA_JARS}
-
-# Retrieve a copy of all command line arguments to pass to the application.
-
-ARGS=""
-for ARG do
-    ARGS="$ARGS $ARG"
-done
-
-# Set the CLASSPATH and invoke the Java loader.
-${JAVA} -classpath $CP PDump $ARGS

ocrmypdf/jhove/userhome

@@ -1,24 +0,0 @@
-#!/bin/sh
-
-########################################################################
-# userhome - JSTOR/Harvard Object Validation Environment
-# Copyright 2004-2006 by the President and Fellows of Harvard College
-# JHOVE is made available under the GNU General Public License (see the
-# file LICENSE for details)
-#
-# Driver script to display the default Java user.home property
-#
-# Usage: userhome
-#
-# Configuration constants:
-
-JHOVE_HOME=/users/stephen/projects/jhove
-
-JAVA_HOME=/usr/java      # Java JRE directory
-JAVA=$JAVA_HOME/bin/java # Java interpreter
-
-# NOTE: Nothing below this line should be edited
-########################################################################
-
-# Set the CLASSPATH and invoke the Java loader.
-${JAVA} -classpath ${JHOVE_HOME}/classes UserHome

ocrmypdf/leptonica.py

@@ -10,11 +10,16 @@
 
 from __future__ import print_function, absolute_import, division
 import argparse
-import ctypes as C
 import sys
 import os
 import logging
 from tempfile import TemporaryFile
+from ctypes.util import find_library
+from .lib._leptonica import ffi
+from functools import lru_cache
+from enum import Enum
+
+lept = ffi.dlopen(find_library('lept'))
 
 logger = logging.getLogger(__name__)
 
@@ -25,67 +30,6 @@ def stderr(*objs):
     print("leptonica.py:", *objs, file=sys.stderr)
 
 
-from ctypes.util import find_library
-lept_lib = find_library('lept')
-if not lept_lib:
-    stderr("Could not find the Leptonica library")
-    sys.exit(3)
-try:
-    lept = C.cdll.LoadLibrary(lept_lib)
-except Exception:
-    stderr("Could not load the Leptonica library from %s", lept_lib)
-    sys.exit(3)
-
-
-class _PIXCOLORMAP(C.Structure):
-    """struct PixColormap from Leptonica src/pix.h
-    """
-
-    _fields_ = [
-        ("array", C.c_void_p),
-        ("depth", C.c_int32),
-        ("nalloc", C.c_int32),
-        ("n", C.c_int32)
-    ]
-
-
-class _PIX(C.Structure):
-    """struct Pix from Leptonica src/pix.h
-    """
-
-    _fields_ = [
-        ("w", C.c_uint32),
-        ("h", C.c_uint32),
-        ("d", C.c_uint32),
-        ("wpl", C.c_uint32),
-        ("refcount", C.c_uint32),
-        ("xres", C.c_int32),
-        ("yres", C.c_int32),
-        ("informat", C.c_int32),
-        ("text", C.POINTER(C.c_char)),
-        ("colormap", C.POINTER(_PIXCOLORMAP)),
-        ("data", C.POINTER(C.c_uint32))
-    ]
-
-
-PIX = C.POINTER(_PIX)
-
-lept.pixRead.argtypes = [C.c_char_p]
-lept.pixRead.restype = PIX
-lept.pixScale.argtypes = [PIX, C.c_float, C.c_float]
-lept.pixScale.restype = PIX
-lept.pixDeskew.argtypes = [PIX, C.c_int32]
-lept.pixDeskew.restype = PIX
-lept.pixFindSkew.argtypes = [PIX, C.POINTER(C.c_float), C.POINTER(C.c_float)]
-lept.pixFindSkew.restype = C.c_int32
-lept.pixWriteImpliedFormat.argtypes = [C.c_char_p, PIX, C.c_int32, C.c_int32]
-lept.pixWriteImpliedFormat.restype = C.c_int32
-lept.pixDestroy.argtypes = [C.POINTER(PIX)]
-lept.pixDestroy.restype = None
-lept.getLeptonicaVersion.argtypes = []
-lept.getLeptonicaVersion.restype = C.c_char_p
-
-
 class LeptonicaErrorTrap(object):
     """Context manager to trap errors reported by Leptonica.
 
@@ -140,103 +84,368 @@ class LeptonicaIOError(LeptonicaError):
     pass
 
 
-def pixRead(filename):
-    """Load an image file into a PIX object.
+class RemoveColormap(Enum):
+    to_binary = 0
+    to_grayscale = 1
+    to_full_color = 2
+    based_on_src = 3
 
-    Leptonica can load TIFF, PNM (PBM, PGM, PPM), PNG, and JPEG.  If loading
-    fails then the object will wrap a C null pointer.
 
+class Pix:
+    """Wrapper around leptonica's PIX object.
+
+    Leptonica uses referencing counting on PIX objects. Also, many Leptonica
+    functions return the original object with an increased reference count
+    if the operation had no effect (for example, image skew was found to be 0).
+    This has complications for memory management in Python. Whenever Leptonica
+    returns a PIX object (new or old), we wrap it in this class, which
+    registers it with the FFI garbage collector. pixDestroy() decrements the
+    reference count and only destroys when the last reference is removed.
+
+    Leptonica's reference counting is not threadsafe. This class can be used
+    in a threadsafe manner if a Python threading.Lock protects the data.
     """
-    with LeptonicaErrorTrap():
-        return lept.pixRead(filename.encode(sys.getfilesystemencoding()))
 
+    def __init__(self, pix):
+        self._pix = ffi.gc(pix, Pix._pix_destroy)
 
-def pixScale(pix, scalex, scaley):
-    """Returns the pix object rescaled according to the proportions given."""
-    with LeptonicaErrorTrap():
-        return lept.pixScale(pix, scalex, scaley)
-
-
-def pixDeskew(pix, reduction_factor=0):
-    """Returns the deskewed pix object.
-
-    A clone of the original is returned when the algorithm cannot find a skew
-    angle with sufficient confidence.
-
-    reduction_factor -- amount to downsample (0 for default) when searching
-        for skew angle
-
-    """
-    with LeptonicaErrorTrap():
-        return lept.pixDeskew(pix, reduction_factor)
-
-
-def pixFindSkew(pix):
-    """Returns a tuple (deskew angle in degrees, confidence value).
-
-    Returns (None, None) if no angle is available.
-
-    """
-    with LeptonicaErrorTrap():
-        angle = C.c_float(0.0)
-        confidence = C.c_float(0.0)
-        result = lept.pixFindSkew(pix, C.byref(angle), C.byref(confidence))
-        if result == 0:
-            return (angle.value, confidence.value)
+    def __repr__(self):
+        if self._pix:
+            s = "<leptonica.Pix image size={0}x{1} depth={2} at 0x{3:x}>"
+            return s.format(self._pix.w, self._pix.h, self._pix.d,
+                            int(ffi.cast("intptr_t", self._pix)))
         else:
-            return (None, None)
+            return "<leptonica.Pix image NULL>"
+
+    def __getstate__(self):
+        data = ffi.new('l_uint32 **')
+        size = ffi.new('size_t *')
+
+        err = lept.pixSerializeToMemory(self._pix, data, size)
+        if err != 0:
+            raise LeptonicaIOError("pixSerializeToMemory")
+
+        char_data = ffi.cast('char *', data[0])
+
+        # Copy from C bytes to python bytes()
+        data_bytes = ffi.buffer(char_data, size[0])[:]
+
+        # Can now free C bytes
+        lept.lept_free(char_data)
+        return dict(data=data_bytes)
+
+    def __setstate__(self, state):
+        cdata_bytes = ffi.new('char[]', state['data'])
+        cdata_uint32 = ffi.cast('l_uint32 *', cdata_bytes)
+
+        pix = lept.pixDeserializeFromMemory(
+            cdata_uint32, len(state['data']))
+        Pix.__init__(self, pix)
+
+    @property
+    def width(self):
+        return self._pix.w
+
+    @property
+    def height(self):
+        return self._pix.h
+
+    @property
+    def depth(self):
+        return self._pix.d
+
+    @property
+    def size(self):
+        return (self._pix.w, self._pix.h)
+
+    @property
+    def info(self):
+        return {'dpi': (self._pix.xres, self._pix.yres)}
+
+    @property
+    def mode(self):
+        "Return mode like PIL.Image"
+        if self.depth == 1:
+            return '1'
+        elif self.depth >= 16:
+            return 'RGB'
+        elif not self._pix.colormap:
+            return 'L'
+        else:
+            return 'P'
+
+    @classmethod
+    def read(cls, filename):
+        """Load an image file into a PIX object.
+
+        Leptonica can load TIFF, PNM (PBM, PGM, PPM), PNG, and JPEG.  If
+        loading fails then the object will wrap a C null pointer.
+        """
+        with LeptonicaErrorTrap():
+            return cls(lept.pixRead(
+                filename.encode(sys.getfilesystemencoding())))
+
+    def write_implied_format(
+            self, filename, jpeg_quality=0, jpeg_progressive=0):
+        """Write pix to the filename, with the extension indicating format.
+
+        jpeg_quality -- quality (iff JPEG; 1 - 100, 0 for default)
+        jpeg_progressive -- (iff JPEG; 0 for baseline seq., 1 for progressive)
+        """
+        with LeptonicaErrorTrap():
+            lept.pixWriteImpliedFormat(
+                filename.encode(sys.getfilesystemencoding()),
+                self._pix, jpeg_quality, jpeg_progressive)
+
+    def topil(self):
+        "Returns a PIL.Image version of this Pix"
+        from PIL import Image
+
+        with LeptonicaErrorTrap():
+            pix_swapped = Pix(lept.pixEndianByteSwapNew(self._pix))
+
+        size = (pix_swapped._pix.wpl * 4, pix_swapped._pix.h)
+        buf = ffi.buffer(pix_swapped._pix.data, size[0] * size[1])
+
+        im_raw = Image.frombytes(self.mode, size, buf, 'raw')
+
+        # Leptonica stores images in 32-bit words
+        # Need to crop the any trailing amount
+        box = (0, 0, self.width, self.height)
+        im = im_raw.crop(box)
+
+        return im
+
+    def show(self):
+        return self.topil().show()
+
+    def deskew(self, reduction_factor=0):
+        """Returns the deskewed pix object.
+
+        A clone of the original is returned when the algorithm cannot find a
+        skew angle with sufficient confidence.
+
+        reduction_factor -- amount to downsample (0 for default) when searching
+            for skew angle
+        """
+        with LeptonicaErrorTrap():
+            return Pix(lept.pixDeskew(self._pix, reduction_factor))
+
+    def scale(self, scalex, scaley):
+        "Returns the pix object rescaled according to the proportions given."
+        with LeptonicaErrorTrap():
+            return Pix(lept.pixScale(self._pix, scalex, scaley))
+
+    def rotate180(self):
+        with LeptonicaErrorTrap():
+            return Pix(lept.pixRotate180(ffi.NULL, self._pix))
+
+    def rotate_orth(self, quads):
+        "Orthographic rotation, quads: 0-3, number of clockwise rotations"
+        with LeptonicaErrorTrap():
+            return Pix(lept.pixRotateOrth(self._pix, quads))
+
+    def find_skew(self):
+        """Returns a tuple (deskew angle in degrees, confidence value).
+
+        Returns (None, None) if no angle is available.
+        """
+        with LeptonicaErrorTrap():
+            angle = ffi.new('float *', 0.0)
+            confidence = ffi.new('float *', 0.0)
+            result = lept.pixFindSkew(self._pix, angle, confidence)
+            if result == 0:
+                return (angle[0], confidence[0])
+            else:
+                return (None, None)
+
+    def convert_rgb_to_luminance(self):
+        with LeptonicaErrorTrap():
+            gray_pix = lept.pixConvertRGBToLuminance(self._pix)
+            if gray_pix:
+                return Pix(gray_pix)
+            return None
+
+    def remove_colormap(self, removal_type):
+        """Remove a palette
+
+            removal_type - RemovalColormap()
+        """
+
+        with LeptonicaErrorTrap():
+            return Pix(lept.pixRemoveColormap(self._pix, removal_type))
+
+    def otsu_adaptive_threshold(
+            self, tile_size=(300, 300), kernel_size=(4, 4), scorefract=0.1):
+        with LeptonicaErrorTrap():
+            sx, sy = tile_size
+            smoothx, smoothy = kernel_size
+            p_pix = ffi.new('PIX **')
+
+            result = lept.pixOtsuAdaptiveThreshold(
+                self._pix,
+                sx, sy,
+                smoothx, smoothy,
+                scorefract,
+                ffi.NULL,
+                p_pix)
+            if result == 0:
+                return Pix(p_pix[0])
+            else:
+                return None
+
+    def otsu_threshold_on_background_norm(
+            self, mask=None, tile_size=(10, 15), thresh=100, mincount=50,
+            bgval=255, kernel_size=(2, 2), scorefract=0.1):
+        with LeptonicaErrorTrap():
+            sx, sy = tile_size
+            smoothx, smoothy = kernel_size
+            if mask is None:
+                mask = ffi.NULL
+            if isinstance(mask, Pix):
+                mask = mask._pix
+
+            thresh_pix = lept.pixOtsuThreshOnBackgroundNorm(
+                self._pix,
+                mask,
+                sx, sy,
+                thresh, mincount, bgval,
+                smoothx, smoothy,
+                scorefract,
+                ffi.NULL
+                )
+            if thresh_pix == ffi.NULL:
+                return None
+            return Pix(thresh_pix)
+
+    def crop_to_foreground(
+            self, threshold=128, mindist=70, erasedist=30, pagenum=0,
+            showmorph=0, display=0, pdfdir=ffi.NULL):
+        with LeptonicaErrorTrap():
+            cropbox = Box(lept.pixFindPageForeground(
+                self._pix,
+                threshold,
+                mindist,
+                erasedist,
+                pagenum,
+                showmorph,
+                display,
+                pdfdir))
+
+            print(repr(cropbox))
+
+            cropped_pix = lept.pixClipRectangle(
+                self._pix,
+                cropbox._box,
+                ffi.NULL)
+
+            return Pix(cropped_pix)
+
+    def clean_background_to_white(
+            self, mask=None, grayscale=None, gamma=1.0, black=0, white=255):
+        with LeptonicaErrorTrap():
+            return Pix(lept.pixCleanBackgroundToWhite(
+                self._pix,
+                mask or ffi.NULL,
+                grayscale or ffi.NULL,
+                gamma,
+                black,
+                white))
+
+    @staticmethod
+    @lru_cache(maxsize=1)
+    def make_pixel_sum_tab8():
+        return lept.makePixelSumTab8()
+
+    @staticmethod
+    def correlation_binary(pix1, pix2):
+        if get_leptonica_version() < 'leptonica-1.72':
+            # Older versions of Leptonica (pre-1.72) have a buggy
+            # implementation of pixCorrelationBinary that overflows on larger
+            # images.
+            pix1_count = ffi.new('l_int32 *')
+            pix2_count = ffi.new('l_int32 *')
+            pixn_count = ffi.new('l_int32 *')
+            tab8 = Pix.make_pixel_sum_tab8()
+
+            lept.pixCountPixels(pix1._pix, pix1_count, tab8)
+            lept.pixCountPixels(pix2._pix, pix2_count, tab8)
+            pixn = Pix(lept.pixAnd(ffi.NULL, pix1._pix, pix2._pix))
+            lept.pixCountPixels(pixn._pix, pixn_count, tab8)
+
+            # Python converts these int32s to larger units as needed
+            # to avoid overflow. Overflow happens easily here.
+            correlation = (
+                    (pixn_count[0] * pixn_count[0]) /
+                    (pix1_count[0] * pix2_count[0])
+                    )
+            return correlation
+        else:
+            correlation = ffi.new('float *', 0.0)
+            result = lept.pixCorrelationBinary(pix1._pix, pix2._pix,
+                                               correlation)
+            if result != 0:
+                raise LeptonicaError("Correlation failed")
+            return correlation[0]
+
+    @staticmethod
+    def _pix_destroy(pix):
+        p_pix = ffi.new('PIX **', pix)
+        lept.pixDestroy(p_pix)
+        # print('pix destroy ' + repr(pix))
 
 
-def pixWriteImpliedFormat(filename, pix, jpeg_quality=0, jpeg_progressive=0):
-    """Write pix to the filename, with the extension indicating format.
-
-    jpeg_quality -- quality (iff JPEG; 1 - 100, 0 for default)
-    jpeg_progressive -- (iff JPEG; 0 for baseline seq., 1 for progressive)
+class Box:
+    """Wrapper around Leptonica's BOX objects.
 
+    See class Pix for notes about reference counting.
     """
-    fileroot, extension = os.path.splitext(filename)
-    fix_pnm = False
-    if extension.lower() in ('.pbm', '.pgm', '.ppm'):
-        # Leptonica does not process handle these extensions correctly, but
-        # does handle .pnm correctly.  Add another .pnm suffix.
-        filename += '.pnm'
-        fix_pnm = True
 
-    with LeptonicaErrorTrap():
-        lept.pixWriteImpliedFormat(
-            filename.encode(sys.getfilesystemencoding()),
-            pix, jpeg_quality, jpeg_progressive)
+    def __init__(self, box):
+        self._box = ffi.gc(box, Box._box_destroy)
 
-    if fix_pnm:
-        from shutil import move
-        move(filename, filename[:-4])   # Remove .pnm suffix
+    def __repr__(self):
+        if self._box:
+            return '<leptonica.Box x={0} y={1} w={2} h={3}>'.format(
+                self.x, self.y, self.w, self.h)
+        return '<leptonica.Box NULL>'
+
+    @property
+    def x(self):
+        return self._box.x
+
+    @property
+    def y(self):
+        return self._box.y
+
+    @property
+    def w(self):
+        return self._box.w
+
+    @property
+    def h(self):
+        return self._box.h
+
+    @staticmethod
+    def _box_destroy(box):
+        p_box = ffi.new('BOX **', box)
+        lept.boxDestroy(p_box)
 
 
-def pixDestroy(pix):
-    """Destroy the pix object.
-
-    Function signature is pixDestroy(struct Pix **), hence C.byref() to pass
-    the address of the pointer.
-
-    """
-    with LeptonicaErrorTrap():
-        lept.pixDestroy(C.byref(pix))
-
-
-def getLeptonicaVersion():
+@lru_cache(maxsize=1)
+def get_leptonica_version():
     """Get Leptonica version string.
 
     Caveat: Leptonica expects the caller to free this memory.  We don't,
     since that would involve binding to libc to access libc.free(),
     a pointless effort to reclaim 100 bytes of memory.
-
     """
-    return lept.getLeptonicaVersion().decode()
+    return ffi.string(lept.getLeptonicaVersion()).decode()
 
 
 def deskew(infile, outfile, dpi):
     try:
-        pix_source = pixRead(infile)
+        pix_source = Pix.read(infile)
     except LeptonicaIOError:
         raise LeptonicaIOError("Failed to open file: %s" % infile)
 
@@ -244,14 +453,12 @@ def deskew(infile, outfile, dpi):
         reduction_factor = 1  # Don't downsample too much if DPI is already low
     else:
         reduction_factor = 0  # Use default
-    pix_deskewed = pixDeskew(pix_source, reduction_factor)
+    pix_deskewed = pix_source.deskew(reduction_factor)
 
     try:
-        pixWriteImpliedFormat(outfile, pix_deskewed)
+        pix_deskewed.write_implied_format(outfile)
     except LeptonicaIOError:
         raise LeptonicaIOError("Failed to open destination file: %s" % outfile)
-    pixDestroy(pix_source)
-    pixDestroy(pix_deskewed)
 
 
 if __name__ == '__main__':
@@ -270,39 +477,12 @@ if __name__ == '__main__':
 
     args = parser.parse_args()
 
-    if getLeptonicaVersion() != u'leptonica-1.69':
+    if get_leptonica_version() != u'leptonica-1.69':
         print("Unexpected leptonica version: %s" % getLeptonicaVersion())
 
     args.func(args)
 
 
-def _test_output(mode, extension, im_format):
-    from PIL import Image
-    from tempfile import NamedTemporaryFile
-
-    with NamedTemporaryFile(prefix='test-lept-pnm', suffix=extension, delete=True) as tmpfile:
-        im = Image.new(mode=mode, size=(100, 100))
-        im.save(tmpfile)
-
-        pix = pixRead(tmpfile.name)
-        pixWriteImpliedFormat(tmpfile.name, pix)
-        pixDestroy(pix)
-
-        im_roundtrip = Image.open(tmpfile.name)
-        assert im_roundtrip.mode == im.mode, "leptonica mode differs"
-        assert im_roundtrip.format == im_format, \
-            "{0}: leptonica produced a {1}".format(
-                extension,
-                im_roundtrip.format)
-
-
-def test_pnm_output():
-    params = [['1', '.pbm', 'PPM'], ['L', '.pgm', 'PPM'],
-              ['RGB', '.ppm', 'PPM']]
-    for param in params:
-        _test_output(*param)
-
-
 def test_skew_angle():
     from PIL import Image, ImageDraw
     from tempfile import NamedTemporaryFile
@@ -325,7 +505,6 @@ def test_skew_angle():
             rotated_im.save(tmpfile)
             pix = pixRead(tmpfile.name)
             angle, confidence = pixFindSkew(pix)
-            pixDestroy(pix)
             print('{0} {1}  {2}'.format(rotate_angle, angle, confidence), file=sys.stderr)
 
 

ocrmypdf/lib/compile_leptonica.py

@@ -0,0 +1,142 @@
+from cffi import FFI
+
+ffi = FFI()
+ffi.set_source("ocrmypdf.lib._leptonica", None)
+ffi.cdef("""
+typedef signed char             l_int8;
+typedef unsigned char           l_uint8;
+typedef short                   l_int16;
+typedef unsigned short          l_uint16;
+typedef int                     l_int32;
+typedef unsigned int            l_uint32;
+typedef float                   l_float32;
+typedef double                  l_float64;
+typedef long long               l_int64;
+typedef unsigned long long      l_uint64;
+
+struct Pix
+{
+    l_uint32             w;           /* width in pixels                   */
+    l_uint32             h;           /* height in pixels                  */
+    l_uint32             d;           /* depth in bits (bpp)               */
+    l_uint32             spp;         /* number of samples per pixel       */
+    l_uint32             wpl;         /* 32-bit words/line                 */
+    l_uint32             refcount;    /* reference count (1 if no clones)  */
+    l_int32              xres;        /* image res (ppi) in x direction    */
+                                      /* (use 0 if unknown)                */
+    l_int32              yres;        /* image res (ppi) in y direction    */
+                                      /* (use 0 if unknown)                */
+    l_int32              informat;    /* input file format, IFF_*          */
+    l_int32              special;     /* special instructions for I/O, etc */
+    char                *text;        /* text string associated with pix   */
+    struct PixColormap  *colormap;    /* colormap (may be null)            */
+    l_uint32            *data;        /* the image data                    */
+};
+typedef struct Pix PIX;
+
+struct PixColormap
+{
+    void            *array;     /* colormap table (array of RGBA_QUAD)     */
+    l_int32          depth;     /* of pix (1, 2, 4 or 8 bpp)               */
+    l_int32          nalloc;    /* number of color entries allocated       */
+    l_int32          n;         /* number of color entries used            */
+};
+typedef struct PixColormap  PIXCMAP;
+
+struct Box
+{
+    l_int32            x;
+    l_int32            y;
+    l_int32            w;
+    l_int32            h;
+    l_uint32           refcount;      /* reference count (1 if no clones)  */
+
+};
+typedef struct Box    BOX;
+
+""")
+
+ffi.cdef("""
+PIX * pixRead ( const char *filename );
+PIX * pixScale ( PIX *pixs, l_float32 scalex, l_float32 scaley );
+l_int32 pixFindSkew ( PIX *pixs, l_float32 *pangle, l_float32 *pconf );
+l_int32 pixWriteImpliedFormat ( const char *filename, PIX *pix, l_int32 quality, l_int32 progressive );
+void pixDestroy ( PIX **ppix );
+
+PIX *
+pixEndianByteSwapNew(PIX  *pixs);
+
+PIX * pixDeskew ( PIX *pixs, l_int32 redsearch );
+char * getLeptonicaVersion (  );
+l_int32 pixCorrelationBinary(PIX *pix1, PIX *pix2, l_float32 *pval);
+PIX *pixRotate180(PIX *pixd, PIX *pixs);
+PIX *
+pixRotateOrth(PIX     *pixs,
+              l_int32  quads);
+
+l_int32 pixCountPixels ( PIX *pix, l_int32 *pcount, l_int32 *tab8 );
+PIX * pixAnd ( PIX *pixd, PIX *pixs1, PIX *pixs2 );
+l_int32 * makePixelSumTab8 ( void );
+
+PIX * pixDeserializeFromMemory ( const l_uint32 *data, size_t nbytes );
+l_int32 pixSerializeToMemory ( PIX *pixs, l_uint32 **pdata, size_t *pnbytes );
+
+PIX * pixConvertRGBToLuminance(PIX *pixs);
+
+PIX * pixRemoveColormap(PIX *pixs, l_int32  type);
+
+l_int32
+pixOtsuAdaptiveThreshold(PIX       *pixs,
+                         l_int32    sx,
+                         l_int32    sy,
+                         l_int32    smoothx,
+                         l_int32    smoothy,
+                         l_float32  scorefract,
+                         PIX      **ppixth,
+                         PIX      **ppixd);
+
+PIX *
+pixOtsuThreshOnBackgroundNorm(PIX       *pixs,
+                              PIX       *pixim,
+                              l_int32    sx,
+                              l_int32    sy,
+                              l_int32    thresh,
+                              l_int32    mincount,
+                              l_int32    bgval,
+                              l_int32    smoothx,
+                              l_int32    smoothy,
+                              l_float32  scorefract,
+                              l_int32   *pthresh);
+
+PIX *
+pixCleanBackgroundToWhite(PIX       *pixs,
+                          PIX       *pixim,
+                          PIX       *pixg,
+                          l_float32  gamma,
+                          l_int32    blackval,
+                          l_int32    whiteval);
+
+BOX *
+pixFindPageForeground(PIX         *pixs,
+                      l_int32      threshold,
+                      l_int32      mindist,
+                      l_int32      erasedist,
+                      l_int32      pagenum,
+                      l_int32      showmorph,
+                      l_int32      display,
+                      const char  *pdfdir);
+
+PIX *
+pixClipRectangle(PIX   *pixs,
+                 BOX   *box,
+                 BOX  **pboxc);
+
+void
+boxDestroy(BOX  **pbox);
+
+void lept_free(void *ptr);
+""")
+
+
+if __name__ == '__main__':
+    ffi.compile()

ocrmypdf/pageinfo.py

@@ -3,10 +3,13 @@
 
 from subprocess import Popen, PIPE
 from decimal import Decimal, getcontext
+from math import hypot
 import re
 import sys
 import PyPDF2 as pypdf
+from collections import namedtuple
 
+matrix_mult = pypdf.pdf.utils.matrixMultiply
 
 FRIENDLY_COLORSPACE = {
     '/DeviceGray': 'gray',
@@ -19,7 +22,11 @@ FRIENDLY_COLORSPACE = {
     '/Indexed': 'index',
     '/Separation': 'sep',
     '/DeviceN': 'devn',
-    '/Pattern': '-'
+    '/Pattern': '-',
+    '/G': 'gray',  # Abbreviations permitted in inline images
+    '/RGB': 'rgb',
+    '/CMYK': 'cmyk',
+    '/I': 'index',
 }
 
 FRIENDLY_ENCODING = {
@@ -27,6 +34,13 @@ FRIENDLY_ENCODING = {
     '/DCTDecode': 'jpeg',
     '/JPXDecode': 'jpx',
     '/JBIG2Decode': 'jbig2',
+    '/CCF': 'ccitt',  # Abbreviations permitted in inline images
+    '/DCT': 'jpeg',
+    '/AHx': 'asciihex',
+    '/A85': 'ascii85',
+    '/LZW': 'lzw',
+    '/Fl': 'flate',
+    '/RL': 'runlength'
 }
 
 FRIENDLY_COMP = {
@@ -34,43 +48,169 @@ FRIENDLY_COMP = {
     'rgb': 3,
     'cmyk': 4,
     'lab': 3,
+    'index': 1
 }
 
 
-def _page_has_inline_images(page):
-    # PDF always uses \r\n for separator regardless of platform
-    # Really basic heuristic that might trigger the odd false positive
-    # This is only finds the first image and is not quite spec compliant
-    contents = page.getContents()
-    data = contents.getData()
-    begin_image, image_data, end_image = False, False, False
-    for data in re.split(b'\s+', data):
-        if data == b'BI':
-            begin_image = True
-        elif data == b'ID':
-            image_data = True
-        elif data == b'EI':
-            end_image = True
-        if all((begin_image, image_data, end_image)):
-            return True
+def _matrix_from_shorthand(shorthand):
+    """Convert from PDF matrix shorthand to full matrix
+
+    PDF 1.7 spec defines a shorthand for describing the entries of a matrix
+    since the last column is always (0, 0, 1).
+    """
+
+    a, b, c, d, e, f = map(float, shorthand)
+    return ((a, b, 0),
+            (c, d, 0),
+            (e, f, 1))
 
 
-def _find_page_images(page, pageinfo):
+def _shorthand_from_matrix(matrix):
+    """Convert from transformation matrix to PDF shorthand."""
+    a, b = matrix[0][0], matrix[0][1]
+    c, d = matrix[1][0], matrix[1][1]
+    e, f = matrix[2][0], matrix[2][1]
+    return tuple(map(float, (a, b, c, d, e, f)))
+
+
+ContentsInfo = namedtuple('ContentsInfo', ['raster_settings', 'inline_images'])
+
+
+def _interpret_contents(contentstream):
+    operations = contentstream.operations
+    stack = []
+    ctm = _matrix_from_shorthand((1, 0, 0, 1, 0, 0))
+    image_raster_settings = []
+    inline_images = []
+
+    for op in operations:
+        operands, command = op
+        if command == b'q':
+            stack.append(ctm)
+            if len(stack) > 32:
+                raise RuntimeError("PDF graphics stack overflow")
+        elif command == b'Q':
+            ctm = stack.pop()
+        elif command == b'cm':
+            ctm = matrix_mult(
+                _matrix_from_shorthand(operands), ctm)
+        elif command == b'Do':
+            image_name = operands[0]
+            image_raster_settings.append(
+                (image_name, _shorthand_from_matrix(ctm)))
+        elif command == b'INLINE IMAGE':
+            settings = operands['settings']
+            inline_images.append(
+                (settings, _shorthand_from_matrix(ctm)))
+
+    return ContentsInfo(
+        raster_settings=image_raster_settings,
+        inline_images=inline_images)
+
+
+def _get_dpi(ctm_shorthand, image_size):
+    """Given the transformation matrix and image size, find the image DPI.
+
+    PDFs do not include image resolution information within image data.
+    Instead, the PDF page content stream describes the location where the
+    image will be rasterized, and the effective resolution is the ratio of the
+    pixel size to raster target size.
+
+    Normally a scanned PDF has the paper size set appropriately but this is
+    not guaranteed. The most common case is a cropped image will change the
+    page size (/CropBox) without altering the page content stream. That means
+    it is not sufficient to assume that the image fills the page, even though
+    that is the most common case.
+
+    A PDF image may be scaled (always), cropped, translated, rotated in place
+    to an arbitrary angle (rarely) and skewed. Only equal area mappings can
+    be expressed, that is, it is not necessary to consider distortions where
+    the effective DPI varies with position.
+
+    To determine the image scale, transform an offset axis vector v0 (0, 0),
+    width-axis vector v0 (1, 0), height-axis vector vh (0, 1) with the matrix,
+    which gives the dimensions of the image in PDF units. From there we can
+    compare to actual image dimensions. PDF uses
+    row vector * matrix_tranposed unlike the traditional
+    matrix * column vector.
+
+    The offset, width and height vectors can be combined in a matrix and
+    multiplied by the transform matrix. Then we want to calculated
+        magnitude(width_vector - offset_vector)
+    and
+        magnitude(height_vector - offset_vector)
+
+    When the above is worked out algebraically, the effect of translation
+    cancels out, and the vector magnitudes become functions of the nonzero
+    transformation matrix indices. The results of the derivation are used
+    in this code.
+
+    pdfimages -list does calculate the DPI in some way that is not completely
+    naive, but it does not get the DPI of rotated images right, so cannot be
+    used anymore to validate this. Photoshop works, or using Acrobat to
+    rotate the image back to normal.
+
+    It does not matter if the image is partially cropped, or even out of the
+    /MediaBox.
+
+    """
+
+    a, b, c, d, _, _ = ctm_shorthand
+
+    # Calculate the width and height of the image in PDF units
+    image_drawn_width = hypot(a, b)
+    image_drawn_height = hypot(c, d)
+
+    # The scale of the image is pixels per PDF unit (1/72")
+    scale_w = image_size[0] / image_drawn_width
+    scale_h = image_size[1] / image_drawn_height
+
+    # DPI = scale * 72
+    dpi_w = scale_w * 72.0
+    dpi_h = scale_h * 72.0
+
+    return (dpi_w, dpi_h)
+
+
+def _find_page_inline_images(page, pageinfo, contentsinfo):
+    "Find inline images on the page"
+
+    for n, im in enumerate(contentsinfo.inline_images):
+        settings, shorthand = im
+        image = {}
+        image['name'] = str('inline-%02d' % n)
+        image['width'] = settings['/W']
+        image['height'] = settings['/H']
+        image['bpc'] = settings['/BPC']
+        image['color'] = FRIENDLY_COLORSPACE.get(settings['/CS'], '-')
+        image['comp'] = FRIENDLY_COMP.get(image['color'], '?')
+        if '/F' in settings:
+            filter_ = settings['/F']
+            if isinstance(filter_, pypdf.generic.ArrayObject):
+                filter_ = filter_[0]
+            image['enc'] = FRIENDLY_ENCODING.get(filter_, 'image')
+        else:
+            image['enc'] = 'image'
+
+        dpi_w, dpi_h = _get_dpi(shorthand, (image['width'], image['height']))
+        image['dpi_w'], image['dpi_h'] = Decimal(dpi_w), Decimal(dpi_h)
+        yield image
+
+
+def _find_page_regular_images(page, pageinfo, contentsinfo):
+    "Find images stored in XObject resources"
+
     try:
         page['/Resources']['/XObject']
     except KeyError:
         return
-
-    # Look for XObject (out of line images)
     for xobj in page['/Resources']['/XObject']:
         # PyPDF2 returns the keys as an iterator
         pdfimage = page['/Resources']['/XObject'][xobj]
         if pdfimage['/Subtype'] != '/Image':
             continue
-        if '/ImageMask' in pdfimage:
-            if pdfimage['/ImageMask']:
-                continue
         image = {}
+        image['name'] = str(xobj)
         image['width'] = pdfimage['/Width']
         image['height'] = pdfimage['/Height']
         image['bpc'] = pdfimage['/BitsPerComponent']
@@ -90,34 +230,78 @@ def _find_page_images(page, pageinfo):
             image['color'] = 'jpx' if image['enc'] == 'jpx' else '?'
 
         image['comp'] = FRIENDLY_COMP.get(image['color'], '?')
-        image['dpi_w'] = image['width'] / pageinfo['width_inches']
-        image['dpi_h'] = image['height'] / pageinfo['height_inches']
+        image['dpi_w'] = image['dpi_h'] = 0
+
+        for raster in contentsinfo.raster_settings:
+            # Loop in case the same image is display multiple times on a page
+            if raster[0] != image['name']:
+                continue
+            shorthand = raster[1]
+
+            dpi_w, dpi_h = _get_dpi(
+                shorthand, (image['width'], image['height']))
+
+            # When image is used multiple times take the highest DPI it is
+            # rendered at
+            image['dpi_w'] = max(dpi_w, image.get('dpi_w', 0))
+            image['dpi_h'] = max(dpi_h, image.get('dpi_h', 0))
+
+        image['dpi_w'] = Decimal(image['dpi_w'])
+        image['dpi_h'] = Decimal(image['dpi_h'])
         image['dpi'] = (image['dpi_w'] * image['dpi_h']) ** Decimal(0.5)
         yield image
 
 
-def _pdf_get_pageinfo(infile, page: int):
+def _find_page_images(page, pageinfo, contentsinfo):
+    yield from _find_page_inline_images(page, pageinfo, contentsinfo)
+    yield from _find_page_regular_images(page, pageinfo, contentsinfo)
+
+
+def _page_has_text(pdf, page):
+    # Simple test
+    text = page.extractText()
+    if text.strip() != '':
+        return True
+
+    # More nuanced test to deal with quirks of Tesseract PDF generation
+    # Check if there's a Glyphless font
+    try:
+        font = page['/Resources']['/Font']
+    except KeyError:
+        pass
+    else:
+        font_objects = list(font.keys())
+        for font_object in font_objects:
+            basefont = font[font_object]['/BaseFont']
+            if basefont.endswith('GlyphLessFont'):
+                return True
+
+    return False
+
+
+def _pdf_get_pageinfo(infile, pageno: int):
     pageinfo = {}
-    pageinfo['pageno'] = page
+    pageinfo['pageno'] = pageno
     pageinfo['images'] = []
 
     pdf = pypdf.PdfFileReader(infile)
-    page = pdf.pages[page - 1]
+    page = pdf.pages[pageno]
 
-    text = page.extractText()
-    pageinfo['has_text'] = (text.strip() != '')
+    pageinfo['has_text'] = _page_has_text(pdf, page)
 
-    width_pt = page['/MediaBox'][2] - page['/MediaBox'][0]
-    height_pt = page['/MediaBox'][3] - page['/MediaBox'][1]
+    width_pt = page.mediaBox.getWidth()
+    height_pt = page.mediaBox.getHeight()
     pageinfo['width_inches'] = width_pt / Decimal(72.0)
     pageinfo['height_inches'] = height_pt / Decimal(72.0)
 
-    pageinfo['images'] = [im for im in _find_page_images(page, pageinfo)]
+    try:
+        contentstream = pypdf.pdf.ContentStream(page.getContents(), pdf)
+    except AttributeError as e:
+        return pageinfo
 
-    # Look for inline images
-    if _page_has_inline_images(page):
-        raise NotImplementedError(
-            "Warning: input PDF contains inline images - not supported")
+    contentsinfo = _interpret_contents(contentstream)
+    pageinfo['images'] = [im for im in _find_page_images(
+                                page, pageinfo, contentsinfo)]
 
     if pageinfo['images']:
         xres = max(image['dpi_w'] for image in pageinfo['images'])
@@ -135,3 +319,17 @@ def pdf_get_all_pageinfo(infile):
     pdf = pypdf.PdfFileReader(infile)
     getcontext().prec = 6
     return [_pdf_get_pageinfo(infile, n) for n in range(pdf.numPages)]
+
+
+def main():
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument('infile')
+    args = parser.parse_args()
+    info = pdf_get_all_pageinfo(args.infile)
+    from pprint import pprint
+    pprint(info)
+
+
+if __name__ == '__main__':
+    main()

ocrmypdf/pdfa.py

@@ -5,9 +5,13 @@
 
 from __future__ import print_function, absolute_import, division
 from string import Template
-from subprocess import Popen, PIPE
-import os
 import codecs
+import pkg_resources
+
+ICC_PROFILE_RELPATH = 'data/sRGB.icc'
+
+SRGB_ICC_PROFILE = pkg_resources.resource_filename(
+    'ocrmypdf', ICC_PROFILE_RELPATH)
 
 
 # This is a template written in PostScript which is needed to create PDF/A
@@ -27,6 +31,7 @@ def
   /Author <$author>
   /Subject <$subject>
   /Keywords <$keywords>
+  /Creator <$creator>
   /DOCINFO pdfmark
 
 % Define an ICC profile :
@@ -86,39 +91,14 @@ def _get_pdfa_def(icc_profile, icc_identifier, pdfmark):
                           title=pdfmark_utf16.get('title', ''),
                           author=pdfmark_utf16.get('author', ''),
                           subject=pdfmark_utf16.get('subject', ''),
+                          creator=pdfmark_utf16.get('creator', ''),
                           keywords=pdfmark_utf16.get('keywords', ''))
     return result
 
 
-def _get_postscript_icc_path():
-    "Parse Ghostscript's help message to find where iccprofiles are stored"
-
-    p_gs = Popen(['gs', '--help'], close_fds=True, universal_newlines=True,
-                 stdout=PIPE, stderr=PIPE)
-    out, _ = p_gs.communicate()
-    lines = out.splitlines()
-
-    def search_paths(lines):
-        seeking = True
-        for line in lines:
-            if seeking:
-                if line.startswith('Search path'):
-                    seeking = False
-                    continue
-            else:
-                if line.strip().startswith('/'):
-                    yield from (
-                        path.strip() for path in line.split(':')
-                        if path.strip() != '')
-    for root in search_paths(lines):
-        path = os.path.realpath(os.path.join(root, '../iccprofiles'))
-        if os.path.exists(path):
-            return path
-
-
 def generate_pdfa_def(target_filename, pdfmark, icc='sRGB'):
     if icc == 'sRGB':
-        icc_profile = os.path.join(_get_postscript_icc_path(), 'srgb.icc')
+        icc_profile = SRGB_ICC_PROFILE
     else:
         raise NotImplementedError("Only supporting sRGB")
 

ocrmypdf/qpdf.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+# © 2015 James R. Barlow: github.com/jbarlow83
+
+from subprocess import CalledProcessError, check_output, STDOUT, check_call
+import sys
+import os
+
+from . import ExitCode, get_program
+
+
+def check(input_file, log):
+    args_qpdf = [
+        get_program('qpdf'),
+        '--check',
+        input_file
+    ]
+
+    try:
+        check_output(args_qpdf, stderr=STDOUT, universal_newlines=True)
+    except CalledProcessError as e:
+        if e.returncode == 2:
+            log.error("{0}: not a valid PDF, and could not repair it.".format(
+                    input_file))
+            log.error("Details:")
+            log.error(e.output)
+        elif e.returncode == 3:
+            log.info("qpdf --check returned warnings:")
+            log.info(e.output)
+        else:
+            log.warning(e.output)
+        return False
+    return True
+
+
+def repair(input_file, output_file, log):
+    args_qpdf = [
+        get_program('qpdf'), input_file, output_file
+    ]
+    try:
+        check_output(args_qpdf, stderr=STDOUT, universal_newlines=True)
+    except CalledProcessError as e:
+        if e.returncode == 3 and e.output.find("operation succeeded"):
+            log.debug('qpdf found and fixed errors: ' + e.output)
+            log.debug(e.output)
+            return
+
+        if e.returncode == 2 and e.output.find("invalid password"):
+            log.error("{0}: this PDF is password-protected - password must "
+                      "be removed for OCR".format(input_file))
+            sys.exit(ExitCode.input_file)
+        elif e.returncode == 2:
+            log.error("{0}: not a valid PDF, and could not repair it.".format(
+                      input_file))
+            log.error("Details: " + e.output)
+            sys.exit(ExitCode.input_file)
+        else:
+            log.error("{0}: unknown error".format(
+                      input_file))
+            log.error(e.output)
+            sys.exit(ExitCode.unknown)
+
+
+def get_npages(input_file, log):
+    try:
+        pages = check_output(
+            [get_program('qpdf'), '--show-npages', input_file],
+             universal_newlines=True, close_fds=True)
+    except CalledProcessError as e:
+        if e.returncode == 2 and e.output.find('No such file'):
+            log.error(e.output)
+            sys.exit(ExitCode.input_file)
+    return int(pages)
+
+
+def split_pages(input_file, work_folder, npages):
+    """Split multipage PDF into individual pages.
+
+    Incredibly enough, this multiple process approach is about 70 times
+    faster than using Ghostscript.
+    """
+    for n in range(int(npages)):
+        args_qpdf = [
+            get_program('qpdf'), input_file,
+            '--pages', input_file, '{0}'.format(n + 1), '--',
+            os.path.join(work_folder, '{0:06d}.page.pdf'.format(n + 1))
+        ]
+        check_call(args_qpdf)

ocrmypdf/tesseract.py

@@ -1,42 +1,25 @@
 #!/usr/bin/env python3
 # © 2015 James R. Barlow: github.com/jbarlow83
 
-from subprocess import STDOUT, CalledProcessError, check_output
 import sys
 import os
 import re
+import shutil
 from functools import lru_cache
+from . import ExitCode, get_program, page_number
+from collections import namedtuple
+
+from subprocess import Popen, PIPE, CalledProcessError, \
+    TimeoutExpired, check_output, STDOUT
+try:
+    from subprocess import DEVNULL
+except ImportError:
+    DEVNULL = open(os.devnull, 'wb')
 
 
-@lru_cache(maxsize=1)
-def version():
-    args_tess = [
-        'tesseract',
-        '--version'
-    ]
-    try:
-        versions = check_output(
-                args_tess, close_fds=True, universal_newlines=True,
-                stderr=STDOUT)
-    except CalledProcessError:
-        print("Could not find Tesseract executable on system PATH.")
-        sys.exit(1)
-
-    tesseract_version = re.match(r'tesseract\s(.+)', versions).group(1)
-    return tesseract_version
-
-
-@lru_cache(maxsize=1)
-def languages():
-    args_tess = [
-        'tesseract',
-        '--list-langs'
-    ]
-    langs = check_output(
-            args_tess, close_fds=True, universal_newlines=True,
-            stderr=STDOUT)
-    return set(lang.strip() for lang in langs.splitlines()[1:])
-
+OrientationConfidence = namedtuple(
+    'OrientationConfidence',
+    ('angle', 'confidence'))
 
 HOCR_TEMPLATE = '''<?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
@@ -59,3 +42,219 @@ HOCR_TEMPLATE = '''<?xml version="1.0" encoding="UTF-8"?>
   </div>
  </body>
 </html>'''
+
+
+@lru_cache(maxsize=1)
+def version():
+    args_tess = [
+        get_program('tesseract'),
+        '--version'
+    ]
+    try:
+        versions = check_output(
+                args_tess, close_fds=True, universal_newlines=True,
+                stderr=STDOUT)
+    except CalledProcessError:
+        print("Could not find Tesseract executable on system PATH.")
+        sys.exit(ExitCode.missing_dependency)
+
+    tesseract_version = re.match(r'tesseract\s(.+)', versions).group(1)
+    return tesseract_version
+
+
+@lru_cache(maxsize=1)
+def languages():
+    args_tess = [
+        get_program('tesseract'),
+        '--list-langs'
+    ]
+    try:
+        langs = check_output(
+                args_tess, close_fds=True, universal_newlines=True,
+                stderr=STDOUT)
+    except CalledProcessError as e:
+        print("Tesseract failed to report available languages.")
+        print("Output from Tesseract:")
+        print("-" * 40)
+        print(e.output)
+        sys.exit(ExitCode.missing_dependency)
+    return set(lang.strip() for lang in langs.splitlines()[1:])
+
+
+def get_orientation(input_file, language: list, timeout: float, log):
+    args_tesseract = [
+        get_program('tesseract'),
+        '-l', '+'.join(language),
+        '-psm', '0',
+        input_file,
+        'stdout'
+    ]
+
+    try:
+        stdout = check_output(
+            args_tesseract, close_fds=True, stderr=STDOUT,
+            universal_newlines=True, timeout=timeout)
+    except TimeoutExpired:
+        return OrientationConfidence(angle=0, confidence=0.0)
+    except CalledProcessError as e:
+        tesseract_log_output(log, e.output, input_file)
+        if ('Too few characters. Skipping this page' in e.output or
+                'Image too large' in e.output):
+            return OrientationConfidence(0, 0)
+        raise e from e
+    else:
+        osd = {}
+        for line in stdout.splitlines():
+            line = line.strip()
+            parts = line.split(':', maxsplit=2)
+            if len(parts) == 2:
+                osd[parts[0].strip()] = parts[1].strip()
+
+        angle = int(osd.get('Orientation in degrees', 0))
+        if 'Orientation' in osd:
+            # Tesseract < 3.04.01
+            # reports "Orientation in degrees" as a counterclockwise angle
+            # We keep it clockwise
+            assert 'Rotate' not in osd
+            angle = -angle % 360
+        else:
+            # Tesseract == 3.04.01, hopefully also Tesseract > 3.04.01
+            # reports "Orientation in degrees" as a clockwise angle
+            assert 'Rotate' in osd
+
+        oc = OrientationConfidence(
+            angle=angle,
+            confidence=float(osd.get('Orientation confidence', 0)))
+        return oc
+
+
+def tesseract_log_output(log, stdout, input_file):
+    lines = stdout.splitlines()
+    prefix = "{0:4d}: [tesseract] ".format(page_number(input_file))
+    for line in lines:
+        if line.startswith("Tesseract Open Source"):
+            continue
+        elif line.startswith("Warning in pixReadMem"):
+            continue
+        elif 'diacritics' in line:
+            log.warning(prefix + "lots of diacritics - possibly poor OCR")
+        elif line.startswith('OSD: Weak margin'):
+            log.warning(prefix + "unsure about page orientation")
+        elif 'error' in line.lower() or 'exception' in line.lower():
+            log.error(prefix + line.strip())
+        else:
+            log.info(prefix + line.strip())
+
+
+def page_timedout(log, input_file):
+    prefix = "{0:4d}: [tesseract] ".format(page_number(input_file))
+    log.warning(prefix + " took too long to OCR - skipping")
+
+
+def _generate_null_hocr(output_hocr, pageinfo):
+    with open(output_hocr, 'w', encoding="utf-8") as f:
+        f.write(HOCR_TEMPLATE.format(
+            pageinfo['width_pixels'],
+            pageinfo['height_pixels']))
+
+
+def generate_hocr(input_file, output_hocr, language: list, tessconfig: list,
+                  timeout: float, pageinfo_getter, pagesegmode: int, log):
+
+    badxml = os.path.splitext(output_hocr)[0] + '.badxml'
+
+    args_tesseract = [
+        get_program('tesseract'),
+        '-l', '+'.join(language)
+    ]
+
+    if pagesegmode is not None:
+        args_tesseract.extend(['-psm', str(pagesegmode)])
+
+    args_tesseract.extend([
+        input_file,
+        badxml,
+        'hocr'
+    ] + tessconfig)
+    try:
+        stdout = check_output(
+            args_tesseract, close_fds=True, stderr=STDOUT,
+            universal_newlines=True, timeout=timeout)
+    except TimeoutExpired:
+        # Generate a HOCR file with no recognized text if tesseract times out
+        # Temporary workaround to hocrTransform not being able to function if
+        # it does not have a valid hOCR file.
+        page_timedout(log, input_file)
+        _generate_null_hocr(output_hocr, pageinfo_getter())
+    except CalledProcessError as e:
+        tesseract_log_output(log, e.output, input_file)
+        if 'Image too large' in e.output:
+            _generate_null_hocr(output_hocr, pageinfo_getter())
+            return
+
+        raise e from e
+    else:
+        tesseract_log_output(log, stdout, input_file)
+        if os.path.exists(badxml + '.html'):
+            # Tesseract 3.02 appends suffix ".html" on its own (.badxml.html)
+            shutil.move(badxml + '.html', badxml)
+        elif os.path.exists(badxml + '.hocr'):
+            # Tesseract 3.03 appends suffix ".hocr" on its own (.badxml.hocr)
+            shutil.move(badxml + '.hocr', badxml)
+
+        # Tesseract 3.03 inserts source filename into hocr file without
+        # escaping it, creating invalid XML and breaking the parser.
+        # As a workaround, rewrite the hocr file, replacing the filename
+        # with a space.  Don't know if Tesseract 3.02 does the same.
+
+        regex_nested_single_quotes = re.compile(
+            r"""title='image "([^"]*)";""")
+        with open(badxml, mode='r', encoding='utf-8') as f_in, \
+                open(output_hocr, mode='w', encoding='utf-8') as f_out:
+            for line in f_in:
+                line = regex_nested_single_quotes.sub(
+                    r"""title='image " ";""", line)
+                f_out.write(line)
+
+
+def generate_pdf(input_image, skip_pdf, output_pdf, language: list,
+                 tessconfig: list, timeout: float, pagesegmode: int, log):
+    '''Use Tesseract to render a PDF.
+
+    input_image -- image to analyze
+    skip_pdf -- if we time out, use this file as output
+    language -- list of languages to consider
+    tessconfig -- tesseract configuration
+    timeout -- timeout (seconds)
+    log -- logger object
+    '''
+
+    args_tesseract = [
+        get_program('tesseract'),
+        '-l', '+'.join(language)
+    ]
+
+    if pagesegmode is not None:
+        args_tesseract.extend(['-psm', str(pagesegmode)])
+
+    args_tesseract.extend([
+        input_image,
+        os.path.splitext(output_pdf)[0],  # Tesseract appends suffix
+        'pdf'
+    ] + tessconfig)
+
+    try:
+        stdout = check_output(
+            args_tesseract, close_fds=True, stderr=STDOUT,
+            universal_newlines=True, timeout=timeout)
+    except TimeoutExpired:
+        page_timedout(log, input_image)
+        shutil.copy(skip_pdf, output_pdf)
+    except CalledProcessError as e:
+        tesseract_log_output(log, e.output, input_image)
+        if 'Image too large' in e.output:
+            shutil.copy(skip_pdf, output_pdf)
+            return
+        raise e from e
+    else:
+        tesseract_log_output(log, stdout, input_image)

ocrmypdf/unpaper.py

@@ -3,23 +3,23 @@
 # unpaper documentation:
 # https://github.com/Flameeyes/unpaper/blob/master/doc/basic-concepts.md
 
-from subprocess import Popen, PIPE
+from subprocess import CalledProcessError, STDOUT, check_output, check_call
 from tempfile import NamedTemporaryFile
 import sys
 import os
 from functools import lru_cache
+from . import ExitCode, get_program
 
 
 @lru_cache(maxsize=1)
 def version():
     args_unpaper = [
-        'unpaper',
+        get_program('unpaper'),
         '--version'
     ]
-    p_unpaper = Popen(args_unpaper, close_fds=True, universal_newlines=True,
-                      stdout=PIPE, stderr=PIPE)
-    version, _ = p_unpaper.communicate(timeout=5)
-
+    version = check_output(
+        args_unpaper, close_fds=True, universal_newlines=True,
+        stderr=STDOUT, timeout=5)
     return version.strip()
 
 
@@ -32,16 +32,33 @@ except ImportError:
 
 def run(input_file, output_file, dpi, log, mode_args):
     args_unpaper = [
-        'unpaper',
+        get_program('unpaper'),
         '-v',
         '--dpi', str(dpi)
     ] + mode_args
 
     SUFFIXES = {'1': '.pbm', 'L': '.pgm', 'RGB': '.ppm'}
-    suffix = ''
 
     im = Image.open(input_file)
-    suffix = SUFFIXES[im.mode]
+    if im.mode not in SUFFIXES.keys():
+        log.info("Converting image to other colorspace")
+        try:
+            if im.mode == 'P' and len(im.getcolors()) == 2:
+                im = im.convert(mode='1')
+            else:
+                im = im.convert(mode='RGB')
+        except IOError:
+            log.error(
+                    "Could not convert image with type " + im.mode)
+            sys.exit(ExitCode.missing_dependency)
+
+    try:
+        suffix = SUFFIXES[im.mode]
+    except KeyError:
+        log.error(
+                "Failed to convert image to a supported format.")
+        sys.exit(ExitCode.missing_dependency)
+
     with NamedTemporaryFile(suffix=suffix) as input_pnm, \
             NamedTemporaryFile(suffix=suffix, mode="r+b") as output_pnm:
         im.save(input_pnm, format='PPM')
@@ -50,15 +67,17 @@ def run(input_file, output_file, dpi, log, mode_args):
         os.unlink(output_pnm.name)
 
         args_unpaper.extend([input_pnm.name, output_pnm.name])
-        p_unpaper = Popen(
-            args_unpaper, close_fds=True,
-            universal_newlines=True, stdout=PIPE, stderr=PIPE
-            )
-        out, err = p_unpaper.communicate()
-        log.debug(out)
-        log.debug(err)
-
-        Image.open(output_pnm.name).save(output_file)
+        try:
+            stdout = check_output(
+                args_unpaper, close_fds=True,
+                universal_newlines=True, stderr=STDOUT,
+                )
+        except CalledProcessError as e:
+            log.debug(e.output)
+            raise e from e
+        else:
+            log.debug(stdout)
+            Image.open(output_pnm.name).save(output_file)
 
 
 def deskew(input_file, output_file, dpi, log):

pipeline.svg

@@ -4,232 +4,294 @@
 <!-- Generated by graphviz version 2.38.0 (20140413.2041)
  -->
 <!-- Title: Pipeline: Pages: 1 -->
-<svg width="728pt" height="651pt"
- viewBox="0.00 0.00 728.00 650.53" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
-<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 646.53)">
+<svg width="1452pt" height="824pt"
+ viewBox="0.00 0.00 1452.00 824.08" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 820.083)">
 <title>Pipeline:</title>
-<polygon fill="white" stroke="none" points="-4,4 -4,-646.53 724,-646.53 724,4 -4,4"/>
+<polygon fill="white" stroke="none" points="-4,4 -4,-820.083 1448,-820.083 1448,4 -4,4"/>
 <g id="clust1" class="cluster"><title>clustertasks</title>
-<polygon fill="none" stroke="black" points="8,-8 8,-634.53 712,-634.53 712,-8 8,-8"/>
-<text text-anchor="middle" x="360" y="-606.53" font-family="Times,serif" font-size="30.00" fill="#ff3232">Pipeline:</text>
+<polygon fill="none" stroke="black" points="8,-8 8,-808.083 1436,-808.083 1436,-8 8,-8"/>
+<text text-anchor="middle" x="722" y="-780.083" font-family="Times,serif" font-size="30.00" fill="#ff3232">Pipeline:</text>
 </g>
 <!-- t0 -->
 <g id="node1" class="node"><title>t0</title>
-<polygon fill="#efa03b" stroke="#006000" points="481.791,-588.53 386.209,-588.53 382.209,-584.53 382.209,-552.53 477.791,-552.53 481.791,-556.53 481.791,-588.53"/>
-<polyline fill="none" stroke="#006000" points="477.791,-584.53 382.209,-584.53 "/>
-<polyline fill="none" stroke="#006000" points="477.791,-584.53 477.791,-552.53 "/>
-<polyline fill="none" stroke="#006000" points="477.791,-584.53 481.791,-588.53 "/>
-<text text-anchor="middle" x="432" y="-564.53" font-family="Times,serif" font-size="20.00" fill="#006000">repair_pdf</text>
+<polygon fill="#efa03b" stroke="black" points="1256.54,-762.083 1033.46,-762.083 1029.46,-758.083 1029.46,-726.083 1252.54,-726.083 1256.54,-730.083 1256.54,-762.083"/>
+<polyline fill="none" stroke="black" points="1252.54,-758.083 1029.46,-758.083 "/>
+<polyline fill="none" stroke="black" points="1252.54,-758.083 1252.54,-726.083 "/>
+<polyline fill="none" stroke="black" points="1252.54,-758.083 1256.54,-762.083 "/>
+<text text-anchor="middle" x="1143" y="-738.083" font-family="Times,serif" font-size="20.00">repair_pdf</text>
 </g>
 <!-- t1 -->
 <g id="node2" class="node"><title>t1</title>
-<polygon fill="#efa03b" stroke="black" points="466.782,-509.564 374,-526.497 281.218,-509.564 281.304,-482.165 466.696,-482.165 466.782,-509.564"/>
-<polygon fill="none" stroke="black" points="470.799,-512.902 374,-530.569 277.201,-512.902 277.311,-478.159 470.689,-478.159 470.799,-512.902"/>
-<text text-anchor="middle" x="374" y="-495.991" font-family="Times,serif" font-size="20.00">split_pages</text>
+<polygon fill="#efa03b" stroke="black" points="1234.11,-683.155 1030,-700.057 825.888,-683.155 826.078,-655.806 1233.92,-655.806 1234.11,-683.155"/>
+<polygon fill="none" stroke="black" points="1238.13,-686.834 1030,-704.069 821.866,-686.834 822.11,-651.808 1237.89,-651.808 1238.13,-686.834"/>
+<text text-anchor="middle" x="1030" y="-669.596" font-family="Times,serif" font-size="20.00">split_pages</text>
 </g>
 <!-- t0&#45;&gt;t1 -->
 <g id="edge1" class="edge"><title>t0&#45;&gt;t1</title>
-<path fill="none" stroke="gray" d="M417.064,-552.394C412.296,-546.925 406.86,-540.689 401.493,-534.532"/>
-<polygon fill="gray" stroke="gray" points="403.996,-532.077 394.787,-526.838 398.719,-536.676 403.996,-532.077"/>
+<path fill="none" stroke="#0044a0" d="M1113.9,-725.961C1103.58,-719.89 1091.66,-712.873 1080.09,-706.069"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1081.75,-702.982 1071.35,-700.927 1078.2,-709.015 1081.75,-702.982"/>
 </g>
-<!-- t10 -->
-<g id="node12" class="node"><title>t10</title>
-<polygon fill="#efa03b" stroke="#006000" points="704.338,-451.452 493.662,-451.452 489.662,-447.452 489.662,-415.452 700.338,-415.452 704.338,-419.452 704.338,-451.452"/>
-<polyline fill="none" stroke="#006000" points="700.338,-447.452 489.662,-447.452 "/>
-<polyline fill="none" stroke="#006000" points="700.338,-447.452 700.338,-415.452 "/>
-<polyline fill="none" stroke="#006000" points="700.338,-447.452 704.338,-451.452 "/>
-<text text-anchor="middle" x="597" y="-427.452" font-family="Times,serif" font-size="20.00" fill="#006000">generate_postscript_stub</text>
+<!-- t14 -->
+<g id="node16" class="node"><title>t14</title>
+<polygon fill="#efa03b" stroke="black" points="1428.08,-625.109 1089.92,-625.109 1085.92,-621.109 1085.92,-589.109 1424.08,-589.109 1428.08,-593.109 1428.08,-625.109"/>
+<polyline fill="none" stroke="black" points="1424.08,-621.109 1085.92,-621.109 "/>
+<polyline fill="none" stroke="black" points="1424.08,-621.109 1424.08,-589.109 "/>
+<polyline fill="none" stroke="black" points="1424.08,-621.109 1428.08,-625.109 "/>
+<text text-anchor="middle" x="1257" y="-601.109" font-family="Times,serif" font-size="20.00">generate_postscript_stub</text>
 </g>
-<!-- t0&#45;&gt;t10 -->
-<g id="edge16" class="edge"><title>t0&#45;&gt;t10</title>
-<path fill="none" stroke="gray" d="M453.048,-552.468C461.441,-545.654 471.185,-537.73 480,-530.53 510.227,-505.84 544.753,-477.467 568.421,-457.99"/>
-<polygon fill="gray" stroke="gray" points="570.818,-460.55 576.315,-451.493 566.369,-455.146 570.818,-460.55"/>
+<!-- t0&#45;&gt;t14 -->
+<g id="edge22" class="edge"><title>t0&#45;&gt;t14</title>
+<path fill="none" stroke="#0044a0" d="M1219.32,-726.004C1229.98,-720.592 1239.75,-713.466 1247,-704.083 1261.92,-684.78 1263.1,-656.385 1261.35,-635.486"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1264.8,-634.881 1260.22,-625.328 1257.84,-635.653 1264.8,-634.881"/>
 </g>
 <!-- t2 -->
 <g id="node3" class="node"><title>t2</title>
-<polygon fill="#efa03b" stroke="black" points="451.555,-451.452 228.445,-451.452 224.445,-447.452 224.445,-415.452 447.555,-415.452 451.555,-419.452 451.555,-451.452"/>
-<polyline fill="none" stroke="black" points="447.555,-447.452 224.445,-447.452 "/>
-<polyline fill="none" stroke="black" points="447.555,-447.452 447.555,-415.452 "/>
-<polyline fill="none" stroke="black" points="447.555,-447.452 451.555,-451.452 "/>
-<text text-anchor="middle" x="338" y="-427.452" font-family="Times,serif" font-size="20.00">rasterize_with_ghostscript</text>
+<polygon fill="#efa03b" stroke="black" points="1029.88,-625.109 748.117,-625.109 744.117,-621.109 744.117,-589.109 1025.88,-589.109 1029.88,-593.109 1029.88,-625.109"/>
+<polyline fill="none" stroke="black" points="1025.88,-621.109 744.117,-621.109 "/>
+<polyline fill="none" stroke="black" points="1025.88,-621.109 1025.88,-589.109 "/>
+<polyline fill="none" stroke="black" points="1025.88,-621.109 1029.88,-625.109 "/>
+<text text-anchor="middle" x="887" y="-601.109" font-family="Times,serif" font-size="20.00">rasterize_preview</text>
 </g>
 <!-- t1&#45;&gt;t2 -->
 <g id="edge2" class="edge"><title>t1&#45;&gt;t2</title>
-<path fill="none" stroke="#0044a0" d="M361.611,-478.092C358.514,-472.369 355.171,-466.19 352.003,-460.333"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="355.064,-458.636 347.227,-451.506 348.907,-461.967 355.064,-458.636"/>
-</g>
-<!-- t11 -->
-<g id="node10" class="node"><title>t11</title>
-<polygon fill="#efa03b" stroke="black" points="644.594,-393.452 551.406,-393.452 547.406,-389.452 547.406,-357.452 640.594,-357.452 644.594,-361.452 644.594,-393.452"/>
-<polyline fill="none" stroke="black" points="640.594,-389.452 547.406,-389.452 "/>
-<polyline fill="none" stroke="black" points="640.594,-389.452 640.594,-357.452 "/>
-<polyline fill="none" stroke="black" points="640.594,-389.452 644.594,-393.452 "/>
-<text text-anchor="middle" x="596" y="-369.452" font-family="Times,serif" font-size="20.00">skip_page</text>
-</g>
-<!-- t1&#45;&gt;t11 -->
-<g id="edge13" class="edge"><title>t1&#45;&gt;t11</title>
-<path fill="none" stroke="#0044a0" d="M425.139,-478.007C437.834,-470.731 450.731,-461.83 461,-451.452 473.874,-438.442 466.946,-427.178 481,-415.452 490.212,-407.766 513.965,-399.205 537.508,-392.067"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="538.592,-395.396 547.189,-389.203 536.607,-388.683 538.592,-395.396"/>
-</g>
-<!-- t9 -->
-<g id="node11" class="node"><title>t9</title>
-<polygon fill="#efa03b" stroke="black" points="272.496,-277.452 19.5039,-277.452 15.5039,-273.452 15.5039,-241.452 268.496,-241.452 272.496,-245.452 272.496,-277.452"/>
-<polyline fill="none" stroke="black" points="268.496,-273.452 15.5039,-273.452 "/>
-<polyline fill="none" stroke="black" points="268.496,-273.452 268.496,-241.452 "/>
-<polyline fill="none" stroke="black" points="268.496,-273.452 272.496,-277.452 "/>
-<text text-anchor="middle" x="144" y="-253.452" font-family="Times,serif" font-size="20.00">tesseract_ocr_and_render_pdf</text>
-</g>
-<!-- t1&#45;&gt;t9 -->
-<g id="edge15" class="edge"><title>t1&#45;&gt;t9</title>
-<path fill="none" stroke="#0044a0" d="M277.341,-486.044C254.708,-478.702 232.21,-467.759 215,-451.452 168.207,-407.114 152.056,-329.127 146.633,-287.855"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="150.084,-287.235 145.422,-277.721 143.133,-288.065 150.084,-287.235"/>
+<path fill="none" stroke="#0044a0" d="M980.788,-651.715C965.266,-644.498 948.187,-636.558 932.905,-629.452"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="934.195,-626.193 923.652,-625.15 931.244,-632.54 934.195,-626.193"/>
 </g>
 <!-- t3 -->
 <g id="node4" class="node"><title>t3</title>
-<polygon fill="#efa03b" stroke="black" points="458.999,-393.452 291.001,-393.452 287.001,-389.452 287.001,-357.452 454.999,-357.452 458.999,-361.452 458.999,-393.452"/>
-<polyline fill="none" stroke="black" points="454.999,-389.452 287.001,-389.452 "/>
-<polyline fill="none" stroke="black" points="454.999,-389.452 454.999,-357.452 "/>
-<polyline fill="none" stroke="black" points="454.999,-389.452 458.999,-393.452 "/>
-<text text-anchor="middle" x="373" y="-369.452" font-family="Times,serif" font-size="20.00">preprocess_deskew</text>
+<polygon fill="#efa03b" stroke="black" points="1128.15,-567.109 893.852,-567.109 889.852,-563.109 889.852,-531.109 1124.15,-531.109 1128.15,-535.109 1128.15,-567.109"/>
+<polyline fill="none" stroke="black" points="1124.15,-563.109 889.852,-563.109 "/>
+<polyline fill="none" stroke="black" points="1124.15,-563.109 1124.15,-531.109 "/>
+<polyline fill="none" stroke="black" points="1124.15,-563.109 1128.15,-567.109 "/>
+<text text-anchor="middle" x="1009" y="-543.109" font-family="Times,serif" font-size="20.00">orient_page</text>
+</g>
+<!-- t1&#45;&gt;t3 -->
+<g id="edge4" class="edge"><title>t1&#45;&gt;t3</title>
+<path fill="none" stroke="#0044a0" d="M1037.63,-651.508C1042.21,-633.96 1045.92,-609.455 1039,-589.109 1037.4,-584.404 1034.98,-579.822 1032.19,-575.552"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1034.85,-573.258 1026.1,-567.282 1029.21,-577.409 1034.85,-573.258"/>
 </g>
 <!-- t2&#45;&gt;t3 -->
 <g id="edge3" class="edge"><title>t2&#45;&gt;t3</title>
-<path fill="none" stroke="#0044a0" d="M348.691,-415.346C351.337,-411.112 354.23,-406.485 357.066,-401.946"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="360.042,-403.788 362.374,-393.453 354.106,-400.078 360.042,-403.788"/>
-</g>
-<!-- t6 -->
-<g id="node6" class="node"><title>t6</title>
-<polygon fill="#efa03b" stroke="black" points="664.375,-277.452 477.625,-277.452 473.625,-273.452 473.625,-241.452 660.375,-241.452 664.375,-245.452 664.375,-277.452"/>
-<polyline fill="none" stroke="black" points="660.375,-273.452 473.625,-273.452 "/>
-<polyline fill="none" stroke="black" points="660.375,-273.452 660.375,-241.452 "/>
-<polyline fill="none" stroke="black" points="660.375,-273.452 664.375,-277.452 "/>
-<text text-anchor="middle" x="569" y="-253.452" font-family="Times,serif" font-size="20.00">select_image_for_pdf</text>
-</g>
-<!-- t2&#45;&gt;t6 -->
-<g id="edge7" class="edge"><title>t2&#45;&gt;t6</title>
-<path fill="none" stroke="#0044a0" d="M423.032,-415.374C438.812,-409.925 454.543,-402.785 468,-393.452 508.201,-365.571 539.287,-316.662 555.808,-286.558"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="559.024,-287.965 560.653,-277.496 552.851,-284.664 559.024,-287.965"/>
+<path fill="none" stroke="#0044a0" d="M924.267,-589.003C936.328,-583.467 949.857,-577.257 962.502,-571.453"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="964.333,-574.463 971.961,-567.111 961.413,-568.102 964.333,-574.463"/>
 </g>
 <!-- t4 -->
 <g id="node5" class="node"><title>t4</title>
-<polygon fill="#efa03b" stroke="black" points="449.705,-335.452 300.295,-335.452 296.295,-331.452 296.295,-299.452 445.705,-299.452 449.705,-303.452 449.705,-335.452"/>
-<polyline fill="none" stroke="black" points="445.705,-331.452 296.295,-331.452 "/>
-<polyline fill="none" stroke="black" points="445.705,-331.452 445.705,-299.452 "/>
-<polyline fill="none" stroke="black" points="445.705,-331.452 449.705,-335.452 "/>
-<text text-anchor="middle" x="373" y="-311.452" font-family="Times,serif" font-size="20.00">preprocess_clean</text>
+<polygon fill="#efa03b" stroke="black" points="1027.3,-509.109 676.701,-509.109 672.701,-505.109 672.701,-473.109 1023.3,-473.109 1027.3,-477.109 1027.3,-509.109"/>
+<polyline fill="none" stroke="black" points="1023.3,-505.109 672.701,-505.109 "/>
+<polyline fill="none" stroke="black" points="1023.3,-505.109 1023.3,-473.109 "/>
+<polyline fill="none" stroke="black" points="1023.3,-505.109 1027.3,-509.109 "/>
+<text text-anchor="middle" x="850" y="-485.109" font-family="Times,serif" font-size="20.00">rasterize_with_ghostscript</text>
 </g>
 <!-- t3&#45;&gt;t4 -->
-<g id="edge4" class="edge"><title>t3&#45;&gt;t4</title>
-<path fill="none" stroke="#0044a0" d="M373,-357.346C373,-353.655 373,-349.665 373,-345.695"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="376.5,-345.453 373,-335.453 369.5,-345.453 376.5,-345.453"/>
+<g id="edge5" class="edge"><title>t3&#45;&gt;t4</title>
+<path fill="none" stroke="#0044a0" d="M960.431,-531.003C943.787,-525.141 924.999,-518.524 907.704,-512.433"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="908.867,-509.132 898.272,-509.111 906.541,-515.734 908.867,-509.132"/>
 </g>
-<!-- t3&#45;&gt;t6 -->
-<g id="edge6" class="edge"><title>t3&#45;&gt;t6</title>
-<path fill="none" stroke="#0044a0" d="M414.399,-357.412C428.785,-351.039 444.852,-343.405 459,-335.452 466.909,-331.007 506.49,-303.805 535.921,-283.431"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="538.073,-286.198 544.299,-277.625 534.086,-280.444 538.073,-286.198"/>
+<!-- t9 -->
+<g id="node11" class="node"><title>t9</title>
+<polygon fill="#efa03b" stroke="black" points="314.109,-277.109 19.8906,-277.109 15.8906,-273.109 15.8906,-241.109 310.109,-241.109 314.109,-245.109 314.109,-277.109"/>
+<polyline fill="none" stroke="black" points="310.109,-273.109 15.8906,-273.109 "/>
+<polyline fill="none" stroke="black" points="310.109,-273.109 310.109,-241.109 "/>
+<polyline fill="none" stroke="black" points="310.109,-273.109 314.109,-277.109 "/>
+<text text-anchor="middle" x="165" y="-253.109" font-family="Times,serif" font-size="20.00">select_image_layer</text>
 </g>
-<!-- t4&#45;&gt;t6 -->
-<g id="edge5" class="edge"><title>t4&#45;&gt;t6</title>
-<path fill="none" stroke="#0044a0" d="M432.605,-299.422C453.701,-293.395 477.617,-286.561 499.465,-280.319"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="500.687,-283.61 509.341,-277.497 498.764,-276.879 500.687,-283.61"/>
+<!-- t3&#45;&gt;t9 -->
+<g id="edge14" class="edge"><title>t3&#45;&gt;t9</title>
+<path fill="none" stroke="#0044a0" d="M889.675,-542.297C675.505,-530.255 248,-498.192 248,-434.109 248,-434.109 248,-434.109 248,-374.109 248,-337.597 219.83,-304.936 196.312,-284.031"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="198.437,-281.243 188.562,-277.404 193.888,-286.563 198.437,-281.243"/>
+</g>
+<!-- t15 -->
+<g id="node14" class="node"><title>t15</title>
+<polygon fill="#efa03b" stroke="black" points="1391.34,-451.109 1170.66,-451.109 1166.66,-447.109 1166.66,-415.109 1387.34,-415.109 1391.34,-419.109 1391.34,-451.109"/>
+<polyline fill="none" stroke="black" points="1387.34,-447.109 1166.66,-447.109 "/>
+<polyline fill="none" stroke="black" points="1387.34,-447.109 1387.34,-415.109 "/>
+<polyline fill="none" stroke="black" points="1387.34,-447.109 1391.34,-451.109 "/>
+<text text-anchor="middle" x="1279" y="-427.109" font-family="Times,serif" font-size="20.00">skip_page</text>
+</g>
+<!-- t3&#45;&gt;t15 -->
+<g id="edge19" class="edge"><title>t3&#45;&gt;t15</title>
+<path fill="none" stroke="#0044a0" d="M1060.32,-531.078C1078.95,-524.591 1100.07,-516.881 1119,-509.109 1159.3,-492.568 1204.11,-471.304 1236.04,-455.645"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1237.67,-458.74 1245.1,-451.181 1234.58,-452.461 1237.67,-458.74"/>
+</g>
+<!-- t13 -->
+<g id="node15" class="node"><title>t13</title>
+<polygon fill="#efa03b" stroke="black" points="1388.24,-277.109 1007.76,-277.109 1003.76,-273.109 1003.76,-241.109 1384.24,-241.109 1388.24,-245.109 1388.24,-277.109"/>
+<polyline fill="none" stroke="black" points="1384.24,-273.109 1003.76,-273.109 "/>
+<polyline fill="none" stroke="black" points="1384.24,-273.109 1384.24,-241.109 "/>
+<polyline fill="none" stroke="black" points="1384.24,-273.109 1388.24,-277.109 "/>
+<text text-anchor="middle" x="1196" y="-253.109" font-family="Times,serif" font-size="20.00">tesseract_ocr_and_render_pdf</text>
+</g>
+<!-- t3&#45;&gt;t13 -->
+<g id="edge21" class="edge"><title>t3&#45;&gt;t13</title>
+<path fill="none" stroke="#0044a0" d="M1040.79,-531.094C1071.69,-511.983 1114,-477.834 1114,-434.109 1114,-434.109 1114,-434.109 1114,-374.109 1114,-337.749 1141.83,-305.063 1165.07,-284.11"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1167.46,-286.664 1172.72,-277.466 1162.87,-281.377 1167.46,-286.664"/>
 </g>
 <!-- t5 -->
-<g id="node7" class="node"><title>t5</title>
-<polygon fill="#efa03b" stroke="black" points="455.922,-277.452 294.078,-277.452 290.078,-273.452 290.078,-241.452 451.922,-241.452 455.922,-245.452 455.922,-277.452"/>
-<polyline fill="none" stroke="black" points="451.922,-273.452 290.078,-273.452 "/>
-<polyline fill="none" stroke="black" points="451.922,-273.452 451.922,-241.452 "/>
-<polyline fill="none" stroke="black" points="451.922,-273.452 455.922,-277.452 "/>
-<text text-anchor="middle" x="373" y="-253.452" font-family="Times,serif" font-size="20.00">ocr_tesseract_hocr</text>
+<g id="node6" class="node"><title>t5</title>
+<polygon fill="#efa03b" stroke="black" points="830.742,-451.109 535.258,-451.109 531.258,-447.109 531.258,-415.109 826.742,-415.109 830.742,-419.109 830.742,-451.109"/>
+<polyline fill="none" stroke="black" points="826.742,-447.109 531.258,-447.109 "/>
+<polyline fill="none" stroke="black" points="826.742,-447.109 826.742,-415.109 "/>
+<polyline fill="none" stroke="black" points="826.742,-447.109 830.742,-451.109 "/>
+<text text-anchor="middle" x="681" y="-427.109" font-family="Times,serif" font-size="20.00">preprocess_deskew</text>
 </g>
 <!-- t4&#45;&gt;t5 -->
-<g id="edge8" class="edge"><title>t4&#45;&gt;t5</title>
-<path fill="none" stroke="#0044a0" d="M373,-299.346C373,-295.655 373,-291.665 373,-287.695"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="376.5,-287.453 373,-277.453 369.5,-287.453 376.5,-287.453"/>
+<g id="edge6" class="edge"><title>t4&#45;&gt;t5</title>
+<path fill="none" stroke="#0044a0" d="M798.376,-473.003C780.521,-467.087 760.346,-460.402 741.823,-454.264"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="742.901,-450.934 732.308,-451.111 740.699,-457.579 742.901,-450.934"/>
 </g>
-<!-- t4&#45;&gt;t9 -->
-<g id="edge14" class="edge"><title>t4&#45;&gt;t9</title>
-<path fill="none" stroke="#0044a0" d="M303.36,-299.422C278.157,-293.259 249.509,-286.253 223.521,-279.898"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="224.249,-276.473 213.704,-277.497 222.586,-283.273 224.249,-276.473"/>
+<!-- t8 -->
+<g id="node10" class="node"><title>t8</title>
+<polygon fill="#efa03b" stroke="black" points="980.119,-335.109 665.881,-335.109 661.881,-331.109 661.881,-299.109 976.119,-299.109 980.119,-303.109 980.119,-335.109"/>
+<polyline fill="none" stroke="black" points="976.119,-331.109 661.881,-331.109 "/>
+<polyline fill="none" stroke="black" points="976.119,-331.109 976.119,-299.109 "/>
+<polyline fill="none" stroke="black" points="976.119,-331.109 980.119,-335.109 "/>
+<text text-anchor="middle" x="821" y="-311.109" font-family="Times,serif" font-size="20.00">select_image_for_pdf</text>
+</g>
+<!-- t4&#45;&gt;t8 -->
+<g id="edge12" class="edge"><title>t4&#45;&gt;t8</title>
+<path fill="none" stroke="#0044a0" d="M848.03,-473.087C846.196,-457.857 843.261,-434.941 840,-415.109 836.1,-391.393 830.78,-364.592 826.761,-345.181"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="830.152,-344.296 824.68,-335.224 823.3,-345.728 830.152,-344.296"/>
+</g>
+<!-- t6 -->
+<g id="node7" class="node"><title>t6</title>
+<polygon fill="#efa03b" stroke="black" points="801.95,-393.109 524.05,-393.109 520.05,-389.109 520.05,-357.109 797.95,-357.109 801.95,-361.109 801.95,-393.109"/>
+<polyline fill="none" stroke="black" points="797.95,-389.109 520.05,-389.109 "/>
+<polyline fill="none" stroke="black" points="797.95,-389.109 797.95,-357.109 "/>
+<polyline fill="none" stroke="black" points="797.95,-389.109 801.95,-393.109 "/>
+<text text-anchor="middle" x="661" y="-369.109" font-family="Times,serif" font-size="20.00">preprocess_clean</text>
+</g>
+<!-- t5&#45;&gt;t6 -->
+<g id="edge7" class="edge"><title>t5&#45;&gt;t6</title>
+<path fill="none" stroke="#0044a0" d="M674.891,-415.003C673.495,-411.095 671.979,-406.852 670.48,-402.652"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="673.731,-401.351 667.072,-393.111 667.139,-403.705 673.731,-401.351"/>
+</g>
+<!-- t5&#45;&gt;t8 -->
+<g id="edge11" class="edge"><title>t5&#45;&gt;t8</title>
+<path fill="none" stroke="#0044a0" d="M781.944,-415.046C792.967,-409.579 803.051,-402.434 811,-393.109 822.009,-380.195 824.454,-361.215 824.174,-345.581"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="827.659,-345.2 823.617,-335.407 820.669,-345.583 827.659,-345.2"/>
 </g>
 <!-- t7 -->
 <g id="node8" class="node"><title>t7</title>
-<polygon fill="#efa03b" stroke="black" points="426.366,-219.452 269.634,-219.452 265.634,-215.452 265.634,-183.452 422.366,-183.452 426.366,-187.452 426.366,-219.452"/>
-<polyline fill="none" stroke="black" points="422.366,-215.452 265.634,-215.452 "/>
-<polyline fill="none" stroke="black" points="422.366,-215.452 422.366,-183.452 "/>
-<polyline fill="none" stroke="black" points="422.366,-215.452 426.366,-219.452 "/>
-<text text-anchor="middle" x="346" y="-195.452" font-family="Times,serif" font-size="20.00">render_hocr_page</text>
+<polygon fill="#efa03b" stroke="black" points="643.666,-335.109 354.334,-335.109 350.334,-331.109 350.334,-299.109 639.666,-299.109 643.666,-303.109 643.666,-335.109"/>
+<polyline fill="none" stroke="black" points="639.666,-331.109 350.334,-331.109 "/>
+<polyline fill="none" stroke="black" points="639.666,-331.109 639.666,-299.109 "/>
+<polyline fill="none" stroke="black" points="639.666,-331.109 643.666,-335.109 "/>
+<text text-anchor="middle" x="497" y="-311.109" font-family="Times,serif" font-size="20.00">ocr_tesseract_hocr</text>
 </g>
 <!-- t6&#45;&gt;t7 -->
-<g id="edge9" class="edge"><title>t6&#45;&gt;t7</title>
-<path fill="none" stroke="#0044a0" d="M501.184,-241.422C476.75,-235.286 448.99,-228.315 423.772,-221.982"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="424.429,-218.538 413.877,-219.497 422.724,-225.328 424.429,-218.538"/>
-</g>
-<!-- t8 -->
-<g id="node9" class="node"><title>t8</title>
-<polygon fill="#efa03b" stroke="black" points="663.742,-219.452 448.258,-219.452 444.258,-215.452 444.258,-183.452 659.742,-183.452 663.742,-187.452 663.742,-219.452"/>
-<polyline fill="none" stroke="black" points="659.742,-215.452 444.258,-215.452 "/>
-<polyline fill="none" stroke="black" points="659.742,-215.452 659.742,-183.452 "/>
-<polyline fill="none" stroke="black" points="659.742,-215.452 663.742,-219.452 "/>
-<text text-anchor="middle" x="554" y="-195.452" font-family="Times,serif" font-size="20.00">render_hocr_debug_page</text>
+<g id="edge8" class="edge"><title>t6&#45;&gt;t7</title>
+<path fill="none" stroke="#0044a0" d="M610.903,-357.003C593.736,-351.141 574.358,-344.524 556.518,-338.433"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="557.384,-335.03 546.79,-335.111 555.122,-341.654 557.384,-335.03"/>
 </g>
 <!-- t6&#45;&gt;t8 -->
-<g id="edge11" class="edge"><title>t6&#45;&gt;t8</title>
-<path fill="none" stroke="#0044a0" d="M564.418,-241.346C563.4,-237.546 562.298,-233.43 561.203,-229.345"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="564.522,-228.207 558.554,-219.453 557.761,-230.018 564.522,-228.207"/>
+<g id="edge10" class="edge"><title>t6&#45;&gt;t8</title>
+<path fill="none" stroke="#0044a0" d="M709.875,-357.003C726.624,-351.141 745.529,-344.524 762.933,-338.433"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="764.142,-341.718 772.425,-335.111 761.83,-335.111 764.142,-341.718"/>
 </g>
-<!-- t5&#45;&gt;t7 -->
-<g id="edge10" class="edge"><title>t5&#45;&gt;t7</title>
-<path fill="none" stroke="#0044a0" d="M364.752,-241.346C362.816,-237.329 360.708,-232.958 358.629,-228.645"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="361.693,-226.941 354.197,-219.453 355.387,-229.981 361.693,-226.941"/>
+<!-- t10 -->
+<g id="node9" class="node"><title>t10</title>
+<polygon fill="#efa03b" stroke="black" points="620.109,-277.109 335.891,-277.109 331.891,-273.109 331.891,-241.109 616.109,-241.109 620.109,-245.109 620.109,-277.109"/>
+<polyline fill="none" stroke="black" points="616.109,-273.109 331.891,-273.109 "/>
+<polyline fill="none" stroke="black" points="616.109,-273.109 616.109,-241.109 "/>
+<polyline fill="none" stroke="black" points="616.109,-273.109 620.109,-277.109 "/>
+<text text-anchor="middle" x="476" y="-253.109" font-family="Times,serif" font-size="20.00">render_hocr_page</text>
 </g>
-<!-- t5&#45;&gt;t8 -->
-<g id="edge12" class="edge"><title>t5&#45;&gt;t8</title>
-<path fill="none" stroke="#0044a0" d="M428.289,-241.346C447.587,-235.375 469.416,-228.622 489.404,-222.437"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="490.53,-225.753 499.049,-219.453 488.461,-219.065 490.53,-225.753"/>
+<!-- t7&#45;&gt;t10 -->
+<g id="edge9" class="edge"><title>t7&#45;&gt;t10</title>
+<path fill="none" stroke="#0044a0" d="M490.585,-299.003C489.12,-295.095 487.528,-290.852 485.953,-286.652"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="489.164,-285.245 482.376,-277.111 482.61,-287.703 489.164,-285.245"/>
+</g>
+<!-- t11 -->
+<g id="node13" class="node"><title>t11</title>
+<polygon fill="#efa03b" stroke="black" points="985.486,-277.109 642.514,-277.109 638.514,-273.109 638.514,-241.109 981.486,-241.109 985.486,-245.109 985.486,-277.109"/>
+<polyline fill="none" stroke="black" points="981.486,-273.109 638.514,-273.109 "/>
+<polyline fill="none" stroke="black" points="981.486,-273.109 981.486,-241.109 "/>
+<polyline fill="none" stroke="black" points="981.486,-273.109 985.486,-277.109 "/>
+<text text-anchor="middle" x="812" y="-253.109" font-family="Times,serif" font-size="20.00">render_hocr_debug_page</text>
+</g>
+<!-- t7&#45;&gt;t11 -->
+<g id="edge18" class="edge"><title>t7&#45;&gt;t11</title>
+<path fill="none" stroke="#0044a0" d="M592.793,-299.079C628.531,-292.726 669.303,-285.478 705.912,-278.969"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="706.887,-282.351 716.12,-277.155 705.661,-275.459 706.887,-282.351"/>
 </g>
 <!-- t12 -->
-<g id="node13" class="node"><title>t12</title>
-<polygon fill="#efa03b" stroke="black" points="473.845,-105.456 554,-78.0208 634.155,-105.456 634.08,-149.848 473.92,-149.848 473.845,-105.456"/>
-<polygon fill="none" stroke="black" points="469.836,-102.581 554,-73.7729 638.164,-102.581 638.078,-153.869 469.922,-153.869 469.836,-102.581"/>
-<text text-anchor="middle" x="554" y="-111.726" font-family="Times,serif" font-size="20.00">merge_pages</text>
-</g>
-<!-- t7&#45;&gt;t12 -->
-<g id="edge20" class="edge"><title>t7&#45;&gt;t12</title>
-<path fill="none" stroke="#0044a0" d="M389.35,-183.419C409.937,-175.33 435.398,-165.326 460.092,-155.624"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="461.479,-158.839 469.507,-151.924 458.919,-152.324 461.479,-158.839"/>
-</g>
-<!-- t8&#45;&gt;t12 -->
-<g id="edge19" class="edge"><title>t8&#45;&gt;t12</title>
-<path fill="none" stroke="#0044a0" d="M554,-183.12C554,-177.585 554,-171.177 554,-164.592"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="557.5,-164.201 554,-154.201 550.5,-164.201 557.5,-164.201"/>
-</g>
-<!-- t11&#45;&gt;t12 -->
-<g id="edge17" class="edge"><title>t11&#45;&gt;t12</title>
-<path fill="none" stroke="#0044a0" d="M615.939,-357.13C634.767,-339.333 661.745,-309.733 673,-277.452 686.754,-238.003 694.347,-219.364 673,-183.452 666.345,-172.256 656.999,-162.875 646.44,-155.047"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="648.378,-152.132 638.152,-149.36 644.417,-157.904 648.378,-152.132"/>
-</g>
-<!-- t9&#45;&gt;t12 -->
-<g id="edge18" class="edge"><title>t9&#45;&gt;t12</title>
-<path fill="none" stroke="#0044a0" d="M164.711,-241.235C186.394,-224.08 222.088,-198.224 257,-183.452 321.895,-155.994 399.893,-139.571 459.67,-130.141"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="460.504,-133.554 469.855,-128.574 459.439,-126.635 460.504,-133.554"/>
+<g id="node12" class="node"><title>t12</title>
+<polygon fill="#efa03b" stroke="black" points="607.082,-219.109 348.918,-219.109 344.918,-215.109 344.918,-183.109 603.082,-183.109 607.082,-187.109 607.082,-219.109"/>
+<polyline fill="none" stroke="black" points="603.082,-215.109 344.918,-215.109 "/>
+<polyline fill="none" stroke="black" points="603.082,-215.109 603.082,-183.109 "/>
+<polyline fill="none" stroke="black" points="603.082,-215.109 607.082,-219.109 "/>
+<text text-anchor="middle" x="476" y="-195.109" font-family="Times,serif" font-size="20.00">add_text_layer</text>
 </g>
 <!-- t10&#45;&gt;t12 -->
-<g id="edge22" class="edge"><title>t10&#45;&gt;t12</title>
-<path fill="none" stroke="gray" d="M629.379,-415.337C660.552,-396.255 703,-362.233 703,-318.452 703,-318.452 703,-318.452 703,-258.452 703,-224.066 705.581,-210.223 684,-183.452 673.976,-171.017 660.952,-160.773 647.026,-152.396"/>
-<polygon fill="gray" stroke="gray" points="648.682,-149.312 638.258,-147.419 645.227,-155.4 648.682,-149.312"/>
+<g id="edge15" class="edge"><title>t10&#45;&gt;t12</title>
+<path fill="none" stroke="#0044a0" d="M476,-241.003C476,-237.312 476,-233.322 476,-229.352"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="479.5,-229.111 476,-219.111 472.5,-229.111 479.5,-229.111"/>
 </g>
-<!-- t13 -->
-<g id="node14" class="node"><title>t13</title>
-<polygon fill="#efa03b" stroke="black" points="616.338,-52 495.662,-52 491.662,-48 491.662,-16 612.338,-16 616.338,-20 616.338,-52"/>
-<polyline fill="none" stroke="black" points="612.338,-48 491.662,-48 "/>
-<polyline fill="none" stroke="black" points="612.338,-48 612.338,-16 "/>
-<polyline fill="none" stroke="black" points="612.338,-48 616.338,-52 "/>
-<text text-anchor="middle" x="554" y="-28" font-family="Times,serif" font-size="20.00">validate_pdfa</text>
+<!-- t8&#45;&gt;t9 -->
+<g id="edge13" class="edge"><title>t8&#45;&gt;t9</title>
+<path fill="none" stroke="#0044a0" d="M661.842,-299.864C658.871,-299.606 655.921,-299.354 653,-299.109 509.812,-287.095 471.159,-289.734 324.542,-277.219"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="324.526,-273.705 314.262,-276.332 323.924,-280.679 324.526,-273.705"/>
 </g>
-<!-- t12&#45;&gt;t13 -->
-<g id="edge21" class="edge"><title>t12&#45;&gt;t13</title>
-<path fill="none" stroke="#0044a0" d="M554,-73.9482C554,-69.9654 554,-66.007 554,-62.2247"/>
-<polygon fill="#0044a0" stroke="#0044a0" points="557.5,-62.1573 554,-52.1573 550.5,-62.1574 557.5,-62.1573"/>
+<!-- t8&#45;&gt;t11 -->
+<g id="edge17" class="edge"><title>t8&#45;&gt;t11</title>
+<path fill="none" stroke="#0044a0" d="M818.251,-299.003C817.64,-295.204 816.979,-291.087 816.322,-287.002"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="819.775,-286.429 814.732,-277.111 812.864,-287.539 819.775,-286.429"/>
+</g>
+<!-- t8&#45;&gt;t13 -->
+<g id="edge20" class="edge"><title>t8&#45;&gt;t13</title>
+<path fill="none" stroke="#0044a0" d="M935.04,-299.079C978.311,-292.618 1027.78,-285.23 1071.94,-278.636"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1072.48,-282.093 1081.86,-277.155 1071.45,-275.17 1072.48,-282.093"/>
+</g>
+<!-- t9&#45;&gt;t12 -->
+<g id="edge16" class="edge"><title>t9&#45;&gt;t12</title>
+<path fill="none" stroke="#0044a0" d="M259.577,-241.079C294.86,-234.726 335.115,-227.478 371.259,-220.969"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="372.116,-224.372 381.337,-219.155 370.875,-217.482 372.116,-224.372"/>
+</g>
+<!-- t16 -->
+<g id="node17" class="node"><title>t16</title>
+<polygon fill="#efa03b" stroke="black" points="1031.47,-105.333 1196,-78.005 1360.53,-105.333 1360.37,-149.551 1031.63,-149.551 1031.47,-105.333"/>
+<polygon fill="none" stroke="black" points="1027.46,-101.94 1196,-73.9453 1364.54,-101.94 1364.36,-153.556 1027.64,-153.556 1027.46,-101.94"/>
+<text text-anchor="middle" x="1196" y="-111.555" font-family="Times,serif" font-size="20.00">merge_pages</text>
+</g>
+<!-- t12&#45;&gt;t16 -->
+<g id="edge26" class="edge"><title>t12&#45;&gt;t16</title>
+<path fill="none" stroke="#0044a0" d="M607.222,-185.246C722.142,-172.229 890.172,-153.196 1017.34,-138.791"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1018.04,-142.235 1027.58,-137.632 1017.25,-135.279 1018.04,-142.235"/>
+</g>
+<!-- t11&#45;&gt;t16 -->
+<g id="edge27" class="edge"><title>t11&#45;&gt;t16</title>
+<path fill="none" stroke="#0044a0" d="M859.086,-240.997C916.223,-220.232 1014.29,-184.591 1089.94,-157.099"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1091.32,-160.323 1099.52,-153.618 1088.92,-153.744 1091.32,-160.323"/>
+</g>
+<!-- t15&#45;&gt;t16 -->
+<g id="edge23" class="edge"><title>t15&#45;&gt;t16</title>
+<path fill="none" stroke="#0044a0" d="M1321.98,-414.938C1359.16,-396.916 1407,-364.681 1407,-318.109 1407,-318.109 1407,-318.109 1407,-258.109 1407,-211.469 1370.69,-179.509 1328.2,-158.076"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1329.57,-154.854 1319.04,-153.66 1326.53,-161.158 1329.57,-154.854"/>
+</g>
+<!-- t13&#45;&gt;t16 -->
+<g id="edge25" class="edge"><title>t13&#45;&gt;t16</title>
+<path fill="none" stroke="#0044a0" d="M1196,-240.997C1196,-221.914 1196,-190.265 1196,-163.926"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1199.5,-163.759 1196,-153.759 1192.5,-163.759 1199.5,-163.759"/>
+</g>
+<!-- t14&#45;&gt;t16 -->
+<g id="edge24" class="edge"><title>t14&#45;&gt;t16</title>
+<path fill="none" stroke="#0044a0" d="M1326.64,-588.956C1373.58,-572.748 1427,-543.103 1427,-492.109 1427,-492.109 1427,-492.109 1427,-258.109 1427,-220.539 1416.55,-207.537 1388,-183.109 1376.82,-173.542 1364.16,-165.377 1350.82,-158.413"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1351.99,-155.087 1341.47,-153.781 1348.88,-161.358 1351.99,-155.087"/>
+</g>
+<!-- t17 -->
+<g id="node18" class="node"><title>t17</title>
+<polygon fill="#efa03b" stroke="black" points="1310.18,-52 1085.82,-52 1081.82,-48 1081.82,-16 1306.18,-16 1310.18,-20 1310.18,-52"/>
+<polyline fill="none" stroke="black" points="1306.18,-48 1081.82,-48 "/>
+<polyline fill="none" stroke="black" points="1306.18,-48 1306.18,-16 "/>
+<polyline fill="none" stroke="black" points="1306.18,-48 1310.18,-52 "/>
+<text text-anchor="middle" x="1196" y="-28" font-family="Times,serif" font-size="20.00">copy_final</text>
+</g>
+<!-- t16&#45;&gt;t17 -->
+<g id="edge28" class="edge"><title>t16&#45;&gt;t17</title>
+<path fill="none" stroke="#0044a0" d="M1196,-73.8665C1196,-69.8921 1196,-65.942 1196,-62.1676"/>
+<polygon fill="#0044a0" stroke="#0044a0" points="1199.5,-62.1213 1196,-52.1214 1192.5,-62.1214 1199.5,-62.1213"/>
 </g>
 </g>
 </svg>

pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+norecursedirs = lib 

requirements.txt

@@ -0,0 +1,6 @@
+ruffus==2.6.3
+Pillow==3.1.1
+reportlab==3.2.0
+PyPDF2==1.25.1
+img2pdf==0.2
+cffi==1.5.2

setup.cfg

@@ -1,2 +1,4 @@
 [bdist_wheel]
-python-tag = py34
+python-tag = py34
+[aliases]
+test=pytest

setup.py

@@ -1,11 +1,18 @@
 #!/usr/bin/env python3
+# -*- coding: utf-8 -*-
 # © 2015 James R. Barlow: github.com/jbarlow83
 
-from setuptools import setup
-from subprocess import Popen, STDOUT, check_output, CalledProcessError
-from string import Template
-import re
+from __future__ import print_function, unicode_literals
+
 import sys
+if sys.version_info < (3, 4):
+    print("Python 3.4 or newer is required")
+    sys.exit(1)
+
+from setuptools import setup  # nopep8
+from subprocess import STDOUT, check_output, CalledProcessError  # nopep8
+from collections.abc import Mapping  # nopep8
+import re  # nopep8
 
 
 missing_program = '''
@@ -48,21 +55,30 @@ commands:
     sudo apt-get install {package}
 
 On RPM-based systems (Red Hat, Fedora), search for instructions on
-installing the RPM for {package}.
+installing the RPM for {program}.
 '''
 
 
-def _error_trailer(program, package, optional, **kwargs):
-    if program == 'java':
-        return  # You're fucked
+def get_platform():
+    if sys.platform.startswith('freebsd'):
+        return 'freebsd'
+    elif sys.platform.startswith('linux'):
+        return 'linux'
+    return sys.platform
 
+
+def _error_trailer(program, package, optional, **kwargs):
     if optional:
         print(okay_its_optional.format(**locals()), file=sys.stderr)
     else:
         print(not_okay_its_required.format(**locals()), file=sys.stderr)
-    if sys.platform.startswith('darwin'):
+
+    if isinstance(package, Mapping):
+        package = package[get_platform()]
+
+    if get_platform() == 'darwin':
         print(osx_install_advice.format(**locals()), file=sys.stderr)
-    elif sys.platform.startswith('linux'):
+    elif get_platform() == 'linux':
         print(linux_install_advice.format(**locals()), file=sys.stderr)
 
 
@@ -130,14 +146,16 @@ def check_external_program(
     print('Found {program} {found_version}'.format(
             program=program, found_version=found_version))
 
+
 command = next((arg for arg in sys.argv[1:] if not arg.startswith('-')), '')
 
+
 if command.startswith('install') or \
-        command in ['check', 'test', 'nosetests', 'easy_install', 'egg_info']:
+        command in ['check', 'test', 'nosetests', 'easy_install']:
     check_external_program(
         program='tesseract',
         need_version='3.02.02',
-        package='tesseract'
+        package={'darwin': 'tesseract', 'linux': 'tesseract-ocr'}
     )
     check_external_program(
         program='gs',
@@ -150,12 +168,6 @@ if command.startswith('install') or \
         package='unpaper',
         optional=True
     )
-    check_external_program(
-        program='java',
-        need_version='1.5.0',
-        package='Java Runtime Environment',
-        version_check_args=['-version']
-    )
     check_external_program(
         program='qpdf',
         need_version='5.0.0',
@@ -163,19 +175,25 @@ if command.startswith('install') or \
         version_check_args=['--version']
     )
 
+
+if 'upload' in sys.argv[1:]:
+    print('Use twine to upload the package - setup.py upload is insecure')
+    sys.exit(1)
+
+tests_require = open('test_requirements.txt').read().splitlines()
+
 setup(
     name='ocrmypdf',
-    version='3.0rc4',  # also update: release notes, main.py
     description='OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched',
-    url='https://github.com/fritz-hh/OCRmyPDF',
-    author='James. R. Barlow',
+    url='https://github.com/jbarlow83/OCRmyPDF',
+    author='James R. Barlow',
     author_email='jim@purplerock.ca',
     license='Public Domain',
     packages=['ocrmypdf'],
     keywords=['PDF', 'OCR', 'optical character recognition', 'PDF/A', 'scanning'],
     classifiers=[
         "Programming Language :: Python :: 3",
-        "Development Status :: 4 - Beta",
+        "Development Status :: 5 - Production/Stable",
         "Environment :: Console",
         "Intended Audience :: End Users/Desktop",
         "Intended Audience :: Science/Research",
@@ -189,26 +207,29 @@ setup(
         "Topic :: Text Processing :: Indexing",
         "Topic :: Text Processing :: Linguistic",
         ],
+    setup_requires=[
+        'setuptools_scm',
+        'cffi>=1.5.0',
+        'pytest-runner'
+    ],
+    use_scm_version={'version_scheme': 'post-release'},
+    cffi_modules=[
+        'ocrmypdf/lib/compile_leptonica.py:ffi'
+    ],
     install_requires=[
-        'ruffus>=2.6.3',
-        'Pillow>=2.7.0',
-        'lxml>=3.4.2',
+        'ruffus==2.6.3',
+        'Pillow>=3.0.0',
         'reportlab>=3.1.44',
-        'PyPDF2>=1.25.1'
-    ],
-    test_requires=[
-        'img2pdf>=0.1.5',
-        'pytest>=2.7.2'
+        'PyPDF2>=1.25.1',
+        'img2pdf>=0.2.0',
+        'cffi>=1.5.0'
     ],
+    tests_require=tests_require,
     entry_points={
         'console_scripts': [
             'ocrmypdf = ocrmypdf.main:run_pipeline'
         ],
     },
-    eager_resources=[
-        'ocrmypdf/jhove/bin/*.jar',
-        'ocrmypdf/jhove/conf/*.conf',
-        'ocrmypdf/jhove/lib/*.jar'
-    ],
+    package_data={'ocrmypdf': ['data/sRGB.icc']},
     include_package_data=True,
     zip_safe=False)

share/etc-apt-sources.list

@@ -0,0 +1,4 @@
+deb http://httpredir.debian.org/debian stretch main
+deb http://httpredir.debian.org/debian stretch-updates main
+deb http://security.debian.org stretch/updates main
+deb http://ftp.de.debian.org/debian sid main contrib non-free

test_requirements.txt

@@ -0,0 +1 @@
+pytest>=2.7.2

tests/resources/NOTE.rst

@@ -1,17 +0,0 @@
-All test resources must come from free public domain sources for
-copyright reasons.
-
-Test files do not necessarily produce perfect (or even good) OCR
-results.
-
-+-------------------+--------------------------------------------------------------------------------+
-| File              | Source                                                                         |
-+===================+================================================================================+
-| graph.pdf         | Wikimedia                                                                      |
-+-------------------+--------------------------------------------------------------------------------+
-| c02-22.pdf        | Project Gutenberg: https://www.gutenberg.org/files/76/76-h/images/c02-22.jpg   |
-+-------------------+--------------------------------------------------------------------------------+
-| LinnSequencer.jpg | Wikimedia: https://upload.wikimedia.org/wikipedia/en/b/b7/LinnSequencer_hardware_MIDI_sequencer_brochure_page_2_300dpi.jpg   |
-+-------------------+--------------------------------------------------------------------------------+
-| congress.jpg      | http://www.baxleystamps.com/litho/meiji/courts_1871.jpg                        |
-+-------------------+--------------------------------------------------------------------------------+

tests/resources/README.rst

@@ -0,0 +1,66 @@
+These test files are used in OCRmyPDF's test suite. They do not necessarily produce OCR results
+at all and are not meant as examples of OCR output. Some are even invalid PDFs that might
+crash certain PDF viewers.
+
+
+Files derived from free sources
+===============================
+
+These test resources come from free sources, under either public domain or Creative Commons licenses.
+In some cases they were converted from one image format to another without other changes.
+
++---------------------+--------------------------------------------------------------------------------+
+| File                | Source                                                                         |
++=====================+================================================================================+
+| c02-22.pdf          | `Project Gutenberg`_, Adventures of Huckleberry Finn, page 22                  |
++---------------------+--------------------------------------------------------------------------------+
+| congress.jpg        | `US Congressional Records`_                                                    |
++---------------------+--------------------------------------------------------------------------------+
+| graph.pdf           | `Wikimedia: Pandas text analysis.png`_                                         |
++---------------------+--------------------------------------------------------------------------------+
+| lichtenstein.pdf    | `Wikimedia: JPEG2000 Lichtenstein`_ (Creative Commons BY-SA 3.0)               |
++---------------------+--------------------------------------------------------------------------------+
+| LinnSequencer.jpg,  | `Wikimedia: LinnSequencer`_ (Creative Commons Attribution-ShareAlike 3.0)      |
+| linn.pdf, linn.txt  |                                                                                |
++---------------------+--------------------------------------------------------------------------------+
+ 
+
+Files generated for this project
+================================
+
+The following test resources were crafted specifically for this project, and can be used
+under the terms of the license in LICENSE.rst.
+
+- blank.pdf (a blank PDF page)
+- cmyk.pdf (a CMYK image created in Photoshop)
+- enormous.pdf (a very lage page)
+- francais.pdf (a page containing French accented characters)
+- hugemono.pdf (large monochrome JBIG2 page with pixel dimensions of 35000x35000)
+- invalid.pdf (a PDF file header followed by EOF marker)
+- missing_docinfo.pdf (PDF file with no /DocumentInfo section)
+
+
+Assemblies
+==========
+
+These test resources are assemblies from other previously mentioned files, released under the same license terms as their input files.
+
+- cardinal.pdf (four cardinal directions, rotated copies of LinnSequencer.jpg)
+- ccitt.pdf (LinnSequencer.jpg, converted to CCITT encoding)
+- graph_ocred.pdf (from graph.pdf)
+- jbig2.pdf (congress.jpg, converted to JBIG2 encoding)
+- multipage.pdf (from several other files)
+- palette.pdf (congress.jpg, converted to a 256-color palette)
+- skew.pdf (from c02-22.pdf)
+- skew-encrypted.pdf (skew.pdf with encrypted applied)
+
+
+.. _`Wikimedia: LinnSequencer`: https://upload.wikimedia.org/wikipedia/en/b/b7/LinnSequencer_hardware_MIDI_sequencer_brochure_page_2_300dpi.jpg
+
+.. _`Project Gutenberg`: https://www.gutenberg.org/files/76/76-h/76-h.htm#c2
+
+.. _`US Congressional Records`: http://www.baxleystamps.com/litho/meiji/courts_1871.jpg
+
+.. _`Wikimedia: Pandas text analysis.png`: https://en.wikipedia.org/wiki/File:Pandas_text_analysis.png
+
+.. _`Wikimedia: JPEG2000 Lichtenstein`: https://en.wikipedia.org/wiki/JPEG_2000#/media/File:Jpeg2000_2-level_wavelet_transform-lichtenstein.png

tests/resources/invalid.pdf

@@ -0,0 +1,3 @@
+%PDF-1.3
+This is not a valid PDF file
+%%EOF

tests/resources/linn.txt

@@ -0,0 +1,123 @@
+The LinnSequencer
+32 Track MIDI Sequence Recorder
+
+The LinnSequencer is a state—of—the-art composition and performance tool for the professional musician. It is
+
+extremely powerful, yet amazingly simple to learn and use. It’s many remarkable features include:
+
+0 Operation is similar to multi-track tape recorder with PLAY, STOP, RECORD, FAST
+FORWARD, REWIND, and LOCATE controls.
+
+0 Each of the 100 sequences contains 32 simultaneous, polyphonic tracks. Each track may
+be assigned to one of 16 MIDI channels. Simultaneously plays up to 16 polyphonic
+
+synthesizers !
+
+0 Ultra-fast 3 1/2 ” disk drive stores complex songs in seconds and holds over 110,000 notes
+
+per disk!
+
+0 One or all tracks may be TRANSPOSED at the touch of a key.
+0 Exclusive real—time ERASE function makes editing FAST.
+0 Exclusive REPEAT function automatically repeats any held notes at a pre-selected
+
+rhythmic value.
+
+0 TIMING CORRECTION works during playback and operates without ‘chopping’ notes.
+
+0 Optional SMPTE time code synchronization.
+
+0 Optional remote control.
+
+Recording a Sequence
+
+To record a sequence, simply press RECORD and PLAY,
+then play your MIDI keyboard in time to the Sequencer’s
+click track. When the sequence loops back around to bar 1,
+you’ll hear what you played—only all timing errors will be
+
+corrected! (Timing correction may be adjusted 0r defeated).
+
+Any additional notes played will be added into the track
+—existing notes are not erased while recording!
+
+FAST FORWARD, REWIND, and LOCATE controls
+may be used at any time to quickly access any location in
+your sequence for spot-recording. To overdub a new part,
+select a different track and start recording—while you
+record, the first‘track will play in perfect sync (unless you
+MUTE it, or SOLO another track). In this way, up to 32
+tracks may be overdubbed! All MIDI effects are recorded
+including pitch bend, modulation, velocity, aftertouch,
+sustain pedal, and program changes!
+
+Editing
+
+To erase a wrong note, simply hold ERASE and press
+the note to be erased just before it plays in the sequence-—
+when played back, it will be gone. Notes may also be
+
+added, erased, or changed using the SINGLE STEP func-
+tion. To overdub notes at specific points within a sequence,
+
+Additional Features
+
+simply use LOCATE, FAST FORWARD, or REWIND to
+find the desired bar number, then start recording.
+
+The INSERT/ COPY function allows you to move bars
+from one location to another—in the same sequence or a
+different one. For example, you might insert a copy of the
+first verse between the second chorus and the bridge.
+DELETE BARS operates the same way to remove
+unwanted sections.
+
+Creating a Song
+
+One way to create a song is to record each track all the
+way through (up to 999 bars). Another way is to record
+each basic section (verse, chorus, etc.) in individual
+sequences, then use the CREATE SONG function to “chain”
+them together. CREATE SONG will then automatically
+copy all the parts into a new sequence. If desired, you can
+even set the last few bars to repeat infinitely, for a fadeout.
+
+Composition Without Compromise
+
+The technology you use should never be so complex that
+it interferes with the creative process. That’s precisely why
+the LinnSequencer is designed to let you compose, record
+and edit while devoting your undivided attention to your
+music. See your Linn dealer today for a demonstration!
+
+0 Simple, easy to learn operation—the 32 character LCD display clearly guides you through all operations. If needed, the
+
+HELP button displays additional explanations.
+
+0 Non-destructive recording—existing notes are not erased while recording.
+0 Two FOOTSWIT CH INPUTS may be assigned to remotely control many of the commonly used functions, including
+
+ERASE, REPEAT, PLAY/ STOP, or LOCATE.
+
+0 Two TRIGGER OUTPUTS may be programmed to output pulses at any selected note value.
+
+0 Will sync to standard LinnDrum or Linn 9000 sync tone.
+
+0 Utilizes ultra high—speed, 8 MHZ 80186 16 bit computer internally for FAST operation.
+0 TEMPO may be specified in BEATS-PER—MINUTE or FRAMES-PER—BEAT at 24, 25, or 30 frames per second,
+
+(even drop frame!)
+
+0 TEMPO may be entered numerically, adjustable in tenths of a Beat-Per-Minute increments, or by tapping quarter notes
+
+on the TAP TEMPO button.
+
+0 TEMPO CHANGES may be programmed into a sequence, with smooth transitions if desired.
+0 Any TIME SIGNATURE may be used, and may be changed within a song.
+
+EDI]
+Linn Electronics, Inc.
+
+18720 Oxnard Street, Tarzana, CA 91356
+(818) 708-8131 TELEX #298949 LINN UR
+

tests/resources/missing_docinfo.pdf

@@ -0,0 +1,41 @@
+%PDF-1.3
+%¿÷¢þ
+1 0 obj
+<< /Outlines 2 0 R /PageMode /UseNone /Pages 3 0 R /Type /Catalog >>
+endobj
+2 0 obj
+<< /Count 0 /Type /Outlines >>
+endobj
+3 0 obj
+<< /Count 1 /Kids [ 4 0 R ] /Type /Pages >>
+endobj
+4 0 obj
+<< /Contents 5 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 3 0 R /Resources << /Font 6 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] >> /Rotate 0 /Trans << >> /Type /Page >>
+endobj
+5 0 obj
+<< /Length 104 /Filter /FlateDecode >>
+stream
+xœ-Œ1
+ƒ@{O1e´ˆûEI¢ Xþè⢉»[¨÷Wƒ<˜bž`ÎÉŸÖÃ[ÉA
+Ô!å³D;jMN/w*Æðªêyè4¯¸y™ú•É?Ñî~[\dX¢ý¥èÍ®’™…endstream
+endobj
+6 0 obj
+<< /F1 7 0 R >>
+endobj
+7 0 obj
+<< /BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font >>
+endobj
+xref
+0 8
+0000000000 65535 f 
+0000000015 00000 n 
+0000000099 00000 n 
+0000000145 00000 n 
+0000000204 00000 n 
+0000000403 00000 n 
+0000000578 00000 n 
+0000000609 00000 n 
+trailer << /Root 1 0 R /Size 8 /ID [<85bf6c3c41ed927eb43bdfdfff4c0cb5><0304c5f87e589cb7b06f6f52dc6e6707>] >>
+startxref
+716
+%%EOF

tests/spoof/qpdf_dummy_return2.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+import sys
+
+
+def main():
+    print('qpdf dummy')
+    sys.exit(2)
+
+
+if __name__ == '__main__':
+    main()

tests/spoof/tesseract_big_image_error.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+import sys
+
+
+VERSION_STRING = '''tesseract 3.04.00
+ leptonica-1.72
+  libjpeg 8d : libpng 1.6.19 : libtiff 4.0.6 : zlib 1.2.5
+SPOOFED: return error claiming image too big
+'''
+
+"""Simulates a Tesseract crash
+
+It isn't strictly necessary to crash the process and that has unwanted
+side effects like triggering core dumps or error reporting, logging and such.
+It's enough to dump some text to stderr and return an error code.
+
+Follows the POSIX? convention of returning 128 + signal number.
+
+"""
+
+
+def main():
+    if sys.argv[1] == '--version':
+        print(VERSION_STRING, file=sys.stderr)
+        sys.exit(0)
+    elif sys.argv[1] == '--list-langs':
+        print('List of available languages (1):\neng', file=sys.stderr)
+        sys.exit(0)
+    elif sys.argv[-1] == 'hocr':
+        print("Image too large: (33830, 14959)\n"
+              "Error during processing.", file=sys.stderr)
+        sys.exit(1)
+    elif sys.argv[-1] == 'pdf':
+        print("Image too large: (33830, 14959)\n"
+              "Error during processing.", file=sys.stderr)
+        sys.exit(1)
+    elif sys.argv[-1] == 'stdout':
+        print("Image too large: (33830, 14959)\n"
+              "Error during processing.", file=sys.stderr)
+        sys.exit(1)
+    else:
+        print("Spoof doesn't understand arguments", file=sys.stderr)
+        print(sys.argv, file=sys.stderr)
+        sys.exit(1)
+
+    sys.exit(0)
+
+
+if __name__ == '__main__':
+    main()

tests/spoof/tesseract_cache.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+import sys
+import os
+import hashlib
+import shutil
+import subprocess
+
+
+CACHE_PATH = os.path.abspath(os.path.join(
+        os.path.dirname(__file__), '..', 'cache'))
+
+
+def real_tesseract():
+    tess_args = ['tesseract'] + sys.argv[1:]
+    os.execvp("tesseract", tess_args)
+    return  # Not reachable
+
+def main():
+    operation = sys.argv[-1]
+    # For anything unexpected operation, defer to real tesseract binary
+    if operation != 'hocr' and operation != 'pdf' and operation != 'stdout':
+        real_tesseract()
+        return  # Not reachable
+
+    try:
+        os.makedirs(CACHE_PATH)
+    except FileExistsError:
+        pass
+
+    m = hashlib.sha1()
+
+    version = subprocess.check_output(
+        ['tesseract', '--version'],
+        stderr=subprocess.STDOUT)
+
+    m.update(version)
+    m.update(operation.encode())
+
+    try:
+        lang = sys.argv[sys.argv.index('-l') + 1]
+        m.update(lang.encode())
+    except ValueError:
+        pass
+    try:
+        psm = sys.argv[sys.argv.index('-psm') + 1]
+        m.update(psm.encode())
+    except ValueError:
+        pass
+
+    if operation == 'stdout' and psm != '0':
+        real_tesseract()
+        return
+
+    if operation == 'stdout':
+        input_file = sys.argv[-2]
+        output_file = 'stdout'
+    else:
+        input_file = sys.argv[-3]
+        output_file = sys.argv[-2]
+
+    if operation == 'hocr':
+        output_file += '.hocr'
+    elif operation == 'pdf':
+        output_file += '.pdf'
+
+    with open(input_file, 'rb') as f:
+        m.update(f.read())
+    cache_name = os.path.join(CACHE_PATH, m.hexdigest())
+    print(cache_name)
+    if os.path.exists(cache_name):
+        # Cache hit
+        print("Tesseract cache hit", file=sys.stderr)
+        if operation != 'stdout':
+            shutil.copy(cache_name, output_file)
+
+        # Replicate output
+        with open(cache_name + '.stdout', 'r') as f:
+            print(f.read(), end='')
+        with open(cache_name + '.stderr', 'r') as f:
+            print(f.read(), end='', file=sys.stderr)
+        sys.exit(0)
+
+    # Cache miss
+    print("Tesseract cache miss", file=sys.stderr)
+
+    # Call tesseract
+    p = subprocess.Popen(
+            ['tesseract'] + sys.argv[1:],
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+            universal_newlines=True)
+    stdout, stderr = p.communicate()
+
+    if p.returncode != 0:
+        # Do not cache errors or crashes
+        print("Tesseract error", file=sys.stderr)
+        print(stdout, end='')
+        print(stderr, end='', file=sys.stderr)
+        return p.returncode
+
+    with open(cache_name + '.stdout', 'w') as f:
+        f.write(stdout)
+    with open(cache_name + '.stderr', 'w') as f:
+        f.write(stderr)
+    print(stdout, end='')
+    print(stderr, end='', file=sys.stderr)
+
+    # Insert file into cache
+    if output_file != 'stdout':
+        if os.path.exists(output_file):
+            shutil.copy(output_file, cache_name)
+        else:
+            print("Could not find output file", file=sys.stderr)
+    else:
+        open(cache_name, 'w').close()
+
+
+if __name__ == '__main__':
+    main()

tests/spoof/tesseract_crash.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+import sys
+import os
+import signal
+
+
+VERSION_STRING = '''tesseract 3.04.00
+ leptonica-1.72
+  libjpeg 8d : libpng 1.6.19 : libtiff 4.0.6 : zlib 1.2.5
+SPOOFED: CRASH ON OCR or -psm 0
+'''
+
+"""Simulates a Tesseract crash
+
+It isn't strictly necessary to crash the process and that has unwanted
+side effects like triggering core dumps or error reporting, logging and such.
+It's enough to dump some text to stderr and return an error code.
+
+Follows the POSIX? convention of returning 128 + signal number.
+
+"""
+
+
+def main():
+    if sys.argv[1] == '--version':
+        print(VERSION_STRING, file=sys.stderr)
+        sys.exit(0)
+    elif sys.argv[1] == '--list-langs':
+        print('List of available languages (1):\neng', file=sys.stderr)
+        sys.exit(0)
+    elif sys.argv[-1] == 'hocr':
+        print("KABOOM! Tesseract failed for some reason", file=sys.stderr)
+        sys.exit(128 + signal.SIGSEGV)
+    elif sys.argv[-1] == 'pdf':
+        print("KABOOM! Tesseract failed for some reason", file=sys.stderr)
+        sys.exit(128 + signal.SIGSEGV)
+    elif sys.argv[-1] == 'stdout':
+        print("libc++abi.dylib: terminating with uncaught exception of type "
+              "std::bad_alloc: std::bad_alloc", file=sys.stderr)
+        sys.exit(128 + signal.SIGABRT)
+    else:
+        print("Spoof doesn't understand arguments", file=sys.stderr)
+        print(sys.argv, file=sys.stderr)
+        sys.exit(1)
+
+    sys.exit(0)
+
+
+if __name__ == '__main__':
+    main()

tests/spoof/tesseract_noop.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+import sys
+import img2pdf
+from PIL import Image
+
+
+VERSION_STRING = '''tesseract 3.04.00
+ leptonica-1.72
+  libjpeg 8d : libpng 1.6.19 : libtiff 4.0.6 : zlib 1.2.5
+SPOOFED
+'''
+
+HOCR_TEMPLATE = '''<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+  <title></title>
+  <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+  <meta name='ocr-system' content='tesseract 3.02.02' />
+  <meta name='ocr-capabilities' content='ocr_page ocr_carea ocr_par ocr_line ocrx_word'/>
+ </head>
+ <body>
+  <div class='ocr_page' id='page_1' title='image "x.tif"; bbox 0 0 {0} {1}; ppageno 0'>
+   <div class='ocr_carea' id='block_1_1' title="bbox 0 1 {0} {1}">
+    <p class='ocr_par' dir='ltr' id='par_1' title="bbox 0 1 {0} {1}">
+     <span class='ocr_line' id='line_1' title="bbox 0 1 {0} {1}"><span class='ocrx_word' id='word_1' title="bbox 0 1 {0} {1}"> </span>
+     </span>
+    </p>
+   </div>
+  </div>
+ </body>
+</html>'''
+
+
+def main():
+    if sys.argv[1] == '--version':
+        print(VERSION_STRING, file=sys.stderr)
+        sys.exit(0)
+    elif sys.argv[1] == '--list-langs':
+        print('List of available languages (1):\neng', file=sys.stderr)
+        sys.exit(0)
+    elif sys.argv[-1] == 'hocr':
+        inputf = sys.argv[-3]
+        output = sys.argv[-2]
+        with Image.open(inputf) as im, \
+                open(output + '.hocr', 'w', encoding='utf-8') as f:
+            w, h = im.size
+            f.write(HOCR_TEMPLATE.format(str(w), str(h)))
+    elif sys.argv[-1] == 'pdf':
+        inputf = sys.argv[-3]
+        output = sys.argv[-2]
+        pdf_bytes = img2pdf.convert([inputf], dpi=300)
+        with open(output + '.pdf', 'wb') as f:
+            f.write(pdf_bytes)
+    elif sys.argv[-1] == 'stdout':
+        inputf = sys.argv[-2]
+        print("""Orientation: 0
+Orientation in degrees: 0
+Orientation confidence: 100.00
+Script: 1
+Script confidence: 100.00""", file=sys.stderr)
+    else:
+        print("Spoof doesn't understand arguments", file=sys.stderr)
+        print(sys.argv, file=sys.stderr)
+        sys.exit(1)
+
+    sys.exit(0)
+
+
+if __name__ == '__main__':
+    main()

tests/spoof/unpaper_oldversion.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+import sys
+
+def main():
+    if sys.argv[1] == '--version':
+        print('0.5')
+        sys.exit(0)
+
+    print("Only supports --version")
+    sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

tests/test_hocrtransform.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+# © 2015 James R. Barlow: github.com/jbarlow83
+
+from ocrmypdf import hocrtransform
+from ocrmypdf.tesseract import HOCR_TEMPLATE
+from reportlab.pdfgen.canvas import Canvas
+from PIL import Image
+from tempfile import NamedTemporaryFile
+from contextlib import suppress
+import os
+import shutil
+import pytest
+import img2pdf
+import pytest
+import sys
+
+
+if sys.version_info.major < 3:
+    print("Requires Python 3.4+")
+    sys.exit(1)
+
+TESTS_ROOT = os.path.abspath(os.path.dirname(__file__))
+SPOOF_PATH = os.path.join(TESTS_ROOT, 'spoof')
+PROJECT_ROOT = os.path.dirname(TESTS_ROOT)
+OCRMYPDF = os.path.join(PROJECT_ROOT, 'OCRmyPDF.sh')
+TEST_RESOURCES = os.path.join(PROJECT_ROOT, 'tests', 'resources')
+TEST_OUTPUT = os.environ.get(
+    'OCRMYPDF_TEST_OUTPUT',
+    default=os.path.join(PROJECT_ROOT, 'tests', 'output', 'hocrtransform'))
+
+
+def setup_module():
+    with suppress(FileNotFoundError):
+        shutil.rmtree(TEST_OUTPUT)
+    with suppress(FileExistsError):
+        os.makedirs(TEST_OUTPUT)
+    with open(_make_output('blank.hocr'), 'w') as f:
+        f.write(HOCR_TEMPLATE)
+
+
+def _make_input(input_basename):
+    return os.path.join(TEST_RESOURCES, input_basename)
+
+
+def _make_output(output_basename):
+    return os.path.join(TEST_OUTPUT, output_basename)
+
+
+def test_mono_image():
+    im = Image.new('1', (8, 8), 0)
+    for n in range(8):
+        im.putpixel((n, n), 1)
+    im.save(_make_output('mono.tif'), format='TIFF')
+
+    hocr = hocrtransform.HocrTransform(_make_output('blank.hocr'), 300)
+    hocr.to_pdf(_make_output('mono.pdf'), imageFileName=_make_output('mono.tif'))
+
+
+
+
+

tests/test_main.py

@@ -2,15 +2,16 @@
 # © 2015 James R. Barlow: github.com/jbarlow83
 
 from __future__ import print_function
-from subprocess import Popen, PIPE, check_output
+from subprocess import Popen, PIPE, check_output, check_call
 import os
 import shutil
 from contextlib import suppress
 import sys
-from unittest.mock import patch, create_autospec
 import pytest
 from ocrmypdf.pageinfo import pdf_get_all_pageinfo
 import PyPDF2 as pypdf
+from ocrmypdf import ExitCode
+from ocrmypdf import leptonica
 
 
 if sys.version_info.major < 3:
@@ -18,54 +19,120 @@ if sys.version_info.major < 3:
     sys.exit(1)
 
 TESTS_ROOT = os.path.abspath(os.path.dirname(__file__))
+SPOOF_PATH = os.path.join(TESTS_ROOT, 'spoof')
 PROJECT_ROOT = os.path.dirname(TESTS_ROOT)
 OCRMYPDF = os.path.join(PROJECT_ROOT, 'OCRmyPDF.sh')
 TEST_RESOURCES = os.path.join(PROJECT_ROOT, 'tests', 'resources')
-TEST_OUTPUT = os.path.join(PROJECT_ROOT, 'tests', 'output')
+TEST_OUTPUT = os.environ.get(
+    'OCRMYPDF_TEST_OUTPUT',
+    default=os.path.join(PROJECT_ROOT, 'tests', 'output', 'main'))
+
+
+def running_in_docker():
+    # Docker creates a file named /.dockerinit
+    return os.path.exists('/.dockerinit')
 
 
 def setup_module():
     with suppress(FileNotFoundError):
         shutil.rmtree(TEST_OUTPUT)
     with suppress(FileExistsError):
-        os.mkdir(TEST_OUTPUT)
+        os.makedirs(TEST_OUTPUT)
 
 
-def run_ocrmypdf_sh(input_file, output_file, *args):
+def run_ocrmypdf_sh(input_file, output_file, *args, env=None):
     sh_args = ['sh', OCRMYPDF] + list(args) + [input_file, output_file]
     sh = Popen(
         sh_args, close_fds=True, stdout=PIPE, stderr=PIPE,
-        universal_newlines=True)
+        universal_newlines=True, env=env)
     out, err = sh.communicate()
     return sh, out, err
 
 
-def _make_input(input_basename):
+def _infile(input_basename):
     return os.path.join(TEST_RESOURCES, input_basename)
 
 
-def _make_output(output_basename):
-    return os.path.join(TEST_OUTPUT, output_basename)
+def _outfile(output_basename):
+    return os.path.join(TEST_OUTPUT, os.path.basename(output_basename))
 
 
-def check_ocrmypdf(input_basename, output_basename, *args):
-    input_file = _make_input(input_basename)
-    output_file = _make_output(output_basename)
+def check_ocrmypdf(input_basename, output_basename, *args, env=None):
+    input_file = _infile(input_basename)
+    output_file = _outfile(output_basename)
 
-    sh, _, err = run_ocrmypdf_sh(input_file, output_file, *args)
-    assert sh.returncode == 0, err
+    sh, out, err = run_ocrmypdf_sh(input_file, output_file, *args, env=env)
+    if sh.returncode != 0:
+        print('stdout\n======')
+        print(out)
+        print('stderr\n======')
+        print(err)
+    assert sh.returncode == 0
     assert os.path.exists(output_file), "Output file not created"
     assert os.stat(output_file).st_size > 100, "PDF too small or empty"
     return output_file
 
 
-def test_quick():
-    check_ocrmypdf('c02-22.pdf', 'test_quick.pdf')
+def run_ocrmypdf_env(input_basename, output_basename, *args, env=None):
+    input_file = _infile(input_basename)
+    output_file = _outfile(output_basename)
+
+    if env is None:
+        env = os.environ
+
+    p_args = ['ocrmypdf'] + list(args) + [input_file, output_file]
+    p = Popen(
+        p_args, close_fds=True, stdout=PIPE, stderr=PIPE,
+        universal_newlines=True, env=env)
+    out, err = p.communicate()
+    return p, out, err
 
 
-def test_deskew():
+def spoof(replace_program, with_spoof):
+    """Modify environment variables to override subprocess executables
+
+    Before running any executable, ocrmypdf checks the environment variable
+    OCRMYPDF_PROGRAMNAME to override default program name/location, e.g.
+    OCRMYPDF_GS redirects from the system path Ghostscript ("gs") to elsewhere.
+    """
+    env = os.environ.copy()
+    spoofer = os.path.join(SPOOF_PATH, with_spoof)
+    if not os.access(spoofer, os.X_OK):
+        os.chmod(spoofer, 0o755)
+    env['OCRMYPDF_' + replace_program.upper()] = spoofer
+    return env
+
+
+@pytest.fixture
+def spoof_tesseract_noop():
+    return spoof('tesseract', 'tesseract_noop.py')
+
+
+@pytest.fixture
+def spoof_tesseract_cache():
+    if running_in_docker():
+        return os.environ.copy()
+    return spoof('tesseract', "tesseract_cache.py")
+
+
+@pytest.fixture
+def spoof_tesseract_crash():
+    return spoof('tesseract', 'tesseract_crash.py')
+
+
+@pytest.fixture
+def spoof_tesseract_big_image_error():
+    return spoof('tesseract', 'tesseract_big_image_error.py')
+
+
+def test_quick(spoof_tesseract_noop):
+    check_ocrmypdf('c02-22.pdf', 'test_quick.pdf', env=spoof_tesseract_noop)
+
+
+def test_deskew(spoof_tesseract_noop):
     # Run with deskew
-    deskewed_pdf = check_ocrmypdf('skew.pdf', 'test_deskew.pdf', '-d')
+    deskewed_pdf = check_ocrmypdf(
+        'skew.pdf', 'test_deskew.pdf', '-d', '-v', '1', env=spoof_tesseract_noop)
 
     # Now render as an image again and use Leptonica to find the skew angle
     # to confirm that it was deskewed
@@ -73,7 +140,7 @@ def test_deskew():
     import logging
     log = logging.getLogger()
 
-    deskewed_png = _make_output('deskewed.png')
+    deskewed_png = _outfile('deskewed.png')
 
     rasterize_pdf(
         deskewed_pdf,
@@ -83,23 +150,44 @@ def test_deskew():
         raster_device='pngmono',
         log=log)
 
-    from ocrmypdf.leptonica import pixRead, pixDestroy, pixFindSkew
-    pix = pixRead(deskewed_png)
-    skew_angle, skew_confidence = pixFindSkew(pix)
-    pix = pixDestroy(pix)
+    from ocrmypdf.leptonica import Pix
+    pix = Pix.read(deskewed_png)
+    skew_angle, skew_confidence = pix.find_skew()
 
     print(skew_angle)
     assert -0.5 < skew_angle < 0.5, "Deskewing failed"
 
 
-def test_clean():
-    check_ocrmypdf('skew.pdf', 'test_clean.pdf', '-c')
+def test_clean(spoof_tesseract_noop):
+    check_ocrmypdf('skew.pdf', 'test_clean.pdf', '-c', env=spoof_tesseract_noop)
 
 
-def test_preserve_metadata():
-    pdf_before = pypdf.PdfFileReader(_make_input('graph.pdf'))
+@pytest.mark.parametrize("pdf,renderer", [
+    ('palette.pdf', 'hocr'),
+    ('palette.pdf', 'tesseract'),
+    ('cmyk.pdf', 'hocr'),
+    ('cmyk.pdf', 'tesseract'),
+    ('ccitt.pdf', 'hocr'),
+    ('ccitt.pdf', 'tesseract'),
+    ('jbig2.pdf', 'hocr'),
+    ('jbig2.pdf', 'tesseract'),
+    ('lichtenstein.pdf', 'hocr'),
+    ('lichtenstein.pdf', 'tesseract')
+])
+def test_exotic_image(spoof_tesseract_cache, pdf, renderer):
+    check_ocrmypdf(
+        pdf,
+        'test_{0}_{1}.pdf'.format(pdf, renderer),
+        '-dc',
+        '-v', '1',
+        '--pdf-renderer', renderer, env=spoof_tesseract_cache)
 
-    output = check_ocrmypdf('graph.pdf', 'test_metadata_preserve.pdf')
+
+def test_preserve_metadata(spoof_tesseract_noop):
+    pdf_before = pypdf.PdfFileReader(_infile('graph.pdf'))
+
+    output = check_ocrmypdf('graph.pdf', 'test_metadata_preserve.pdf',
+                            env=spoof_tesseract_noop)
 
     pdf_after = pypdf.PdfFileReader(output)
 
@@ -107,16 +195,24 @@ def test_preserve_metadata():
         assert pdf_before.documentInfo[key] == pdf_after.documentInfo[key]
 
 
-def test_override_metadata():
+def test_override_metadata(spoof_tesseract_noop):
+    input_file = _infile('c02-22.pdf')
+    output_file = _outfile('test_override_metadata.pdf')
+
     german = 'Du siehst den Wald vor lauter Bäumen nicht.'
     chinese = '孔子'
     high_unicode = 'U+1030C is: 𐌌'
 
-    pdf = check_ocrmypdf(
-        'c02-22.pdf', 'test_metadata.pdf',
+    p, out, err = run_ocrmypdf_env(
+        input_file, output_file,
         '--title', german,
         '--author', chinese,
-        '--subject', high_unicode)
+        '--subject', high_unicode,
+        env=spoof_tesseract_noop)
+
+    assert p.returncode == ExitCode.ok
+
+    pdf = output_file
 
     out_pdfinfo = check_output(['pdfinfo', pdf], universal_newlines=True)
     lines_pdfinfo = out_pdfinfo.splitlines()
@@ -131,20 +227,20 @@ def test_override_metadata():
     assert pdfinfo.get('Keywords', '') == ''
 
 
-def check_oversample(renderer):
+@pytest.mark.parametrize('renderer', [
+    'hocr',
+    'tesseract',
+    ])
+def test_oversample(spoof_tesseract_cache, renderer):
     oversampled_pdf = check_ocrmypdf(
-        'skew.pdf', 'test_oversample_%s.pdf' % renderer, '--oversample', '300',
-        '--pdf-renderer', renderer)
+        'skew.pdf', 'test_oversample_%s.pdf' % renderer, '--oversample', '350',
+        '-f',
+        '--pdf-renderer', renderer, env=spoof_tesseract_cache)
 
     pdfinfo = pdf_get_all_pageinfo(oversampled_pdf)
 
     print(pdfinfo[0]['xres'])
-    assert abs(pdfinfo[0]['xres'] - 300) < 1
-
-
-def test_oversample():
-    yield check_oversample, 'hocr'
-    yield check_oversample, 'tesseract'
+    assert abs(pdfinfo[0]['xres'] - 350) < 1
 
 
 def test_repeat_ocr():
@@ -152,51 +248,306 @@ def test_repeat_ocr():
     assert sh.returncode != 0
 
 
-def test_force_ocr():
-    out = check_ocrmypdf('graph_ocred.pdf', 'test_force.pdf', '-f')
+def test_force_ocr(spoof_tesseract_cache):
+    out = check_ocrmypdf('graph_ocred.pdf', 'test_force.pdf', '-f',
+                         env=spoof_tesseract_cache)
     pdfinfo = pdf_get_all_pageinfo(out)
     assert pdfinfo[0]['has_text']
 
 
-def test_skip_ocr():
-    check_ocrmypdf('graph_ocred.pdf', 'test_skip.pdf', '-s')
+def test_skip_ocr(spoof_tesseract_cache):
+    check_ocrmypdf('graph_ocred.pdf', 'test_skip.pdf', '-s',
+                   env=spoof_tesseract_cache)
 
 
-def test_argsfile():
-    with open(_make_output('test_argsfile.txt'), 'w') as argsfile:
+def test_argsfile(spoof_tesseract_noop):
+    with open(_outfile('test_argsfile.txt'), 'w') as argsfile:
         print('--title', 'ArgsFile Test', '--author', 'Test Cases',
               sep='\n', end='\n', file=argsfile)
     check_ocrmypdf('graph.pdf', 'test_argsfile.pdf',
-                   '@' + _make_output('test_argsfile.txt'))
+                   '@' + _outfile('test_argsfile.txt'),
+                   env=spoof_tesseract_noop)
 
 
-def check_ocr_timeout(renderer):
+def check_monochrome_correlation(
+        reference_pdf, reference_pageno, test_pdf, test_pageno):
+
+    import ocrmypdf.ghostscript as ghostscript
+    import logging
+
+    gslog = logging.getLogger()
+
+    reference_png = _outfile('{}.ref{:04d}.png'.format(
+        reference_pdf, reference_pageno))
+    test_png = _outfile('{}.test{:04d}.png'.format(
+        test_pdf, test_pageno))
+
+    def rasterize(pdf, pageno, png):
+        if os.path.exists(png):
+            print(png)
+            return
+        ghostscript.rasterize_pdf(
+            pdf,
+            png,
+            xres=100, yres=100,
+            raster_device='pngmono', log=gslog, pageno=pageno)
+
+    rasterize(reference_pdf, reference_pageno, reference_png)
+    rasterize(test_pdf, test_pageno, test_png)
+
+    pix_ref = leptonica.Pix.read(reference_png)
+    pix_test = leptonica.Pix.read(test_png)
+
+    return leptonica.Pix.correlation_binary(pix_ref, pix_test)
+
+
+def test_monochrome_correlation():
+    # Verify leptonica: check that an incorrect rotated image has poor
+    # correlation with reference
+    corr = check_monochrome_correlation(
+        reference_pdf=_infile('cardinal.pdf'),
+        reference_pageno=1,  # north facing page
+        test_pdf=_infile('cardinal.pdf'),
+        test_pageno=3,  # south facing page
+        )
+    assert corr < 0.10
+    corr = check_monochrome_correlation(
+        reference_pdf=_infile('cardinal.pdf'),
+        reference_pageno=2,
+        test_pdf=_infile('cardinal.pdf'),
+        test_pageno=2,
+        )
+    assert corr > 0.90
+
+
+@pytest.mark.parametrize('renderer', [
+    'hocr',
+    'tesseract',
+    ])
+def test_autorotate(spoof_tesseract_cache, renderer):
+    # cardinal.pdf contains four copies of an image rotated in each cardinal
+    # direction - these ones are "burned in" not tagged with /Rotate
+    out = check_ocrmypdf('cardinal.pdf', 'test_autorotate_%s.pdf' % renderer,
+                         '-r', '-v', '1', env=spoof_tesseract_cache)
+    for n in range(1, 4+1):
+        correlation = check_monochrome_correlation(
+            reference_pdf=_infile('cardinal.pdf'),
+            reference_pageno=1,
+            test_pdf=out,
+            test_pageno=n)
+        assert correlation > 0.80
+
+
+def test_autorotate_threshold_low(spoof_tesseract_cache):
+    out = check_ocrmypdf('cardinal.pdf', 'test_autorotate_threshold_low.pdf',
+                         '--rotate-pages-threshold', '1',
+                         '-r', '-v', '1', env=spoof_tesseract_cache)
+
+    # Low threshold -> always rotate -> expect high correlation between
+    # reference page and test page
+    correlation = check_monochrome_correlation(
+        reference_pdf=_infile('cardinal.pdf'),
+        reference_pageno=1,
+        test_pdf=out,
+        test_pageno=3)
+    assert correlation > 0.80
+
+
+def test_autorotate_threshold_high(spoof_tesseract_cache):
+    out = check_ocrmypdf('cardinal.pdf', 'test_autorotate_threshold_high.pdf',
+                         '--rotate-pages-threshold', '99',
+                         '-r', '-v', '1', env=spoof_tesseract_cache)
+
+    # High threshold -> never rotate -> expect low correlation since
+    # test page will not be rotated
+    correlation = check_monochrome_correlation(
+        reference_pdf=_infile('cardinal.pdf'),
+        reference_pageno=1,
+        test_pdf=out,
+        test_pageno=3)
+    assert correlation < 0.10
+
+
+@pytest.mark.parametrize('renderer', [
+    'hocr',
+    'tesseract',
+    ])
+def test_ocr_timeout(renderer):
     out = check_ocrmypdf('skew.pdf', 'test_timeout_%s.pdf' % renderer,
                          '--tesseract-timeout', '1.0')
     pdfinfo = pdf_get_all_pageinfo(out)
-    assert pdfinfo[0]['has_text'] == False
+    assert not pdfinfo[0]['has_text']
 
 
-def test_ocr_timeout():
-    yield check_ocr_timeout, 'hocr'
-    yield check_ocr_timeout, 'tesseract'
-
-
-def test_skip_big():
+def test_skip_big(spoof_tesseract_cache):
     out = check_ocrmypdf('enormous.pdf', 'test_enormous.pdf',
-                         '--skip-big', '10')
+                         '--skip-big', '10', env=spoof_tesseract_cache)
     pdfinfo = pdf_get_all_pageinfo(out)
-    assert pdfinfo[0]['has_text'] == False
+    assert not pdfinfo[0]['has_text']
 
 
-def check_maximum_options(renderer):
+@pytest.mark.parametrize('renderer', [
+    'hocr',
+    'tesseract',
+    ])
+def test_maximum_options(spoof_tesseract_cache, renderer):
     check_ocrmypdf(
         'multipage.pdf', 'test_multipage%s.pdf' % renderer,
         '-d', '-c', '-i', '-g', '-f', '-k', '--oversample', '300',
         '--skip-big', '10', '--title', 'Too Many Weird Files',
-        '--author', 'py.test', '--pdf-renderer', renderer)
+        '--author', 'py.test', '--pdf-renderer', renderer,
+        env=spoof_tesseract_cache)
 
 
-def test_maximum_options():
-    yield check_maximum_options, 'hocr'
-    yield check_maximum_options, 'tesseract'
+def test_tesseract_missing_tessdata():
+    env = os.environ.copy()
+    env['TESSDATA_PREFIX'] = '/tmp'
+
+    p, _, err = run_ocrmypdf_env(
+        'graph_ocred.pdf', 'not_a_pdfa.pdf', '-v', '1', '--skip-text', env=env)
+    assert p.returncode == ExitCode.missing_dependency, err
+
+
+def test_invalid_input_pdf():
+    p, out, err = run_ocrmypdf_env(
+        'invalid.pdf', 'wont_be_created.pdf')
+    assert p.returncode == ExitCode.input_file, err
+
+
+def test_blank_input_pdf():
+    p, out, err = run_ocrmypdf_env(
+        'blank.pdf', 'still_blank.pdf')
+    assert p.returncode == ExitCode.ok
+
+
+def test_french(spoof_tesseract_cache):
+    p, out, err = run_ocrmypdf_env(
+        'francais.pdf', 'francais.pdf', '-l', 'fra', env=spoof_tesseract_cache)
+    assert p.returncode == ExitCode.ok, \
+        "This test may fail if Tesseract language packs are missing"
+
+
+def test_klingon():
+    p, out, err = run_ocrmypdf_env(
+        'francais.pdf', 'francais.pdf', '-l', 'klz')
+    assert p.returncode == ExitCode.bad_args
+
+
+def test_missing_docinfo(spoof_tesseract_noop):
+    p, out, err = run_ocrmypdf_env(
+        'missing_docinfo.pdf', 'missing_docinfo.pdf', '-l', 'eng', '-c',
+        env=spoof_tesseract_noop)
+    assert p.returncode == ExitCode.ok, err
+
+
+@pytest.mark.skipif(running_in_docker(),
+                    reason="writes to tests/resources")
+def test_uppercase_extension(spoof_tesseract_noop):
+    shutil.copy(_infile("skew.pdf"), _infile("UPPERCASE.PDF"))
+    try:
+        check_ocrmypdf("UPPERCASE.PDF", "UPPERCASE_OUT.PDF",
+                       env=spoof_tesseract_noop)
+    finally:
+        os.unlink(_infile("UPPERCASE.PDF"))
+
+
+def test_input_file_not_found():
+    input_file = "does not exist.pdf"
+    sh, out, err = run_ocrmypdf_sh(
+        _infile(input_file),
+        _outfile("will not happen.pdf"))
+    assert sh.returncode == ExitCode.input_file
+    assert (input_file in out or input_file in err)
+
+
+def test_input_file_not_a_pdf():
+    input_file = __file__  # Try to OCR this file
+    sh, out, err = run_ocrmypdf_sh(
+        _infile(input_file),
+        _outfile("will not happen.pdf"))
+    assert sh.returncode == ExitCode.input_file
+    assert (input_file in out or input_file in err)
+
+
+def test_qpdf_repair_fails():
+    env = os.environ.copy()
+    env['OCRMYPDF_QPDF'] = os.path.abspath('./spoof/qpdf_dummy_return2.py')
+    p, out, err = run_ocrmypdf_env(
+        '-v', '1',
+        'c02-22.pdf', 'wont_be_created.pdf', env=env)
+    print(out)
+    print(err)
+    assert p.returncode == ExitCode.input_file
+
+
+def test_encrypted():
+    p, out, err = run_ocrmypdf_env('skew-encrypted.pdf', 'wont_be_created.pdf')
+    assert p.returncode == ExitCode.input_file
+    assert out.find('password')
+
+
+@pytest.mark.parametrize('renderer', [
+    'hocr',
+    'tesseract',
+    ])
+def test_pagesegmode(renderer, spoof_tesseract_cache):
+    check_ocrmypdf(
+        'skew.pdf', 'test_psm_%s.pdf' % renderer,
+        '--tesseract-pagesegmode', '7',
+        '-v', '1',
+        '--pdf-renderer', renderer, env=spoof_tesseract_cache)
+
+
+@pytest.mark.parametrize('renderer', [
+    'hocr',
+    'tesseract',
+    ])
+def test_tesseract_crash(renderer, spoof_tesseract_crash):
+    sh, out, err = run_ocrmypdf_env(
+        'ccitt.pdf', 'wontwork.pdf', '-v', '1',
+        '--pdf-renderer', renderer, env=spoof_tesseract_crash)
+    assert sh.returncode == ExitCode.child_process_error
+    assert not os.path.exists(_outfile('wontwork.pdf'))
+    assert "ERROR" in err
+
+
+def test_tesseract_crash_autorotate(spoof_tesseract_crash):
+    sh, out, err = run_ocrmypdf_env(
+        'ccitt.pdf', 'wontwork.pdf',
+        '-r', env=spoof_tesseract_crash)
+    assert sh.returncode == ExitCode.child_process_error
+    assert not os.path.exists(_outfile('wontwork.pdf'))
+    assert "ERROR" in err
+    print(out)
+    print(err)
+
+
+@pytest.mark.parametrize('renderer', [
+    'hocr',
+    'tesseract',
+    ])
+def test_tesseract_image_too_big(renderer, spoof_tesseract_big_image_error):
+    check_ocrmypdf(
+        'hugemono.pdf', 'hugemono_%s.pdf' % renderer, '-r',
+        '--pdf-renderer', renderer, env=spoof_tesseract_big_image_error)
+
+
+def test_no_unpaper():
+    env = os.environ.copy()
+    env['OCRMYPDF_UNPAPER'] = os.path.abspath('./spoof/no_unpaper_here.py')
+    sh, out, err = run_ocrmypdf_env(
+        'c02-22.pdf', 'wont_be_created.pdf', '--clean', env=env)
+    assert sh.returncode == ExitCode.missing_dependency
+
+
+def test_old_unpaper():
+    env = os.environ.copy()
+    env['OCRMYPDF_UNPAPER'] = os.path.abspath('./spoof/unpaper_oldversion.py')
+    sh, out, err = run_ocrmypdf_env(
+        'c02-22.pdf', 'wont_be_created.pdf', '--clean', env=env)
+    assert sh.returncode == ExitCode.missing_dependency
+
+
+def test_algo4():
+    sh, _, _ = run_ocrmypdf_env('encrypted_algo4.pdf', 'wontwork.pdf')
+    assert sh.returncode == ExitCode.encrypted_pdf

tests/test_pageinfo.py

@@ -7,22 +7,40 @@ from PIL import Image
 from tempfile import NamedTemporaryFile
 from contextlib import suppress
 import os
-import sys
 import shutil
 import pytest
 import img2pdf
-from pkg_resources import Requirement, resource_filename
+import pytest
+import sys
 
-req = Requirement.parse('ocrmypdf')
 
-TEST_OUTPUT = os.path.join(os.path.dirname(__file__), 'output')
+if sys.version_info.major < 3:
+    print("Requires Python 3.4+")
+    sys.exit(1)
+
+TESTS_ROOT = os.path.abspath(os.path.dirname(__file__))
+SPOOF_PATH = os.path.join(TESTS_ROOT, 'spoof')
+PROJECT_ROOT = os.path.dirname(TESTS_ROOT)
+OCRMYPDF = os.path.join(PROJECT_ROOT, 'OCRmyPDF.sh')
+TEST_RESOURCES = os.path.join(PROJECT_ROOT, 'tests', 'resources')
+TEST_OUTPUT = os.environ.get(
+    'OCRMYPDF_TEST_OUTPUT',
+    default=os.path.join(PROJECT_ROOT, 'tests', 'output', 'pageinfo'))
 
 
 def setup_module():
     with suppress(FileNotFoundError):
         shutil.rmtree(TEST_OUTPUT)
     with suppress(FileExistsError):
-        os.mkdir(TEST_OUTPUT)
+        os.makedirs(TEST_OUTPUT)
+
+
+def _make_input(input_basename):
+    return os.path.join(TEST_RESOURCES, input_basename)
+
+
+def _make_output(output_basename):
+    return os.path.join(TEST_OUTPUT, output_basename)
 
 
 def test_single_page_text():
@@ -49,13 +67,21 @@ def test_single_page_text():
 def test_single_page_image():
     filename = os.path.join(TEST_OUTPUT, 'image-mono.pdf')
 
-    with NamedTemporaryFile() as im_tmp:
+    with NamedTemporaryFile(mode='wb+', suffix='.png') as im_tmp:
         im = Image.new('1', (8, 8), 0)
         for n in range(8):
             im.putpixel((n, n), 1)
         im.save(im_tmp.name, format='PNG')
 
-        pdf_bytes = img2pdf.convert([im_tmp.name], dpi=8)
+        imgsize = ((img2pdf.ImgSize.dpi, 8), (img2pdf.ImgSize.dpi, 8))
+        layout_fun = img2pdf.get_layout_fun(None, imgsize, None, None, None)
+
+        im_tmp.seek(0)
+        im_bytes = im_tmp.read()
+        pdf_bytes = img2pdf.convert(
+                im_bytes, producer="img2pdf", with_pdfrw=False,
+                layout_fun=layout_fun)
+
         with open(filename, 'wb') as pdf:
             pdf.write(pdf_bytes)
 
@@ -77,8 +103,8 @@ def test_single_page_image():
     assert pdfimage['bpc'] == 8
 
     # DPI in a 1"x1" is the image width
-    assert pdfimage['dpi_w'] == 8
-    assert pdfimage['dpi_h'] == 8
+    assert abs(pdfimage['dpi_w'] - 8) < 1e-5
+    assert abs(pdfimage['dpi_h'] - 8) < 1e-5
 
 
 def test_single_page_inline_image():
@@ -94,15 +120,20 @@ def test_single_page_inline_image():
         pdf.showPage()
         pdf.save()
 
-    with pytest.raises(NotImplementedError):
-        pageinfo.pdf_get_all_pageinfo(filename)
+    pdfinfo = pageinfo.pdf_get_all_pageinfo(filename)
+    print(pdfinfo)
+    pdfimage = pdfinfo[0]['images'][0]
+    assert (pdfimage['dpi_w'] - 8) < 1e-5
+    assert pdfimage['color'] != '-'
+    assert pdfimage['width'] == 8
 
 
 def test_jpeg():
-    filename = resource_filename(req, 'tests/resources/c02-22.pdf')
+    filename = _make_input('c02-22.pdf')
 
     pdfinfo = pageinfo.pdf_get_all_pageinfo(filename)
 
     pdfimage = pdfinfo[0]['images'][0]
     assert pdfimage['enc'] == 'jpeg'
+    assert (pdfimage['dpi_w'] - 150) < 1e-5