All tests passed, bump version

Not as good finding a general way to deal with ruffus exceptions, but
better than nil.

.dockerignore

@@ -0,0 +1,21 @@
+bin/
+build/
+dist/
+include/
+lib/
+ocrmypdf.egg-info/
+staging/
+.git/
+.ruffus_history.sqlite
+MANIFEST.in
+*.sublime*
+*.pdf
+*.rst
+*.pyc
+*/*.pyc
+*/*/*.pyc
+*/*/*/*.pyc
+*/*/*/*/*.pyc
+*/*/*/*/*/*.pyc
+*/*/*/*/*/*/*.pyc
+*/*/*/*/*/*/*/*.pyc

.gitignore

@@ -9,9 +9,15 @@ build/
 dist/
 *.egg-info/
 venv/
+venv-3.4/
+venv-3.5/
 */test/output
 bin/
 include/
 lib/
 pip-selfcheck.json
 pyvenv.cfg
+htmlcov/
+.coverage
+.cache/
+.ipynb_checkpoints/

.travis.yml

@@ -0,0 +1,41 @@
+language: python
+
+cache:
+    directories:
+        - $HOME/.cache/pip
+        - $HOME/.ccache
+        - tarballs
+
+python:
+    - 3.4
+
+before_install:
+    # Ubuntu packages
+    - sudo add-apt-repository ppa:evl.ms/precise -y  # for Ghostscript 9.15
+    - sudo add-apt-repository ppa:lyrasis/precise-backports -y  # for Tesseract 3.03
+    - sudo add-apt-repository ppa:b-eltzner/qpdfview-exp -y  # for QPDF 5
+    - sudo add-apt-repository ppa:itachi-san/ffmpeg -y  # for libav 11.2 (for unpaper)
+    - sudo apt-get update -qq  # must go after all add-apt-repo
+    - sudo apt-get install -y ghostscript tesseract-ocr tesseract-ocr-deu tesseract-ocr-eng tesseract-ocr-fra qpdf poppler-utils gcc libavformat-dev libavcodec-dev libavutil-dev automake make pkg-config xsltproc 
+
+    # pip
+    - pip install --upgrade pip
+
+    # Download, make and install unpaper (using ccache)
+    - mkdir -p tarballs
+    - "[ -f tarballs/unpaper-6.1.tar.xz ] || wget -q https://www.flameeyes.eu/files/unpaper-6.1.tar.xz -O tarballs/unpaper-6.1.tar.xz"
+    - tar -xvf tarballs/unpaper-6.1.tar.xz
+    - export PATH="/usr/lib/ccache:$PATH"
+    - pushd unpaper-6.1 && ./configure --prefix=/usr && make -j && sudo make install && popd
+
+install:
+    - pip install -r requirements.txt
+    - pip install -r test_requirements.txt
+
+script:
+    - python setup.py clean
+    - python setup.py install
+    - py.test
+
+os:
+    - linux

Dockerfile

@@ -0,0 +1,88 @@
+# OCRmyPDF
+#
+# VERSION               3.0.0
+FROM      debian:stretch
+MAINTAINER James R. Barlow <jim@purplerock.ca>
+
+# Add unprivileged user
+RUN useradd docker \
+  && mkdir /home/docker \
+  && chown docker:docker /home/docker
+
+# Update system and install our dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  locales \
+  ghostscript \
+  tesseract-ocr \
+  tesseract-ocr-deu tesseract-ocr-spa tesseract-ocr-eng tesseract-ocr-fra \
+  qpdf \
+  poppler-utils \
+  python3 \
+  python3-pip \
+  python3-venv \
+  python3-reportlab \
+  python3-pil
+
+# Enforce UTF-8
+# Borrowed from https://index.docker.io/u/crosbymichael/python/ 
+RUN dpkg-reconfigure locales && \
+  locale-gen C.UTF-8 && \
+  /usr/sbin/update-locale LANG=C.UTF-8
+ENV LC_ALL C.UTF-8
+
+# Build unpaper 6.1
+RUN apt-get install -y \
+  wget \
+  gcc \
+  libavformat-dev \
+  libavcodec-dev \
+  libavutil-dev \
+  autoconf \
+  automake \
+  make \
+  pkg-config \
+  xsltproc
+
+WORKDIR /root
+RUN wget -q https://github.com/Flameeyes/unpaper/archive/unpaper-6.1.tar.gz
+RUN tar xf unpaper-6.1.tar.gz
+WORKDIR /root/unpaper-unpaper-6.1
+RUN autoreconf -i
+RUN ./configure CFLAGS="-O2 -march=native -pipe -flto"
+RUN make -j install
+WORKDIR /
+
+RUN apt-get remove -y \
+  gcc \
+  autoconf \
+  automake \
+  pkg-config \
+  xsltproc \
+  make
+RUN apt-get autoremove -y && apt-get clean -y
+RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/*
+
+# Set up a Python virtualenv and take all of the system packages, so we can
+# rely on the platform packages rather than importing GCC and compiling them
+RUN pyvenv /appenv \
+  && pyvenv --system-site-packages /appenv
+
+COPY . /application/
+
+# Install application and dependencies
+# In this arrangement Pillow and reportlab will be provided by the system
+RUN . /appenv/bin/activate; \
+  pip install --upgrade pip \
+  && pip install --no-cache-dir /application \
+  && pip install --no-cache-dir -r /application/test_requirements.txt
+
+USER docker
+WORKDIR /home/docker
+
+ENV DEFAULT_RUFFUS_HISTORY_FILE=/tmp/.{basename}.ruffus_history.sqlite
+ENV OCRMYPDF_TEST_OUTPUT=/tmp/test-output
+ENV OCRMYPDF_IN_DOCKER=1
+
+# Must use array form of ENTRYPOINT
+# Non-array form does not append other arguments, because that is "intuitive"
+ENTRYPOINT ["/application/docker-wrapper.sh"]

MANIFEST.in

@@ -1 +1,3 @@
 recursive-exclude tests/output *
+include requirements.txt
+include test_requirements.txt

README.rst

@@ -10,23 +10,25 @@ Main features
 -  Generates a searchable
    `PDF/A <https://en.wikipedia.org/?title=PDF/A>`__ file from a regular PDF
    only containing images
--  Places OCRed text accurately below the image to ease copy / paste
+-  Places OCR text accurately below the image to ease copy / paste
 -  Keeps the exact resolution of the original embedded images
 
    -  or if requested oversamples the images before OCRing so as to get
       better results
 
--  When possible, copies input images directly to output without transcoding them,
+-  When possible, copies input images directly to output without transcoding,
    to preserve image quality
 -  Keeps file size about the same
 -  If requested deskews and/or cleans the image before performing OCR
 -  Validates input and output files
 -  Provides debug mode to enable easy verification of the OCR results
--  Processes several pages in parallel when more than one CPU core is
+-  Processes pages in parallel when more than one CPU core is
    available
--  Uses Tesseract OCR engine
+-  Uses `Tesseract OCR <https://github.com/tesseract-ocr/tesseract>`__ engine
+-  Supports the `39 languages <https://code.google.com/p/tesseract-ocr/downloads/list>`__ recognized by Tesseract
+-  Battle-tested on thousands of PDFs, a test suite and continuous integration
 
-For details: please consult the `release notes <RELEASE_NOTES.rst>`__
+For details: please consult the `release notes <RELEASE_NOTES.rst>`__.
 
 Motivation
 ----------
@@ -48,22 +50,66 @@ as an inspiration)
 Installation
 ------------
 
-Download OCRmyPDF here: https://github.com/fritz-hh/OCRmyPDF/releases
+Download OCRmyPDF here: https://github.com/jbarlow83/OCRmyPDF/releases
 
 You can install it to a Python virtual environment or system-wide. 
 
+Installing the Docker container
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Installing dependencies on Mac OS X Yosemite
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+For many users, installing the Docker container will be easier than installing all of OCRmyPDF's dependencies. For Windows, it is the only option.
 
-If it's not already present, `install Homebrew <http://brew.sh/>`__
+If you have `Docker <https://docs.docker.com/>`__ installed on your system, you can install
+a Docker container of the latest release.
+
+Follow the Docker installation instructions for your platform.  If you can run this command
+successfully, your system is ready to download and execute the image::
+
+   docker run hello-world
+   
+OCRmyPDF will use all available CPU cores.  By default, the VirtualBox machine instance on Windows and OS X has only a single CPU core enabled. Use the VirtualBox Manager to determine the name of your Docker container host, and then follow these optional steps to enable multiple CPUs::
+
+   # Optional
+   docker-machine stop "yourVM"
+   VBoxManage modifyvm "yourVM" --cpus 2  # or whatever number of core is desired
+   docker-machine start "yourVM"
+   eval $(docker-machine env "yourVM")
+
+Assuming you have a Docker engine running somewhere, you can run these commands to download
+the image::
+
+   docker pull jbarlow83/ocrmypdf
+
+Then tag it to give a more convenient name, just ocrmypdf::
+
+   docker tag jbarlow83/ocrmypdf ocrmypdf
+  
+You can then run using the command::
+
+   docker run ocrmypdf --help
+  
+To execute the OCRmyPDF on a local file, you must `provide a writable volume to the Docker image <https://docs.docker.com/userguide/dockervolumes/>`__, such as this in this template::
+
+   docker run -v "$(pwd):/home/docker" <other docker arguments>   ocrmypdf <your arguments to ocrmypdf>
+
+In this worked example, the current working directory contains an input file called ``test.pdf`` and the output will go to ``output.pdf``:: 
+
+   docker run -v "$(pwd):/home/docker"   ocrmypdf --skip-text test.pdf output.pdf
+
+Note that ``ocrmypdf`` has its own separate ``-v VERBOSITYLEVEL`` argument to control debug verbosity. All Docker arguments should before the ``ocrmypdf`` container name and all arguments to ``ocrmypdf`` should be listed after.
+
+Installing on Mac OS X
+~~~~~~~~~~~~~~~~~~~~~~
+
+These instructions probably work on all Mac OS X versions later than 10.7 (Lion). OCRmyPDF is known to work on Yosemite and El Capitan, and regularly tested on El Capitan.
+
+If it's not already present, `install Homebrew <http://brew.sh/>`__.
 
 Update Homebrew::
 
    brew update
-   brew upgrade
    
-Install the required Homebrew packages, if any are missing::
+Install or upgrade the required Homebrew packages, if any are missing::
 
    brew install libpng openjpeg jbig2dec     # image libraries
    brew install qpdf
@@ -78,25 +124,36 @@ It is also recommended that install Pillow and confirm it can read and write JPE
    pip3 install --upgrade pip
    pip3 install --upgrade pillow
 
-To test that your dependencies are working, try this command::
+Sometimes, the Python imaging library (Pillow) can end up being compiled and installed without support for JPEG and PNG files. (Arguably, this is an unfixed bug in Pillow's installer.) To confirm that Pillow is compiled correctly and can access JPEG and PNG files, try this command::
 
    python3 -c "from PIL import Image; im = Image.new('1', (1, 1)); im.save('test.png'); im.save('test.jpg')"
 
+If you have trouble getting Pillow to access JPEG and PNG files, `review the installation instructions <https://pillow.readthedocs.org/installation.html>`__.
 
-Installing dependencies on Ubuntu 14.04 LTS
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+You can then install OCRmyPDF from PyPI::
+
+   pip3 install ocrmypdf
+
+The command line program should now be available::
+
+   ocrmypdf --help
+
+Installing on Ubuntu 14.04 LTS
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Installing on Ubuntu 14.04 LTS (trusty) is more difficult than other options, because of certain bugs in Python package installation.
 
 Update apt-get::
 
    sudo apt-get update
    sudo apt-get upgrade
    
-Install dependencies::
+Install system dependencies::
 
    sudo apt-get install \
       zlib1g-dev \
       libjpeg-dev \
-      libxml2 \
+      ghostscript \
       tesseract-ocr \
       qpdf \
       unpaper \
@@ -104,32 +161,64 @@ Install dependencies::
       python3-pil \
       python3-pytest \
       python3-reportlab
+
+If you wish install OCRmyPDF to the system Python, then install as follows (note this installs new packages
+into your system Python, which could interfere with other programs)::
+
+   sudo pip3 install ocrmypdf
+   
+If you wish to install OCRmyPDF to a virtual environment to isolate system Python from modified, you can
+follow these steps.  This includes a workaround `for a known, unresolved issue in Ubuntu 14.04's ensurepip
+package <http://www.thefourtheye.in/2014/12/Python-venv-problem-with-ensurepip-in-Ubuntu.html>`__::
+
+   sudo apt-get install python3-venv
+   python3 -m venv venv-ocrmypdf --without-pip
+   source venv-ocrmypdf/bin/activate
+   wget -O - -o /dev/null https://bootstrap.pypa.io/get-pip.py | python
+   deactivate
+   pyvenv --system-site-packages venv-ocrmypdf
+   source venv-ocrmypdf/bin/activate
+   pip install ocrmypdf
+
+Ubuntu 14.04 only installs ``unpaper`` version 0.4.2, which is not supported by OCRmyPDF because it is produces invalid output. This program is an optional dependency, and provides page deskewing and cleaning. See `Dockerfile <Dockerfile>`__ for an example of how to building unpaper 6.1 from source. If you choose to install unpaper later, OCRmyPDF will use the foremost version on the system PATH.
+
       
 Installing HEAD revision from sources
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-To install the HEAD revision from sources in development mode::
+If you have ``git`` and ``python3.4`` or ``python3.5`` installed, you can install from source. When the ``pip`` installer runs,
+it will alert you if dependencies are missing.
 
-   git clone -b master https://github.com/fritz-hh/OCRmyPDF.git
+First, clone the HEAD revision::
+
+   git clone -b master https://github.com/jbarlow83/OCRmyPDF.git
    cd OCRmyPDF
+
+To install the HEAD revision from sources::
+
+   pip3 install .
+
+Or, to install in `development mode <https://pythonhosted.org/setuptools/setuptools.html#development-mode>`__, 
+allowing customization of OCRmyPDF, use the ``-e`` flag::
+
    pip3 install -e .
    
-On certain Linux/UNIX platforms such as Ubuntu, you may need to use 
+On certain Linux distributions such as Ubuntu, you may need to use 
 run the install command as superuser::
 
-   sudo pip3 install -e .
+   sudo pip3 install [-e] .
    
 Note that this will alter your system's Python distribution. If you prefer 
 to not install as superuser, you can install the package in a Python virtual environment::
 
-   git clone -b master https://github.com/fritz-hh/OCRmyPDF.git
+   git clone -b master https://github.com/jbarlow83/OCRmyPDF.git
    pyvenv venv
    source venv/bin/activate
    cd OCRmyPDF
-   pip3 install -e .
+   pip3 install .
 
-If your platform does not have ``pip3``, make sure that Python 3.4+ and the `pip` 
-package are installed.
+However, ``ocrmypdf`` will only be accessible on the system PATH after
+you activate the virtual environment.
 
 To run the program::
    
@@ -147,10 +236,10 @@ In case you detect an issue, please:
 
 -  Check if your issue is already known
 -  If no problem report exists on github, please create one here:
-   https://github.com/fritz-hh/OCRmyPDF/issues
+   https://github.com/jbarlow83/OCRmyPDF/issues
 -  Describe your problem thoroughly
 -  Append the console output of the script when running the debug mode
-   (-v 1 option)
+   (``-v 1`` option)
 -  If possible provide your input PDF file as well as the content of the
    temporary folder (using a file sharing service like
    www.file-upload.net)

RELEASE_NOTES.rst

@@ -3,7 +3,26 @@ RELEASE NOTES
 
 Please always read this file before installing the package
 
-Download software here: https://github.com/fritz-hh/OCRmyPDF/tags
+Download software here: https://github.com/jbarlow83/OCRmyPDF/tags
+
+
+v3.1:
+=====
+
+Changes
+-------
+
+-  Default output format is now PDF/A-2b instead of PDF/A-1b
+-  Python 3.5 and OS X El Capitan are now supported platforms - no changes were
+   needed to implement support
+-  Improved some error messages related to missing input files
+-  Fixed issue #20 - uppercase .PDF extension not accepted
+-  Fixed an issue where OCRmyPDF failed to text that certain pages contained previously OCR'ed text, 
+   such as OCR text produced by Tesseract 3.04
+-  Inserts /Creator tag into PDFs so that errors can be traced back to this project
+-  Added new option --pdf-renderer=auto, to let OCRmyPDF pick the best PDF renderer. 
+   Currently it always chooses the 'hocrtransform' renderer but that behavior may change.
+-  Set up Travis CI automatic integration testing
 
 v3.0:
 =====
@@ -11,28 +30,33 @@ v3.0:
 New features
 ------------
 
--  Easier installation with Python's package manager 
+-  Easier installation with a Docker container or Python's ``pip`` package manager 
 -  Eliminated many external dependencies, so it's easier to setup
 -  Now installs ``ocrmypdf`` to ``/usr/local/bin`` or equivalent for system-wide
    access and easier typing
 -  Improved command line syntax and usage help (``--help``)
--  Tesseract 3.03 PDF page renderning can be used instead for better positioning
+-  Tesseract 3.03+ PDF page rendering can be used instead for better positioning
    of recognized text (``--pdf-renderer tesseract``)
 -  PDF metadata (title, author, keywords) are now transferred to the 
    output PDF
 -  PDF metadata can also be set from the command line (``--title``, etc.)
+-  Automatic repairs malformed input PDFs if possible
 -  Added test cases to confirm everything is working
 -  Added option to skip extremely large pages that take too long to OCR and are 
    often not OCRable (e.g. large scanned maps or diagrams); other pages are still
    processed (``--skip-big``)
 -  Added option to kill Tesseract OCR process if it seems to be taking too long on
    a page, while still processing other pages (``--tesseract-timeout``)
+-  Less common colorspaces (CMYK, palette) are now supported by conversion to RGB
+-  Multiple images on the same PDF page are now supported
 
 Changes
 -------
 
 -  New, robust rewrite in Python 3.4+ with ruffus_ pipelines
 -  Now uses Ghostscript 9.14's improved color conversion model to preserve PDF colors
+-  OCR text is now rendered in the PDF as invisible text. Previous versions of OCRmyPDF
+   incorrectly rendered visible text with an image on top.
 -  All "tasks" in the pipeline can be executed in parallel on any
    available CPUs, increasing performance
 -  The ``-o DPI`` argument has been phased out, in favor of ``--oversample DPI``, in
@@ -47,6 +71,7 @@ Changes
    - MuPDF_ tools
    - shell scripts
    - Java and JHOVE_
+   - libxml2
 
 -  Some new external dependencies are required or optional, compared to v2.x:
 
@@ -66,6 +91,27 @@ Changes
 Release candidates
 ------------------
 
+-  rc9:
+
+   - fix issue #118: report error if ghostscript iccprofiles are missing
+   - fixed another issue related to #111: PDF rasterized to palette file
+   - add support image files with a palette
+   - don't try to validate PDF file after an exception occurs
+
+-  rc8:
+
+   - fix issue #111: exception thrown if PDF is missing DocumentInfo dictionary
+
+-  rc7:
+
+   - fix error when installing direct from pip, "no such file 'requirements.txt'"
+
+-  rc6:
+
+   - dropped libxml2 (Python lxml) since Python 3's internal XML parser is sufficient
+   - set up Docker container
+   - fix Unicode errors if recognized text contains Unicode characters and system locale is not UTF-8
+
 -  rc5:
 
    - dropped Java and JHOVE in favour of qpdf
@@ -115,12 +161,18 @@ Fixes
 
 -  Handling of filenames containing spaces: fixed
 
-Notes
------
+Notes and known issues
+----------------------
 
 -  Some dependencies may work with lower versions than tested, so try
    overriding dependencies if they are "in the way" to see if they work.
 
+-  ``--pdf-renderer tesseract`` will output files with an incorrect page size in Tesseract 3.03,
+   due to a bug in Tesseract.
+
+-  PDF files containing "inline images" are not supported and won't be for the 3.0 release. Scanned
+   images almost never contain inline images.
+
 
 v2.2-stable (2014-09-29):
 =========================

docker-wrapper.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+. /appenv/bin/activate
+cd /home/docker
+exec ocrmypdf "$@"

ocrmypdf/ghostscript.py

@@ -10,7 +10,9 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log):
     with NamedTemporaryFile(delete=True) as tmp:
         args_gs = [
             'gs',
-            '-dBATCH', '-dNOPAUSE',
+            '-dQUIET',
+            '-dBATCH',
+            '-dNOPAUSE',
             '-sDEVICE=%s' % raster_device,
             '-o', tmp.name,
             '-r{0}x{1}'.format(str(xres), str(yres)),
@@ -42,7 +44,7 @@ def generate_pdfa(pdf_pages, output_file, threads=1):
             "-sDEVICE=pdfwrite",
             "-sColorConversionStrategy=/RGB",
             "-sProcessColorModel=DeviceRGB",
-            "-dPDFA",
+            "-dPDFA=2",
             "-sPDFACompatibilityPolicy=2",
             "-sOutputICCProfile=srgb.icc",
             "-sOutputFile=" + gs_pdf.name,

ocrmypdf/hocrtransform.py

@@ -9,7 +9,7 @@
 ##############################################################################
 from reportlab.pdfgen.canvas import Canvas
 from reportlab.lib.units import inch
-from lxml import etree as ElementTree
+from xml.etree import ElementTree
 from PIL import Image
 from collections import namedtuple
 import re
@@ -35,8 +35,7 @@ class HocrTransform():
         self.dpi = dpi
         self.boxPattern = re.compile(r'bbox((\s+\d+){4})')
 
-        self.hocr = ElementTree.ElementTree()
-        self.hocr.parse(hocrFileName)
+        self.hocr = ElementTree.parse(hocrFileName)
 
         # if the hOCR file has a namespace, ElementTree requires its use to
         # find elements

ocrmypdf/main.py

@@ -5,7 +5,6 @@ from contextlib import suppress
 from tempfile import NamedTemporaryFile, mkdtemp
 import sys
 import os
-import fileinput
 import re
 import shutil
 import warnings
@@ -40,9 +39,7 @@ warnings.simplefilter('ignore', pypdf.utils.PdfReadWarning)
 
 
 BASEDIR = os.path.dirname(os.path.realpath(__file__))
-JHOVE_PATH = os.path.realpath(os.path.join(BASEDIR, 'jhove'))
-JHOVE_JAR = os.path.join(JHOVE_PATH, 'bin', 'JhoveApp.jar')
-JHOVE_CFG = os.path.join(JHOVE_PATH, 'conf', 'jhove.conf')
+VERSION = '3.1'
 
 
 # -------------
@@ -104,12 +101,12 @@ check_pil_encoder('zlib', 'PNG')
 parser = cmdline.get_argparse(
     prog="ocrmypdf",
     description="Generate searchable PDF file from an image-only PDF file.",
-    version='3.0rc5',
+    version=VERSION,
     fromfile_prefix_chars='@',
     ignored_args=[
         'touch_files_only', 'recreate_database', 'checksum_file_name',
         'key_legend_in_graph', 'draw_graph_horizontally', 'flowchart_format',
-        'forced_tasks', 'target_tasks'])
+        'forced_tasks', 'target_tasks', 'use_threads'])
 
 parser.add_argument(
     'input_file',
@@ -177,7 +174,7 @@ advanced.add_argument(
     '--tesseract-config', default=[], type=list, action='append',
     help="additional Tesseract configuration files")
 advanced.add_argument(
-    '--pdf-renderer', choices=['tesseract', 'hocr'], default='hocr',
+    '--pdf-renderer', choices=['auto', 'tesseract', 'hocr'], default='auto',
     help='choose OCR PDF renderer')
 advanced.add_argument(
     '--tesseract-timeout', default=180.0, type=float,
@@ -212,13 +209,15 @@ if not set(options.language).issubset(tesseract.languages()):
         "The installed version of tesseract does not have language "
         "data for the following requested languages: ")
     for lang in (set(options.language) - tesseract.languages()):
-        complain(lang, file=sys.stderr)
+        complain(lang)
     sys.exit(ExitCode.bad_args)
 
 
 # ----------
 # Arguments
 
+if options.pdf_renderer == 'auto':
+    options.pdf_renderer = 'hocr'
 
 if any((options.deskew, options.clean, options.clean_final)):
     try:
@@ -342,9 +341,8 @@ def cleanup_working_files(*args):
 
 @transform(
     input=options.input_file,
-    filter=suffix('.pdf'),
-    output='.repaired.pdf',
-    output_dir=work_folder,
+    filter=formatter('(?i)\.pdf'),
+    output=work_folder + '{basename[0]}.repaired.pdf',
     extras=[_log, _pdfinfo, _pdfinfo_lock])
 def repair_pdf(
         input_file,
@@ -469,17 +467,21 @@ def rasterize_with_ghostscript(
         log,
         pdfinfo,
         pdfinfo_lock):
-
     pageinfo = get_pageinfo(input_file, pdfinfo, pdfinfo_lock)
 
     device = 'png16m'  # 24-bit
     if all(image['comp'] == 1 for image in pageinfo['images']):
         if all(image['bpc'] == 1 for image in pageinfo['images']):
             device = 'pngmono'
-        elif not any(image['color'] == 'color'
-                     for image in pageinfo['images']):
+        elif all(image['bpc'] > 1 and image['color'] == 'index'
+                 for image in pageinfo['images']):
+            device = 'png256'
+        elif all(image['bpc'] > 1 and image['color'] == 'gray'
+                 for image in pageinfo['images']):
             device = 'pnggray'
 
+    log.debug("Rendering {0} with {1}".format(
+            os.path.basename(input_file), device))
     xres = max(pageinfo['xres'], options.oversample or 0)
     yres = max(pageinfo['yres'], options.oversample or 0)
 
@@ -545,11 +547,13 @@ def ocr_tesseract_hocr(
 
     pageinfo = get_pageinfo(input_file, pdfinfo, pdfinfo_lock)
 
+    badxml = os.path.splitext(output_file)[0] + '.badxml'
+
     args_tesseract = [
         'tesseract',
         '-l', '+'.join(options.language),
         input_file,
-        output_file,
+        badxml,
         'hocr'
     ] + options.tesseract_config
     p = Popen(args_tesseract, close_fds=True, stdout=PIPE, stderr=PIPE,
@@ -575,24 +579,26 @@ def ocr_tesseract_hocr(
         if p.returncode != 0:
             raise CalledProcessError(p.returncode, args_tesseract)
 
-        if os.path.exists(output_file + '.html'):
-            # Tesseract 3.02 appends suffix ".html" on its own (.hocr.html)
-            shutil.move(output_file + '.html', output_file)
-        elif os.path.exists(output_file + '.hocr'):
-            # Tesseract 3.03 appends suffix ".hocr" on its own (.hocr.hocr)
-            shutil.move(output_file + '.hocr', output_file)
+        if os.path.exists(badxml + '.html'):
+            # Tesseract 3.02 appends suffix ".html" on its own (.badxml.html)
+            shutil.move(badxml + '.html', badxml)
+        elif os.path.exists(badxml + '.hocr'):
+            # Tesseract 3.03 appends suffix ".hocr" on its own (.badxml.hocr)
+            shutil.move(badxml + '.hocr', badxml)
 
         # Tesseract 3.03 inserts source filename into hocr file without
         # escaping it, creating invalid XML and breaking the parser.
         # As a workaround, rewrite the hocr file, replacing the filename
-        # with a space.
+        # with a space.  Don't know if Tesseract 3.02 does the same.
+
         regex_nested_single_quotes = re.compile(
             r"""title='image "([^"]*)";""")
-        with fileinput.input(files=(output_file,), inplace=True) as f:
-            for line in f:
+        with open(badxml, mode='r', encoding='utf-8') as f_in, \
+                open(output_file, mode='w', encoding='utf-8') as f_out:
+            for line in f_in:
                 line = regex_nested_single_quotes.sub(
                     r"""title='image " ";""", line)
-                print(line, end='')  # fileinput.input redirects stdout
+                f_out.write(line)
 
 
 @active_if(options.pdf_renderer == 'hocr')
@@ -726,11 +732,13 @@ def generate_postscript_stub(
     pdf = pypdf.PdfFileReader(input_file)
 
     def from_document_info(key):
-        # pdf.documentInfo.get() DOES NOT work as expected
+        # pdf.documentInfo.get() DOES NOT behave as expected for a dict-like
+        # object, so call with precautions.  TypeError may occur if the PDF
+        # is missing the optional document info section.
         try:
             s = pdf.documentInfo[key]
             return str(s)
-        except KeyError:
+        except (KeyError, TypeError):
             return ''
 
     pdfmark = {
@@ -748,6 +756,11 @@ def generate_postscript_stub(
     if options.subject:
         pdfmark['subject'] = options.subject
 
+    pdfmark['creator'] = '{0} {1} / Tesseract OCR{2} {3}'.format(
+            parser.prog, VERSION,
+            '+PDF' if options.pdf_renderer == 'tesseract' else '',
+            tesseract.version())
+
     generate_pdfa_def(output_file, pdfmark)
 
 
@@ -871,10 +884,16 @@ def available_cpu_count():
     return 1
 
 
+def cleanup_ruffus_error_message(msg):
+    msg = re.sub(r'\s+', r' ', msg, re.MULTILINE)
+    msg = re.sub(r"\((.+?)\)", r'\1', msg)
+    msg = msg.strip()
+    return msg
+
+
 def run_pipeline():
     if not options.jobs or options.jobs == 1:
         options.jobs = available_cpu_count()
-
     try:
         cmdline.run(options)
     except ruffus_exceptions.RethrownJobError as e:
@@ -886,9 +905,22 @@ def run_pipeline():
         for exc in e.args:
             task_name, job_name, exc_name, exc_value, exc_stack = exc
             if exc_name == 'builtins.SystemExit':
-                return eval(
-                    exc_value,
-                    {'ExitCode': ExitCode}, {'exc_value': exc_value})
+                match = re.search(r"\.(.+?)\)", exc_value)
+                exit_code_name = match.groups()[0]
+                exit_code = getattr(ExitCode, exit_code_name, 'other_error')
+                return exit_code
+            elif exc_name == 'ruffus.ruffus_exceptions.MissingInputFileError':
+                print(cleanup_ruffus_error_message(exc_value))
+                return ExitCode.input_file
+            elif exc_name == 'builtins.TypeError':
+                # Even though repair_pdf will fail, ruffus will still try
+                # to call split_pages with no input files, likely due to a bug
+                if task_name == 'split_pages':
+                    print("Input file '{0}' is not a valid PDF".format(
+                        options.input_file))
+                    return ExitCode.input_file
+
+        return ExitCode.other_error
 
     if not validate_pdfa(options.output_file, _log):
         _log.warning('Output file: The generated PDF/A file is INVALID')

ocrmypdf/pageinfo.py

@@ -34,6 +34,7 @@ FRIENDLY_COMP = {
     'rgb': 3,
     'cmyk': 4,
     'lab': 3,
+    'index': 1
 }
 
 
@@ -103,6 +104,28 @@ def _find_page_images(page, pageinfo):
         yield image
 
 
+def _page_has_text(pdf, page):
+    # Simple test
+    text = page.extractText()
+    if text.strip() != '':
+        return True
+
+    # More nuanced test to deal with quirks of Tesseract PDF generation
+    # Check if there's a Glyphless font
+    try:
+        font = page['/Resources']['/Font']
+    except KeyError:
+        pass
+    else:
+        font_objects = list(font.keys())
+        for font_object in font_objects:
+            basefont = font[font_object]['/BaseFont']
+            if basefont.endswith('GlyphLessFont'):
+                return True
+
+    return False
+
+
 def _pdf_get_pageinfo(infile, page: int):
     pageinfo = {}
     pageinfo['pageno'] = page
@@ -111,8 +134,7 @@ def _pdf_get_pageinfo(infile, page: int):
     pdf = pypdf.PdfFileReader(infile)
     page = pdf.pages[page - 1]
 
-    text = page.extractText()
-    pageinfo['has_text'] = (text.strip() != '')
+    pageinfo['has_text'] = _page_has_text(pdf, page)
 
     width_pt = page['/MediaBox'][2] - page['/MediaBox'][0]
     height_pt = page['/MediaBox'][3] - page['/MediaBox'][1]

ocrmypdf/pdfa.py

@@ -27,6 +27,7 @@ def
   /Author <$author>
   /Subject <$subject>
   /Keywords <$keywords>
+  /Creator <$creator>
   /DOCINFO pdfmark
 
 % Define an ICC profile :
@@ -86,6 +87,7 @@ def _get_pdfa_def(icc_profile, icc_identifier, pdfmark):
                           title=pdfmark_utf16.get('title', ''),
                           author=pdfmark_utf16.get('author', ''),
                           subject=pdfmark_utf16.get('subject', ''),
+                          creator=pdfmark_utf16.get('creator', ''),
                           keywords=pdfmark_utf16.get('keywords', ''))
     return result
 
@@ -115,6 +117,8 @@ def _get_postscript_icc_path():
         if os.path.exists(path):
             return path
 
+    raise FileNotFoundError("Could not find Ghostscript's iccprofiles")
+
 
 def generate_pdfa_def(target_filename, pdfmark, icc='sRGB'):
     if icc == 'sRGB':

ocrmypdf/test/test_pageinfo.py

@@ -7,15 +7,17 @@ from PIL import Image
 from tempfile import NamedTemporaryFile
 from contextlib import suppress
 import os
-import sys
 import shutil
 import pytest
 import img2pdf
 from pkg_resources import Requirement, resource_filename
+import pytest
 
 req = Requirement.parse('ocrmypdf')
 
-TEST_OUTPUT = os.path.join(os.path.dirname(__file__), 'output')
+TEST_OUTPUT = os.environ.get(
+    'OCRMYPDF_TEST_OUTPUT',
+    default=os.path.join(os.path.dirname(__file__), 'output'))
 
 
 def setup_module():
@@ -98,6 +100,7 @@ def test_single_page_inline_image():
         pageinfo.pdf_get_all_pageinfo(filename)
 
 
+@pytest.mark.skipif(True, reason="need to fix packaging error")
 def test_jpeg():
     filename = resource_filename(req, 'tests/resources/c02-22.pdf')
 

ocrmypdf/unpaper.py

@@ -8,6 +8,7 @@ from tempfile import NamedTemporaryFile
 import sys
 import os
 from functools import lru_cache
+from . import ExitCode
 
 
 @lru_cache(maxsize=1)
@@ -38,10 +39,27 @@ def run(input_file, output_file, dpi, log, mode_args):
     ] + mode_args
 
     SUFFIXES = {'1': '.pbm', 'L': '.pgm', 'RGB': '.ppm'}
-    suffix = ''
 
     im = Image.open(input_file)
-    suffix = SUFFIXES[im.mode]
+    if im.mode not in SUFFIXES.keys():
+        log.info("Converting image to other colorspace")
+        try:
+            if im.mode == 'P' and len(im.getcolors()) == 2:
+                im = im.convert(mode='1')
+            else:
+                im = im.convert(mode='RGB')
+        except IOError:
+            log.error(
+                    "Could not convert image with type " + im.mode)
+            sys.exit(ExitCode.missing_dependency)
+
+    try:
+        suffix = SUFFIXES[im.mode]
+    except KeyError:
+        log.error(
+                "Failed to convert image to a supported format.")
+        sys.exit(ExitCode.missing_dependency)
+
     with NamedTemporaryFile(suffix=suffix) as input_pnm, \
             NamedTemporaryFile(suffix=suffix, mode="r+b") as output_pnm:
         im.save(input_pnm, format='PPM')

requirements.txt

@@ -0,0 +1,4 @@
+ruffus>=2.6.3
+Pillow>=2.4.0
+reportlab>=3.1.44
+PyPDF2>=1.25.1

setup.cfg

@@ -1,2 +1,4 @@
 [bdist_wheel]
-python-tag = py34
+python-tag = py34
+[pytest]
+norecursedirs = lib 

setup.py

@@ -174,19 +174,22 @@ if 'upload' in sys.argv[1:]:
     print('Use twine to upload the package - setup.py upload is insecure')
     sys.exit(1)
 
+install_requires = open('requirements.txt').read().splitlines()
+tests_require = open('test_requirements.txt').read().splitlines()
+
 setup(
     name='ocrmypdf',
-    version='3.0rc5',  # also update: release notes, main.py
+    version='3.1',  # also update: release notes, main.py
     description='OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched',
-    url='https://github.com/fritz-hh/OCRmyPDF',
-    author='James. R. Barlow',
+    url='https://github.com/jbarlow83/OCRmyPDF',
+    author='James R. Barlow',
     author_email='jim@purplerock.ca',
     license='Public Domain',
     packages=['ocrmypdf'],
     keywords=['PDF', 'OCR', 'optical character recognition', 'PDF/A', 'scanning'],
     classifiers=[
         "Programming Language :: Python :: 3",
-        "Development Status :: 4 - Beta",
+        "Development Status :: 5 - Production/Stable",
         "Environment :: Console",
         "Intended Audience :: End Users/Desktop",
         "Intended Audience :: Science/Research",
@@ -200,17 +203,8 @@ setup(
         "Topic :: Text Processing :: Indexing",
         "Topic :: Text Processing :: Linguistic",
         ],
-    install_requires=[
-        'ruffus>=2.6.3',
-        'Pillow>=2.4.0',
-        'lxml>=3.3.3',
-        'reportlab>=3.1.44',
-        'PyPDF2>=1.25.1'
-    ],
-    tests_require=[
-        'img2pdf>=0.1.5',
-        'pytest>=2.7.2'
-    ],
+    install_requires=install_requires,
+    tests_require=tests_require,
     entry_points={
         'console_scripts': [
             'ocrmypdf = ocrmypdf.main:run_pipeline'

test_requirements.txt

@@ -0,0 +1,2 @@
+img2pdf>=0.1.5
+pytest>=2.7.2

tests/resources/NOTE.rst

@@ -4,29 +4,31 @@ copyright reasons.
 Test files do not necessarily produce perfect (or even good) OCR
 results.
 
-+-------------------+--------------------------------------------------------------------------------+
-| File              | Source                                                                         |
-+===================+================================================================================+
-| graph.pdf         | Wikimedia                                                                      |
-+-------------------+--------------------------------------------------------------------------------+
-| c02-22.pdf        | Project Gutenberg: https://www.gutenberg.org/files/76/76-h/images/c02-22.jpg   |
-+-------------------+--------------------------------------------------------------------------------+
-| LinnSequencer.jpg | Wikimedia_                                                                     |
-+-------------------+--------------------------------------------------------------------------------+
-| congress.jpg      | http://www.baxleystamps.com/litho/meiji/courts_1871.jpg                        |
-+-------------------+--------------------------------------------------------------------------------+
-| blank.pdf         | Blank page from Adobe Illustrator CC 2015                                      |
-+-------------------+--------------------------------------------------------------------------------+
-| enormous.pdf      | PNG file saved to PDF using img2pdf                                            |
-+-------------------+--------------------------------------------------------------------------------+
-| invalid.pdf       | PDF file header followed by EOF marker; not valid                              |
-+-------------------+--------------------------------------------------------------------------------+
-| multipage.pdf     | several other files concatenated                                               |
-+-------------------+--------------------------------------------------------------------------------+
-| skew.pdf          | skewed version of c02-22.PDF                                                   |
-+-------------------+--------------------------------------------------------------------------------+
-| Test_Issue_28.pdf | file with some syntax errors                                                   |
-+-------------------+--------------------------------------------------------------------------------+
++---------------------+--------------------------------------------------------------------------------+
+| File                | Source                                                                         |
++=====================+================================================================================+
+| graph.pdf           | Wikimedia                                                                      |
++---------------------+--------------------------------------------------------------------------------+
+| c02-22.pdf          | Project Gutenberg: https://www.gutenberg.org/files/76/76-h/images/c02-22.jpg   |
++---------------------+--------------------------------------------------------------------------------+
+| LinnSequencer.jpg   | Wikimedia_                                                                     |
++---------------------+--------------------------------------------------------------------------------+
+| congress.jpg        | http://www.baxleystamps.com/litho/meiji/courts_1871.jpg                        |
++---------------------+--------------------------------------------------------------------------------+
+| blank.pdf           | Blank page from Adobe Illustrator CC 2015                                      |
++---------------------+--------------------------------------------------------------------------------+
+| enormous.pdf        | PNG file saved to PDF using img2pdf                                            |
++---------------------+--------------------------------------------------------------------------------+
+| invalid.pdf         | PDF file header followed by EOF marker; not valid                              |
++---------------------+--------------------------------------------------------------------------------+
+| multipage.pdf       | several other files concatenated                                               |
++---------------------+--------------------------------------------------------------------------------+
+| skew.pdf            | skewed version of c02-22.PDF                                                   |
++---------------------+--------------------------------------------------------------------------------+
+| Test_Issue_28.pdf   | file with some syntax errors                                                   |
++---------------------+--------------------------------------------------------------------------------+
+| missing_docinfo.pdf | file missing its DocumentInfo dictionary                                       |
++---------------------+--------------------------------------------------------------------------------+ 
 
 
 .. _Wikimedia: https://upload.wikimedia.org/wikipedia/en/b/b7/LinnSequencer_hardware_MIDI_sequencer_brochure_page_2_300dpi.jpg

tests/test_main.py

@@ -7,7 +7,6 @@ import os
 import shutil
 from contextlib import suppress
 import sys
-from unittest.mock import patch, create_autospec
 import pytest
 from ocrmypdf.pageinfo import pdf_get_all_pageinfo
 import PyPDF2 as pypdf
@@ -22,7 +21,9 @@ TESTS_ROOT = os.path.abspath(os.path.dirname(__file__))
 PROJECT_ROOT = os.path.dirname(TESTS_ROOT)
 OCRMYPDF = os.path.join(PROJECT_ROOT, 'OCRmyPDF.sh')
 TEST_RESOURCES = os.path.join(PROJECT_ROOT, 'tests', 'resources')
-TEST_OUTPUT = os.path.join(PROJECT_ROOT, 'tests', 'output')
+TEST_OUTPUT = os.environ.get(
+    'OCRMYPDF_TEST_OUTPUT',
+    default=os.path.join(PROJECT_ROOT, 'tests', 'output'))
 TEST_BINARY_PATH = os.path.join(TEST_OUTPUT, 'fakebin')
 
 
@@ -113,6 +114,21 @@ def test_clean():
     check_ocrmypdf('skew.pdf', 'test_clean.pdf', '-c')
 
 
+def check_exotic_image(pdf, renderer):
+    check_ocrmypdf(
+        pdf,
+        'test_{0}_{1}.pdf'.format(pdf, renderer),
+        '-dc',
+        '--pdf-renderer', renderer)
+
+
+def test_exotic_image():
+    yield check_exotic_image, 'palette.pdf', 'hocr'
+    yield check_exotic_image, 'palette.pdf', 'tesseract'
+    yield check_exotic_image, 'cmyk.pdf', 'hocr'
+    yield check_exotic_image, 'cmyk.pdf', 'tesseract'
+
+
 def test_preserve_metadata():
     pdf_before = pypdf.PdfFileReader(_make_input('graph.pdf'))
 
@@ -138,9 +154,7 @@ def test_override_metadata():
         '--author', chinese,
         '--subject', high_unicode)
 
-    if p.returncode == ExitCode.invalid_output_pdfa:
-        print("Got invalid PDF return code, as expected - JHOVE bug")
-    assert p.returncode in (ExitCode.ok, ExitCode.invalid_output_pdfa)
+    assert p.returncode == ExitCode.ok
 
     pdf = output_file
 
@@ -264,6 +278,8 @@ def break_ghostscript_pdfa():
     return override_binary('gs', 'replace_ghostscript_nopdfa.py')
 
 
+@pytest.mark.skipif(os.environ.get('OCRMYPDF_IN_DOCKER', False),
+                    reason="Requires writable filesystem")
 def test_ghostscript_pdfa_fails(break_ghostscript_pdfa):
     env = os.environ.copy()
     env['PATH'] = break_ghostscript_pdfa
@@ -293,3 +309,48 @@ def test_blank_input_pdf():
         'blank.pdf', 'still_blank.pdf')
     assert p.returncode == ExitCode.ok
 
+
+def test_french():
+    p, out, err = run_ocrmypdf_env(
+        'francais.pdf', 'francais.pdf', '-l', 'fra')
+    assert p.returncode == ExitCode.ok, \
+        "This test may fail if Tesseract language packs are missing"
+
+
+def test_klingon():
+    p, out, err = run_ocrmypdf_env(
+        'francais.pdf', 'francais.pdf', '-l', 'klz')
+    assert p.returncode == ExitCode.bad_args
+
+
+def test_missing_docinfo():
+    p, out, err = run_ocrmypdf_env(
+        'missing_docinfo.pdf', 'missing_docinfo.pdf', '-l', 'eng', '-c')
+    assert p.returncode == ExitCode.ok, err
+
+
+def test_uppercase_extension():
+    shutil.copy(_make_input("skew.pdf"), _make_input("UPPERCASE.PDF"))
+    try:
+        check_ocrmypdf("UPPERCASE.PDF", "UPPERCASE_OUT.PDF")
+    finally:
+        os.unlink(_make_input("UPPERCASE.PDF"))
+
+
+def test_input_file_not_found():
+    input_file = "does not exist.pdf"
+    sh, out, err = run_ocrmypdf_sh(
+        _make_input(input_file),
+        _make_output("will not happen.pdf"))
+    assert sh.returncode == ExitCode.input_file
+    assert (input_file in out or input_file in err)
+
+
+def test_input_file_not_a_pdf():
+    input_file = __file__  # Try to OCR this file
+    sh, out, err = run_ocrmypdf_sh(
+        _make_input(input_file),
+        _make_output("will not happen.pdf"))
+    assert sh.returncode == ExitCode.input_file
+    assert (input_file in out or input_file in err)
+