Re-enable macos

Reorganize and see if this is better now

.coveragerc

@@ -0,0 +1,16 @@
+# Coverage isn't really compatible with subprocesses so results are unreliable
+
+[run]
+branch = True
+#concurrency = multiprocessing
+source = ocrmypdf/
+
+[report]
+exclude_lines =
+	pragma: no cover
+	def __repr__
+	raise AssertionError
+	raise NotImplementedError
+	if 0:
+	if False:
+	if __name__ == .__main__.:

.gitignore

@@ -38,4 +38,7 @@ tests/cache/
 tests/output/
 tests/resources/private/
 tmp/
-pdfbox-app*.jar
+pdfbox-app*.jar
+.vscode/
+IDEAS
+_Dockerfile.local

.travis.yml

@@ -44,7 +44,8 @@ install:
 script:
 - mv ocrmypdf dont_import_this_ocrmypdf
 - tesseract --version
-- pytest -n auto
+- qpdf --version
+- pytest
 - mv dont_import_this_ocrmypdf ocrmypdf
 
 deploy:
@@ -85,6 +86,7 @@ deploy:
   script: /usr/bin/true
   on:
     branch: master
+    tags: true
     condition: $TRAVIS_OS_NAME == "osx"
 
 after_deploy: |

.travis/autobrew.py

@@ -23,6 +23,10 @@ class Ocrmypdf < Formula
   depends_on "unpaper"
   depends_on "qpdf"
 
+  # mactex installs its own ghostscript by default which causes problems
+  # mactex users should use caskroom/cask/mactex-no-ghostscript instead
+  conflicts_with cask: "caskroom/cask/mactex"
+
   # For Pillow source install
   depends_on "openjpeg"
   depends_on "freetype"
@@ -30,6 +34,7 @@ class Ocrmypdf < Formula
   depends_on "libjpeg"
   depends_on "webp"
   depends_on "little-cms2"
+  depends_on "zlib"
 
 ${resources}
   def install

Dockerfile

@@ -1,7 +1,6 @@
 # OCRmyPDF
 #
-# VERSION               4.4.2
-FROM      ubuntu:16.10
+FROM      ubuntu:17.04
 MAINTAINER James R. Barlow <jim@purplerock.ca>
 
 RUN apt-get update && apt-get install -y --no-install-recommends \

Dockerfile.polyglot

@@ -1,8 +1,6 @@
 # OCRmyPDF polyglot
 #
-# VERSION               4.4.2
 FROM      jbarlow83/ocrmypdf:latest
-MAINTAINER James R. Barlow <jim@purplerock.ca>
 
 USER root
 

Dockerfile.tess4

@@ -1,7 +1,6 @@
 # OCRmyPDF
 #
-# VERSION               4.4.2
-FROM 	ubuntu:16.10
+FROM 	ubuntu:17.04
 MAINTAINER James R. Barlow <jim@purplerock.ca>
 
 RUN apt-get update && apt-get install -y --no-install-recommends \

LICENSE.rst

@@ -1,4 +1,7 @@
-Copyright (c) 2013-2016, The OCRmyPDF Authors
+Copyright (c) 2014-2017, James R. Barlow
+
+Copyright (c) 2013-2014, Julien Pfefferkorn
+Copyright (c) 2013-2017, The OCRmyPDF Authors
 
 Permission is hereby granted, free of charge, to any person obtaining a
 copy of this software and associated documentation files (the

MANIFEST.in

@@ -7,12 +7,12 @@ include dev_requirements.txt
 include .git_archival.txt
 
 # docker
-include Dockerfile
-include Dockerfile.polyglot
+include Dockerfile*
 include .dockerignore
 
 # tests
 include pytest.ini
+include .coveragerc
 recursive-include tests *.jpg
 recursive-include tests *.png
 recursive-include tests *.pdf
@@ -38,11 +38,14 @@ recursive-exclude docs/_build *
 recursive-include ocrmypdf/data *
 include *.py
 exclude tasks.py
+exclude debug-runner.py
+recursive-exclude .travis *
+exclude .travis*
+
 
 # code
 recursive-include ocrmypdf *.py
 exclude ocrmypdf/lib/_leptonica.py
-include OCRmyPDF.sh
 include docker-wrapper.sh
 
 

README.rst

@@ -122,6 +122,13 @@ If you detect an issue, please:
 -  If possible provide your input PDF file as well as the content of the
    temporary folder (using a file sharing service like Dropbox)
 
+Requirements
+------------
+
+Runs on CPython 3.6, and requires external program installations of Ghostscript, Tesseract OCR, QPDF, and Leptonica. ocrmypdf is pure Python, but uses CFFI to portably generate library bindings.
+
+Python 3.5 is fully supported, but support will end at the end of 2017, to allow f-strings in the code base.
+
 Press & Media
 -------------
 

docs/batch.rst

@@ -90,13 +90,83 @@ This user contributed script also provides an example of batch processing.
 API
 """
 
-OCRmyPDF is currently supported as a command line interface. Due to limitations in one of the libraries OCRmyPDF depends on, it is not yet usable as an API.
+OCRmyPDF is currently supported as a command line interface. This means that even if you are using OCRmyPDF in a Python script, you should run it in a subprocess rather importing the ocrmypdf package.
+
+The reason for this limitation is that the `ruffus <https://github.com/bunbun/ruffus/>`_ library that OCRmyPDF depends on is unfortunately not reentrant. OCRmyPDF works by defining each operation it does as a ruffus task that takes one or more files as input and generates one or more files as output. As such ruffus is fairly fundamental.
+
+(If you find individual functions implemented in OCRmyPDF useful (such as ``ocrmypdf.pdfinfo``), you can use these if you wish to.)
+
+
+Synology DiskStations
+"""""""""""""""""""""
+
+Synology DiskStations (Network Attached Storage devices) can run the Docker image of OCRmyPDF if the Synology `Docker package <https://www.synology.com/en-global/dsm/packages/Docker>`_ is installed. Attached is a script to address particular quirks of using OCRmyPDF on one of these devices.
+
+This is only possible for x86-based Synology products. Some Synology products use ARM or Power processors and do not support Docker. Further adjustments might be needed to deal with the Synology's relatively limited CPU and RAM.
+
+.. code-block:: python
+
+	#!/bin/env python3
+	# Contributed by github.com/Enantiomerie
+
+	# script needs 2 arguments
+	# 1. source dir with *.pdf - default is location of script
+	# 2. move dir where *.pdf and *_OCR.pdf are moved to
+
+	import logging
+	import os
+	import subprocess
+	import sys
+	import time
+	import shutil
+
+	script_dir = os.path.dirname(os.path.realpath(__file__))
+	timestamp = time.strftime("%Y-%m-%d-%H%M_")
+	log_file = script_dir + '/' + timestamp + 'ocrmypdf.log'
+	logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s', filename=log_file, filemode='w')
+
+	if len(sys.argv) > 1:
+	    start_dir = sys.argv[1]
+	else:
+	    start_dir = '.'
+
+	for dir_name, subdirs, file_list in os.walk(start_dir):
+	    logging.info('\n')
+	    logging.info(dir_name + '\n')
+	    os.chdir(dir_name)
+	    for filename in file_list:
+	        file_ext = os.path.splitext(filename)[1]
+	        if file_ext == '.pdf':
+	            full_path = dir_name + '/' + filename
+	            file_noext = os.path.splitext(filename)[0]
+	            timestamp_OCR = time.strftime("%Y-%m-%d-%H%M_OCR_")
+	            filename_OCR = timestamp_OCR + file_noext + '.pdf'
+	            docker_mount = dir_name + ':/home/docker'
+	# create string for pdf processing 
+	# diskstation needs a user:group docker:docker. find uid:gid of your diskstation docker:docker with id docker.
+	# use this uid:gid in -u flag
+	# rw rights for docker:docker at source dir are also necessary
+	# the script is processed as root user via chron 
+	            cmd = ['docker', 'run', '--rm', '-v', docker_mount, '-u=1030:65538', 'jbarlow83/ocrmypdf', , '--deskew' , filename, filename_OCR]
+	            logging.info(cmd)
+	            proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+	            result = proc.stdout.read()
+	            logging.info(result)
+	            full_path_OCR = dir_name + '/' + filename_OCR
+	            os.chmod(full_path_OCR, 0o666)
+	            os.chmod(full_path, 0o666)
+	            full_path_OCR_archive = sys.argv[2]
+	            full_path_archive = sys.argv[2] + '/no_ocr'
+	            shutil.move(full_path_OCR,full_path_OCR_archive)
+	            shutil.move(full_path, full_path_archive)
+	logging.info('Finished.\n')
+
 
 
 Huge batch jobs
 """""""""""""""
 
-If you have thousands of files to work with, contact the author.
+If you have thousands of files to work with, contact the author. Consulting work related to OCRmyPDF helps fund this open source project and all inquiries are appreciated.
 
 
 Hot (watched) folders

docs/installation.rst

@@ -105,6 +105,12 @@ In this worked example, the current working directory contains an input file cal
 
 Note that ``ocrmypdf`` has its own separate ``-v VERBOSITYLEVEL`` argument to control debug verbosity. All Docker arguments should before the ``ocrmypdf`` image name and all arguments to ``ocrmypdf`` should be listed after.
 
+In some environments the permissions associated with Docker can be complex to configure. The process that executes Docker may end up not having the permissions to write the specified file system. In that case one can stream the file into and out of the Docker process and avoid all permission hassles, using ``-`` as the input and output filename:
+
+.. code-block:: bash
+
+    docker run --rm -i   ocrmypdf <other arguments to ocrmypdf> - - <input.pdf >output.pdf
+
 For convenience, a shell alias can hide the docker command:
 
 .. code-block:: bash
@@ -119,6 +125,9 @@ Or in the wonderful `fish shell <https://fishshell.com/>`_:
     alias ocrmypdf 'docker run --rm -v (pwd):/home/docker ocrmypdf'
     funcsave ocrmypdf
 
+.. note::
+
+    The ocrmypdf Docker containers are designed to be used for a single OCR job. The ``docker run --rm`` argument tells Docker to delete temporary storage associated with container when it is done executing.
 
 Manual installation on macOS
 ----------------------------
@@ -165,7 +174,13 @@ Update the homebrew pip and install Pillow:
     pip3 install --upgrade pip
     pip3 install --upgrade pillow
 
-You can then install OCRmyPDF from PyPI:
+You can then install OCRmyPDF from PyPI, for the current user:
+
+.. code-block:: bash
+
+    pip3 install --user ocrmypdf
+
+or system-wide:
 
 .. code-block:: bash
 
@@ -194,8 +209,13 @@ No package is currently available for Ubuntu 16.04, but you can install the depe
         python3-pip \
         python3-cffi
 
-If you wish install OCRmyPDF to the system Python, then install as follows (note this installs new packages
-into your system Python, which could interfere with other programs):
+If you wish install OCRmyPDF for the current user:
+
+.. code-block:: bash
+
+    pip3 install --user ocrmypdf
+
+Alternately, system-wide. Note that this may modify the system Python environment:
 
 .. code-block:: bash
 
@@ -215,7 +235,7 @@ Installing on Ubuntu 14.04 LTS
 ------------------------------
 
 Installing on Ubuntu 14.04 LTS (trusty) is more difficult than some other options,
-because it is older.
+because it is older and does not provide ``pip``.
 
 Update apt-get:
 

docs/languages.rst

@@ -33,6 +33,34 @@ Docker users
 
 Users of the Docker image may use the alternative :ref:`"polyglot" container <docker-polyglot>` which includes all languages.
 
+Adding individual language packs to a Docker image
+""""""""""""""""""""""""""""""""""""""""""""""""""
+
+If you wish to add a single language pack, you could do the following:
+
+* Download the desired ``.trainedata`` file from the `tessdata <https://github.com/tesseract-ocr/tessdata>`_ repository. Let's use Hebrew in this example (``heb.traineddata``)
+
+* Copy the file to ``/home/user/downloads/heb.traineddata``.
+
+* Create a new container based on the ocrmypdf-tess4 image and jump into it with a terminal:
+
+.. code-block:: bash
+
+	host$ docker run  -v /home/user/downloads:/home/docker -it --entrypoint /bin/bash ocrmypdf-tess4
+
+* Put the file where Tesseract expects it:
+
+.. code-block:: bash
+
+	docker$ cp /home/docker/heb.traineddata /usr/share/tesseract-ocr/tessdata
+
+* Note the container id, and save it as a new image (in this example, ``ocrmypdf-tess4-heb``)
+
+.. code-block:: bash
+
+    host$ docker commit <container_id> ocrmypdf-tess4-heb
+
+
 Known limitations
 -----------------
 

docs/release_notes.rst

@@ -5,6 +5,46 @@ OCRmyPDF uses `semantic versioning <http://semver.org/>`_ for its command line i
 
 The OCRmyPDF package itself does not contain a public API, although it is fairly stable and breaking changes are usually timed with a major release. A future release will clearly define the stable public API.
 
+v5.4.4
+------
+
+-   Fix issue #181: fix final merge failure for PDFs with more pages than the system file handle limit (``ulimit -n``)
+-   Fix issue #200: an uncommon syntax for formatting decimal numbers in a PDF would cause qpdf to issue a warning, which ocrmypdf treated as an error. Now this the warning is relayed.
+-   Fix an issue where intermediate PDFs would be created at version 1.3 instead of the version of the original file. It's possible but unlikely this had side effects.
+-   A warning is now issued when older versions of qpdf are used since issues like #200 cause qpdf to infinite-loop
+-   Address issue #140: if Tesseract outputs invalid UTF-8, escape it and print its message instead of aborting with a Unicode error 
+-   Adding previously unlisted setup requirement, pytest-runner
+-   Update documentation: fix an error in the example script for Synology with Docker images, improved security guidance, advised ``pip install --user``
+
+
+v5.4.3
+------
+
+-   If a subprocess fails to report its version when queried, exit cleanly with an error instead of throwing an exception
+-   Added test to confirm that the system locale is Unicode-aware and fail early if it's not
+-   Clarified some copyright information
+-   Updated pinned requirements.txt so the homebrew formula captures more recent versions
+
+
+v5.4.2
+------
+
+-   Fixed a regression from v5.4.1 that caused sidecar files to be created as empty files
+
+
+v5.4.1
+------
+
+-   Add workaround for Tesseract v4.00alpha crash when trying to obtain orientation and the latest language packs are installed
+
+
+v5.4
+----
+
+-   Change wording of a deprecation warning to improve clarity
+-   Added option to generate PDF/A-1b output if desired (``--output-type pdfa-1``); default remains PDF/A-2b generation
+-   Update documentation
+
 
 v5.3.3
 ------

docs/security.rst

@@ -10,7 +10,7 @@ The disclaimer applies: this software has no warranties of any kind.
 PDFs may contain malware
 ------------------------
 
-PDF is a rich, complex file format. The official PDF 1.7 specification, ISO 32000:2008, is hundreds of packages long and references several annexes each of which are similar in length. PDFs can contain video, audio, JavaScript and other programming, and forms. In some cases, they can open internet connections to pre-selected URLs. All of these possible attack vectors.
+PDF is a rich, complex file format. The official PDF 1.7 specification, ISO 32000:2008, is hundreds of pages long and references several annexes each of which are similar in length. PDFs can contain video, audio, XML, JavaScript and other programming, and forms. In some cases, they can open internet connections to pre-selected URLs. All of these possible attack vectors.
 
 In short, PDFs `may contain viruses <https://security.stackexchange.com/questions/64052/can-a-pdf-file-contain-a-virus>`_.
 
@@ -21,18 +21,45 @@ How OCRmyPDF processes PDFs
 
 OCRmyPDF must open and interpret your PDF in order to insert an OCR layer. First, it runs all PDFs through `qpdf <https://github.com/qpdf/qpdf>`_, a program that repairs PDFs with syntax errors. This is done because, in the author's experience, a significant number of PDFs in the wild especially those created by scanners are not well-formed files. qpdf makes it more likely that OCRmyPDF will succeed, but offers no security guarantees. qpdf is also used to split the PDF into single page PDFs.
 
-After qpdf, OCRmyPDF examines each page using `PyPDF2 <https://github.com/mstamy2/PyPDF2>`_. This library also has no warranties or guarantees.
+After qpdf, OCRmyPDF examines each page using `PyPDF2 <https://github.com/mstamy2/PyPDF2>`_. This library also has no warranties or guarantees. OCRmyPDF works with qpdf 5.0 and up, but version 7.0 is recommended because of known security vulnerabilities in early versions.
 
-Finally, OCRmyPDF rasterizes each page of the PDF using `Ghostscript <http://ghostscript.com/>`_ in ``-dSAFER`` mode. 
+Finally, OCRmyPDF rasterizes each page of the PDF using `Ghostscript <http://ghostscript.com/>`_ in ``-dSAFER`` mode.
 
 Depending on the options specified, OCRmyPDF may graft the OCR layer into the existing PDF or it may essentially reconstruct ("re-fry") a visually identical PDF that may be quite different at the binary level. That said, OCRmyPDF is not a tool designed for sanitizing PDFs.
 
-Using OCRmyPDF online
----------------------
+Using OCRmyPDF online or as a service
+-------------------------------------
 
-OCRmyPDF is not designed to be deployed "as a service", in a setting where a user/attacker could upload a file for OCR processing online. It is not designed to be secure in this case.
+OCRmyPDF should not be deployed as a public-facing service, like a website where a potential attacker could upload a PDF of their choice for OCR. OCRmyPDF is not designed to be secure against PDF malware. Another concern is PDFs specifically designed to be a denial of service attack: PDFs can contain recursive data structures that sometimes send parsers into infinite loops, and issue complex graphics drawing commands.
+
+OCRmyPDF should be relatively safe to use in a trusted intranet, with some considerations:
+
+Limiting CPU usage
+^^^^^^^^^^^^^^^^^^
+
+OCRmyPDF will attempt to use all available CPUs and storage, so executing ``nice ocrmypdf`` or limiting the number of jobs with the ``-j`` argument may ensure the server remains available. Another option would be run OCRmyPDF jobs inside a Docker container or virtual machine, which can impose its own limits on CPU usage.
+
+Temporary storage requirements
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+OCRmyPDF will use a large amount of temporary storage for its work, proportional to the total number of pixels needed to rasterize the PDF. The raster image of a 8.5×11" color page at 300 DPI takes 25 MB uncompressed; OCRmyPDF saves its intermediates as PNG, but that still means it requires about 9 MB per intermediate based on average compression ratios. Multiple intermediates per page are also required, depending on the command line given. A rule of thumb would be to allow 100 MB of temporary storage per page in a file – meaning that a small cloud servers or small VM partitions should be provisioned with plenty of extra space, if say, a 500 page file might be sent.
+
+To check temporary storage usage on actual files, run ``ocrmypdf -k ...`` which will preserve and print the path to temporary storage when the job is done.
+
+To change where temporary files are stored, change the ``TMPDIR`` environment variable for ocrmypdf's environment. (Python's ``tempfile.gettempdir()`` returns the root directory in which temporary files will be stored.)
+
+Timeouts
+^^^^^^^^
+
+To prevent excessively long OCR jobs consider setting ``--tesseract-timeout`` and/or ``--skip-big`` arguments. ``--skip-big`` is particularly helpful if your PDFs include documents such as reports on standard page sizes with large images attached - often large images are not worth OCR'ing anyway.
+
+Commercial alternatives
+^^^^^^^^^^^^^^^^^^^^^^^
+
+The author also provides professional services that include OCR and building databases around PDFs, and is happy to provide consultation.
+
+Abbyy Cloud OCR is a viable commercial alternative with a web services API. 
 
-Abbyy Cloud OCR is a viable commercial alternative with a web services API. The author also provides professional services that include OCR and building databases around PDFs, and is happy to provide consultation.
 
 Password protection, digital signatures and certification
 ---------------------------------------------------------

ocrmypdf/__init__.py

@@ -4,6 +4,7 @@ import pkg_resources
 
 PROGRAM_NAME = 'ocrmypdf'
 
-VERSION = pkg_resources.get_distribution('ocrmypdf').version
-
+# Official PEP 396
+__version__ = pkg_resources.get_distribution('ocrmypdf').version
 
+VERSION = __version__

ocrmypdf/__main__.py

@@ -1,29 +1,25 @@
 #!/usr/bin/env python3
 # © 2015-17 James R. Barlow: github.com/jbarlow83
 
-from contextlib import suppress
 from tempfile import mkdtemp
 from collections.abc import Sequence
 import sys
 import os
 import re
-import shutil
 import warnings
 import multiprocessing
 import atexit
 import textwrap
-import img2pdf
 import logging
 import argparse
 
 import PyPDF2 as pypdf
-from PIL import Image
 
 import ruffus.ruffus_exceptions as ruffus_exceptions
 import ruffus.cmdline as cmdline
 import ruffus.proxy_logger as proxy_logger
 
-from .pipeline import JobContext, JobContextManager, re_symlink, \
+from .pipeline import JobContext, JobContextManager, \
     cleanup_working_files, build_pipeline
 from .pdfa import file_claims_pdfa
 from .helpers import is_iterable_notstr, re_symlink, is_file_writable
@@ -32,6 +28,7 @@ from . import PROGRAM_NAME, VERSION
 
 from .exceptions import *
 from . import exceptions as ocrmypdf_exceptions
+from ._unicodefun import verify_python3_env
 
 warnings.simplefilter('ignore', pypdf.utils.PdfReadWarning)
 
@@ -53,6 +50,11 @@ def complain(message):
 if 'IDE_PROJECT_ROOTS' in os.environ:
     os.environ['PATH'] = '/usr/local/bin:' + os.environ['PATH']
 
+# -------- 
+# Critical environment tests
+
+verify_python3_env()
+
 if tesseract.version() < MINIMUM_TESS_VERSION:
     complain(
         "Please install tesseract {0} or newer "
@@ -134,12 +136,15 @@ parser.add_argument(
     '--image-dpi', metavar='DPI', type=int,
     help="For input image instead of PDF, use this DPI instead of file's.")
 parser.add_argument(
-    '--output-type', choices=['pdfa', 'pdf'], default='pdfa',
+    '--output-type', choices=['pdfa', 'pdf', 'pdfa-1', 'pdfa-2'], 
+    default='pdfa',
     help="Choose output type. 'pdfa' creates a PDF/A-2b compliant file for "
          "long term archiving (default, recommended) but may not suitable "
          "for users who want their file altered as little as possible. 'pdfa' "
          "also has problems with full Unicode text. 'pdf' attempts to "
-         "preserve file contents as much as possible.")
+         "preserve file contents as much as possible. 'pdf-a1' creates a "
+         "PDF/A1-b file. 'pdf-a2' is equivalent to 'pdfa'."
+         )
 
 # Use null string '\0' as sentinel to indicate the user supplied no argument,
 # since that is the only invalid character for filepaths on all platforms
@@ -343,13 +348,18 @@ def check_options_output(options, log):
         options.pdf_renderer = 'sandwich'
 
     if options.pdf_renderer == 'tesseract':
-        log.warning("The 'tesseract' PDF renderer is deprecated.")
         if tesseract.version() < '3.05' and options.output_type == 'pdfa':
             log.warning(
                 "For best results use --pdf-renderer=tesseract "
                 "--output-type=pdf to disable PDF/A generation via "
                 "Ghostscript, which is known to corrupt the OCR text of "
                 "some PDFs produced your version of Tesseract.")
+        elif tesseract.has_textonly_pdf():
+            log.warning(
+                "The argument --pdf-renderer=tesseract provides support for "
+                "versions of tesseract older than your version. For best "
+                "results omit this argument and let OCRmyPDF choose the "
+                "best available renderer.")
 
     if options.debug_rendering and options.pdf_renderer != 'hocr':
         log.info(
@@ -422,7 +432,7 @@ def check_options_ocr_behavior(options, log):
 def check_options_advanced(options, log):
     if tesseract.v4():
         log.info(
-            "Tesseract v4.x.alpha found. OCRmyPDF support is experimental.")
+            "Tesseract v4.x.alpha found.")
     if options.tesseract_oem and not tesseract.v4():
         log.warning(
             "--tesseract-oem requires Tesseract 4.x -- argument ignored")
@@ -440,9 +450,9 @@ def check_options_advanced(options, log):
 
 def check_options_metadata(options, log):
     import unicodedata
-    metadata = [options.title, options.author, options.keywords,
-                options.subject]
-    for s in (m for m in metadata if m):
+    docinfo = [options.title, options.author, options.keywords,
+               options.subject]
+    for s in (m for m in docinfo if m):
         for c in s:
             if unicodedata.category(c) == 'Co' or ord(c) >= 0x10000:
                 raise ValueError(
@@ -524,31 +534,31 @@ def cleanup_ruffus_error_message(msg):
 def do_ruffus_exception(ruffus_five_tuple, options, log):
     """Replace the elaborate ruffus stack trace with a user friendly
     description of the error message that occurred."""
+    exit_code = None
 
     task_name, job_name, exc_name, exc_value, exc_stack = ruffus_five_tuple
     if exc_name == 'builtins.SystemExit':
         match = re.search(r"\.(.+?)\)", exc_value)
         exit_code_name = match.groups()[0]
-        exit_code = getattr(ExitCode, exit_code_name, 'other_error')
-        return exit_code
+        exit_code = getattr(ExitCode, exit_code_name, 'other_error')        
     elif exc_name == 'ruffus.ruffus_exceptions.MissingInputFileError':
         log.error(cleanup_ruffus_error_message(exc_value))
-        return ExitCode.input_file
+        exit_code = ExitCode.input_file
     elif exc_name == 'builtins.TypeError':
         # Even though repair_pdf will fail, ruffus will still try
         # to call split_pages with no input files, likely due to a bug
         if task_name == 'split_pages':
             log.error("Input file '{0}' is not a valid PDF".format(
                 options.input_file))
-            return ExitCode.input_file
+            exit_code = ExitCode.input_file
     elif exc_name == 'builtins.KeyboardInterrupt':
         log.error("Interrupted by user")
-        return ExitCode.ctrl_c
+        exit_code = ExitCode.ctrl_c
     elif exc_name == 'subprocess.CalledProcessError':
         # It's up to the subprocess handler to report something useful
         msg = "Error occurred while running this command:"
         log.error(msg + '\n' + exc_value)
-        return ExitCode.child_process_error
+        exit_code = ExitCode.child_process_error
     elif exc_name == 'ocrmypdf.exceptions.PdfMergeFailedError':
         log.error(textwrap.dedent("""\
             Failed to merge PDF image layer with OCR layer
@@ -559,11 +569,11 @@ def do_ruffus_exception(ruffus_five_tuple, options, log):
             Try using
                 ocrmypdf --pdf-renderer tesseract  [..other args..]
             """))
-        return ExitCode.input_file
+        exit_code = ExitCode.input_file
     elif exc_name.startswith('ocrmypdf.exceptions.'):
         base_exc_name = exc_name.replace('ocrmypdf.exceptions.', '')
         exc_class = getattr(ocrmypdf_exceptions, base_exc_name)
-        return exc_class.exit_code
+        exit_code = exc_class.exit_code
     elif exc_name == 'PyPDF2.utils.PdfReadError' and \
             'not been decrypted' in exc_value:
         log.error(textwrap.dedent("""\
@@ -576,7 +586,10 @@ def do_ruffus_exception(ruffus_five_tuple, options, log):
             (Only algorithms "R = 1" and "R = 2" are supported.)
 
             """))
-        return ExitCode.encrypted_pdf
+        exit_code = ExitCode.encrypted_pdf
+
+    if exit_code is not None:
+        return exit_code
 
     if not options.verbose:
         log.error(exc_stack)
@@ -642,6 +655,20 @@ def check_closed_streams(options):
     return True
 
 
+def log_page_orientations(pdfinfo, _log):
+    direction = {0: 'n', 90: 'e',
+                180: 's', 270: 'w'}
+    orientations = []
+    for n, page in enumerate(pdfinfo):
+        angle = pdfinfo[n].rotation or 0
+        if angle != 0:
+            orientations.append('{0}{1}'.format(
+                n + 1,
+                direction.get(angle, '')))
+    if orientations:
+        _log.info('Page orientations detected: ' + ' '.join(orientations))
+
+
 def run_pipeline():
     options = parser.parse_args()
     options.verbose_abbreviated_path = 1
@@ -655,9 +682,19 @@ def run_pipeline():
         logging_factory, __name__, logger_args)
     _log.debug('ocrmypdf ' + VERSION)
     _log.debug('tesseract ' + tesseract.version())
+    _log.debug('qpdf ' + qpdf.version())
 
     check_options(options, _log)
 
+    # Complain about qpdf version < 7.0.0
+    # Suppress the warning if in the test suite, since there are no PPAs
+    # for qpdf 7.0.0 for Ubuntu trusty (i.e. Travis)
+    if qpdf.version() < '7.0.0' and not os.environ.get('PYTEST_CURRENT_TEST'):
+        complain(
+            "You are using qpdf version {0} which has known issues including "
+            "security vulnerabilities with certain malformed PDFs. Consider "
+            "upgrading to version 7.0.0 or newer.".format(qpdf.version()))
+
     # Any changes to options will not take effect for options that are already
     # bound to function parameters in the pipeline. (For example
     # options.input_file, options.pdf_renderer are already bound.)
@@ -731,8 +768,7 @@ def run_pipeline():
             _log.error("Unexpected ruffus exception: " + str(e))
             _log.error(repr(e))
             return ExitCode.other_error
-        else:
-            return exitcode
+        return exitcode
     except ExitCodeException as e:
         return e.exit_code
     except Exception as e:
@@ -742,7 +778,7 @@ def run_pipeline():
     if options.flowchart:
         _log.info("Flowchart saved to {}".format(options.flowchart))
     elif options.output_file != '-':
-        if options.output_type == 'pdfa':
+        if options.output_type.startswith('pdfa'):
             pdfa_info = file_claims_pdfa(options.output_file)
             if pdfa_info['pass']:
                 msg = 'Output file is a {} (as expected)'
@@ -750,7 +786,6 @@ def run_pipeline():
             else:
                 msg = 'Output file is okay but is not PDF/A (seems to be {})'
                 _log.warning(msg.format(pdfa_info['conformance']))
-
                 return ExitCode.invalid_output_pdf
         if not qpdf.check(options.output_file, _log):
             _log.warning('Output file: The generated PDF is INVALID')
@@ -762,17 +797,8 @@ def run_pipeline():
     if options.verbose:
         from pprint import pformat
         _log.debug(pformat(pdfinfo))
-    direction = {0: 'n', 90: 'e',
-                 180: 's', 270: 'w'}
-    orientations = []
-    for n, page in enumerate(pdfinfo):
-        angle = pdfinfo[n].rotation or 0
-        if angle != 0:
-            orientations.append('{0}{1}'.format(
-                n + 1,
-                direction.get(angle, '')))
-    if orientations:
-        _log.info('Page orientations detected: ' + ' '.join(orientations))
+
+    log_page_orientations(pdfinfo, _log)
 
     return ExitCode.ok
 

ocrmypdf/_unicodefun.py

@@ -0,0 +1,108 @@
+# Copyright (c) 2014, Armin Ronacher
+#
+# Copyright (c) 2017, James R Barlow
+#
+# Some rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#
+#     * Redistributions in binary form must reproduce the above
+#       copyright notice, this list of conditions and the following
+#       disclaimer in the documentation and/or other materials provided
+#       with the distribution.
+#
+#     * The names of the contributors may not be used to endorse or
+#       promote products derived from this software without specific
+#       prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import os
+import sys
+import codecs
+
+
+def verify_python3_env():
+    """Ensures that the environment is good for unicode on Python 3."""
+    try:
+        import locale
+        fs_enc = codecs.lookup(locale.getpreferredencoding()).name
+    except Exception:
+        fs_enc = 'ascii'
+    if fs_enc != 'ascii':
+        return
+
+    extra = ''
+    if os.name == 'posix':
+        import subprocess
+        rv = subprocess.Popen(['locale', '-a'], stdout=subprocess.PIPE,
+                              stderr=subprocess.PIPE).communicate()[0]
+        good_locales = set()
+        has_c_utf8 = False
+
+        # Make sure we're operating on text here.
+        if isinstance(rv, bytes):
+            rv = rv.decode('ascii', 'replace')
+
+        for line in rv.splitlines():
+            locale = line.strip()
+            if locale.lower().endswith(('.utf-8', '.utf8')):
+                good_locales.add(locale)
+                if locale.lower() in ('c.utf8', 'c.utf-8'):
+                    has_c_utf8 = True
+
+        extra += '\n\n'
+        if not good_locales:
+            extra += (
+                'Additional information: on this system no suitable UTF-8\n'
+                'locales were discovered.  This most likely requires resolving\n'
+                'by reconfiguring the locale system.'
+            )
+        elif has_c_utf8:
+            extra += (
+                'This system supports the C.UTF-8 locale which is recommended.\n'
+                'You might be able to resolve your issue by exporting the\n'
+                'following environment variables:\n\n'
+                '    export LC_ALL=C.UTF-8\n'
+                '    export LANG=C.UTF-8'
+            )
+        else:
+            extra += (
+                'This system lists a couple of UTF-8 supporting locales that\n'
+                'you can pick from.  The following suitable locales were\n'
+                'discovered: %s'
+            ) % ', '.join(sorted(good_locales))
+
+        bad_locale = None
+        for locale in os.environ.get('LC_ALL'), os.environ.get('LANG'):
+            if locale and locale.lower().endswith(('.utf-8', '.utf8')):
+                bad_locale = locale
+            if locale is not None:
+                break
+        if bad_locale is not None:
+            extra += (
+                '\n\ocrmypdf discovered that you exported a UTF-8 locale\n'
+                'but the locale system could not pick up from it because\n'
+                'it does not exist.  The exported locale is "%s" but it\n'
+                'is not supported'
+            ) % bad_locale
+
+    raise RuntimeError('ocrmypdf will abort further execution because Python 3 '
+                       'was configured to use ASCII as encoding for the '
+                       'environment.' + extra)

ocrmypdf/exceptions.py

@@ -20,7 +20,7 @@ class ExitCode(IntEnum):
 
 
 class ExitCodeException(Exception):
-    pass
+    exit_code = ExitCode.other_error
 
 
 class PdfMergeFailedError(ExitCodeException):

ocrmypdf/exec/__init__.py

@@ -4,8 +4,40 @@
 """Wrappers to manage subprocess calls"""
 
 import os
+import re
+import sys
+from subprocess import run, STDOUT, PIPE, CalledProcessError
+from ..exceptions import MissingDependencyError
 
 
 def get_program(name):
+    "Check environment variables for overrides to this program"
     envvar = 'OCRMYPDF_' + name.upper()
     return os.environ.get(envvar, name)
+
+
+def get_version(program, *, 
+        version_arg='--version', regex=r'(\d+(\.\d+)*)'):
+    "Get the version of the specified program, "
+    args_prog = [
+        get_program(program),
+        version_arg
+    ]
+    try:
+        proc = run(
+            args_prog, close_fds=True, universal_newlines=True,
+            stdout=PIPE, stderr=STDOUT, check=True)
+        output = proc.stdout
+    except CalledProcessError as e:
+        raise MissingDependencyError(
+            "Could not find program '{}' on the PATH".format(program)) from e
+
+    try:
+        version = re.match(regex, output.strip()).group(1)
+    except AttributeError as e:
+        raise MissingDependencyError(
+            ("The program '{}' did not report its version. "
+            "Message was:\n{}").format(program, output)
+        )
+
+    return version

ocrmypdf/exec/ghostscript.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# © 2015 James R. Barlow: github.com/jbarlow83
+# © 2017 James R. Barlow: github.com/jbarlow83
 
 from tempfile import NamedTemporaryFile
 from subprocess import run, PIPE, STDOUT, CalledProcessError
@@ -7,28 +7,15 @@ from shutil import copy
 from functools import lru_cache
 import re
 import sys
-from . import get_program
-from ..exceptions import SubprocessOutputError
 from PIL import Image
+from . import get_program, get_version
+from ..exceptions import SubprocessOutputError, MissingDependencyError
 from ..helpers import fspath
 
 
 @lru_cache(maxsize=1)
 def version():
-    args_gs = [
-        get_program('gs'),
-        '--version'
-    ]
-    try:
-        version = check_output(
-                args_gs, close_fds=True, universal_newlines=True,
-                stderr=STDOUT)
-    except CalledProcessError as e:
-        print("Could not find Ghostscript executable on system PATH.",
-              file=sys.stderr)
-        raise MissingDependencyError from e
-
-    return version.strip()
+    return get_version('gs')
 
 
 def _gs_error_reported(stream):
@@ -103,7 +90,7 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
 
 
 def generate_pdfa(pdf_pages, output_file, compression, log,
-                  threads=1, pdf_version='1.5'):
+                  threads=1, pdf_version='1.5', pdfa_part='2'):
     compression_args = []
     if compression == 'jpeg':
         compression_args = [
@@ -139,7 +126,7 @@ def generate_pdfa(pdf_pages, output_file, compression, log,
             "-sProcessColorModel=DeviceRGB"
         ] + compression_args + [
             "-dJPEGQ=95",
-            "-dPDFA=2",
+            "-dPDFA=" + pdfa_part,
             "-dPDFACompatibilityPolicy=1",
             "-sOutputFile=" + gs_pdf.name,
         ]
@@ -166,4 +153,4 @@ def generate_pdfa(pdf_pages, output_file, compression, log,
             copy(gs_pdf.name, output_file)
         else:
             log.error('Ghostscript PDF/A rendering failed')
-            raise SubprocessOutputError()
+            raise SubprocessOutputError()

ocrmypdf/exec/qpdf.py

@@ -1,33 +1,22 @@
 #!/usr/bin/env python3
-# © 2015 James R. Barlow: github.com/jbarlow83
+# © 2017 James R. Barlow: github.com/jbarlow83
 
-from subprocess import CalledProcessError, STDOUT, PIPE, run, check_output
+from subprocess import CalledProcessError, STDOUT, PIPE, run
 from functools import lru_cache
 import sys
 import os
 import re
+import resource
 
 from ..exceptions import InputFileError, SubprocessOutputError, \
     MissingDependencyError, EncryptedPdfError
-from . import get_program
+from . import get_program, get_version
+from ..helpers import re_symlink
 
 
 @lru_cache(maxsize=1)
 def version():
-    args_qpdf = [
-        get_program('qpdf'),
-        '--version'
-    ]
-    try:
-        p = run(args_qpdf, universal_newlines=True, stderr=STDOUT,
-                stdout=PIPE)
-    except CalledProcessError as e:
-        print("Could not find qpdf executable on system PATH.",
-              file=sys.stderr)
-        raise MissingDependencyError() from e
-
-    qpdf_version = re.match(r'qpdf version (.+)', p.stdout).group(1)
-    return qpdf_version
+    return get_version('qpdf', regex=r'qpdf version (.+)')    
 
 
 def check(input_file, log=None):
@@ -41,7 +30,8 @@ def check(input_file, log=None):
         import logging as log
 
     try:
-        check_output(args_qpdf, stderr=STDOUT, universal_newlines=True)
+        run(args_qpdf, stderr=STDOUT, stdout=PIPE, universal_newlines=True, 
+            check=True)
     except CalledProcessError as e:
         if e.returncode == 2:
             log.error("{0}: not a valid PDF, and could not repair it.".format(
@@ -70,11 +60,11 @@ def repair(input_file, output_file, log):
         get_program('qpdf'), input_file, output_file
     ]
     try:
-        check_output(args_qpdf, stderr=STDOUT, universal_newlines=True)
+        run(args_qpdf, stderr=STDOUT, stdout=PIPE, universal_newlines=True, 
+            check=True)
     except CalledProcessError as e:
         if e.returncode == 3 and e.output.find("operation succeeded"):
             log.debug('qpdf found and fixed errors: ' + e.output)
-            log.debug(e.output)
             return
 
         if _probably_encrypted(e):
@@ -95,9 +85,9 @@ def repair(input_file, output_file, log):
 
 def get_npages(input_file, log):
     try:
-        pages = check_output(
+        pages = run(
             [get_program('qpdf'), '--show-npages', input_file],
-             universal_newlines=True, close_fds=True)
+            universal_newlines=True, check=True, stdout=PIPE, stderr=STDOUT)
     except CalledProcessError as e:
         if e.returncode == 2 and e.output.find('No such file'):
             log.error(e.output)
@@ -120,18 +110,97 @@ def split_pages(input_file, work_folder, npages):
         run(args_qpdf, check=True)
 
 
-def merge(input_files, output_file, min_version=None):
+def _merge_inner(input_files, output_file, min_version=None, log=None):
     """Merge the list of input files (all filenames) into the output file.
 
     The input files may contain one or more pages.
     """
+
+    # Single page 'merges' should still be attempted to that the same error
+    # checking is applied to single page case
+
     version_arg = ['--min-version={}'.format(min_version)] \
                   if min_version else []
 
+    if log is None:
+        import logging as log
+
     args_qpdf = [
         get_program('qpdf')
     ] + version_arg + [
         input_files[0], '--pages'
     ] + input_files + ['--', output_file]
-    run(args_qpdf, check=True)
+
+    try:
+        run(args_qpdf, check=True, stderr=PIPE, universal_newlines=True)
+    except CalledProcessError as e:
+        if e.returncode == 3 and \
+                e.stderr.find("unknown token while reading object") and \
+                e.stderr.find("operation succeeded"):
+            # Only whitelist the 'unknown token' problem (decimal/string issue)
+            # qpdf issue #165
+            log.warning('qpdf found and fixed errors: ' + e.stderr)
+            return
+        raise e from e
+
+
+def merge(input_files, output_file, min_version=None, log=None, max_files=None):
+    """Merge the list of input files (all filenames) into the output file.
+
+    The input files may contain one or more pages.
+
+    """
+    # qpdf requires that every file that contributes to the output has a file 
+    # descriptor that remains open. That means, given our approach of one 
+    # intermediate PDF per, we can practically hit the number of file 
+    # descriptors. 
+
+    if max_files is None or max_files < 2:
+        # Find out how many open file descriptors we can get away with
+        ulimits = resource.getrlimit(resource.RLIMIT_NOFILE)
+        max_open_files = ulimits[0]
+        max_files = max_open_files // 2  # Conservative guess
+
+    # We'll write things alongside the output file
+    output_dir = os.path.dirname(output_file)
+
+    import random
+    import string    
+
+    def randstr():
+        return ''.join(random.sample(string.ascii_lowercase, 6))
+
+    # How many files to grab at once, merging all their contents
+    step_size = max_files
+
+    workqueue = input_files.copy()
+    counter = 1
+    next_workqueue = []
+    while len(workqueue) > 1 or len(next_workqueue) > 0:
+        # Take n files out of the queue
+        n = min(step_size, len(workqueue))
+        job = workqueue[0:n]
+        del workqueue[0:n]
+        print('merging ' + repr(job))
+
+        # Merge them into 1 file, which will contain n^depth pages
+        merge_file = os.path.join(
+            output_dir, "merge-{:06d}-{}.pdf".format(counter, randstr()))
+        counter += 1
+        _merge_inner(job, merge_file, min_version=min_version, log=log)
+
+        # On the next 
+        next_workqueue.append(merge_file)
+        print('next_workqueue ' + repr(next_workqueue))
+
+        # If we're out of things to do in this queue, move on to the next
+        # queue. On the counter-th pass of the workqueue we can chew through
+        # (step_size)**N pages, so on most systems the second pass finishes
+        # the job.
+        if len(workqueue) == 0:
+            workqueue = next_workqueue
+            next_workqueue = []
+
+    re_symlink(workqueue.pop(), output_file)
+
 

ocrmypdf/exec/tesseract.py

@@ -1,21 +1,20 @@
 #!/usr/bin/env python3
-# © 2015 James R. Barlow: github.com/jbarlow83
+# © 2017 James R. Barlow: github.com/jbarlow83
 
 import sys
 import os
 import re
 import shutil
 from functools import lru_cache
-from ..exceptions import MissingDependencyError, TesseractConfigError
-from ..helpers import page_number
-from . import get_program
 from collections import namedtuple
 from textwrap import dedent
 import PyPDF2 as pypdf
+from subprocess import PIPE, CalledProcessError, \
+    TimeoutExpired, check_output, STDOUT
 
-from subprocess import Popen, PIPE, CalledProcessError, \
-    TimeoutExpired, check_output, STDOUT, DEVNULL
-
+from ..exceptions import MissingDependencyError, TesseractConfigError
+from ..helpers import page_number
+from . import get_program, get_version
 
 OrientationConfidence = namedtuple(
     'OrientationConfidence',
@@ -41,26 +40,12 @@ HOCR_TEMPLATE = """<?xml version="1.0" encoding="UTF-8"?>
 
 @lru_cache(maxsize=1)
 def version():
-    args_tess = [
-        get_program('tesseract'),
-        '--version'
-    ]
-    try:
-        versions = check_output(
-                args_tess, close_fds=True, universal_newlines=True,
-                stderr=STDOUT)
-    except CalledProcessError as e:
-        print("Could not find Tesseract executable on system PATH.",
-              file=sys.stderr)
-        raise MissingDependencyError from e
-
-    tesseract_version = re.match(r'tesseract\s(.+)', versions).group(1)
-    return tesseract_version
+    return get_version('tesseract', regex=r'tesseract\s(.+)')
 
 
 def v4():
     "Is this Tesseract v4.0?"
-    return (version() >= '4')
+    return version() >= '4'
 
 
 @lru_cache(maxsize=1)
@@ -74,10 +59,10 @@ def has_textonly_pdf():
         get_program('tesseract'),
         '--print-parameters'
     ]
+    params = ''
     try:
         params = check_output(
-                args_tess, close_fds=True, universal_newlines=True,
-                stderr=STDOUT)
+                args_tess, universal_newlines=True, stderr=STDOUT)
     except CalledProcessError as e:
         print("Could not --print-parameters from tesseract",
               file=sys.stderr)
@@ -100,8 +85,7 @@ def languages():
     ]
     try:
         langs = check_output(
-                args_tess, close_fds=True, universal_newlines=True,
-                stderr=STDOUT)
+                args_tess, universal_newlines=True, stderr=STDOUT)
     except CalledProcessError as e:
         msg = dedent("""Tesseract failed to report available languages.
         Output from Tesseract:
@@ -113,12 +97,12 @@ def languages():
     return set(lang.strip() for lang in langs.splitlines()[1:])
 
 
-def tess_base_args(languages, engine_mode):
+def tess_base_args(langs, engine_mode):
     args = [
         get_program('tesseract'),
     ]
-    if languages:
-        args.extend(['-l', '+'.join(languages)])
+    if langs:
+        args.extend(['-l', '+'.join(langs)])
     if engine_mode is not None and v4():
         args.extend(['--oem', str(engine_mode)])
     return args
@@ -126,7 +110,7 @@ def tess_base_args(languages, engine_mode):
 
 def get_orientation(input_file, language: list, engine_mode, timeout: float,
                     log):
-    args_tesseract = tess_base_args(language, engine_mode) + [
+    args_tesseract = tess_base_args(['osd'], engine_mode) + [
         psm(), '0',
         input_file,
         'stdout'
@@ -134,19 +118,18 @@ def get_orientation(input_file, language: list, engine_mode, timeout: float,
 
     try:
         stdout = check_output(
-            args_tesseract, close_fds=True, stderr=STDOUT,
-            universal_newlines=True, timeout=timeout)
+            args_tesseract, stderr=STDOUT, timeout=timeout)
     except TimeoutExpired:
         return OrientationConfidence(angle=0, confidence=0.0)
     except CalledProcessError as e:
         tesseract_log_output(log, e.output, input_file)
-        if ('Too few characters. Skipping this page' in e.output or
-                'Image too large' in e.output):
+        if (b'Too few characters. Skipping this page' in e.output or
+                b'Image too large' in e.output):
             return OrientationConfidence(0, 0)
         raise e from e
     else:
         osd = {}
-        for line in stdout.splitlines():
+        for line in stdout.decode().splitlines():
             line = line.strip()
             parts = line.split(':', maxsplit=2)
             if len(parts) == 2:
@@ -171,8 +154,17 @@ def get_orientation(input_file, language: list, engine_mode, timeout: float,
 
 
 def tesseract_log_output(log, stdout, input_file):
-    lines = stdout.splitlines()
     prefix = "{0:4d}: [tesseract] ".format(page_number(input_file))
+
+    try:
+        text = stdout.decode()
+    except UnicodeDecodeError as e:
+        log.error(prefix + "command line output was not utf-8. " +
+            "This usually means Tesseract's language packs do not match "
+            "the installed version of Tesseract.")
+        text = stdout.decode('utf-8', 'backslashreplace')
+
+    lines = text.splitlines()
     for line in lines:
         if line.startswith("Tesseract Open Source"):
             continue
@@ -245,8 +237,8 @@ def generate_hocr(input_file, output_files, language: list, engine_mode,
     try:
         log.debug(args_tesseract)
         stdout = check_output(
-            args_tesseract, close_fds=True, stderr=STDOUT,
-            universal_newlines=True, timeout=timeout)
+            args_tesseract, stderr=STDOUT,
+            timeout=timeout)
     except TimeoutExpired:
         # Generate a HOCR file with no recognized text if tesseract times out
         # Temporary workaround to hocrTransform not being able to function if
@@ -255,9 +247,9 @@ def generate_hocr(input_file, output_files, language: list, engine_mode,
         _generate_null_hocr(output_hocr, output_sidecar, input_file)
     except CalledProcessError as e:
         tesseract_log_output(log, e.output, input_file)
-        if 'read_params_file: parameter not found' in e.output:
+        if b'read_params_file: parameter not found' in e.output:
             raise TesseractConfigError() from e
-        if 'Image too large' in e.output:
+        if b'Image too large' in e.output:
             _generate_null_hocr(output_hocr, output_sidecar, input_file)
             return
 
@@ -338,8 +330,8 @@ def generate_pdf(*, input_image, skip_pdf, output_pdf, output_text,
     try:
         log.debug(args_tesseract)
         stdout = check_output(
-            args_tesseract, close_fds=True, stderr=STDOUT,
-            universal_newlines=True, timeout=timeout)
+            args_tesseract, stderr=STDOUT,
+            timeout=timeout)
         if os.path.exists(prefix + '.txt'):
             shutil.move(prefix + '.txt', output_text)
     except TimeoutExpired:
@@ -347,10 +339,10 @@ def generate_pdf(*, input_image, skip_pdf, output_pdf, output_text,
         use_skip_page(text_only, skip_pdf, output_pdf, output_text)
     except CalledProcessError as e:
         tesseract_log_output(log, e.output, input_image)
-        if 'read_params_file: parameter not found' in e.output:
+        if b'read_params_file: parameter not found' in e.output:
             raise TesseractConfigError() from e
 
-        if 'Image too large' in e.output:
+        if b'Image too large' in e.output:
             use_skip_page(text_only, skip_pdf, output_pdf, output_text)
             return
         raise e from e

ocrmypdf/exec/unpaper.py

@@ -3,25 +3,13 @@
 # unpaper documentation:
 # https://github.com/Flameeyes/unpaper/blob/master/doc/basic-concepts.md
 
-from subprocess import CalledProcessError, STDOUT, check_output, check_call
+from subprocess import CalledProcessError, STDOUT, check_output
 from tempfile import NamedTemporaryFile
 import sys
 import os
 from functools import lru_cache
 from ..exceptions import MissingDependencyError
-from . import get_program
-
-
-@lru_cache(maxsize=1)
-def version():
-    args_unpaper = [
-        get_program('unpaper'),
-        '--version'
-    ]
-    version = check_output(
-        args_unpaper, close_fds=True, universal_newlines=True,
-        stderr=STDOUT, timeout=5)
-    return version.strip()
+from . import get_program, get_version
 
 
 try:
@@ -31,6 +19,11 @@ except ImportError:
     raise
 
 
+@lru_cache(maxsize=1)
+def version():
+    return get_version('unpaper')
+
+
 def run(input_file, output_file, dpi, log, mode_args):
     args_unpaper = [
         get_program('unpaper'),

ocrmypdf/helpers.py

@@ -32,7 +32,7 @@ def re_symlink(input_file, soft_link_name, log=None):
                 "%s exists and is not a link" % soft_link_name)
         try:
             os.unlink(soft_link_name)
-        except:
+        except OSError:
             prdebug("Can't unlink %s" % (soft_link_name))
 
     if not os.path.exists(input_file):

ocrmypdf/hocrtransform.py

@@ -1,12 +1,33 @@
 #!/usr/bin/env python3
-##############################################################################
-# Copyright (c) 2013-14: fritz-hh from Github
-#   (https://github.com/fritz-hh)
 #
-# Copyright (c) 2010: Jonathan Brinley from Github
-#   (https://github.com/jbrinley/HocrConverter)
-#   Initial version by Jonathan Brinley, jonathanbrinley@gmail.com
-##############################################################################
+# Copyright (c) 2010, Jonathan Brinley
+#   Original version from: https://github.com/jbrinley/HocrConverter
+#
+# Copyright (c) 2013-14, Julien Pfefferkorn
+#   Modifications
+#
+# Copyright (c) 2015-16, James R. Barlow
+#   Set text to transparent
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
 from reportlab.pdfgen.canvas import Canvas
 from reportlab.lib.units import inch
 from xml.etree import ElementTree

ocrmypdf/lib/compile_leptonica.py

@@ -183,10 +183,39 @@ pixColorMagnitude(PIX     *pixs,
                   l_int32  bwhite,
                   l_int32  type);
 
-void
+PIX *
+pixMaskOverColorPixels(PIX     *pixs,
+                       l_int32  threshdiff,
+                       l_int32  mindist);
+
+l_int32
+pixGetAverageMaskedRGB(PIX        *pixs,
+                       PIX        *pixm,
+                       l_int32     x,
+                       l_int32     y,
+                       l_int32     factor,
+                       l_int32     type,
+                       l_float32  *prval,
+                       l_float32  *pgval,
+                       l_float32  *pbval);
+
+PIX * 
+pixGlobalNormRGB(PIX * 	pixd,
+                 PIX * 	pixs,
+                 l_int32 	rval,
+                 l_int32 	gval,
+                 l_int32 	bval,
+                 l_int32 	mapval); 
+
+PIX *
+pixInvert(PIX * pixd,
+          PIX * pixs);
+
+void                 
 boxDestroy(BOX  **pbox);
 
-void lept_free(void *ptr);
+void
+lept_free(void *ptr);  
 """)
 
 

ocrmypdf/pipeline.py

@@ -2,20 +2,11 @@
 # © 2016 James R. Barlow: github.com/jbarlow83
 
 from contextlib import suppress
-from tempfile import mkdtemp
-from functools import partial
 import sys
 import os
-import re
 import shutil
-import warnings
-import multiprocessing
-import atexit
-import textwrap
 import img2pdf
-import logging
-import argparse
-
+import re
 import PyPDF2 as pypdf
 from PIL import Image
 
@@ -23,7 +14,7 @@ from ruffus import formatter, regex, Pipeline, suffix
 
 from .hocrtransform import HocrTransform
 from .pdfinfo import PdfInfo, Encoding, Colorspace
-from .pdfa import generate_pdfa_ps, file_claims_pdfa
+from .pdfa import generate_pdfa_ps
 from .helpers import re_symlink, is_iterable_notstr, page_number
 from .exec import ghostscript, tesseract, qpdf
 from .exceptions import *
@@ -51,6 +42,8 @@ class JobContext:
 
     def __init__(self):
         self.pdfinfo = None
+        self.options = None
+        self.work_folder = None
 
     def generate_pdfinfo(self, infile):
         self.pdfinfo = PdfInfo(infile)
@@ -156,6 +149,22 @@ def triage_image_file(input_file, output_file, log, options):
         raise UnsupportedImageFormatError() from e
 
 
+def _pdf_guess_version(input_file, search_window=1024):
+    """Try to find version signature at start of file.
+    
+    Not robust enough to deal with appended files.
+
+    Returns empty string if not found, indicating file is probably not PDF.
+    """
+
+    with open(input_file, 'rb') as f:
+        signature = f.read(1024)
+    m = re.search(b'%PDF-(\d\.\d)', signature)
+    if m:
+        return m.group(1)
+    return ''
+
+
 def triage(
         input_file,
         output_file,
@@ -164,14 +173,12 @@ def triage(
 
     options = context.get_options()
     try:
-        with open(input_file, 'rb') as f:
-            signature = f.read(1024)
-            if b'%PDF' in signature:
-                if options.image_dpi:
-                    log.warning("Argument --image-dpi ignored because the "
-                                "input file is a PDF, not an image.")
-                re_symlink(input_file, output_file, log)
-                return
+        if _pdf_guess_version(input_file):
+            if options.image_dpi:
+                log.warning("Argument --image-dpi ignored because the "
+                            "input file is a PDF, not an image.")
+            re_symlink(input_file, output_file, log)
+            return
     except EnvironmentError as e:
         log.error(e)
         raise InputFileError() from e
@@ -189,7 +196,8 @@ def repair_pdf(
     pdfinfo = PdfInfo(output_file)
 
     if pdfinfo.has_userunit and options.output_type == 'pdfa':
-        log.error("This input file uses a PDF feature that is not supported "
+        log.error(
+            "This input file uses a PDF feature that is not supported "
             "by Ghostscript, so you cannot use --output-type=pdfa for this "
             "file. (Specifically, it uses the PDF-1.6 /UserUnit feature to "
             "support very large or small page sizes, and Ghostscript cannot "
@@ -706,7 +714,7 @@ def combine_layers(
 
     if rotation != 0:
         log.info("{0:4d}: rotating image layer {1} degrees".format(
-            page_number(image), rotation, tx, ty))
+            page_number(image), rotation))
 
     try:
         page_text.mergeRotatedScaledTranslatedPage(
@@ -826,9 +834,9 @@ def get_pdfmark(base_pdf, options):
         renderer_tag = 'OCR'
 
     pdfmark['/Creator'] = '{0} {1} / Tesseract {2} {3}'.format(
-            PROGRAM_NAME, VERSION,
-            renderer_tag,
-            tesseract.version())
+        PROGRAM_NAME, VERSION,
+        renderer_tag,
+        tesseract.version())
     return pdfmark
 
 
@@ -885,7 +893,8 @@ def merge_pages_ghostscript(
         output_file=output_file,
         compression=options.pdfa_image_compression,
         log=log,
-        threads=options.jobs or 1)
+        threads=options.jobs or 1,
+        pdfa_part=('1' if options.output_type == 'pdfa-1' else '2'))
 
 
 def merge_pages_qpdf(
@@ -919,6 +928,7 @@ def merge_pages_qpdf(
     first_page = pypdf.PdfFileReader(pdf_pages[0])
 
     writer = pypdf.PdfFileWriter()
+    writer._header = b'%PDF-' + _pdf_guess_version(pdf_pages[0])  # copy version from source
     writer.appendPagesFromReader(first_page)
     writer.addMetadata(pdfmark)
     writer_file = pdf_pages[0].replace('.pdf', '.metadata.pdf')
@@ -928,7 +938,8 @@ def merge_pages_qpdf(
     pdf_pages[0] = writer_file
 
     qpdf.merge(input_files=pdf_pages, output_file=output_file,
-               min_version=context.get_pdfinfo().min_version)
+               min_version=context.get_pdfinfo().min_version,
+               log=log)
 
 
 def merge_sidecars(
@@ -936,7 +947,6 @@ def merge_sidecars(
         output_file,
         log,
         context):
-    options = context.get_options()
     pdfinfo = context.get_pdfinfo()
 
     txt_files = [None] * len(pdfinfo)
@@ -947,21 +957,30 @@ def merge_sidecars(
             txt_files[idx] = infile
 
     def write_pages(stream):
-        for page_number, txt_file in enumerate(txt_files):
-            if page_number != 0:
+        for page_num, txt_file in enumerate(txt_files):
+            if page_num != 0:
                 stream.write('\f')  # Form feed between pages
             if txt_file:
-                with open(txt_file, 'r') as in_:
-                    stream.write(in_.read())
+                with open(txt_file, 'r', encoding="utf-8") as in_:
+                    txt = in_.read()
+                    # Tesseract v4 alpha started adding form feeds in 
+                    # commit aa6eb6b
+                    # No obvious way to detect what binaries will do this, so
+                    # for consistency just ignore its form feeds and insert our
+                    # own
+                    if txt.endswith('\f'):
+                        stream.write(txt[:-1])
+                    else:
+                        stream.write(txt)
             else:
                 stream.write('[OCR skipped on page {}]'.format(
-                        page_number + 1))
+                        page_num + 1))
 
     if output_file == '-':
         write_pages(sys.stdout)
         sys.stdout.flush()
     else:
-        with open(output_file, 'w', encoding='utf-8') as out:
+        with open(output_file, 'w', encoding="utf-8") as out:
             write_pages(out)
 
 
@@ -1139,7 +1158,8 @@ def build_pipeline(options, work_folder, log, context):
         output=os.path.join(work_folder, r'\1.rendered.pdf'),
         extras=[log, context])
     task_combine_layers.graphviz(fillcolor='"#00cc66"')
-    task_combine_layers.active_if(options.pdf_renderer == 'hocr' or options.pdf_renderer == 'sandwich')
+    task_combine_layers.active_if(options.pdf_renderer == 'hocr' or 
+                                  options.pdf_renderer == 'sandwich')
 
     # Tesseract OCR+PDF
     task_ocr_tesseract_and_render_pdf = main_pipeline.collate(
@@ -1161,7 +1181,7 @@ def build_pipeline(options, work_folder, log, context):
         filter=formatter(r'\.repaired\.pdf'),
         output=os.path.join(work_folder, 'pdfa.ps'),
         extras=[log, context])
-    task_generate_postscript_stub.active_if(options.output_type == 'pdfa')
+    task_generate_postscript_stub.active_if(options.output_type.startswith('pdfa'))
 
 
     # Bypass valve
@@ -1183,7 +1203,7 @@ def build_pipeline(options, work_folder, log, context):
                task_generate_postscript_stub],
         output=os.path.join(work_folder, 'merged.pdf'),
         extras=[log, context])
-    task_merge_pages_ghostscript.active_if(options.output_type == 'pdfa')
+    task_merge_pages_ghostscript.active_if(options.output_type.startswith('pdfa'))
 
     task_merge_pages_qpdf = main_pipeline.merge(
         task_func=merge_pages_qpdf,

requirements.txt

@@ -2,8 +2,8 @@
 # setup.py lists a separate set of requirements that are looser to simplify
 # installation
 ruffus == 2.6.3
-Pillow == 4.1.1
+Pillow == 4.3.0
 reportlab == 3.4.0
 PyPDF2 == 1.26.0
-img2pdf == 0.2.3
-cffi == 1.10.0
+img2pdf == 0.2.4
+cffi == 1.11.2

setup.py

@@ -217,9 +217,11 @@ setup(
         "Topic :: Text Processing :: Indexing",
         "Topic :: Text Processing :: Linguistic",
         ],
+    python_requires='>=3.5',
     setup_requires=[
         'setuptools_scm',       # so that version will work
-        'cffi>=1.9.1'           # to build the leptonica module
+        'cffi>=1.9.1',          # to build the leptonica module
+        'pytest-runner'         # to enable python setup.py test
     ],
     use_scm_version={'version_scheme': 'post-release'},
     cffi_modules=[

test_requirements.txt

@@ -1,4 +1,5 @@
-pytest >= 3.0
+pytest >= 3.2
 pytest-helpers-namespace
 pytest-xdist
-pytest-cov
+pytest-cov
+pytest-timeout

tests/resources/README.rst

@@ -57,10 +57,13 @@ under the terms of the license in LICENSE.rst.
         - test image with 200 x 100 DPI resolution
     *   - blank.pdf
         - @jbarlow83
-        - blank PDF
+        - blank PDF generated by Adobe Illustrator CC 17, containing a lot of application-specific metadata/bloat
     *   - cmyk.pdf
         - @jbarlow83
         - a CMYK image created in Photoshop
+    *   - crom.png
+        - @jbarlow83
+        - test for non-dictionary words        
     *   - enormous.pdf
         - @jbarlow83
         - very large PDF page
@@ -69,7 +72,7 @@ under the terms of the license in LICENSE.rst.
         - a linearized PDF containing some unusual indirect objects, created by an Epson printer; printout of a Wikipedia article (CC BY-SA)
     *   - formxobject.pdf
         - @jbarlow83
-        - hand-crafted exotic PDF containing an image inside a Form XObject
+        - hand-crafted PDF containing an image inside a Form XObject
     *   - francais.pdf
         - @jbarlow83
         - a page containing French accents (diacritics)  
@@ -88,12 +91,15 @@ under the terms of the license in LICENSE.rst.
     *   - overlay.pdf
         - @maxandersen
         - PDF file generated by PDFPen pro that triggered content stream parse errors
+    *   - negzero.pdf
+        - @jbarlow83
+        - copy of formxobject.pdf with token that qpdf doesn't like
     *   - no_contents.pdf
         - @jbarlow83
         - synthetic PDF with a blank page that has no /Contents entry
-    *   - crom.png
+    *   - trivial.pdf
         - @jbarlow83
-        - test for non-dictionary words
+        - smallest possible valid PDF-1.3 with all required fields
 
 
 Assemblies

tests/resources/trivial.pdf

@@ -0,0 +1,27 @@
+%PDF-1.3
+%¿÷¢þ
+1 0 obj
+<< /Pages 2 0 R /Type /Catalog >>
+endobj
+2 0 obj
+<< /Count 1 /Kids [ 3 0 R ] /Type /Pages >>
+endobj
+3 0 obj
+<< /Contents 4 0 R /MediaBox [ 0 0 300 300 ] /Parent 2 0 R /Resources << >> /Type /Page >>
+endobj
+4 0 obj
+<< /Length 0 >>
+stream
+endstream
+endobj
+xref
+0 5
+0000000000 65535 f 
+0000000015 00000 n 
+0000000064 00000 n 
+0000000123 00000 n 
+0000000229 00000 n 
+trailer << /Root 1 0 R /Size 5 /ID [<8088c23e0edc07ef4fbed3daa55f52ed><8088c23e0edc07ef4fbed3daa55f52ed>] >>
+startxref
+277
+%%EOF

tests/spoof/qpdf_dummy_return2.py

@@ -5,6 +5,9 @@ import sys
 
 
 def main():
+    if sys.argv[-1] == '--version':
+        print('qpdf version 7.0.0')
+        sys.exit(0)
     print('qpdf dummy')
     sys.exit(2)
 

tests/spoof/tesseract_badutf8.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+# © 2017 James R. Barlow: github.com/jbarlow83
+import sys
+import img2pdf
+import PyPDF2 as pypdf
+from PIL import Image
+
+
+"""Tesseract bad utf8 spoof
+
+In 'hocr' mode or 'pdf' mode, return error code 1 and some non-Unicode
+text because tesseract seems to do that in some cases related to 
+language pack version mismatches
+
+"""
+
+
+VERSION_STRING = '''tesseract 3.05.01
+ leptonica-1.72
+  libjpeg 8d : libpng 1.6.19 : libtiff 4.0.6 : zlib 1.2.5
+SPOOFED
+'''
+
+# Japanese "Invalid UTF-8" encoded in Shift JIS
+BAD_UTF8 = b'\x96\xb3\x8c\xf8\x82\xc8UTF-8\x0a'
+
+
+def main():
+    if sys.argv[1] == '--version':
+        print(VERSION_STRING, file=sys.stderr)
+        sys.exit(0)
+    elif sys.argv[1] == '--list-langs':
+        print('List of available languages (1):\neng', file=sys.stderr)
+        sys.exit(0)
+    elif sys.argv[1] == '--print-parameters':
+        print("Some parameters", file=sys.stderr)
+        print("textonly_pdf\t1\tSome help text")
+        sys.exit(0)
+    elif sys.argv[-2] in ('hocr', 'pdf'):
+        sys.stdout.buffer.write(BAD_UTF8)
+        sys.exit(1)
+    elif sys.argv[-1] == 'stdout':
+        inputf = sys.argv[-2]
+        print("""Orientation: 0
+Orientation in degrees: 0
+Orientation confidence: 100.00
+Script: 1
+Script confidence: 100.00""", file=sys.stderr)
+    else:
+        print("Spoof doesn't understand arguments", file=sys.stderr)
+        print(sys.argv, file=sys.stderr)
+        sys.exit(1)
+
+    sys.exit(0)
+
+
+if __name__ == '__main__':
+    main()

tests/test_main.py

@@ -1,16 +1,16 @@
-#!/usr/bin/env python3
-# © 2015 James R. Barlow: github.com/jbarlow83
+# © 2015-17 James R. Barlow: github.com/jbarlow83
 
 from subprocess import Popen, PIPE, check_output, check_call, DEVNULL
 import os
 import shutil
+import resource
 import pytest
 from ocrmypdf.pdfinfo import PdfInfo, Colorspace, Encoding
 import PyPDF2 as pypdf
 from ocrmypdf.exceptions import ExitCode
 from ocrmypdf import leptonica
 from ocrmypdf.pdfa import file_claims_pdfa
-from ocrmypdf.exec import ghostscript, tesseract
+from ocrmypdf.exec import ghostscript, tesseract, qpdf
 import logging
 from math import isclose
 
@@ -55,6 +55,11 @@ def spoof_no_tess_gs_raster_fail():
     return spoof(tesseract='tesseract_noop.py', gs='gs_raster_failure.py')
 
 
+@pytest.fixture
+def spoof_tess_bad_utf8():
+    return spoof(tesseract='tesseract_badutf8.py')
+
+
 @pytest.fixture
 def spoof_qpdf_always_error():
     return spoof(qpdf='qpdf_dummy_return2.py')
@@ -64,6 +69,7 @@ def test_quick(spoof_tesseract_cache, resources, outpdf):
     check_ocrmypdf(resources / 'ccitt.pdf', outpdf, env=spoof_tesseract_cache)
 
 
+@pytest.mark.filterwarnings('ignore:Image size')
 def test_deskew(spoof_tesseract_noop, resources, outdir):
     # Run with deskew
     deskewed_pdf = check_ocrmypdf(
@@ -805,7 +811,6 @@ def test_user_words(resources, outdir):
             '--image-dpi', 150,
             '--sidecar', sidecar_before
         )
-
         assert 'cromulent' not in sidecar_before.open().read()
 
     with word_list.open('w') as f:
@@ -982,3 +987,53 @@ def test_sidecar_pagecount(spoof_tesseract_cache, resources, outpdf):
     # formfeeds is the page count less one
     assert ocr_text.count('\f') == num_pages - 1, \
         "Sidecar page count does not match PDF page count"
+
+
+def test_sidecar_nonempty(spoof_tesseract_cache, resources, outpdf):
+    sidecar = outpdf + '.txt'
+    check_ocrmypdf(
+        resources / 'ccitt.pdf', outpdf,
+        '--sidecar', sidecar,
+        env=spoof_tesseract_cache
+    )
+
+    with open(sidecar, 'r') as f:
+        ocr_text = f.read()
+    assert 'the' in ocr_text
+
+
+def test_pdfa_1(spoof_tesseract_cache, resources, outpdf):
+    check_ocrmypdf(
+        resources / 'ccitt.pdf', outpdf,
+        '--output-type', 'pdfa-1',
+        env=spoof_tesseract_cache
+    )
+
+    pdfa_info = file_claims_pdfa(outpdf)
+    assert pdfa_info['conformance'] == 'PDF/A-1B'
+
+
+def test_bad_locale():
+    env = os.environ.copy()
+    env['LC_ALL'] = 'C'
+
+    p, out, err = run_ocrmypdf(
+        'a', 'b', env=env
+    )
+    assert out == '', "stdout not clean"
+    assert p.returncode != 0
+    assert 'configured to use ASCII as encoding' in err, "should whine"
+
+
+@pytest.mark.parametrize('renderer', RENDERERS)
+def test_bad_utf8(spoof_tess_bad_utf8, renderer, resources, no_outpdf):
+    p, out, err = run_ocrmypdf(
+        resources / 'ccitt.pdf', no_outpdf,
+        '--pdf-renderer', renderer,
+        env=spoof_tess_bad_utf8
+    )
+
+    assert out == '', "stdout not clean"
+    assert p.returncode != 0
+    assert 'not utf-8' in err, "should whine about utf-8"
+    assert '\\x96' in err, 'should repeat backslash encoded output'

tests/test_qpdf.py

@@ -0,0 +1,55 @@
+# © 2017 James R. Barlow: github.com/jbarlow83
+
+import logging
+import resource
+import pytest
+
+from ocrmypdf.exec import ghostscript, tesseract, qpdf
+from ocrmypdf.pdfinfo import PdfInfo
+
+
+@pytest.mark.skipif(
+    qpdf.version() < '7.0.0',
+    reason="negzero.pdf crashes earlier versions")
+def test_qpdf_negative_zero(resources, outpdf):
+    negzero = resources / 'negzero.pdf'
+    hugemono = resources / 'hugemono.pdf'
+    # raises exception on err
+    qpdf.merge([str(negzero), str(hugemono)], outpdf, log=logging.getLogger())
+    
+
+@pytest.mark.timeout(15)
+@pytest.mark.parametrize('max_files,skip', [
+    (2, 0),  # Can we merge correctly without opening more than 2 files at once?
+    (16, 0), # And does this work properly when we can one-shot it?
+    (2, 1),  # Or playing with even/odd
+    (3, 0)   # Or odd step size
+    ])
+def test_qpdf_merge_correctness(resources, outpdf, max_files, skip):
+    # All of these must be only one page long
+    inputs = [
+        '2400dpi.pdf', 'aspect.pdf', 'blank.pdf', 'ccitt.pdf', 
+        'linn.pdf', 'masks.pdf', 'poster.pdf', 'overlay.pdf',
+        'skew.pdf', 'trivial.pdf']
+    
+    input_files = [str(resources / f) for f in inputs]
+
+    qpdf.merge(
+        input_files[skip:], outpdf, log=logging.getLogger(), 
+        max_files=max_files)
+    assert len(PdfInfo(outpdf).pages) == len(input_files[skip:])
+    
+
+@pytest.mark.timeout(15)
+@pytest.mark.skipif(
+    True, 
+    reason='qpdf binary cannot open multiple files multiple times')
+def test_page_merge_ulimit(resources, outpdf):
+    # Ensure we can merge pages without opening one file descriptor per page
+    ulimits = resource.getrlimit(resource.RLIMIT_NOFILE)
+    page_count = ulimits[0]
+    print(page_count)
+    input_files = [str(resources / 'trivial.pdf')] * page_count
+
+    qpdf.merge(input_files, outpdf, log=logging.getLogger())
+