v4.3.5: Python 3.6 compatibility

Revert to exactly what the previous passing build specified.

.gitignore

@@ -1,7 +1,7 @@
 # Development environment
 *.pyc
 *.sublime-*
-venv-*/
+venv*/
 pyvenv.cfg
 tasks.py
 

.travis.yml

@@ -11,8 +11,9 @@ cache:
         - tests/cache
 
 python:
-    - 3.4
-    - 3.5
+    - "3.4"
+    - "3.5"
+    - "3.6-dev"  # 3.6 not available yet
 
 before_cache:
     - rm -f $HOME/.cache/pip/log/debug.log

MANIFEST.in

@@ -14,6 +14,7 @@ include .dockerignore
 # tests
 include pytest.ini
 recursive-include tests *.jpg
+recursive-include tests *.png
 recursive-include tests *.pdf
 recursive-include tests *.py
 recursive-include tests *.rst

RELEASE_NOTES.rst

@@ -3,6 +3,41 @@ RELEASE NOTES
 
 OCRmyPDF uses `semantic versioning <http://semver.org/>`_.
 
+v4.3.5:
+=======
+
+-  Update documentation to confirm Python 3.6.0 compatibility. No code changes were needed, so many earlier versions are likely supported.
+
+
+v4.3.4:
+=======
+
+-  Fixed "decimal.InvalidOperation: quantize result has too many digits" for high DPI images
+
+
+v4.3.3:
+=======
+
+-  Fixed PDF/A creation with Ghostscript 9.20 properly
+-  Fixed an exception on inline stencil masks with a missing optional parameter
+
+
+v4.3.2:
+=======
+
+-  Fixed a PDF/A creation issue with Ghostscript 9.20 (note: this fix did not actually work)
+
+
+v4.3.1:
+=======
+
+-  Fixed an issue where pages produced by the "hocr" renderer after a Tesseract timeout would be rotated incorrectly if the input page was rotated with a /Rotate marker
+-  Fixed a file handle leak in LeptonicaErrorTrap that would cause a "too many open files" error for files around hundred pages of pages long when ``--deskew`` or ``--remove-background`` or other Leptonica based image processing features were in use, depending on the system value of ``ulimit -n``
+-  Ability to specify multiple languages for multilingual documents is now advertised in documentation
+-  Reduced the file sizes of some test resources
+-  Cleaned up debug output
+-  Tesseract caching in test cases is now more cautious about false cache hits and reproducing exact output, not that any problems were observed
+
 
 v4.3:
 =====
@@ -266,7 +301,7 @@ Changes
 -------
 
 -  Default output format is now PDF/A-2b instead of PDF/A-1b
--  Python 3.5 and OS X El Capitan are now supported platforms - no changes were
+-  Python 3.5 and macOS El Capitan are now supported platforms - no changes were
    needed to implement support
 -  Improved some error messages related to missing input files
 -  Fixed issue #20 - uppercase .PDF extension not accepted

dev_requirements.txt

@@ -1,4 +1,9 @@
-check-manifest>=0.33
-setuptools-scm>=1.11.1
-twine>=1.8.1
-coverage>=4.2
+check-manifest >= 0.34
+twine >= 1.8.1
+coverage >= 4.3.1
+pytest-xdist >= 1.15.0
+
+# Known good versions: 1.11.1
+# Known broken versions: 1.15.0
+setuptools-scm == 1.11.1
+setuptools-scm-git-archive == 1.0

docs/conf.py

@@ -31,7 +31,7 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-    'sphinx.ext.mathjax',
+    # 'sphinx.ext.mathjax',
 ]
 
 # Add any paths that contain templates here, relative to this directory.

docs/cookbook.rst

@@ -49,10 +49,23 @@ OCR will attempt to automatic correct the rotation of each page. This can help f
 You can increase (decrease) the parameter ``--rotate-pages-threshold`` to make page rotation more (less) aggressive.
 
 
+OCR languages other than English
+""""""""""""""""""""""""""""""""
+
+By default OCRmyPDF assumes the document is English. 
+
+.. code-block:: bash
+
+	ocrmypdf -l fre LeParisien.pdf LeParisien.pdf
+	ocrmypdf -l eng+fre Bilingual-English-French.pdf Bilingual-English-French.pdf
+
+Language packs must be installed for all languages specified. See :ref:`Installing additional language packs <lang-packs>`.
+
+
 OCR images, not PDFs
 --------------------
 
-Use a program like `img2pdf <https://gitlab.mister-muffin.de/josch/img2pdf>`_ to convert your images to PDFs, and then pipe the resutls to run ocrmypdf:
+Use a program like `img2pdf <https://gitlab.mister-muffin.de/josch/img2pdf>`_ to convert your images to PDFs, and then pipe the results to run ocrmypdf:
 
 .. code-block:: bash
 
@@ -107,6 +120,7 @@ watchdog installs the command line program ``watchmedo``, which can be told to r
 	mkdir out
 	watchmedo shell-command \
 		--patterns="*.pdf" \
+		--ignore-directories \
 		--command='ocrmypdf "${watch_src_path}" "out/${watch_src_path}" ' \
 		.  # don't forget the final dot
 
@@ -114,12 +128,12 @@ For more complex behavior you can write a Python script around to use the watchd
 
 On file servers, you could configure watchmedo as a system service so it will run all the time.
 
-
 Caveats
 """""""
 
 * ``watchmedo`` may not work properly on a networked file system, depending on the capabilities of the file system client and server.
 * This simple recipe does not filter for the type of file system event, so file copies, deletes and moves, and directory operations, will all be sent to ocrmypdf, producing errors in several cases. Disable your watched folder if you are doing anything other than copying files to it.
+* If the source and destination directory are the same, watchmedo may create an infinite loop.
 
 
 Batch jobs

docs/index.rst

@@ -20,6 +20,7 @@ Contents:
    installation
    languages
    cookbook
+   security
    errors
 
 

docs/installation.rst

@@ -78,6 +78,8 @@ In this worked example, the current working directory contains an input file cal
 
    docker run --rm -v "$(pwd):/home/docker"   ocrmypdf --skip-text test.pdf output.pdf
 
+.. note:: The working directory should be a writable local volume or Docker may not have permission to access it.
+
 Note that ``ocrmypdf`` has its own separate ``-v VERBOSITYLEVEL`` argument to control debug verbosity. All Docker arguments should before the ``ocrmypdf`` image name and all arguments to ``ocrmypdf`` should be listed after.
 
 
@@ -105,6 +107,8 @@ Install or upgrade the required Homebrew packages, if any are missing:
    brew install libxml2 libffi leptonica
    brew install unpaper   # optional
    
+Python 3.4, 3.5 and 3.6 are supported.
+
 Install the required Tesseract OCR engine with the language packs you plan to use:
    
 .. code-block:: bash

docs/introduction.rst

@@ -28,11 +28,16 @@ Rasterizing a PDF is the process of generating an image suitable for display or
 About PDF/A
 -----------
 
-`PDF/A <https://en.wikipedia.org/wiki/PDF/A>`_ is a standardized subset of the full PDF specification that is designed for archiving.  PDF/A differs from PDF primarily by omitting features that would make it difficult to read the file in the future, such as embedded Javascript or references to external fonts.  All fonts and resources needed to interpret the PDF must be contained within it.  Generally speaking, scanned documents should be converted to PDF/A. There are various conformance levels and versions, such as "PDF/A-2b".
+`PDF/A <https://en.wikipedia.org/wiki/PDF/A>`_ is an ISO-standardized subset of the full PDF specification that is designed for archiving (the 'A' stands for Archive).  PDF/A differs from PDF primarily by omitting features that would make it difficult to read the file in the future, such as embedded Javascript, video, audio and references to external fonts.  All fonts and resources needed to interpret the PDF must be contained within it. Because PDF/A disables Javascript and other types of embedded content, it is probably more secure.
 
-Since most people who scan documents are interested in reading them in the future, OCRmyPDF generates PDF/A-2b by default.
+There are various conformance levels and versions, such as "PDF/A-2b".
+
+Generally speaking, the best format for scanned documents is PDF/A. Some governments and jurisdictions, US Courts in particular, `mandate the use of PDF/A <https://pdfblog.com/2012/02/13/what-is-pdfa/>`_ for scanned documents.
+
+Since most people who scan documents are interested in reading them indefinitely into the future, OCRmyPDF generates PDF/A-2b by default.
+
+PDF/A has a few drawbacks.  Some PDF viewers include an alert that the file is a PDF/A, which may confuse some users.  It also tends to produce larger files than PDF, because it embeds certain resources even if they are commonly available. PDF/A files can be digitally signed, but may not be encrypted, to ensure they can be read in the future.  Fortunately, converting from PDF/A to a regular PDF is trivial, and any PDF viewer can view PDF/A.
 
-PDF/A has a few drawbacks.  Some PDF viewers include an alert that the file is a PDF/A, which may confuse some users.  It also tends to produce larger files than PDF, because it embeds certain resources even if they are commonly available. 
 
 What OCRmyPDF does
 ------------------

docs/languages.rst

@@ -1,3 +1,5 @@
+.. _lang-packs:
+
 Installing additional language packs
 ====================================
 
@@ -19,7 +21,7 @@ Debian and Ubuntu users
    apt-get install tesseract-ocr-chi-sim  # Example: Install Chinese Simplified language back
    
 You can then pass the ``-l LANG`` argument to OCRmyPDF to give a hint as to what languages it should search for. Multiple
-languages can be requested.
+languages can be requested using either ``-l eng+fre`` (English and French) or ``-l eng -l fre``.
 
 Mac OS X (macOS) users
 ----------------------
@@ -38,7 +40,7 @@ As of v4.2, users of ocrmypdf working languages outside the Latin alphabet shoul
 
 .. code-block:: bash
 
-	ocrmypdf --output-type pdf --pdf-renderer tesseract
+	ocrmypdf -l eng+gre --output-type pdf --pdf-renderer tesseract
 
 The reasons for this are:
 

docs/security.rst

@@ -0,0 +1,44 @@
+PDF Security Issues
+===================
+
+	OCRmyPDF should only be used on PDFs you trust. It is not designed to protect you against malware. 
+
+Recognizing that many users have an interest in handling PDFs and applying OCR to PDFs they did not generate themselves, this article discusses the security implications of PDFs and how users can protect themselves.
+
+The disclaimer applies: this software has no warranties of any kind.
+
+PDFs may contain malware
+------------------------
+
+PDF is a rich, complex file format. The official PDF 1.7 specification, ISO 32000:2008, is hundreds of packages long and references several annexes each of which are similar in length. PDFs can contain video, audio, JavaScript and other programming, and forms. In some cases, they can open internet connections to pre-selected URLs. All of these possible attack vectors.
+
+In short, PDFs `may contain viruses <https://security.stackexchange.com/questions/64052/can-a-pdf-file-contain-a-virus>`_.
+
+This `article <https://theinvisiblethings.blogspot.ca/2013/02/converting-untrusted-pdfs-into-trusted.html>`_ describes a method which allows potentially hostile PDFs to be viewed and rasterized safely in a disposable virtual machine. A trusted PDF created in this manner is converted to images and loses all information making it searchable. OCRmyPDF could be used restore searchability.
+
+How OCRmyPDF processes PDFs
+---------------------------
+
+OCRmyPDF must open and interpret your PDF in order to insert an OCR layer. First, it runs all PDFs through `qpdf <https://github.com/qpdf/qpdf>`_, a program that repairs PDFs with syntax errors. This is done because, in the author's experience, a significant number of PDFs in the wild especially those created by scanners are not well-formed files. qpdf makes it more likely that OCRmyPDF will succeed, but offers no security guarantees. qpdf is also used to split the PDF into single page PDFs.
+
+After qpdf, OCRmyPDF examines each page using `PyPDF2 <https://github.com/mstamy2/PyPDF2>`_. This library also has no warranties or guarantees.
+
+Finally, OCRmyPDF rasterizes each page of the PDF using `Ghostscript <http://ghostscript.com/>`_ in ``-dSAFER`` mode. 
+
+Depending on the options specified, OCRmyPDF may graft the OCR layer into the existing PDF or it may essentially reconstruct ("re-fry") a visually identical PDF that may be quite different at the binary level. That said, OCRmyPDF is not a tool designed for sanitizing PDFs.
+
+Using OCRmyPDF online
+---------------------
+
+OCRmyPDF is not designed to be deployed "as a service", in a setting where a user/attacker could upload a file for OCR processing online. It is not designed to be secure in this case.
+
+Abbyy Cloud OCR is a viable commercial alternative with a web services API. The author also provides professional services that include OCR and building databases around PDFs, and is happy to provide consultation.
+
+Password protection, digital signatures and certification
+---------------------------------------------------------
+
+OCRmyPDF cannot remove password protection from a PDF. ``qpdf``, one of its dependencies, has this capability. After OCR is applied, password protection is not permitted on PDF/A documents but the file can be converted to regular PDF.
+
+Many programs exist which are capable of inserting an image of someone's signature. On its own, this offers no security guarantees. It is trivial to remove the signature image and apply it to other files. This practice offers no real security.
+
+Important documents can be digitally signed and certified to attest to their authorship. OCRmyPDF cannot do this. Open source tools such as pdfbox (Java) have this capability as does Adobe Acrobat. 

ocrmypdf/__main__.py

@@ -21,7 +21,7 @@ from PIL import Image
 from functools import partial
 
 from ruffus import transform, suffix, merge, active_if, regex, jobs_limit, \
-    formatter, follows, split, collate, check_if_uptodate, graphviz
+    formatter, follows, split, collate, check_if_uptodate, graphviz, posttask
 import ruffus.ruffus_exceptions as ruffus_exceptions
 import ruffus.cmdline as cmdline
 import ruffus.proxy_logger as proxy_logger
@@ -166,8 +166,10 @@ parser.add_argument(
     help="output searchable PDF file (or '-' to write to standard output)")
 parser.add_argument(
     '-l', '--language', action='append',
-    help="languages of the file to be OCRed (see tesseract --list-langs for "
-         "all language packs installed in your system)")
+    help="Language(s) of the file to be OCRed (see tesseract --list-langs for "
+         "all language packs installed in your system). To specify multiple "
+         "languages, join them with '+' or issue this argument once for each "
+         "language.")
 parser.add_argument(
     '-j', '--jobs', metavar='N', type=int,
     help="Use up to N CPU cores simultaneously (default: use all)")
@@ -304,6 +306,8 @@ if not set(options.language).issubset(tesseract.languages()):
 # ----------
 # Arguments
 
+options.verbose_abbreviated_path = 1
+
 if options.pdf_renderer == 'auto':
     options.pdf_renderer = 'hocr'
 
@@ -414,6 +418,7 @@ class WrappedLogger:
             self.logger.critical(*args, **kwargs)
 
 _log = WrappedLogger(_logger, _logger_mutex)
+_log.debug('ocrmypdf ' + VERSION)
 
 
 def re_symlink(input_file, soft_link_name, log=_log):
@@ -460,6 +465,11 @@ _pdfinfo_lock = manager.Lock()
 work_folder = mkdtemp(prefix="com.github.ocrmypdf.")
 
 
+def done_task(caller):
+    "Useful as debug hook"
+    pass
+
+
 @atexit.register
 def cleanup_working_files(*args):
     if options.keep_temporary_files:
@@ -533,6 +543,7 @@ def triage_image_file(input_file, output_file, log):
         sys.exit(ExitCode.input_file)
 
 
+@posttask(partial(done_task, 'triage'))
 @transform(
     input=os.path.join(work_folder, 'origin'),
     filter=formatter('(?i)'),
@@ -555,6 +566,7 @@ def triage(
     triage_image_file(input_file, output_file, log)
 
 
+@posttask(partial(done_task, 'repair_pdf'))
 @transform(
     input=triage,
     filter=suffix('.pdf'),
@@ -648,6 +660,7 @@ def is_ocr_required(pageinfo, log):
     return ocr_required
 
 
+@posttask(partial(done_task, 'split_pages'))
 @split(
     repair_pdf,
     os.path.join(work_folder, '*.page.pdf'),
@@ -690,6 +703,7 @@ def split_pages(
                 os.path.basename(filename)[0:6] + alt_suffix))
 
 
+@posttask(partial(done_task, 'rasterize_preview'))
 @active_if(options.rotate_pages)
 @transform(
     input=split_pages,
@@ -712,6 +726,7 @@ def rasterize_preview(
         log=log)
 
 
+@posttask(partial(done_task, 'orient_page'))
 @collate(
     input=[split_pages, rasterize_preview],
     filter=regex(r".*/(\d{6})(\.ocr|\.skip)(?:\.page\.pdf|\.preview\.jpg)"),
@@ -786,6 +801,7 @@ def orient_page(
             pdfinfo[pageno] = pageinfo
 
 
+@posttask(partial(done_task, 'rasterize_with_ghostscript'))
 @transform(
     input=orient_page,
     filter=suffix('.ocr.oriented.pdf'),
@@ -822,6 +838,7 @@ def rasterize_with_ghostscript(
         log=log)
 
 
+@posttask(partial(done_task, 'preprocess_remove_background'))
 @transform(
     input=rasterize_with_ghostscript,
     filter=suffix(".page.png"),
@@ -848,6 +865,7 @@ def preprocess_remove_background(
         re_symlink(input_file, output_file, log)
 
 
+@posttask(partial(done_task, 'preprocess_deskew'))
 @transform(
     input=preprocess_remove_background,
     filter=suffix(".pp-background.png"),
@@ -870,6 +888,7 @@ def preprocess_deskew(
     leptonica.deskew(input_file, output_file, dpi)
 
 
+@posttask(partial(done_task, 'preprocess_clean'))
 @transform(
     input=preprocess_deskew,
     filter=suffix(".pp-deskew.png"),
@@ -892,6 +911,7 @@ def preprocess_clean(
     unpaper.clean(input_file, output_file, dpi, log)
 
 
+@posttask(partial(done_task, 'ocr_tesseract_hocr'))
 @active_if(options.pdf_renderer == 'hocr')
 @transform(
     input=preprocess_clean,
@@ -919,6 +939,7 @@ def ocr_tesseract_hocr(
         )
 
 
+@posttask(partial(done_task, 'select_image_for_pdf'))
 @collate(
     input=[rasterize_with_ghostscript, preprocess_remove_background,
            preprocess_deskew, preprocess_clean],
@@ -962,6 +983,7 @@ def select_image_for_pdf(
         re_symlink(image, output_file)
 
 
+@posttask(partial(done_task, 'select_image_layer'))
 @active_if(options.pdf_renderer == 'hocr')
 @collate(
     input=[select_image_for_pdf, orient_page],
@@ -992,11 +1014,14 @@ def select_image_layer(
         with open(image, 'rb') as imfile, \
                 open(output_file, 'wb') as pdf:
             rawdata = imfile.read()
+            log.debug('{:4d}: convert'.format(page_number(page_pdf)))
             img2pdf.convert(
                 rawdata, with_pdfrw=False,
                 layout_fun=layout_fun, outputstream=pdf)
+            log.debug('{:4d}: convert done'.format(page_number(page_pdf)))
 
 
+@posttask(partial(done_task, 'render_hocr_page'))
 @active_if(options.pdf_renderer == 'hocr')
 @transform(
     input=ocr_tesseract_hocr,
@@ -1019,6 +1044,7 @@ def render_hocr_page(
                          showBoundingboxes=False, invisibleText=True)
 
 
+@posttask(partial(done_task, 'render_hocr_debug_page'))
 @active_if(options.pdf_renderer == 'hocr')
 @active_if(options.debug_rendering)
 @collate(
@@ -1048,6 +1074,7 @@ class PdfMergeFailedError(Exception):
     pass
 
 
+@posttask(partial(done_task, 'add_text_layer'))
 @active_if(options.pdf_renderer == 'hocr')
 @collate(
     input=[render_hocr_page, select_image_layer],
@@ -1126,6 +1153,7 @@ def add_text_layer(
         pdf_output.write(out)
 
 
+@posttask(partial(done_task, 'tesseract_ocr_and_render_pdf'))
 @active_if(options.pdf_renderer == 'tesseract')
 @collate(
     input=[select_image_for_pdf, orient_page],
@@ -1191,6 +1219,7 @@ def get_pdfmark(base_pdf):
     return pdfmark
 
 
+@posttask(partial(done_task, 'generate_postscript_stub'))
 @active_if(options.output_type == 'pdfa')
 @transform(
     input=repair_pdf,
@@ -1207,6 +1236,7 @@ def generate_postscript_stub(
     generate_pdfa_def(output_file, pdfmark)
 
 
+@posttask(partial(done_task, 'skip_page'))
 @transform(
     input=orient_page,
     filter=suffix('.skip.oriented.pdf'),
@@ -1224,6 +1254,7 @@ def skip_page(
     re_symlink(input_file, output_file, log)
 
 
+@posttask(partial(done_task, 'merge_pages_ghostscript'))
 @active_if(options.output_type == 'pdfa')
 @merge(
     input=[add_text_layer, render_hocr_debug_page, skip_page,
@@ -1252,9 +1283,10 @@ def merge_pages_ghostscript(
 
     pdf_pages = sorted(input_files, key=input_file_order)
     log.debug("Final pages: " + "\n".join(pdf_pages))
-    ghostscript.generate_pdfa(pdf_pages, output_file, options.jobs or 1)
+    ghostscript.generate_pdfa(pdf_pages, output_file, log, options.jobs or 1)
 
 
+@posttask(partial(done_task, 'merge_pages_qpdf'))
 @active_if(options.output_type == 'pdf')
 @merge(
     input=[add_text_layer, render_hocr_debug_page, skip_page,
@@ -1301,6 +1333,7 @@ def merge_pages_qpdf(
     qpdf.merge(pdf_pages, output_file)
 
 
+@posttask(partial(done_task, 'copy_final'))
 @merge(
     input=[merge_pages_ghostscript, merge_pages_qpdf],
     output=options.output_file,
@@ -1501,7 +1534,10 @@ def run_pipeline():
         _log.info("Output sent to stdout")
 
     with _pdfinfo_lock:
-        _log.debug(_pdfinfo)
+        if options.verbose:
+            from pprint import pformat
+            referent = _pdfinfo._getvalue()  # get the real list out of proxy
+            _log.debug(pformat(referent))
         direction = {0: 'n', 90: 'e',
                      180: 's', 270: 'w'}
         orientations = []

ocrmypdf/ghostscript.py

@@ -2,7 +2,7 @@
 # © 2015 James R. Barlow: github.com/jbarlow83
 
 from tempfile import NamedTemporaryFile
-from subprocess import Popen, PIPE, check_call
+from subprocess import Popen, PIPE, STDOUT, check_call
 from shutil import copy
 from . import get_program
 from .pdfa import SRGB_ICC_PROFILE
@@ -25,16 +25,13 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
             input_file
         ]
 
-        p = Popen(args_gs, close_fds=True, stdout=PIPE, stderr=PIPE,
+        p = Popen(args_gs, close_fds=True, stdout=PIPE, stderr=STDOUT,
                   universal_newlines=True)
-        stdout, stderr = p.communicate()
-        if stdout:
-            if 'error' in stdout:
-                log.error(stdout)  # Ghostscript puts errors in stdout
-            else:
-                log.debug(stdout)
-        if stderr:
-            log.error(stderr)
+        stdout, _ = p.communicate()
+        if 'error' in stdout:
+            log.error(stdout)  # Ghostscript puts errors in stdout
+        else:
+            log.debug(stdout)
 
         if p.returncode == 0:
             copy(tmp.name, output_file)
@@ -42,7 +39,7 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
             log.error('Ghostscript rendering failed')
 
 
-def generate_pdfa(pdf_pages, output_file, threads=1):
+def generate_pdfa(pdf_pages, output_file, log, threads=1):
     with NamedTemporaryFile(delete=True) as gs_pdf:
         args_gs = [
             get_program("gs"),
@@ -56,9 +53,30 @@ def generate_pdfa(pdf_pages, output_file, threads=1):
             "-sProcessColorModel=DeviceRGB",
             "-dJPEGQ=95",
             "-dPDFA=2",
-            "-sPDFACompatibilityPolicy=2",
+            "-dPDFACompatibilityPolicy=1",
             "-sOutputFile=" + gs_pdf.name,
         ]
         args_gs.extend(pdf_pages)
-        check_call(args_gs)
-        copy(gs_pdf.name, output_file)
+        p = Popen(args_gs, close_fds=True, stdout=PIPE, stderr=STDOUT,
+                  universal_newlines=True)
+        stdout, _ = p.communicate()
+
+        if 'error' in stdout:
+            log.error(stdout)
+        elif 'overprint mode not set' in stdout:
+            # Unless someone is going to print PDF/A documents on a
+            # magical sRGB printer I can't see the removal of overprinting
+            # being a problem....
+            log.debug(
+                "Ghostscript had to remove PDF 'overprinting' from the "
+                "input file to complete PDF/A conversion. "
+                )
+        else:
+            log.debug(stdout)
+
+        if p.returncode == 0:
+            # Ghostscript does not change return code when it fails to create
+            # PDF/A - check PDF/A status elsewhere
+            copy(gs_pdf.name, output_file)
+        else:
+            log.error('Ghostscript PDF/A failed')

ocrmypdf/leptonica.py

@@ -5,7 +5,6 @@
 #
 # Python FFI wrapper for Leptonica library
 
-from __future__ import print_function, absolute_import, division
 import argparse
 import sys
 import os
@@ -45,23 +44,30 @@ class LeptonicaErrorTrap(object):
         self.tmpfile = TemporaryFile()
 
         # Save the old stderr, and redirect stderr to temporary file
+        sys.stderr.flush()
         try:
-            self.old_stderr_fileno = os.dup(sys.stderr.fileno())
-            os.dup2(self.tmpfile.fileno(), sys.stderr.fileno())
+            self.copy_of_stderr = os.dup(sys.stderr.fileno())
+            os.dup2(self.tmpfile.fileno(), sys.stderr.fileno(),
+                    inheritable=False)
         except UnsupportedOperation:
-            self.old_stderr_fileno = None
+            self.copy_of_stderr = None
         return
 
     def __exit__(self, exc_type, exc_value, traceback):
         # Restore old stderr
-        if self.old_stderr_fileno is not None:
-            os.dup2(self.old_stderr_fileno, sys.stderr.fileno())
+        sys.stderr.flush()
+        if self.copy_of_stderr is not None:
+            os.dup2(self.copy_of_stderr, sys.stderr.fileno())
+            os.close(self.copy_of_stderr)
 
         # Get data from tmpfile (in with block to ensure it is closed)
         with self.tmpfile as tmpfile:
             tmpfile.seek(0)  # Cursor will be at end, so move back to beginning
             leptonica_output = tmpfile.read().decode(errors='replace')
 
+        assert self.tmpfile.closed
+        assert not sys.stderr.closed
+
         # If there are Python errors, let them bubble up
         if exc_type:
             logger.warning(leptonica_output)

ocrmypdf/pageinfo.py

@@ -2,7 +2,7 @@
 # © 2015 James R. Barlow: github.com/jbarlow83
 
 from subprocess import Popen, PIPE
-from decimal import Decimal, getcontext
+from decimal import Decimal
 from math import hypot
 import re
 import sys
@@ -232,7 +232,10 @@ def _find_page_inline_images(page, pageinfo, contentsinfo):
             image['bpc'] = inline.settings['/BPC']
         else:
             image['bpc'] = 8
-        image['color'] = FRIENDLY_COLORSPACE.get(inline.settings['/CS'], '-')
+        if '/CS' in inline.settings:
+            image['color'] = FRIENDLY_COLORSPACE.get(inline.settings['/CS'], '-')
+        else:
+            image['color'] = '-'
         image['comp'] = FRIENDLY_COMP.get(image['color'], '?')
         if '/F' in inline.settings:
             filter_ = inline.settings['/F']
@@ -326,9 +329,11 @@ def _find_page_regular_images(page, pageinfo, contentsinfo):
             image['dpi_w'] = max(dpi_w, image.get('dpi_w', 0))
             image['dpi_h'] = max(dpi_h, image.get('dpi_h', 0))
 
-        image['dpi_w'] = Decimal(image['dpi_w'])
-        image['dpi_h'] = Decimal(image['dpi_h'])
-        image['dpi'] = (image['dpi_w'] * image['dpi_h']) ** Decimal(0.5)
+        DPI_PREC = Decimal('1.000')
+        dpi = Decimal(image['dpi_w'] * image['dpi_h']).sqrt()
+        image['dpi_w'] = Decimal(image['dpi_w']).quantize(DPI_PREC)
+        image['dpi_h'] = Decimal(image['dpi_h']).quantize(DPI_PREC)
+        image['dpi'] = dpi.quantize(DPI_PREC)
         yield image
 
 
@@ -374,6 +379,11 @@ def _pdf_get_pageinfo(infile, pageno: int):
     pageinfo['width_inches'] = width_pt / Decimal(72.0)
     pageinfo['height_inches'] = height_pt / Decimal(72.0)
 
+    try:
+        pageinfo['rotate'] = int(page['/Rotate'])
+    except KeyError:
+        pageinfo['rotate'] = 0
+
     try:
         contentstream = pypdf.pdf.ContentStream(page.getContents(), pdf)
     except AttributeError as e:
@@ -397,7 +407,6 @@ def _pdf_get_pageinfo(infile, pageno: int):
 
 def pdf_get_all_pageinfo(infile):
     pdf = pypdf.PdfFileReader(infile)
-    getcontext().prec = 6
     return [_pdf_get_pageinfo(infile, n) for n in range(pdf.numPages)]
 
 

ocrmypdf/pdfa.py

@@ -140,21 +140,20 @@ def file_claims_pdfa(filename):
                 'conformance': 'No XMP metadata'}
 
     pdfa_dict = {attr.localName: attr.value for attr in pdfa_nodes}
-    pdfa_dict['pass'] = False
-    pdfa_dict['output'] = 'pdf'
-    if pdfa_dict:
-        part_conformance = pdfa_dict['part'] + pdfa_dict['conformance']
-        valid_part_conforms = {'1A', '1B', '2A', '2B', '2U', '3A', '3B', '3U'}
+    if not pdfa_dict:
+        return {'pass': False, 'output': 'pdf',
+                'conformance': 'No XMP metadata'}
 
-        conformance = 'PDF/A-{}'.format(
-            part_conformance)
+    part_conformance = pdfa_dict['part'] + pdfa_dict['conformance']
+    valid_part_conforms = {'1A', '1B', '2A', '2B', '2U', '3A', '3B', '3U'}
 
-        if part_conformance in valid_part_conforms:
-            pdfa_dict['pass'] = True
-            pdfa_dict['output'] = 'pdfa'
-        pdfa_dict['conformance'] = conformance
-    else:
-        pdfa_dict['conformance'] = 'PDF'
+    conformance = 'PDF/A-{}'.format(
+        part_conformance)
+
+    if part_conformance in valid_part_conforms:
+        pdfa_dict['pass'] = True
+        pdfa_dict['output'] = 'pdfa'
+    pdfa_dict['conformance'] = conformance
 
     return pdfa_dict
 

ocrmypdf/tesseract.py

@@ -147,11 +147,16 @@ def page_timedout(log, input_file):
     log.warning(prefix + " took too long to OCR - skipping")
 
 
-def _generate_null_hocr(output_hocr, pageinfo):
+def _generate_null_hocr(output_hocr, image):
+    """Produce a .hocr file that reports no text detected on a page that is
+    the same size as the input image."""
+    from PIL import Image
+
+    im = Image.open(image)
+    w, h = im.size
+
     with open(output_hocr, 'w', encoding="utf-8") as f:
-        f.write(HOCR_TEMPLATE.format(
-            pageinfo['width_pixels'],
-            pageinfo['height_pixels']))
+        f.write(HOCR_TEMPLATE.format(w, h))
 
 
 def generate_hocr(input_file, output_hocr, language: list, tessconfig: list,
@@ -181,11 +186,11 @@ def generate_hocr(input_file, output_hocr, language: list, tessconfig: list,
         # Temporary workaround to hocrTransform not being able to function if
         # it does not have a valid hOCR file.
         page_timedout(log, input_file)
-        _generate_null_hocr(output_hocr, pageinfo_getter())
+        _generate_null_hocr(output_hocr, input_file)
     except CalledProcessError as e:
         tesseract_log_output(log, e.output, input_file)
         if 'Image too large' in e.output:
-            _generate_null_hocr(output_hocr, pageinfo_getter())
+            _generate_null_hocr(output_hocr, input_file)
             return
 
         raise e from e

requirements.txt

@@ -2,8 +2,8 @@
 # setup.py lists a separate set of requirements that are looser to simplify
 # installation
 ruffus==2.6.3
-Pillow==3.3.0
-reportlab==3.2.0
+Pillow==4.0.0
+reportlab==3.3.0
 PyPDF2==1.26
 img2pdf==0.2.1
-cffi==1.5.2
+cffi==1.9.1

setup.py

@@ -191,11 +191,14 @@ setup(
     url='https://github.com/jbarlow83/OCRmyPDF',
     author='James R. Barlow',
     author_email='jim@purplerock.ca',
-    license='Public Domain',
+    license='MIT',
     packages=['ocrmypdf'],
     keywords=['PDF', 'OCR', 'optical character recognition', 'PDF/A', 'scanning'],
     classifiers=[
         "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.4",
+        "Programming Language :: Python :: 3.5",
+        "Programming Language :: Python :: 3.6",
         "Development Status :: 5 - Production/Stable",
         "Environment :: Console",
         "Intended Audience :: End Users/Desktop",

test_requirements.txt

@@ -1 +1 @@
-pytest>=2.7.2
+pytest >= 2.7.2

tests/resources/README.rst

@@ -31,6 +31,9 @@ In some cases they were converted from one image format to another without other
     *   - LinnSequencer.jpg, linn.pdf, linn.txt
         - `Wikimedia: LinnSequencer`_
         - Creative Commons BY-SA 3.0
+    *   - typewriter.png, 2400dpi.pdf
+        - `Wikimedia: Triumph typewrtier text Linzensoep`_
+        * Creative Commons BY-SA 2.5
 
 
 Files generated for this project
@@ -82,14 +85,15 @@ Assemblies
 
 These test resources are assemblies from other previously mentioned files, released under the same license terms as their input files.
 
-- cardinal.pdf (four cardinal directions, rotated copies of LinnSequencer.jpg)
+- cardinal.pdf (four cardinal directions, baked-in rotated copies of LinnSequencer.jpg)
 - ccitt.pdf (LinnSequencer.jpg, converted to CCITT encoding)
 - encrypted_algo4.pdf (congress.jpg, encrypted with algorithm 4 - not supported by PyPDF2)
 - graph_ocred.pdf (from graph.pdf)
 - jbig2.pdf (congress.jpg, converted to JBIG2 encoding)
 - multipage.pdf (from several other files)
 - palette.pdf (congress.jpg, converted to a 256-color palette)
-- skew.pdf (from c02-22.pdf)
+- rotated_skew.pdf (a /Rotate'd and skewed document from LinnSequencer.jpg)
+- skew.pdf (from LinnSequencer.jpg, skew simulated by adjusting the transformation matrix)
 - skew-encrypted.pdf (skew.pdf with encryption - access supported by PyPDF2)
 
 
@@ -103,4 +107,6 @@ These test resources are assemblies from other previously mentioned files, relea
 
 .. _`Wikimedia: JPEG2000 Lichtenstein`: https://en.wikipedia.org/wiki/JPEG_2000#/media/File:Jpeg2000_2-level_wavelet_transform-lichtenstein.png
 
-.. _`Linux (Wikipedia Article)`: https://de.wikipedia.org/wiki/Linux 
+.. _`Linux (Wikipedia Article)`: https://de.wikipedia.org/wiki/Linux 
+
+.. _`Wikimedia: Triumph typewrtier text Linzensoep`: https://commons.wikimedia.org/wiki/File:Triumph.typewriter_text_Linzensoep.gif

tests/spoof/gs_feature_elision.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+import sys
+import os
+from subprocess import check_call
+
+"""Replicate one type of Ghostscript feature elision warning during
+PDF/A creation."""
+
+
+def real_ghostscript(argv):
+    gs_args = ['gs'] + argv[1:]
+    os.execvp("gs", gs_args)
+    return  # Not reachable
+
+
+elision_warning = """GPL Ghostscript 9.20: Setting Overprint Mode to 1
+not permitted in PDF/A-2, overprint mode not set"""
+
+
+def main():
+    if '--version' in sys.argv:
+        print('9.20')
+        print('SPOOFED: ' + os.path.basename(__filename__))
+        sys.exit(0)
+
+    gs_args = ['gs'] + sys.argv[1:]
+    check_call(gs_args)
+
+    if '-sDEVICE=pdfwrite' in sys.argv[1:]:
+        print(elision_warning)
+
+    sys.exit(0)
+
+if __name__ == '__main__':
+    main()

tests/spoof/gs_pdfa_failure.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+import sys
+import os
+
+"""Replicate Ghostscript PDF/A conversion failure by suppressing some
+arguments"""
+
+
+def real_ghostscript(argv):
+    gs_args = ['gs'] + argv[1:]
+    os.execvp("gs", gs_args)
+    return  # Not reachable
+
+
+def main():
+    if '--version' in sys.argv:
+        print('9.20')
+        print('SPOOFED: ' + os.path.basename(__filename__))
+        sys.exit(0)
+
+    # Unless some argument is calling for PDFA generation, forward to
+    # real ghostscript
+    if not any(arg.startswith('-dPDFA') for arg in sys.argv):
+        real_ghostscript(sys.argv)
+        return
+
+    # Remove the two arguments that tell ghostscript to create a PDF/A
+    # Does not remove the Postscript definition file - not necessary
+    # to cause PDF/A creation failure
+    argv = []
+    for arg in sys.argv:
+        if arg.startswith('-dPDFA'):
+            continue
+        elif arg.startswith('-dPDFACompatibilityPolicy'):
+            continue
+        argv.append(arg)
+
+    real_ghostscript(argv)
+
+
+if __name__ == '__main__':
+    main()

tests/spoof/tesseract_cache.py

@@ -48,11 +48,17 @@ def main():
 
     m = hashlib.sha1()
 
-    version = subprocess.check_output(
+    tess_version = subprocess.check_output(
         ['tesseract', '--version'],
         stderr=subprocess.STDOUT)
 
-    m.update(version)
+    m.update(tess_version)
+
+    # Insert this source file into the hash function, to ensure that any
+    # changes to this file invalidate previous hashes
+    with open(__file__, 'rb') as f:
+        m.update(f.read())
+
     m.update(operation.encode())
 
     try:
@@ -93,10 +99,10 @@ def main():
             shutil.copy(cache_name, output_file)
 
         # Replicate output
-        with open(cache_name + '.stdout', 'r') as f:
-            print(f.read(), end='')
-        with open(cache_name + '.stderr', 'r') as f:
-            print(f.read(), end='', file=sys.stderr)
+        with open(cache_name + '.stdout', 'rb') as f:
+            sys.stdout.buffer.write(f.read())
+        with open(cache_name + '.stderr', 'rb') as f:
+            sys.stderr.buffer.write(f.read())
         sys.exit(0)
 
     # Cache miss
@@ -105,23 +111,22 @@ def main():
     # Call tesseract
     p = subprocess.Popen(
             ['tesseract'] + sys.argv[1:],
-            stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-            universal_newlines=True)
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     stdout, stderr = p.communicate()
 
     if p.returncode != 0:
         # Do not cache errors or crashes
         print("Tesseract error", file=sys.stderr)
-        print(stdout, end='')
-        print(stderr, end='', file=sys.stderr)
+        sys.stdout.buffer.write(stdout)
+        sys.stderr.buffer.write(stderr)
         return p.returncode
 
-    with open(cache_name + '.stdout', 'w') as f:
+    with open(cache_name + '.stdout', 'wb') as f:
         f.write(stdout)
-    with open(cache_name + '.stderr', 'w') as f:
+    with open(cache_name + '.stderr', 'wb') as f:
         f.write(stderr)
-    print(stdout, end='')
-    print(stderr, end='', file=sys.stderr)
+    sys.stdout.buffer.write(stdout)
+    sys.stderr.buffer.write(stderr)
 
     # Insert file into cache
     if output_file != 'stdout':

tests/test_main.py

@@ -60,6 +60,7 @@ def check_ocrmypdf(input_basename, output_basename, *args, env=None):
     output_file = _outfile(output_basename)
 
     p, out, err = run_ocrmypdf(input_basename, output_basename, *args, env=env)
+    print(err)  # ensure py.test collects the output, use -s to view
     if p.returncode != 0:
         print('stdout\n======')
         print(out)
@@ -90,41 +91,56 @@ def run_ocrmypdf(input_basename, output_basename, *args, env=None):
     return p, out, err
 
 
-def spoof(replace_program, with_spoof):
+def spoof(**kwargs):
     """Modify environment variables to override subprocess executables
 
+    spoof(program1='replacement', ...)
+
     Before running any executable, ocrmypdf checks the environment variable
     OCRMYPDF_PROGRAMNAME to override default program name/location, e.g.
     OCRMYPDF_GS redirects from the system path Ghostscript ("gs") to elsewhere.
+
     """
     env = os.environ.copy()
-    spoofer = os.path.join(SPOOF_PATH, with_spoof)
-    if not os.access(spoofer, os.X_OK):
-        os.chmod(spoofer, 0o755)
-    env['OCRMYPDF_' + replace_program.upper()] = spoofer
+
+    for replace_program, with_spoof in kwargs.items():
+        spoofer = os.path.join(SPOOF_PATH, with_spoof)
+        if not os.access(spoofer, os.X_OK):
+            os.chmod(spoofer, 0o755)
+        env['OCRMYPDF_' + replace_program.upper()] = spoofer
     return env
 
 
 @pytest.fixture
 def spoof_tesseract_noop():
-    return spoof('tesseract', 'tesseract_noop.py')
+    return spoof(tesseract='tesseract_noop.py')
 
 
 @pytest.fixture
 def spoof_tesseract_cache():
     if running_in_docker():
         return os.environ.copy()
-    return spoof('tesseract', "tesseract_cache.py")
+    return spoof(tesseract="tesseract_cache.py")
 
 
 @pytest.fixture
 def spoof_tesseract_crash():
-    return spoof('tesseract', 'tesseract_crash.py')
+    return spoof(tesseract='tesseract_crash.py')
 
 
 @pytest.fixture
 def spoof_tesseract_big_image_error():
-    return spoof('tesseract', 'tesseract_big_image_error.py')
+    return spoof(tesseract='tesseract_big_image_error.py')
+
+
+@pytest.fixture
+def spoof_no_tess_no_pdfa():
+    return spoof(tesseract='tesseract_noop.py', gs='gs_pdfa_failure.py')
+
+
+@pytest.fixture
+def spoof_no_tess_pdfa_warning():
+    return spoof(tesseract='tesseract_noop.py', gs='gs_feature_elision.py')
 
 
 def test_quick(spoof_tesseract_cache):
@@ -688,3 +704,54 @@ def test_linearized_pdf_and_indirect_object(spoof_tesseract_noop):
     check_ocrmypdf(
         'epson.pdf', 'test_epson.pdf',
         env=spoof_tesseract_noop)
+
+
+def test_rotated_skew_timeout():
+    """This document contains an image that is rotated 90 into place with a
+    /Rotate tag and intentionally skewed by altering the transformation matrix.
+
+    This tests for a bug where the combinatino of preprocessing and a tesseract
+    timeout produced a page whose dimensions did not match the original's.
+    """
+
+    input_file = _infile('rotated_skew.pdf')
+    in_pageinfo = pdf_get_all_pageinfo(input_file)[0]
+
+    assert in_pageinfo['height_pixels'] < in_pageinfo['width_pixels'], \
+        "Expected the input page to be landscape"
+    assert in_pageinfo['rotate'] == 90, "Expected a rotated page"
+
+    out = check_ocrmypdf(
+        'rotated_skew.pdf', 'test_rotated_skew.pdf',
+        '--pdf-renderer', 'hocr',
+        '--deskew', '--tesseract-timeout', '0')
+
+    out_pageinfo = pdf_get_all_pageinfo(out)[0]
+
+    assert out_pageinfo['height_pixels'] > out_pageinfo['width_pixels'], \
+        "Expected the output page to be portrait"
+
+    assert out_pageinfo['rotate'] == 0, \
+        "Expected no page rotation for output"
+
+    assert in_pageinfo['width_pixels'] == out_pageinfo['height_pixels'] and \
+        in_pageinfo['height_pixels'] == out_pageinfo['width_pixels'], \
+        "Expected page rotation to be baked in"
+
+
+def test_ghostscript_pdfa_failure(spoof_no_tess_no_pdfa):
+    p, out, err = run_ocrmypdf(
+        'ccitt.pdf', 'test_pdfa_failure.pdf',
+        env=spoof_no_tess_no_pdfa)
+    assert p.returncode == 4, "Expected return code 4 when PDF/A fails"
+
+
+def test_ghostscript_feature_elision(spoof_no_tess_pdfa_warning):
+    check_ocrmypdf('ccitt.pdf', 'test_feature_elision.pdf',
+                   env=spoof_no_tess_pdfa_warning)
+
+
+def test_very_high_dpi(spoof_tesseract_cache):
+    "Checks for a Decimal quantize error with high DPI, etc"
+    check_ocrmypdf('2400dpi.pdf', 'test_2400dpi.pdf',
+                   env=spoof_tesseract_cache)

tests/test_pageinfo.py

@@ -97,11 +97,6 @@ def test_single_page_image():
     assert pdfimage['width'] == 8
     assert pdfimage['color'] == 'gray'
 
-    # While unexpected, this is correct
-    # PDF spec says /FlateDecode image must have /BitsPerComponent 8
-    # So mono images get upgraded to 8-bit
-    assert pdfimage['bpc'] == 8
-
     # DPI in a 1"x1" is the image width
     assert abs(pdfimage['dpi_w'] - 8) < 1e-5
     assert abs(pdfimage['dpi_h'] - 8) < 1e-5