Tweak release notes

Update documentation to match.

.docker/Dockerfile

@@ -1,7 +1,6 @@
 # OCRmyPDF
 #
-FROM      ubuntu:17.04
-MAINTAINER James R. Barlow <jim@purplerock.ca>
+FROM      ubuntu:17.10
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
   software-properties-common python-software-properties \
@@ -52,4 +51,4 @@ WORKDIR /home/docker
 
 # Must use array form of ENTRYPOINT
 # Non-array form does not append other arguments, because that is "intuitive"
-ENTRYPOINT ["/application/docker-wrapper.sh"]
+ENTRYPOINT ["/application/.docker/docker-wrapper.sh"]

.docker/Dockerfile.polyglot

@@ -14,4 +14,4 @@ USER docker
 
 # Must use array form of ENTRYPOINT
 # Non-array form does not append other arguments, because that is "intuitive"
-ENTRYPOINT ["/application/docker-wrapper.sh"]
+ENTRYPOINT ["/application/.docker/docker-wrapper.sh"]

.docker/Dockerfile.tess4

@@ -1,7 +1,6 @@
 # OCRmyPDF
 #
-FROM 	ubuntu:17.04
-MAINTAINER James R. Barlow <jim@purplerock.ca>
+FROM 	ubuntu:17.10
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
   software-properties-common python-software-properties \
@@ -62,4 +61,4 @@ WORKDIR /home/docker
 
 # Must use array form of ENTRYPOINT
 # Non-array form does not append other arguments, because that is "intuitive"
-ENTRYPOINT ["/application/docker-wrapper.sh"]
+ENTRYPOINT ["/application/.docker/docker-wrapper.sh"]

.dockerignore

@@ -3,18 +3,9 @@
 *.pyc
 *.rst
 *.sublime*
-*/*.pyc
-*/*/*.pyc
-*/*/*/*.pyc
-*/*/*/*/*.pyc
-*/*/*/*/*/*.pyc
-*/*/*/*/*/*/*.pyc
-*/*/*/*/*/*/*/*.pyc
-.cache/
-.eggs/
-.github/
-.git/
-.ipynb_checkpoints/
+**/*.pyc
+.*/
+!.docker/
 .ruffus_history.sqlite
 bin/
 build/

.gitignore

@@ -7,6 +7,7 @@ tasks.py
 .bash_history
 .ruffus_history.sqlite
 .idea/
+.pytest_cache/
 
 # Package building
 *.egg-info/

.travis.yml

@@ -34,7 +34,9 @@ before_install: |
   if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
     bash .travis/linux_before_install.sh
   elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-    bash .travis/osx_before_install.sh
+    brew update && brew bundle --file=.travis/Brewfile
+    pip3 install --upgrade pip
+    pip3 install wheel
   fi
 
 install:

.travis/Brewfile

@@ -0,0 +1,12 @@
+brew 'python'
+
+brew 'ghostscript'
+brew 'jbig2dec'
+brew 'leptonica'
+brew 'libffi'
+brew 'libtiff'
+brew 'libxml2'
+brew 'openjpeg'
+brew 'qpdf'
+brew 'tesseract'
+brew 'unpaper'

.travis/autobrew.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# © 2016-7 James R. Barlow: github.com/jbarlow83
+# © 2017-18 James R. Barlow: github.com/jbarlow83
 
 from string import Template
 from subprocess import run, PIPE
@@ -14,12 +14,12 @@ class Ocrmypdf < Formula
   ${ocrmypdf_url}
   ${ocrmypdf_sha256}
 
+  depends_on "pkg-config" => :build
   depends_on "freetype"
   depends_on "ghostscript"
   depends_on "jpeg"
   depends_on "libpng"
-  depends_on "pkg-config" => :build
-  depends_on "python3"
+  depends_on "python"
   depends_on "qpdf"
   depends_on "tesseract"
   depends_on "unpaper"
@@ -55,28 +55,9 @@ ${resources}
     # Since we use Python 3, we require a UTF-8 locale
     ENV["LC_ALL"] = "en_US.UTF-8"
 
-    # One page Postscript with the wording "Testing" on the page
-    # This is more compact than including a test PDF
-    (testpath/"test.ps").write(
-      <<~EOS
-        %!PS
-        /Times-Roman findfont
-        20 scalefont
-        setfont
-        gsave
-        newpath
-        200 400 moveto
-        (Testing) show
-        closepath
-        stroke
-        showpage
-      EOS
-    )
-
-    system "#{Formula["ghostscript"].opt_bin}/ps2pdf", testpath/"test.ps", testpath/"test.pdf"
-
-    # Use ocrmypdf -f to rasterize the PDF to image before doing OCR
-    system "#{bin}/ocrmypdf", "-f", "-q", "--deskew", testpath/"test.pdf", testpath/"ocr.pdf"
+    system "#{bin}/ocrmypdf", "-f", "-q", "--deskew",
+                              test_fixtures("test.pdf"), "ocr.pdf"
+    assert_predicate testpath/"ocr.pdf", :exist?
   end
 end
 """)

.travis/osx_before_install.sh

@@ -1,17 +0,0 @@
-#!/bin/bash
-# © 2017 James R. Barlow: github.com/jbarlow83
-set -euo pipefail
-set -x
-
-brew update
-
-brew install openjpeg jbig2dec libtiff     # image libraries
-brew install qpdf
-brew install ghostscript
-brew install python3
-brew install libxml2 libffi leptonica
-brew install unpaper   # optional
-brew install tesseract
-
-pip3 install --upgrade pip
-pip3 install wheel

README.rst

@@ -74,11 +74,10 @@ Users of Debian 9 or later or Ubuntu 16.10 or later may simply
 
    apt-get install ocrmypdf
 
-and macOS users may simply
+and macOS users with Homebrew may simply
 
 .. code-block:: bash
 
-   brew tap jbarlow83/ocrmypdf
    brew install ocrmypdf
 
 For everyone else, `see our documentation <https://ocrmypdf.readthedocs.io/en/latest/installation.html>`_ for installation steps.
@@ -127,7 +126,7 @@ Requirements
 
 Runs on CPython 3.6, and requires external program installations of Ghostscript, Tesseract OCR, QPDF, and Leptonica. ocrmypdf is pure Python, but uses CFFI to portably generate library bindings.
 
-Python 3.5 is fully supported, but support will end at the end of 2017, to allow f-strings in the code base.
+Python 3.5 is also supported.
 
 Press & Media
 -------------

docs/advanced.rst

@@ -43,7 +43,7 @@ OCRmyPDF checks the environment variable ``OCRMYPDF_TESSERACT`` for the full pat
 
 	A Tesseract environment variable that overrides the path to Tesseract's data files.
 
-For example, if you are testing tesseract 4.00 and don't wish to disturb use an existing tesseract 3.04 installation, you can launch OCRmyPDF as follows:
+For example, if you are testing tesseract 4.00 and don't wish to use an existing tesseract 3.04 installation, you can launch OCRmyPDF as follows:
 
 .. code-block:: bash
 
@@ -115,7 +115,7 @@ rendering
   Creating a new PDF from other data (such as an existing PDF).
 
 
-OCRmyPDF has three PDF renderers: ``sandwich``, ``hocr``, ``tesseract``. The renderer may be selected using ``--pdf-renderer``. The default is ``auto`` which lets OCRmyPDF select the renderer to use. Currently, ``auto`` selects ``sandwich`` for Tesseract 3.05.01, and newer, ``hocr`` for older versions of Tesseract.
+OCRmyPDF has three PDF renderers: ``sandwich``, ``hocr``, ``tesseract``. The renderer may be selected using ``--pdf-renderer``. The default is ``auto`` which lets OCRmyPDF select the renderer to use. Currently, ``auto`` selects ``sandwich`` for Tesseract 3.05.01 or newer, or ``hocr`` for older versions of Tesseract.
 
 The ``sandwich`` renderer
 """""""""""""""""""""""""
@@ -136,7 +136,7 @@ This works in all versions of Tesseract.
 The ``tesseract`` renderer
 """"""""""""""""""""""""""
 
-The ``tesseract`` renderer creates a PDF with the image and text layers precomposed, meaning that it always transcodes, loses image quality and rasterizes and vector objects. It does a better job on non-Latin text and document structure than ``hocr``.
+The ``tesseract`` renderer creates a PDF with the image and text layers precomposed, meaning that it always transcodes, loses image quality and rasterizes any vector objects. It does a better job on non-Latin text and document structure than ``hocr``.
 
 If a PDF created with this renderer using Tesseract versions older than 3.05.00 is then passed through Ghostscript's pdfwrite feature, the OCR text *may* be corrupted. The ``--output-type=pdfa`` argument will produce a warning in this situation.
 

docs/index.rst

@@ -9,7 +9,7 @@ OCRmyPDF documentation
 OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to
 be searched.
 
-PDFs are the best format for scanned documents.  Unfortunately, PDFs can be difficult to work with. OCRmyPDF makes it easy to apply image processing and OCR to existing PDFs.
+PDF is the best format for storing and exchanging scanned documents.  Unfortunately, PDFs can be difficult to modify. OCRmyPDF makes it easy to apply image processing and OCR to existing PDFs.
 
 .. toctree::
    :maxdepth: 1

docs/installation.rst

@@ -15,18 +15,20 @@ Users of Debian 9 ("stretch") or later or Ubuntu 16.10 or later may simply
 Installing on macOS
 -------------------
 
-A `Homebrew <https://brew.sh>`_ tap is available for macOS:
+OCRmyPDF is now a standard `Homebrew <https://brew.sh>`_ formula. To install on macOS:
 
 .. code-block:: bash
 
-    brew tap jbarlow83/ocrmypdf
     brew install ocrmypdf
 
-
-.. warning::
+.. note::
 
     Users who previously installed OCRmyPDF on macOS using ``pip install ocrmypdf`` should remove the pip version (``pip3 uninstall ocrmypdf``) before switching to the Homebrew version.
 
+.. note::
+
+    Users who previously installed OCRmyPDF from the private tap should switch to the mainline version (``brew untap jbarlow83/ocrmypdf``) and install from there.
+
 .. _Docker-install:
 
 Installing the Docker image
@@ -285,6 +287,8 @@ Now we need to install ``pip`` and let it install ocrmypdf:
     wget -O - -o /dev/null https://bootstrap.pypa.io/get-pip.py | python3.6
     pip3.6 install ocrmypdf
 
+The ``wget`` command will download a program and run it.
+
 These installation instructions omit the optional dependency ``unpaper``, which is only available at version 0.4.2 in Ubuntu 14.04. The author could not find a backport of ``unpaper``, and created a .deb package to do the job of installing unpaper 6.1 (for x86 64-bit only):
 
 .. code-block:: bash

docs/introduction.rst

@@ -16,7 +16,7 @@ OCRmyPDF uses `Tesseract <https://github.com/tesseract-ocr/tesseract>`_, the bes
 About PDFs
 ----------
 
-PDFs are page description files that attempts to preserve a layout exactly. They can contain `vector graphic files <http://vector-conversions.com/vectorizing/raster_vs_vector.html>`_ that can contain raster objects such as scanned images. Because PDFs can contain multiple pages (unlike many image formats) and can contain fonts and text, it is a good formats for exchanging scanned documents.
+PDFs are page description files that attempts to preserve a layout exactly. They  contain `vector graphics <http://vector-conversions.com/vectorizing/raster_vs_vector.html>`_ that can contain raster objects such as scanned images. Because PDFs can contain multiple pages (unlike many image formats) and can contain fonts and text, it is a good formats for exchanging scanned documents.
 
 .. image:: bitmap_vs_svg.svg
 
@@ -42,15 +42,13 @@ PDF/A has a few drawbacks.  Some PDF viewers include an alert that the file is a
 What OCRmyPDF does
 ------------------
 
-OCRmyPDF analyzes each page of a PDF to determine the colorspace and resolution (DPI) needed to capture all of the information on that page without losing content.  It uses `Ghostscript <http://ghostscript.com/>`_ to rasterize the page, and then performs on OCR on the rasterized image.  It is not enough to simply extract the images from each page and run OCR on them individually.  Of course one could use Ghostscript or another PDF rasterizer and then pass the image to Tesseract.  OCRmyPDF automates this process and produces a minimally changed output file that contains the same information, colorspace and resolution.
+OCRmyPDF analyzes each page of a PDF to determine the colorspace and resolution (DPI) needed to capture all of the information on that page without losing content.  It uses `Ghostscript <http://ghostscript.com/>`_ to rasterize the page, and then performs on OCR on the rasterized image to create an OCR "layer". The layer is then grafted back onto the original PDF.
 
-The Tesseract OCR engine can output 'hOCR' files, which are XML files that contain a description of the text it found on the page.  OCRmyPDF will render a new PDF that contains only the hidden text layer, and merge this with the original page.
+While one can use a program like Ghostscript or ImageMagick to get an image and put the image through Tesseract, that actually creates a new PDF and many details may be lost. OCRmyPDF can produce a minimally changed PDF as output.
 
-Alternately, OCRmyPDF can use the Tesseract OCR engine to directly output PDFs for each page, then merge them.
+OCRmyPDF also some image processing options like deskew which improve the appearance of files and quality of OCR. When these are used, the OCR layer is grafted onto the processed image instead.
 
-By default, OCRmyPDF will convert the file to a PDF/A.  This behavior can be disabled with the ``--output-type pdf`` argument.
-
-Depending on the settings selected, OCRmyPDF may "graft" the OCR layer into the existing PDF, or reconstruct a visually equivalent new PDF.
+By default, OCRmyPDF produces archival PDFs – PDF/A, which are a stricter subset of PDF features designed for long term archives. If regular PDFs are desired, this can be disabled with ``--output-type pdf``.
 
 
 Why you shouldn't do this manually
@@ -85,6 +83,7 @@ OCRmyPDF is limited by the Tesseract OCR engine.  As such it experiences these l
 OCRmyPDF is also limited by the PDF specification:
 
 * PDF encodes the position of text glyphs but does not encode document structure.  There is no markup that divides a document in sections, paragraphs, sentences, or even words (since blank spaces are not represented). As such all elements of document structure including the spaces between words must be derived heuristically.  Some PDF viewers do a better job of this than others.
+* Because some popular open source PDF viewers have a particularly hard time with spaces betweem words, OCRmyPDF appends a space to each text element as a workaround. While this mixes document structure with graphical information that ideally should be left to the PDF viewer to interpret, it improves compatibility with some viewers and does not cause problems for better ones.
 
 Ghostscript also imposes some limitations:
 

docs/release_notes.rst

@@ -5,6 +5,49 @@ OCRmyPDF uses `semantic versioning <http://semver.org/>`_ for its command line i
 
 The OCRmyPDF package itself does not contain a public API, although it is fairly stable and breaking changes are usually timed with a major release. A future release will clearly define the stable public API.
 
+v5.7.0
+------
+
+-   Fixed an issue that caused poor CPU utilization on machines more than 4 cores when running Tesseract 4. (Related to issue #217.)
+
+-   The 'hocr' renderer has been improved. The 'sandwich' and 'tesseract' renderers are still better for most use cases, but 'hocr' may be useful for people who work with the PDF.js renderer in English/ASCII languages. (#225)
+
+    + It now formats text in a matter that is easier for certain PDF viewers to select and extract copy and paste text. This should help macOS Preview and PDF.js in particular.
+    + The appearance of selected text and behavior of selecting text is improved.
+    + The PDF content stream now uses relative moves, making it more compact and easier for viewers to determine when two words on the same line.
+    + It can now deal with text on a skewed baseline.
+    + Thanks to @cforcey for the pull request, @jbreiden for many helpful suggestions, @ctbarbour for another round of improvements, and @acaloiaro for an independent review.
+
+v5.6.3
+------
+
+-   Suppress two debug messages that were too verbose
+
+
+v5.6.2
+------
+
+-   Development branch accidentally tagged as release. Do not use.
+
+
+v5.6.1
+------
+
+-   Fix issue #219: change how the final output file is created to avoid triggering permission errors when the output is a special file such as ``/dev/null``
+-   Fix test suite failures due to a qpdf 8.0.0 regression and Python 3.5's handling of symlink
+-   The "encrypted PDF" error message was different depending on the type of PDF encryption. Now a single clear message appears for all types of PDF encryption.
+-   ocrmypdf is now in Homebrew. Homebrew users are advised to the version of ocrmypdf in the official homebrew-core formulas rather than the private tap.
+-   Some linting
+
+
+v5.6.0
+------
+
+-   Fix issue #216: preserve "text as curves" PDFs without rasterizing file
+-   Related to the above, messages about rasterizing are more consistent
+-   For consistency versions minor releases will now get the trailing .0 they always should have had.
+
+
 v5.5
 ----
 

docs/security.rst

@@ -30,14 +30,16 @@ Depending on the options specified, OCRmyPDF may graft the OCR layer into the ex
 Using OCRmyPDF online or as a service
 -------------------------------------
 
-OCRmyPDF should not be deployed as a public-facing service, like a website where a potential attacker could upload a PDF of their choice for OCR. OCRmyPDF is not designed to be secure against PDF malware. Another concern is PDFs specifically designed to be a denial of service attack: PDFs can contain recursive data structures that sometimes send parsers into infinite loops, and issue complex graphics drawing commands.
+OCRmyPDF should not be deployed as a public-facing service, such as a website where a potential attacker could upload a PDF of their choice for OCR. OCRmyPDF is not designed to be secure against PDF malware. Another concern is PDFs specifically designed to be a denial of service attack: PDFs can contain recursive data structures that sometimes send parsers into infinite loops, and issue complex graphics drawing commands.
+
+Setting aside these concerns, a side effect of OCRmyPDF is it may incidentally sanitize PDFs that contain malware. It runs ``qpdf`` to repair the PDF, which could correct malformed PDF structures that are part of an attack. When PDF/A output is selected (the default), the input PDF is partially reconstructed by Ghostscript. When ``--force-ocr`` is used, all pages are rasterized and reconverted to PDF, which could remove malware in embedded images. No guarantees.
 
 OCRmyPDF should be relatively safe to use in a trusted intranet, with some considerations:
 
 Limiting CPU usage
 ^^^^^^^^^^^^^^^^^^
 
-OCRmyPDF will attempt to use all available CPUs and storage, so executing ``nice ocrmypdf`` or limiting the number of jobs with the ``-j`` argument may ensure the server remains available. Another option would be run OCRmyPDF jobs inside a Docker container or virtual machine, which can impose its own limits on CPU usage.
+OCRmyPDF will attempt to use all available CPUs and storage, so executing ``nice ocrmypdf`` or limiting the number of jobs with the ``-j`` argument may ensure the server remains available. Another option would be run OCRmyPDF jobs inside a Docker container, a virtual machine, or a cloud instance, which can impose its own limits on CPU usage and be terminated "from orbit" if it fails to complete.
 
 Temporary storage requirements
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -46,7 +48,7 @@ OCRmyPDF will use a large amount of temporary storage for its work, proportional
 
 To check temporary storage usage on actual files, run ``ocrmypdf -k ...`` which will preserve and print the path to temporary storage when the job is done.
 
-To change where temporary files are stored, change the ``TMPDIR`` environment variable for ocrmypdf's environment. (Python's ``tempfile.gettempdir()`` returns the root directory in which temporary files will be stored.)
+To change where temporary files are stored, change the ``TMPDIR`` environment variable for ocrmypdf's environment. (Python's ``tempfile.gettempdir()`` returns the root directory in which temporary files will be stored.) For example, one could redirect ``TMPDIR`` to a large RAM disk to avoid wear on HDD/SSD and potentially improve performance. On Amazon Web Services, ``TMPDIR`` can be set to `empheral storage <https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/InstanceStorage.html>`_.
 
 Timeouts
 ^^^^^^^^
@@ -64,7 +66,13 @@ Abbyy Cloud OCR is a viable commercial alternative with a web services API.
 Password protection, digital signatures and certification
 ---------------------------------------------------------
 
-OCRmyPDF cannot remove password protection from a PDF. ``qpdf``, one of its dependencies, has this capability. After OCR is applied, password protection is not permitted on PDF/A documents but the file can be converted to regular PDF.
+Password protected PDFs usually have two passwords, and owner and user password. When the user password is set to empty, PDF readers will open the file automatically and marked it as "(SECURED)". While not as reliable as a digital signature, this indicates that whoever set the password approved of the file at that time. When the user password is set, the document cannot be viewed without the password. 
+
+Either way, OCRmyPDF does not remove passwords from PDFs and exits with an error on encountering them.
+
+``qpdf``, one of OCRmyPDF's dependencies, can remove passwords. If the owner and user password are set, a password is required for ``qpdf``. If only the owner password is set, then the password can be stripped, even if one does not have the owner password.
+
+After OCR is applied, password protection is not permitted on PDF/A documents but the file can be converted to regular PDF.
 
 Many programs exist which are capable of inserting an image of someone's signature. On its own, this offers no security guarantees. It is trivial to remove the signature image and apply it to other files. This practice offers no real security.
 

ocrmypdf/__main__.py

@@ -27,7 +27,8 @@ from .helpers import is_iterable_notstr, re_symlink, is_file_writable
 from .exec import tesseract, qpdf, ghostscript
 from . import PROGRAM_NAME, VERSION
 
-from .exceptions import *
+from .exceptions import ExitCode, ExitCodeException, MissingDependencyError, \
+    InputFileError, BadArgsError, OutputFileAccessError
 from . import exceptions as ocrmypdf_exceptions
 from ._unicodefun import verify_python3_env
 
@@ -479,6 +480,12 @@ def check_options_metadata(options, log):
                 ))
 
 
+def check_options_pillow(options, log):
+    PIL.Image.MAX_IMAGE_PIXELS = int(options.max_image_mpixels * 1000000)
+    if PIL.Image.MAX_IMAGE_PIXELS == 0:
+        PIL.Image.MAX_IMAGE_PIXELS = None
+
+
 def check_options(options, log):
     try:
         check_options_languages(options, log)
@@ -488,6 +495,7 @@ def check_options(options, log):
         check_options_preprocessing(options, log)
         check_options_ocr_behavior(options, log)
         check_options_advanced(options, log)
+        check_options_pillow(options, log)
     except ValueError as e:
         log.error(e)
         sys.exit(ExitCode.bad_args)
@@ -554,6 +562,7 @@ def do_ruffus_exception(ruffus_five_tuple, options, log):
     exit_code = None
 
     task_name, job_name, exc_name, exc_value, exc_stack = ruffus_five_tuple
+    job_name = job_name  # unused
     if exc_name == 'builtins.SystemExit':
         match = re.search(r"\.(.+?)\)", exc_value)
         exit_code_name = match.groups()[0]
@@ -576,6 +585,21 @@ def do_ruffus_exception(ruffus_five_tuple, options, log):
         msg = "Error occurred while running this command:"
         log.error(msg + '\n' + exc_value)
         exit_code = ExitCode.child_process_error
+    elif (exc_name == 'PyPDF2.utils.PdfReadError' and \
+            'not been decrypted' in exc_value) or \
+            (exc_name == 'ocrmypdf.exceptions.EncryptedPdfError'):
+        log.error(textwrap.dedent("""\
+            Input PDF is encrypted. The encryption must be removed to
+            perform OCR. 
+
+            For information about this PDF's security use
+                qpdf --show-encryption infilename
+
+            You can remove the encryption using
+                qpdf --decrypt [--password=[password]] infilename
+
+            """))
+        exit_code = ExitCode.encrypted_pdf        
     elif exc_name == 'ocrmypdf.exceptions.PdfMergeFailedError':
         log.error(textwrap.dedent("""\
             Failed to merge PDF image layer with OCR layer
@@ -591,19 +615,6 @@ def do_ruffus_exception(ruffus_five_tuple, options, log):
         base_exc_name = exc_name.replace('ocrmypdf.exceptions.', '')
         exc_class = getattr(ocrmypdf_exceptions, base_exc_name)
         exit_code = exc_class.exit_code
-    elif exc_name == 'PyPDF2.utils.PdfReadError' and \
-            'not been decrypted' in exc_value:
-        log.error(textwrap.dedent("""\
-            Input PDF uses either an encryption algorithm or a PDF security
-            handler that is not supported by ocrmypdf.
-
-            For information about this PDF's security use
-                qpdf --show-encryption [...input PDF...]
-
-            (Only algorithms "R = 1" and "R = 2" are supported.)
-
-            """))
-        exit_code = ExitCode.encrypted_pdf
     elif exc_name == 'PIL.Image.DecompressionBombError':
         msg = cleanup_ruffus_error_message(exc_value)
         msg += ("\nUse the --max-image-mpixels argument to set increase the "
@@ -622,6 +633,21 @@ def do_ruffus_exception(ruffus_five_tuple, options, log):
 def traverse_ruffus_exception(e_args, options, log):
     """Walk through a RethrownJobError and find the first exception.
 
+    Ruffus flattens exception to 5 element tuples. Because of a bug
+    in <= 2.6.3 it may present either the single:
+      (task, job, exc, value, stack)
+    or something like:
+      [[(task, job, exc, value, stack)]]
+    
+    Generally cross-process exception marshalling doesn't work well
+    and ruffus doesn't support because BaseException has its own
+    implementation of __reduce__ that attempts to reconstruct the
+    exception based on e.__init__(e.args).
+    
+    Attempting to log the exception directly marshalls it to the logger
+    which is probably in another process, so it's better to log only
+    data from the exception at this point.
+
     The exit code will be based on this, even if multiple exceptions occurred
     at the same time."""
 
@@ -653,6 +679,9 @@ def check_closed_streams(options):
 
     """
 
+    if sys.version_info[0:3] >= (3, 6, 4):
+        return True  # Issued fixed in Python 3.6.4+
+
     if sys.stderr is None:
         sys.stderr = open(os.devnull, 'w')
 
@@ -683,7 +712,7 @@ def log_page_orientations(pdfinfo, _log):
                 180: 's', 270: 'w'}
     orientations = []
     for n, page in enumerate(pdfinfo):
-        angle = pdfinfo[n].rotation or 0
+        angle = page.rotation or 0
         if angle != 0:
             orientations.append('{0}{1}'.format(
                 n + 1,
@@ -692,6 +721,42 @@ def log_page_orientations(pdfinfo, _log):
         _log.info('Page orientations detected: ' + ' '.join(orientations))
 
 
+def preamble(_log):
+    _log.debug('ocrmypdf ' + VERSION)
+    _log.debug('tesseract ' + tesseract.version())
+    _log.debug('qpdf ' + qpdf.version())
+
+
+def check_input_file(options, _log, start_input_file):
+    if options.input_file == '-':
+        # stdin
+        _log.info('reading file from standard input')
+        with open(start_input_file, 'wb') as stream_buffer:
+            from shutil import copyfileobj
+            copyfileobj(sys.stdin.buffer, stream_buffer)
+    else:
+        try:
+            re_symlink(options.input_file, start_input_file, _log)
+        except FileNotFoundError:
+            _log.error("File not found - " + options.input_file)
+            raise InputFileError()
+
+
+def check_output_file(options, _log):
+    if options.output_file == '-':
+        if sys.stdout.isatty():
+            _log.error(textwrap.dedent("""\
+                Output was set to stdout '-' but it looks like stdout
+                is connected to a terminal.  Please redirect stdout to a
+                file."""))
+            raise BadArgsError()
+    elif not is_file_writable(options.output_file):
+        _log.error(
+            "Output file location (" + options.output_file + ") " +
+            "is not a writable file.")
+        raise OutputFileAccessError()
+
+
 def run_pipeline():
     options = parser.parse_args()
     options.verbose_abbreviated_path = 1
@@ -703,16 +768,9 @@ def run_pipeline():
 
     _log, _log_mutex = proxy_logger.make_shared_logger_and_proxy(
         logging_factory, __name__, logger_args)
-    _log.debug('ocrmypdf ' + VERSION)
-    _log.debug('tesseract ' + tesseract.version())
-    _log.debug('qpdf ' + qpdf.version())
-
+    preamble(_log)
     check_options(options, _log)
 
-    PIL.Image.MAX_IMAGE_PIXELS = int(options.max_image_mpixels * 1000000)
-    if PIL.Image.MAX_IMAGE_PIXELS == 0:
-        PIL.Image.MAX_IMAGE_PIXELS = None
-
     # Complain about qpdf version < 7.0.0
     # Suppress the warning if in the test suite, since there are no PPAs
     # for qpdf 7.0.0 for Ubuntu trusty (i.e. Travis)
@@ -727,6 +785,13 @@ def run_pipeline():
     # options.input_file, options.pdf_renderer are already bound.)
     if not options.jobs:
         options.jobs = available_cpu_count()
+
+    # Performance is improved by setting Tesseract to single threaded. In tests
+    # this gives better throughput than letting a smaller number of Tesseract
+    # jobs run multithreaded.
+    if tesseract.v4():
+        os.environ.setdefault('OMP_THREAD_LIMIT', '1')
+
     try:
         work_folder = mkdtemp(prefix="com.github.ocrmypdf.")
         options.history_file = os.path.join(
@@ -734,37 +799,14 @@ def run_pipeline():
         start_input_file = os.path.join(
             work_folder, 'origin')
 
-        if options.input_file == '-':
-            # stdin
-            _log.info('reading file from standard input')
-            with open(start_input_file, 'wb') as stream_buffer:
-                from shutil import copyfileobj
-                copyfileobj(sys.stdin.buffer, stream_buffer)
-        else:
-            try:
-                re_symlink(options.input_file, start_input_file, _log)
-            except FileNotFoundError:
-                _log.error("File not found - " + options.input_file)
-                return ExitCode.input_file
-
-        if options.output_file == '-':
-            if sys.stdout.isatty():
-                _log.error(textwrap.dedent("""\
-                    Output was set to stdout '-' but it looks like stdout
-                    is connected to a terminal.  Please redirect stdout to a
-                    file."""))
-                return ExitCode.bad_args
-        elif not is_file_writable(options.output_file):
-            _log.error(
-                "Output file location (" + options.output_file + ") " +
-                "is not writable.")
-            return ExitCode.file_access_error
+        check_input_file(options, _log, start_input_file)
+        check_output_file(options, _log)
 
         manager = JobContextManager()
-        manager.register('JobContext', JobContext)
+        manager.register('JobContext', JobContext)  # pylint: disable=no-member
         manager.start()
 
-        context = manager.JobContext()
+        context = manager.JobContext()  # pylint: disable=no-member
         context.set_options(options)
         context.set_work_folder(work_folder)
 
@@ -774,22 +816,6 @@ def run_pipeline():
     except ruffus_exceptions.RethrownJobError as e:
         if options.verbose:
             _log.debug(str(e))  # stringify exception so logger doesn't have to
-
-        # Ruffus flattens exception to 5 element tuples. Because of a bug
-        # in <= 2.6.3 it may present either the single:
-        #   (task, job, exc, value, stack)
-        # or something like:
-        #   [[(task, job, exc, value, stack)]]
-        #
-        # Generally cross-process exception marshalling doesn't work well
-        # and ruffus doesn't support because BaseException has its own
-        # implementation of __reduce__ that attempts to reconstruct the
-        # exception based on e.__init__(e.args).
-        #
-        # Attempting to log the exception directly marshalls it to the logger
-        # which is probably in another process, so it's better to log only
-        # data from the exception at this point.
-
         exitcode = traverse_ruffus_exception(e.args, options, _log)
         if exitcode is None:
             _log.error("Unexpected ruffus exception: " + str(e))
@@ -804,7 +830,11 @@ def run_pipeline():
 
     if options.flowchart:
         _log.info("Flowchart saved to {}".format(options.flowchart))
-    elif options.output_file != '-':
+    elif options.output_file == '-':
+        _log.info("Output sent to stdout")
+    elif os.path.samefile(options.output_file, os.devnull):
+        pass  # Say nothing when sending to dev null
+    else:
         if options.output_type.startswith('pdfa'):
             pdfa_info = file_claims_pdfa(options.output_file)
             if pdfa_info['pass']:
@@ -817,8 +847,6 @@ def run_pipeline():
         if not qpdf.check(options.output_file, _log):
             _log.warning('Output file: The generated PDF is INVALID')
             return ExitCode.invalid_output_pdf
-    else:
-        _log.info("Output sent to stdout")
 
     pdfinfo = context.get_pdfinfo()
     if options.verbose:

ocrmypdf/_unicodefun.py

@@ -97,7 +97,7 @@ def verify_python3_env():
                 break
         if bad_locale is not None:
             extra += (
-                '\n\ocrmypdf discovered that you exported a UTF-8 locale\n'
+                '\nocrmypdf discovered that you exported a UTF-8 locale\n'
                 'but the locale system could not pick up from it because\n'
                 'it does not exist.  The exported locale is "%s" but it\n'
                 'is not supported'

ocrmypdf/exceptions.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # © 2016 James R. Barlow: github.com/jbarlow83
 
 
@@ -23,6 +22,10 @@ class ExitCodeException(Exception):
     exit_code = ExitCode.other_error
 
 
+class BadArgsError(ExitCodeException):
+    exit_code = ExitCode.bad_args
+
+
 class PdfMergeFailedError(ExitCodeException):
     exit_code = ExitCode.input_file
 
@@ -39,6 +42,10 @@ class DpiError(ExitCodeException):
     exit_code = ExitCode.input_file
 
 
+class OutputFileAccessError(ExitCodeException):
+    exit_code = ExitCode.file_access_error
+
+
 class PriorOcrFoundError(ExitCodeException):
     exit_code = ExitCode.already_done_ocr
 

ocrmypdf/exec/__init__.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # © 2016 James R. Barlow: github.com/jbarlow83
 
 """Wrappers to manage subprocess calls"""
@@ -46,4 +45,4 @@ def get_version(program, *,
             "Message was:\n{}").format(program, output)
         )
 
-    return version
+    return version

ocrmypdf/exec/ghostscript.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # © 2017 James R. Barlow: github.com/jbarlow83
 
 from tempfile import NamedTemporaryFile

ocrmypdf/exec/qpdf.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # © 2017 James R. Barlow: github.com/jbarlow83
 
 from subprocess import CalledProcessError, STDOUT, PIPE, run
@@ -68,8 +67,6 @@ def repair(input_file, output_file, log):
             return
 
         if _probably_encrypted(e):
-            log.error("{0}: this PDF is password-protected - password must "
-                      "be removed for OCR".format(input_file))
             raise EncryptedPdfError() from e
         elif e.returncode == 2:
             log.error("{0}: not a valid PDF, and could not repair it.".format(
@@ -181,7 +178,7 @@ def merge(input_files, output_file, min_version=None, log=None, max_files=None):
         n = min(step_size, len(workqueue))
         job = workqueue[0:n]
         del workqueue[0:n]
-        print('merging ' + repr(job))
+        log.debug('merging ' + repr(job))
 
         # Merge them into 1 file, which will contain n^depth pages
         merge_file = os.path.join(
@@ -191,7 +188,7 @@ def merge(input_files, output_file, min_version=None, log=None, max_files=None):
 
         # On the next 
         next_workqueue.append(merge_file)
-        print('next_workqueue ' + repr(next_workqueue))
+        log.debug('next_workqueue ' + repr(next_workqueue))
 
         # If we're out of things to do in this queue, move on to the next
         # queue. On the counter-th pass of the workqueue we can chew through
@@ -201,6 +198,6 @@ def merge(input_files, output_file, min_version=None, log=None, max_files=None):
             workqueue = next_workqueue
             next_workqueue = []
 
-    re_symlink(workqueue.pop(), output_file)
+    re_symlink(workqueue.pop(), output_file, log)
 
 

ocrmypdf/exec/tesseract.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # © 2017 James R. Barlow: github.com/jbarlow83
 
 import sys
@@ -158,7 +157,7 @@ def tesseract_log_output(log, stdout, input_file):
 
     try:
         text = stdout.decode()
-    except UnicodeDecodeError as e:
+    except UnicodeDecodeError:
         log.error(prefix + "command line output was not utf-8. " +
             "This usually means Tesseract's language packs do not match "
             "the installed version of Tesseract.")

ocrmypdf/exec/unpaper.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # © 2015 James R. Barlow: github.com/jbarlow83
 # unpaper documentation:
 # https://github.com/Flameeyes/unpaper/blob/master/doc/basic-concepts.md

ocrmypdf/helpers.py

@@ -1,9 +1,9 @@
-#!/usr/bin/env python3
 # © 2016 James R. Barlow: github.com/jbarlow83
 
 from functools import partial
 from collections.abc import Iterable
 from contextlib import suppress, contextmanager
+from pathlib import Path
 import sys
 import os
 
@@ -63,19 +63,32 @@ def is_file_writable(test_file):
     can replace it atomically. Before doing the OCR work, make sure
     the location is writable.
     """
-    if os.path.exists(test_file):
+    p = Path(test_file)
+
+    if p.is_symlink():
+        # Python 3.5 does not accept parameters for Path.resolve() and behaves
+        # as if strict=True (throws an exception on failure). Python 3.6 
+        # defaults to strict=False. This implements strict=False like behavior
+        # for Python 3.5.
+        if sys.version_info[0:2] <= (3, 5):
+            p = Path(os.path.realpath(str(p)))
+        else:
+            p = p.resolve(strict=False)
+
+    # p.is_file() throws an exception in some cases
+    if p.exists() and p.is_file():
         return os.access(
-            test_file, os.W_OK,
+            str(p), os.W_OK,
             effective_ids=(os.access in os.supports_effective_ids))
     else:
         try:
-            fp = open(test_file, 'wb')
-        except OSError as e:
+            fp = p.open('wb')
+        except OSError:
             return False
         else:
             fp.close()
             with suppress(OSError):
-                os.unlink(test_file)
+                p.unlink()
         return True
 
 
@@ -118,4 +131,4 @@ if sys.version_info[0:2] <= (3, 5):
 
 else:
     universal_open = open
-    fspath = os.fspath
+    fspath = os.fspath

ocrmypdf/hocrtransform.py

@@ -33,6 +33,7 @@ from reportlab.lib.units import inch
 from xml.etree import ElementTree
 from PIL import Image
 from collections import namedtuple
+from math import atan, sin, cos
 import re
 import argparse
 
@@ -49,13 +50,24 @@ class HocrTransform():
     """
     A class for converting documents from the hOCR format.
     For details of the hOCR format, see:
-    http://docs.google.com/View?docid=dfxcv4vc_67g844kf
+    http://kba.cloud/hocr-spec/
     """
 
+    box_pattern = re.compile(r'bbox((\s+\d+){4})')
+    baseline_pattern = re.compile(r'''
+        baseline \s+ 
+        ([\-\+]?\d*\.?\d*) \s+  # +/- decimal float
+        ([\-\+]?\d+)            # +/- int''', re.VERBOSE)
+    ligatures = str.maketrans({
+        'ff': 'ff',
+        'ffi': 'f‌f‌i',
+        'ffl': 'f‌f‌l',
+        'fi': 'fi',
+        'fl': 'fl',
+    })
+
     def __init__(self, hocrFileName, dpi):
         self.dpi = dpi
-        self.boxPattern = re.compile(r'bbox((\s+\d+){4})')
-
         self.hocr = ElementTree.parse(hocrFileName)
 
         # if the hOCR file has a namespace, ElementTree requires its use to
@@ -104,19 +116,31 @@ class HocrTransform():
             text += element.tail
         return text
 
-    def element_coordinates(self, element):
+    @classmethod
+    def element_coordinates(cls, element):
         """
         Returns a tuple containing the coordinates of the bounding box around
         an element
         """
         out = (0, 0, 0, 0)
         if 'title' in element.attrib:
-            matches = self.boxPattern.search(element.attrib['title'])
+            matches = cls.box_pattern.search(element.attrib['title'])
             if matches:
                 coords = matches.group(1).split()
                 out = Rect._make(int(coords[n]) for n in range(4))
         return out
 
+    @classmethod
+    def baseline(cls, element):
+        """
+        Returns a tuple containing the baseline slope and intercept.
+        """
+        if 'title' in element.attrib:
+            matches = cls.baseline_pattern.search(element.attrib['title'])
+            if matches:
+                return float(matches.group(1)), int(matches.group(2))
+        return (0, 0) 
+
     def pt_from_pixel(self, pxl):
         """
         Returns the quantity in PDF units (pt) given quantity in pixels
@@ -124,20 +148,17 @@ class HocrTransform():
         return Rect._make(
             (c / self.dpi * inch) for c in pxl)
 
-    def replace_unsupported_chars(self, s):
+    @classmethod
+    def replace_unsupported_chars(cls, s):
         """
         Given an input string, returns the corresponding string that:
         - is available in the helvetica facetype
         - does not contain any ligature (to allow easy search in the PDF file)
         """
-        # The 'u' before the character to replace indicates that it is a
-        # unicode character
-        s = s.replace(u"fl", "fl")
-        s = s.replace(u"fi", "fi")
-        return s
+        return s.translate(cls.ligatures)
 
     def to_pdf(self, outFileName, imageFileName=None, showBoundingboxes=False,
-               fontname="Helvetica", invisibleText=False):
+               fontname="Helvetica", invisibleText=False, interwordSpaces=False):
         """
         Creates a PDF file with an image superimposed on top of the text.
         Text is positioned according to the bounding box of the lines in
@@ -172,57 +193,19 @@ class HocrTransform():
                 pdf.rect(
                     pt.x1, self.height - pt.y2, pt.x2 - pt.x1, pt.y2 - pt.y1,
                     fill=1)
+                    
+        found_lines = False
+        for line in self.hocr.findall(
+                ".//%sspan[@class='%s']" % (self.xmlns, "ocr_line")):
+            found_lines = True
+            self._do_line(pdf, line, "ocrx_word", fontname, invisibleText, 
+                          interwordSpaces, showBoundingboxes)
 
-        # check if element with class 'ocrx_word' are available
-        # otherwise use 'ocr_line' as fallback
-        elemclass = "ocr_line"
-        if self.hocr.find(
-                ".//%sspan[@class='ocrx_word']" % (self.xmlns)) is not None:
-            elemclass = "ocrx_word"
-
-        # itterate all text elements
-        # light green for bounding box of word/line
-        pdf.setStrokeColorRGB(1, 0, 0)
-        pdf.setLineWidth(0.5)		# bounding box line width
-        pdf.setDash(6, 3)		# bounding box is dashed
-        pdf.setFillColorRGB(0, 0, 0)  # text in black
-        for elem in self.hocr.findall(
-                ".//%sspan[@class='%s']" % (self.xmlns, elemclass)):
-
-            elemtxt = self._get_element_text(elem).rstrip()
-
-            elemtxt = self.replace_unsupported_chars(elemtxt)
-
-            if len(elemtxt) == 0:
-                continue
-
-            pxl_coords = self.element_coordinates(elem)
-            pt = self.pt_from_pixel(pxl_coords)
-
-            # draw the bbox border
-            if showBoundingboxes:
-                pdf.rect(
-                    pt.x1, self.height - pt.y2, pt.x2 - pt.x1, pt.y2 - pt.y1,
-                    fill=0)
-
-            text = pdf.beginText()
-            fontsize = pt.y2 - pt.y1
-            text.setFont(fontname, fontsize)
-            if invisibleText:
-                text.setTextRenderMode(3)  # Invisible (indicates OCR text)
-
-            # set cursor to bottom left corner of bbox (adjust for dpi)
-            text.setTextOrigin(pt.x1, self.height - pt.y2)
-
-            # scale the width of the text to fill the width of the bbox
-            text.setHorizScale(
-                100 * (pt.x2 - pt.x1) / pdf.stringWidth(
-                    elemtxt, fontname, fontsize))
-
-            # write the text to the page
-            text.textLine(elemtxt)
-            pdf.drawText(text)
-
+        if not found_lines:
+            # Tesseract did not report any lines (just words)
+            root = self.hocr.find(".//%sdiv[@class='%s']" % (self.xmlns, "ocr_page"))
+            self._do_line(pdf, root, "ocrx_word", fontname, invisibleText,
+                          interwordSpaces, showBoundingboxes)
         # put the image on the page, scaled to fill the page
         if imageFileName is not None:
             pdf.drawImage(imageFileName, 0, 0,
@@ -233,6 +216,117 @@ class HocrTransform():
         pdf.save()
 
 
+    @classmethod
+    def polyval(cls, poly, x):
+        return x * poly[0] + poly[1]
+
+
+    def _do_line(self, pdf, line, elemclass, fontname, invisibleText, 
+                 interwordSpaces, showBoundingboxes):
+        pxl_line_coords = self.element_coordinates(line)
+        line_box = self.pt_from_pixel(pxl_line_coords)
+        line_height = line_box.y2 - line_box.y1
+
+        slope, pxl_intercept = self.baseline(line)
+        if abs(slope) < 0.005:
+            slope = 0.0
+        angle = atan(slope)
+        cos_a, sin_a = cos(angle), sin(angle)
+
+        text = pdf.beginText()
+        intercept = pxl_intercept / self.dpi * inch
+
+        # Don't allow the font to break out of the bounding box. Division by
+        # cos_a accounts for extra clearance between the glyph's vertical axis
+        # on a sloped baseline and the edge of the bounding box.
+        fontsize = (line_height - abs(intercept)) / cos_a
+        text.setFont(fontname, fontsize)
+        if invisibleText:
+            text.setTextRenderMode(3)  # Invisible (indicates OCR text)
+
+        # Intercept is normally negative, so this places it above the bottom
+        # of the line box
+        baseline_y2 = self.height - (line_box.y2 + intercept)
+
+        if showBoundingboxes:
+            # draw the baseline in magenta, dashed
+            pdf.setDash()
+            pdf.setStrokeColorRGB(0.95, 0.65, 0.95)
+            pdf.setLineWidth(0.5)
+            # negate slope because it is defined as a rise/run in pixel
+            # coordinates and page coordinates have the y axis flipped
+            pdf.line(line_box.x1, 
+                     baseline_y2,
+                     line_box.x2,
+                     self.polyval((-slope, baseline_y2), 
+                                  line_box.x2 - line_box.x1))
+            # light green for bounding box of word/line
+            pdf.setDash(6, 3)
+            pdf.setStrokeColorRGB(1, 0, 0)
+
+        text.setTextTransform(
+            cos_a, -sin_a, sin_a, cos_a,
+            line_box.x1, baseline_y2
+        )
+        pdf.setFillColorRGB(0, 0, 0)  # text in black
+
+        elements = line.findall(
+                ".//%sspan[@class='%s']" % (self.xmlns, elemclass))
+        for elem in elements:
+            elemtxt = self._get_element_text(elem).strip()
+            elemtxt = self.replace_unsupported_chars(elemtxt)
+            if elemtxt == '':
+                continue
+
+            pxl_coords = self.element_coordinates(elem)
+            box = self.pt_from_pixel(pxl_coords)
+            if interwordSpaces:
+                # if  `--interword-spaces` is true, append a space
+                # to the end of each text element to allow simpler PDF viewers 
+                # such as PDF.js to better recognize words in search and copy 
+                # and paste. Do not remove space from last word in line, even
+                # though it would look better, because it will interfere with
+                # naive text extraction. \n does not work either.
+                elemtxt += ' '
+                box = Rect._make((
+                    box.x1, 
+                    line_box.y1,
+                    box.x2 + pdf.stringWidth(' ', fontname, line_height), 
+                    line_box.y2))
+            box_width = box.x2 - box.x1
+            font_width = pdf.stringWidth(elemtxt, fontname, fontsize)
+
+            # draw the bbox border
+            if showBoundingboxes:
+                pdf.rect(
+                    box.x1,
+                    self.height - line_box.y2,
+                    box_width,
+                    line_height,
+                    fill=0)
+
+            # Adjust relative position of cursor
+            # This is equivalent to:
+            #   text.setTextOrigin(pt.x1, self.height - line_box.y2)
+            # but the former generates a full text reposition matrix (Tm) in the
+            # content stream while this issues a "offset" (Td) command.
+            # .moveCursor() is relative to start of the text line, where the
+            # "text line" means whatever reportlab defines it as. Do not use
+            # use .getCursor(), since moveCursor() rather unintuitively plans 
+            # its moves relative to .getStartOfLine().
+            # For skewed lines, in the text transform we set up a rotated
+            # coordinate system, so we don't have to account for the 
+            # incremental offset. Surprisingly most PDF viewers can handle this.
+            cursor = text.getStartOfLine()
+            dx = box.x1 - cursor[0]
+            dy = baseline_y2 - cursor[1]            
+            text.moveCursor(dx, dy)
+
+            text.setHorizScale(100 * box_width / font_width)
+            text.textOut(elemtxt)
+        pdf.drawText(text)
+
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description='Convert hocr file to PDF')
     parser.add_argument('-b', '--boundingboxes', action="store_true",
@@ -242,10 +336,12 @@ if __name__ == "__main__":
                         help='Resolution of the image that was OCRed')
     parser.add_argument('-i', '--image', default=None,
                         help='Path to the image to be placed above the text')
+    parser.add_argument('--interword-spaces', action='store_true',
+                         default=False, help='Add spaces between words')
     parser.add_argument('hocrfile', help='Path to the hocr file to be parsed')
     parser.add_argument(
         'outputfile', help='Path to the PDF file to be generated')
     args = parser.parse_args()
 
     hocr = HocrTransform(args.hocrfile, args.resolution)
-    hocr.to_pdf(args.outputfile, args.image, args.boundingboxes)
+    hocr.to_pdf(args.outputfile, args.image, args.boundingboxes, interwordSpaces=args.interword_spaces)

ocrmypdf/leptonica.py

@@ -566,35 +566,4 @@ if __name__ == '__main__':
     parser_deskew.set_defaults(func=deskew)
 
     args = parser.parse_args()
-
-    if get_leptonica_version() != u'leptonica-1.69':
-        print("Unexpected leptonica version: %s" % getLeptonicaVersion())
-
-    args.func(args)
-
-
-def test_skew_angle():
-    from PIL import Image, ImageDraw
-    from tempfile import NamedTemporaryFile
-
-    im = Image.new(mode='1', size=(1000, 1000), color=1)
-
-    draw = ImageDraw.Draw(im)
-    for n in range(20):
-        draw.line([(50, 25 + 50*n), (950, 25 + 50*n)], width=1)
-    del draw
-
-    test_angles = [0.1 * ang for ang in range(1, 10)] + \
-                  [float(ang) for ang in range(1, 7)]
-    test_angles += [-ang for ang in test_angles]
-    test_angles = sorted(test_angles)
-
-    for rotate_angle in test_angles:
-        rotated_im = im.rotate(rotate_angle)
-        with NamedTemporaryFile(prefix='lept-skew', suffix='.png', delete=True) as tmpfile:
-            rotated_im.save(tmpfile)
-            pix = pixRead(tmpfile.name)
-            angle, confidence = pixFindSkew(pix)
-            print('{0} {1}  {2}'.format(rotate_angle, angle, confidence), file=sys.stderr)
-
-
+    args.func(args)

ocrmypdf/lib/compile_leptonica.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 # © 2017 James R. Barlow: github.com/jbarlow83
 
 from cffi import FFI

ocrmypdf/pdfa.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # © 2015 James R. Barlow: github.com/jbarlow83
 #
 # Generate a PDFA_def.ps file for Ghostscript >= 9.14

ocrmypdf/pdfinfo.py

@@ -662,10 +662,6 @@ class PageInfo:
         else:
             return '1.5'
 
-    @property
-    def images(self):
-        return self._pageinfo['images']
-
     def __repr__(self):
         return (
             '<PageInfo '

ocrmypdf/pipeline.py

@@ -1,7 +1,7 @@
-#!/usr/bin/env python3
 # © 2016 James R. Barlow: github.com/jbarlow83
 
 from contextlib import suppress
+from shutil import copyfileobj
 import sys
 import os
 import shutil
@@ -17,7 +17,8 @@ from .pdfinfo import PdfInfo, Encoding, Colorspace
 from .pdfa import generate_pdfa_ps
 from .helpers import re_symlink, is_iterable_notstr, page_number
 from .exec import ghostscript, tesseract, qpdf
-from .exceptions import *
+from .exceptions import PdfMergeFailedError, UnsupportedImageFormatError, \
+    DpiError, PriorOcrFoundError, InputFileError
 from . import leptonica
 from . import PROGRAM_NAME, VERSION
 
@@ -159,7 +160,7 @@ def _pdf_guess_version(input_file, search_window=1024):
 
     with open(input_file, 'rb') as f:
         signature = f.read(1024)
-    m = re.search(b'%PDF-(\d\.\d)', signature)
+    m = re.search(br'%PDF-(\d\.\d)', signature)
     if m:
         return m.group(1)
     return ''
@@ -244,7 +245,30 @@ def get_canvas_square_dpi(pageinfo, options):
 def is_ocr_required(pageinfo, log, options):
     page = pageinfo.pageno + 1
     ocr_required = True
-    if not pageinfo.images:
+
+    if pageinfo.has_text:
+        msg = "{0:4d}: page already has text! – {1}"
+
+        if not options.force_ocr and not options.skip_text:
+            log.error(msg.format(page,
+                                 "aborting (use --force-ocr to force OCR)"))
+            raise PriorOcrFoundError()
+        elif options.force_ocr:
+            log.info(msg.format(page,
+                                "rasterizing text and running OCR anyway"))
+            ocr_required = True
+        elif options.skip_text:
+            log.info(msg.format(page,
+                                "skipping all processing on this page"))
+            ocr_required = False
+    elif not pageinfo.images and not options.lossless_reconstruction:
+        # We found a page with no images and no text. That means it may
+        # have vector art that the user wants to OCR. If we determined
+        # lossless reconstruction is not possible then we have to rasterize
+        # the image. So if OCR is being forced, take that to mean YES, go 
+        # ahead and rasterize. If not forced, then pretend there's no text
+        # on the page at all so we don't lose anything.
+        # This could be made smarter by explicitly searching for vector art.
         if options.force_ocr and options.oversample:
             # The user really wants to reprocess this file
             log.info(
@@ -263,23 +287,9 @@ def is_ocr_required(pageinfo, log, options):
         else:
             log.info(
                 "{0:4d}: page has no images - "
-                "skipping all processing on this page".format(page))
-            ocr_required = False
-
-    elif pageinfo.has_text:
-        msg = "{0:4d}: page already has text! – {1}"
-
-        if not options.force_ocr and not options.skip_text:
-            log.error(msg.format(page,
-                                 "aborting (use --force-ocr to force OCR)"))
-            raise PriorOcrFoundError()
-        elif options.force_ocr:
-            log.info(msg.format(page,
-                                "rasterizing text and running OCR anyway"))
-            ocr_required = True
-        elif options.skip_text:
-            log.info(msg.format(page,
-                                "skipping all processing on this page"))
+                "skipping all processing on this page to avoid losing detail. "
+                "Use --force-ocr if you wish to perform OCR on pages that "
+                "have vector content.".format(page))
             ocr_required = False
 
     if ocr_required and options.skip_big and pageinfo.images:
@@ -630,8 +640,8 @@ def render_hocr_page(
 
     hocrtransform = HocrTransform(hocr, dpi)
     hocrtransform.to_pdf(output_file, imageFileName=None,
-                         showBoundingboxes=False, invisibleText=True)
-
+                         showBoundingboxes=False, invisibleText=True,
+                         interwordSpaces=True)
 
 def flatten_groups(groups):
     for obj in groups:
@@ -655,8 +665,8 @@ def render_hocr_debug_page(
 
     hocrtransform = HocrTransform(hocr, dpi)
     hocrtransform.to_pdf(output_file, imageFileName=None,
-                         showBoundingboxes=True, invisibleText=False)
-
+                         showBoundingboxes=True, invisibleText=False,
+                         interwordSpaces=True)
 
 def combine_layers(
         infiles,
@@ -991,13 +1001,16 @@ def copy_final(
         context):
     input_file = next((ii for ii in input_files if ii.endswith('.pdf')))
 
-    if output_file == '-':
-        from shutil import copyfileobj
-        with open(input_file, 'rb') as input_stream:
+    with open(input_file, 'rb') as input_stream:
+        if output_file == '-':
             copyfileobj(input_stream, sys.stdout.buffer)
             sys.stdout.flush()
-    else:
-        shutil.copy(input_file, output_file)
+        else:
+            # At this point we overwrite the output_file specified by the user
+            # use copyfileobj because then we use open() to create the file and 
+            # get the appropriate umask, ownership, etc.
+            with open(output_file, 'wb') as output_stream:
+                copyfileobj(input_stream, output_stream)
 
 
 def build_pipeline(options, work_folder, log, context):
@@ -1092,8 +1105,6 @@ def build_pipeline(options, work_folder, log, context):
         extras=[log, context])
     task_ocr_tesseract_hocr.graphviz(fillcolor='"#00cc66"')
     task_ocr_tesseract_hocr.active_if(options.pdf_renderer == 'hocr')
-    if tesseract.v4():
-        task_ocr_tesseract_hocr.jobs_limit(2)  # Uses multi-core on its own
 
     task_select_visible_page_image = main_pipeline.collate(
         task_func=select_visible_page_image,
@@ -1146,8 +1157,6 @@ def build_pipeline(options, work_folder, log, context):
         extras=[log, context])
     task_ocr_tesseract_textonly_pdf.graphviz(fillcolor='"#ff69b4"')
     task_ocr_tesseract_textonly_pdf.active_if(options.pdf_renderer == 'sandwich')
-    if tesseract.v4():
-        task_ocr_tesseract_textonly_pdf.jobs_limit(2)
 
     task_combine_layers = main_pipeline.collate(
         task_func=combine_layers,
@@ -1171,8 +1180,6 @@ def build_pipeline(options, work_folder, log, context):
         extras=[log, context])
     task_ocr_tesseract_and_render_pdf.graphviz(fillcolor='"#66ccff"')
     task_ocr_tesseract_and_render_pdf.active_if(options.pdf_renderer == 'tesseract')
-    if tesseract.v4():
-        task_ocr_tesseract_and_render_pdf.jobs_limit(2)  # Uses multi-core
 
     # PDF/A
     task_generate_postscript_stub = main_pipeline.transform(
@@ -1226,7 +1233,7 @@ def build_pipeline(options, work_folder, log, context):
     task_merge_sidecars.active_if(options.sidecar)
 
     # Finalize
-    task_copy_final = main_pipeline.merge(
+    main_pipeline.merge(
         task_func=copy_final,
         input=[task_merge_pages_ghostscript, task_merge_pages_qpdf],
         output=options.output_file,

tests/conftest.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # © 2017 James R. Barlow: github.com/jbarlow83
 
 import sys

tests/resources/README.rst

@@ -33,7 +33,7 @@ In some cases they were converted from one image format to another without other
         - Creative Commons BY-SA 3.0
     *   - typewriter.png, 2400dpi.pdf
         - `Wikimedia: Triumph typewrtier text Linzensoep`_
-        * Creative Commons BY-SA 2.5
+        - Creative Commons BY-SA 2.5
     *   - baiona.png
         - `Wikimedia: Baionako udalerri mugakideak`_
         - Creative Commons BY-SA 4.0
@@ -100,6 +100,9 @@ under the terms of the license in LICENSE.rst.
     *   - trivial.pdf
         - @jbarlow83
         - smallest possible valid PDF-1.3 with all required fields
+    *   - vector.pdf
+        - @Catscratch
+        - a PDF with vector art and text rendered as curves with no fonts
 
 
 Assemblies
@@ -116,7 +119,7 @@ These test resources are assemblies or derivatives from other previously mention
 - palette.pdf (congress.jpg, converted to a 256-color palette)
 - rotated_skew.pdf (a /Rotate'd and skewed document from LinnSequencer.jpg)
 - skew.pdf (from LinnSequencer.jpg, skew simulated by adjusting the transformation matrix)
-- skew-encrypted.pdf (skew.pdf with encryption - access supported by PyPDF2)
+- skew-encrypted.pdf (skew.pdf with encryption - access supported by PyPDF2, password is "password")
 - baiona_gray.png (from baiona.png)
 - poster.pdf (from LinnSequencer.jpg)
 

tests/spoof/gs_render_failure.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# © 2016 James R. Barlow: github.com/jbarlow83
+# © 2016-18 James R. Barlow: github.com/jbarlow83
 
 import sys
 import os

tests/spoof/tesseract_badutf8.py

@@ -40,7 +40,7 @@ def main():
         sys.stdout.buffer.write(BAD_UTF8)
         sys.exit(1)
     elif sys.argv[-1] == 'stdout':
-        inputf = sys.argv[-2]
+        # input file is at sys.argv[-2] but we don't look at it
         print("""Orientation: 0
 Orientation in degrees: 0
 Orientation confidence: 100.00

tests/test_hocrtransform.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # © 2015 James R. Barlow: github.com/jbarlow83
 
 from ocrmypdf import hocrtransform

tests/test_main.py

@@ -1,6 +1,7 @@
 # © 2015-17 James R. Barlow: github.com/jbarlow83
 
 from subprocess import Popen, PIPE, check_output, check_call, DEVNULL
+from pathlib import Path
 import os
 import shutil
 import resource
@@ -16,6 +17,9 @@ from math import isclose
 
 import PIL
 
+# pytest.helpers is dynamic
+# pylint: disable=no-member
+# pylint: disable=w0612
 
 check_ocrmypdf = pytest.helpers.check_ocrmypdf
 run_ocrmypdf = pytest.helpers.run_ocrmypdf
@@ -433,7 +437,7 @@ def test_klingon(resources, outpdf):
 
 def test_missing_docinfo(spoof_tesseract_noop, resources, outpdf):
     p, out, err = run_ocrmypdf(
-        resources / 'missing_docinfo.pdf', outpdf, '-l', 'eng', '-c',
+        resources / 'missing_docinfo.pdf', outpdf, '-l', 'eng', '--skip-text',
         env=spoof_tesseract_noop)
     assert p.returncode == ExitCode.ok, err
 
@@ -480,7 +484,7 @@ def test_encrypted(resources, no_outpdf):
     p, out, err = run_ocrmypdf(
         resources / 'skew-encrypted.pdf', no_outpdf)
     assert p.returncode == ExitCode.encrypted_pdf
-    assert out.find('password')
+    assert out.find('encrypted')
 
 
 @pytest.mark.parametrize('renderer', [
@@ -881,6 +885,8 @@ def test_gs_raster_failure(spoof_no_tess_gs_raster_fail, resources, outpdf):
     assert p.returncode == ExitCode.child_process_error
 
 
+@pytest.mark.skipif('8.0.0' <= qpdf.version() <= '8.0.1',
+                    reason="qpdf regression")
 def test_no_contents(spoof_tesseract_noop, resources, outpdf):
     check_ocrmypdf(resources / 'no_contents.pdf', outpdf, '--force-ocr',
                    env=spoof_tesseract_noop)
@@ -1081,4 +1087,55 @@ def test_decompression_bomb(resources, outpdf):
         outpdf,
         '--max-image-mpixels', '2000'
     )
-    assert p.returncode == 0
+    assert p.returncode == 0
+
+
+def test_text_curves(spoof_tesseract_noop, resources, outpdf):
+    check_ocrmypdf(
+        resources / 'vector.pdf', outpdf, env=spoof_tesseract_noop)
+
+    info = PdfInfo(outpdf)
+    assert len(info.pages[0].images) == 0, "added images to the vector PDF"
+
+    check_ocrmypdf(
+        resources / 'vector.pdf', outpdf, '--force-ocr',
+        env=spoof_tesseract_noop)
+    
+    info = PdfInfo(outpdf)
+    assert len(info.pages[0].images) != 0, "force did not rasterize"
+    
+
+def test_dev_null(spoof_tesseract_noop, resources):
+    p, out, err = run_ocrmypdf(
+        resources / 'trivial.pdf',
+        os.devnull,
+        '--force-ocr',
+        env=spoof_tesseract_noop
+    )
+    assert p.returncode == 0, "could not send output to /dev/null"
+    assert len(out) == 0, "wrote to stdout"
+
+
+def test_output_is_dir(spoof_tesseract_noop, resources, outdir):
+    p, out, err = run_ocrmypdf(
+        resources / 'trivial.pdf',
+        outdir,
+        '--force-ocr',
+        env=spoof_tesseract_noop
+    )
+    assert p.returncode == ExitCode.file_access_error
+    assert 'is not a writable file' in err
+
+
+def test_output_is_symlink(spoof_tesseract_noop, resources, outdir):
+    sym = Path(outdir / 'this_is_a_symlink')
+    sym.symlink_to(outdir / 'out.pdf')
+    p, out, err = run_ocrmypdf(
+        resources / 'trivial.pdf',
+        sym,
+        '--force-ocr',
+        env=spoof_tesseract_noop
+    )
+    assert p.returncode == ExitCode.ok, err
+    assert (outdir / 'out.pdf').stat().st_size > 0, 'target file not created'
+    

tests/test_pageinfo.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # © 2015 James R. Barlow: github.com/jbarlow83
 
 from ocrmypdf import pdfinfo
@@ -122,4 +121,4 @@ def test_no_contents(resources):
 def test_oversized_page(resources):
     pdf = pdfinfo.PdfInfo(resources / 'poster.pdf')
     image = pdf[0].images[0]
-    assert image.width * image.xres > 200, "this is supposed to be oversized"
+    assert image.width * image.xres > 200, "this is supposed to be oversized"

tests/test_tess3.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # © 2017 James R. Barlow: github.com/jbarlow83
 
 import pytest

tests/test_tess4.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # © 2017 James R. Barlow: github.com/jbarlow83
 
 import pytest
@@ -10,7 +9,7 @@ import os
 import PyPDF2 as pypdf
 from contextlib import contextmanager
 
-
+# pylint: disable=no-member
 spoof = pytest.helpers.spoof
 
 
@@ -128,4 +127,4 @@ def test_content_preservation(ensure_tess4, resources, outpdf):
 
     info = pdfinfo.PdfInfo(outpdf)
     page = info[0]
-    assert len(page.images) > 1, "masks were rasterized"
+    assert len(page.images) > 1, "masks were rasterized"

tests/test_userunit.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 # © 2017 James R. Barlow: github.com/jbarlow83
 
 from subprocess import Popen, PIPE, check_output, check_call, DEVNULL