Ghostcript 9.21 seems to have a regression related to Unicode metadata

Not needed since reportlab 3.4 comes with a wheel, and that was the main difficulty.

.dockerignore

@@ -11,11 +11,14 @@
 */*/*/*/*/*/*.pyc
 */*/*/*/*/*/*/*.pyc
 .cache/
+.eggs/
+.github/
 .git/
 .ipynb_checkpoints/
 .ruffus_history.sqlite
 bin/
 build/
+docs/
 dist/
 htmlcov/
 include/
@@ -27,5 +30,5 @@ tests/cache/
 tests/output/
 tests/resources/private/
 tmp/
-venv-3.4/
-venv-3.5/
+venv*/
+wheelhouse/

.gitignore

@@ -4,6 +4,8 @@
 venv*/
 pyvenv.cfg
 tasks.py
+.bash_history
+.ruffus_history.sqlite
 
 # Package building
 *.egg-info/
@@ -27,8 +29,11 @@ htmlcov/
 # Testing
 log/
 /*.pdf
+/*.qdf
+*.ipynb
 .ipynb_checkpoints/
 tests/cache/
 tests/output/
 tests/resources/private/
 tmp/
+pdfbox-app*.jar

.travis.yml

@@ -1,46 +1,60 @@
-sudo: required
 dist: trusty
 language: python
-
-cache: pip
-
 cache:
-    directories:
-        - $HOME/.ccache
-        - packages
-        - tests/cache
+  pip: true
+  directories:
+  - packages
+  - tests/cache
+  - $HOME/Library/Caches/Homebrew
 
-python:
-    - "3.4"
-    - "3.5"
-    - "3.6"
+matrix:
+  include:
+    - os: linux
+      sudo: required
+      language: python
+      python: 3.4
+    - os: linux
+      sudo: required
+      language: python
+      python: 3.5
+    - os: linux
+      sudo: required
+      language: python
+      python: 3.6
+    - os: osx
+      osx_image: xcode8
+      language: generic
 
 before_cache:
-    - rm -f $HOME/.cache/pip/log/debug.log
+- rm -f $HOME/.cache/pip/log/debug.log
 
 before_install:
-    # Ubuntu packages
-    - sudo add-apt-repository ppa:vshn/ghostscript -y  # for ghostscript 9.16 (trusty has 9.10)
-    - sudo add-apt-repository ppa:heyarje/libav-11 -y  # for libav11, which is for unpaper
-    - sudo apt-get update -qq  # must go after all add-apt-repo
-    - sudo apt-get install -y ghostscript tesseract-ocr tesseract-ocr-deu tesseract-ocr-eng tesseract-ocr-fra qpdf poppler-utils libavformat56 libavcodec56 libavutil54 libffi-dev
-
-    # pip
-    - pip install --upgrade pip
-
-    # Download, make and install unpaper (using ccache)
-    - mkdir -p packages
-    - "[ -f packages/unpaper_6.1-1.deb ] || wget -q https://dl.dropboxusercontent.com/u/28971240/unpaper_6.1-1.deb -O packages/unpaper_6.1-1.deb"
-    - sudo dpkg -i packages/unpaper_6.1-1.deb
+- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then bash .travis/linux_before_install.sh ; fi
+- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then bash .travis/osx_before_install.sh ; fi
 
 install:
-    - pip install -r requirements.txt
-    - pip install -r test_requirements.txt
+- pip3 install .
+- pip3 install -r requirements.txt -r test_requirements.txt
 
 script:
-    - python setup.py clean
-    - python setup.py install
-    - python setup.py test
+- mv ocrmypdf dont_import_this_ocrmypdf
+- pytest
+- mv dont_import_this_ocrmypdf ocrmypdf
 
-os:
-    - linux
+after_success:
+- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then bash .travis/osx_brew.sh ; fi
+
+# See https://www.appneta.com/blog/pypi-deployment-with-travis-ci/ for
+# steps to set up testpypi deploy for untagged builds if desired
+
+deploy:
+  provider: pypi
+  user: ocrmypdf-travis
+  password:
+    secure: DTFOmmNL6olA0+yXvp4u9jXZlZeqrJsJ0526jzqf4a3gZ6jnGTq5UI6WzRsslSyoMMfXKtHQebqHM6ogSgCZinyZ3ufHJo8fn9brxbEc2gsiWkbj5o3bGwdWMT1vNNE7XW0VCpw87rZ1EEwjl4FJHFudMlPR1yfU5+uq0k0PACo=
+  distributions: "sdist bdist_wheel"
+  on:
+    branch: master
+    tags: true
+    condition: $TRAVIS_PYTHON_VERSION == "3.6" && $TRAVIS_OS_NAME == "linux"
+  skip_upload_docs: true

.travis/autobrew.py

@@ -0,0 +1,75 @@
+from string import Template
+from subprocess import run, PIPE
+import re
+
+recipe_template = Template("""
+class Ocrmypdf < Formula
+  include Language::Python::Virtualenv
+
+  desc "Adds an OCR text layer to scanned PDF files"
+  homepage "https://github.com/jbarlow83/OCRmyPDF"
+  ${ocrmypdf_url}
+  ${ocrmypdf_sha256}
+
+  depends_on :python3
+  depends_on "pkg-config" => :build
+  depends_on "zlib"
+  depends_on "libffi"
+  depends_on "tesseract"
+  depends_on "ghostscript"
+  depends_on "unpaper"
+  depends_on "qpdf"
+
+  # For Pillow source install
+  depends_on "openjpeg"
+  depends_on "freetype"
+  depends_on "libpng"
+
+${resources}
+
+  def install
+    ENV.append ["SETUPTOOLS_SCM_PRETEND_VERSION"], "v${ocrmypdf_version}"
+    ENV.each do |key, value|
+      puts "#{key}:#{value}"
+    end
+    virtualenv_install_with_resources
+  end
+
+  test do
+    # `test do` will create, run in and delete a temporary directory.
+    #
+    # The installed folder is not in the path, so use the entire path to any
+    # executables being tested: `system "#{bin}/program", "do", "something"`.
+    system "#{bin}/ocrmypdf", "--version"
+  end
+end
+""")
+
+def main():
+    p = run(['poet', '--single', 'ocrmypdf'],
+            encoding='utf-8', stdout=PIPE, check=True)
+
+    ocrmypdf_lines = p.stdout.splitlines()
+    ocrmypdf_url = ocrmypdf_lines[1].strip()
+    ocrmypdf_sha256 = ocrmypdf_lines[2].strip()
+
+    ocrmypdf_version = re.search(
+        r'ocrmypdf-(.+)\.tar.*', ocrmypdf_url).group(1)
+    print(f"Autobrewing {ocrmypdf_version}")
+
+    p = run(['poet', '--resources', 'ocrmypdf'],
+            encoding='utf-8', stdout=PIPE, check=True)
+
+    poet_resources = p.stdout
+
+    # Remove the duplicate "ocrmypdf" resource block
+    all_resources = poet_resources.split('resource')
+    kept_resources = [block for block in all_resources if 'ocrmypdf' not in block]
+    resources = 'resource'.join(kept_resources)
+
+    with open('ocrmypdf.rb', 'w') as out:
+        out.write(recipe_template.substitute(**locals()))
+
+
+if __name__ == '__main__':
+    main()

.travis/linux_before_install.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+set -euo pipefail
+set -x
+
+sudo add-apt-repository ppa:vshn/ghostscript -y
+sudo add-apt-repository ppa:heyarje/libav-11 -y
+sudo apt-get update -qq
+sudo apt-get install -y \
+	ghostscript \
+	tesseract-ocr \
+	tesseract-ocr-deu \
+	tesseract-ocr-eng \
+	tesseract-ocr-fra \
+	qpdf \
+	poppler-utils \
+	libavformat56 \
+	libavcodec56 \
+	libavutil54 \
+	libffi-dev
+
+pip install --upgrade pip
+mkdir -p packages
+[ -f packages/unpaper_6.1-1.deb ] || wget -q https://dl.dropboxusercontent.com/u/28971240/unpaper_6.1-1.deb -O packages/unpaper_6.1-1.deb
+sudo dpkg -i packages/unpaper_6.1-1.deb

.travis/osx_before_install.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -euo pipefail
+set -x
+
+brew update
+
+brew install openjpeg jbig2dec libtiff     # image libraries
+brew install qpdf
+brew install ghostscript
+brew install python3
+brew install libxml2 libffi leptonica
+brew install unpaper   # optional
+brew install tesseract
+
+pip3 install --upgrade pip
+pip3 install wheel

.travis/osx_brew.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -euo pipefail
+set -x
+
+pip3 install homebrew-pypi-poet
+python3 .travis/autobrew.py
+brew audit ocrmypdf.rb
+cat ocrmypdf.rb

Dockerfile

@@ -1,91 +1,54 @@
 # OCRmyPDF
 #
-# VERSION               3.2
-FROM      debian:stretch
+# VERSION               4.4.2
+FROM      ubuntu:16.10
 MAINTAINER James R. Barlow <jim@purplerock.ca>
 
-# Add unprivileged user
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  software-properties-common python-software-properties \
+  python3-wheel \
+  python3-reportlab \
+  python3-venv \
+  ghostscript \
+  qpdf \
+  poppler-utils \
+  unpaper \
+  libffi-dev \ 
+  tesseract-ocr \
+  tesseract-ocr-eng \
+  tesseract-ocr-fra \
+  tesseract-ocr-spa \
+  tesseract-ocr-deu
+
+RUN python3 -m venv --system-site-packages /appenv
+
+# This installs the latest binary wheel instead of the code in the current
+# folder. Installing from source will fail, apparently because cffi needs
+# build-essentials (gcc) to do a source installation 
+# (i.e. "pip install ."). It's unclear to me why this is the case.
+RUN . /appenv/bin/activate; \
+  pip install --upgrade pip \
+  && pip install ocrmypdf
+
+# Now copy the application in, mainly to get the test suite.
+# Do this now to make the best use of Docker cache.
+COPY . /application
+RUN . /appenv/bin/activate; \
+  pip install -r /application/test_requirements.txt
+
+# Remove the junk, including the source version of application since it was
+# already installed
+RUN rm -rf /tmp/* /var/tmp/* /root/* /application/ocrmypdf \
+  && apt-get autoremove -y \
+  && apt-get autoclean -y
+
 RUN useradd docker \
   && mkdir /home/docker \
   && chown docker:docker /home/docker
 
-# Update system and install our dependencies
-# If this command takes too Docker hub's automated build will timeout,
-# so try it in portions
-RUN apt-get update && apt-get install -y --no-install-recommends \
-  locales \
-  python3 \
-  python3-pip \
-  python3-venv \
-  python3-reportlab \
-  python3-pil \
-  python3-wheel
-
-RUN apt-get install -y --no-install-recommends \
-  unpaper \
-  qpdf \
-  poppler-utils \
-  tesseract-ocr \
-  tesseract-ocr-deu tesseract-ocr-spa tesseract-ocr-eng tesseract-ocr-fra
-
-RUN apt-get install -qy --no-install-recommends \
-  libffi-dev \
-  libpython3-dev \
-  gcc
-
-# Install Ghostscript from Debian sid to work around JPEG 2000 issue in
-# Debian stretch libgs9 or gs 9.16~dfsg-2.1
-
-COPY ./share/etc-apt-sources.list /etc/apt/sources.list
-
-RUN apt-get update && apt-get install -y ghostscript/sid
-
-
-# Enforce UTF-8
-# Borrowed from https://index.docker.io/u/crosbymichael/python/ 
-RUN dpkg-reconfigure locales && \
-  locale-gen C.UTF-8 && \
-  /usr/sbin/update-locale LANG=C.UTF-8
-ENV LC_ALL C.UTF-8
-
-
-# Set up a Python virtualenv and take all of the system packages, so we can
-# rely on the platform packages rather than importing GCC and compiling them
-RUN pyvenv /appenv \
-  && pyvenv --system-site-packages /appenv
-
-COPY . /application/
-
-# Replace stock Tesseract 3.04.00 font with improved sharp2.ttf that resolves
-# issues in many PDF viewers.
-# Discussion is in https://github.com/tesseract-ocr/tesseract/issues/182
-COPY ./share/sharp2.ttf /usr/share/tesseract-ocr/tessdata/pdf.ttf
-RUN chmod 644 /usr/share/tesseract-ocr/tessdata/pdf.ttf
-
-# Set this here to force a docker version, allowing non-tagged versions to
-# be built
-# ENV SETUPTOOLS_SCM_PRETEND_VERSION=v3.3.0
-
-# Install application and dependencies
-# In this arrangement Pillow and reportlab will be provided by the system
-# Even though ocrmypdf is locally present, pull from PyPI because
-# Dockerhub and setuptools_scm clash
-RUN . /appenv/bin/activate; \
-  pip install --upgrade pip \
-  && pip install ocrmypdf \
-  && pip install --no-cache-dir -r /application/test_requirements.txt
-
-# Remove the junk
-RUN apt-get remove -qy gcc
-RUN apt-get autoremove -y && apt-get clean -y
-RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/*
-
 USER docker
 WORKDIR /home/docker
 
-ENV OCRMYPDF_TEST_OUTPUT=/tmp/test-output
-ENV OCRMYPDF_SHARP_TTF=1
-
 # Must use array form of ENTRYPOINT
 # Non-array form does not append other arguments, because that is "intuitive"
-ENTRYPOINT ["/application/docker-wrapper.sh"]
+ENTRYPOINT ["/application/docker-wrapper.sh"]

Dockerfile.polyglot

@@ -1,14 +1,17 @@
 # OCRmyPDF polyglot
 #
-# VERSION               3.2
+# VERSION               4.4.2
 FROM      jbarlow83/ocrmypdf:latest
 MAINTAINER James R. Barlow <jim@purplerock.ca>
 
-# Update system and install our dependencies
 USER root
+
+# Update system and install our dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends \
   tesseract-ocr-all 
 
+RUN apt-get autoremove -y && apt-get clean -y
+
 USER docker
 
 # Must use array form of ENTRYPOINT

Dockerfile.tess4

@@ -0,0 +1,64 @@
+# OCRmyPDF
+#
+# VERSION               4.4.2
+FROM 	ubuntu:16.10
+MAINTAINER James R. Barlow <jim@purplerock.ca>
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  software-properties-common python-software-properties \
+  python3-wheel \
+  python3-reportlab \
+  python3-venv \
+  ghostscript \
+  qpdf \
+  poppler-utils \
+  unpaper \
+  libffi-dev
+
+RUN add-apt-repository ppa:alex-p/tesseract-ocr
+
+RUN apt-get update \
+	&& apt-get autoremove -y \
+	&& apt-get install -y --no-install-recommends \
+		tesseract-ocr \
+		tesseract-ocr-eng \
+		tesseract-ocr-fra \
+		tesseract-ocr-deu \
+		tesseract-ocr-spa \
+		tesseract-ocr-por \
+		tesseract-ocr-ara \
+		tesseract-ocr-rus \
+		tesseract-ocr-chi-sim 
+
+RUN python3 -m venv --system-site-packages /appenv
+
+# This installs the latest binary wheel instead of the code in the current
+# folder. Installing from source will fail, apparently because cffi needs
+# build-essentials (gcc) to do a source installation 
+# (i.e. "pip install ."). It's unclear to me why this is the case.
+RUN . /appenv/bin/activate; \
+  pip install --upgrade pip \
+  && pip install ocrmypdf
+
+# Now copy the application in, mainly to get the test suite.
+# Do this now to make the best use of Docker cache.
+COPY . /application
+RUN . /appenv/bin/activate; \
+  pip install -r /application/test_requirements.txt
+
+# Remove the junk, including the source version of application since it was
+# already installed
+RUN rm -rf /tmp/* /var/tmp/* /root/* /application/ocrmypdf \
+  && apt-get autoremove -y \
+  && apt-get autoclean -y
+
+RUN useradd docker \
+  && mkdir /home/docker \
+  && chown docker:docker /home/docker
+
+USER docker
+WORKDIR /home/docker
+
+# Must use array form of ENTRYPOINT
+# Non-array form does not append other arguments, because that is "intuitive"
+ENTRYPOINT ["/application/docker-wrapper.sh"]

MANIFEST.in

@@ -36,7 +36,6 @@ recursive-exclude docs/_build *
 
 # support files
 recursive-include ocrmypdf/data *
-recursive-include share *
 include *.py
 exclude tasks.py
 

README.rst

@@ -60,7 +60,17 @@ Installation
 Linux, UNIX, and macOS are supported. Windows is not directly supported but there is a Docker image available that runs on Windows.
 
 Users of Debian 9 or later or Ubuntu 16.10 or later may simply
-``apt-get install ocrmypdf``.
+
+.. code-block:: bash
+
+   apt-get install ocrmypdf
+
+and macOS users may simply
+
+.. code-block:: bash
+
+   brew tap jbarlow83/ocrmypdf
+   brew install ocrmypdf
 
 For everyone else, `see our documentation <https://ocrmypdf.readthedocs.io/en/latest/installation.html>`_ for installation steps.
 

RELEASE_NOTES.rst

@@ -3,51 +3,94 @@ RELEASE NOTES
 
 OCRmyPDF uses `semantic versioning <http://semver.org/>`_.
 
-v4.4:
-=====
+
+v4.5.2
+======
+
+-  Fix issue #147. ``--pdf-renderer tess4 --clean`` will produce an oversized page containing the original image in the bottom left corner, due to loss DPI information.
+-  Make "using Tesseract 4.0" warning less ominous
+-  Set up machinery for homebrew OCRmyPDF tap
+
+
+v4.5.1
+======
+
+-  Fix issue #137, proportions of images with a non-square pixel aspect ratio would be distorted in output for ``--force-ocr`` and some other combinations of flags
+
+
+v4.5
+====
+
+-  Exotic PDFs containing "Form XObjects" are now supported (issue #134; PDF reference manual 8.10), and images they contain are taken into account when determining the resolution for rasterizing
+-  The Tesseract 4 Docker image no longer includes all languages, because it took so long to build something would tend to fail
+-  OCRmyPDF now warns about using ``--pdf-renderer tesseract`` with Tesseract 3.04 or lower due to issues with Ghostscript corrupting the OCR text in these cases
+
+
+v4.4.2
+======
+
+-  The Docker images (ocrmypdf, ocrmypdf-polyglot, ocrmypdf-tess4) are now based on Ubuntu 16.10 instead of Debian stretch
+
+   + This makes supporting the Tesseract 4 image easier
+   + This could be a disruptive change for any Docker users who built customized these images with their own changes, and made those changes in a way that depends on Debian and not Ubuntu
+
+-  OCRmyPDF now prevents running the Tesseract 4 renderer with Tesseract 3.04, which was permitted in v4.4 and v4.4.1 but will not work
+
+
+v4.4.1
+======
+
+-  To prevent a `TIFF output error <https://github.com/python-pillow/Pillow/issues/2206>`_ caused by img2pdf >= 0.2.1 and Pillow <= 3.4.2, dependencies have been tightened
+-  The Tesseract 4.00 simultaneous process limit was increased from 1 to 2, since it was observed that 1 lowers performance
+-  Documentation improvements to describe the ``--tesseract-config`` feature 
+-  Added test cases and fixed error handling for ``--tesseract-config``
+-  Tweaks to setup.py to deal with issues in the v4.4 release
+
+v4.4
+====
 
 -  Tesseract 4.00 is now supported on an experimental basis.
 
    +  A new rendering option ``--pdf-renderer tess4`` exploits Tesseract 4's new text-only output PDF mode. See the documentation on PDF Renderers for details.
-   +  The ``--tesseract-oem`` argument allows control over the Tesseract 4 OCR 
-   engine mode (tesseract's ``--oem``). Use ``--tesseract-oem 2`` to enforce the new LSTM mode.
+   +  The ``--tesseract-oem`` argument allows control over the Tesseract 4 OCR engine mode (tesseract's ``--oem``). Use ``--tesseract-oem 2`` to enforce the new LSTM mode.
    +  Fixed poor performance with Tesseract 4.00 on Linux
 
 -  Fixed an issue that caused corruption of output to stdout in some cases
 -  Removed test for Pillow JPEG and PNG support, as the minimum supported version of Pillow now enforces this
 -  OCRmyPDF now tests that the intended destination file is writable before proceeding
+-  The test suite now requires ``pytest-helpers-namespace`` to run (but not install)
 -  Significant code reorganization to make OCRmyPDF re-entrant and improve performance. All changes should be backward compatible for the v4.x series.
 
    + However, OCRmyPDF's dependency "ruffus" is not re-entrant, so no Python API is available. Scripts should continue to use the command line interface.
 
 
-v4.3.5:
-=======
+v4.3.5
+======
 
 -  Update documentation to confirm Python 3.6.0 compatibility. No code changes were needed, so many earlier versions are likely supported.
 
 
-v4.3.4:
-=======
+v4.3.4
+======
 
 -  Fixed "decimal.InvalidOperation: quantize result has too many digits" for high DPI images
 
 
-v4.3.3:
-=======
+v4.3.3
+======
 
 -  Fixed PDF/A creation with Ghostscript 9.20 properly
 -  Fixed an exception on inline stencil masks with a missing optional parameter
 
 
-v4.3.2:
-=======
+v4.3.2
+======
 
 -  Fixed a PDF/A creation issue with Ghostscript 9.20 (note: this fix did not actually work)
 
 
-v4.3.1:
-=======
+v4.3.1
+======
 
 -  Fixed an issue where pages produced by the "hocr" renderer after a Tesseract timeout would be rotated incorrectly if the input page was rotated with a /Rotate marker
 -  Fixed a file handle leak in LeptonicaErrorTrap that would cause a "too many open files" error for files around hundred pages of pages long when ``--deskew`` or ``--remove-background`` or other Leptonica based image processing features were in use, depending on the system value of ``ulimit -n``
@@ -57,8 +100,8 @@ v4.3.1:
 -  Tesseract caching in test cases is now more cautious about false cache hits and reproducing exact output, not that any problems were observed
 
 
-v4.3:
-=====
+v4.3
+====
 
 -  New feature ``--remove-background`` to detect and erase the background of color and grayscale images
 -  Better documentation
@@ -68,21 +111,21 @@ v4.3:
    +  This does not improve performance since temporary files are still used for buffering
    +  Some output validation is disabled in this mode
 
-v4.2.5:
-=======
+v4.2.5
+======
 
 -  Fixed an issue (#100) with PDFs that omit the optional /BitsPerComponent parameter on images
 -  Removed non-free file milk.pdf
 
 
-v4.2.4:
-=======
+v4.2.4
+======
 
 -  Fixed an error (#90) caused by PDFs that use stencil masks properly
 -  Fixed handling of PDFs that try to draw images or stencil masks without properly setting up the graphics state (such images are now ignored for the purposes of calculating DPI)
 
-v4.2.3:
-=======
+v4.2.3
+======
 
 -  Fixed an issue with PDFs that store page rotation (/Rotate) in an indirect object
 -  Integrated a few fixes to simplify downstream packaging (Debian)
@@ -95,21 +138,21 @@ v4.2.3:
 -  Deprecated the OCRmyPDF.sh shell script
 
 
-v4.2.2:
-=======
+v4.2.2
+======
 
 -  Improvements to documentation
 
 
-v4.2.1:
-=======
+v4.2.1
+======
 
 -  Fixed an issue where PDF pages that contained stencil masks would report an incorrect DPI and cause Ghostscript to abort
 -  Implemented stdin streaming
 
 
-v4.2:
-=====
+v4.2
+====
 
 -  ocrmypdf will now try to convert single image files to PDFs if they are provided as input (#15)
 
@@ -141,14 +184,14 @@ v4.2:
 
 -  Ghostscript now runs in "safer" mode where possible
 
-v4.1.4:
-=======
+v4.1.4
+======
 
 -  Bug fix: monochrome images with an ICC profile attached were incorrectly converted to full color images if lossless reconstruction was not possible due to other settings; consequence was increased file size for these images
 
 
-v4.1.3:
-=======
+v4.1.3
+======
 
 -  More helpful error message for PDFs with version 4 security handler
 -  Update usage instructions for Windows/Docker users
@@ -156,50 +199,50 @@ v4.1.3:
 -  Add a few leptonica wrapper functions (no effect on most users)
 
 
-v4.1.2:
-=======
+v4.1.2
+======
 
 -  Replace IEC sRGB ICC profile with Debian's sRGB (from icc-profiles-free) which is more compatible with the MIT license
 -  More helpful error message for an error related to certain types of malformed PDFs
 
 
-v4.1:
-=====
+v4.1
+====
 
 -  ``--rotate-pages`` now only rotates pages when reasonably confidence in the orientation. This behavior can be adjusted with the new argument ``--rotate-pages-threshold``
 -  Fixed problems in error checking if ``unpaper`` is uninstalled or missing at run-time
 -  Fixed problems with "RethrownJobError" errors during error handling that suppressed the useful error messages
 
 
-v4.0.7:
-=======
+v4.0.7
+======
 
 -  Minor correction to Ghostscript output settings
 
 
-v4.0.6:
-=======
+v4.0.6
+======
 
 -  Update install instructions
 -  Provide a sRGB profile instead of using Ghostscript's
 
 
-v4.0.5:
-=======
+v4.0.5
+======
 
 -  Remove some verbose debug messages from v4.0.4
 -  Fixed temporary that wasn't being deleted
 -  DPI is now calculated correctly for cropped images, along with other image transformations
 -  Inline images are now checked during DPI calculation instead of rejecting the image
 
-v4.0.4:
-=======
+v4.0.4
+======
 
 Released with verbose debug message turned on. Do not use. Skip to v4.0.5.
 
 
-v4.0.3:
-=======
+v4.0.3
+======
 
 New features
 ------------
@@ -216,8 +259,8 @@ Fixes
 -  Docker: fix blank JPEG2000 issue by insisting on Ghostscript versions that have this fixed
 
 
-v4.0.2:
-=======
+v4.0.2
+======
 
 Fixes
 -----
@@ -228,8 +271,8 @@ Fixes
 -  Fixed use of chmod on Docker that broke most test cases
 
 
-v4.0.1:
-=======
+v4.0.1
+======
 
 Fixes
 -----
@@ -237,8 +280,8 @@ Fixes
 -  Fixed a KeyError if tesseract fails to find page orientation information
 
 
-v4.0:
-=====
+v4.0
+====
 
 New features
 ------------
@@ -272,8 +315,8 @@ Changes
    to correct the problem.
 
 
-v3.2.1:
-=======
+v3.2.1
+======
 
 Changes
 -------
@@ -282,8 +325,8 @@ Changes
 -  Tweaked the Dockerfiles
 
 
-v3.2:
-=====
+v3.2
+====
 
 New features
 ------------
@@ -304,16 +347,16 @@ Changes
 
 
 
-v3.1.1:
-=======
+v3.1.1
+======
 
 Changes
 -------
 
 -  Fixed bug that caused incorrect page size and DPI calculations on documents with mixed page sizes
 
-v3.1:
-=====
+v3.1
+====
 
 Changes
 -------
@@ -330,8 +373,8 @@ Changes
    Currently it always chooses the 'hocrtransform' renderer but that behavior may change.
 -  Set up Travis CI automatic integration testing
 
-v3.0:
-=====
+v3.0
+====
 
 New features
 ------------
@@ -480,9 +523,9 @@ Notes and known issues
    images almost never contain inline images.
 
 
-v2.2-stable (2014-09-29):
-=========================
+v2.2-stable (2014-09-29)
+========================
 
 OCRmyPDF versions 1 and 2 were implemented as shell scripts. OCRmyPDF 3.0+ is a fork that gradually replaced all shell scripts with Python while maintaining the existing command line arguments. No one is maintaining old versions.
 
-For details on older versions, see the `final version of its release notes <https://github.com/fritz-hh/OCRmyPDF/blob/7fd3dbdf42ca53a619412ce8add7532c5e81a9d1/RELEASE_NOTES.md>`_.
+For details on older versions, see the `final version of its release notes <https://github.com/fritz-hh/OCRmyPDF/blob/7fd3dbdf42ca53a619412ce8add7532c5e81a9d1/RELEASE_NOTES.md>`_.

docs/advanced.rst

@@ -0,0 +1,127 @@
+Advanced features
+=================
+
+Control of OCR options
+----------------------
+
+OCRmyPDF provides many features to control the behavior of the OCR engine, Tesseract.
+
+When OCR is skipped
+"""""""""""""""""""
+
+If a page in a PDF seems to have text, by default OCRmyPDF will exit without modifying the PDF. This is to ensure that PDFs that were previously OCRed or were "born digital" rather than scanned are not processed. 
+
+If ``--skip-text`` is issued, then no OCR will be performed on pages that already have text. The page will be copied to the output. This may be useful for documents that contain both "born digital" and scanned content, or to use OCRmyPDF to normalize and convert to PDF/A regardless of their contents.
+
+If ``--force-ocr`` is issued, then all pages will be rasterized to images, discarding any hidden OCR text, and rasterizing any printable text. This is useful for redoing OCR, for fixing OCR text with a damaged character map (text is selectable but not searchable), and destroying redacted information.
+
+
+Time and image size limits
+""""""""""""""""""""""""""
+
+By default, OCRmyPDF permits tesseract to run for only three minutes (180 seconds) per page. This is usually more than enough time to find all text on a reasonably sized page with modern hardware. 
+
+If a page is skipped, it will be inserted without OCR. If preprocessing was requested, the preprocessed image layer will be inserted.
+
+If you want to adjust the amount of time spent on OCR, change ``--tesseract-timeout``.  You can also automatically skip images that exceed a certain number of megapixels with ``--skip-big``. (A 300 DPI, 8.5×11" page is 8.4 megapixels.)
+
+.. code-block:: bash
+
+	# Allow 300 seconds for OCR; skip any page larger than 50 megapixels
+	ocrmypdf --tesseract-timeout 300 --skip-big 50 bigfile.pdf output.pdf
+
+Overriding default tesseract
+""""""""""""""""""""""""""""
+
+OCRmyPDF checks the environment variable ``OCRMYPDF_TESSERACT`` for the full path *to the tesseract executable* first. 
+
+For example, if you are testing tesseract 4.00 and don't wish to disturb your tesseract 3.04 installation, you can launch OCRmyPDF as follows:
+
+.. code-block:: bash
+
+	env \
+		OCRMYPDF_TESSERACT=/home/user/src/tesseract4/api/tesseract \
+		TESSDATA_PREFIX=/home/user/src/tesseract4 \
+		ocrmypdf --pdf-renderer tess4 --tesseract-oem 2 input.pdf output.pdf
+
+* ``TESSDATA_PREFIX`` directs tesseract 4.0 to use LSTM training data. This is a tesseract environment variable.
+* ``--pdf-renderer tess4`` takes advantage of new tesseract 4.0 PDF renderer in OCRmyPDF. (Tesseract 4.0 only.)
+* ``--tesseract-oem 1`` requests tesseract 4.0's new LSTM engine. (Tesseract 4.0 only.)
+
+Overriding other support programs
+"""""""""""""""""""""""""""""""""
+
+In addition to tesseract, OCRmyPDF uses the following external binaries:
+
+* ``gs`` (Ghostscript)
+* ``unpaper``
+* ``qpdf``
+
+In each case OCRmyPDF will check the environment variable ``OCRMYPDF_{program}`` before asking the system to find ``{program}`` on the PATH. For example, you could redirect OCRmyPDF to ``OCRMYPDF_GS`` to override Ghostscript.
+
+Changing tesseract configuration variables
+""""""""""""""""""""""""""""""""""""""""""
+
+You can override tesseract's default `control parameters <https://github.com/tesseract-ocr/tesseract/wiki/ControlParams>`_ with a configuration file.
+
+As an example, this configuration will disable Tesseract's dictionary for current language. Normally the dictionary is helpful for interpolating words that are unclear, but it may interfere with OCR if the document does not contain many words (for example, a list of part numbers).
+
+Create a file named "no-dict.cfg" with these contents:
+
+::
+
+	load_system_dawg 0
+	language_model_penalty_non_dict_word 0
+	language_model_penalty_non_freq_dict_word 0
+
+then run ocrmypdf as follows (along with any other desired arguments):
+
+.. code-block:: bash
+
+	ocrmypdf --tesseract-config no-dict.cfg input.pdf output.pdf
+
+.. warning::
+
+	Some combinations of control parameters will break Tesseract or break assumptions that OCRmyPDF makes about Tesseract's output.
+
+
+Changing the PDF renderer
+-------------------------
+
+rasterizing
+  Converting a PDF to an image for display.
+
+rendering
+  Creating a new PDF from other data (such as an existing PDF).
+
+
+OCRmyPDF has three PDF renderers: ``hocr``, ``tesseract`` and ``tess4``. The renderer may be selected using ``--pdf-renderer``. The default is ``auto`` which lets OCRmyPDF select the renderer to use. Currently, ``auto`` always selects ``hocr``. 
+
+The ``hocr`` renderer
+"""""""""""""""""""""
+
+The ``hocr`` renderer is the default because it works in most cases. In this mode the whole PDF is rasterized, the raster image is run through OCR to generate a .hocr file, which is an HTML-like file that specifies the location of all identified words.
+
+The .hocr file is then rendered as a PDF and merged with the image layer.
+
+The image layer is copied from the original PDF page if possible, avoiding potentially lossy transcoding or loss of other PDF information. If preprocessing is specified, then the image layer is a new PDF.
+
+This is the only option for tesseract 3.02 and older.
+
+
+The ``tesseract`` renderer
+""""""""""""""""""""""""""
+
+The tesseract renderer uses tesseract's capability to produce a PDF directly. In version 3, tesseract automatically combined the image layer and text, meaning that this mode *always* transcodes and loses potentially loses image quality and other PDF information.
+
+It does a much better job on non-Latin text.
+
+In a future release this will become the "tess3" renderer and ultimately will be dropped.
+
+
+The ``tess4`` renderer
+""""""""""""""""""""""
+
+The tess4 renderer uses tesseract 4.00 alpha's text-only PDF feature added in January 2017. This combines the advantages of the tesseract and hocr renderers, transcoding the image layer only if required by preprocessing options.
+
+Ghostscript PDF/A still sometimes inserts spaces between words when the tess4 renderer is used, affecting search quality.  ``--output-pdf pdf`` may be used to avoid this issue.

docs/batch.rst

@@ -10,11 +10,29 @@ Consider using the excellent `GNU Parallel <https://www.gnu.org/software/paralle
 
 Both ``parallel`` and ``ocrmypdf`` will try to use all available processors. To maximize parallelism without overloading your system with processes, consider using ``parallel -j 2`` to limit parallel to running two jobs at once.
 
-This command will run all ocrmypdf all files named ``*.pdf`` in the current directory and write them to the previous created ``output/`` folder.
+This command will run all ocrmypdf all files named ``*.pdf`` in the current directory and write them to the previous created ``output/`` folder. It will not search subdirectories.
+
+The ``--tag`` argument tells parallel to print the filename as a prefix whenever a message is printed, so that one can trace any errors to the file that produced them.
 
 .. code-block:: bash
 
-	parallel -j 2 ocrmypdf '{}' 'output/{}' ::: *.pdf
+	parallel --tag -j 2 ocrmypdf '{}' 'output/{}' ::: *.pdf
+
+Directory trees
+---------------
+
+This will walk through a directory tree and run OCR on all files in place, printing the output in a way that makes 
+
+.. code-block:: bash
+
+	find . --printf '%p' -name '*.pdf' -exec ocrmypdf '{}' '{}' \;
+
+This only runs one ``ocrmypdf`` process at a time. This variation uses ``find`` to create a directory list and ``parallel`` to parallelize runs of ``ocrmypdf``, again updating files in place.
+
+.. code-block:: bash
+
+	find . -name '*.pdf' | parallel --tag -j 2 ocrmypdf '{}' '{}'
+
 
 Sample script
 """""""""""""

docs/cookbook.rst

@@ -59,7 +59,7 @@ By default OCRmyPDF assumes the document is English.
 	ocrmypdf -l fre LeParisien.pdf LeParisien.pdf
 	ocrmypdf -l eng+fre Bilingual-English-French.pdf Bilingual-English-French.pdf
 
-Language packs must be installed for all languages specified. See :ref:`Installing additional language packs <languages>`.
+Language packs must be installed for all languages specified. See :ref:`Installing additional language packs <lang-packs>`.
 
 
 Produce PDF and text file containing OCR text
@@ -68,10 +68,16 @@ Produce PDF and text file containing OCR text
 This produces a file named "output.pdf" and a companion text file named "output.txt". The ``pdftotext`` program from 
 `Poppler <https://poppler.freedesktop.org/>`_ is used to extract text from the finished PDF.
 
+
 .. code-block:: bash
 
 	ocrmypdf input.pdf - | tee output.pdf | pdftotext - output.txt
 
+.. note::
+
+	To get pdftotext, Debian/Ubuntu users may ``apt-get install poppler-utils`` 
+	and macOS users may ``brew install poppler`` respectively.
+
 
 OCR images, not PDFs
 --------------------
@@ -111,6 +117,14 @@ OCRmyPDF perform some image processing on each page of a PDF, if desired.  The s
 
 * ``--clean-final`` uses unpaper to clean up pages before OCR and inserts the page into the final output.  You will want to review each page to ensure that unpaper did not remove something important.
 
+.. note::
+
+	In many cases image processing will rasterize PDF pages as images, potentially losing quality.
+
+.. warning::
+
+	``--clean-final`` and ``-remove-background`` may leave undesirable visual artifacts in some images where their algorithms have shortcomings. Files should be visually reviewed after using these options.
+
 
 OCR and correct document skew (crooked scan)
 """"""""""""""""""""""""""""""""""""""""""""
@@ -127,16 +141,4 @@ Image processing commands can be combined. The order in which options are given
 
 	ocrmypdf --deskew --clean --rotate-pages input.pdf output.pdf
 
-Control of OCR options
-----------------------
-
-By default, OCRmyPDF permits tesseract to run for only three minutes (180 seconds) per page. This is usually more than enough time to find all text on a reasonably sized page with modern hardware. A skipped page will be inserted into the output without any OCR text.
-
-If you want to adjust the amount of time spent on OCR, change ``--tesseract-timeout``.  You can also automatically skip images that exceed a certain number of megapixels.
-
-.. code-block:: bash
-
-	# Allow 300 seconds for OCR; skip any page larger than 50 megapixels
-	ocrmypdf --tesseract-timeout 300 --skip-big 50 bigfile.pdf output.pdf
-
 

docs/index.rst

@@ -20,8 +20,8 @@ Contents:
    installation
    languages
    cookbook
+   advanced
    batch
-   renderers
    security
    errors
 

docs/installation.rst

@@ -10,6 +10,18 @@ Users of Debian 9 ("stretch") or later or Ubuntu 16.10 or later may simply
 
 	apt-get install ocrmypdf
 
+
+Installing on macOS
+-------------------
+
+.. code-block:: bash
+
+   brew tap jbarlow83/ocrmypdf
+   brew install ocrmypdf
+
+
+Users who previously installed OCRmyPDF on macOS may need to remove the ``pip`` based installation (``pip3 uninstall ocrmypdf``).
+
 .. _Docker:
 
 Installing the Docker image
@@ -27,7 +39,7 @@ successfully, your system is ready to download and execute the image:
 
    docker run hello-world
    
-OCRmyPDF will use all available CPU cores.  By default, the VirtualBox machine instance on Windows and OS X has only a single CPU core enabled. Use the VirtualBox Manager to determine the name of your Docker engine host, and then follow these optional steps to enable multiple CPUs:
+OCRmyPDF will use all available CPU cores.  By default, the VirtualBox machine instance on Windows and macOS has only a single CPU core enabled. Use the VirtualBox Manager to determine the name of your Docker engine host, and then follow these optional steps to enable multiple CPUs:
 
 .. code-block:: bash
 
@@ -37,28 +49,34 @@ OCRmyPDF will use all available CPU cores.  By default, the VirtualBox machine i
    docker-machine start "yourVM"
    eval $(docker-machine env "yourVM")
 
-Assuming you have a Docker engine running somewhere, you can run these commands to download
-the image:
+Assuming you have a Docker engine running, you can download one of the three available images:
+
++-----------------------------+---------------------------------------------+---------------------------------------------------------------------------------+
+| Image name                  | Download command                            | Notes                                                                           |
++-----------------------------+---------------------------------------------+---------------------------------------------------------------------------------+
+| ocrmypdf                    | ``docker pull jbarlow83/ocrmypdf``          | Latest ocrmypdf with Tesseract 3.04. Includes English, French, German, Spanish. |
++-----------------------------+---------------------------------------------+---------------------------------------------------------------------------------+
+| ocrmypdf-polyglot           | ``docker pull jbarlow83/ocrmypdf-polyglot`` | As above, with all available language packs.                                    |
++-----------------------------+---------------------------------------------+---------------------------------------------------------------------------------+
+| ocrmypdf-tess4              | ``docker pull jbarlow83/ocrmypdf-tess4``    | Latest ocrmypdf with Tesseract 4.00.00alpha and English, French, German,        |
+|                             |                                             | Spanish, Portuguese, Chinese Simplified, Arabic and Russian (the top 8).        |
++-----------------------------+---------------------------------------------+---------------------------------------------------------------------------------+
+
+For example:
 
 .. code-block:: bash
 
-   docker pull jbarlow83/ocrmypdf
+   docker pull jbarlow83/ocrmypdf-tess4
 
 Then tag it to give a more convenient name, just ocrmypdf:
 
 .. code-block:: bash
 
-   docker tag jbarlow83/ocrmypdf ocrmypdf
+   docker tag jbarlow83/ocrmypdf-tess4 ocrmypdf
 
 .. _docker-polyglot:
 
-This image contains language packs for English, French, Spanish and German. The alternative "polyglot" image provides `all available language packs <https://github.com/tesseract-ocr/tesseract/blob/master/doc/tesseract.1.asc#languages>`_:
-
-.. code-block:: bash
-
-   # Alternative step: If you need all language packs
-   docker pull jbarlow83/ocrmypdf-polyglot
-   docker tag jbarlow83/ocrmypdf-polyglot ocrmypdf
+The alternative "polyglot" image provides `all available language packs <https://github.com/tesseract-ocr/tesseract/blob/master/doc/tesseract.1.asc#languages>`_.
 
 You can then run ocrmypdf using the command:
 
@@ -82,11 +100,25 @@ In this worked example, the current working directory contains an input file cal
 
 Note that ``ocrmypdf`` has its own separate ``-v VERBOSITYLEVEL`` argument to control debug verbosity. All Docker arguments should before the ``ocrmypdf`` image name and all arguments to ``ocrmypdf`` should be listed after.
 
+For convenience, a shell alias can hide the docker command:
 
-Installing on macOS (formerly Mac OS X)
----------------------------------------
+.. code-block:: bash
 
-These instructions probably work on all macOS supported by Homebrew. OCRmyPDF is known to work on Yosemite and El Capitan, and regularly tested on El Capitan.
+   alias ocrmypdf='docker run --rm -v "$(pwd):/home/docker" ocrmypdf' 
+   ocrmypdf --version  # runs docker version
+
+Or in the wonderful `fish shell <https://fishshell.com/>`_:
+
+.. code-block:: fish
+
+   alias ocrmypdf 'docker run --rm -v (pwd):/home/docker ocrmypdf'
+   funcsave ocrmypdf
+
+
+Manual installation on macOS
+----------------------------
+
+These instructions probably work on all macOS supported by Homebrew.
 
 If it's not already present, `install Homebrew <http://brew.sh/>`_.
 
@@ -155,10 +187,6 @@ No package is currently available for Ubuntu 16.04, but you can install the depe
       tesseract-ocr \
       qpdf \
       python3-pip \
-      python3-pil \
-      python3-pytest \
-      python3-reportlab \
-      python3-ruffus \
       python3-cffi
 
 If you wish install OCRmyPDF to the system Python, then install as follows (note this installs new packages
@@ -173,7 +201,7 @@ follow these steps.
 
 .. code-block:: bash
 
-   python3 -m venv --system-site-packages venv-ocrmypdf
+   python3 -m venv venv-ocrmypdf
    source venv-ocrmypdf/bin/activate
    pip3 install ocrmypdf
 

docs/renderers.rst

@@ -1,40 +0,0 @@
-PDF Renderers
-=============
-
-rasterizing
-  Converting a PDF to an image for display.
-
-rendering
-  Creating a new PDF from other data (such as an existing PDF).
-
-
-OCRmyPDF has three PDF renderers: ``hocr``, ``tesseract`` and ``tess4``. The renderer may be selected using ``--pdf-renderer``. The default is ``auto`` which lets OCRmyPDF select the renderer to use. Currently it always uses ``hocr``. 
-
-The hocr renderer
------------------
-
-The ``hocr`` renderer is the default because it works in most cases. In this mode the whole PDF is rasterized, the raster image is run through OCR to generate a .hocr file, which is an HTML-like file that specifies the location of all identified words.
-
-The .hocr file is then rendered as a PDF and merged with the image layer.
-
-The image layer is copied from the original PDF page if possible, avoiding potentially lossy transcoding or loss of other PDF information. If preprocessing is specified, then the image layer is a new PDF.
-
-This is the only option for tesseract 3.02 and older.
-
-
-The tesseract renderer
-----------------------
-
-The tesseract renderer uses tesseract's capability to produce a PDF directly. In version 3, tesseract automatically combined the image layer and text, meaning that this mode always transcodes and loses potentially loses quality and other PDF information.
-
-It does a much better job on non-Latin text.
-
-In a future release this will become the "tess3" renderer and ultimately will be dropped.
-
-
-The tess4 renderer
-------------------
-
-The tess4 renderer uses tesseract 4.00 alpha's text-only PDF feature added in January 2017. This combines the advantages of the tesseract and hocr renderers, transcoding the image layer only if required by preprocessing options.
-
-Ghostscript PDF/A still sometimes inserts spaces between words when the tess4 renderer is used, affecting search quality.  ``--output-pdf pdf`` may be used to avoid this issue.

ocrmypdf/__main__.py

@@ -207,7 +207,7 @@ advanced = parser.add_argument_group(
     "Advanced options for power users")
 advanced.add_argument(
     '--tesseract-config', action='append', metavar='CFG', default=[],
-    help="additional Tesseract configuration files")
+    help="additional Tesseract configuration files -- see documentation")
 advanced.add_argument(
     '--tesseract-pagesegmode', action='store', type=int, metavar='PSM',
     choices=range(0, 14),
@@ -272,13 +272,18 @@ def check_options_output(options, log):
     if options.pdf_renderer == 'auto':
         options.pdf_renderer = 'hocr'
 
-    if options.pdf_renderer == 'tesseract' and \
-            tesseract.version() < '3.04.01' and \
-            os.environ.get('OCRMYPDF_SHARP_TTF', '') != '1':
-        log.warning(
-            "Your version of tesseract has problems with PDF output."
-            " Some PDF viewers will fail to find searchable text.\n"
-            "--pdf-renderer=tesseract is not recommended.")
+    if options.pdf_renderer in ('tesseract', 'tess4'):
+        if tesseract.version() < '3.05':
+            log.warning(
+                "The setting --pdf-renderer=tesseract is not recommend for "
+                " use with tesseract versions less than 3.05, because it "
+                " produces OCR text that is incompatible with Ghostscript and "
+                " some other software.")
+        elif tesseract.version() == '4.00.00alpha':
+            log.warning(
+                "The setting --pdf-renderer={tesseract,tess4} is not"
+                " recommended for builds of tesseract 4.00.00alpha older than"
+                " February 2017. Make sure you are using a recent build.")
 
     if options.debug_rendering and options.pdf_renderer == 'tesseract':
         log.info(
@@ -333,6 +338,10 @@ def check_options_advanced(options, log):
     if options.tesseract_oem and not tesseract.v4():
         log.warning(
             "--tesseract-oem requires Tesseract 4.x -- argument ignored")
+    if options.pdf_renderer == 'tess4' and not tesseract.has_textonly_pdf():
+        raise MissingDependencyError(
+            "--pdf-renderer tess4 requires Tesseract 4.x "
+            "commit 3d9fb3b or later")
 
 
 def check_options(options, log):
@@ -350,9 +359,6 @@ def check_options(options, log):
         sys.exit(ExitCode.missing_dependency)
 
 
-
-
-
 # ----------
 # Logging
 
@@ -475,10 +481,58 @@ def traverse_ruffus_exception(e_args, options, log):
             return traverse_ruffus_exception(exc, options, log)
 
 
+def check_closed_streams(options):
+    """Work around Python issue with multiprocessing forking on closed streams
+
+    https://bugs.python.org/issue28326
+
+    Attempting to a fork/exec a new Python process when any of std{in,out,err}
+    are closed or not flushable for some reason may raise an exception.
+    Fix this by opening devnull if the handle seems to be closed.  Do this
+    globally to avoid tracking places all places that fork.
+
+    Seems to be specific to multiprocessing.Process not all Python process
+    forkers.
+
+    The error actually occurs when the stream object is not flushable,
+    but replacing an open stream object that is not flushable with
+    /dev/null is a bad idea since it will create a silent failure.  Replacing
+    a closed handle with /dev/null seems safe.
+
+    """
+
+    if sys.stderr is None:
+        sys.stderr = open(os.devnull, 'w')
+
+    if sys.stdin is None:
+        if options.input_file == '-':
+            print("Trying to read from stdin but stdin seems closed",
+                  file=sys.stderr)
+            return False
+        sys.stdin = open(os.devnull, 'r')
+
+    if sys.stdout is None:
+        if options.output_file == '-':
+            # Can't replace stdout if the user is piping
+            # If this case can even happen, it must be some kind of weird
+            # stream.
+            print(textwrap.dedent("""\
+                Output was set to stdout '-' but the stream attached to
+                stdout does not support the flush() system call.  This
+                will fail."""), file=sys.stderr)
+            return False
+        sys.stdout = open(os.devnull, 'w')
+
+    return True
+
+
 def run_pipeline():
     options = parser.parse_args()
     options.verbose_abbreviated_path = 1
 
+    if not check_closed_streams(options):
+        return ExitCode.bad_args
+
     _log, _log_mutex = proxy_logger.make_shared_logger_and_proxy(
         logging_factory, __name__, [None, options.verbose])
     _log.debug('ocrmypdf ' + VERSION)
@@ -519,9 +573,9 @@ def run_pipeline():
                     file."""))
                 return ExitCode.bad_args
         elif not is_file_writable(options.output_file):
-                _log.error(textwrap.dedent("""\
-                    Cutput file location is not writable."""))
-                return ExitCode.file_access_error
+            _log.error(textwrap.dedent("""\
+                Cutput file location is not writable."""))
+            return ExitCode.file_access_error
 
         manager = JobContextManager()
         manager.register('JobContext', JobContext)

ocrmypdf/exceptions.py

@@ -14,6 +14,7 @@ class ExitCode(IntEnum):
     already_done_ocr = 6
     child_process_error = 7
     encrypted_pdf = 8
+    invalid_config = 9
     other_error = 15
     ctrl_c = 130
 
@@ -52,3 +53,7 @@ class SubprocessOutputError(ExitCodeException):
 
 class EncryptedPdfError(ExitCodeException):
     exit_code = ExitCode.encrypted_pdf
+
+
+class TesseractConfigError(ExitCodeException):
+    exit_code = ExitCode.invalid_config

ocrmypdf/exec/ghostscript.py

@@ -2,12 +2,32 @@
 # © 2015 James R. Barlow: github.com/jbarlow83
 
 from tempfile import NamedTemporaryFile
-from subprocess import Popen, PIPE, STDOUT, check_call
+from subprocess import Popen, PIPE, STDOUT, check_call, CalledProcessError, \
+    check_output
 from shutil import copy
+from functools import lru_cache
 from . import get_program
 from ..pdfa import SRGB_ICC_PROFILE
 
 
+@lru_cache(maxsize=1)
+def version():
+    args_gs = [
+        get_program('gs'),
+        '--version'
+    ]
+    try:
+        version = check_output(
+                args_gs, close_fds=True, universal_newlines=True,
+                stderr=STDOUT)
+    except CalledProcessError as e:
+        print("Could not find Ghostscript executable on system PATH.",
+              file=sys.stderr)
+        raise MissingDependencyError from e
+
+    return version.strip()
+
+
 def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
                   pageno=1):
     with NamedTemporaryFile(delete=True) as tmp:

ocrmypdf/exec/tesseract.py

@@ -6,7 +6,7 @@ import os
 import re
 import shutil
 from functools import lru_cache
-from ..exceptions import MissingDependencyError
+from ..exceptions import MissingDependencyError, TesseractConfigError
 from ..helpers import page_number
 from . import get_program
 from collections import namedtuple
@@ -186,6 +186,8 @@ def tesseract_log_output(log, stdout, input_file):
             log.warning(prefix + "unsure about page orientation")
         elif 'error' in line.lower() or 'exception' in line.lower():
             log.error(prefix + line.strip())
+        elif 'read_params_file' in line.lower():
+            log.error(prefix + line.strip())
         else:
             log.info(prefix + line.strip())
 
@@ -236,6 +238,8 @@ def generate_hocr(input_file, output_hocr, language: list, engine_mode,
         _generate_null_hocr(output_hocr, input_file)
     except CalledProcessError as e:
         tesseract_log_output(log, e.output, input_file)
+        if 'read_params_file: parameter not found' in e.output:
+            raise TesseractConfigError() from e
         if 'Image too large' in e.output:
             _generate_null_hocr(output_hocr, input_file)
             return
@@ -243,6 +247,7 @@ def generate_hocr(input_file, output_hocr, language: list, engine_mode,
         raise e from e
     else:
         tesseract_log_output(log, stdout, input_file)
+
         if os.path.exists(badxml + '.html'):
             # Tesseract 3.02 appends suffix ".html" on its own (.badxml.html)
             shutil.move(badxml + '.html', badxml)
@@ -305,6 +310,9 @@ def generate_pdf(input_image, skip_pdf, output_pdf, language: list,
         shutil.copy(skip_pdf, output_pdf)
     except CalledProcessError as e:
         tesseract_log_output(log, e.output, input_image)
+        if 'read_params_file: parameter not found' in e.output:
+            raise TesseractConfigError() from e
+
         if 'Image too large' in e.output:
             shutil.copy(skip_pdf, output_pdf)
             return

ocrmypdf/exec/unpaper.py

@@ -78,7 +78,8 @@ def run(input_file, output_file, dpi, log, mode_args):
             raise e from e
         else:
             log.debug(stdout)
-            Image.open(output_pnm.name).save(output_file)
+            # unpaper sets dpi to 72
+            Image.open(output_pnm.name).save(output_file, dpi=(dpi, dpi))
 
 
 def deskew(input_file, output_file, dpi, log):

ocrmypdf/lib/compile_leptonica.py

@@ -1,8 +1,8 @@
 from cffi import FFI
 
-ffi = FFI()
-ffi.set_source("ocrmypdf.lib._leptonica", None)
-ffi.cdef("""
+ffibuilder = FFI()
+ffibuilder.set_source("ocrmypdf.lib._leptonica", None)
+ffibuilder.cdef("""
 typedef signed char             l_int8;
 typedef unsigned char           l_uint8;
 typedef short                   l_int16;
@@ -56,7 +56,7 @@ typedef struct Box    BOX;
 
 """)
 
-ffi.cdef("""
+ffibuilder.cdef("""
 PIX * pixRead ( const char *filename );
 PIX * pixScale ( PIX *pixs, l_float32 scalex, l_float32 scaley );
 l_int32 pixFindSkew ( PIX *pixs, l_float32 *pangle, l_float32 *pconf );
@@ -189,4 +189,4 @@ void lept_free(void *ptr);
 
 
 if __name__ == '__main__':
-    ffi.compile()
+    ffibuilder.compile(verbose=True)

ocrmypdf/pageinfo.py

@@ -89,13 +89,13 @@ def _is_unit_square(shorthand):
     pairwise = zip(values, UNIT_SQUARE)
     return all([isclose(a, b, rel_tol=1e-3) for a, b in pairwise])
 
-RasterSettings = namedtuple('RasterSettings',
+XobjectSettings = namedtuple('XobjectSettings',
     ['name', 'shorthand', 'stack_depth'])
 
 InlineSettings = namedtuple('InlineSettings',
     ['settings', 'shorthand', 'stack_depth'])
 
-ContentsInfo = namedtuple('ContentsInfo', ['raster_settings', 'inline_images'])
+ContentsInfo = namedtuple('ContentsInfo', ['xobject_settings', 'inline_images'])
 
 
 def _normalize_stack(operations):
@@ -114,7 +114,7 @@ def _normalize_stack(operations):
             yield (operands, command)
 
 
-def _interpret_contents(contentstream):
+def _interpret_contents(contentstream, initial_shorthand=UNIT_SQUARE):
     """Interpret the PDF content stream
 
     The stack represents the state of the PDF graphics stack.  We are only
@@ -139,8 +139,8 @@ def _interpret_contents(contentstream):
 
     operations = contentstream.operations
     stack = []
-    ctm = _matrix_from_shorthand(UNIT_SQUARE)
-    image_raster_settings = []
+    ctm = _matrix_from_shorthand(initial_shorthand)
+    xobject_settings = []
     inline_images = []
 
     for n, op in enumerate(_normalize_stack(operations)):
@@ -161,10 +161,10 @@ def _interpret_contents(contentstream):
                 _matrix_from_shorthand(operands), ctm)
         elif command == b'Do':
             image_name = operands[0]
-            raster = RasterSettings(
+            settings = XobjectSettings(
                 name=image_name, shorthand=_shorthand_from_matrix(ctm),
                 stack_depth=len(stack))
-            image_raster_settings.append(raster)
+            xobject_settings.append(settings)
         elif command == b'INLINE IMAGE':
             settings = operands['settings']
             inline = InlineSettings(
@@ -173,7 +173,7 @@ def _interpret_contents(contentstream):
             inline_images.append(inline)
 
     return ContentsInfo(
-        raster_settings=image_raster_settings,
+        xobject_settings=xobject_settings,
         inline_images=inline_images)
 
 
@@ -241,8 +241,8 @@ def _get_dpi(ctm_shorthand, image_size):
     return (dpi_w, dpi_h)
 
 
-def _find_page_inline_images(page, pageinfo, contentsinfo):
-    "Find inline images on the page"
+def _find_inline_images(contentsinfo):
+    "Find inline images in the contentstream"
 
     for n, inline in enumerate(contentsinfo.inline_images):
         image = {}
@@ -272,20 +272,44 @@ def _find_page_inline_images(page, pageinfo, contentsinfo):
         yield image
 
 
-def _find_page_regular_images(page, pageinfo, contentsinfo):
-    "Find images stored in XObject resources"
+def _image_xobjects(container):
+    """Search for all XObject-based images in the container
 
-    try:
-        page['/Resources']['/XObject']
-    except KeyError:
+    Usually the container is a page, but it could also be a Form XObject
+    that contains images. Filter out the Form XObjects which are dealt with
+    elsewhere.
+
+    Generate a sequence of tuples (image, xobj container), where container,
+    where xobj is the name of the object and image is the object itself,
+    since the object does not know its own name.
+
+    """
+
+    if '/Resources' not in container:
         return
-    for xobj in page['/Resources']['/XObject']:
-        # PyPDF2 returns the keys as an iterator
-        pdfimage = page['/Resources']['/XObject'][xobj]
-        if pdfimage['/Subtype'] != '/Image':
-            continue
+    resources = container['/Resources']
+    if '/XObject' not in resources:
+        return
+    for xobj in resources['/XObject']:
+        candidate = resources['/XObject'][xobj]
+        if candidate['/Subtype'] == '/Image':
+            image = candidate
+            yield (image, xobj)
+
+
+def _find_regular_images(container, contentsinfo):
+    """Find images stored in the container's /Resources /XObject
+
+    Usually the container is a page, but it could also be a Form XObject
+    that contains images.
+
+    Generates images with their DPI at time of drawing.
+
+    """
+
+    for pdfimage, xobj in _image_xobjects(container):
         image = {}
-        image['name'] = str(xobj)
+        image['name'] = xobj
         image['width'] = pdfimage['/Width']
         image['height'] = pdfimage['/Height']
         if '/BitsPerComponent' in pdfimage:
@@ -329,12 +353,12 @@ def _find_page_regular_images(page, pageinfo, contentsinfo):
 
         image['dpi_w'] = image['dpi_h'] = 0
 
-        for raster in contentsinfo.raster_settings:
+        for xobj in contentsinfo.xobject_settings:
             # Loop in case the same image is display multiple times on a page
-            if raster.name != image['name']:
+            if xobj.name != image['name']:
                 continue
 
-            if raster.stack_depth == 0 and _is_unit_square(raster.shorthand):
+            if xobj.stack_depth == 0 and _is_unit_square(xobj.shorthand):
                 # At least one PDF in the wild (and test suite) draws an image
                 # when the graphics stack depth is 0, meaning that the image
                 # gets drawn into a square of 1x1 PDF units (or 1/72",
@@ -343,7 +367,7 @@ def _find_page_regular_images(page, pageinfo, contentsinfo):
                 continue
 
             dpi_w, dpi_h = _get_dpi(
-                raster.shorthand, (image['width'], image['height']))
+                xobj.shorthand, (image['width'], image['height']))
 
             # When image is used multiple times take the highest DPI it is
             # rendered at
@@ -358,9 +382,85 @@ def _find_page_regular_images(page, pageinfo, contentsinfo):
         yield image
 
 
-def _find_page_images(page, pageinfo, contentsinfo):
-    yield from _find_page_inline_images(page, pageinfo, contentsinfo)
-    yield from _find_page_regular_images(page, pageinfo, contentsinfo)
+def _find_form_xobject_images(pdf, container, contentsinfo):
+    """Find any images that are in Form XObjects in the container
+
+    The container may be a page, or a parent Form XObject.
+
+    """
+    if '/Resources' not in container:
+        return
+    resources = container['/Resources']
+    if '/XObject' not in resources:
+        return
+    for xobj in resources['/XObject']:
+        candidate = resources['/XObject'][xobj]
+        if candidate['/Subtype'] != '/Form':
+            continue
+
+        form_xobject = candidate
+        for settings in contentsinfo.xobject_settings:
+            if settings.name != xobj:
+                continue
+
+            # Find images once for each time this Form XObject is drawn.
+            # This could be optimized to cache the multiple drawing events
+            # but in practice both Form XObjects and multiple drawing of the
+            # same object are both very rare.
+            ctm_shorthand = settings.shorthand
+            yield from _find_images(pdf, form_xobject, ctm_shorthand)
+
+
+def _find_images(pdf, container, shorthand=None):
+    """Find all individual instances of images drawn in the container
+
+    Usually the container is a page, but it may also be a Form XObject.
+
+    On a typical page images are stored inline or as regular images
+    in an XObject.
+
+    Form XObjects may include inline images, XObject images,
+    and recursively, other Form XObjects; and also vector drawing commands.
+
+    Every instance of an image being drawn somewhere is flattened and
+    treated as a unique image, since if the same image is drawn multiple times
+    on one page it may be drawn at differing resolutions, and our objective
+    is to find the resolution at which the page can be rastered without
+    downsampling.
+
+    """
+
+    if container.get('/Type') == '/Page':
+        # For a /Page the content stream is attached to the page's /Contents
+        page = container
+        contentstream = pypdf.pdf.ContentStream(page.getContents(), pdf)
+        initial_shorthand = shorthand or UNIT_SQUARE
+    elif container.get('/Type') == '/XObject' and \
+            container['/Subtype'] == '/Form':
+        # For a Form XObject that content stream is attached to the XObject
+        contentstream = pypdf.pdf.ContentStream(container, pdf)
+
+        # Set the CTM to the state it was when the "Do" operator was
+        # encountered that is drawing this instance of the Form XObject
+        ctm = _matrix_from_shorthand(shorthand or UNIT_SQUARE)
+
+        # A Form XObject may provide its own matrix to map form space into
+        # user space. Get this if one exists
+        form_matrix = _matrix_from_shorthand(
+                container.get('/Matrix', UNIT_SQUARE))
+
+        # Concatenate form matrix with CTM to ensure CTM is correct for
+        # drawing this instance of the XObject
+        ctm = matrix_mult(form_matrix, ctm)
+        initial_shorthand = _shorthand_from_matrix(ctm)
+    else:
+        return
+
+    contentsinfo = _interpret_contents(contentstream, initial_shorthand)
+
+    yield from _find_inline_images(contentsinfo)
+    yield from _find_regular_images(container, contentsinfo)
+    yield from _find_form_xobject_images(pdf, container, contentsinfo)
 
 
 def _page_has_text(pdf, page):
@@ -405,15 +505,8 @@ def _pdf_get_pageinfo(infile, pageno: int):
     except KeyError:
         pageinfo['rotate'] = 0
 
-    try:
-        contentstream = pypdf.pdf.ContentStream(page.getContents(), pdf)
-    except AttributeError as e:
-        return pageinfo
-
-    contentsinfo = _interpret_contents(contentstream)
-    pageinfo['images'] = [im for im in _find_page_images(
-                                page, pageinfo, contentsinfo)]
-
+    pageinfo['images'] = [im for im in
+                          _find_images(pdf, page)]
     if pageinfo['images']:
         xres = max(image['dpi_w'] for image in pageinfo['images'])
         yres = max(image['dpi_h'] for image in pageinfo['images'])

ocrmypdf/pipeline.py

@@ -555,16 +555,21 @@ def select_image_layer(
         re_symlink(page_pdf, output_file, log)
     else:
         pageinfo = get_pageinfo(image, context)
-        dpi = get_page_dpi(pageinfo, options)
-        dpi = float(dpi[0]), float(dpi[1])
-        layout_fun = img2pdf.get_fixed_dpi_layout_fun(dpi)
+
+        # We rasterize a square DPI version of each page because most image
+        # processing tools don't support rectangular DPI. Use the square DPI
+        # as it accurately describes the image. It would be possible to
+        # resample the image at this stage back to non-square DPI to more
+        # closely resemble the input, except that the hocr renderer does not
+        # understand non-square DPI. The tess4 renderer would be fine.
+        dpi = get_page_square_dpi(pageinfo, options)
+        layout_fun = img2pdf.get_fixed_dpi_layout_fun((dpi, dpi))
 
         with open(image, 'rb') as imfile, \
                 open(output_file, 'wb') as pdf:
-            rawdata = imfile.read()
             log.debug('{:4d}: convert'.format(page_number(page_pdf)))
             img2pdf.convert(
-                rawdata, with_pdfrw=False,
+                imfile, with_pdfrw=False,
                 layout_fun=layout_fun, outputstream=pdf)
             log.debug('{:4d}: convert done'.format(page_number(page_pdf)))
 
@@ -948,7 +953,7 @@ def build_pipeline(options, work_folder, log, context):
     task_ocr_tesseract_hocr.graphviz(fillcolor='"#00cc66"')
     task_ocr_tesseract_hocr.active_if(options.pdf_renderer == 'hocr')
     if tesseract.v4():
-        task_ocr_tesseract_hocr.jobs_limit(1)  # Uses multi-core on its own
+        task_ocr_tesseract_hocr.jobs_limit(2)  # Uses multi-core on its own
 
     task_select_visible_page_image = main_pipeline.collate(
         task_func=select_visible_page_image,
@@ -1001,7 +1006,7 @@ def build_pipeline(options, work_folder, log, context):
     task_ocr_tesseract_textonly_pdf.graphviz(fillcolor='"#ff69b4"')
     task_ocr_tesseract_textonly_pdf.active_if(options.pdf_renderer == 'tess4')
     if tesseract.v4():
-        task_ocr_tesseract_textonly_pdf.jobs_limit(1)
+        task_ocr_tesseract_textonly_pdf.jobs_limit(2)
 
     task_combine_layers = main_pipeline.collate(
         task_func=combine_layers,
@@ -1024,7 +1029,7 @@ def build_pipeline(options, work_folder, log, context):
     task_ocr_tesseract_and_render_pdf.graphviz(fillcolor='"#66ccff"')
     task_ocr_tesseract_and_render_pdf.active_if(options.pdf_renderer == 'tesseract')
     if tesseract.v4():
-        task_ocr_tesseract_and_render_pdf.jobs_limit(1)  # Uses multi-core
+        task_ocr_tesseract_and_render_pdf.jobs_limit(2)  # Uses multi-core
 
     # PDF/A
     task_generate_postscript_stub = main_pipeline.transform(

requirements.txt

@@ -2,8 +2,8 @@
 # setup.py lists a separate set of requirements that are looser to simplify
 # installation
 ruffus == 2.6.3
-Pillow == 3.4.2
+Pillow == 4.0.0
 reportlab == 3.3.0
 PyPDF2 == 1.26
-img2pdf == 0.2.1
+img2pdf == 0.2.3
 cffi == 1.9.1

setup.py

@@ -9,7 +9,7 @@ if sys.version_info < (3, 4):
     print("Python 3.4 or newer is required", file=sys.stderr)
     sys.exit(1)
 
-from setuptools import setup  # nopep8
+from setuptools import setup, find_packages  # nopep8
 from subprocess import STDOUT, check_output, CalledProcessError  # nopep8
 from collections.abc import Mapping  # nopep8
 import re  # nopep8
@@ -192,7 +192,7 @@ setup(
     author='James R. Barlow',
     author_email='jim@purplerock.ca',
     license='MIT',
-    packages=['ocrmypdf'],
+    packages=find_packages(exclude=["tests", "tests.*"]),
     keywords=['PDF', 'OCR', 'optical character recognition', 'PDF/A', 'scanning'],
     classifiers=[
         "Programming Language :: Python :: 3",
@@ -214,22 +214,20 @@ setup(
         "Topic :: Text Processing :: Linguistic",
         ],
     setup_requires=[
-        'setuptools_scm',
-        'cffi>=1.5.0',
-        'pytest-runner',
-        'pytest-helpers-namespace'
+        'setuptools_scm',       # so that version will work
+        'cffi>=1.9.1'           # to build the leptonica module
     ],
     use_scm_version={'version_scheme': 'post-release'},
     cffi_modules=[
-        'ocrmypdf/lib/compile_leptonica.py:ffi'
+        'ocrmypdf/lib/compile_leptonica.py:ffibuilder'
     ],
     install_requires=[
         'ruffus==2.6.3',        # pinned - ocrmypdf implements a 2.6.3 workaround
-        'Pillow>=3.1.1',        # Pillow is pretty stable
+        'Pillow>=4.0.0',        # Pillow < 4 has BytesIO/TIFF bug w/img2pdf 0.2.3
         'reportlab>=3.2.0',     # oldest released version with sane image handling
         'PyPDF2>=1.26',         # pure Python, so track HEAD closely
-        'img2pdf>=0.2.1',       # pure Python, so track HEAD closely
-        'cffi>=1.5.0'           # oldest version ever tested
+        'img2pdf>=0.2.3',       # pure Python, so track HEAD closely
+        'cffi>=1.9.1'           # must be a setup and install requirement
     ],
     tests_require=tests_require,
     entry_points={

share/etc-apt-sources.list

@@ -1,4 +0,0 @@
-deb http://httpredir.debian.org/debian stretch main
-deb http://httpredir.debian.org/debian stretch-updates main
-deb http://security.debian.org stretch/updates main
-deb http://ftp.de.debian.org/debian sid main contrib non-free

test_requirements.txt

@@ -1,2 +1,2 @@
-pytest >= 2.8
+pytest >= 3.0
 pytest-helpers-namespace

tests/conftest.py

@@ -22,10 +22,21 @@ def is_linux():
     return platform.system() == 'Linux'
 
 
+@pytest.helpers.register
+def is_macos():
+    return platform.system() == 'Darwin'
+
+
 @pytest.helpers.register
 def running_in_docker():
-    # Docker creates a file named /.dockerinit
-    return os.path.exists('/.dockerinit')
+    # Docker creates a file named /.dockerenv (newer versions) or
+    # /.dockerinit (older) -- this is undocumented, not an offical test
+    return os.path.exists('/.dockerenv') or os.path.exists('/.dockerinit')
+
+
+@pytest.helpers.register
+def running_in_travis():
+    return os.environ.get('TRAVIS') == 'true'
 
 
 TESTS_ROOT = os.path.abspath(os.path.dirname(__file__))
@@ -90,7 +101,7 @@ def check_ocrmypdf(input_file, output_file, *args, env=None):
 
     p, out, err = run_ocrmypdf(input_file, output_file, *args, env=env)
     #print(err)  # ensure py.test collects the output, use -s to view
-    assert p.returncode == 0
+    assert p.returncode == 0, "<stderr>\n" + err + "\n</stderr>"
     assert os.path.exists(str(output_file)), "Output file not created"
     assert os.stat(str(output_file)).st_size > 100, "PDF too small or empty"
     assert out == "", \
@@ -114,3 +125,11 @@ def run_ocrmypdf(input_file, output_file, *args, env=None):
     #print(err)
 
     return p, out, err
+
+
+@pytest.helpers.register
+def first_page_dimensions(pdf):
+    from ocrmypdf import pageinfo
+    info = pageinfo.pdf_get_all_pageinfo(str(pdf))
+    page0 = info[0]
+    return (page0['width_inches'], page0['height_inches'])

tests/resources/README.rst

@@ -64,6 +64,9 @@ under the terms of the license in LICENSE.rst.
     *   - epson.pdf
         - @lowesjam
         - a linearized PDF containing some unusual indirect objects, created by an Epson printer; printout of a Wikipedia article (CC BY-SA)
+    *   - formxobject.pdf
+        - @jbarlow83
+        - hand-crafted exotic PDF containing an image inside a Form XObject
     *   - francais.pdf
         - @jbarlow83
         - a page containing French accents (diacritics)  

tests/spoof/tesseract_cache.py

@@ -37,6 +37,7 @@ def real_tesseract():
 def main():
     operation = sys.argv[-1]
     # For anything unexpected operation, defer to real tesseract binary
+    # Currently this includes all use of "--tesseract-config"
     if operation != 'hocr' and operation != 'pdf' and operation != 'stdout':
         real_tesseract()
         return  # Not reachable
@@ -52,6 +53,11 @@ def main():
         ['tesseract', '--version'],
         stderr=subprocess.STDOUT)
 
+    if b'4.00.00alpha' in tess_version:
+        # Tesseract 4.x alpha is a moving target, don't cache it
+        real_tesseract()
+        return
+
     m.update(tess_version)
 
     # Insert this source file into the hash function, to ensure that any
@@ -65,12 +71,21 @@ def main():
         lang = sys.argv[sys.argv.index('-l') + 1]
         m.update(lang.encode())
     except ValueError:
-        pass
-    try:
-        psm = sys.argv[sys.argv.index('-psm') + 1]
-        m.update(psm.encode())
-    except ValueError:
-        pass
+        m.update(b'default-lang')
+
+    psm_arg = ''
+    if '--psm' in sys.argv:
+        psm_arg = '--psm'
+    elif '-psm' in sys.argv:
+        psm_arg = '-psm'
+    if psm_arg:
+        try:
+            psm = sys.argv[sys.argv.index(psm_arg) + 1]
+            m.update(psm.encode())
+        except ValueError:
+            m.update(b'default-psm')
+    else:
+        m.update(b'default-psm')
 
     if operation == 'stdout' and psm != '0':
         real_tesseract()

tests/test_main.py

@@ -12,6 +12,7 @@ import PyPDF2 as pypdf
 from ocrmypdf.exceptions import ExitCode
 from ocrmypdf import leptonica
 from ocrmypdf.pdfa import file_claims_pdfa
+from ocrmypdf.exec import ghostscript
 
 
 check_ocrmypdf = pytest.helpers.check_ocrmypdf
@@ -169,6 +170,13 @@ def test_preserve_metadata(spoof_tesseract_noop, output_type,
 @pytest.mark.skipif(
     pytest.helpers.is_linux() and not pytest.helpers.running_in_docker(),
     reason="likely to fail if Linux locale is not configured correctly")
+@pytest.mark.skipif(
+    pytest.helpers.is_macos() and pytest.helpers.running_in_travis(),
+    reason="save Travis the trouble of installing poppler")
+@pytest.mark.xfail(
+    ghostscript.version() == '9.21',
+    reason="gs 9.21 has a regression that affects this"
+    )
 @pytest.mark.parametrize("output_type", [
     'pdfa', 'pdf'
     ])
@@ -188,7 +196,7 @@ def test_override_metadata(spoof_tesseract_noop, output_type, resources,
         '--output-type', output_type,
         env=spoof_tesseract_noop)
 
-    assert p.returncode == ExitCode.ok
+    assert p.returncode == ExitCode.ok, err
 
     pdf = str(outpdf)
 
@@ -409,10 +417,14 @@ def test_force_ocr_on_pdf_with_no_images(spoof_tesseract_crash, resources,
     assert not os.path.exists(no_outpdf)
 
 
+@pytest.mark.skipif(
+    pytest.helpers.is_macos() and pytest.helpers.running_in_travis(),
+    reason="takes too long to install language packs in Travis macOS homebrew")
 def test_french(spoof_tesseract_cache, resources, outpdf):
     p, out, err = run_ocrmypdf(
         resources / 'francais.pdf', outpdf, '-l', 'fra',
         env=spoof_tesseract_cache)
+    print(os.environ)
     assert p.returncode == ExitCode.ok, \
         "This test may fail if Tesseract language packs are missing"
 
@@ -564,6 +576,46 @@ def test_non_square_resolution(renderer, spoof_tesseract_cache,
     assert in_pageinfo[0]['yres'] == out_pageinfo[0]['yres']
 
 
+@pytest.mark.skipif(sys.version_info < (3, 5), reason="needs math.isclose")
+@pytest.mark.parametrize('renderer', [
+    'hocr',
+    'tesseract'
+    ])
+def test_convert_to_square_resolution(renderer, spoof_tesseract_cache,
+                                      resources, outpdf):
+    from math import isclose
+
+    # Confirm input image is non-square resolution
+    in_pageinfo = pdf_get_all_pageinfo(str(resources / 'aspect.pdf'))
+    assert in_pageinfo[0]['xres'] != in_pageinfo[0]['yres']
+
+    # --force-ocr requires means forced conversion to square resolution
+    check_ocrmypdf(
+        resources / 'aspect.pdf', outpdf,
+        '--force-ocr',
+        '--pdf-renderer', renderer, env=spoof_tesseract_cache)
+
+    out_pageinfo = pdf_get_all_pageinfo(str(outpdf))
+
+    in_p0, out_p0 = in_pageinfo[0], out_pageinfo[0]
+
+    # Resolution show now be equal
+    assert out_p0['xres'] == out_p0['yres']
+
+    # Page size should match input page size
+    assert isclose(in_p0['width_inches'],
+                   out_p0['width_inches'])
+    assert isclose(in_p0['height_inches'],
+                   out_p0['height_inches'])
+
+    # Because we rasterized the page to produce a new image, it should occupy
+    # the entire page
+    out_im_w = out_p0['images'][0]['width'] / out_p0['images'][0]['dpi_w']
+    out_im_h = out_p0['images'][0]['height'] / out_p0['images'][0]['dpi_h']
+    assert isclose(out_p0['width_inches'], out_im_w)
+    assert isclose(out_p0['height_inches'], out_im_h)
+
+
 def test_image_to_pdf(spoof_tesseract_noop, resources, outpdf):
     check_ocrmypdf(
         resources / 'LinnSequencer.jpg', outpdf, '--image-dpi', '200',
@@ -614,6 +666,23 @@ def test_stdout(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf):
     assert qpdf.check(output_file, log=None)
 
 
+def test_closed_streams(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf):
+    input_file = str(resources / 'francais.pdf')
+    output_file = str(outpdf)
+
+    def evil_closer():
+        os.close(0)
+        os.close(1)
+
+    p_args = ocrmypdf_exec + [input_file, output_file]
+    p = Popen(
+        p_args, close_fds=True, stdout=None, stderr=PIPE, stdin=None,
+        env=spoof_tesseract_noop, preexec_fn=evil_closer)
+    out, err = p.communicate()
+    print(err.decode())
+    assert p.returncode == ExitCode.ok
+
+
 def test_masks(spoof_tesseract_noop, resources, outpdf):
     check_ocrmypdf(resources / 'masks.pdf', outpdf, env=spoof_tesseract_noop)
 
@@ -682,6 +751,10 @@ def test_overlay(spoof_tesseract_noop, resources, outpdf):
                    env=spoof_tesseract_noop)
 
 
+@pytest.mark.skipif(
+    os.getuid() == 0 or os.geteuid() == 0,
+    reason="root can write to anything"
+    )
 def test_destination_not_writable(spoof_tesseract_noop, resources, outdir):
     protected_file = outdir / 'protected.pdf'
     protected_file.touch()
@@ -690,3 +763,82 @@ def test_destination_not_writable(spoof_tesseract_noop, resources, outdir):
         resources / 'jbig2.pdf', protected_file,
         env=spoof_tesseract_noop)
     assert p.returncode == ExitCode.file_access_error, "Expected error"
+
+
+def test_tesseract_config_valid(resources, outdir):
+    cfg_file = outdir / 'test.cfg'
+    with cfg_file.open('w') as f:
+        f.write('''\
+load_system_dawg 0
+language_model_penalty_non_dict_word 0
+language_model_penalty_non_freq_dict_word 0
+''')
+
+    check_ocrmypdf(
+        resources / 'ccitt.pdf', outdir / 'out.pdf',
+        '--tesseract-config', str(cfg_file))
+
+
+@pytest.mark.parametrize('renderer', [
+    'hocr',
+    'tesseract',
+    ])
+def test_tesseract_config_notfound(renderer, resources, outdir):
+    cfg_file = outdir / 'nofile.cfg'
+
+    p, out, err = run_ocrmypdf(
+        resources / 'ccitt.pdf', outdir / 'out.pdf',
+        '--pdf-renderer', renderer,
+        '--tesseract-config', str(cfg_file))
+    assert "Can't open" in err, "No error message about missing config file"
+    assert p.returncode == ExitCode.ok
+
+
+@pytest.mark.parametrize('renderer', [
+    'hocr',
+    'tesseract',
+    ])
+def test_tesseract_config_invalid(renderer, resources, outdir):
+    cfg_file = outdir / 'test.cfg'
+    with cfg_file.open('w') as f:
+        f.write('''\
+THIS FILE IS INVALID
+''')
+
+    p, out, err = run_ocrmypdf(
+        resources / 'ccitt.pdf', outdir / 'out.pdf',
+        '--pdf-renderer', renderer,
+        '--tesseract-config', str(cfg_file))
+    assert "parameter not found" in err, "No error message"
+    assert p.returncode == ExitCode.invalid_config
+
+
+def test_form_xobject(spoof_tesseract_noop, resources, outpdf):
+    check_ocrmypdf(resources / 'formxobject.pdf', outpdf,
+                   '--force-ocr',
+                   env=spoof_tesseract_noop)
+
+
+@pytest.mark.skipif(sys.version_info < (3, 5), reason="needs math.isclose")
+@pytest.mark.parametrize('renderer', [
+    'hocr',
+    'tesseract',
+])
+def test_pagesize_consistency(renderer, resources, outpdf):
+    from math import isclose
+
+    first_page_dimensions = pytest.helpers.first_page_dimensions
+
+    infile = resources / 'linn.pdf'
+
+    before_dims = first_page_dimensions(infile)
+
+    check_ocrmypdf(
+        infile,
+        outpdf, '--pdf-renderer', renderer,
+        '--clean', '--deskew', '--remove-background', '--clean-final')
+
+    after_dims = first_page_dimensions(outpdf)
+
+    assert isclose(before_dims[0], after_dims[0])
+    assert isclose(before_dims[1], after_dims[1])

tests/test_pageinfo.py

@@ -101,3 +101,10 @@ def test_jpeg(resources, outdir):
     assert pdfimage['enc'] == 'jpeg'
     assert (pdfimage['dpi_w'] - 150) < 1e-5
 
+
+def test_form_xobject(resources):
+    filename = resources / 'formxobject.pdf'
+
+    pdfinfo = pageinfo.pdf_get_all_pageinfo(str(filename))
+    pdfimage = pdfinfo[0]['images'][0]
+    assert pdfimage['width'] == 50

tests/test_tess3.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+# © 2017 James R. Barlow: github.com/jbarlow83
+
+import pytest
+from ocrmypdf.exceptions import ExitCode
+from ocrmypdf.exec import tesseract
+
+
+# Skip all tests in this file if not tesseract 3
+pytestmark = pytest.mark.skipif(tesseract.v4(),
+                                reason="tesseract 3.x required")
+
+
+def test_textonly_pdf_on_tess3(resources, no_outpdf):
+    p, _, _ = pytest.helpers.run_ocrmypdf(
+        resources / 'linn.pdf',
+        no_outpdf, '--pdf-renderer', 'tess4')
+
+    assert p.returncode == ExitCode.missing_dependency
+
+
+def test_oem_on_tess3(resources, no_outpdf):
+    p, _, err = pytest.helpers.run_ocrmypdf(
+        resources / 'aspect.pdf',
+        no_outpdf, '--tesseract-oem', '1')
+
+    assert p.returncode == ExitCode.ok
+    assert 'argument ignored' in err

tests/test_tess4.py

@@ -1,31 +1,43 @@
 #!/usr/bin/env python3
 # © 2017 James R. Barlow: github.com/jbarlow83
 
-from subprocess import Popen, PIPE, check_output, check_call, DEVNULL
-import os
-import shutil
-from contextlib import suppress
-import sys
 import pytest
-from ocrmypdf.pageinfo import pdf_get_all_pageinfo
-import PyPDF2 as pypdf
 from ocrmypdf.exceptions import ExitCode
-from ocrmypdf import leptonica
-from ocrmypdf.pdfa import file_claims_pdfa
 from ocrmypdf.exec import tesseract
+from ocrmypdf import pageinfo
+import sys
 
 
 # Skip all tests in this file if not tesseract 4
-pytestmark = pytest.mark.skipif(not tesseract.v4(),
-                                reason="tesseract 4.0 required")
+pytestmark = pytest.mark.skipif(
+    not (tesseract.v4() and tesseract.has_textonly_pdf()),
+    reason="tesseract 4.0 with textonly_pdf feature required")
+
+check_ocrmypdf = pytest.helpers.check_ocrmypdf
+run_ocrmypdf = pytest.helpers.run_ocrmypdf
+spoof = pytest.helpers.spoof
 
 
-@pytest.mark.skipif(not tesseract.has_textonly_pdf(),
-                    reason="requires textonly_pdf feature")
 def test_textonly_pdf(resources, outdir):
-    pytest.helpers.check_ocrmypdf(
+    check_ocrmypdf(
         resources / 'linn.pdf',
         outdir / 'linn_textonly.pdf', '--pdf-renderer', 'tess4')
 
 
+@pytest.mark.skipif(sys.version_info < (3, 5), reason="needs math.isclose")
+def test_pagesize_consistency_tess4(resources, outpdf):
+    from math import isclose
 
+    infile = resources / 'linn.pdf'
+
+    before_dims = pytest.helpers.first_page_dimensions(infile)
+
+    check_ocrmypdf(
+        infile,
+        outpdf, '--pdf-renderer', 'tess4',
+        '--clean', '--deskew', '--remove-background', '--clean-final')
+
+    after_dims = pytest.helpers.first_page_dimensions(outpdf)
+
+    assert isclose(before_dims[0], after_dims[0])
+    assert isclose(before_dims[1], after_dims[1])