Ghostcript 9.21 seems to have a regression related to Unicode metadata

Not needed since reportlab 3.4 comes with a wheel, and that was the main difficulty.

.dockerignore

@@ -11,11 +11,14 @@
 */*/*/*/*/*/*.pyc
 */*/*/*/*/*/*/*.pyc
 .cache/
+.eggs/
+.github/
 .git/
 .ipynb_checkpoints/
 .ruffus_history.sqlite
 bin/
 build/
+docs/
 dist/
 htmlcov/
 include/
@@ -27,5 +30,5 @@ tests/cache/
 tests/output/
 tests/resources/private/
 tmp/
-venv-3.4/
-venv-3.5/
+venv*/
+wheelhouse/

.gitignore

@@ -4,6 +4,8 @@
 venv*/
 pyvenv.cfg
 tasks.py
+.bash_history
+.ruffus_history.sqlite
 
 # Package building
 *.egg-info/
@@ -27,8 +29,11 @@ htmlcov/
 # Testing
 log/
 /*.pdf
+/*.qdf
+*.ipynb
 .ipynb_checkpoints/
 tests/cache/
 tests/output/
 tests/resources/private/
 tmp/
+pdfbox-app*.jar

.travis.yml

@@ -1,47 +1,60 @@
-sudo: required
 dist: trusty
 language: python
-
 cache:
-    pip: true
-    directories:
-        - packages
-        - tests/cache
+  pip: true
+  directories:
+  - packages
+  - tests/cache
+  - $HOME/Library/Caches/Homebrew
 
-python:
-    - "3.4"
-    - "3.5"
-    - "3.6"
+matrix:
+  include:
+    - os: linux
+      sudo: required
+      language: python
+      python: 3.4
+    - os: linux
+      sudo: required
+      language: python
+      python: 3.5
+    - os: linux
+      sudo: required
+      language: python
+      python: 3.6
+    - os: osx
+      osx_image: xcode8
+      language: generic
 
 before_cache:
-    - rm -f $HOME/.cache/pip/log/debug.log
+- rm -f $HOME/.cache/pip/log/debug.log
 
 before_install:
-    # Ubuntu packages
-    - sudo add-apt-repository ppa:vshn/ghostscript -y  # for ghostscript 9.16 (trusty has 9.10)
-    - sudo add-apt-repository ppa:heyarje/libav-11 -y  # for libav11, which is for unpaper
-    - sudo apt-get update -qq  # must go after all add-apt-repo
-    - sudo apt-get install -y ghostscript tesseract-ocr tesseract-ocr-deu tesseract-ocr-eng tesseract-ocr-fra qpdf poppler-utils libavformat56 libavcodec56 libavutil54 libffi-dev
-
-    # pip
-    - pip install --upgrade pip
-
-    # install unpaper from a previously built .deb since ubuntu's version is too old
-    - mkdir -p packages
-    - "[ -f packages/unpaper_6.1-1.deb ] || wget -q https://dl.dropboxusercontent.com/u/28971240/unpaper_6.1-1.deb -O packages/unpaper_6.1-1.deb"
-    - sudo dpkg -i packages/unpaper_6.1-1.deb
+- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then bash .travis/linux_before_install.sh ; fi
+- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then bash .travis/osx_before_install.sh ; fi
 
 install:
-    - pip install .
-    - pip install -r requirements.txt -r test_requirements.txt
+- pip3 install .
+- pip3 install -r requirements.txt -r test_requirements.txt
 
 script:
-    # "pip install" installs the ocrmypdf into the system environment.
-    # The working copy of ocrmypdf is not an installed version and its cffi
-    # module has not been compiled. As such, move the local copy out of the
-    # way and ensure that the system copy gets tested.
-    - mv ocrmypdf dont_import_this_ocrmypdf
-    - pytest
+- mv ocrmypdf dont_import_this_ocrmypdf
+- pytest
+- mv dont_import_this_ocrmypdf ocrmypdf
 
-os:
-    - linux
+after_success:
+- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then bash .travis/osx_brew.sh ; fi
+
+# See https://www.appneta.com/blog/pypi-deployment-with-travis-ci/ for
+# steps to set up testpypi deploy for untagged builds if desired
+
+deploy:
+  provider: pypi
+  user: ocrmypdf-travis
+  password:
+    secure: DTFOmmNL6olA0+yXvp4u9jXZlZeqrJsJ0526jzqf4a3gZ6jnGTq5UI6WzRsslSyoMMfXKtHQebqHM6ogSgCZinyZ3ufHJo8fn9brxbEc2gsiWkbj5o3bGwdWMT1vNNE7XW0VCpw87rZ1EEwjl4FJHFudMlPR1yfU5+uq0k0PACo=
+  distributions: "sdist bdist_wheel"
+  on:
+    branch: master
+    tags: true
+    condition: $TRAVIS_PYTHON_VERSION == "3.6" && $TRAVIS_OS_NAME == "linux"
+  skip_upload_docs: true

.travis/autobrew.py

@@ -0,0 +1,75 @@
+from string import Template
+from subprocess import run, PIPE
+import re
+
+recipe_template = Template("""
+class Ocrmypdf < Formula
+  include Language::Python::Virtualenv
+
+  desc "Adds an OCR text layer to scanned PDF files"
+  homepage "https://github.com/jbarlow83/OCRmyPDF"
+  ${ocrmypdf_url}
+  ${ocrmypdf_sha256}
+
+  depends_on :python3
+  depends_on "pkg-config" => :build
+  depends_on "zlib"
+  depends_on "libffi"
+  depends_on "tesseract"
+  depends_on "ghostscript"
+  depends_on "unpaper"
+  depends_on "qpdf"
+
+  # For Pillow source install
+  depends_on "openjpeg"
+  depends_on "freetype"
+  depends_on "libpng"
+
+${resources}
+
+  def install
+    ENV.append ["SETUPTOOLS_SCM_PRETEND_VERSION"], "v${ocrmypdf_version}"
+    ENV.each do |key, value|
+      puts "#{key}:#{value}"
+    end
+    virtualenv_install_with_resources
+  end
+
+  test do
+    # `test do` will create, run in and delete a temporary directory.
+    #
+    # The installed folder is not in the path, so use the entire path to any
+    # executables being tested: `system "#{bin}/program", "do", "something"`.
+    system "#{bin}/ocrmypdf", "--version"
+  end
+end
+""")
+
+def main():
+    p = run(['poet', '--single', 'ocrmypdf'],
+            encoding='utf-8', stdout=PIPE, check=True)
+
+    ocrmypdf_lines = p.stdout.splitlines()
+    ocrmypdf_url = ocrmypdf_lines[1].strip()
+    ocrmypdf_sha256 = ocrmypdf_lines[2].strip()
+
+    ocrmypdf_version = re.search(
+        r'ocrmypdf-(.+)\.tar.*', ocrmypdf_url).group(1)
+    print(f"Autobrewing {ocrmypdf_version}")
+
+    p = run(['poet', '--resources', 'ocrmypdf'],
+            encoding='utf-8', stdout=PIPE, check=True)
+
+    poet_resources = p.stdout
+
+    # Remove the duplicate "ocrmypdf" resource block
+    all_resources = poet_resources.split('resource')
+    kept_resources = [block for block in all_resources if 'ocrmypdf' not in block]
+    resources = 'resource'.join(kept_resources)
+
+    with open('ocrmypdf.rb', 'w') as out:
+        out.write(recipe_template.substitute(**locals()))
+
+
+if __name__ == '__main__':
+    main()

.travis/linux_before_install.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+set -euo pipefail
+set -x
+
+sudo add-apt-repository ppa:vshn/ghostscript -y
+sudo add-apt-repository ppa:heyarje/libav-11 -y
+sudo apt-get update -qq
+sudo apt-get install -y \
+	ghostscript \
+	tesseract-ocr \
+	tesseract-ocr-deu \
+	tesseract-ocr-eng \
+	tesseract-ocr-fra \
+	qpdf \
+	poppler-utils \
+	libavformat56 \
+	libavcodec56 \
+	libavutil54 \
+	libffi-dev
+
+pip install --upgrade pip
+mkdir -p packages
+[ -f packages/unpaper_6.1-1.deb ] || wget -q https://dl.dropboxusercontent.com/u/28971240/unpaper_6.1-1.deb -O packages/unpaper_6.1-1.deb
+sudo dpkg -i packages/unpaper_6.1-1.deb

.travis/osx_before_install.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -euo pipefail
+set -x
+
+brew update
+
+brew install openjpeg jbig2dec libtiff     # image libraries
+brew install qpdf
+brew install ghostscript
+brew install python3
+brew install libxml2 libffi leptonica
+brew install unpaper   # optional
+brew install tesseract
+
+pip3 install --upgrade pip
+pip3 install wheel

.travis/osx_brew.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -euo pipefail
+set -x
+
+pip3 install homebrew-pypi-poet
+python3 .travis/autobrew.py
+brew audit ocrmypdf.rb
+cat ocrmypdf.rb

Dockerfile

@@ -1,91 +1,54 @@
 # OCRmyPDF
 #
-# VERSION               3.2
-FROM      debian:stretch
+# VERSION               4.4.2
+FROM      ubuntu:16.10
 MAINTAINER James R. Barlow <jim@purplerock.ca>
 
-# Add unprivileged user
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  software-properties-common python-software-properties \
+  python3-wheel \
+  python3-reportlab \
+  python3-venv \
+  ghostscript \
+  qpdf \
+  poppler-utils \
+  unpaper \
+  libffi-dev \ 
+  tesseract-ocr \
+  tesseract-ocr-eng \
+  tesseract-ocr-fra \
+  tesseract-ocr-spa \
+  tesseract-ocr-deu
+
+RUN python3 -m venv --system-site-packages /appenv
+
+# This installs the latest binary wheel instead of the code in the current
+# folder. Installing from source will fail, apparently because cffi needs
+# build-essentials (gcc) to do a source installation 
+# (i.e. "pip install ."). It's unclear to me why this is the case.
+RUN . /appenv/bin/activate; \
+  pip install --upgrade pip \
+  && pip install ocrmypdf
+
+# Now copy the application in, mainly to get the test suite.
+# Do this now to make the best use of Docker cache.
+COPY . /application
+RUN . /appenv/bin/activate; \
+  pip install -r /application/test_requirements.txt
+
+# Remove the junk, including the source version of application since it was
+# already installed
+RUN rm -rf /tmp/* /var/tmp/* /root/* /application/ocrmypdf \
+  && apt-get autoremove -y \
+  && apt-get autoclean -y
+
 RUN useradd docker \
   && mkdir /home/docker \
   && chown docker:docker /home/docker
 
-# Update system and install our dependencies
-# If this command takes too Docker hub's automated build will timeout,
-# so try it in portions
-RUN apt-get update && apt-get install -y --no-install-recommends \
-  locales \
-  python3 \
-  python3-pip \
-  python3-venv \
-  python3-reportlab \
-  python3-pil \
-  python3-wheel
-
-RUN apt-get install -y --no-install-recommends \
-  unpaper \
-  qpdf \
-  poppler-utils \
-  tesseract-ocr \
-  tesseract-ocr-deu tesseract-ocr-spa tesseract-ocr-eng tesseract-ocr-fra
-
-RUN apt-get install -qy --no-install-recommends \
-  libffi-dev \
-  libpython3-dev \
-  gcc
-
-# Install Ghostscript from Debian sid to work around JPEG 2000 issue in
-# Debian stretch libgs9 or gs 9.16~dfsg-2.1
-
-COPY ./share/etc-apt-sources.list /etc/apt/sources.list
-
-RUN apt-get update && apt-get install -y ghostscript/sid
-
-
-# Enforce UTF-8
-# Borrowed from https://index.docker.io/u/crosbymichael/python/ 
-RUN dpkg-reconfigure locales && \
-  locale-gen C.UTF-8 && \
-  /usr/sbin/update-locale LANG=C.UTF-8
-ENV LC_ALL C.UTF-8
-
-
-# Set up a Python virtualenv and take all of the system packages, so we can
-# rely on the platform packages rather than importing GCC and compiling them
-RUN pyvenv /appenv \
-  && pyvenv --system-site-packages /appenv
-
-COPY . /application/
-
-# Replace stock Tesseract 3.04.00 font with improved sharp2.ttf that resolves
-# issues in many PDF viewers.
-# Discussion is in https://github.com/tesseract-ocr/tesseract/issues/182
-COPY ./share/sharp2.ttf /usr/share/tesseract-ocr/tessdata/pdf.ttf
-RUN chmod 644 /usr/share/tesseract-ocr/tessdata/pdf.ttf
-
-# Set this here to force a docker version, allowing non-tagged versions to
-# be built
-# ENV SETUPTOOLS_SCM_PRETEND_VERSION=v3.3.0
-
-# Install application and dependencies
-# In this arrangement Pillow and reportlab will be provided by the system
-# Even though ocrmypdf is locally present, pull from PyPI because
-# Dockerhub and setuptools_scm clash
-RUN . /appenv/bin/activate; \
-  pip install --upgrade pip \
-  && pip install ocrmypdf \
-  && pip install --no-cache-dir -r /application/test_requirements.txt
-
-# Remove the junk
-RUN apt-get remove -qy gcc
-RUN apt-get autoremove -y && apt-get clean -y
-RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/*
-
 USER docker
 WORKDIR /home/docker
 
-ENV OCRMYPDF_TEST_OUTPUT=/tmp/test-output
-ENV OCRMYPDF_SHARP_TTF=1
-
 # Must use array form of ENTRYPOINT
 # Non-array form does not append other arguments, because that is "intuitive"
-ENTRYPOINT ["/application/docker-wrapper.sh"]
+ENTRYPOINT ["/application/docker-wrapper.sh"]

Dockerfile.polyglot

@@ -1,14 +1,17 @@
 # OCRmyPDF polyglot
 #
-# VERSION               3.2
+# VERSION               4.4.2
 FROM      jbarlow83/ocrmypdf:latest
 MAINTAINER James R. Barlow <jim@purplerock.ca>
 
-# Update system and install our dependencies
 USER root
+
+# Update system and install our dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends \
   tesseract-ocr-all 
 
+RUN apt-get autoremove -y && apt-get clean -y
+
 USER docker
 
 # Must use array form of ENTRYPOINT

Dockerfile.tess4

@@ -0,0 +1,64 @@
+# OCRmyPDF
+#
+# VERSION               4.4.2
+FROM 	ubuntu:16.10
+MAINTAINER James R. Barlow <jim@purplerock.ca>
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  software-properties-common python-software-properties \
+  python3-wheel \
+  python3-reportlab \
+  python3-venv \
+  ghostscript \
+  qpdf \
+  poppler-utils \
+  unpaper \
+  libffi-dev
+
+RUN add-apt-repository ppa:alex-p/tesseract-ocr
+
+RUN apt-get update \
+	&& apt-get autoremove -y \
+	&& apt-get install -y --no-install-recommends \
+		tesseract-ocr \
+		tesseract-ocr-eng \
+		tesseract-ocr-fra \
+		tesseract-ocr-deu \
+		tesseract-ocr-spa \
+		tesseract-ocr-por \
+		tesseract-ocr-ara \
+		tesseract-ocr-rus \
+		tesseract-ocr-chi-sim 
+
+RUN python3 -m venv --system-site-packages /appenv
+
+# This installs the latest binary wheel instead of the code in the current
+# folder. Installing from source will fail, apparently because cffi needs
+# build-essentials (gcc) to do a source installation 
+# (i.e. "pip install ."). It's unclear to me why this is the case.
+RUN . /appenv/bin/activate; \
+  pip install --upgrade pip \
+  && pip install ocrmypdf
+
+# Now copy the application in, mainly to get the test suite.
+# Do this now to make the best use of Docker cache.
+COPY . /application
+RUN . /appenv/bin/activate; \
+  pip install -r /application/test_requirements.txt
+
+# Remove the junk, including the source version of application since it was
+# already installed
+RUN rm -rf /tmp/* /var/tmp/* /root/* /application/ocrmypdf \
+  && apt-get autoremove -y \
+  && apt-get autoclean -y
+
+RUN useradd docker \
+  && mkdir /home/docker \
+  && chown docker:docker /home/docker
+
+USER docker
+WORKDIR /home/docker
+
+# Must use array form of ENTRYPOINT
+# Non-array form does not append other arguments, because that is "intuitive"
+ENTRYPOINT ["/application/docker-wrapper.sh"]

MANIFEST.in

@@ -36,7 +36,6 @@ recursive-exclude docs/_build *
 
 # support files
 recursive-include ocrmypdf/data *
-recursive-include share *
 include *.py
 exclude tasks.py
 

README.rst

@@ -60,7 +60,17 @@ Installation
 Linux, UNIX, and macOS are supported. Windows is not directly supported but there is a Docker image available that runs on Windows.
 
 Users of Debian 9 or later or Ubuntu 16.10 or later may simply
-``apt-get install ocrmypdf``.
+
+.. code-block:: bash
+
+   apt-get install ocrmypdf
+
+and macOS users may simply
+
+.. code-block:: bash
+
+   brew tap jbarlow83/ocrmypdf
+   brew install ocrmypdf
 
 For everyone else, `see our documentation <https://ocrmypdf.readthedocs.io/en/latest/installation.html>`_ for installation steps.
 

RELEASE_NOTES.rst

@@ -3,17 +3,51 @@ RELEASE NOTES
 
 OCRmyPDF uses `semantic versioning <http://semver.org/>`_.
 
-v4.4.1:
-=======
+
+v4.5.2
+======
+
+-  Fix issue #147. ``--pdf-renderer tess4 --clean`` will produce an oversized page containing the original image in the bottom left corner, due to loss DPI information.
+-  Make "using Tesseract 4.0" warning less ominous
+-  Set up machinery for homebrew OCRmyPDF tap
+
+
+v4.5.1
+======
+
+-  Fix issue #137, proportions of images with a non-square pixel aspect ratio would be distorted in output for ``--force-ocr`` and some other combinations of flags
+
+
+v4.5
+====
+
+-  Exotic PDFs containing "Form XObjects" are now supported (issue #134; PDF reference manual 8.10), and images they contain are taken into account when determining the resolution for rasterizing
+-  The Tesseract 4 Docker image no longer includes all languages, because it took so long to build something would tend to fail
+-  OCRmyPDF now warns about using ``--pdf-renderer tesseract`` with Tesseract 3.04 or lower due to issues with Ghostscript corrupting the OCR text in these cases
+
+
+v4.4.2
+======
+
+-  The Docker images (ocrmypdf, ocrmypdf-polyglot, ocrmypdf-tess4) are now based on Ubuntu 16.10 instead of Debian stretch
+
+   + This makes supporting the Tesseract 4 image easier
+   + This could be a disruptive change for any Docker users who built customized these images with their own changes, and made those changes in a way that depends on Debian and not Ubuntu
+
+-  OCRmyPDF now prevents running the Tesseract 4 renderer with Tesseract 3.04, which was permitted in v4.4 and v4.4.1 but will not work
+
+
+v4.4.1
+======
 
 -  To prevent a `TIFF output error <https://github.com/python-pillow/Pillow/issues/2206>`_ caused by img2pdf >= 0.2.1 and Pillow <= 3.4.2, dependencies have been tightened
--  The Tesseract 4.00 simultaenous process limit was increased from 1 to 2, since it was observed that 1 lowers performance
+-  The Tesseract 4.00 simultaneous process limit was increased from 1 to 2, since it was observed that 1 lowers performance
 -  Documentation improvements to describe the ``--tesseract-config`` feature 
 -  Added test cases and fixed error handling for ``--tesseract-config``
 -  Tweaks to setup.py to deal with issues in the v4.4 release
 
-v4.4:
-=====
+v4.4
+====
 
 -  Tesseract 4.00 is now supported on an experimental basis.
 
@@ -30,33 +64,33 @@ v4.4:
    + However, OCRmyPDF's dependency "ruffus" is not re-entrant, so no Python API is available. Scripts should continue to use the command line interface.
 
 
-v4.3.5:
-=======
+v4.3.5
+======
 
 -  Update documentation to confirm Python 3.6.0 compatibility. No code changes were needed, so many earlier versions are likely supported.
 
 
-v4.3.4:
-=======
+v4.3.4
+======
 
 -  Fixed "decimal.InvalidOperation: quantize result has too many digits" for high DPI images
 
 
-v4.3.3:
-=======
+v4.3.3
+======
 
 -  Fixed PDF/A creation with Ghostscript 9.20 properly
 -  Fixed an exception on inline stencil masks with a missing optional parameter
 
 
-v4.3.2:
-=======
+v4.3.2
+======
 
 -  Fixed a PDF/A creation issue with Ghostscript 9.20 (note: this fix did not actually work)
 
 
-v4.3.1:
-=======
+v4.3.1
+======
 
 -  Fixed an issue where pages produced by the "hocr" renderer after a Tesseract timeout would be rotated incorrectly if the input page was rotated with a /Rotate marker
 -  Fixed a file handle leak in LeptonicaErrorTrap that would cause a "too many open files" error for files around hundred pages of pages long when ``--deskew`` or ``--remove-background`` or other Leptonica based image processing features were in use, depending on the system value of ``ulimit -n``
@@ -66,8 +100,8 @@ v4.3.1:
 -  Tesseract caching in test cases is now more cautious about false cache hits and reproducing exact output, not that any problems were observed
 
 
-v4.3:
-=====
+v4.3
+====
 
 -  New feature ``--remove-background`` to detect and erase the background of color and grayscale images
 -  Better documentation
@@ -77,21 +111,21 @@ v4.3:
    +  This does not improve performance since temporary files are still used for buffering
    +  Some output validation is disabled in this mode
 
-v4.2.5:
-=======
+v4.2.5
+======
 
 -  Fixed an issue (#100) with PDFs that omit the optional /BitsPerComponent parameter on images
 -  Removed non-free file milk.pdf
 
 
-v4.2.4:
-=======
+v4.2.4
+======
 
 -  Fixed an error (#90) caused by PDFs that use stencil masks properly
 -  Fixed handling of PDFs that try to draw images or stencil masks without properly setting up the graphics state (such images are now ignored for the purposes of calculating DPI)
 
-v4.2.3:
-=======
+v4.2.3
+======
 
 -  Fixed an issue with PDFs that store page rotation (/Rotate) in an indirect object
 -  Integrated a few fixes to simplify downstream packaging (Debian)
@@ -104,21 +138,21 @@ v4.2.3:
 -  Deprecated the OCRmyPDF.sh shell script
 
 
-v4.2.2:
-=======
+v4.2.2
+======
 
 -  Improvements to documentation
 
 
-v4.2.1:
-=======
+v4.2.1
+======
 
 -  Fixed an issue where PDF pages that contained stencil masks would report an incorrect DPI and cause Ghostscript to abort
 -  Implemented stdin streaming
 
 
-v4.2:
-=====
+v4.2
+====
 
 -  ocrmypdf will now try to convert single image files to PDFs if they are provided as input (#15)
 
@@ -150,14 +184,14 @@ v4.2:
 
 -  Ghostscript now runs in "safer" mode where possible
 
-v4.1.4:
-=======
+v4.1.4
+======
 
 -  Bug fix: monochrome images with an ICC profile attached were incorrectly converted to full color images if lossless reconstruction was not possible due to other settings; consequence was increased file size for these images
 
 
-v4.1.3:
-=======
+v4.1.3
+======
 
 -  More helpful error message for PDFs with version 4 security handler
 -  Update usage instructions for Windows/Docker users
@@ -165,50 +199,50 @@ v4.1.3:
 -  Add a few leptonica wrapper functions (no effect on most users)
 
 
-v4.1.2:
-=======
+v4.1.2
+======
 
 -  Replace IEC sRGB ICC profile with Debian's sRGB (from icc-profiles-free) which is more compatible with the MIT license
 -  More helpful error message for an error related to certain types of malformed PDFs
 
 
-v4.1:
-=====
+v4.1
+====
 
 -  ``--rotate-pages`` now only rotates pages when reasonably confidence in the orientation. This behavior can be adjusted with the new argument ``--rotate-pages-threshold``
 -  Fixed problems in error checking if ``unpaper`` is uninstalled or missing at run-time
 -  Fixed problems with "RethrownJobError" errors during error handling that suppressed the useful error messages
 
 
-v4.0.7:
-=======
+v4.0.7
+======
 
 -  Minor correction to Ghostscript output settings
 
 
-v4.0.6:
-=======
+v4.0.6
+======
 
 -  Update install instructions
 -  Provide a sRGB profile instead of using Ghostscript's
 
 
-v4.0.5:
-=======
+v4.0.5
+======
 
 -  Remove some verbose debug messages from v4.0.4
 -  Fixed temporary that wasn't being deleted
 -  DPI is now calculated correctly for cropped images, along with other image transformations
 -  Inline images are now checked during DPI calculation instead of rejecting the image
 
-v4.0.4:
-=======
+v4.0.4
+======
 
 Released with verbose debug message turned on. Do not use. Skip to v4.0.5.
 
 
-v4.0.3:
-=======
+v4.0.3
+======
 
 New features
 ------------
@@ -225,8 +259,8 @@ Fixes
 -  Docker: fix blank JPEG2000 issue by insisting on Ghostscript versions that have this fixed
 
 
-v4.0.2:
-=======
+v4.0.2
+======
 
 Fixes
 -----
@@ -237,8 +271,8 @@ Fixes
 -  Fixed use of chmod on Docker that broke most test cases
 
 
-v4.0.1:
-=======
+v4.0.1
+======
 
 Fixes
 -----
@@ -246,8 +280,8 @@ Fixes
 -  Fixed a KeyError if tesseract fails to find page orientation information
 
 
-v4.0:
-=====
+v4.0
+====
 
 New features
 ------------
@@ -281,8 +315,8 @@ Changes
    to correct the problem.
 
 
-v3.2.1:
-=======
+v3.2.1
+======
 
 Changes
 -------
@@ -291,8 +325,8 @@ Changes
 -  Tweaked the Dockerfiles
 
 
-v3.2:
-=====
+v3.2
+====
 
 New features
 ------------
@@ -313,16 +347,16 @@ Changes
 
 
 
-v3.1.1:
-=======
+v3.1.1
+======
 
 Changes
 -------
 
 -  Fixed bug that caused incorrect page size and DPI calculations on documents with mixed page sizes
 
-v3.1:
-=====
+v3.1
+====
 
 Changes
 -------
@@ -339,8 +373,8 @@ Changes
    Currently it always chooses the 'hocrtransform' renderer but that behavior may change.
 -  Set up Travis CI automatic integration testing
 
-v3.0:
-=====
+v3.0
+====
 
 New features
 ------------
@@ -489,8 +523,8 @@ Notes and known issues
    images almost never contain inline images.
 
 
-v2.2-stable (2014-09-29):
-=========================
+v2.2-stable (2014-09-29)
+========================
 
 OCRmyPDF versions 1 and 2 were implemented as shell scripts. OCRmyPDF 3.0+ is a fork that gradually replaced all shell scripts with Python while maintaining the existing command line arguments. No one is maintaining old versions.
 

docs/batch.rst

@@ -10,11 +10,29 @@ Consider using the excellent `GNU Parallel <https://www.gnu.org/software/paralle
 
 Both ``parallel`` and ``ocrmypdf`` will try to use all available processors. To maximize parallelism without overloading your system with processes, consider using ``parallel -j 2`` to limit parallel to running two jobs at once.
 
-This command will run all ocrmypdf all files named ``*.pdf`` in the current directory and write them to the previous created ``output/`` folder.
+This command will run all ocrmypdf all files named ``*.pdf`` in the current directory and write them to the previous created ``output/`` folder. It will not search subdirectories.
+
+The ``--tag`` argument tells parallel to print the filename as a prefix whenever a message is printed, so that one can trace any errors to the file that produced them.
 
 .. code-block:: bash
 
-	parallel -j 2 ocrmypdf '{}' 'output/{}' ::: *.pdf
+	parallel --tag -j 2 ocrmypdf '{}' 'output/{}' ::: *.pdf
+
+Directory trees
+---------------
+
+This will walk through a directory tree and run OCR on all files in place, printing the output in a way that makes 
+
+.. code-block:: bash
+
+	find . --printf '%p' -name '*.pdf' -exec ocrmypdf '{}' '{}' \;
+
+This only runs one ``ocrmypdf`` process at a time. This variation uses ``find`` to create a directory list and ``parallel`` to parallelize runs of ``ocrmypdf``, again updating files in place.
+
+.. code-block:: bash
+
+	find . -name '*.pdf' | parallel --tag -j 2 ocrmypdf '{}' '{}'
+
 
 Sample script
 """""""""""""

docs/installation.rst

@@ -10,6 +10,18 @@ Users of Debian 9 ("stretch") or later or Ubuntu 16.10 or later may simply
 
 	apt-get install ocrmypdf
 
+
+Installing on macOS
+-------------------
+
+.. code-block:: bash
+
+   brew tap jbarlow83/ocrmypdf
+   brew install ocrmypdf
+
+
+Users who previously installed OCRmyPDF on macOS may need to remove the ``pip`` based installation (``pip3 uninstall ocrmypdf``).
+
 .. _Docker:
 
 Installing the Docker image
@@ -27,7 +39,7 @@ successfully, your system is ready to download and execute the image:
 
    docker run hello-world
    
-OCRmyPDF will use all available CPU cores.  By default, the VirtualBox machine instance on Windows and OS X has only a single CPU core enabled. Use the VirtualBox Manager to determine the name of your Docker engine host, and then follow these optional steps to enable multiple CPUs:
+OCRmyPDF will use all available CPU cores.  By default, the VirtualBox machine instance on Windows and macOS has only a single CPU core enabled. Use the VirtualBox Manager to determine the name of your Docker engine host, and then follow these optional steps to enable multiple CPUs:
 
 .. code-block:: bash
 
@@ -37,28 +49,34 @@ OCRmyPDF will use all available CPU cores.  By default, the VirtualBox machine i
    docker-machine start "yourVM"
    eval $(docker-machine env "yourVM")
 
-Assuming you have a Docker engine running somewhere, you can run these commands to download
-the image:
+Assuming you have a Docker engine running, you can download one of the three available images:
+
++-----------------------------+---------------------------------------------+---------------------------------------------------------------------------------+
+| Image name                  | Download command                            | Notes                                                                           |
++-----------------------------+---------------------------------------------+---------------------------------------------------------------------------------+
+| ocrmypdf                    | ``docker pull jbarlow83/ocrmypdf``          | Latest ocrmypdf with Tesseract 3.04. Includes English, French, German, Spanish. |
++-----------------------------+---------------------------------------------+---------------------------------------------------------------------------------+
+| ocrmypdf-polyglot           | ``docker pull jbarlow83/ocrmypdf-polyglot`` | As above, with all available language packs.                                    |
++-----------------------------+---------------------------------------------+---------------------------------------------------------------------------------+
+| ocrmypdf-tess4              | ``docker pull jbarlow83/ocrmypdf-tess4``    | Latest ocrmypdf with Tesseract 4.00.00alpha and English, French, German,        |
+|                             |                                             | Spanish, Portuguese, Chinese Simplified, Arabic and Russian (the top 8).        |
++-----------------------------+---------------------------------------------+---------------------------------------------------------------------------------+
+
+For example:
 
 .. code-block:: bash
 
-   docker pull jbarlow83/ocrmypdf
+   docker pull jbarlow83/ocrmypdf-tess4
 
 Then tag it to give a more convenient name, just ocrmypdf:
 
 .. code-block:: bash
 
-   docker tag jbarlow83/ocrmypdf ocrmypdf
+   docker tag jbarlow83/ocrmypdf-tess4 ocrmypdf
 
 .. _docker-polyglot:
 
-This image contains language packs for English, French, Spanish and German. The alternative "polyglot" image provides `all available language packs <https://github.com/tesseract-ocr/tesseract/blob/master/doc/tesseract.1.asc#languages>`_:
-
-.. code-block:: bash
-
-   # Alternative step: If you need all language packs
-   docker pull jbarlow83/ocrmypdf-polyglot
-   docker tag jbarlow83/ocrmypdf-polyglot ocrmypdf
+The alternative "polyglot" image provides `all available language packs <https://github.com/tesseract-ocr/tesseract/blob/master/doc/tesseract.1.asc#languages>`_.
 
 You can then run ocrmypdf using the command:
 
@@ -82,11 +100,25 @@ In this worked example, the current working directory contains an input file cal
 
 Note that ``ocrmypdf`` has its own separate ``-v VERBOSITYLEVEL`` argument to control debug verbosity. All Docker arguments should before the ``ocrmypdf`` image name and all arguments to ``ocrmypdf`` should be listed after.
 
+For convenience, a shell alias can hide the docker command:
 
-Installing on macOS (formerly Mac OS X)
----------------------------------------
+.. code-block:: bash
 
-These instructions probably work on all macOS supported by Homebrew. OCRmyPDF is known to work on Yosemite and El Capitan, and regularly tested on El Capitan.
+   alias ocrmypdf='docker run --rm -v "$(pwd):/home/docker" ocrmypdf' 
+   ocrmypdf --version  # runs docker version
+
+Or in the wonderful `fish shell <https://fishshell.com/>`_:
+
+.. code-block:: fish
+
+   alias ocrmypdf 'docker run --rm -v (pwd):/home/docker ocrmypdf'
+   funcsave ocrmypdf
+
+
+Manual installation on macOS
+----------------------------
+
+These instructions probably work on all macOS supported by Homebrew.
 
 If it's not already present, `install Homebrew <http://brew.sh/>`_.
 
@@ -155,10 +187,6 @@ No package is currently available for Ubuntu 16.04, but you can install the depe
       tesseract-ocr \
       qpdf \
       python3-pip \
-      python3-pil \
-      python3-pytest \
-      python3-reportlab \
-      python3-ruffus \
       python3-cffi
 
 If you wish install OCRmyPDF to the system Python, then install as follows (note this installs new packages
@@ -173,7 +201,7 @@ follow these steps.
 
 .. code-block:: bash
 
-   python3 -m venv --system-site-packages venv-ocrmypdf
+   python3 -m venv venv-ocrmypdf
    source venv-ocrmypdf/bin/activate
    pip3 install ocrmypdf
 

ocrmypdf/__main__.py

@@ -272,13 +272,18 @@ def check_options_output(options, log):
     if options.pdf_renderer == 'auto':
         options.pdf_renderer = 'hocr'
 
-    if options.pdf_renderer == 'tesseract' and \
-            tesseract.version() < '3.04.01' and \
-            os.environ.get('OCRMYPDF_SHARP_TTF', '') != '1':
-        log.warning(
-            "Your version of tesseract has problems with PDF output."
-            " Some PDF viewers will fail to find searchable text.\n"
-            "--pdf-renderer=tesseract is not recommended.")
+    if options.pdf_renderer in ('tesseract', 'tess4'):
+        if tesseract.version() < '3.05':
+            log.warning(
+                "The setting --pdf-renderer=tesseract is not recommend for "
+                " use with tesseract versions less than 3.05, because it "
+                " produces OCR text that is incompatible with Ghostscript and "
+                " some other software.")
+        elif tesseract.version() == '4.00.00alpha':
+            log.warning(
+                "The setting --pdf-renderer={tesseract,tess4} is not"
+                " recommended for builds of tesseract 4.00.00alpha older than"
+                " February 2017. Make sure you are using a recent build.")
 
     if options.debug_rendering and options.pdf_renderer == 'tesseract':
         log.info(
@@ -333,6 +338,10 @@ def check_options_advanced(options, log):
     if options.tesseract_oem and not tesseract.v4():
         log.warning(
             "--tesseract-oem requires Tesseract 4.x -- argument ignored")
+    if options.pdf_renderer == 'tess4' and not tesseract.has_textonly_pdf():
+        raise MissingDependencyError(
+            "--pdf-renderer tess4 requires Tesseract 4.x "
+            "commit 3d9fb3b or later")
 
 
 def check_options(options, log):
@@ -350,9 +359,6 @@ def check_options(options, log):
         sys.exit(ExitCode.missing_dependency)
 
 
-
-
-
 # ----------
 # Logging
 
@@ -475,10 +481,58 @@ def traverse_ruffus_exception(e_args, options, log):
             return traverse_ruffus_exception(exc, options, log)
 
 
+def check_closed_streams(options):
+    """Work around Python issue with multiprocessing forking on closed streams
+
+    https://bugs.python.org/issue28326
+
+    Attempting to a fork/exec a new Python process when any of std{in,out,err}
+    are closed or not flushable for some reason may raise an exception.
+    Fix this by opening devnull if the handle seems to be closed.  Do this
+    globally to avoid tracking places all places that fork.
+
+    Seems to be specific to multiprocessing.Process not all Python process
+    forkers.
+
+    The error actually occurs when the stream object is not flushable,
+    but replacing an open stream object that is not flushable with
+    /dev/null is a bad idea since it will create a silent failure.  Replacing
+    a closed handle with /dev/null seems safe.
+
+    """
+
+    if sys.stderr is None:
+        sys.stderr = open(os.devnull, 'w')
+
+    if sys.stdin is None:
+        if options.input_file == '-':
+            print("Trying to read from stdin but stdin seems closed",
+                  file=sys.stderr)
+            return False
+        sys.stdin = open(os.devnull, 'r')
+
+    if sys.stdout is None:
+        if options.output_file == '-':
+            # Can't replace stdout if the user is piping
+            # If this case can even happen, it must be some kind of weird
+            # stream.
+            print(textwrap.dedent("""\
+                Output was set to stdout '-' but the stream attached to
+                stdout does not support the flush() system call.  This
+                will fail."""), file=sys.stderr)
+            return False
+        sys.stdout = open(os.devnull, 'w')
+
+    return True
+
+
 def run_pipeline():
     options = parser.parse_args()
     options.verbose_abbreviated_path = 1
 
+    if not check_closed_streams(options):
+        return ExitCode.bad_args
+
     _log, _log_mutex = proxy_logger.make_shared_logger_and_proxy(
         logging_factory, __name__, [None, options.verbose])
     _log.debug('ocrmypdf ' + VERSION)
@@ -519,9 +573,9 @@ def run_pipeline():
                     file."""))
                 return ExitCode.bad_args
         elif not is_file_writable(options.output_file):
-                _log.error(textwrap.dedent("""\
-                    Cutput file location is not writable."""))
-                return ExitCode.file_access_error
+            _log.error(textwrap.dedent("""\
+                Cutput file location is not writable."""))
+            return ExitCode.file_access_error
 
         manager = JobContextManager()
         manager.register('JobContext', JobContext)

ocrmypdf/exec/ghostscript.py

@@ -2,12 +2,32 @@
 # © 2015 James R. Barlow: github.com/jbarlow83
 
 from tempfile import NamedTemporaryFile
-from subprocess import Popen, PIPE, STDOUT, check_call
+from subprocess import Popen, PIPE, STDOUT, check_call, CalledProcessError, \
+    check_output
 from shutil import copy
+from functools import lru_cache
 from . import get_program
 from ..pdfa import SRGB_ICC_PROFILE
 
 
+@lru_cache(maxsize=1)
+def version():
+    args_gs = [
+        get_program('gs'),
+        '--version'
+    ]
+    try:
+        version = check_output(
+                args_gs, close_fds=True, universal_newlines=True,
+                stderr=STDOUT)
+    except CalledProcessError as e:
+        print("Could not find Ghostscript executable on system PATH.",
+              file=sys.stderr)
+        raise MissingDependencyError from e
+
+    return version.strip()
+
+
 def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
                   pageno=1):
     with NamedTemporaryFile(delete=True) as tmp:

ocrmypdf/exec/unpaper.py

@@ -78,7 +78,8 @@ def run(input_file, output_file, dpi, log, mode_args):
             raise e from e
         else:
             log.debug(stdout)
-            Image.open(output_pnm.name).save(output_file)
+            # unpaper sets dpi to 72
+            Image.open(output_pnm.name).save(output_file, dpi=(dpi, dpi))
 
 
 def deskew(input_file, output_file, dpi, log):

ocrmypdf/pageinfo.py

@@ -89,13 +89,13 @@ def _is_unit_square(shorthand):
     pairwise = zip(values, UNIT_SQUARE)
     return all([isclose(a, b, rel_tol=1e-3) for a, b in pairwise])
 
-RasterSettings = namedtuple('RasterSettings',
+XobjectSettings = namedtuple('XobjectSettings',
     ['name', 'shorthand', 'stack_depth'])
 
 InlineSettings = namedtuple('InlineSettings',
     ['settings', 'shorthand', 'stack_depth'])
 
-ContentsInfo = namedtuple('ContentsInfo', ['raster_settings', 'inline_images'])
+ContentsInfo = namedtuple('ContentsInfo', ['xobject_settings', 'inline_images'])
 
 
 def _normalize_stack(operations):
@@ -114,7 +114,7 @@ def _normalize_stack(operations):
             yield (operands, command)
 
 
-def _interpret_contents(contentstream):
+def _interpret_contents(contentstream, initial_shorthand=UNIT_SQUARE):
     """Interpret the PDF content stream
 
     The stack represents the state of the PDF graphics stack.  We are only
@@ -139,8 +139,8 @@ def _interpret_contents(contentstream):
 
     operations = contentstream.operations
     stack = []
-    ctm = _matrix_from_shorthand(UNIT_SQUARE)
-    image_raster_settings = []
+    ctm = _matrix_from_shorthand(initial_shorthand)
+    xobject_settings = []
     inline_images = []
 
     for n, op in enumerate(_normalize_stack(operations)):
@@ -161,10 +161,10 @@ def _interpret_contents(contentstream):
                 _matrix_from_shorthand(operands), ctm)
         elif command == b'Do':
             image_name = operands[0]
-            raster = RasterSettings(
+            settings = XobjectSettings(
                 name=image_name, shorthand=_shorthand_from_matrix(ctm),
                 stack_depth=len(stack))
-            image_raster_settings.append(raster)
+            xobject_settings.append(settings)
         elif command == b'INLINE IMAGE':
             settings = operands['settings']
             inline = InlineSettings(
@@ -173,7 +173,7 @@ def _interpret_contents(contentstream):
             inline_images.append(inline)
 
     return ContentsInfo(
-        raster_settings=image_raster_settings,
+        xobject_settings=xobject_settings,
         inline_images=inline_images)
 
 
@@ -241,8 +241,8 @@ def _get_dpi(ctm_shorthand, image_size):
     return (dpi_w, dpi_h)
 
 
-def _find_page_inline_images(page, pageinfo, contentsinfo):
-    "Find inline images on the page"
+def _find_inline_images(contentsinfo):
+    "Find inline images in the contentstream"
 
     for n, inline in enumerate(contentsinfo.inline_images):
         image = {}
@@ -272,20 +272,44 @@ def _find_page_inline_images(page, pageinfo, contentsinfo):
         yield image
 
 
-def _find_page_regular_images(page, pageinfo, contentsinfo):
-    "Find images stored in XObject resources"
+def _image_xobjects(container):
+    """Search for all XObject-based images in the container
 
-    try:
-        page['/Resources']['/XObject']
-    except KeyError:
+    Usually the container is a page, but it could also be a Form XObject
+    that contains images. Filter out the Form XObjects which are dealt with
+    elsewhere.
+
+    Generate a sequence of tuples (image, xobj container), where container,
+    where xobj is the name of the object and image is the object itself,
+    since the object does not know its own name.
+
+    """
+
+    if '/Resources' not in container:
         return
-    for xobj in page['/Resources']['/XObject']:
-        # PyPDF2 returns the keys as an iterator
-        pdfimage = page['/Resources']['/XObject'][xobj]
-        if pdfimage['/Subtype'] != '/Image':
-            continue
+    resources = container['/Resources']
+    if '/XObject' not in resources:
+        return
+    for xobj in resources['/XObject']:
+        candidate = resources['/XObject'][xobj]
+        if candidate['/Subtype'] == '/Image':
+            image = candidate
+            yield (image, xobj)
+
+
+def _find_regular_images(container, contentsinfo):
+    """Find images stored in the container's /Resources /XObject
+
+    Usually the container is a page, but it could also be a Form XObject
+    that contains images.
+
+    Generates images with their DPI at time of drawing.
+
+    """
+
+    for pdfimage, xobj in _image_xobjects(container):
         image = {}
-        image['name'] = str(xobj)
+        image['name'] = xobj
         image['width'] = pdfimage['/Width']
         image['height'] = pdfimage['/Height']
         if '/BitsPerComponent' in pdfimage:
@@ -329,12 +353,12 @@ def _find_page_regular_images(page, pageinfo, contentsinfo):
 
         image['dpi_w'] = image['dpi_h'] = 0
 
-        for raster in contentsinfo.raster_settings:
+        for xobj in contentsinfo.xobject_settings:
             # Loop in case the same image is display multiple times on a page
-            if raster.name != image['name']:
+            if xobj.name != image['name']:
                 continue
 
-            if raster.stack_depth == 0 and _is_unit_square(raster.shorthand):
+            if xobj.stack_depth == 0 and _is_unit_square(xobj.shorthand):
                 # At least one PDF in the wild (and test suite) draws an image
                 # when the graphics stack depth is 0, meaning that the image
                 # gets drawn into a square of 1x1 PDF units (or 1/72",
@@ -343,7 +367,7 @@ def _find_page_regular_images(page, pageinfo, contentsinfo):
                 continue
 
             dpi_w, dpi_h = _get_dpi(
-                raster.shorthand, (image['width'], image['height']))
+                xobj.shorthand, (image['width'], image['height']))
 
             # When image is used multiple times take the highest DPI it is
             # rendered at
@@ -358,9 +382,85 @@ def _find_page_regular_images(page, pageinfo, contentsinfo):
         yield image
 
 
-def _find_page_images(page, pageinfo, contentsinfo):
-    yield from _find_page_inline_images(page, pageinfo, contentsinfo)
-    yield from _find_page_regular_images(page, pageinfo, contentsinfo)
+def _find_form_xobject_images(pdf, container, contentsinfo):
+    """Find any images that are in Form XObjects in the container
+
+    The container may be a page, or a parent Form XObject.
+
+    """
+    if '/Resources' not in container:
+        return
+    resources = container['/Resources']
+    if '/XObject' not in resources:
+        return
+    for xobj in resources['/XObject']:
+        candidate = resources['/XObject'][xobj]
+        if candidate['/Subtype'] != '/Form':
+            continue
+
+        form_xobject = candidate
+        for settings in contentsinfo.xobject_settings:
+            if settings.name != xobj:
+                continue
+
+            # Find images once for each time this Form XObject is drawn.
+            # This could be optimized to cache the multiple drawing events
+            # but in practice both Form XObjects and multiple drawing of the
+            # same object are both very rare.
+            ctm_shorthand = settings.shorthand
+            yield from _find_images(pdf, form_xobject, ctm_shorthand)
+
+
+def _find_images(pdf, container, shorthand=None):
+    """Find all individual instances of images drawn in the container
+
+    Usually the container is a page, but it may also be a Form XObject.
+
+    On a typical page images are stored inline or as regular images
+    in an XObject.
+
+    Form XObjects may include inline images, XObject images,
+    and recursively, other Form XObjects; and also vector drawing commands.
+
+    Every instance of an image being drawn somewhere is flattened and
+    treated as a unique image, since if the same image is drawn multiple times
+    on one page it may be drawn at differing resolutions, and our objective
+    is to find the resolution at which the page can be rastered without
+    downsampling.
+
+    """
+
+    if container.get('/Type') == '/Page':
+        # For a /Page the content stream is attached to the page's /Contents
+        page = container
+        contentstream = pypdf.pdf.ContentStream(page.getContents(), pdf)
+        initial_shorthand = shorthand or UNIT_SQUARE
+    elif container.get('/Type') == '/XObject' and \
+            container['/Subtype'] == '/Form':
+        # For a Form XObject that content stream is attached to the XObject
+        contentstream = pypdf.pdf.ContentStream(container, pdf)
+
+        # Set the CTM to the state it was when the "Do" operator was
+        # encountered that is drawing this instance of the Form XObject
+        ctm = _matrix_from_shorthand(shorthand or UNIT_SQUARE)
+
+        # A Form XObject may provide its own matrix to map form space into
+        # user space. Get this if one exists
+        form_matrix = _matrix_from_shorthand(
+                container.get('/Matrix', UNIT_SQUARE))
+
+        # Concatenate form matrix with CTM to ensure CTM is correct for
+        # drawing this instance of the XObject
+        ctm = matrix_mult(form_matrix, ctm)
+        initial_shorthand = _shorthand_from_matrix(ctm)
+    else:
+        return
+
+    contentsinfo = _interpret_contents(contentstream, initial_shorthand)
+
+    yield from _find_inline_images(contentsinfo)
+    yield from _find_regular_images(container, contentsinfo)
+    yield from _find_form_xobject_images(pdf, container, contentsinfo)
 
 
 def _page_has_text(pdf, page):
@@ -405,15 +505,8 @@ def _pdf_get_pageinfo(infile, pageno: int):
     except KeyError:
         pageinfo['rotate'] = 0
 
-    try:
-        contentstream = pypdf.pdf.ContentStream(page.getContents(), pdf)
-    except AttributeError as e:
-        return pageinfo
-
-    contentsinfo = _interpret_contents(contentstream)
-    pageinfo['images'] = [im for im in _find_page_images(
-                                page, pageinfo, contentsinfo)]
-
+    pageinfo['images'] = [im for im in
+                          _find_images(pdf, page)]
     if pageinfo['images']:
         xres = max(image['dpi_w'] for image in pageinfo['images'])
         yres = max(image['dpi_h'] for image in pageinfo['images'])

ocrmypdf/pipeline.py

@@ -555,16 +555,21 @@ def select_image_layer(
         re_symlink(page_pdf, output_file, log)
     else:
         pageinfo = get_pageinfo(image, context)
-        dpi = get_page_dpi(pageinfo, options)
-        dpi = float(dpi[0]), float(dpi[1])
-        layout_fun = img2pdf.get_fixed_dpi_layout_fun(dpi)
+
+        # We rasterize a square DPI version of each page because most image
+        # processing tools don't support rectangular DPI. Use the square DPI
+        # as it accurately describes the image. It would be possible to
+        # resample the image at this stage back to non-square DPI to more
+        # closely resemble the input, except that the hocr renderer does not
+        # understand non-square DPI. The tess4 renderer would be fine.
+        dpi = get_page_square_dpi(pageinfo, options)
+        layout_fun = img2pdf.get_fixed_dpi_layout_fun((dpi, dpi))
 
         with open(image, 'rb') as imfile, \
                 open(output_file, 'wb') as pdf:
-            rawdata = imfile.read()
             log.debug('{:4d}: convert'.format(page_number(page_pdf)))
             img2pdf.convert(
-                rawdata, with_pdfrw=False,
+                imfile, with_pdfrw=False,
                 layout_fun=layout_fun, outputstream=pdf)
             log.debug('{:4d}: convert done'.format(page_number(page_pdf)))
 

share/etc-apt-sources.list

@@ -1,4 +0,0 @@
-deb http://httpredir.debian.org/debian stretch main
-deb http://httpredir.debian.org/debian stretch-updates main
-deb http://security.debian.org stretch/updates main
-deb http://ftp.de.debian.org/debian sid main contrib non-free

tests/conftest.py

@@ -22,10 +22,21 @@ def is_linux():
     return platform.system() == 'Linux'
 
 
+@pytest.helpers.register
+def is_macos():
+    return platform.system() == 'Darwin'
+
+
 @pytest.helpers.register
 def running_in_docker():
-    # Docker creates a file named /.dockerinit
-    return os.path.exists('/.dockerinit')
+    # Docker creates a file named /.dockerenv (newer versions) or
+    # /.dockerinit (older) -- this is undocumented, not an offical test
+    return os.path.exists('/.dockerenv') or os.path.exists('/.dockerinit')
+
+
+@pytest.helpers.register
+def running_in_travis():
+    return os.environ.get('TRAVIS') == 'true'
 
 
 TESTS_ROOT = os.path.abspath(os.path.dirname(__file__))
@@ -90,7 +101,7 @@ def check_ocrmypdf(input_file, output_file, *args, env=None):
 
     p, out, err = run_ocrmypdf(input_file, output_file, *args, env=env)
     #print(err)  # ensure py.test collects the output, use -s to view
-    assert p.returncode == 0
+    assert p.returncode == 0, "<stderr>\n" + err + "\n</stderr>"
     assert os.path.exists(str(output_file)), "Output file not created"
     assert os.stat(str(output_file)).st_size > 100, "PDF too small or empty"
     assert out == "", \
@@ -114,3 +125,11 @@ def run_ocrmypdf(input_file, output_file, *args, env=None):
     #print(err)
 
     return p, out, err
+
+
+@pytest.helpers.register
+def first_page_dimensions(pdf):
+    from ocrmypdf import pageinfo
+    info = pageinfo.pdf_get_all_pageinfo(str(pdf))
+    page0 = info[0]
+    return (page0['width_inches'], page0['height_inches'])

tests/resources/README.rst

@@ -64,6 +64,9 @@ under the terms of the license in LICENSE.rst.
     *   - epson.pdf
         - @lowesjam
         - a linearized PDF containing some unusual indirect objects, created by an Epson printer; printout of a Wikipedia article (CC BY-SA)
+    *   - formxobject.pdf
+        - @jbarlow83
+        - hand-crafted exotic PDF containing an image inside a Form XObject
     *   - francais.pdf
         - @jbarlow83
         - a page containing French accents (diacritics)  

tests/test_main.py

@@ -12,6 +12,7 @@ import PyPDF2 as pypdf
 from ocrmypdf.exceptions import ExitCode
 from ocrmypdf import leptonica
 from ocrmypdf.pdfa import file_claims_pdfa
+from ocrmypdf.exec import ghostscript
 
 
 check_ocrmypdf = pytest.helpers.check_ocrmypdf
@@ -169,6 +170,13 @@ def test_preserve_metadata(spoof_tesseract_noop, output_type,
 @pytest.mark.skipif(
     pytest.helpers.is_linux() and not pytest.helpers.running_in_docker(),
     reason="likely to fail if Linux locale is not configured correctly")
+@pytest.mark.skipif(
+    pytest.helpers.is_macos() and pytest.helpers.running_in_travis(),
+    reason="save Travis the trouble of installing poppler")
+@pytest.mark.xfail(
+    ghostscript.version() == '9.21',
+    reason="gs 9.21 has a regression that affects this"
+    )
 @pytest.mark.parametrize("output_type", [
     'pdfa', 'pdf'
     ])
@@ -188,7 +196,7 @@ def test_override_metadata(spoof_tesseract_noop, output_type, resources,
         '--output-type', output_type,
         env=spoof_tesseract_noop)
 
-    assert p.returncode == ExitCode.ok
+    assert p.returncode == ExitCode.ok, err
 
     pdf = str(outpdf)
 
@@ -409,10 +417,14 @@ def test_force_ocr_on_pdf_with_no_images(spoof_tesseract_crash, resources,
     assert not os.path.exists(no_outpdf)
 
 
+@pytest.mark.skipif(
+    pytest.helpers.is_macos() and pytest.helpers.running_in_travis(),
+    reason="takes too long to install language packs in Travis macOS homebrew")
 def test_french(spoof_tesseract_cache, resources, outpdf):
     p, out, err = run_ocrmypdf(
         resources / 'francais.pdf', outpdf, '-l', 'fra',
         env=spoof_tesseract_cache)
+    print(os.environ)
     assert p.returncode == ExitCode.ok, \
         "This test may fail if Tesseract language packs are missing"
 
@@ -564,6 +576,46 @@ def test_non_square_resolution(renderer, spoof_tesseract_cache,
     assert in_pageinfo[0]['yres'] == out_pageinfo[0]['yres']
 
 
+@pytest.mark.skipif(sys.version_info < (3, 5), reason="needs math.isclose")
+@pytest.mark.parametrize('renderer', [
+    'hocr',
+    'tesseract'
+    ])
+def test_convert_to_square_resolution(renderer, spoof_tesseract_cache,
+                                      resources, outpdf):
+    from math import isclose
+
+    # Confirm input image is non-square resolution
+    in_pageinfo = pdf_get_all_pageinfo(str(resources / 'aspect.pdf'))
+    assert in_pageinfo[0]['xres'] != in_pageinfo[0]['yres']
+
+    # --force-ocr requires means forced conversion to square resolution
+    check_ocrmypdf(
+        resources / 'aspect.pdf', outpdf,
+        '--force-ocr',
+        '--pdf-renderer', renderer, env=spoof_tesseract_cache)
+
+    out_pageinfo = pdf_get_all_pageinfo(str(outpdf))
+
+    in_p0, out_p0 = in_pageinfo[0], out_pageinfo[0]
+
+    # Resolution show now be equal
+    assert out_p0['xres'] == out_p0['yres']
+
+    # Page size should match input page size
+    assert isclose(in_p0['width_inches'],
+                   out_p0['width_inches'])
+    assert isclose(in_p0['height_inches'],
+                   out_p0['height_inches'])
+
+    # Because we rasterized the page to produce a new image, it should occupy
+    # the entire page
+    out_im_w = out_p0['images'][0]['width'] / out_p0['images'][0]['dpi_w']
+    out_im_h = out_p0['images'][0]['height'] / out_p0['images'][0]['dpi_h']
+    assert isclose(out_p0['width_inches'], out_im_w)
+    assert isclose(out_p0['height_inches'], out_im_h)
+
+
 def test_image_to_pdf(spoof_tesseract_noop, resources, outpdf):
     check_ocrmypdf(
         resources / 'LinnSequencer.jpg', outpdf, '--image-dpi', '200',
@@ -614,6 +666,23 @@ def test_stdout(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf):
     assert qpdf.check(output_file, log=None)
 
 
+def test_closed_streams(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf):
+    input_file = str(resources / 'francais.pdf')
+    output_file = str(outpdf)
+
+    def evil_closer():
+        os.close(0)
+        os.close(1)
+
+    p_args = ocrmypdf_exec + [input_file, output_file]
+    p = Popen(
+        p_args, close_fds=True, stdout=None, stderr=PIPE, stdin=None,
+        env=spoof_tesseract_noop, preexec_fn=evil_closer)
+    out, err = p.communicate()
+    print(err.decode())
+    assert p.returncode == ExitCode.ok
+
+
 def test_masks(spoof_tesseract_noop, resources, outpdf):
     check_ocrmypdf(resources / 'masks.pdf', outpdf, env=spoof_tesseract_noop)
 
@@ -682,6 +751,10 @@ def test_overlay(spoof_tesseract_noop, resources, outpdf):
                    env=spoof_tesseract_noop)
 
 
+@pytest.mark.skipif(
+    os.getuid() == 0 or os.geteuid() == 0,
+    reason="root can write to anything"
+    )
 def test_destination_not_writable(spoof_tesseract_noop, resources, outdir):
     protected_file = outdir / 'protected.pdf'
     protected_file.touch()
@@ -738,3 +811,34 @@ THIS FILE IS INVALID
         '--tesseract-config', str(cfg_file))
     assert "parameter not found" in err, "No error message"
     assert p.returncode == ExitCode.invalid_config
+
+
+def test_form_xobject(spoof_tesseract_noop, resources, outpdf):
+    check_ocrmypdf(resources / 'formxobject.pdf', outpdf,
+                   '--force-ocr',
+                   env=spoof_tesseract_noop)
+
+
+@pytest.mark.skipif(sys.version_info < (3, 5), reason="needs math.isclose")
+@pytest.mark.parametrize('renderer', [
+    'hocr',
+    'tesseract',
+])
+def test_pagesize_consistency(renderer, resources, outpdf):
+    from math import isclose
+
+    first_page_dimensions = pytest.helpers.first_page_dimensions
+
+    infile = resources / 'linn.pdf'
+
+    before_dims = first_page_dimensions(infile)
+
+    check_ocrmypdf(
+        infile,
+        outpdf, '--pdf-renderer', renderer,
+        '--clean', '--deskew', '--remove-background', '--clean-final')
+
+    after_dims = first_page_dimensions(outpdf)
+
+    assert isclose(before_dims[0], after_dims[0])
+    assert isclose(before_dims[1], after_dims[1])

tests/test_pageinfo.py

@@ -101,3 +101,10 @@ def test_jpeg(resources, outdir):
     assert pdfimage['enc'] == 'jpeg'
     assert (pdfimage['dpi_w'] - 150) < 1e-5
 
+
+def test_form_xobject(resources):
+    filename = resources / 'formxobject.pdf'
+
+    pdfinfo = pageinfo.pdf_get_all_pageinfo(str(filename))
+    pdfimage = pdfinfo[0]['images'][0]
+    assert pdfimage['width'] == 50

tests/test_tess3.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+# © 2017 James R. Barlow: github.com/jbarlow83
+
+import pytest
+from ocrmypdf.exceptions import ExitCode
+from ocrmypdf.exec import tesseract
+
+
+# Skip all tests in this file if not tesseract 3
+pytestmark = pytest.mark.skipif(tesseract.v4(),
+                                reason="tesseract 3.x required")
+
+
+def test_textonly_pdf_on_tess3(resources, no_outpdf):
+    p, _, _ = pytest.helpers.run_ocrmypdf(
+        resources / 'linn.pdf',
+        no_outpdf, '--pdf-renderer', 'tess4')
+
+    assert p.returncode == ExitCode.missing_dependency
+
+
+def test_oem_on_tess3(resources, no_outpdf):
+    p, _, err = pytest.helpers.run_ocrmypdf(
+        resources / 'aspect.pdf',
+        no_outpdf, '--tesseract-oem', '1')
+
+    assert p.returncode == ExitCode.ok
+    assert 'argument ignored' in err

tests/test_tess4.py

@@ -1,31 +1,43 @@
 #!/usr/bin/env python3
 # © 2017 James R. Barlow: github.com/jbarlow83
 
-from subprocess import Popen, PIPE, check_output, check_call, DEVNULL
-import os
-import shutil
-from contextlib import suppress
-import sys
 import pytest
-from ocrmypdf.pageinfo import pdf_get_all_pageinfo
-import PyPDF2 as pypdf
 from ocrmypdf.exceptions import ExitCode
-from ocrmypdf import leptonica
-from ocrmypdf.pdfa import file_claims_pdfa
 from ocrmypdf.exec import tesseract
+from ocrmypdf import pageinfo
+import sys
 
 
 # Skip all tests in this file if not tesseract 4
-pytestmark = pytest.mark.skipif(not tesseract.v4(),
-                                reason="tesseract 4.0 required")
+pytestmark = pytest.mark.skipif(
+    not (tesseract.v4() and tesseract.has_textonly_pdf()),
+    reason="tesseract 4.0 with textonly_pdf feature required")
+
+check_ocrmypdf = pytest.helpers.check_ocrmypdf
+run_ocrmypdf = pytest.helpers.run_ocrmypdf
+spoof = pytest.helpers.spoof
 
 
-@pytest.mark.skipif(not tesseract.has_textonly_pdf(),
-                    reason="requires textonly_pdf feature")
 def test_textonly_pdf(resources, outdir):
-    pytest.helpers.check_ocrmypdf(
+    check_ocrmypdf(
         resources / 'linn.pdf',
         outdir / 'linn_textonly.pdf', '--pdf-renderer', 'tess4')
 
 
+@pytest.mark.skipif(sys.version_info < (3, 5), reason="needs math.isclose")
+def test_pagesize_consistency_tess4(resources, outpdf):
+    from math import isclose
 
+    infile = resources / 'linn.pdf'
+
+    before_dims = pytest.helpers.first_page_dimensions(infile)
+
+    check_ocrmypdf(
+        infile,
+        outpdf, '--pdf-renderer', 'tess4',
+        '--clean', '--deskew', '--remove-background', '--clean-final')
+
+    after_dims = pytest.helpers.first_page_dimensions(outpdf)
+
+    assert isclose(before_dims[0], after_dims[0])
+    assert isclose(before_dims[1], after_dims[1])