v4.5 notes

Installing Tess4 PPA over Tess3 proved too much pain, so sever the link
between this and the jbarlow83/ocrmypdf image, starting each from
scratch. Also the complete set of language packs proves too much - the
build seems likely to fail when trying to install so many.

.dockerignore

@@ -11,11 +11,14 @@
 */*/*/*/*/*/*.pyc
 */*/*/*/*/*/*/*.pyc
 .cache/
+.eggs/
+.github/
 .git/
 .ipynb_checkpoints/
 .ruffus_history.sqlite
 bin/
 build/
+docs/
 dist/
 htmlcov/
 include/
@@ -27,5 +30,5 @@ tests/cache/
 tests/output/
 tests/resources/private/
 tmp/
-venv-3.4/
-venv-3.5/
+venv*/
+wheelhouse/

.travis.yml

@@ -1,47 +1,55 @@
 sudo: required
 dist: trusty
 language: python
-
 cache:
-    pip: true
-    directories:
-        - packages
-        - tests/cache
+  pip: true
+  directories:
+  - packages
+  - tests/cache
 
 python:
-    - "3.4"
-    - "3.5"
-    - "3.6"
-
-before_cache:
-    - rm -f $HOME/.cache/pip/log/debug.log
-
-before_install:
-    # Ubuntu packages
-    - sudo add-apt-repository ppa:vshn/ghostscript -y  # for ghostscript 9.16 (trusty has 9.10)
-    - sudo add-apt-repository ppa:heyarje/libav-11 -y  # for libav11, which is for unpaper
-    - sudo apt-get update -qq  # must go after all add-apt-repo
-    - sudo apt-get install -y ghostscript tesseract-ocr tesseract-ocr-deu tesseract-ocr-eng tesseract-ocr-fra qpdf poppler-utils libavformat56 libavcodec56 libavutil54 libffi-dev
-
-    # pip
-    - pip install --upgrade pip
-
-    # install unpaper from a previously built .deb since ubuntu's version is too old
-    - mkdir -p packages
-    - "[ -f packages/unpaper_6.1-1.deb ] || wget -q https://dl.dropboxusercontent.com/u/28971240/unpaper_6.1-1.deb -O packages/unpaper_6.1-1.deb"
-    - sudo dpkg -i packages/unpaper_6.1-1.deb
-
-install:
-    - pip install .
-    - pip install -r requirements.txt -r test_requirements.txt
-
-script:
-    # "pip install" installs the ocrmypdf into the system environment.
-    # The working copy of ocrmypdf is not an installed version and its cffi
-    # module has not been compiled. As such, move the local copy out of the
-    # way and ensure that the system copy gets tested.
-    - mv ocrmypdf dont_import_this_ocrmypdf
-    - pytest
+- '3.4'
+- '3.5'
+- '3.6'
 
 os:
-    - linux
+- linux
+
+before_cache:
+- rm -f $HOME/.cache/pip/log/debug.log
+
+before_install:
+- sudo add-apt-repository ppa:vshn/ghostscript -y
+- sudo add-apt-repository ppa:heyarje/libav-11 -y
+- sudo apt-get update -qq
+- sudo apt-get install -y ghostscript tesseract-ocr tesseract-ocr-deu tesseract-ocr-eng
+  tesseract-ocr-fra qpdf poppler-utils libavformat56 libavcodec56 libavutil54 libffi-dev
+- pip install --upgrade pip
+- mkdir -p packages
+- "[ -f packages/unpaper_6.1-1.deb ] || wget -q https://dl.dropboxusercontent.com/u/28971240/unpaper_6.1-1.deb
+  -O packages/unpaper_6.1-1.deb"
+- sudo dpkg -i packages/unpaper_6.1-1.deb
+
+install:
+- pip install .
+- pip install -r requirements.txt -r test_requirements.txt
+
+script:
+- mv ocrmypdf dont_import_this_ocrmypdf
+- pytest
+- mv dont_import_this_ocrmypdf ocrmypdf
+
+# See https://www.appneta.com/blog/pypi-deployment-with-travis-ci/ for
+# steps to set up testpypi deploy for untagged builds if desired
+
+deploy:
+  provider: pypi
+  user: ocrmypdf-travis
+  password:
+    secure: DTFOmmNL6olA0+yXvp4u9jXZlZeqrJsJ0526jzqf4a3gZ6jnGTq5UI6WzRsslSyoMMfXKtHQebqHM6ogSgCZinyZ3ufHJo8fn9brxbEc2gsiWkbj5o3bGwdWMT1vNNE7XW0VCpw87rZ1EEwjl4FJHFudMlPR1yfU5+uq0k0PACo=
+  distributions: "sdist bdist_wheel"
+  on:
+    branch: master
+    tags: true
+    condition: $TRAVIS_PYTHON_VERSION = "3.6"
+  skip_upload_docs: true

Dockerfile

@@ -1,91 +1,54 @@
 # OCRmyPDF
 #
-# VERSION               3.2
-FROM      debian:stretch
+# VERSION               4.4.2
+FROM      ubuntu:16.10
 MAINTAINER James R. Barlow <jim@purplerock.ca>
 
-# Add unprivileged user
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  software-properties-common python-software-properties \
+  python3-wheel \
+  python3-reportlab \
+  python3-venv \
+  ghostscript \
+  qpdf \
+  poppler-utils \
+  unpaper \
+  libffi-dev \ 
+  tesseract-ocr \
+  tesseract-ocr-eng \
+  tesseract-ocr-fra \
+  tesseract-ocr-spa \
+  tesseract-ocr-deu
+
+RUN python3 -m venv --system-site-packages /appenv
+
+# This installs the latest binary wheel instead of the code in the current
+# folder. Installing from source will fail, apparently because cffi needs
+# build-essentials (gcc) to do a source installation 
+# (i.e. "pip install ."). It's unclear to me why this is the case.
+RUN . /appenv/bin/activate; \
+  pip install --upgrade pip \
+  && pip install ocrmypdf
+
+# Now copy the application in, mainly to get the test suite.
+# Do this now to make the best use of Docker cache.
+COPY . /application
+RUN . /appenv/bin/activate; \
+  pip install -r /application/test_requirements.txt
+
+# Remove the junk, including the source version of application since it was
+# already installed
+RUN rm -rf /tmp/* /var/tmp/* /root/* /application/ocrmypdf \
+  && apt-get autoremove -y \
+  && apt-get autoclean -y
+
 RUN useradd docker \
   && mkdir /home/docker \
   && chown docker:docker /home/docker
 
-# Update system and install our dependencies
-# If this command takes too Docker hub's automated build will timeout,
-# so try it in portions
-RUN apt-get update && apt-get install -y --no-install-recommends \
-  locales \
-  python3 \
-  python3-pip \
-  python3-venv \
-  python3-reportlab \
-  python3-pil \
-  python3-wheel
-
-RUN apt-get install -y --no-install-recommends \
-  unpaper \
-  qpdf \
-  poppler-utils \
-  tesseract-ocr \
-  tesseract-ocr-deu tesseract-ocr-spa tesseract-ocr-eng tesseract-ocr-fra
-
-RUN apt-get install -qy --no-install-recommends \
-  libffi-dev \
-  libpython3-dev \
-  gcc
-
-# Install Ghostscript from Debian sid to work around JPEG 2000 issue in
-# Debian stretch libgs9 or gs 9.16~dfsg-2.1
-
-COPY ./share/etc-apt-sources.list /etc/apt/sources.list
-
-RUN apt-get update && apt-get install -y ghostscript/sid
-
-
-# Enforce UTF-8
-# Borrowed from https://index.docker.io/u/crosbymichael/python/ 
-RUN dpkg-reconfigure locales && \
-  locale-gen C.UTF-8 && \
-  /usr/sbin/update-locale LANG=C.UTF-8
-ENV LC_ALL C.UTF-8
-
-
-# Set up a Python virtualenv and take all of the system packages, so we can
-# rely on the platform packages rather than importing GCC and compiling them
-RUN pyvenv /appenv \
-  && pyvenv --system-site-packages /appenv
-
-COPY . /application/
-
-# Replace stock Tesseract 3.04.00 font with improved sharp2.ttf that resolves
-# issues in many PDF viewers.
-# Discussion is in https://github.com/tesseract-ocr/tesseract/issues/182
-COPY ./share/sharp2.ttf /usr/share/tesseract-ocr/tessdata/pdf.ttf
-RUN chmod 644 /usr/share/tesseract-ocr/tessdata/pdf.ttf
-
-# Set this here to force a docker version, allowing non-tagged versions to
-# be built
-# ENV SETUPTOOLS_SCM_PRETEND_VERSION=v3.3.0
-
-# Install application and dependencies
-# In this arrangement Pillow and reportlab will be provided by the system
-# Even though ocrmypdf is locally present, pull from PyPI because
-# Dockerhub and setuptools_scm clash
-RUN . /appenv/bin/activate; \
-  pip install --upgrade pip \
-  && pip install ocrmypdf \
-  && pip install --no-cache-dir -r /application/test_requirements.txt
-
-# Remove the junk
-RUN apt-get remove -qy gcc
-RUN apt-get autoremove -y && apt-get clean -y
-RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/*
-
 USER docker
 WORKDIR /home/docker
 
-ENV OCRMYPDF_TEST_OUTPUT=/tmp/test-output
-ENV OCRMYPDF_SHARP_TTF=1
-
 # Must use array form of ENTRYPOINT
 # Non-array form does not append other arguments, because that is "intuitive"
-ENTRYPOINT ["/application/docker-wrapper.sh"]
+ENTRYPOINT ["/application/docker-wrapper.sh"]

Dockerfile.polyglot

@@ -1,14 +1,17 @@
 # OCRmyPDF polyglot
 #
-# VERSION               3.2
+# VERSION               4.4.2
 FROM      jbarlow83/ocrmypdf:latest
 MAINTAINER James R. Barlow <jim@purplerock.ca>
 
-# Update system and install our dependencies
 USER root
+
+# Update system and install our dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends \
   tesseract-ocr-all 
 
+RUN apt-get autoremove -y && apt-get clean -y
+
 USER docker
 
 # Must use array form of ENTRYPOINT

Dockerfile.tess4

@@ -0,0 +1,64 @@
+# OCRmyPDF
+#
+# VERSION               4.4.2
+FROM 	ubuntu:16.10
+MAINTAINER James R. Barlow <jim@purplerock.ca>
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  software-properties-common python-software-properties \
+  python3-wheel \
+  python3-reportlab \
+  python3-venv \
+  ghostscript \
+  qpdf \
+  poppler-utils \
+  unpaper \
+  libffi-dev
+
+RUN add-apt-repository ppa:alex-p/tesseract-ocr
+
+RUN apt-get update \
+	&& apt-get autoremove -y \
+	&& apt-get install -y --no-install-recommends \
+		tesseract-ocr \
+		tesseract-ocr-eng \
+		tesseract-ocr-fra \
+		tesseract-ocr-deu \
+		tesseract-ocr-spa \
+		tesseract-ocr-por \
+		tesseract-ocr-ara \
+		tesseract-ocr-rus \
+		tesseract-ocr-chi-sim 
+
+RUN python3 -m venv --system-site-packages /appenv
+
+# This installs the latest binary wheel instead of the code in the current
+# folder. Installing from source will fail, apparently because cffi needs
+# build-essentials (gcc) to do a source installation 
+# (i.e. "pip install ."). It's unclear to me why this is the case.
+RUN . /appenv/bin/activate; \
+  pip install --upgrade pip \
+  && pip install ocrmypdf
+
+# Now copy the application in, mainly to get the test suite.
+# Do this now to make the best use of Docker cache.
+COPY . /application
+RUN . /appenv/bin/activate; \
+  pip install -r /application/test_requirements.txt
+
+# Remove the junk, including the source version of application since it was
+# already installed
+RUN rm -rf /tmp/* /var/tmp/* /root/* /application/ocrmypdf \
+  && apt-get autoremove -y \
+  && apt-get autoclean -y
+
+RUN useradd docker \
+  && mkdir /home/docker \
+  && chown docker:docker /home/docker
+
+USER docker
+WORKDIR /home/docker
+
+# Must use array form of ENTRYPOINT
+# Non-array form does not append other arguments, because that is "intuitive"
+ENTRYPOINT ["/application/docker-wrapper.sh"]

MANIFEST.in

@@ -36,7 +36,6 @@ recursive-exclude docs/_build *
 
 # support files
 recursive-include ocrmypdf/data *
-recursive-include share *
 include *.py
 exclude tasks.py
 

RELEASE_NOTES.rst

@@ -3,17 +3,37 @@ RELEASE NOTES
 
 OCRmyPDF uses `semantic versioning <http://semver.org/>`_.
 
-v4.4.1:
-=======
+
+v4.5
+====
+
+-  Exotic PDFs containing "Form XObjects" are now supported (issue #134; PDF reference manual 8.10), and images they contain are taken into account when determining the resolution for rasterizing
+-  The Tesseract 4 Docker image no longer includes all languages, because it took so long to build something would tend to fail
+-  OCRmyPDF now warns about using ``--pdf-renderer tesseract`` with Tesseract 3.04 or lower due to issues with Ghostscript corrupting the OCR text in these cases
+
+
+v4.4.2
+======
+
+-  The Docker images (ocrmypdf, ocrmypdf-polyglot, ocrmypdf-tess4) are now based on Ubuntu 16.10 instead of Debian stretch
+
+   + This makes supporting the Tesseract 4 image easier
+   + This could be a disruptive change for any Docker users who built customized these images with their own changes, and made those changes in a way that depends on Debian and not Ubuntu
+
+-  OCRmyPDF now prevents running the Tesseract 4 renderer with Tesseract 3.04, which was permitted in v4.4 and v4.4.1 but will not work
+
+
+v4.4.1
+======
 
 -  To prevent a `TIFF output error <https://github.com/python-pillow/Pillow/issues/2206>`_ caused by img2pdf >= 0.2.1 and Pillow <= 3.4.2, dependencies have been tightened
--  The Tesseract 4.00 simultaenous process limit was increased from 1 to 2, since it was observed that 1 lowers performance
+-  The Tesseract 4.00 simultaneous process limit was increased from 1 to 2, since it was observed that 1 lowers performance
 -  Documentation improvements to describe the ``--tesseract-config`` feature 
 -  Added test cases and fixed error handling for ``--tesseract-config``
 -  Tweaks to setup.py to deal with issues in the v4.4 release
 
-v4.4:
-=====
+v4.4
+====
 
 -  Tesseract 4.00 is now supported on an experimental basis.
 
@@ -30,33 +50,33 @@ v4.4:
    + However, OCRmyPDF's dependency "ruffus" is not re-entrant, so no Python API is available. Scripts should continue to use the command line interface.
 
 
-v4.3.5:
-=======
+v4.3.5
+======
 
 -  Update documentation to confirm Python 3.6.0 compatibility. No code changes were needed, so many earlier versions are likely supported.
 
 
-v4.3.4:
-=======
+v4.3.4
+======
 
 -  Fixed "decimal.InvalidOperation: quantize result has too many digits" for high DPI images
 
 
-v4.3.3:
-=======
+v4.3.3
+======
 
 -  Fixed PDF/A creation with Ghostscript 9.20 properly
 -  Fixed an exception on inline stencil masks with a missing optional parameter
 
 
-v4.3.2:
-=======
+v4.3.2
+======
 
 -  Fixed a PDF/A creation issue with Ghostscript 9.20 (note: this fix did not actually work)
 
 
-v4.3.1:
-=======
+v4.3.1
+======
 
 -  Fixed an issue where pages produced by the "hocr" renderer after a Tesseract timeout would be rotated incorrectly if the input page was rotated with a /Rotate marker
 -  Fixed a file handle leak in LeptonicaErrorTrap that would cause a "too many open files" error for files around hundred pages of pages long when ``--deskew`` or ``--remove-background`` or other Leptonica based image processing features were in use, depending on the system value of ``ulimit -n``
@@ -66,8 +86,8 @@ v4.3.1:
 -  Tesseract caching in test cases is now more cautious about false cache hits and reproducing exact output, not that any problems were observed
 
 
-v4.3:
-=====
+v4.3
+====
 
 -  New feature ``--remove-background`` to detect and erase the background of color and grayscale images
 -  Better documentation
@@ -77,21 +97,21 @@ v4.3:
    +  This does not improve performance since temporary files are still used for buffering
    +  Some output validation is disabled in this mode
 
-v4.2.5:
-=======
+v4.2.5
+======
 
 -  Fixed an issue (#100) with PDFs that omit the optional /BitsPerComponent parameter on images
 -  Removed non-free file milk.pdf
 
 
-v4.2.4:
-=======
+v4.2.4
+======
 
 -  Fixed an error (#90) caused by PDFs that use stencil masks properly
 -  Fixed handling of PDFs that try to draw images or stencil masks without properly setting up the graphics state (such images are now ignored for the purposes of calculating DPI)
 
-v4.2.3:
-=======
+v4.2.3
+======
 
 -  Fixed an issue with PDFs that store page rotation (/Rotate) in an indirect object
 -  Integrated a few fixes to simplify downstream packaging (Debian)
@@ -104,21 +124,21 @@ v4.2.3:
 -  Deprecated the OCRmyPDF.sh shell script
 
 
-v4.2.2:
-=======
+v4.2.2
+======
 
 -  Improvements to documentation
 
 
-v4.2.1:
-=======
+v4.2.1
+======
 
 -  Fixed an issue where PDF pages that contained stencil masks would report an incorrect DPI and cause Ghostscript to abort
 -  Implemented stdin streaming
 
 
-v4.2:
-=====
+v4.2
+====
 
 -  ocrmypdf will now try to convert single image files to PDFs if they are provided as input (#15)
 
@@ -150,14 +170,14 @@ v4.2:
 
 -  Ghostscript now runs in "safer" mode where possible
 
-v4.1.4:
-=======
+v4.1.4
+======
 
 -  Bug fix: monochrome images with an ICC profile attached were incorrectly converted to full color images if lossless reconstruction was not possible due to other settings; consequence was increased file size for these images
 
 
-v4.1.3:
-=======
+v4.1.3
+======
 
 -  More helpful error message for PDFs with version 4 security handler
 -  Update usage instructions for Windows/Docker users
@@ -165,50 +185,50 @@ v4.1.3:
 -  Add a few leptonica wrapper functions (no effect on most users)
 
 
-v4.1.2:
-=======
+v4.1.2
+======
 
 -  Replace IEC sRGB ICC profile with Debian's sRGB (from icc-profiles-free) which is more compatible with the MIT license
 -  More helpful error message for an error related to certain types of malformed PDFs
 
 
-v4.1:
-=====
+v4.1
+====
 
 -  ``--rotate-pages`` now only rotates pages when reasonably confidence in the orientation. This behavior can be adjusted with the new argument ``--rotate-pages-threshold``
 -  Fixed problems in error checking if ``unpaper`` is uninstalled or missing at run-time
 -  Fixed problems with "RethrownJobError" errors during error handling that suppressed the useful error messages
 
 
-v4.0.7:
-=======
+v4.0.7
+======
 
 -  Minor correction to Ghostscript output settings
 
 
-v4.0.6:
-=======
+v4.0.6
+======
 
 -  Update install instructions
 -  Provide a sRGB profile instead of using Ghostscript's
 
 
-v4.0.5:
-=======
+v4.0.5
+======
 
 -  Remove some verbose debug messages from v4.0.4
 -  Fixed temporary that wasn't being deleted
 -  DPI is now calculated correctly for cropped images, along with other image transformations
 -  Inline images are now checked during DPI calculation instead of rejecting the image
 
-v4.0.4:
-=======
+v4.0.4
+======
 
 Released with verbose debug message turned on. Do not use. Skip to v4.0.5.
 
 
-v4.0.3:
-=======
+v4.0.3
+======
 
 New features
 ------------
@@ -225,8 +245,8 @@ Fixes
 -  Docker: fix blank JPEG2000 issue by insisting on Ghostscript versions that have this fixed
 
 
-v4.0.2:
-=======
+v4.0.2
+======
 
 Fixes
 -----
@@ -237,8 +257,8 @@ Fixes
 -  Fixed use of chmod on Docker that broke most test cases
 
 
-v4.0.1:
-=======
+v4.0.1
+======
 
 Fixes
 -----
@@ -246,8 +266,8 @@ Fixes
 -  Fixed a KeyError if tesseract fails to find page orientation information
 
 
-v4.0:
-=====
+v4.0
+====
 
 New features
 ------------
@@ -281,8 +301,8 @@ Changes
    to correct the problem.
 
 
-v3.2.1:
-=======
+v3.2.1
+======
 
 Changes
 -------
@@ -291,8 +311,8 @@ Changes
 -  Tweaked the Dockerfiles
 
 
-v3.2:
-=====
+v3.2
+====
 
 New features
 ------------
@@ -313,16 +333,16 @@ Changes
 
 
 
-v3.1.1:
-=======
+v3.1.1
+======
 
 Changes
 -------
 
 -  Fixed bug that caused incorrect page size and DPI calculations on documents with mixed page sizes
 
-v3.1:
-=====
+v3.1
+====
 
 Changes
 -------
@@ -339,8 +359,8 @@ Changes
    Currently it always chooses the 'hocrtransform' renderer but that behavior may change.
 -  Set up Travis CI automatic integration testing
 
-v3.0:
-=====
+v3.0
+====
 
 New features
 ------------
@@ -489,8 +509,8 @@ Notes and known issues
    images almost never contain inline images.
 
 
-v2.2-stable (2014-09-29):
-=========================
+v2.2-stable (2014-09-29)
+========================
 
 OCRmyPDF versions 1 and 2 were implemented as shell scripts. OCRmyPDF 3.0+ is a fork that gradually replaced all shell scripts with Python while maintaining the existing command line arguments. No one is maintaining old versions.
 

docs/batch.rst

@@ -10,11 +10,29 @@ Consider using the excellent `GNU Parallel <https://www.gnu.org/software/paralle
 
 Both ``parallel`` and ``ocrmypdf`` will try to use all available processors. To maximize parallelism without overloading your system with processes, consider using ``parallel -j 2`` to limit parallel to running two jobs at once.
 
-This command will run all ocrmypdf all files named ``*.pdf`` in the current directory and write them to the previous created ``output/`` folder.
+This command will run all ocrmypdf all files named ``*.pdf`` in the current directory and write them to the previous created ``output/`` folder. It will not search subdirectories.
+
+The ``--tag`` argument tells parallel to print the filename as a prefix whenever a message is printed, so that one can trace any errors to the file that produced them.
 
 .. code-block:: bash
 
-	parallel -j 2 ocrmypdf '{}' 'output/{}' ::: *.pdf
+	parallel --tag -j 2 ocrmypdf '{}' 'output/{}' ::: *.pdf
+
+Directory trees
+---------------
+
+This will walk through a directory tree and run OCR on all files in place, printing the output in a way that makes 
+
+.. code-block:: bash
+
+	find . --printf '%p' -name '*.pdf' -exec ocrmypdf '{}' '{}' \;
+
+This only runs one ``ocrmypdf`` process at a time. This variation uses ``find`` to create a directory list and ``parallel`` to parallelize runs of ``ocrmypdf``, again updating files in place.
+
+.. code-block:: bash
+
+	find . -name '*.pdf' | parallel --tag -j 2 ocrmypdf '{}' '{}'
+
 
 Sample script
 """""""""""""

docs/installation.rst

@@ -27,7 +27,7 @@ successfully, your system is ready to download and execute the image:
 
    docker run hello-world
    
-OCRmyPDF will use all available CPU cores.  By default, the VirtualBox machine instance on Windows and OS X has only a single CPU core enabled. Use the VirtualBox Manager to determine the name of your Docker engine host, and then follow these optional steps to enable multiple CPUs:
+OCRmyPDF will use all available CPU cores.  By default, the VirtualBox machine instance on Windows and macOS has only a single CPU core enabled. Use the VirtualBox Manager to determine the name of your Docker engine host, and then follow these optional steps to enable multiple CPUs:
 
 .. code-block:: bash
 
@@ -37,28 +37,34 @@ OCRmyPDF will use all available CPU cores.  By default, the VirtualBox machine i
    docker-machine start "yourVM"
    eval $(docker-machine env "yourVM")
 
-Assuming you have a Docker engine running somewhere, you can run these commands to download
-the image:
+Assuming you have a Docker engine running, you can download one of the three available images:
+
++-----------------------------+---------------------------------------------+---------------------------------------------------------------------------------+
+| Image name                  | Download command                            | Notes                                                                           |
++-----------------------------+---------------------------------------------+---------------------------------------------------------------------------------+
+| ocrmypdf                    | ``docker pull jbarlow83/ocrmypdf``          | Latest ocrmypdf with Tesseract 3.04. Includes English, French, German, Spanish. |
++-----------------------------+---------------------------------------------+---------------------------------------------------------------------------------+
+| ocrmypdf-polyglot           | ``docker pull jbarlow83/ocrmypdf-polyglot`` | As above, with all available language packs.                                    |
++-----------------------------+---------------------------------------------+---------------------------------------------------------------------------------+
+| ocrmypdf-tess4              | ``docker pull jbarlow83/ocrmypdf-tess4``    | Latest ocrmypdf with Tesseract 4.00.00alpha and English, French, German,        |
+|                             |                                             | Spanish, Portuguese, Chinese Simplified, Arabic and Russian (the top 8).        |
++-----------------------------+---------------------------------------------+---------------------------------------------------------------------------------+
+
+For example:
 
 .. code-block:: bash
 
-   docker pull jbarlow83/ocrmypdf
+   docker pull jbarlow83/ocrmypdf-tess4
 
 Then tag it to give a more convenient name, just ocrmypdf:
 
 .. code-block:: bash
 
-   docker tag jbarlow83/ocrmypdf ocrmypdf
+   docker tag jbarlow83/ocrmypdf-tess4 ocrmypdf
 
 .. _docker-polyglot:
 
-This image contains language packs for English, French, Spanish and German. The alternative "polyglot" image provides `all available language packs <https://github.com/tesseract-ocr/tesseract/blob/master/doc/tesseract.1.asc#languages>`_:
-
-.. code-block:: bash
-
-   # Alternative step: If you need all language packs
-   docker pull jbarlow83/ocrmypdf-polyglot
-   docker tag jbarlow83/ocrmypdf-polyglot ocrmypdf
+The alternative "polyglot" image provides `all available language packs <https://github.com/tesseract-ocr/tesseract/blob/master/doc/tesseract.1.asc#languages>`_.
 
 You can then run ocrmypdf using the command:
 
@@ -82,11 +88,25 @@ In this worked example, the current working directory contains an input file cal
 
 Note that ``ocrmypdf`` has its own separate ``-v VERBOSITYLEVEL`` argument to control debug verbosity. All Docker arguments should before the ``ocrmypdf`` image name and all arguments to ``ocrmypdf`` should be listed after.
 
+For convenience, a shell alias can hide the docker command:
 
-Installing on macOS (formerly Mac OS X)
----------------------------------------
+.. code-block:: bash
 
-These instructions probably work on all macOS supported by Homebrew. OCRmyPDF is known to work on Yosemite and El Capitan, and regularly tested on El Capitan.
+   alias ocrmypdf='docker run --rm -v "$(pwd):/home/docker" ocrmypdf' 
+   ocrmypdf --version  # runs docker version
+
+Or in the wonderful `fish shell <https://fishshell.com/>`_:
+
+.. code-block:: fish
+
+   alias ocrmypdf 'docker run --rm -v (pwd):/home/docker ocrmypdf'
+   funcsave ocrmypdf
+
+
+Installing on macOS
+-------------------
+
+These instructions probably work on all macOS supported by Homebrew.
 
 If it's not already present, `install Homebrew <http://brew.sh/>`_.
 

ocrmypdf/__main__.py

@@ -272,13 +272,15 @@ def check_options_output(options, log):
     if options.pdf_renderer == 'auto':
         options.pdf_renderer = 'hocr'
 
-    if options.pdf_renderer == 'tesseract' and \
-            tesseract.version() < '3.04.01' and \
-            os.environ.get('OCRMYPDF_SHARP_TTF', '') != '1':
-        log.warning(
-            "Your version of tesseract has problems with PDF output."
-            " Some PDF viewers will fail to find searchable text.\n"
-            "--pdf-renderer=tesseract is not recommended.")
+    if options.pdf_renderer in ('tesseract', 'tess4'):
+        if tesseract.version() < '3.05':
+            log.warning(
+                "tesseract < 3.05 may corrupt PDF output. "
+                "--pdf-renderer=tesseract is not recommend.")
+        elif tesseract.version() == '4.00.00alpha':
+            log.warning(
+                "tesseract 4.00.00alpha may corrupt PDF output. "
+                "--pdf-renderer={tesseract,tess4} is not recommend.")
 
     if options.debug_rendering and options.pdf_renderer == 'tesseract':
         log.info(
@@ -333,6 +335,10 @@ def check_options_advanced(options, log):
     if options.tesseract_oem and not tesseract.v4():
         log.warning(
             "--tesseract-oem requires Tesseract 4.x -- argument ignored")
+    if options.pdf_renderer == 'tess4' and not tesseract.has_textonly_pdf():
+        raise MissingDependencyError(
+            "--pdf-renderer tess4 requires Tesseract 4.x "
+            "commit 3d9fb3b or later")
 
 
 def check_options(options, log):
@@ -350,9 +356,6 @@ def check_options(options, log):
         sys.exit(ExitCode.missing_dependency)
 
 
-
-
-
 # ----------
 # Logging
 

ocrmypdf/pageinfo.py

@@ -89,13 +89,13 @@ def _is_unit_square(shorthand):
     pairwise = zip(values, UNIT_SQUARE)
     return all([isclose(a, b, rel_tol=1e-3) for a, b in pairwise])
 
-RasterSettings = namedtuple('RasterSettings',
+XobjectSettings = namedtuple('XobjectSettings',
     ['name', 'shorthand', 'stack_depth'])
 
 InlineSettings = namedtuple('InlineSettings',
     ['settings', 'shorthand', 'stack_depth'])
 
-ContentsInfo = namedtuple('ContentsInfo', ['raster_settings', 'inline_images'])
+ContentsInfo = namedtuple('ContentsInfo', ['xobject_settings', 'inline_images'])
 
 
 def _normalize_stack(operations):
@@ -114,7 +114,7 @@ def _normalize_stack(operations):
             yield (operands, command)
 
 
-def _interpret_contents(contentstream):
+def _interpret_contents(contentstream, initial_shorthand=UNIT_SQUARE):
     """Interpret the PDF content stream
 
     The stack represents the state of the PDF graphics stack.  We are only
@@ -139,8 +139,8 @@ def _interpret_contents(contentstream):
 
     operations = contentstream.operations
     stack = []
-    ctm = _matrix_from_shorthand(UNIT_SQUARE)
-    image_raster_settings = []
+    ctm = _matrix_from_shorthand(initial_shorthand)
+    xobject_settings = []
     inline_images = []
 
     for n, op in enumerate(_normalize_stack(operations)):
@@ -161,10 +161,10 @@ def _interpret_contents(contentstream):
                 _matrix_from_shorthand(operands), ctm)
         elif command == b'Do':
             image_name = operands[0]
-            raster = RasterSettings(
+            settings = XobjectSettings(
                 name=image_name, shorthand=_shorthand_from_matrix(ctm),
                 stack_depth=len(stack))
-            image_raster_settings.append(raster)
+            xobject_settings.append(settings)
         elif command == b'INLINE IMAGE':
             settings = operands['settings']
             inline = InlineSettings(
@@ -173,7 +173,7 @@ def _interpret_contents(contentstream):
             inline_images.append(inline)
 
     return ContentsInfo(
-        raster_settings=image_raster_settings,
+        xobject_settings=xobject_settings,
         inline_images=inline_images)
 
 
@@ -241,8 +241,8 @@ def _get_dpi(ctm_shorthand, image_size):
     return (dpi_w, dpi_h)
 
 
-def _find_page_inline_images(page, pageinfo, contentsinfo):
-    "Find inline images on the page"
+def _find_inline_images(contentsinfo):
+    "Find inline images in the contentstream"
 
     for n, inline in enumerate(contentsinfo.inline_images):
         image = {}
@@ -272,20 +272,44 @@ def _find_page_inline_images(page, pageinfo, contentsinfo):
         yield image
 
 
-def _find_page_regular_images(page, pageinfo, contentsinfo):
-    "Find images stored in XObject resources"
+def _image_xobjects(container):
+    """Search for all XObject-based images in the container
 
-    try:
-        page['/Resources']['/XObject']
-    except KeyError:
+    Usually the container is a page, but it could also be a Form XObject
+    that contains images. Filter out the Form XObjects which are dealt with
+    elsewhere.
+
+    Generate a sequence of tuples (image, xobj container), where container,
+    where xobj is the name of the object and image is the object itself,
+    since the object does not know its own name.
+
+    """
+
+    if '/Resources' not in container:
         return
-    for xobj in page['/Resources']['/XObject']:
-        # PyPDF2 returns the keys as an iterator
-        pdfimage = page['/Resources']['/XObject'][xobj]
-        if pdfimage['/Subtype'] != '/Image':
-            continue
+    resources = container['/Resources']
+    if '/XObject' not in resources:
+        return
+    for xobj in resources['/XObject']:
+        candidate = resources['/XObject'][xobj]
+        if candidate['/Subtype'] == '/Image':
+            image = candidate
+            yield (image, xobj)
+
+
+def _find_regular_images(container, contentsinfo):
+    """Find images stored in the container's /Resources /XObject
+
+    Usually the container is a page, but it could also be a Form XObject
+    that contains images.
+
+    Generates images with their DPI at time of drawing.
+
+    """
+
+    for pdfimage, xobj in _image_xobjects(container):
         image = {}
-        image['name'] = str(xobj)
+        image['name'] = xobj
         image['width'] = pdfimage['/Width']
         image['height'] = pdfimage['/Height']
         if '/BitsPerComponent' in pdfimage:
@@ -329,12 +353,12 @@ def _find_page_regular_images(page, pageinfo, contentsinfo):
 
         image['dpi_w'] = image['dpi_h'] = 0
 
-        for raster in contentsinfo.raster_settings:
+        for xobj in contentsinfo.xobject_settings:
             # Loop in case the same image is display multiple times on a page
-            if raster.name != image['name']:
+            if xobj.name != image['name']:
                 continue
 
-            if raster.stack_depth == 0 and _is_unit_square(raster.shorthand):
+            if xobj.stack_depth == 0 and _is_unit_square(xobj.shorthand):
                 # At least one PDF in the wild (and test suite) draws an image
                 # when the graphics stack depth is 0, meaning that the image
                 # gets drawn into a square of 1x1 PDF units (or 1/72",
@@ -343,7 +367,7 @@ def _find_page_regular_images(page, pageinfo, contentsinfo):
                 continue
 
             dpi_w, dpi_h = _get_dpi(
-                raster.shorthand, (image['width'], image['height']))
+                xobj.shorthand, (image['width'], image['height']))
 
             # When image is used multiple times take the highest DPI it is
             # rendered at
@@ -358,9 +382,85 @@ def _find_page_regular_images(page, pageinfo, contentsinfo):
         yield image
 
 
-def _find_page_images(page, pageinfo, contentsinfo):
-    yield from _find_page_inline_images(page, pageinfo, contentsinfo)
-    yield from _find_page_regular_images(page, pageinfo, contentsinfo)
+def _find_form_xobject_images(pdf, container, contentsinfo):
+    """Find any images that are in Form XObjects in the container
+
+    The container may be a page, or a parent Form XObject.
+
+    """
+    if '/Resources' not in container:
+        return
+    resources = container['/Resources']
+    if '/XObject' not in resources:
+        return
+    for xobj in resources['/XObject']:
+        candidate = resources['/XObject'][xobj]
+        if candidate['/Subtype'] != '/Form':
+            continue
+
+        form_xobject = candidate
+        for settings in contentsinfo.xobject_settings:
+            if settings.name != xobj:
+                continue
+
+            # Find images once for each time this Form XObject is drawn.
+            # This could be optimized to cache the multiple drawing events
+            # but in practice both Form XObjects and multiple drawing of the
+            # same object are both very rare.
+            ctm_shorthand = settings.shorthand
+            yield from _find_images(pdf, form_xobject, ctm_shorthand)
+
+
+def _find_images(pdf, container, shorthand=None):
+    """Find all individual instances of images drawn in the container
+
+    Usually the container is a page, but it may also be a Form XObject.
+
+    On a typical page images are stored inline or as regular images
+    in an XObject.
+
+    Form XObjects may include inline images, XObject images,
+    and recursively, other Form XObjects; and also vector drawing commands.
+
+    Every instance of an image being drawn somewhere is flattened and
+    treated as a unique image, since if the same image is drawn multiple times
+    on one page it may be drawn at differing resolutions, and our objective
+    is to find the resolution at which the page can be rastered without
+    downsampling.
+
+    """
+
+    if container.get('/Type') == '/Page':
+        # For a /Page the content stream is attached to the page's /Contents
+        page = container
+        contentstream = pypdf.pdf.ContentStream(page.getContents(), pdf)
+        initial_shorthand = shorthand or UNIT_SQUARE
+    elif container.get('/Type') == '/XObject' and \
+            container['/Subtype'] == '/Form':
+        # For a Form XObject that content stream is attached to the XObject
+        contentstream = pypdf.pdf.ContentStream(container, pdf)
+
+        # Set the CTM to the state it was when the "Do" operator was
+        # encountered that is drawing this instance of the Form XObject
+        ctm = _matrix_from_shorthand(shorthand or UNIT_SQUARE)
+
+        # A Form XObject may provide its own matrix to map form space into
+        # user space. Get this if one exists
+        form_matrix = _matrix_from_shorthand(
+                container.get('/Matrix', UNIT_SQUARE))
+
+        # Concatenate form matrix with CTM to ensure CTM is correct for
+        # drawing this instance of the XObject
+        ctm = matrix_mult(form_matrix, ctm)
+        initial_shorthand = _shorthand_from_matrix(ctm)
+    else:
+        return
+
+    contentsinfo = _interpret_contents(contentstream, initial_shorthand)
+
+    yield from _find_inline_images(contentsinfo)
+    yield from _find_regular_images(container, contentsinfo)
+    yield from _find_form_xobject_images(pdf, container, contentsinfo)
 
 
 def _page_has_text(pdf, page):
@@ -405,15 +505,8 @@ def _pdf_get_pageinfo(infile, pageno: int):
     except KeyError:
         pageinfo['rotate'] = 0
 
-    try:
-        contentstream = pypdf.pdf.ContentStream(page.getContents(), pdf)
-    except AttributeError as e:
-        return pageinfo
-
-    contentsinfo = _interpret_contents(contentstream)
-    pageinfo['images'] = [im for im in _find_page_images(
-                                page, pageinfo, contentsinfo)]
-
+    pageinfo['images'] = [im for im in
+                          _find_images(pdf, page)]
     if pageinfo['images']:
         xres = max(image['dpi_w'] for image in pageinfo['images'])
         yres = max(image['dpi_h'] for image in pageinfo['images'])

share/etc-apt-sources.list

@@ -1,4 +0,0 @@
-deb http://httpredir.debian.org/debian stretch main
-deb http://httpredir.debian.org/debian stretch-updates main
-deb http://security.debian.org stretch/updates main
-deb http://ftp.de.debian.org/debian sid main contrib non-free

tests/conftest.py

@@ -24,8 +24,9 @@ def is_linux():
 
 @pytest.helpers.register
 def running_in_docker():
-    # Docker creates a file named /.dockerinit
-    return os.path.exists('/.dockerinit')
+    # Docker creates a file named /.dockerenv (newer versions) or
+    # /.dockerinit (older) -- this is undocumented, not an offical test
+    return os.path.exists('/.dockerenv') or os.path.exists('/.dockerinit')
 
 
 TESTS_ROOT = os.path.abspath(os.path.dirname(__file__))

tests/resources/README.rst

@@ -64,6 +64,9 @@ under the terms of the license in LICENSE.rst.
     *   - epson.pdf
         - @lowesjam
         - a linearized PDF containing some unusual indirect objects, created by an Epson printer; printout of a Wikipedia article (CC BY-SA)
+    *   - formxobject.pdf
+        - @jbarlow83
+        - hand-crafted exotic PDF containing an image inside a Form XObject
     *   - francais.pdf
         - @jbarlow83
         - a page containing French accents (diacritics)  

tests/test_main.py

@@ -682,6 +682,10 @@ def test_overlay(spoof_tesseract_noop, resources, outpdf):
                    env=spoof_tesseract_noop)
 
 
+@pytest.mark.skipif(
+    os.getuid() == 0 or os.geteuid() == 0,
+    reason="root can write to anything"
+    )
 def test_destination_not_writable(spoof_tesseract_noop, resources, outdir):
     protected_file = outdir / 'protected.pdf'
     protected_file.touch()
@@ -738,3 +742,9 @@ THIS FILE IS INVALID
         '--tesseract-config', str(cfg_file))
     assert "parameter not found" in err, "No error message"
     assert p.returncode == ExitCode.invalid_config
+
+
+def test_form_xobject(spoof_tesseract_noop, resources, outpdf):
+    check_ocrmypdf(resources / 'formxobject.pdf', outpdf,
+                   '--force-ocr',
+                   env=spoof_tesseract_noop)

tests/test_pageinfo.py

@@ -101,3 +101,10 @@ def test_jpeg(resources, outdir):
     assert pdfimage['enc'] == 'jpeg'
     assert (pdfimage['dpi_w'] - 150) < 1e-5
 
+
+def test_form_xobject(resources):
+    filename = resources / 'formxobject.pdf'
+
+    pdfinfo = pageinfo.pdf_get_all_pageinfo(str(filename))
+    pdfimage = pdfinfo[0]['images'][0]
+    assert pdfimage['width'] == 50

tests/test_tess3.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+# © 2017 James R. Barlow: github.com/jbarlow83
+
+import pytest
+from ocrmypdf.exceptions import ExitCode
+from ocrmypdf.exec import tesseract
+
+
+# Skip all tests in this file if not tesseract 3
+pytestmark = pytest.mark.skipif(tesseract.v4(),
+                                reason="tesseract 3.x required")
+
+
+def test_textonly_pdf_on_tess3(resources, no_outpdf):
+    p, _, _ = pytest.helpers.run_ocrmypdf(
+        resources / 'linn.pdf',
+        no_outpdf, '--pdf-renderer', 'tess4')
+
+    assert p.returncode == ExitCode.missing_dependency
+
+
+def test_oem_on_tess3(resources, no_outpdf):
+    p, _, err = pytest.helpers.run_ocrmypdf(
+        resources / 'aspect.pdf',
+        no_outpdf, '--tesseract-oem', '1')
+
+    assert p.returncode == ExitCode.ok
+    assert 'argument ignored' in err

tests/test_tess4.py

@@ -1,17 +1,8 @@
 #!/usr/bin/env python3
 # © 2017 James R. Barlow: github.com/jbarlow83
 
-from subprocess import Popen, PIPE, check_output, check_call, DEVNULL
-import os
-import shutil
-from contextlib import suppress
-import sys
 import pytest
-from ocrmypdf.pageinfo import pdf_get_all_pageinfo
-import PyPDF2 as pypdf
 from ocrmypdf.exceptions import ExitCode
-from ocrmypdf import leptonica
-from ocrmypdf.pdfa import file_claims_pdfa
 from ocrmypdf.exec import tesseract