v4.5.6 release notes

Fix #156 - NoneType has no ‘getObject’ for pages with no /Contents
Ensure that ocrmypdf stops and reports an error if Ghostscript fails
2017-05-01 15:55:02 -07:00 · 2017-05-01 15:46:15 -07:00 · 2017-05-01 15:44:21 -07:00 · 2017-05-01 14:55:47 -07:00 · 2017-04-28 17:18:01 -07:00 · 2017-04-28 15:27:41 -07:00
47 changed files with 1588 additions and 454 deletions
@@ -11,11 +11,14 @@
 */*/*/*/*/*/*.pyc
 */*/*/*/*/*/*/*.pyc
 .cache/
+.eggs/
+.github/
 .git/
 .ipynb_checkpoints/
 .ruffus_history.sqlite
 bin/
 build/
+docs/
 dist/
 htmlcov/
 include/
@@ -27,5 +30,5 @@ tests/cache/
 tests/output/
 tests/resources/private/
 tmp/
-venv-3.4/
-venv-3.5/
+venv*/
+wheelhouse/
@@ -4,6 +4,9 @@
 venv*/
 pyvenv.cfg
 tasks.py
+.bash_history
+.ruffus_history.sqlite
+.idea/

 # Package building
 *.egg-info/
@@ -27,8 +30,11 @@ htmlcov/
 # Testing
 log/
 /*.pdf
+/*.qdf
+*.ipynb
 .ipynb_checkpoints/
 tests/cache/
 tests/output/
 tests/resources/private/
 tmp/
+pdfbox-app*.jar
@@ -1,46 +1,82 @@
-sudo: required
 dist: trusty
 language: python
-
-cache: pip
-
 cache:
-    directories:
-        - $HOME/.ccache
-        - packages
-        - tests/cache
+  pip: true
+  directories:
+  - packages
+  - tests/cache
+  - $HOME/Library/Caches/Homebrew

-python:
-    - "3.4"
-    - "3.5"
-    - "3.6"
+matrix:
+  include:
+    - os: linux
+      sudo: required
+      language: python
+      python: 3.4
+    - os: linux
+      sudo: required
+      language: python
+      python: 3.5
+    - os: linux
+      sudo: required
+      language: python
+      python: 3.6
+      env:
+        - secure: "oyX5xesoHD7qcDXKIxMyGZPi+H/WxcvfFkaviEmq84K1DDyHk48+9e92IKgrw8/lcTADnEo/AgVKfnhCPflFimk1xTkgaK4sUg1WLI2YjmaHcwl5SlBHa2rN3uGBwy1hyP92qyv/mMc9R59NtRJ8u76lbn6eN9wi7lkFWdE6BTw="  # DOCKERHUB_OCRMYPDF_TOKEN
+    - os: osx
+      osx_image: xcode8
+      language: generic
+      env:
+        - secure: "hsf6MT+n2x3OiDM2fQyJZdV0/PWYmv81LdVqC6cfnHBE/8N3DloJRqQ7WfO14TxhiK9PEC7MpyCj0lSabUHEO7gSH6Vks6I1asoSkt8S9/bSMlhT4hei+pwVpeGEiU5xHVATNjY+D919VC3IFvc3XmjT74h/2SLhaZ+jhEmDggM="  # HOMEBREW_OCRMYPDF_TOKEN

 before_cache:
-    - rm -f $HOME/.cache/pip/log/debug.log
+- rm -f $HOME/.cache/pip/log/debug.log

-before_install:
-    # Ubuntu packages
-    - sudo add-apt-repository ppa:vshn/ghostscript -y  # for ghostscript 9.16 (trusty has 9.10)
-    - sudo add-apt-repository ppa:heyarje/libav-11 -y  # for libav11, which is for unpaper
-    - sudo apt-get update -qq  # must go after all add-apt-repo
-    - sudo apt-get install -y ghostscript tesseract-ocr tesseract-ocr-deu tesseract-ocr-eng tesseract-ocr-fra qpdf poppler-utils libavformat56 libavcodec56 libavutil54 libffi-dev
-
-    # pip
-    - pip install --upgrade pip
-
-    # Download, make and install unpaper (using ccache)
-    - mkdir -p packages
-    - "[ -f packages/unpaper_6.1-1.deb ] || wget -q https://dl.dropboxusercontent.com/u/28971240/unpaper_6.1-1.deb -O packages/unpaper_6.1-1.deb"
-    - sudo dpkg -i packages/unpaper_6.1-1.deb
+before_install: |
+  if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
+    bash .travis/linux_before_install.sh
+  elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+    bash .travis/osx_before_install.sh
+  fi

 install:
-    - pip install -r requirements.txt
-    - pip install -r test_requirements.txt
+- pip3 install .
+- pip3 install -r requirements.txt -r test_requirements.txt

 script:
-    - python setup.py clean
-    - python setup.py install
-    - python setup.py test
+- mv ocrmypdf dont_import_this_ocrmypdf
+- pytest
+- mv dont_import_this_ocrmypdf ocrmypdf

-os:
-    - linux
+deploy:
+  # release for main pypi
+- provider: pypi
+  user: ocrmypdf-travis
+  password:
+    secure: "DTFOmmNL6olA0+yXvp4u9jXZlZeqrJsJ0526jzqf4a3gZ6jnGTq5UI6WzRsslSyoMMfXKtHQebqHM6ogSgCZinyZ3ufHJo8fn9brxbEc2gsiWkbj5o3bGwdWMT1vNNE7XW0VCpw87rZ1EEwjl4FJHFudMlPR1yfU5+uq0k0PACo="
+  distributions: "sdist bdist_wheel"
+  on:
+    branch: master
+    tags: true
+    condition: $TRAVIS_PYTHON_VERSION == "3.6" && $TRAVIS_OS_NAME == "linux"
+  skip_upload_docs: true
+
+  # test pypi
+- provider: pypi
+  server: https://testpypi.python.org/pypi
+  user: ocrmypdf-travis
+  password:
+    secure: "DTFOmmNL6olA0+yXvp4u9jXZlZeqrJsJ0526jzqf4a3gZ6jnGTq5UI6WzRsslSyoMMfXKtHQebqHM6ogSgCZinyZ3ufHJo8fn9brxbEc2gsiWkbj5o3bGwdWMT1vNNE7XW0VCpw87rZ1EEwjl4FJHFudMlPR1yfU5+uq0k0PACo="
+  distributions: "sdist"
+  on:
+    branch: develop
+    tags: false
+    condition: $TRAVIS_OS_NAME == "osx"
+  skip_upload_docs: true
+
+after_deploy: |
+  if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+    bash .travis/osx_brew.sh
+  elif [[ "$TRAVIS_PYTHON_VERSION" == "3.6" && "$TRAVIS_OS_NAME" == "linux" ]]; then
+    curl -H "Content-Type: application/json" --data '{"build": true}' -X POST https://registry.hub.docker.com/u/jbarlow83/ocrmypdf/trigger/$DOCKERHUB_OCRMYPDF_TOKEN/
+  fi
@@ -0,0 +1,74 @@
+from string import Template
+from subprocess import run, PIPE
+import re
+
+recipe_template = Template("""
+class Ocrmypdf < Formula
+  include Language::Python::Virtualenv
+
+  desc "Adds an OCR text layer to scanned PDF files"
+  homepage "https://github.com/jbarlow83/OCRmyPDF"
+  ${ocrmypdf_url}
+  ${ocrmypdf_sha256}
+
+  depends_on :python3
+  depends_on "pkg-config" => :build
+  depends_on "libffi"
+  depends_on "tesseract"
+  depends_on "ghostscript"
+  depends_on "unpaper"
+  depends_on "qpdf"
+
+  # For Pillow source install
+  depends_on "openjpeg"
+  depends_on "freetype"
+  depends_on "libpng"
+
+${resources}
+
+  def install
+    ENV.append ["SETUPTOOLS_SCM_PRETEND_VERSION"], "v${ocrmypdf_version}"
+    ENV.each do |key, value|
+      puts "#{key}:#{value}"
+    end
+    virtualenv_install_with_resources
+  end
+
+  test do
+    # `test do` will create, run in and delete a temporary directory.
+    #
+    # The installed folder is not in the path, so use the entire path to any
+    # executables being tested: `system "#{bin}/program", "do", "something"`.
+    system "#{bin}/ocrmypdf", "--version"
+  end
+end
+""")
+
+def main():
+    p = run(['poet', '--single', 'ocrmypdf'],
+            encoding='utf-8', stdout=PIPE, check=True)
+
+    ocrmypdf_lines = p.stdout.splitlines()
+    ocrmypdf_url = ocrmypdf_lines[1].strip()
+    ocrmypdf_sha256 = ocrmypdf_lines[2].strip()
+
+    ocrmypdf_version = re.search(
+        r'ocrmypdf-(.+)\.tar.*', ocrmypdf_url).group(1)
+    print(f"Autobrewing {ocrmypdf_version}")
+
+    p = run(['poet', '--resources', 'ocrmypdf'],
+            encoding='utf-8', stdout=PIPE, check=True)
+
+    poet_resources = p.stdout
+
+    # Remove the duplicate "ocrmypdf" resource block
+    all_resources = poet_resources.split('resource')
+    kept_resources = [block for block in all_resources if 'ocrmypdf' not in block]
+    resources = 'resource'.join(kept_resources)
+
+    with open('ocrmypdf.rb', 'w') as out:
+        out.write(recipe_template.substitute(**locals()))
+
+
+if __name__ == '__main__':
+    main()
@@ -0,0 +1,24 @@
+#!/bin/bash
+set -euo pipefail
+set -x
+
+sudo add-apt-repository ppa:vshn/ghostscript -y
+sudo add-apt-repository ppa:heyarje/libav-11 -y
+sudo apt-get update -qq
+sudo apt-get install -y \
+	ghostscript \
+	tesseract-ocr \
+	tesseract-ocr-deu \
+	tesseract-ocr-eng \
+	tesseract-ocr-fra \
+	qpdf \
+	poppler-utils \
+	libavformat56 \
+	libavcodec56 \
+	libavutil54 \
+	libffi-dev
+
+pip install --upgrade pip
+mkdir -p packages
+[ -f packages/unpaper_6.1-1.deb ] || wget -q https://dl.dropboxusercontent.com/u/28971240/unpaper_6.1-1.deb -O packages/unpaper_6.1-1.deb
+sudo dpkg -i packages/unpaper_6.1-1.deb
@@ -0,0 +1,16 @@
+#!/bin/bash
+set -euo pipefail
+set -x
+
+brew update
+
+brew install openjpeg jbig2dec libtiff     # image libraries
+brew install qpdf
+brew install ghostscript
+brew install python3
+brew install libxml2 libffi leptonica
+brew install unpaper   # optional
+brew install tesseract
+
+pip3 install --upgrade pip
+pip3 install wheel
@@ -0,0 +1,20 @@
+#!/bin/bash
+set -euo pipefail
+set -x
+
+pip3 install homebrew-pypi-poet
+python3 .travis/autobrew.py
+cat ocrmypdf.rb
+brew audit ocrmypdf.rb
+
+# Important: disable debug output so token is hidden
+set +x
+git clone https://$HOMEBREW_OCRMYPDF_TOKEN@github.com/jbarlow83/homebrew-ocrmypdf.git
+set -x
+
+pushd homebrew-ocrmypdf
+cp ../ocrmypdf.rb Formula/ocrmypdf.rb
+git add Formula/ocrmypdf.rb
+git commit -m "homebrew-ocrmypdf: automatic release $TRAVIS_BUILD_NUMBER $TRAVIS_TAG"
+git push origin master
+popd
@@ -1,91 +1,54 @@
 # OCRmyPDF
 #
-# VERSION               3.2
-FROM      debian:stretch
+# VERSION               4.4.2
+FROM      ubuntu:16.10
 MAINTAINER James R. Barlow <jim@purplerock.ca>

-# Add unprivileged user
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  software-properties-common python-software-properties \
+  python3-wheel \
+  python3-reportlab \
+  python3-venv \
+  ghostscript \
+  qpdf \
+  poppler-utils \
+  unpaper \
+  libffi-dev \ 
+  tesseract-ocr \
+  tesseract-ocr-eng \
+  tesseract-ocr-fra \
+  tesseract-ocr-spa \
+  tesseract-ocr-deu
+
+RUN python3 -m venv --system-site-packages /appenv
+
+# This installs the latest binary wheel instead of the code in the current
+# folder. Installing from source will fail, apparently because cffi needs
+# build-essentials (gcc) to do a source installation 
+# (i.e. "pip install ."). It's unclear to me why this is the case.
+RUN . /appenv/bin/activate; \
+  pip install --upgrade pip \
+  && pip install ocrmypdf
+
+# Now copy the application in, mainly to get the test suite.
+# Do this now to make the best use of Docker cache.
+COPY . /application
+RUN . /appenv/bin/activate; \
+  pip install -r /application/test_requirements.txt
+
+# Remove the junk, including the source version of application since it was
+# already installed
+RUN rm -rf /tmp/* /var/tmp/* /root/* /application/ocrmypdf \
+  && apt-get autoremove -y \
+  && apt-get autoclean -y
+
 RUN useradd docker \
  && mkdir /home/docker \
  && chown docker:docker /home/docker

-# Update system and install our dependencies
-# If this command takes too Docker hub's automated build will timeout,
-# so try it in portions
-RUN apt-get update && apt-get install -y --no-install-recommends \
-  locales \
-  python3 \
-  python3-pip \
-  python3-venv \
-  python3-reportlab \
-  python3-pil \
-  python3-wheel
-
-RUN apt-get install -y --no-install-recommends \
-  unpaper \
-  qpdf \
-  poppler-utils \
-  tesseract-ocr \
-  tesseract-ocr-deu tesseract-ocr-spa tesseract-ocr-eng tesseract-ocr-fra
-
-RUN apt-get install -qy --no-install-recommends \
-  libffi-dev \
-  libpython3-dev \
-  gcc
-
-# Install Ghostscript from Debian sid to work around JPEG 2000 issue in
-# Debian stretch libgs9 or gs 9.16~dfsg-2.1
-
-COPY ./share/etc-apt-sources.list /etc/apt/sources.list
-
-RUN apt-get update && apt-get install -y ghostscript/sid
-
-
-# Enforce UTF-8
-# Borrowed from https://index.docker.io/u/crosbymichael/python/ 
-RUN dpkg-reconfigure locales && \
-  locale-gen C.UTF-8 && \
-  /usr/sbin/update-locale LANG=C.UTF-8
-ENV LC_ALL C.UTF-8
-
-
-# Set up a Python virtualenv and take all of the system packages, so we can
-# rely on the platform packages rather than importing GCC and compiling them
-RUN pyvenv /appenv \
-  && pyvenv --system-site-packages /appenv
-
-COPY . /application/
-
-# Replace stock Tesseract 3.04.00 font with improved sharp2.ttf that resolves
-# issues in many PDF viewers.
-# Discussion is in https://github.com/tesseract-ocr/tesseract/issues/182
-COPY ./share/sharp2.ttf /usr/share/tesseract-ocr/tessdata/pdf.ttf
-RUN chmod 644 /usr/share/tesseract-ocr/tessdata/pdf.ttf
-
-# Set this here to force a docker version, allowing non-tagged versions to
-# be built
-# ENV SETUPTOOLS_SCM_PRETEND_VERSION=v3.3.0
-
-# Install application and dependencies
-# In this arrangement Pillow and reportlab will be provided by the system
-# Even though ocrmypdf is locally present, pull from PyPI because
-# Dockerhub and setuptools_scm clash
-RUN . /appenv/bin/activate; \
-  pip install --upgrade pip \
-  && pip install ocrmypdf \
-  && pip install --no-cache-dir -r /application/test_requirements.txt
-
-# Remove the junk
-RUN apt-get remove -qy gcc
-RUN apt-get autoremove -y && apt-get clean -y
-RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/*
-
 USER docker
 WORKDIR /home/docker

-ENV OCRMYPDF_TEST_OUTPUT=/tmp/test-output
-ENV OCRMYPDF_SHARP_TTF=1
-
 # Must use array form of ENTRYPOINT
 # Non-array form does not append other arguments, because that is "intuitive"
-ENTRYPOINT ["/application/docker-wrapper.sh"]
+ENTRYPOINT ["/application/docker-wrapper.sh"]
@@ -1,14 +1,17 @@
 # OCRmyPDF polyglot
 #
-# VERSION               3.2
+# VERSION               4.4.2
 FROM      jbarlow83/ocrmypdf:latest
 MAINTAINER James R. Barlow <jim@purplerock.ca>

-# Update system and install our dependencies
 USER root
+
+# Update system and install our dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends \
  tesseract-ocr-all 

+RUN apt-get autoremove -y && apt-get clean -y
+
 USER docker

 # Must use array form of ENTRYPOINT
@@ -0,0 +1,64 @@
+# OCRmyPDF
+#
+# VERSION               4.4.2
+FROM 	ubuntu:16.10
+MAINTAINER James R. Barlow <jim@purplerock.ca>
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  software-properties-common python-software-properties \
+  python3-wheel \
+  python3-reportlab \
+  python3-venv \
+  ghostscript \
+  qpdf \
+  poppler-utils \
+  unpaper \
+  libffi-dev
+
+RUN add-apt-repository ppa:alex-p/tesseract-ocr
+
+RUN apt-get update \
+	&& apt-get autoremove -y \
+	&& apt-get install -y --no-install-recommends \
+		tesseract-ocr \
+		tesseract-ocr-eng \
+		tesseract-ocr-fra \
+		tesseract-ocr-deu \
+		tesseract-ocr-spa \
+		tesseract-ocr-por \
+		tesseract-ocr-ara \
+		tesseract-ocr-rus \
+		tesseract-ocr-chi-sim 
+
+RUN python3 -m venv --system-site-packages /appenv
+
+# This installs the latest binary wheel instead of the code in the current
+# folder. Installing from source will fail, apparently because cffi needs
+# build-essentials (gcc) to do a source installation 
+# (i.e. "pip install ."). It's unclear to me why this is the case.
+RUN . /appenv/bin/activate; \
+  pip install --upgrade pip \
+  && pip install ocrmypdf
+
+# Now copy the application in, mainly to get the test suite.
+# Do this now to make the best use of Docker cache.
+COPY . /application
+RUN . /appenv/bin/activate; \
+  pip install -r /application/test_requirements.txt
+
+# Remove the junk, including the source version of application since it was
+# already installed
+RUN rm -rf /tmp/* /var/tmp/* /root/* /application/ocrmypdf \
+  && apt-get autoremove -y \
+  && apt-get autoclean -y
+
+RUN useradd docker \
+  && mkdir /home/docker \
+  && chown docker:docker /home/docker
+
+USER docker
+WORKDIR /home/docker
+
+# Must use array form of ENTRYPOINT
+# Non-array form does not append other arguments, because that is "intuitive"
+ENTRYPOINT ["/application/docker-wrapper.sh"]
@@ -36,7 +36,6 @@ recursive-exclude docs/_build *

 # support files
 recursive-include ocrmypdf/data *
-recursive-include share *
 include *.py
 exclude tasks.py

@@ -35,7 +35,7 @@ Main features
 -  Supports more than `100 languages <https://github.com/tesseract-ocr/tessdata>`_ recognized by Tesseract
 -  Battle-tested on thousands of PDFs, a test suite and continuous integration

-For details: please consult the `release notes <RELEASE_NOTES.rst>`_.
+For details: please consult the `documentation <https://ocrmypdf.readthedocs.io/en/latest/>`_.

 Motivation
 ----------
@@ -60,7 +60,17 @@ Installation
 Linux, UNIX, and macOS are supported. Windows is not directly supported but there is a Docker image available that runs on Windows.

 Users of Debian 9 or later or Ubuntu 16.10 or later may simply
-``apt-get install ocrmypdf``.
+
+.. code-block:: bash
+
+   apt-get install ocrmypdf
+
+and macOS users may simply
+
+.. code-block:: bash
+
+   brew tap jbarlow83/ocrmypdf
+   brew install ocrmypdf

 For everyone else, `see our documentation <https://ocrmypdf.readthedocs.io/en/latest/installation.html>`_ for installation steps.

@@ -0,0 +1,127 @@
+Advanced features
+=================
+
+Control of OCR options
+----------------------
+
+OCRmyPDF provides many features to control the behavior of the OCR engine, Tesseract.
+
+When OCR is skipped
+"""""""""""""""""""
+
+If a page in a PDF seems to have text, by default OCRmyPDF will exit without modifying the PDF. This is to ensure that PDFs that were previously OCRed or were "born digital" rather than scanned are not processed. 
+
+If ``--skip-text`` is issued, then no OCR will be performed on pages that already have text. The page will be copied to the output. This may be useful for documents that contain both "born digital" and scanned content, or to use OCRmyPDF to normalize and convert to PDF/A regardless of their contents.
+
+If ``--force-ocr`` is issued, then all pages will be rasterized to images, discarding any hidden OCR text, and rasterizing any printable text. This is useful for redoing OCR, for fixing OCR text with a damaged character map (text is selectable but not searchable), and destroying redacted information.
+
+
+Time and image size limits
+""""""""""""""""""""""""""
+
+By default, OCRmyPDF permits tesseract to run for only three minutes (180 seconds) per page. This is usually more than enough time to find all text on a reasonably sized page with modern hardware. 
+
+If a page is skipped, it will be inserted without OCR. If preprocessing was requested, the preprocessed image layer will be inserted.
+
+If you want to adjust the amount of time spent on OCR, change ``--tesseract-timeout``.  You can also automatically skip images that exceed a certain number of megapixels with ``--skip-big``. (A 300 DPI, 8.5×11" page is 8.4 megapixels.)
+
+.. code-block:: bash
+
+	# Allow 300 seconds for OCR; skip any page larger than 50 megapixels
+	ocrmypdf --tesseract-timeout 300 --skip-big 50 bigfile.pdf output.pdf
+
+Overriding default tesseract
+""""""""""""""""""""""""""""
+
+OCRmyPDF checks the environment variable ``OCRMYPDF_TESSERACT`` for the full path *to the tesseract executable* first. 
+
+For example, if you are testing tesseract 4.00 and don't wish to disturb your tesseract 3.04 installation, you can launch OCRmyPDF as follows:
+
+.. code-block:: bash
+
+	env \
+		OCRMYPDF_TESSERACT=/home/user/src/tesseract4/api/tesseract \
+		TESSDATA_PREFIX=/home/user/src/tesseract4 \
+		ocrmypdf --pdf-renderer tess4 --tesseract-oem 2 input.pdf output.pdf
+
+* ``TESSDATA_PREFIX`` directs tesseract 4.0 to use LSTM training data. This is a tesseract environment variable.
+* ``--pdf-renderer tess4`` takes advantage of new tesseract 4.0 PDF renderer in OCRmyPDF. (Tesseract 4.0 only.)
+* ``--tesseract-oem 1`` requests tesseract 4.0's new LSTM engine. (Tesseract 4.0 only.)
+
+Overriding other support programs
+"""""""""""""""""""""""""""""""""
+
+In addition to tesseract, OCRmyPDF uses the following external binaries:
+
+* ``gs`` (Ghostscript)
+* ``unpaper``
+* ``qpdf``
+
+In each case OCRmyPDF will check the environment variable ``OCRMYPDF_{program}`` before asking the system to find ``{program}`` on the PATH. For example, you could redirect OCRmyPDF to ``OCRMYPDF_GS`` to override Ghostscript.
+
+Changing tesseract configuration variables
+""""""""""""""""""""""""""""""""""""""""""
+
+You can override tesseract's default `control parameters <https://github.com/tesseract-ocr/tesseract/wiki/ControlParams>`_ with a configuration file.
+
+As an example, this configuration will disable Tesseract's dictionary for current language. Normally the dictionary is helpful for interpolating words that are unclear, but it may interfere with OCR if the document does not contain many words (for example, a list of part numbers).
+
+Create a file named "no-dict.cfg" with these contents:
+
+::
+
+	load_system_dawg 0
+	language_model_penalty_non_dict_word 0
+	language_model_penalty_non_freq_dict_word 0
+
+then run ocrmypdf as follows (along with any other desired arguments):
+
+.. code-block:: bash
+
+	ocrmypdf --tesseract-config no-dict.cfg input.pdf output.pdf
+
+.. warning::
+
+	Some combinations of control parameters will break Tesseract or break assumptions that OCRmyPDF makes about Tesseract's output.
+
+
+Changing the PDF renderer
+-------------------------
+
+rasterizing
+  Converting a PDF to an image for display.
+
+rendering
+  Creating a new PDF from other data (such as an existing PDF).
+
+
+OCRmyPDF has three PDF renderers: ``hocr``, ``tesseract`` and ``tess4``. The renderer may be selected using ``--pdf-renderer``. The default is ``auto`` which lets OCRmyPDF select the renderer to use. Currently, ``auto`` always selects ``hocr``. 
+
+The ``hocr`` renderer
+"""""""""""""""""""""
+
+The ``hocr`` renderer is the default because it works in most cases. In this mode the whole PDF is rasterized, the raster image is run through OCR to generate a .hocr file, which is an HTML-like file that specifies the location of all identified words.
+
+The .hocr file is then rendered as a PDF and merged with the image layer.
+
+The image layer is copied from the original PDF page if possible, avoiding potentially lossy transcoding or loss of other PDF information. If preprocessing is specified, then the image layer is a new PDF.
+
+This is the only option for tesseract 3.02 and older.
+
+
+The ``tesseract`` renderer
+""""""""""""""""""""""""""
+
+The tesseract renderer uses tesseract's capability to produce a PDF directly. In version 3, tesseract automatically combined the image layer and text, meaning that this mode *always* transcodes and loses potentially loses image quality and other PDF information.
+
+It does a much better job on non-Latin text.
+
+In a future release this will become the "tess3" renderer and ultimately will be dropped.
+
+
+The ``tess4`` renderer
+""""""""""""""""""""""
+
+The tess4 renderer uses tesseract 4.00 alpha's text-only PDF feature added in January 2017. This combines the advantages of the tesseract and hocr renderers, transcoding the image layer only if required by preprocessing options.
+
+Ghostscript PDF/A still sometimes inserts spaces between words when the tess4 renderer is used, affecting search quality.  ``--output-pdf pdf`` may be used to avoid this issue.
@@ -10,11 +10,29 @@ Consider using the excellent `GNU Parallel <https://www.gnu.org/software/paralle

 Both ``parallel`` and ``ocrmypdf`` will try to use all available processors. To maximize parallelism without overloading your system with processes, consider using ``parallel -j 2`` to limit parallel to running two jobs at once.

-This command will run all ocrmypdf all files named ``*.pdf`` in the current directory and write them to the previous created ``output/`` folder.
+This command will run all ocrmypdf all files named ``*.pdf`` in the current directory and write them to the previous created ``output/`` folder. It will not search subdirectories.
+
+The ``--tag`` argument tells parallel to print the filename as a prefix whenever a message is printed, so that one can trace any errors to the file that produced them.

 .. code-block:: bash

-	parallel -j 2 ocrmypdf '{}' 'output/{}' ::: *.pdf
+	parallel --tag -j 2 ocrmypdf '{}' 'output/{}' ::: *.pdf
+
+Directory trees
+---------------
+
+This will walk through a directory tree and run OCR on all files in place, printing the output in a way that makes 
+
+.. code-block:: bash
+
+	find . --printf '%p' -name '*.pdf' -exec ocrmypdf '{}' '{}' \;
+
+This only runs one ``ocrmypdf`` process at a time. This variation uses ``find`` to create a directory list and ``parallel`` to parallelize runs of ``ocrmypdf``, again updating files in place.
+
+.. code-block:: bash
+
+	find . -name '*.pdf' | parallel --tag -j 2 ocrmypdf '{}' '{}'
+

 Sample script
 """""""""""""
@@ -59,7 +59,7 @@ By default OCRmyPDF assumes the document is English.
 	ocrmypdf -l fre LeParisien.pdf LeParisien.pdf
 	ocrmypdf -l eng+fre Bilingual-English-French.pdf Bilingual-English-French.pdf

-Language packs must be installed for all languages specified. See :ref:`Installing additional language packs <languages>`.
+Language packs must be installed for all languages specified. See :ref:`Installing additional language packs <lang-packs>`.


 Produce PDF and text file containing OCR text
@@ -68,10 +68,16 @@ Produce PDF and text file containing OCR text
 This produces a file named "output.pdf" and a companion text file named "output.txt". The ``pdftotext`` program from 
 `Poppler <https://poppler.freedesktop.org/>`_ is used to extract text from the finished PDF.

+
 .. code-block:: bash

 	ocrmypdf input.pdf - | tee output.pdf | pdftotext - output.txt

+.. note::
+
+	To get pdftotext, Debian/Ubuntu users may ``apt-get install poppler-utils`` 
+	and macOS users may ``brew install poppler`` respectively.
+

 OCR images, not PDFs
 --------------------
@@ -111,6 +117,14 @@ OCRmyPDF perform some image processing on each page of a PDF, if desired.  The s

 * ``--clean-final`` uses unpaper to clean up pages before OCR and inserts the page into the final output.  You will want to review each page to ensure that unpaper did not remove something important.

+.. note::
+
+	In many cases image processing will rasterize PDF pages as images, potentially losing quality.
+
+.. warning::
+
+	``--clean-final`` and ``-remove-background`` may leave undesirable visual artifacts in some images where their algorithms have shortcomings. Files should be visually reviewed after using these options.
+

 OCR and correct document skew (crooked scan)
 """"""""""""""""""""""""""""""""""""""""""""
@@ -127,16 +141,4 @@ Image processing commands can be combined. The order in which options are given

 	ocrmypdf --deskew --clean --rotate-pages input.pdf output.pdf

-Control of OCR options
----------------------
-
-By default, OCRmyPDF permits tesseract to run for only three minutes (180 seconds) per page. This is usually more than enough time to find all text on a reasonably sized page with modern hardware. A skipped page will be inserted into the output without any OCR text.
-
-If you want to adjust the amount of time spent on OCR, change ``--tesseract-timeout``.  You can also automatically skip images that exceed a certain number of megapixels.
-
-.. code-block:: bash
-
-	# Allow 300 seconds for OCR; skip any page larger than 50 megapixels
-	ocrmypdf --tesseract-timeout 300 --skip-big 50 bigfile.pdf output.pdf
-

@@ -11,22 +11,25 @@ be searched.

 PDFs are the best format for scanned documents.  Unfortunately, PDFs can be difficult to work with. OCRmyPDF makes it easy to apply image processing and OCR to existing PDFs.

-Contents:
-
 .. toctree::
-   :maxdepth: 2
+   :maxdepth: 1

   introduction
+   release_notes
   installation
   languages
+
+.. toctree::
+   :caption: Usage
+   :maxdepth: 2
+
   cookbook
+   advanced
   batch
-   renderers
   security
   errors


-
 Indices and tables
 ==================

@@ -10,6 +10,18 @@ Users of Debian 9 ("stretch") or later or Ubuntu 16.10 or later may simply

 	apt-get install ocrmypdf

+
+Installing on macOS
+-------------------
+
+.. code-block:: bash
+
+   brew tap jbarlow83/ocrmypdf
+   brew install ocrmypdf
+
+
+Users who previously installed OCRmyPDF on macOS may need to remove the ``pip`` based installation (``pip3 uninstall ocrmypdf``).
+
 .. _Docker:

 Installing the Docker image
@@ -27,7 +39,7 @@ successfully, your system is ready to download and execute the image:

   docker run hello-world
   
-OCRmyPDF will use all available CPU cores.  By default, the VirtualBox machine instance on Windows and OS X has only a single CPU core enabled. Use the VirtualBox Manager to determine the name of your Docker engine host, and then follow these optional steps to enable multiple CPUs:
+OCRmyPDF will use all available CPU cores.  By default, the VirtualBox machine instance on Windows and macOS has only a single CPU core enabled. Use the VirtualBox Manager to determine the name of your Docker engine host, and then follow these optional steps to enable multiple CPUs:

 .. code-block:: bash

@@ -37,28 +49,34 @@ OCRmyPDF will use all available CPU cores.  By default, the VirtualBox machine i
   docker-machine start "yourVM"
   eval $(docker-machine env "yourVM")

-Assuming you have a Docker engine running somewhere, you can run these commands to download
-the image:
+Assuming you have a Docker engine running, you can download one of the three available images:
+
+-----------------------------+---------------------------------------------+---------------------------------------------------------------------------------+
+| Image name                  | Download command                            | Notes                                                                           |
+-----------------------------+---------------------------------------------+---------------------------------------------------------------------------------+
+| ocrmypdf                    | ``docker pull jbarlow83/ocrmypdf``          | Latest ocrmypdf with Tesseract 3.04. Includes English, French, German, Spanish. |
+-----------------------------+---------------------------------------------+---------------------------------------------------------------------------------+
+| ocrmypdf-polyglot           | ``docker pull jbarlow83/ocrmypdf-polyglot`` | As above, with all available language packs.                                    |
+-----------------------------+---------------------------------------------+---------------------------------------------------------------------------------+
+| ocrmypdf-tess4              | ``docker pull jbarlow83/ocrmypdf-tess4``    | Latest ocrmypdf with Tesseract 4.00.00alpha and English, French, German,        |
+|                             |                                             | Spanish, Portuguese, Chinese Simplified, Arabic and Russian (the top 8).        |
+-----------------------------+---------------------------------------------+---------------------------------------------------------------------------------+
+
+For example:

 .. code-block:: bash

-   docker pull jbarlow83/ocrmypdf
+   docker pull jbarlow83/ocrmypdf-tess4

 Then tag it to give a more convenient name, just ocrmypdf:

 .. code-block:: bash

-   docker tag jbarlow83/ocrmypdf ocrmypdf
+   docker tag jbarlow83/ocrmypdf-tess4 ocrmypdf

 .. _docker-polyglot:

-This image contains language packs for English, French, Spanish and German. The alternative "polyglot" image provides `all available language packs <https://github.com/tesseract-ocr/tesseract/blob/master/doc/tesseract.1.asc#languages>`_:
-
-.. code-block:: bash
-
-   # Alternative step: If you need all language packs
-   docker pull jbarlow83/ocrmypdf-polyglot
-   docker tag jbarlow83/ocrmypdf-polyglot ocrmypdf
+The alternative "polyglot" image provides `all available language packs <https://github.com/tesseract-ocr/tesseract/blob/master/doc/tesseract.1.asc#languages>`_.

 You can then run ocrmypdf using the command:

@@ -82,11 +100,25 @@ In this worked example, the current working directory contains an input file cal

 Note that ``ocrmypdf`` has its own separate ``-v VERBOSITYLEVEL`` argument to control debug verbosity. All Docker arguments should before the ``ocrmypdf`` image name and all arguments to ``ocrmypdf`` should be listed after.

+For convenience, a shell alias can hide the docker command:

-Installing on macOS (formerly Mac OS X)
---------------------------------------
+.. code-block:: bash

-These instructions probably work on all macOS supported by Homebrew. OCRmyPDF is known to work on Yosemite and El Capitan, and regularly tested on El Capitan.
+   alias ocrmypdf='docker run --rm -v "$(pwd):/home/docker" ocrmypdf' 
+   ocrmypdf --version  # runs docker version
+
+Or in the wonderful `fish shell <https://fishshell.com/>`_:
+
+.. code-block:: fish
+
+   alias ocrmypdf 'docker run --rm -v (pwd):/home/docker ocrmypdf'
+   funcsave ocrmypdf
+
+
+Manual installation on macOS
+----------------------------
+
+These instructions probably work on all macOS supported by Homebrew.

 If it's not already present, `install Homebrew <http://brew.sh/>`_.

@@ -155,10 +187,6 @@ No package is currently available for Ubuntu 16.04, but you can install the depe
      tesseract-ocr \
      qpdf \
      python3-pip \
-      python3-pil \
-      python3-pytest \
-      python3-reportlab \
-      python3-ruffus \
      python3-cffi

 If you wish install OCRmyPDF to the system Python, then install as follows (note this installs new packages
@@ -173,7 +201,7 @@ follow these steps.

 .. code-block:: bash

-   python3 -m venv --system-site-packages venv-ocrmypdf
+   python3 -m venv venv-ocrmypdf
   source venv-ocrmypdf/bin/activate
   pip3 install ocrmypdf

@@ -53,6 +53,20 @@ By default, OCRmyPDF will convert the file to a PDF/A.  This behavior can be dis
 Depending on the settings selected, OCRmyPDF may "graft" the OCR layer into the existing PDF, or reconstruct a visually equivalent new PDF.


+Why you shouldn't do this manually
+----------------------------------
+
+There are two routes to manually applying OCR to an existing PDF, both of which destroy information in the original PDF.
+
+1. Rasterize each page as an image, OCR the images, and combine the output into a PDF. This preserves the appearance of each page, but resamples all images (possibly losing quality, increasing file size, introducing compression artifacts, etc.)
+
+2. Extract each image, OCR, and combine the output into a PDF. This loses the context in which images are used in the PDF, meaning that cropping, rotation and scaling of pages may be lost. Some PDFs use multiple images per page with stencil masks, which would quite difficult to reassemble correctly. This also loses and text or vector art on any pages in a PDF with both scanned and pure digital content.
+
+In the case of a PDF that is nothing other than a container of images (no rotation, scaling, cropping, one image per page), the second approach is can be lossless.
+
+OCRmyPDF uses several strategies depending on input options and the input PDF itself, but generally speaking it rasterizes a page for OCR and then grafts the OCR back onto the original. As such it can handle complex PDFs and still preserve their contents as much as possible.
+
+
 Limitations
 -----------

@@ -73,4 +87,15 @@ Ghostscript also imposes some limitations:

 * PDFs containing JBIG2-encoded content will be converted to CCITT Group4 encoding, which has lower compression ratios, if Ghostscript PDF/A is enabled.
  
-OCRmyPDF is currently not designed to be used as a Python API; it is designed to be run as a command line tool. ``import ocrmypf`` currently attempts to process the command line on ``sys.argv`` at import time so it has side effects that will interfere with its use as a package. The API it presents should not be considered stable.
+OCRmyPDF is currently not designed to be used as a Python API; it is designed to be run as a command line tool. ``import ocrmypf`` currently attempts to process the command line on ``sys.argv`` at import time so it has side effects that will interfere with its use as a package. The API it presents should not be considered stable.
+
+
+Similar programs
+----------------
+
+To the author's knowledge, OCRmyPDF is the most feature-rich and thoroughly tested command line OCR PDF conversion tool. If it doesn't meet your needs, consider of these similar open source programs:
+
+* pdf2pdfocr
+* pdfsandwich
+* pypdfocr
+* pdfbeads
@@ -1,53 +1,127 @@
-RELEASE NOTES
+Release notes
 =============

 OCRmyPDF uses `semantic versioning <http://semver.org/>`_.

-v4.4:
-=====
+
+v4.5.6
+------
+
+-   Fixed issue #156, 'NoneType' object has no attribute 'getObject' on pages with no optional /Contents record.  This should resolve all issues related to pages with no /Contents record.
+-   Fixed issue #158, ocrmypdf now stops and terminates if Ghostscript fails on an intermediate step, as it is not possible to proceed.
+-   Fixed issue #160, exception thrown on certain invalid arguments instead of error message
+
+
+v4.5.5
+------
+
+-   Automated update of macOS homebrew tap
+-   Fixed issue #154, KeyError '/Contents' when searching for text on blank pages that have no /Contents record.  Note: incomplete fix for this issue.
+
+
+v4.5.4
+------
+
+-   Fix ``--skip-big`` raising an exception if a page contains no images (#152) (thanks to @TomRaz)
+-   Fix an issue where pages with no images might trigger "cannot write mode P as JPEG" (#151)
+
+
+v4.5.3
+------
+
+-  Added a workaround for Ghostscript 9.21 and probably earlier versions would fail with the error message "VMerror -25", due to a Ghostscript bug in XMP metadata handling
+-  High Unicode characters (U+10000 and up) are no longer accepted for setting metadata on the command line, as Ghostscript may not handle them correctly.
+-  Fixed an issue where the ``tess4`` renderer would duplicate content onto output pages if tesseract failed or timed out
+-  Fixed ``tess4`` renderer not recognized when lossless reconstruction is possible
+
+
+v4.5.2
+------
+
+-  Fix issue #147. ``--pdf-renderer tess4 --clean`` will produce an oversized page containing the original image in the bottom left corner, due to loss DPI information.
+-  Make "using Tesseract 4.0" warning less ominous
+-  Set up machinery for homebrew OCRmyPDF tap
+
+
+v4.5.1
+------
+
+-  Fix issue #137, proportions of images with a non-square pixel aspect ratio would be distorted in output for ``--force-ocr`` and some other combinations of flags
+
+
+v4.5
+----
+
+-  Exotic PDFs containing "Form XObjects" are now supported (issue #134; PDF reference manual 8.10), and images they contain are taken into account when determining the resolution for rasterizing
+-  The Tesseract 4 Docker image no longer includes all languages, because it took so long to build something would tend to fail
+-  OCRmyPDF now warns about using ``--pdf-renderer tesseract`` with Tesseract 3.04 or lower due to issues with Ghostscript corrupting the OCR text in these cases
+
+
+v4.4.2
+------
+
+-  The Docker images (ocrmypdf, ocrmypdf-polyglot, ocrmypdf-tess4) are now based on Ubuntu 16.10 instead of Debian stretch
+
+   + This makes supporting the Tesseract 4 image easier
+   + This could be a disruptive change for any Docker users who built customized these images with their own changes, and made those changes in a way that depends on Debian and not Ubuntu
+
+-  OCRmyPDF now prevents running the Tesseract 4 renderer with Tesseract 3.04, which was permitted in v4.4 and v4.4.1 but will not work
+
+
+v4.4.1
+------
+
+-  To prevent a `TIFF output error <https://github.com/python-pillow/Pillow/issues/2206>`_ caused by img2pdf >= 0.2.1 and Pillow <= 3.4.2, dependencies have been tightened
+-  The Tesseract 4.00 simultaneous process limit was increased from 1 to 2, since it was observed that 1 lowers performance
+-  Documentation improvements to describe the ``--tesseract-config`` feature 
+-  Added test cases and fixed error handling for ``--tesseract-config``
+-  Tweaks to setup.py to deal with issues in the v4.4 release
+
+v4.4
+----

 -  Tesseract 4.00 is now supported on an experimental basis.

   +  A new rendering option ``--pdf-renderer tess4`` exploits Tesseract 4's new text-only output PDF mode. See the documentation on PDF Renderers for details.
-   +  The ``--tesseract-oem`` argument allows control over the Tesseract 4 OCR 
-   engine mode (tesseract's ``--oem``). Use ``--tesseract-oem 2`` to enforce the new LSTM mode.
+   +  The ``--tesseract-oem`` argument allows control over the Tesseract 4 OCR engine mode (tesseract's ``--oem``). Use ``--tesseract-oem 2`` to enforce the new LSTM mode.
   +  Fixed poor performance with Tesseract 4.00 on Linux

 -  Fixed an issue that caused corruption of output to stdout in some cases
 -  Removed test for Pillow JPEG and PNG support, as the minimum supported version of Pillow now enforces this
 -  OCRmyPDF now tests that the intended destination file is writable before proceeding
+-  The test suite now requires ``pytest-helpers-namespace`` to run (but not install)
 -  Significant code reorganization to make OCRmyPDF re-entrant and improve performance. All changes should be backward compatible for the v4.x series.

   + However, OCRmyPDF's dependency "ruffus" is not re-entrant, so no Python API is available. Scripts should continue to use the command line interface.


-v4.3.5:
-=======
+v4.3.5
+------

 -  Update documentation to confirm Python 3.6.0 compatibility. No code changes were needed, so many earlier versions are likely supported.


-v4.3.4:
-=======
+v4.3.4
+------

 -  Fixed "decimal.InvalidOperation: quantize result has too many digits" for high DPI images


-v4.3.3:
-=======
+v4.3.3
+------

 -  Fixed PDF/A creation with Ghostscript 9.20 properly
 -  Fixed an exception on inline stencil masks with a missing optional parameter


-v4.3.2:
-=======
+v4.3.2
+------

 -  Fixed a PDF/A creation issue with Ghostscript 9.20 (note: this fix did not actually work)


-v4.3.1:
-=======
+v4.3.1
+------

 -  Fixed an issue where pages produced by the "hocr" renderer after a Tesseract timeout would be rotated incorrectly if the input page was rotated with a /Rotate marker
 -  Fixed a file handle leak in LeptonicaErrorTrap that would cause a "too many open files" error for files around hundred pages of pages long when ``--deskew`` or ``--remove-background`` or other Leptonica based image processing features were in use, depending on the system value of ``ulimit -n``
@@ -57,8 +131,8 @@ v4.3.1:
 -  Tesseract caching in test cases is now more cautious about false cache hits and reproducing exact output, not that any problems were observed


-v4.3:
-=====
+v4.3
+----

 -  New feature ``--remove-background`` to detect and erase the background of color and grayscale images
 -  Better documentation
@@ -68,21 +142,21 @@ v4.3:
   +  This does not improve performance since temporary files are still used for buffering
   +  Some output validation is disabled in this mode

-v4.2.5:
-=======
+v4.2.5
+------

 -  Fixed an issue (#100) with PDFs that omit the optional /BitsPerComponent parameter on images
 -  Removed non-free file milk.pdf


-v4.2.4:
-=======
+v4.2.4
+------

 -  Fixed an error (#90) caused by PDFs that use stencil masks properly
 -  Fixed handling of PDFs that try to draw images or stencil masks without properly setting up the graphics state (such images are now ignored for the purposes of calculating DPI)

-v4.2.3:
-=======
+v4.2.3
+------

 -  Fixed an issue with PDFs that store page rotation (/Rotate) in an indirect object
 -  Integrated a few fixes to simplify downstream packaging (Debian)
@@ -95,21 +169,21 @@ v4.2.3:
 -  Deprecated the OCRmyPDF.sh shell script


-v4.2.2:
-=======
+v4.2.2
+------

 -  Improvements to documentation


-v4.2.1:
-=======
+v4.2.1
+------

 -  Fixed an issue where PDF pages that contained stencil masks would report an incorrect DPI and cause Ghostscript to abort
 -  Implemented stdin streaming


-v4.2:
-=====
+v4.2
+----

 -  ocrmypdf will now try to convert single image files to PDFs if they are provided as input (#15)

@@ -141,14 +215,14 @@ v4.2:

 -  Ghostscript now runs in "safer" mode where possible

-v4.1.4:
-=======
+v4.1.4
+------

 -  Bug fix: monochrome images with an ICC profile attached were incorrectly converted to full color images if lossless reconstruction was not possible due to other settings; consequence was increased file size for these images


-v4.1.3:
-=======
+v4.1.3
+------

 -  More helpful error message for PDFs with version 4 security handler
 -  Update usage instructions for Windows/Docker users
@@ -156,59 +230,59 @@ v4.1.3:
 -  Add a few leptonica wrapper functions (no effect on most users)


-v4.1.2:
-=======
+v4.1.2
+------

 -  Replace IEC sRGB ICC profile with Debian's sRGB (from icc-profiles-free) which is more compatible with the MIT license
 -  More helpful error message for an error related to certain types of malformed PDFs


-v4.1:
-=====
+v4.1
+----

 -  ``--rotate-pages`` now only rotates pages when reasonably confidence in the orientation. This behavior can be adjusted with the new argument ``--rotate-pages-threshold``
 -  Fixed problems in error checking if ``unpaper`` is uninstalled or missing at run-time
 -  Fixed problems with "RethrownJobError" errors during error handling that suppressed the useful error messages


-v4.0.7:
-=======
+v4.0.7
+------

 -  Minor correction to Ghostscript output settings


-v4.0.6:
-=======
+v4.0.6
+------

 -  Update install instructions
 -  Provide a sRGB profile instead of using Ghostscript's


-v4.0.5:
-=======
+v4.0.5
+------

 -  Remove some verbose debug messages from v4.0.4
 -  Fixed temporary that wasn't being deleted
 -  DPI is now calculated correctly for cropped images, along with other image transformations
 -  Inline images are now checked during DPI calculation instead of rejecting the image

-v4.0.4:
-=======
+v4.0.4
+------

 Released with verbose debug message turned on. Do not use. Skip to v4.0.5.


-v4.0.3:
-=======
+v4.0.3
+------

 New features
------------
+^^^^^^^^^^^^

 -  Page orientations detected are now reported in a summary comment


 Fixes
-----
+^^^^^

 -  Show stack trace if unexpected errors occur
 -  Treat "too few characters" error message from Tesseract as a reason to skip that page rather than
@@ -216,11 +290,11 @@ Fixes
 -  Docker: fix blank JPEG2000 issue by insisting on Ghostscript versions that have this fixed


-v4.0.2:
-=======
+v4.0.2
+------

 Fixes
-----
+^^^^^

 -  Fixed compatibility with Tesseract 3.04.01 release, particularly its different way of outputting
   orientation information
@@ -228,20 +302,20 @@ Fixes
 -  Fixed use of chmod on Docker that broke most test cases


-v4.0.1:
-=======
+v4.0.1
+------

 Fixes
-----
+^^^^^

 -  Fixed a KeyError if tesseract fails to find page orientation information


-v4.0:
-=====
+v4.0
+----

 New features
------------
+^^^^^^^^^^^^

 -  Automatic page rotation (``-r``) is now available. It uses ignores any prior rotation information
   on PDFs and sets rotation based on the dominant orientation of detectable text. This feature is
@@ -251,7 +325,7 @@ New features


 Fixes
-----
+^^^^^

 -  Fixed an issue where lossless reconstruction could cause some pages to be appear incorrectly
   if the page was rotated by the user in Acrobat after being scanned (specifically if it a /Rotate tag)
@@ -260,7 +334,7 @@ Fixes


 Changes
-------
+^^^^^^^

 -  Logging output is now much easier to read
 -  ``--deskew`` is now performed by Leptonica instead of unpaper (#25)
@@ -272,21 +346,21 @@ Changes
   to correct the problem.


-v3.2.1:
-=======
+v3.2.1
+------

 Changes
-------
+^^^^^^^

 -  Fixed issue #47 "convert() got and unexpected keyword argument 'dpi'" by upgrading to img2pdf 0.2
 -  Tweaked the Dockerfiles


-v3.2:
-=====
+v3.2
+----

 New features
------------
+^^^^^^^^^^^^

 -  Lossless reconstruction: when possible, OCRmyPDF will inject text layers without 
   otherwise manipulating the content and layout of a PDF page. For example, a PDF containing a mix
@@ -298,25 +372,25 @@ New features
   for the polyglots among us. It is much larger.

 Changes
-------
+^^^^^^^

 -  JPEG transcoding quality is now 95 instead of the default 75. Bigger file sizes for less degradation.



-v3.1.1:
-=======
+v3.1.1
+------

 Changes
-------
+^^^^^^^

 -  Fixed bug that caused incorrect page size and DPI calculations on documents with mixed page sizes

-v3.1:
-=====
+v3.1
+----

 Changes
-------
+^^^^^^^

 -  Default output format is now PDF/A-2b instead of PDF/A-1b
 -  Python 3.5 and macOS El Capitan are now supported platforms - no changes were
@@ -330,11 +404,11 @@ Changes
   Currently it always chooses the 'hocrtransform' renderer but that behavior may change.
 -  Set up Travis CI automatic integration testing

-v3.0:
-=====
+v3.0
+----

 New features
------------
+^^^^^^^^^^^^

 -  Easier installation with a Docker container or Python's ``pip`` package manager 
 -  Eliminated many external dependencies, so it's easier to setup
@@ -357,7 +431,7 @@ New features
 -  Multiple images on the same PDF page are now supported

 Changes
-------
+^^^^^^^

 -  New, robust rewrite in Python 3.4+ with ruffus_ pipelines
 -  Now uses Ghostscript 9.14's improved color conversion model to preserve PDF colors
@@ -395,7 +469,7 @@ Changes
 .. _JHOVE: http://jhove.sourceforge.net/

 Release candidates
------------------
+^^^^^^^^^^^^^^^^^^

 -  rc9:

@@ -463,12 +537,12 @@ where ``settings.txt`` contains *one argument per line*, for example:


 Fixes
-----
+^^^^^

 -  Handling of filenames containing spaces: fixed

 Notes and known issues
----------------------
+^^^^^^^^^^^^^^^^^^^^^^

 -  Some dependencies may work with lower versions than tested, so try
   overriding dependencies if they are "in the way" to see if they work.
@@ -480,9 +554,9 @@ Notes and known issues
   images almost never contain inline images.


-v2.2-stable (2014-09-29):
-=========================
+v2.2-stable (2014-09-29)
+------------------------

 OCRmyPDF versions 1 and 2 were implemented as shell scripts. OCRmyPDF 3.0+ is a fork that gradually replaced all shell scripts with Python while maintaining the existing command line arguments. No one is maintaining old versions.

-For details on older versions, see the `final version of its release notes <https://github.com/fritz-hh/OCRmyPDF/blob/7fd3dbdf42ca53a619412ce8add7532c5e81a9d1/RELEASE_NOTES.md>`_.
+For details on older versions, see the `final version of its release notes <https://github.com/fritz-hh/OCRmyPDF/blob/7fd3dbdf42ca53a619412ce8add7532c5e81a9d1/RELEASE_NOTES.md>`_.
@@ -1,40 +0,0 @@
-PDF Renderers
-=============
-
-rasterizing
-  Converting a PDF to an image for display.
-
-rendering
-  Creating a new PDF from other data (such as an existing PDF).
-
-
-OCRmyPDF has three PDF renderers: ``hocr``, ``tesseract`` and ``tess4``. The renderer may be selected using ``--pdf-renderer``. The default is ``auto`` which lets OCRmyPDF select the renderer to use. Currently it always uses ``hocr``. 
-
-The hocr renderer
-----------------
-
-The ``hocr`` renderer is the default because it works in most cases. In this mode the whole PDF is rasterized, the raster image is run through OCR to generate a .hocr file, which is an HTML-like file that specifies the location of all identified words.
-
-The .hocr file is then rendered as a PDF and merged with the image layer.
-
-The image layer is copied from the original PDF page if possible, avoiding potentially lossy transcoding or loss of other PDF information. If preprocessing is specified, then the image layer is a new PDF.
-
-This is the only option for tesseract 3.02 and older.
-
-
-The tesseract renderer
----------------------
-
-The tesseract renderer uses tesseract's capability to produce a PDF directly. In version 3, tesseract automatically combined the image layer and text, meaning that this mode always transcodes and loses potentially loses quality and other PDF information.
-
-It does a much better job on non-Latin text.
-
-In a future release this will become the "tess3" renderer and ultimately will be dropped.
-
-
-The tess4 renderer
------------------
-
-The tess4 renderer uses tesseract 4.00 alpha's text-only PDF feature added in January 2017. This combines the advantages of the tesseract and hocr renderers, transcoding the image layer only if required by preprocessing options.
-
-Ghostscript PDF/A still sometimes inserts spaces between words when the tess4 renderer is used, affecting search quality.  ``--output-pdf pdf`` may be used to avoid this issue.
@@ -1,4 +1,4 @@
-PDF Security Issues
+PDF security issues
 ===================

 	OCRmyPDF should only be used on PDFs you trust. It is not designed to protect you against malware. 
@@ -46,6 +46,10 @@ def complain(message):
    print(*textwrap.wrap(message), file=sys.stderr)


+# Hack to help debugger context find /usr/local/bin
+if 'IDE_PROJECT_ROOTS' in os.environ:
+    os.environ['PATH'] = '/usr/local/bin:' + os.environ['PATH']
+
 if tesseract.version() < MINIMUM_TESS_VERSION:
    complain(
        "Please install tesseract {0} or newer "
@@ -53,7 +57,6 @@ if tesseract.version() < MINIMUM_TESS_VERSION:
            MINIMUM_TESS_VERSION, tesseract.version()))
    sys.exit(ExitCode.missing_dependency)

-
 # -------------
 # Parser

@@ -207,7 +210,7 @@ advanced = parser.add_argument_group(
    "Advanced options for power users")
 advanced.add_argument(
    '--tesseract-config', action='append', metavar='CFG', default=[],
-    help="additional Tesseract configuration files")
+    help="additional Tesseract configuration files -- see documentation")
 advanced.add_argument(
    '--tesseract-pagesegmode', action='store', type=int, metavar='PSM',
    choices=range(0, 14),
@@ -265,20 +268,25 @@ def check_options_languages(options, _log):
            "data for the following requested languages: \n")
        for lang in (set(options.language) - tesseract.languages()):
            msg += lang + '\n'
-        raise argparse.ArgumentError(msg)
+        raise argparse.ArgumentError(None, msg)


 def check_options_output(options, log):
    if options.pdf_renderer == 'auto':
        options.pdf_renderer = 'hocr'

-    if options.pdf_renderer == 'tesseract' and \
-            tesseract.version() < '3.04.01' and \
-            os.environ.get('OCRMYPDF_SHARP_TTF', '') != '1':
-        log.warning(
-            "Your version of tesseract has problems with PDF output."
-            " Some PDF viewers will fail to find searchable text.\n"
-            "--pdf-renderer=tesseract is not recommended.")
+    if options.pdf_renderer in ('tesseract', 'tess4'):
+        if tesseract.version() < '3.05':
+            log.warning(
+                "The setting --pdf-renderer=tesseract is not recommend for "
+                " use with tesseract versions less than 3.05, because it "
+                " produces OCR text that is incompatible with Ghostscript and "
+                " some other software.")
+        elif tesseract.version() == '4.00.00alpha':
+            log.warning(
+                "The setting --pdf-renderer={tesseract,tess4} is not"
+                " recommended for builds of tesseract 4.00.00alpha older than"
+                " February 2017. Make sure you are using a recent build.")

    if options.debug_rendering and options.pdf_renderer == 'tesseract':
        log.info(
@@ -286,7 +294,7 @@ def check_options_output(options, log):
            "--pdf-renderer=tesseract.")

    lossless_reconstruction = False
-    if options.pdf_renderer == 'hocr':
+    if options.pdf_renderer in ('hocr', 'tess4'):
        if not any((options.deskew, options.clean_final, options.force_ocr,
                   options.remove_background)):
            lossless_reconstruction = True
@@ -316,6 +324,7 @@ def check_options_preprocessing(options, log):
 def check_options_ocr_behavior(options, log):
    if options.force_ocr and options.skip_text:
        raise argparse.ArgumentError(
+            None,
            "Error: --force-ocr and --skip-text are mutually incompatible.")

    if set(options.language) & {'chi_sim', 'chi_tra'} and \
@@ -333,15 +342,37 @@ def check_options_advanced(options, log):
    if options.tesseract_oem and not tesseract.v4():
        log.warning(
            "--tesseract-oem requires Tesseract 4.x -- argument ignored")
+    if options.pdf_renderer == 'tess4' and not tesseract.has_textonly_pdf():
+        raise MissingDependencyError(
+            "--pdf-renderer tess4 requires Tesseract 4.x "
+            "commit 3d9fb3b or later")
+
+
+def check_options_metadata(options, log):
+    import unicodedata
+    metadata = [options.title, options.author, options.keywords,
+                options.subject]
+    for s in (m for m in metadata if m):
+        for c in s:
+            if unicodedata.category(c) == 'Co' or ord(c) >= 0x10000:
+                raise ValueError(
+                    "One of the metadata strings contains "
+                    "an unsupported Unicode character: '{}' (U+{})".format(
+                        c, hex(ord(c))[2:].upper()
+                ))


 def check_options(options, log):
    try:
        check_options_languages(options, log)
+        check_options_metadata(options, log)
        check_options_output(options, log)
        check_options_preprocessing(options, log)
        check_options_ocr_behavior(options, log)
        check_options_advanced(options, log)
+    except ValueError as e:
+        log.error(e)
+        sys.exit(ExitCode.bad_args)
    except argparse.ArgumentError as e:
        log.error(e)
        sys.exit(ExitCode.bad_args)
@@ -350,9 +381,6 @@ def check_options(options, log):
        sys.exit(ExitCode.missing_dependency)


-
-
-
 # ----------
 # Logging

@@ -393,7 +421,7 @@ def available_cpu_count():


 def cleanup_ruffus_error_message(msg):
-    msg = re.sub(r'\s+', r' ', msg, re.MULTILINE)
+    msg = re.sub(r'\s+', r' ', msg)
    msg = re.sub(r"\((.+?)\)", r'\1', msg)
    msg = msg.strip()
    return msg
@@ -475,10 +503,58 @@ def traverse_ruffus_exception(e_args, options, log):
            return traverse_ruffus_exception(exc, options, log)


+def check_closed_streams(options):
+    """Work around Python issue with multiprocessing forking on closed streams
+
+    https://bugs.python.org/issue28326
+
+    Attempting to a fork/exec a new Python process when any of std{in,out,err}
+    are closed or not flushable for some reason may raise an exception.
+    Fix this by opening devnull if the handle seems to be closed.  Do this
+    globally to avoid tracking places all places that fork.
+
+    Seems to be specific to multiprocessing.Process not all Python process
+    forkers.
+
+    The error actually occurs when the stream object is not flushable,
+    but replacing an open stream object that is not flushable with
+    /dev/null is a bad idea since it will create a silent failure.  Replacing
+    a closed handle with /dev/null seems safe.
+
+    """
+
+    if sys.stderr is None:
+        sys.stderr = open(os.devnull, 'w')
+
+    if sys.stdin is None:
+        if options.input_file == '-':
+            print("Trying to read from stdin but stdin seems closed",
+                  file=sys.stderr)
+            return False
+        sys.stdin = open(os.devnull, 'r')
+
+    if sys.stdout is None:
+        if options.output_file == '-':
+            # Can't replace stdout if the user is piping
+            # If this case can even happen, it must be some kind of weird
+            # stream.
+            print(textwrap.dedent("""\
+                Output was set to stdout '-' but the stream attached to
+                stdout does not support the flush() system call.  This
+                will fail."""), file=sys.stderr)
+            return False
+        sys.stdout = open(os.devnull, 'w')
+
+    return True
+
+
 def run_pipeline():
    options = parser.parse_args()
    options.verbose_abbreviated_path = 1

+    if not check_closed_streams(options):
+        return ExitCode.bad_args
+
    _log, _log_mutex = proxy_logger.make_shared_logger_and_proxy(
        logging_factory, __name__, [None, options.verbose])
    _log.debug('ocrmypdf ' + VERSION)
@@ -519,9 +595,9 @@ def run_pipeline():
                    file."""))
                return ExitCode.bad_args
        elif not is_file_writable(options.output_file):
-                _log.error(textwrap.dedent("""\
-                    Cutput file location is not writable."""))
-                return ExitCode.file_access_error
+            _log.error(textwrap.dedent("""\
+                Cutput file location is not writable."""))
+            return ExitCode.file_access_error

        manager = JobContextManager()
        manager.register('JobContext', JobContext)
@@ -14,6 +14,7 @@ class ExitCode(IntEnum):
    already_done_ocr = 6
    child_process_error = 7
    encrypted_pdf = 8
+    invalid_config = 9
    other_error = 15
    ctrl_c = 130

@@ -47,8 +48,12 @@ class InputFileError(ExitCodeException):


 class SubprocessOutputError(ExitCodeException):
-    exit_code = ExitCode.other_error
+    exit_code = ExitCode.child_process_error


 class EncryptedPdfError(ExitCodeException):
    exit_code = ExitCode.encrypted_pdf
+
+
+class TesseractConfigError(ExitCodeException):
+    exit_code = ExitCode.invalid_config
@@ -2,10 +2,35 @@
 # © 2015 James R. Barlow: github.com/jbarlow83

 from tempfile import NamedTemporaryFile
-from subprocess import Popen, PIPE, STDOUT, check_call
+from subprocess import Popen, PIPE, STDOUT, check_call, CalledProcessError, \
+    check_output
 from shutil import copy
+from functools import lru_cache
+import re
 from . import get_program
-from ..pdfa import SRGB_ICC_PROFILE
+from ..exceptions import SubprocessOutputError
+
+
+@lru_cache(maxsize=1)
+def version():
+    args_gs = [
+        get_program('gs'),
+        '--version'
+    ]
+    try:
+        version = check_output(
+                args_gs, close_fds=True, universal_newlines=True,
+                stderr=STDOUT)
+    except CalledProcessError as e:
+        print("Could not find Ghostscript executable on system PATH.",
+              file=sys.stderr)
+        raise MissingDependencyError from e
+
+    return version.strip()
+
+
+def _gs_error_reported(stream):
+    return re.search(r'error', stream, flags=re.IGNORECASE)


 def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
@@ -28,15 +53,16 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
        p = Popen(args_gs, close_fds=True, stdout=PIPE, stderr=STDOUT,
                  universal_newlines=True)
        stdout, _ = p.communicate()
-        if 'error' in stdout:
-            log.error(stdout)  # Ghostscript puts errors in stdout
+        if _gs_error_reported(stdout):
+            log.error(stdout)
        else:
            log.debug(stdout)

        if p.returncode == 0:
            copy(tmp.name, output_file)
        else:
-            log.error('Ghostscript rendering failed')
+            log.error('Ghostscript rasterizing failed')
+            raise SubprocessOutputError()


 def generate_pdfa(pdf_pages, output_file, log, threads=1):
@@ -61,7 +87,7 @@ def generate_pdfa(pdf_pages, output_file, log, threads=1):
                  universal_newlines=True)
        stdout, _ = p.communicate()

-        if 'error' in stdout:
+        if _gs_error_reported(stdout):
            log.error(stdout)
        elif 'overprint mode not set' in stdout:
            # Unless someone is going to print PDF/A documents on a
@@ -79,4 +105,5 @@ def generate_pdfa(pdf_pages, output_file, log, threads=1):
            # PDF/A - check PDF/A status elsewhere
            copy(gs_pdf.name, output_file)
        else:
-            log.error('Ghostscript PDF/A failed')
+            log.error('Ghostscript PDF/A rendering failed')
+            raise SubprocessOutputError()
@@ -6,11 +6,12 @@ import os
 import re
 import shutil
 from functools import lru_cache
-from ..exceptions import MissingDependencyError
+from ..exceptions import MissingDependencyError, TesseractConfigError
 from ..helpers import page_number
 from . import get_program
 from collections import namedtuple
 from textwrap import dedent
+import PyPDF2 as pypdf

 from subprocess import Popen, PIPE, CalledProcessError, \
    TimeoutExpired, check_output, STDOUT, DEVNULL
@@ -186,6 +187,10 @@ def tesseract_log_output(log, stdout, input_file):
            log.warning(prefix + "unsure about page orientation")
        elif 'error' in line.lower() or 'exception' in line.lower():
            log.error(prefix + line.strip())
+        elif 'warning' in line.lower():
+            log.warning(prefix + line.strip())
+        elif 'read_params_file' in line.lower():
+            log.error(prefix + line.strip())
        else:
            log.info(prefix + line.strip())

@@ -236,6 +241,8 @@ def generate_hocr(input_file, output_hocr, language: list, engine_mode,
        _generate_null_hocr(output_hocr, input_file)
    except CalledProcessError as e:
        tesseract_log_output(log, e.output, input_file)
+        if 'read_params_file: parameter not found' in e.output:
+            raise TesseractConfigError() from e
        if 'Image too large' in e.output:
            _generate_null_hocr(output_hocr, input_file)
            return
@@ -243,6 +250,7 @@ def generate_hocr(input_file, output_hocr, language: list, engine_mode,
        raise e from e
    else:
        tesseract_log_output(log, stdout, input_file)
+
        if os.path.exists(badxml + '.html'):
            # Tesseract 3.02 appends suffix ".html" on its own (.badxml.html)
            shutil.move(badxml + '.html', badxml)
@@ -265,6 +273,24 @@ def generate_hocr(input_file, output_hocr, language: list, engine_mode,
                f_out.write(line)


+def use_skip_page(text_only, skip_pdf, output_pdf):
+    if not text_only:
+        os.symlink(skip_pdf, output_pdf)
+        return
+
+    # For text only we must create a blank page with dimensions identical
+    # to the skip page because this is equivalent to a page with no text
+
+    pdf_in = pypdf.PdfFileReader(skip_pdf)
+    page0 = pdf_in.pages[0]
+
+    with open(output_pdf, 'wb') as out:
+        pdf_out = pypdf.PdfFileWriter()
+        w, h = page0.mediaBox.getWidth(), page0.mediaBox.getHeight()
+        pdf_out.addBlankPage(w, h)
+        pdf_out.write(out)
+
+
 def generate_pdf(input_image, skip_pdf, output_pdf, language: list,
                 engine_mode, text_only: bool,
                 tessconfig: list, timeout: float, pagesegmode: int, log):
@@ -302,11 +328,14 @@ def generate_pdf(input_image, skip_pdf, output_pdf, language: list,
            universal_newlines=True, timeout=timeout)
    except TimeoutExpired:
        page_timedout(log, input_image)
-        shutil.copy(skip_pdf, output_pdf)
+        use_skip_page(text_only, skip_pdf, output_pdf)
    except CalledProcessError as e:
        tesseract_log_output(log, e.output, input_image)
+        if 'read_params_file: parameter not found' in e.output:
+            raise TesseractConfigError() from e
+
        if 'Image too large' in e.output:
-            shutil.copy(skip_pdf, output_pdf)
+            use_skip_page(text_only, skip_pdf, output_pdf)
            return
        raise e from e
    else:
@@ -78,7 +78,8 @@ def run(input_file, output_file, dpi, log, mode_args):
            raise e from e
        else:
            log.debug(stdout)
-            Image.open(output_pnm.name).save(output_file)
+            # unpaper sets dpi to 72
+            Image.open(output_pnm.name).save(output_file, dpi=(dpi, dpi))


 def deskew(input_file, output_file, dpi, log):
@@ -1,8 +1,8 @@
 from cffi import FFI

-ffi = FFI()
-ffi.set_source("ocrmypdf.lib._leptonica", None)
-ffi.cdef("""
+ffibuilder = FFI()
+ffibuilder.set_source("ocrmypdf.lib._leptonica", None)
+ffibuilder.cdef("""
 typedef signed char             l_int8;
 typedef unsigned char           l_uint8;
 typedef short                   l_int16;
@@ -56,7 +56,7 @@ typedef struct Box    BOX;

 """)

-ffi.cdef("""
+ffibuilder.cdef("""
 PIX * pixRead ( const char *filename );
 PIX * pixScale ( PIX *pixs, l_float32 scalex, l_float32 scaley );
 l_int32 pixFindSkew ( PIX *pixs, l_float32 *pangle, l_float32 *pconf );
@@ -189,4 +189,4 @@ void lept_free(void *ptr);


 if __name__ == '__main__':
-    ffi.compile()
+    ffibuilder.compile(verbose=True)
@@ -89,13 +89,13 @@ def _is_unit_square(shorthand):
    pairwise = zip(values, UNIT_SQUARE)
    return all([isclose(a, b, rel_tol=1e-3) for a, b in pairwise])

-RasterSettings = namedtuple('RasterSettings',
+XobjectSettings = namedtuple('XobjectSettings',
    ['name', 'shorthand', 'stack_depth'])

 InlineSettings = namedtuple('InlineSettings',
    ['settings', 'shorthand', 'stack_depth'])

-ContentsInfo = namedtuple('ContentsInfo', ['raster_settings', 'inline_images'])
+ContentsInfo = namedtuple('ContentsInfo', ['xobject_settings', 'inline_images'])


 def _normalize_stack(operations):
@@ -114,7 +114,7 @@ def _normalize_stack(operations):
            yield (operands, command)


-def _interpret_contents(contentstream):
+def _interpret_contents(contentstream, initial_shorthand=UNIT_SQUARE):
    """Interpret the PDF content stream

    The stack represents the state of the PDF graphics stack.  We are only
@@ -139,8 +139,8 @@ def _interpret_contents(contentstream):

    operations = contentstream.operations
    stack = []
-    ctm = _matrix_from_shorthand(UNIT_SQUARE)
-    image_raster_settings = []
+    ctm = _matrix_from_shorthand(initial_shorthand)
+    xobject_settings = []
    inline_images = []

    for n, op in enumerate(_normalize_stack(operations)):
@@ -161,10 +161,10 @@ def _interpret_contents(contentstream):
                _matrix_from_shorthand(operands), ctm)
        elif command == b'Do':
            image_name = operands[0]
-            raster = RasterSettings(
+            settings = XobjectSettings(
                name=image_name, shorthand=_shorthand_from_matrix(ctm),
                stack_depth=len(stack))
-            image_raster_settings.append(raster)
+            xobject_settings.append(settings)
        elif command == b'INLINE IMAGE':
            settings = operands['settings']
            inline = InlineSettings(
@@ -173,7 +173,7 @@ def _interpret_contents(contentstream):
            inline_images.append(inline)

    return ContentsInfo(
-        raster_settings=image_raster_settings,
+        xobject_settings=xobject_settings,
        inline_images=inline_images)


@@ -241,8 +241,8 @@ def _get_dpi(ctm_shorthand, image_size):
    return (dpi_w, dpi_h)


-def _find_page_inline_images(page, pageinfo, contentsinfo):
-    "Find inline images on the page"
+def _find_inline_images(contentsinfo):
+    "Find inline images in the contentstream"

    for n, inline in enumerate(contentsinfo.inline_images):
        image = {}
@@ -272,20 +272,44 @@ def _find_page_inline_images(page, pageinfo, contentsinfo):
        yield image


-def _find_page_regular_images(page, pageinfo, contentsinfo):
-    "Find images stored in XObject resources"
+def _image_xobjects(container):
+    """Search for all XObject-based images in the container

-    try:
-        page['/Resources']['/XObject']
-    except KeyError:
+    Usually the container is a page, but it could also be a Form XObject
+    that contains images. Filter out the Form XObjects which are dealt with
+    elsewhere.
+
+    Generate a sequence of tuples (image, xobj container), where container,
+    where xobj is the name of the object and image is the object itself,
+    since the object does not know its own name.
+
+    """
+
+    if '/Resources' not in container:
        return
-    for xobj in page['/Resources']['/XObject']:
-        # PyPDF2 returns the keys as an iterator
-        pdfimage = page['/Resources']['/XObject'][xobj]
-        if pdfimage['/Subtype'] != '/Image':
-            continue
+    resources = container['/Resources']
+    if '/XObject' not in resources:
+        return
+    for xobj in resources['/XObject']:
+        candidate = resources['/XObject'][xobj]
+        if candidate['/Subtype'] == '/Image':
+            image = candidate
+            yield (image, xobj)
+
+
+def _find_regular_images(container, contentsinfo):
+    """Find images stored in the container's /Resources /XObject
+
+    Usually the container is a page, but it could also be a Form XObject
+    that contains images.
+
+    Generates images with their DPI at time of drawing.
+
+    """
+
+    for pdfimage, xobj in _image_xobjects(container):
        image = {}
-        image['name'] = str(xobj)
+        image['name'] = xobj
        image['width'] = pdfimage['/Width']
        image['height'] = pdfimage['/Height']
        if '/BitsPerComponent' in pdfimage:
@@ -329,12 +353,12 @@ def _find_page_regular_images(page, pageinfo, contentsinfo):

        image['dpi_w'] = image['dpi_h'] = 0

-        for raster in contentsinfo.raster_settings:
+        for xobj in contentsinfo.xobject_settings:
            # Loop in case the same image is display multiple times on a page
-            if raster.name != image['name']:
+            if xobj.name != image['name']:
                continue

-            if raster.stack_depth == 0 and _is_unit_square(raster.shorthand):
+            if xobj.stack_depth == 0 and _is_unit_square(xobj.shorthand):
                # At least one PDF in the wild (and test suite) draws an image
                # when the graphics stack depth is 0, meaning that the image
                # gets drawn into a square of 1x1 PDF units (or 1/72",
@@ -343,7 +367,7 @@ def _find_page_regular_images(page, pageinfo, contentsinfo):
                continue

            dpi_w, dpi_h = _get_dpi(
-                raster.shorthand, (image['width'], image['height']))
+                xobj.shorthand, (image['width'], image['height']))

            # When image is used multiple times take the highest DPI it is
            # rendered at
@@ -358,12 +382,91 @@ def _find_page_regular_images(page, pageinfo, contentsinfo):
        yield image


-def _find_page_images(page, pageinfo, contentsinfo):
-    yield from _find_page_inline_images(page, pageinfo, contentsinfo)
-    yield from _find_page_regular_images(page, pageinfo, contentsinfo)
+def _find_form_xobject_images(pdf, container, contentsinfo):
+    """Find any images that are in Form XObjects in the container
+
+    The container may be a page, or a parent Form XObject.
+
+    """
+    if '/Resources' not in container:
+        return
+    resources = container['/Resources']
+    if '/XObject' not in resources:
+        return
+    for xobj in resources['/XObject']:
+        candidate = resources['/XObject'][xobj]
+        if candidate['/Subtype'] != '/Form':
+            continue
+
+        form_xobject = candidate
+        for settings in contentsinfo.xobject_settings:
+            if settings.name != xobj:
+                continue
+
+            # Find images once for each time this Form XObject is drawn.
+            # This could be optimized to cache the multiple drawing events
+            # but in practice both Form XObjects and multiple drawing of the
+            # same object are both very rare.
+            ctm_shorthand = settings.shorthand
+            yield from _find_images(pdf, form_xobject, ctm_shorthand)
+
+
+def _find_images(pdf, container, shorthand=None):
+    """Find all individual instances of images drawn in the container
+
+    Usually the container is a page, but it may also be a Form XObject.
+
+    On a typical page images are stored inline or as regular images
+    in an XObject.
+
+    Form XObjects may include inline images, XObject images,
+    and recursively, other Form XObjects; and also vector drawing commands.
+
+    Every instance of an image being drawn somewhere is flattened and
+    treated as a unique image, since if the same image is drawn multiple times
+    on one page it may be drawn at differing resolutions, and our objective
+    is to find the resolution at which the page can be rastered without
+    downsampling.
+
+    """
+
+    if container.get('/Type') == '/Page' and '/Contents' in container:
+        # For a /Page the content stream is attached to the page's /Contents
+        page = container
+        contentstream = pypdf.pdf.ContentStream(page.getContents(), pdf)
+        initial_shorthand = shorthand or UNIT_SQUARE
+    elif container.get('/Type') == '/XObject' and \
+            container['/Subtype'] == '/Form':
+        # For a Form XObject that content stream is attached to the XObject
+        contentstream = pypdf.pdf.ContentStream(container, pdf)
+
+        # Set the CTM to the state it was when the "Do" operator was
+        # encountered that is drawing this instance of the Form XObject
+        ctm = _matrix_from_shorthand(shorthand or UNIT_SQUARE)
+
+        # A Form XObject may provide its own matrix to map form space into
+        # user space. Get this if one exists
+        form_matrix = _matrix_from_shorthand(
+                container.get('/Matrix', UNIT_SQUARE))
+
+        # Concatenate form matrix with CTM to ensure CTM is correct for
+        # drawing this instance of the XObject
+        ctm = matrix_mult(form_matrix, ctm)
+        initial_shorthand = _shorthand_from_matrix(ctm)
+    else:
+        return
+
+    contentsinfo = _interpret_contents(contentstream, initial_shorthand)
+
+    yield from _find_inline_images(contentsinfo)
+    yield from _find_regular_images(container, contentsinfo)
+    yield from _find_form_xobject_images(pdf, container, contentsinfo)


 def _page_has_text(pdf, page):
+    if not '/Contents' in page:
+        return False
+
    # Simple test
    text = page.extractText()
    if text.strip() != '':
@@ -405,15 +508,8 @@ def _pdf_get_pageinfo(infile, pageno: int):
    except KeyError:
        pageinfo['rotate'] = 0

-    try:
-        contentstream = pypdf.pdf.ContentStream(page.getContents(), pdf)
-    except AttributeError as e:
-        return pageinfo
-
-    contentsinfo = _interpret_contents(contentstream)
-    pageinfo['images'] = [im for im in _find_page_images(
-                                page, pageinfo, contentsinfo)]
-
+    pageinfo['images'] = [im for im in
+                          _find_images(pdf, page)]
    if pageinfo['images']:
        xres = max(image['dpi_w'] for image in pageinfo['images'])
        yres = max(image['dpi_h'] for image in pageinfo['images'])
@@ -26,11 +26,7 @@ pdfa_def_template = u"""%!
 /ICCProfile ($icc_profile)
 def

-[ /Title <$title>
-  /Author <$author>
-  /Subject <$subject>
-  /Keywords <$keywords>
-  /Creator <$creator>
+[$docinfo
  /DOCINFO pdfmark

 % Define an ICC profile :
@@ -89,20 +85,30 @@ def encode_text_string(s: str) -> str:


 def _get_pdfa_def(icc_profile, icc_identifier, pdfmark):
-    pdfmark_utf16 = {k: encode_text_string(v) for k, v in pdfmark.items()}
+    # Ghostscript <= 9.21 has a bug where null entries in DOCINFO might produce
+    # ERROR: VMerror (-25) on closing pdfwrite device.
+    # https://bugs.ghostscript.com/show_bug.cgi?id=697684
+    # Work around this by only adding keys that have a nontrivial value
+    docinfo_keys = ('/Title', '/Author', '/Subject', '/Creator', '/Keywords')
+    docinfo_line_template = '  {key} <{value}>'
+
+    def docinfo_gen():
+        for key in docinfo_keys:
+            if key in pdfmark and pdfmark[key].strip() != '':
+                line = docinfo_line_template.format(
+                    key=key, value=encode_text_string(pdfmark[key]))
+                yield line
+
+    docinfo = '\n'.join(docinfo_gen())

    t = Template(pdfa_def_template)
    result = t.substitute(icc_profile=icc_profile,
                          icc_identifier=icc_identifier,
-                          title=pdfmark_utf16.get('/Title', ''),
-                          author=pdfmark_utf16.get('/Author', ''),
-                          subject=pdfmark_utf16.get('/Subject', ''),
-                          creator=pdfmark_utf16.get('/Creator', ''),
-                          keywords=pdfmark_utf16.get('/Keywords', ''))
+                          docinfo=docinfo)
    return result


-def generate_pdfa_def(target_filename, pdfmark, icc='sRGB'):
+def generate_pdfa_ps(target_filename, pdfmark, icc='sRGB'):
    if icc == 'sRGB':
        icc_profile = SRGB_ICC_PROFILE
    else:
@@ -116,6 +122,10 @@ def generate_pdfa_def(target_filename, pdfmark, icc='sRGB'):
        f.write(ps)


+# The old name is generate_pdfa_def -- now deprecated
+generate_pdfa_def = generate_pdfa_ps
+
+
 def file_claims_pdfa(filename):
    """Determines if the file claims to be PDF/A compliant

@@ -23,7 +23,7 @@ from ruffus import formatter, regex, Pipeline, suffix

 from .hocrtransform import HocrTransform
 from .pageinfo import pdf_get_all_pageinfo
-from .pdfa import generate_pdfa_def, file_claims_pdfa
+from .pdfa import generate_pdfa_ps, file_claims_pdfa
 from .helpers import re_symlink, is_iterable_notstr, page_number
 from .exec import ghostscript, tesseract, qpdf
 from .exceptions import *
@@ -247,7 +247,7 @@ def is_ocr_required(pageinfo, log, options):
                                "skipping all processing on this page"))
            ocr_required = False

-    if ocr_required and options.skip_big:
+    if ocr_required and options.skip_big and pageinfo['images']:
        pixel_count = pageinfo['width_pixels'] * pageinfo['height_pixels']
        if pixel_count > (options.skip_big * 1000000):
            ocr_required = False
@@ -393,15 +393,16 @@ def rasterize_with_ghostscript(
    pageinfo = get_pageinfo(input_file, context)

    device = 'png16m'  # 24-bit
-    if all(image['comp'] == 1 for image in pageinfo['images']):
-        if all(image['bpc'] == 1 for image in pageinfo['images']):
-            device = 'pngmono'
-        elif all(image['bpc'] > 1 and image['color'] == 'index'
-                 for image in pageinfo['images']):
-            device = 'png256'
-        elif all(image['bpc'] > 1 and image['color'] == 'gray'
-                 for image in pageinfo['images']):
-            device = 'pnggray'
+    if pageinfo['images']:
+        if all(image['comp'] == 1 for image in pageinfo['images']):
+            if all(image['bpc'] == 1 for image in pageinfo['images']):
+                device = 'pngmono'
+            elif all(image['bpc'] > 1 and image['color'] == 'index'
+                     for image in pageinfo['images']):
+                device = 'png256'
+            elif all(image['bpc'] > 1 and image['color'] == 'gray'
+                     for image in pageinfo['images']):
+                device = 'pnggray'

    log.debug("Rasterize {0} with {1}".format(
              os.path.basename(input_file), device))
@@ -517,7 +518,8 @@ def select_visible_page_image(
    image = next(ii for ii in infiles if ii.endswith(image_suffix))

    pageinfo = get_pageinfo(image, context)
-    if all(orig_image['enc'] == 'jpeg' for orig_image in pageinfo['images']):
+    if pageinfo['images'] and \
+            all(im['enc'] == 'jpeg' for im in pageinfo['images']):
        # If all images were JPEGs originally, produce a JPEG as output
        im = Image.open(image)

@@ -555,16 +557,21 @@ def select_image_layer(
        re_symlink(page_pdf, output_file, log)
    else:
        pageinfo = get_pageinfo(image, context)
-        dpi = get_page_dpi(pageinfo, options)
-        dpi = float(dpi[0]), float(dpi[1])
-        layout_fun = img2pdf.get_fixed_dpi_layout_fun(dpi)
+
+        # We rasterize a square DPI version of each page because most image
+        # processing tools don't support rectangular DPI. Use the square DPI
+        # as it accurately describes the image. It would be possible to
+        # resample the image at this stage back to non-square DPI to more
+        # closely resemble the input, except that the hocr renderer does not
+        # understand non-square DPI. The tess4 renderer would be fine.
+        dpi = get_page_square_dpi(pageinfo, options)
+        layout_fun = img2pdf.get_fixed_dpi_layout_fun((dpi, dpi))

        with open(image, 'rb') as imfile, \
                open(output_file, 'wb') as pdf:
-            rawdata = imfile.read()
            log.debug('{:4d}: convert'.format(page_number(page_pdf)))
            img2pdf.convert(
-                rawdata, with_pdfrw=False,
+                imfile, with_pdfrw=False,
                layout_fun=layout_fun, outputstream=pdf)
            log.debug('{:4d}: convert done'.format(page_number(page_pdf)))

@@ -762,13 +769,14 @@ def generate_postscript_stub(
    options = context.get_options()
    pdf = pypdf.PdfFileReader(input_file)
    pdfmark = get_pdfmark(pdf, options)
-    generate_pdfa_def(output_file, pdfmark)
+    generate_pdfa_ps(output_file, pdfmark)


 def skip_page(
        input_file,
        output_file,
-        log):
+        log,
+        context):
    # The purpose of this step is its filter to forward only the skipped
    # files (.skip.oriented.pdf) while disregarding the processed ones
    # (.ocr.oriented.pdf).  Alternative would be for merge_pages to filter
@@ -948,7 +956,7 @@ def build_pipeline(options, work_folder, log, context):
    task_ocr_tesseract_hocr.graphviz(fillcolor='"#00cc66"')
    task_ocr_tesseract_hocr.active_if(options.pdf_renderer == 'hocr')
    if tesseract.v4():
-        task_ocr_tesseract_hocr.jobs_limit(1)  # Uses multi-core on its own
+        task_ocr_tesseract_hocr.jobs_limit(2)  # Uses multi-core on its own

    task_select_visible_page_image = main_pipeline.collate(
        task_func=select_visible_page_image,
@@ -1001,7 +1009,7 @@ def build_pipeline(options, work_folder, log, context):
    task_ocr_tesseract_textonly_pdf.graphviz(fillcolor='"#ff69b4"')
    task_ocr_tesseract_textonly_pdf.active_if(options.pdf_renderer == 'tess4')
    if tesseract.v4():
-        task_ocr_tesseract_textonly_pdf.jobs_limit(1)
+        task_ocr_tesseract_textonly_pdf.jobs_limit(2)

    task_combine_layers = main_pipeline.collate(
        task_func=combine_layers,
@@ -1024,14 +1032,14 @@ def build_pipeline(options, work_folder, log, context):
    task_ocr_tesseract_and_render_pdf.graphviz(fillcolor='"#66ccff"')
    task_ocr_tesseract_and_render_pdf.active_if(options.pdf_renderer == 'tesseract')
    if tesseract.v4():
-        task_ocr_tesseract_and_render_pdf.jobs_limit(1)  # Uses multi-core
+        task_ocr_tesseract_and_render_pdf.jobs_limit(2)  # Uses multi-core

    # PDF/A
    task_generate_postscript_stub = main_pipeline.transform(
        task_func=generate_postscript_stub,
        input=task_repair_pdf,
        filter=formatter(r'\.repaired\.pdf'),
-        output=os.path.join(work_folder, 'pdfa_def.ps'),
+        output=os.path.join(work_folder, 'pdfa.ps'),
        extras=[log, context])
    task_generate_postscript_stub.active_if(options.output_type == 'pdfa')

@@ -1043,7 +1051,7 @@ def build_pipeline(options, work_folder, log, context):
        filter=suffix('.skip.oriented.pdf'),
        output='.done.pdf',
        output_dir=work_folder,
-        extras=[log])
+        extras=[log, context])

    # Merge pages
    task_merge_pages_ghostscript = main_pipeline.merge(
@@ -2,8 +2,8 @@
 # setup.py lists a separate set of requirements that are looser to simplify
 # installation
 ruffus == 2.6.3
-Pillow == 3.4.2
+Pillow == 4.0.0
 reportlab == 3.3.0
 PyPDF2 == 1.26
-img2pdf == 0.2.1
+img2pdf == 0.2.3
 cffi == 1.9.1
@@ -9,7 +9,7 @@ if sys.version_info < (3, 4):
    print("Python 3.4 or newer is required", file=sys.stderr)
    sys.exit(1)

-from setuptools import setup  # nopep8
+from setuptools import setup, find_packages  # nopep8
 from subprocess import STDOUT, check_output, CalledProcessError  # nopep8
 from collections.abc import Mapping  # nopep8
 import re  # nopep8
@@ -192,7 +192,7 @@ setup(
    author='James R. Barlow',
    author_email='jim@purplerock.ca',
    license='MIT',
-    packages=['ocrmypdf'],
+    packages=find_packages(exclude=["tests", "tests.*"]),
    keywords=['PDF', 'OCR', 'optical character recognition', 'PDF/A', 'scanning'],
    classifiers=[
        "Programming Language :: Python :: 3",
@@ -214,22 +214,20 @@ setup(
        "Topic :: Text Processing :: Linguistic",
        ],
    setup_requires=[
-        'setuptools_scm',
-        'cffi>=1.5.0',
-        'pytest-runner',
-        'pytest-helpers-namespace'
+        'setuptools_scm',       # so that version will work
+        'cffi>=1.9.1'           # to build the leptonica module
    ],
    use_scm_version={'version_scheme': 'post-release'},
    cffi_modules=[
-        'ocrmypdf/lib/compile_leptonica.py:ffi'
+        'ocrmypdf/lib/compile_leptonica.py:ffibuilder'
    ],
    install_requires=[
        'ruffus==2.6.3',        # pinned - ocrmypdf implements a 2.6.3 workaround
-        'Pillow>=3.1.1',        # Pillow is pretty stable
+        'Pillow>=4.0.0',        # Pillow < 4 has BytesIO/TIFF bug w/img2pdf 0.2.3
        'reportlab>=3.2.0',     # oldest released version with sane image handling
        'PyPDF2>=1.26',         # pure Python, so track HEAD closely
-        'img2pdf>=0.2.1',       # pure Python, so track HEAD closely
-        'cffi>=1.5.0'           # oldest version ever tested
+        'img2pdf>=0.2.3',       # pure Python, so track HEAD closely
+        'cffi>=1.9.1'           # must be a setup and install requirement
    ],
    tests_require=tests_require,
    entry_points={
@@ -1,4 +0,0 @@
-deb http://httpredir.debian.org/debian stretch main
-deb http://httpredir.debian.org/debian stretch-updates main
-deb http://security.debian.org stretch/updates main
-deb http://ftp.de.debian.org/debian sid main contrib non-free
@@ -1,2 +1,2 @@
-pytest >= 2.8
+pytest >= 3.0
 pytest-helpers-namespace
@@ -22,10 +22,21 @@ def is_linux():
    return platform.system() == 'Linux'


+@pytest.helpers.register
+def is_macos():
+    return platform.system() == 'Darwin'
+
+
@pytest.helpers.register
 def running_in_docker():
-    # Docker creates a file named /.dockerinit
-    return os.path.exists('/.dockerinit')
+    # Docker creates a file named /.dockerenv (newer versions) or
+    # /.dockerinit (older) -- this is undocumented, not an offical test
+    return os.path.exists('/.dockerenv') or os.path.exists('/.dockerinit')
+
+
+@pytest.helpers.register
+def running_in_travis():
+    return os.environ.get('TRAVIS') == 'true'


 TESTS_ROOT = os.path.abspath(os.path.dirname(__file__))
@@ -90,7 +101,7 @@ def check_ocrmypdf(input_file, output_file, *args, env=None):

    p, out, err = run_ocrmypdf(input_file, output_file, *args, env=env)
    #print(err)  # ensure py.test collects the output, use -s to view
-    assert p.returncode == 0
+    assert p.returncode == 0, "<stderr>\n" + err + "\n</stderr>"
    assert os.path.exists(str(output_file)), "Output file not created"
    assert os.stat(str(output_file)).st_size > 100, "PDF too small or empty"
    assert out == "", \
@@ -114,3 +125,11 @@ def run_ocrmypdf(input_file, output_file, *args, env=None):
    #print(err)

    return p, out, err
+
+
+@pytest.helpers.register
+def first_page_dimensions(pdf):
+    from ocrmypdf import pageinfo
+    info = pageinfo.pdf_get_all_pageinfo(str(pdf))
+    page0 = info[0]
+    return (page0['width_inches'], page0['height_inches'])
@@ -64,6 +64,9 @@ under the terms of the license in LICENSE.rst.
    *   - epson.pdf
        - @lowesjam
        - a linearized PDF containing some unusual indirect objects, created by an Epson printer; printout of a Wikipedia article (CC BY-SA)
+    *   - formxobject.pdf
+        - @jbarlow83
+        - hand-crafted exotic PDF containing an image inside a Form XObject
    *   - francais.pdf
        - @jbarlow83
        - a page containing French accents (diacritics)  
@@ -82,6 +85,9 @@ under the terms of the license in LICENSE.rst.
    *   - overlay.pdf
        - @maxandersen
        - PDF file generated by PDFPen pro that triggered content stream parse errors
+    *   - no_conentes.pdf
+        - @jbarlow83
+        - synthetic PDF with a blank page that has no /Contents entry

 Assemblies
 ==========
@@ -0,0 +1,21 @@
+%PDF-1.3
+%¿÷¢þ
+1 0 obj
+<< /Pages 2 0 R /Type /Catalog >>
+endobj
+2 0 obj
+<< /Count 1 /Kids [ 3 0 R ] /Type /Pages >>
+endobj
+3 0 obj
+<< /MediaBox [ 0 0 720 720 ] /Parent 2 0 R /Resources << >> /Type /Page >>
+endobj
+xref
+0 4
+0000000000 65535 f 
+0000000015 00000 n 
+0000000064 00000 n 
+0000000123 00000 n 
+trailer << /Root 1 0 R /Size 4 /ID [<52bba3c78160d0c6e851b59110e5d076><52bba3c78160d0c6e851b59110e5d076>] >>
+startxref
+213
+%%EOF
@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+import sys
+import os
+
+"""Replicate Ghostscript raster failure while allowing rendering"""
+
+
+def real_ghostscript(argv):
+    gs_args = ['gs'] + argv[1:]
+    os.execvp("gs", gs_args)
+    return  # Not reachable
+
+
+def main():
+    if '--version' in sys.argv:
+        print('9.20')
+        print('SPOOFED: ' + os.path.basename(__filename__))
+        sys.exit(0)
+
+    # For any rendering calls (device == pdfwrite) call real ghostscript
+    if '-sDEVICE=pdfwrite' in sys.argv:
+        real_ghostscript(sys.argv)
+        return
+
+    # Fail
+    print("ERROR: Ghost story archive not found")
+    sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()
@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+import sys
+import os
+
+"""Replicate Ghostscript render failure while allowing rasterizing"""
+
+
+def real_ghostscript(argv):
+    gs_args = ['gs'] + argv[1:]
+    os.execvp("gs", gs_args)
+    return  # Not reachable
+
+
+def main():
+    if '--version' in sys.argv:
+        print('9.20')
+        print('SPOOFED: ' + os.path.basename(__filename__))
+        sys.exit(0)
+
+    # For any rasterize calls (device != pdfwrite) call real ghostscript
+    if '-sDEVICE=pdfwrite' not in sys.argv:
+        real_ghostscript(sys.argv)
+        return
+
+    # Fail
+    print("ERROR: Casper is not a friendly ghost")
+    sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()
@@ -37,6 +37,7 @@ def real_tesseract():
 def main():
    operation = sys.argv[-1]
    # For anything unexpected operation, defer to real tesseract binary
+    # Currently this includes all use of "--tesseract-config"
    if operation != 'hocr' and operation != 'pdf' and operation != 'stdout':
        real_tesseract()
        return  # Not reachable
@@ -52,6 +53,11 @@ def main():
        ['tesseract', '--version'],
        stderr=subprocess.STDOUT)

+    if b'4.00.00alpha' in tess_version:
+        # Tesseract 4.x alpha is a moving target, don't cache it
+        real_tesseract()
+        return
+
    m.update(tess_version)

    # Insert this source file into the hash function, to ensure that any
@@ -65,12 +71,21 @@ def main():
        lang = sys.argv[sys.argv.index('-l') + 1]
        m.update(lang.encode())
    except ValueError:
-        pass
-    try:
-        psm = sys.argv[sys.argv.index('-psm') + 1]
-        m.update(psm.encode())
-    except ValueError:
-        pass
+        m.update(b'default-lang')
+
+    psm_arg = ''
+    if '--psm' in sys.argv:
+        psm_arg = '--psm'
+    elif '-psm' in sys.argv:
+        psm_arg = '-psm'
+    if psm_arg:
+        try:
+            psm = sys.argv[sys.argv.index(psm_arg) + 1]
+            m.update(psm.encode())
+        except ValueError:
+            m.update(b'default-psm')
+    else:
+        m.update(b'default-psm')

    if operation == 'stdout' and psm != '0':
        real_tesseract()
@@ -12,6 +12,7 @@ import PyPDF2 as pypdf
 from ocrmypdf.exceptions import ExitCode
 from ocrmypdf import leptonica
 from ocrmypdf.pdfa import file_claims_pdfa
+from ocrmypdf.exec import ghostscript


 check_ocrmypdf = pytest.helpers.check_ocrmypdf
@@ -50,6 +51,16 @@ def spoof_no_tess_pdfa_warning():
    return spoof(tesseract='tesseract_noop.py', gs='gs_feature_elision.py')


+@pytest.fixture
+def spoof_no_tess_gs_render_fail():
+    return spoof(tesseract='tesseract_noop.py', gs='gs_render_failure.py')
+
+
+@pytest.fixture
+def spoof_no_tess_gs_raster_fail():
+    return spoof(tesseract='tesseract_noop.py', gs='gs_raster_failure.py')
+
+
@pytest.fixture
 def spoof_qpdf_always_error():
    return spoof(qpdf='qpdf_dummy_return2.py')
@@ -166,48 +177,50 @@ def test_preserve_metadata(spoof_tesseract_noop, output_type,
    assert pdfa_info['output'] == output_type


-@pytest.mark.skipif(
-    pytest.helpers.is_linux() and not pytest.helpers.running_in_docker(),
-    reason="likely to fail if Linux locale is not configured correctly")
@pytest.mark.parametrize("output_type", [
    'pdfa', 'pdf'
    ])
 def test_override_metadata(spoof_tesseract_noop, output_type, resources,
                           outpdf):
    input_file = resources / 'c02-22.pdf'
-
    german = 'Du siehst den Wald vor lauter Bäumen nicht.'
    chinese = '孔子'
-    high_unicode = 'U+1030C is: 𐌌'

    p, out, err = run_ocrmypdf(
        input_file, outpdf,
        '--title', german,
        '--author', chinese,
-        '--subject', high_unicode,
        '--output-type', output_type,
        env=spoof_tesseract_noop)

-    assert p.returncode == ExitCode.ok
+    assert p.returncode == ExitCode.ok, err

-    pdf = str(outpdf)
+    reader = pypdf.PdfFileReader(outpdf)

-    out_pdfinfo = check_output(['pdfinfo', pdf], universal_newlines=True)
-    lines_pdfinfo = out_pdfinfo.splitlines()
-    pdfinfo = {}
-    for line in lines_pdfinfo:
-        k, v = line.strip().split(':', maxsplit=1)
-        pdfinfo[k.strip()] = v.strip()
+    assert reader.documentInfo['/Title'] == german
+    assert reader.documentInfo['/Author'] == chinese
+    assert reader.documentInfo.get('/Keywords', '') == ''

-    assert pdfinfo['Title'] == german
-    assert pdfinfo['Author'] == chinese
-    assert pdfinfo['Subject'] == high_unicode
-    assert pdfinfo.get('Keywords', '') == ''
-
-    pdfa_info = file_claims_pdfa(pdf)
+    pdfa_info = file_claims_pdfa(outpdf)
    assert pdfa_info['output'] == output_type


+def test_high_unicode(spoof_tesseract_noop, resources, no_outpdf):
+
+    # Ghostscript doesn't support high Unicode, so neither do we, to be
+    # safe
+    input_file = resources / 'c02-22.pdf'
+    high_unicode = 'U+1030C is: 𐌌'
+
+    p, out, err = run_ocrmypdf(
+        input_file, no_outpdf,
+        '--subject', high_unicode,
+        '--output-type', 'pdfa',
+        env=spoof_tesseract_noop)
+
+    assert p.returncode == ExitCode.bad_args, err
+
+
@pytest.mark.parametrize('renderer', [
    'hocr',
    'tesseract',
@@ -409,10 +422,14 @@ def test_force_ocr_on_pdf_with_no_images(spoof_tesseract_crash, resources,
    assert not os.path.exists(no_outpdf)


+@pytest.mark.skipif(
+    pytest.helpers.is_macos() and pytest.helpers.running_in_travis(),
+    reason="takes too long to install language packs in Travis macOS homebrew")
 def test_french(spoof_tesseract_cache, resources, outpdf):
    p, out, err = run_ocrmypdf(
        resources / 'francais.pdf', outpdf, '-l', 'fra',
        env=spoof_tesseract_cache)
+    print(os.environ)
    assert p.returncode == ExitCode.ok, \
        "This test may fail if Tesseract language packs are missing"

@@ -564,6 +581,46 @@ def test_non_square_resolution(renderer, spoof_tesseract_cache,
    assert in_pageinfo[0]['yres'] == out_pageinfo[0]['yres']


+@pytest.mark.skipif(sys.version_info < (3, 5), reason="needs math.isclose")
+@pytest.mark.parametrize('renderer', [
+    'hocr',
+    'tesseract'
+    ])
+def test_convert_to_square_resolution(renderer, spoof_tesseract_cache,
+                                      resources, outpdf):
+    from math import isclose
+
+    # Confirm input image is non-square resolution
+    in_pageinfo = pdf_get_all_pageinfo(str(resources / 'aspect.pdf'))
+    assert in_pageinfo[0]['xres'] != in_pageinfo[0]['yres']
+
+    # --force-ocr requires means forced conversion to square resolution
+    check_ocrmypdf(
+        resources / 'aspect.pdf', outpdf,
+        '--force-ocr',
+        '--pdf-renderer', renderer, env=spoof_tesseract_cache)
+
+    out_pageinfo = pdf_get_all_pageinfo(str(outpdf))
+
+    in_p0, out_p0 = in_pageinfo[0], out_pageinfo[0]
+
+    # Resolution show now be equal
+    assert out_p0['xres'] == out_p0['yres']
+
+    # Page size should match input page size
+    assert isclose(in_p0['width_inches'],
+                   out_p0['width_inches'])
+    assert isclose(in_p0['height_inches'],
+                   out_p0['height_inches'])
+
+    # Because we rasterized the page to produce a new image, it should occupy
+    # the entire page
+    out_im_w = out_p0['images'][0]['width'] / out_p0['images'][0]['dpi_w']
+    out_im_h = out_p0['images'][0]['height'] / out_p0['images'][0]['dpi_h']
+    assert isclose(out_p0['width_inches'], out_im_w)
+    assert isclose(out_p0['height_inches'], out_im_h)
+
+
 def test_image_to_pdf(spoof_tesseract_noop, resources, outpdf):
    check_ocrmypdf(
        resources / 'LinnSequencer.jpg', outpdf, '--image-dpi', '200',
@@ -614,6 +671,23 @@ def test_stdout(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf):
    assert qpdf.check(output_file, log=None)


+def test_closed_streams(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf):
+    input_file = str(resources / 'francais.pdf')
+    output_file = str(outpdf)
+
+    def evil_closer():
+        os.close(0)
+        os.close(1)
+
+    p_args = ocrmypdf_exec + [input_file, output_file]
+    p = Popen(
+        p_args, close_fds=True, stdout=None, stderr=PIPE, stdin=None,
+        env=spoof_tesseract_noop, preexec_fn=evil_closer)
+    out, err = p.communicate()
+    print(err.decode())
+    assert p.returncode == ExitCode.ok
+
+
 def test_masks(spoof_tesseract_noop, resources, outpdf):
    check_ocrmypdf(resources / 'masks.pdf', outpdf, env=spoof_tesseract_noop)

@@ -682,6 +756,10 @@ def test_overlay(spoof_tesseract_noop, resources, outpdf):
                   env=spoof_tesseract_noop)


+@pytest.mark.skipif(
+    os.getuid() == 0 or os.geteuid() == 0,
+    reason="root can write to anything"
+    )
 def test_destination_not_writable(spoof_tesseract_noop, resources, outdir):
    protected_file = outdir / 'protected.pdf'
    protected_file.touch()
@@ -690,3 +768,110 @@ def test_destination_not_writable(spoof_tesseract_noop, resources, outdir):
        resources / 'jbig2.pdf', protected_file,
        env=spoof_tesseract_noop)
    assert p.returncode == ExitCode.file_access_error, "Expected error"
+
+
+def test_tesseract_config_valid(resources, outdir):
+    cfg_file = outdir / 'test.cfg'
+    with cfg_file.open('w') as f:
+        f.write('''\
+load_system_dawg 0
+language_model_penalty_non_dict_word 0
+language_model_penalty_non_freq_dict_word 0
+''')
+
+    check_ocrmypdf(
+        resources / 'ccitt.pdf', outdir / 'out.pdf',
+        '--tesseract-config', str(cfg_file))
+
+
+@pytest.mark.parametrize('renderer', [
+    'hocr',
+    'tesseract',
+    ])
+def test_tesseract_config_notfound(renderer, resources, outdir):
+    cfg_file = outdir / 'nofile.cfg'
+
+    p, out, err = run_ocrmypdf(
+        resources / 'ccitt.pdf', outdir / 'out.pdf',
+        '--pdf-renderer', renderer,
+        '--tesseract-config', str(cfg_file))
+    assert "Can't open" in err, "No error message about missing config file"
+    assert p.returncode == ExitCode.ok
+
+
+@pytest.mark.parametrize('renderer', [
+    'hocr',
+    'tesseract',
+    ])
+def test_tesseract_config_invalid(renderer, resources, outdir):
+    cfg_file = outdir / 'test.cfg'
+    with cfg_file.open('w') as f:
+        f.write('''\
+THIS FILE IS INVALID
+''')
+
+    p, out, err = run_ocrmypdf(
+        resources / 'ccitt.pdf', outdir / 'out.pdf',
+        '--pdf-renderer', renderer,
+        '--tesseract-config', str(cfg_file))
+    assert "parameter not found" in err, "No error message"
+    assert p.returncode == ExitCode.invalid_config
+
+
+def test_form_xobject(spoof_tesseract_noop, resources, outpdf):
+    check_ocrmypdf(resources / 'formxobject.pdf', outpdf,
+                   '--force-ocr',
+                   env=spoof_tesseract_noop)
+
+
+@pytest.mark.skipif(sys.version_info < (3, 5), reason="needs math.isclose")
+@pytest.mark.parametrize('renderer', [
+    'hocr',
+    'tesseract',
+])
+def test_pagesize_consistency(renderer, resources, outpdf):
+    from math import isclose
+
+    first_page_dimensions = pytest.helpers.first_page_dimensions
+
+    infile = resources / 'linn.pdf'
+
+    before_dims = first_page_dimensions(infile)
+
+    check_ocrmypdf(
+        infile,
+        outpdf, '--pdf-renderer', renderer,
+        '--clean', '--deskew', '--remove-background', '--clean-final')
+
+    after_dims = first_page_dimensions(outpdf)
+
+    assert isclose(before_dims[0], after_dims[0])
+    assert isclose(before_dims[1], after_dims[1])
+
+
+def test_skip_big_with_no_images(spoof_tesseract_noop, resources, outpdf):
+    check_ocrmypdf(resources / 'blank.pdf', outpdf,
+                   '--skip-big', '5',
+                   '--force-ocr',
+                   env=spoof_tesseract_noop)
+
+
+def test_gs_render_failure(spoof_no_tess_gs_render_fail, resources, outpdf):
+    p, out, err = run_ocrmypdf(
+        resources / 'blank.pdf', outpdf,
+        env=spoof_no_tess_gs_render_fail)
+    print(err)
+    assert p.returncode == ExitCode.child_process_error
+
+
+def test_gs_raster_failure(spoof_no_tess_gs_raster_fail, resources, outpdf):
+    p, out, err = run_ocrmypdf(
+        resources / 'ccitt.pdf', outpdf,
+        env=spoof_no_tess_gs_raster_fail)
+    print(err)
+    assert p.returncode == ExitCode.child_process_error
+
+
+def test_no_contents(spoof_tesseract_noop, resources, outpdf):
+    check_ocrmypdf(resources / 'no_contents.pdf', outpdf, '--force-ocr',
+                   env=spoof_tesseract_noop)
@@ -101,3 +101,18 @@ def test_jpeg(resources, outdir):
    assert pdfimage['enc'] == 'jpeg'
    assert (pdfimage['dpi_w'] - 150) < 1e-5

+
+def test_form_xobject(resources):
+    filename = resources / 'formxobject.pdf'
+
+    pdfinfo = pageinfo.pdf_get_all_pageinfo(str(filename))
+    pdfimage = pdfinfo[0]['images'][0]
+    assert pdfimage['width'] == 50
+
+
+def test_no_contents(resources):
+    filename = resources / 'no_contents.pdf'
+
+    pdfinfo = pageinfo.pdf_get_all_pageinfo(str(filename))
+    assert len(pdfinfo[0]['images']) == 0
+    assert pdfinfo[0]['has_text'] == False
@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+# © 2017 James R. Barlow: github.com/jbarlow83
+
+import pytest
+from ocrmypdf.exceptions import ExitCode
+from ocrmypdf.exec import tesseract
+
+
+# Skip all tests in this file if not tesseract 3
+pytestmark = pytest.mark.skipif(tesseract.v4(),
+                                reason="tesseract 3.x required")
+
+
+def test_textonly_pdf_on_tess3(resources, no_outpdf):
+    p, _, _ = pytest.helpers.run_ocrmypdf(
+        resources / 'linn.pdf',
+        no_outpdf, '--pdf-renderer', 'tess4')
+
+    assert p.returncode == ExitCode.missing_dependency
+
+
+def test_oem_on_tess3(resources, no_outpdf):
+    p, _, err = pytest.helpers.run_ocrmypdf(
+        resources / 'aspect.pdf',
+        no_outpdf, '--tesseract-oem', '1')
+
+    assert p.returncode == ExitCode.ok
+    assert 'argument ignored' in err
@@ -1,31 +1,113 @@
 #!/usr/bin/env python3
 # © 2017 James R. Barlow: github.com/jbarlow83

-from subprocess import Popen, PIPE, check_output, check_call, DEVNULL
-import os
-import shutil
-from contextlib import suppress
-import sys
 import pytest
-from ocrmypdf.pageinfo import pdf_get_all_pageinfo
-import PyPDF2 as pypdf
 from ocrmypdf.exceptions import ExitCode
-from ocrmypdf import leptonica
-from ocrmypdf.pdfa import file_claims_pdfa
 from ocrmypdf.exec import tesseract
+from ocrmypdf import pageinfo
+import sys
+import os
+import PyPDF2 as pypdf
+
+
+spoof = pytest.helpers.spoof
+
+
+def tess4_possible_location():
+    """The location of tesseract 4 may be OCRMYPDF_TESS4, OCRMYPDF_TESSERACT,
+    or the installed version on PATH."""
+    return os.environ.get('OCRMYPDF_TESS4') or \
+           os.environ.get('OCRMYPDF_TESSERACT') or \
+            'tesseract'
+
+
+@pytest.fixture
+def ensure_tess4():
+    return spoof(tesseract=tess4_possible_location())
+
+
+def tess4_available():
+    """Check if a tesseract 4 binary is available, even if it's not the
+    official "tesseract" on PATH
+
+    """
+    old_environ = os.environ.copy()
+    try:
+        os.environ['OCRMYPDF_TESSERACT'] = tess4_possible_location()
+        return tesseract.v4() and tesseract.has_textonly_pdf()
+    finally:
+        os.environ = old_environ


 # Skip all tests in this file if not tesseract 4
-pytestmark = pytest.mark.skipif(not tesseract.v4(),
-                                reason="tesseract 4.0 required")
+pytestmark = pytest.mark.skipif(
+    not tess4_available(),
+    reason="tesseract 4.0 with textonly_pdf feature required")
+
+check_ocrmypdf = pytest.helpers.check_ocrmypdf
+run_ocrmypdf = pytest.helpers.run_ocrmypdf
+spoof = pytest.helpers.spoof


-@pytest.mark.skipif(not tesseract.has_textonly_pdf(),
-                    reason="requires textonly_pdf feature")
-def test_textonly_pdf(resources, outdir):
-    pytest.helpers.check_ocrmypdf(
+def test_textonly_pdf(ensure_tess4, resources, outdir):
+    check_ocrmypdf(
        resources / 'linn.pdf',
-        outdir / 'linn_textonly.pdf', '--pdf-renderer', 'tess4')
+        outdir / 'linn_textonly.pdf', '--pdf-renderer', 'tess4',
+        env=ensure_tess4)


+@pytest.mark.skipif(sys.version_info < (3, 5), reason="needs math.isclose")
+def test_pagesize_consistency_tess4(ensure_tess4, resources, outpdf):
+    from math import isclose

+    infile = resources / 'linn.pdf'
+
+    before_dims = pytest.helpers.first_page_dimensions(infile)
+
+    check_ocrmypdf(
+        infile,
+        outpdf, '--pdf-renderer', 'tess4',
+        '--clean', '--deskew', '--remove-background', '--clean-final',
+        env=ensure_tess4)
+
+    after_dims = pytest.helpers.first_page_dimensions(outpdf)
+
+    assert isclose(before_dims[0], after_dims[0])
+    assert isclose(before_dims[1], after_dims[1])
+
+
+@pytest.mark.parametrize('basename', ['graph_ocred.pdf', 'cardinal.pdf'])
+def test_skip_pages_does_not_replicate(
+        ensure_tess4, resources, basename, outdir):
+    infile = resources / basename
+    outpdf = outdir / basename
+
+    check_ocrmypdf(
+        infile,
+        outpdf, '--pdf-renderer', 'tess4', '--force-ocr',
+        '--tesseract-timeout', '0',
+        env=ensure_tess4
+    )
+
+    info_in = pageinfo.pdf_get_all_pageinfo(str(infile))
+
+    info = pageinfo.pdf_get_all_pageinfo(str(outpdf))
+    for page in info:
+        assert len(page['images']) == 1, "skipped page was replicated"
+
+    for n in range(len(info_in)):
+        assert info[n]['width_inches'] == info_in[n]['width_inches']
+
+
+def test_content_preservation(ensure_tess4, resources, outpdf):
+    infile = resources / 'masks.pdf'
+
+    check_ocrmypdf(
+        infile,
+        outpdf, '--pdf-renderer', 'tess4', '--tesseract-timeout', '0',
+        env=ensure_tess4
+    )
+
+    info = pageinfo.pdf_get_all_pageinfo(str(outpdf))
+    page = info[0]
+    assert len(page['images']) > 1, "masked were rasterized"
Author	SHA1	Message	Date
James R. Barlow	fae2119b1e	v4.5.6 release notes	2017-05-01 15:55:02 -07:00
James R. Barlow	aa859a4139	Fix #156 - NoneType has no ‘getObject’ for pages with no /Contents	2017-05-01 15:46:15 -07:00
James R. Barlow	b9b12e2879	Ensure that ocrmypdf stops and reports an error if Ghostscript fails Past behavior was to continue and let ruffus puke eventually	2017-05-01 15:44:21 -07:00
James R. Barlow	cf643c9f43	Fix argparse.ArgumentError needs two positional args	2017-05-01 14:55:47 -07:00
James R. Barlow	5b1a7880a9	Switch to Travis triggering Docker build to skip race condition with PyPI	2017-04-28 17:18:01 -07:00
James R. Barlow	474b6b0500	v4.5.5 release notes	2017-04-28 15:27:41 -07:00
James R. Barlow	6c8c1d8173	Fix #154 : KeyError ‘/Contents’ on blank pages with /Contents record	2017-04-28 15:22:05 -07:00
James R. Barlow	6a91fa637f	Squash merge improvements to auto-homebrewing macOS version	2017-04-28 15:17:38 -07:00
Jakub Wilkandjbarlow83	2846fb4e31	Remove misplaced flags from re.sub() call (#153 ) The 4th argument of re.sub() is maximum number of substitutions, not flags. Moreover, re.MULTILINE affects only semantics of ^ and $, so it wouldn't have any effect on this regular expression.	2017-04-20 00:38:23 -07:00
James R. Barlow	a1033cdc64	osx_brew: show output before letting “brew audit” check it	2017-04-18 23:38:07 -07:00
James R. Barlow	204336e1a5	Move release notes into the rest of documentation	2017-04-18 18:26:31 -07:00
James R. Barlow	8954e6c3b9	v4.5.4 Update release notes	2017-04-18 18:09:15 -07:00
James R. Barlow	fee22b6b0b	Fix #151 , cannot write mode P as JPEG all(<empty generator>) is True.	2017-04-18 18:08:45 -07:00
James R. Barlow	2b82c31b85	Update documentation	2017-04-18 18:07:19 -07:00
James R. Barlow	9a4813089c	autobrew: remove homebrew dependency “zlib”, causes audit failure	2017-04-18 15:28:30 -07:00
James R. Barlow	554fcc8b9d	Add test case for #152	2017-04-18 15:20:25 -07:00
Tomandjbarlow83	345256ee99	Fix --skip-big when there are no images in pdf (#152 ) * fixed skip-big when there are no images in pdf * added only_text pdf * updated only_text pdf * Delete only_text.pdf	2017-04-18 15:19:24 -07:00
James R. Barlow	58d1042147	v4.5.3 release notes update	2017-03-29 23:50:31 -07:00
James R. Barlow	7b7e3a3e03	Enable lossless reconstruction for —pdf-renderer tess4 where appropriate	2017-03-29 23:44:12 -07:00
James R. Barlow	1e7fbd4202	Fix issues with —pdf-renderer tess4 page skipping If tess4 renderer needed to skip OCR on a page it would end up duplicating the page contents onto the new page, rather than creating a blank OCR layer and placing it on the output page. This created duplicated content in output files.	2017-03-29 23:43:26 -07:00
James R. Barlow	4a9e9e9db2	v4.5.3 release notes	2017-03-29 13:19:34 -07:00
James R. Barlow	88ef2718f1	Reject high Unicode metadata at command line Ghostscript 9.21 does not seem to accept Unicode above U+FFFF. Previous versions did, but it now exits with a rangecheck error (-15). Reject on the command line for now. Complete fix would also need to check input PDF’s metadata.	2017-03-28 11:08:38 -07:00
James R. Barlow	e71e8ca3ad	Workaround for GS VMerror -25 bug Avoid inserting docinfo keys that would be translated to null strings, to avoid running afoul of https://bugs.ghostscript.com/show_bug.cgi?id=697684	2017-03-28 11:05:43 -07:00
James R. Barlow	45e9257d6e	Don’t use filename “pdfa_def.ps” for GS file At recommendation of Artifex people, don’t use the filename pdfa_def.ps because if given without a full path Ghostscript will prioritize searching its own resources over the current directory, and will pick up the example file pdfa_def.ps instead. We used the full path anyway, but this is a prudent defensive change to avoid odd Ghostscript behavior.	2017-03-28 11:02:46 -07:00
James R. Barlow	2954e72652	Some examples of Ghostscript and Tesseract warnings/errors were not tagged properly	2017-03-28 10:59:53 -07:00
James R. Barlow	199de96cff	Ghostcript 9.21 seems to have a regression related to Unicode metadata	2017-03-24 15:15:46 -07:00
James R. Barlow	8ddbe81513	Fix issue #147 : unpaper loses DPI information, affects —pdf-renderer tess4	2017-03-24 13:23:03 -07:00
James R. Barlow	a3e26e0498	Make —pdf-renderer tess4 more informative, less FUD	2017-03-24 12:58:40 -07:00
James R. Barlow	4ad129d8d8	docs: Don't recommend system-site-packages anymore (Ubuntu 16.04) Not needed since reportlab 3.4 comes with a wheel, and that was the main difficulty.	2017-03-23 08:13:28 -07:00
James R. Barlow	dfb9fa0736	autobrew: missing deps	2017-03-16 00:18:07 -07:00
James R. Barlow	eb036898e9	Update documentation with macOS homebrew tap [ci skip]	2017-03-13 22:25:25 -07:00
James R. Barlow	7c6aa76a2a	Fix brew audit failure	2017-03-13 17:58:31 -07:00
James R. Barlow	f035cb1088	Fixed issue #142 — closed streams raise an exception on fork attempt	2017-03-13 15:52:57 -07:00
James R. Barlow	35162166c5	Fix UnboundLocalError in autobrew.py	2017-03-13 15:51:39 -07:00
James R. Barlow	107f6abcb1	Fix autobrew build issues - missing deps	2017-03-12 17:07:38 -07:00
James R. Barlow	760a939e7d	Further autobrew tweaks	2017-03-11 17:05:26 -08:00
James R. Barlow	72660d0dec	MacOS skip the one test that needs poppler, to save installing poppler	2017-03-11 17:03:26 -08:00
James R. Barlow	8444a8f211	before_deploy doesn’t run unless something is going to be deployed	2017-03-11 17:03:20 -08:00
James R. Barlow	4a1fec8328	Improvements to macOS test and work on homebrew tap autobrew Squashed commits: [3f06c1e] Try setting up homebrew tap autobuilding [01532f1] Strict mode error in brew	2017-03-11 17:00:54 -08:00
James R. Barlow	42547f6017	Revert "Finalize Dockerfile move; unfortunately not supported by Docker Hub" Unfortunately because of this issue https://github.com/docker/hub-feedback/issues/292 Docker Hub effectively automatically changes directory to the location of Dockerfile, meaning needed files are not in the right places, so this has to be reverted. This reverts commit `65c9a07dde`. [ci skip]	2017-03-10 22:49:08 -08:00
James R. Barlow	0ccf564f03	Revert "Move Dockerfiles out of root" This reverts commit `3d3b3abc1b`.	2017-03-10 22:46:03 -08:00
James R. Barlow	65c9a07dde	Finalize Dockerfile move; unfortunately not supported by Docker Hub Unfortunately because of this issue https://github.com/docker/hub-feedback/issues/292 Docker Hub effectively automatically changes directory to the location of Dockerfile, meaning needed files are not in the right places, so this has to be reverted.	2017-03-10 22:45:31 -08:00
James R. Barlow	4700a19322	Move pipeline.svg out of root	2017-03-10 17:18:38 -08:00
James R. Barlow	3d3b3abc1b	Move Dockerfiles out of root	2017-03-10 17:17:36 -08:00
James R. Barlow	7cd2770a13	Fix issue #137 - proportions of non-square resolution distorted Distortion mainly affected —force-ocr	2017-02-26 17:13:16 -08:00
James R. Barlow	7b94129d9e	v4.5 notes	2017-02-14 13:03:48 -08:00
James R. Barlow	d1a0065ef8	Create test case for Form XObjects	2017-02-14 12:51:15 -08:00
James R. Barlow	5a817370fd	Warn more strongly about —pdf-renderer tesseract until fix is widely propagated	2017-02-14 11:33:07 -08:00
James R. Barlow	ab0a210763	Update dockerfile.tess4 yet again Installing Tess4 PPA over Tess3 proved too much pain, so sever the link between this and the jbarlow83/ocrmypdf image, starting each from scratch. Also the complete set of language packs proves too much - the build seems likely to fail when trying to install so many.	2017-02-14 11:32:05 -08:00
James R. Barlow	9f800736bc	Fix running_in_docker() check failing on newer Docker This test has to work to ensure spoof/tesseract_cache.py has a writable directory to put cache into. Otherwise those tests fail.	2017-02-13 02:16:06 -08:00
James R. Barlow	c9a83afad6	Improve batch processing examples	2017-02-13 02:14:32 -08:00
James R. Barlow	5e14274f10	pageinfo: learn to extract image information from Form XObjects	2017-02-11 16:48:59 -08:00
James R. Barlow	167470b4bd	Re-fix Dockerfile.tess4 [ci skip]	2017-02-10 15:43:49 -08:00
James R. Barlow	f06d3c2ec2	Fix tesseract 3.04 on tesseract 4 on image [skip ci]	2017-02-10 08:38:22 -08:00
James R. Barlow	74c99a8a77	v4.4.2 release notes	2017-02-06 21:56:55 -08:00
James R. Barlow	0e4d312ee2	Adjust Travis deploy to PyPI settings -only on master branch -only Python 3.6 build uploads, so the others don’t compete -don’t upload docs to PyPI	2017-02-06 21:27:59 -08:00
James R. Barlow	589f19559d	Rewrite Dockerfiles to use ubuntu 16.10 base system Debian now has a few disadvantages: -there is no convenient PPA for Debian tesseract 4.0, but there is for Ubuntu -Ubuntu sets locale to UTF-8 automatically removing the need to do this All three ocrmypdf docker images are now based on a common Ubuntu 16.10 image, derived from the one used to build ocrmypdf-tess4. -polyglot now differs from -tess4 only by opting into the tess4 PPA. Both Ubuntu 16.10 and Debian stretch use tesseract 3.04.01 now making the sharp.ttf patch unnecessary. /etc/apt/sources has been unused for a while now both have newer Ghostscripts.	2017-02-06 14:39:29 -08:00
James R. Barlow	f28bc25dc0	Configure travis to handle deployment to PyPI; also lint .travis.yml	2017-02-06 13:50:53 -08:00
James R. Barlow	a0657ad937	Prevent use of —pdf-renderer tess4 on tesseract 3	2017-02-06 13:49:43 -08:00
James R. Barlow	5b8d88af4c	Suggest use of aliases to hide docker run	2017-01-30 15:08:02 -08:00
James R. Barlow	fa82b50340	Adding missing file Dockerfile.tess4	2017-01-29 18:34:01 -08:00
James R. Barlow	005216bc57	Support ocrmypdf-tess4	2017-01-29 18:26:52 -08:00
James R. Barlow	e748fdcf6f	v4.4.1 release notes	2017-01-28 22:23:35 -08:00
James R. Barlow	8c17c9918e	Add documentation and test cases for —tesseract-config This parameter has existed for along time but never really got any attention.	2017-01-28 22:06:51 -08:00
James R. Barlow	ea0dd99d0b	More documentation updates	2017-01-28 15:35:59 -08:00
James R. Barlow	e0cc67afae	docs: suggest —oem 1	2017-01-28 14:58:25 -08:00
James R. Barlow	04f9cbe364	Describe how to use tesseract 4.0 while 3.04 is installed	2017-01-27 18:13:59 -08:00
James R. Barlow	99afebd033	tesseract jobs_limit(2) At least on macOS with my quadcore performance improves with two tesseracts in parallel (20% gain). Hard to say how this will affect Linux, but stepping up to 2 jobs seems justifiable.	2017-01-27 18:13:12 -08:00
James R. Barlow	a6feacc810	travis: fix ‘pip install’ by moving working code out of the way	2017-01-27 14:33:23 -08:00
James R. Barlow	65e4b1672f	cffi: verbose=True	2017-01-27 14:17:13 -08:00
James R. Barlow	46cc0dd190	Revert "Do we need to exclude ocrmypdf.lib?" This reverts commit `678b9fb603`.	2017-01-27 13:51:30 -08:00
James R. Barlow	678b9fb603	Do we need to exclude ocrmypdf.lib?	2017-01-27 13:49:11 -08:00
James R. Barlow	49ab0c1f0b	setup.py: cffi is definitely needed in setup_requires	2017-01-27 13:43:47 -08:00
James R. Barlow	ab490a7736	Experiment: update *requirements.txt, use more current travis build steps Perhaps this works around the pip/setup.py asymmetry that broke the 4.4 release.	2017-01-27 13:13:14 -08:00
James R. Barlow	e4ce1dae35	setup.py: for some reason, subpackages must be explicitly specified	2017-01-27 00:37:05 -08:00
James R. Barlow	179b812acb	Fix readthedocs build error	2017-01-26 23:57:51 -08:00
jbarlow83andGitHub	7f170517ec	Note about pytest-helpers-namespace	2017-01-26 23:15:32 -08:00