Travis: nevermind xenial, then

Gave a weird build error

.docker/Dockerfile

@@ -1,22 +1,18 @@
 # OCRmyPDF
 #
-FROM      ubuntu:17.10
+FROM      ubuntu:18.04
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
-  software-properties-common python-software-properties \
-  python3-wheel \
-  python3-reportlab \
-  python3-venv \
-  ghostscript \
-  qpdf \
-  poppler-utils \
+  ocrmypdf \
   unpaper \
-  libffi-dev \ 
-  tesseract-ocr \
   tesseract-ocr-eng \
   tesseract-ocr-fra \
+  tesseract-ocr-deu \
   tesseract-ocr-spa \
-  tesseract-ocr-deu
+  tesseract-ocr-por \
+  tesseract-ocr-chi-sim \
+  python3-venv \
+  python3-pip
 
 ENV LANG=C.UTF-8
 

.docker/Dockerfile.tess4

@@ -1,64 +0,0 @@
-# OCRmyPDF
-#
-FROM 	ubuntu:17.10
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-  software-properties-common python-software-properties \
-  python3-wheel \
-  python3-reportlab \
-  python3-venv \
-  ghostscript \
-  qpdf \
-  poppler-utils \
-  unpaper \
-  libffi-dev
-
-RUN add-apt-repository ppa:alex-p/tesseract-ocr
-
-RUN apt-get update \
-	&& apt-get autoremove -y \
-	&& apt-get install -y --no-install-recommends \
-		tesseract-ocr \
-		tesseract-ocr-eng \
-		tesseract-ocr-fra \
-		tesseract-ocr-deu \
-		tesseract-ocr-spa \
-		tesseract-ocr-por \
-		tesseract-ocr-ara \
-		tesseract-ocr-rus \
-		tesseract-ocr-chi-sim 
-
-ENV LANG=C.UTF-8
-
-RUN python3 -m venv --system-site-packages /appenv
-
-# This installs the latest binary wheel instead of the code in the current
-# folder. Installing from source will fail, apparently because cffi needs
-# build-essentials (gcc) to do a source installation 
-# (i.e. "pip install ."). It's unclear to me why this is the case.
-RUN . /appenv/bin/activate; \
-  pip install --upgrade pip \
-  && pip install ocrmypdf[fitz]
-
-# Now copy the application in, mainly to get the test suite.
-# Do this now to make the best use of Docker cache.
-COPY . /application
-RUN . /appenv/bin/activate; \
-  pip install -r /application/test_requirements.txt
-
-# Remove the junk, including the source version of application since it was
-# already installed
-RUN rm -rf /tmp/* /var/tmp/* /root/* /application/ocrmypdf \
-  && apt-get autoremove -y \
-  && apt-get autoclean -y
-
-RUN useradd docker \
-  && mkdir /home/docker \
-  && chown docker:docker /home/docker
-
-USER docker
-WORKDIR /home/docker
-
-# Must use array form of ENTRYPOINT
-# Non-array form does not append other arguments, because that is "intuitive"
-ENTRYPOINT ["/application/.docker/docker-wrapper.sh"]

.github/ISSUE_TEMPLATE

@@ -1,3 +1,5 @@
-Please include the command line and (if needed) a test file with your issue report.
+Please include the command line and a test file with your issue report.
 
 If possible, please use a test file that we can include in future test cases (no personal information, no copyrighted material).
+
+If you wish to encrypt a test file for the OCRmyPDF maintainer only, see the [Wiki](https://github.com/jbarlow83/OCRmyPDF/wiki).

.gitignore

@@ -2,12 +2,14 @@
 *.pyc
 *.sublime-*
 venv*/
+.venv/
 pyvenv.cfg
 tasks.py
 .bash_history
 .ruffus_history.sqlite
 .idea/
 .pytest_cache/
+.pylintrc
 
 # Package building
 *.egg-info/
@@ -41,4 +43,5 @@ tmp/
 pdfbox-app*.jar
 .vscode/
 IDEAS
-_Dockerfile.local
+_Dockerfile.local
+/scratch.py

.travis.yml

@@ -1,32 +1,51 @@
 dist: trusty
 language: python
 cache:
-  ccache: true
   pip: true
   directories:
   - $HOME/Library/Caches/Homebrew
 
-env:
-  global:
-    - secure: "hsf6MT+n2x3OiDM2fQyJZdV0/PWYmv81LdVqC6cfnHBE/8N3DloJRqQ7WfO14TxhiK9PEC7MpyCj0lSabUHEO7gSH6Vks6I1asoSkt8S9/bSMlhT4hei+pwVpeGEiU5xHVATNjY+D919VC3IFvc3XmjT74h/2SLhaZ+jhEmDggM="  # HOMEBREW_OCRMYPDF_TOKEN
+addons:
+  apt:
+    update: true
+    sources:
+    - sourceline: 'ppa:alex-p/tesseract-ocr'
+    - sourceline: 'ppa:heyarje/libav-11'
+    - sourceline: 'ppa:vshn/ghostscript'
+    packages:
+    - ghostscript
+    - libavcodec56
+    - libavformat56
+    - libavutil54
+    - libffi-dev
+    - poppler-utils
+    - qpdf
+    - tesseract-ocr
+    - tesseract-ocr-deu
+    - tesseract-ocr-eng
+    - tesseract-ocr-fra
 
 matrix:
   include:
     - os: linux
       sudo: required
       language: python
-      python: 3.5
+      python: "3.5"
       env: EXTRAS=
     - os: linux
       sudo: required
       language: python
-      python: 3.6
+      python: "3.6"
       env: EXTRAS=
     - os: linux
       sudo: required
       language: python
-      python: 3.6
+      python: "3.6"
       env: EXTRAS=[fitz]
+    - os: linux
+      sudo: required
+      language: python
+      python: "3.7-dev"
     - os: osx
       osx_image: xcode8
       language: generic
@@ -41,7 +60,10 @@ before_cache:
 
 before_install: |
   if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
-    bash .travis/linux_before_install.sh
+    pip install --upgrade pip
+    mkdir -p packages
+    wget -q 'https://www.dropbox.com/s/vaq0kbwi6e6au80/unpaper_6.1-1.deb?raw=1' -O packages/unpaper_6.1-1.deb
+    sudo dpkg -i packages/unpaper_6.1-1.deb
   elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
     brew update && brew bundle --file=.travis/Brewfile
     pip3 install --upgrade pip
@@ -49,6 +71,7 @@ before_install: |
   fi
 
 install:
+- pip3 install pycparser # py3.7 workaround for https://github.com/eliben/pycparser/issues/251
 - pip3 install ".$EXTRAS"
 - pip3 install -r test_requirements.txt
 
@@ -73,31 +96,3 @@ deploy:
     tags: true
     condition: $TRAVIS_PYTHON_VERSION == "3.6" && $TRAVIS_OS_NAME == "linux" && $EXTRAS == ""
   skip_upload_docs: true
-
-  # test pypi
-- provider: pypi
-  server: https://testpypi.pypi.org/legacy/
-  user: ocrmypdf-travis
-  password:
-    secure: "DTFOmmNL6olA0+yXvp4u9jXZlZeqrJsJ0526jzqf4a3gZ6jnGTq5UI6WzRsslSyoMMfXKtHQebqHM6ogSgCZinyZ3ufHJo8fn9brxbEc2gsiWkbj5o3bGwdWMT1vNNE7XW0VCpw87rZ1EEwjl4FJHFudMlPR1yfU5+uq0k0PACo="
-  distributions: "sdist"
-  on:
-    branch: develop
-    tags: false
-    condition: $TRAVIS_OS_NAME == "osx"
-  skip_upload_docs: true
-
-  # null deploy for osx
-  # we really just want to run after_deploy *after* pypi upload is done, but
-  # after_deploy on runs if a given box deployed
-- provider: script
-  script: /usr/bin/true
-  on:
-    branch: master
-    tags: true
-    condition: $TRAVIS_OS_NAME == "osx"
-
-after_deploy: |
-  if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-    bash .travis/osx_brew.sh
-  fi

.travis/autobrew.py

@@ -1,93 +0,0 @@
-#!/usr/bin/env python3
-# © 2017-18 James R. Barlow: github.com/jbarlow83
-
-from string import Template
-from subprocess import run, PIPE
-import re
-
-recipe_template = Template("""
-class Ocrmypdf < Formula
-  include Language::Python::Virtualenv
-
-  desc "Adds an OCR text layer to scanned PDF files"
-  homepage "https://github.com/jbarlow83/OCRmyPDF"
-  ${ocrmypdf_url}
-  ${ocrmypdf_sha256}
-
-  depends_on "pkg-config" => :build
-  depends_on "mupdf-tools" => :build  # statically links libmupdf.a
-  depends_on "freetype"
-  depends_on "ghostscript"
-  depends_on "jpeg"
-  depends_on "libpng"
-  depends_on "python"
-  depends_on "qpdf"
-  depends_on "tesseract"
-  depends_on "unpaper"
-
-${resources}
-  def install
-    venv = virtualenv_create(libexec, "python3")
-
-    resource("Pillow").stage do
-      inreplace "setup.py" do |s|
-        sdkprefix = MacOS::CLT.installed? ? "" : MacOS.sdk_path
-        s.gsub! "openjpeg.h", "probably_not_a_header_called_this_eh.h"
-        s.gsub! "ZLIB_ROOT = None", "ZLIB_ROOT = ('#{sdkprefix}/usr/lib', '#{sdkprefix}/usr/include')"
-        s.gsub! "JPEG_ROOT = None", "JPEG_ROOT = ('#{Formula["jpeg"].opt_prefix}/lib', '#{Formula["jpeg"].opt_prefix}/include')"
-        s.gsub! "FREETYPE_ROOT = None", "FREETYPE_ROOT = ('#{Formula["freetype"].opt_prefix}/lib', '#{Formula["freetype"].opt_prefix}/include')"
-      end
-
-      # avoid triggering "helpful" distutils code that doesn't recognize Xcode 7 .tbd stubs
-      ENV.append "CFLAGS", "-I#{MacOS.sdk_path}/System/Library/Frameworks/Tk.framework/Versions/8.5/Headers" unless MacOS::CLT.installed?
-      venv.pip_install Pathname.pwd
-    end
-
-    res = resources.map(&:name).to_set - ["Pillow"]
-
-    res.each do |r|
-      venv.pip_install resource(r)
-    end
-
-    venv.pip_install_and_link buildpath
-  end
-
-  test do
-    # Since we use Python 3, we require a UTF-8 locale
-    ENV["LC_ALL"] = "en_US.UTF-8"
-
-    system "#{bin}/ocrmypdf", "-f", "-q", "--deskew",
-                              test_fixtures("test.pdf"), "ocr.pdf"
-    assert_predicate testpath/"ocr.pdf", :exist?
-  end
-end
-""")
-
-def main():
-    p = run(['poet', '--single', 'ocrmypdf'],
-            encoding='utf-8', stdout=PIPE, check=True)
-
-    ocrmypdf_lines = p.stdout.splitlines()
-    ocrmypdf_url = ocrmypdf_lines[1].strip()
-    ocrmypdf_sha256 = ocrmypdf_lines[2].strip()
-
-    ocrmypdf_version = re.search(
-        r'ocrmypdf-(.+)\.tar.*', ocrmypdf_url).group(1)
-    print(f"Autobrewing {ocrmypdf_version}")
-
-    p = run(['poet', '--resources', 'ocrmypdf'],
-            encoding='utf-8', stdout=PIPE, check=True)
-
-    poet_resources = p.stdout
-
-    # Remove the duplicate "ocrmypdf" resource block
-    all_resources = poet_resources.split('resource')
-    kept_resources = [block for block in all_resources if 'ocrmypdf' not in block]
-    resources = 'resource'.join(kept_resources)
-
-    with open('ocrmypdf.rb', 'w') as out:
-        out.write(recipe_template.substitute(**locals()))
-
-
-if __name__ == '__main__':
-    main()

.travis/linux_before_install.sh

@@ -1,42 +0,0 @@
-#!/bin/bash
-# © 2017 James R. Barlow: github.com/jbarlow83
-set -euo pipefail
-set -x
-
-sudo add-apt-repository ppa:vshn/ghostscript -y
-sudo add-apt-repository ppa:heyarje/libav-11 -y
-sudo apt-get update -qq
-sudo apt-get install -y \
-	ghostscript \
-	poppler-utils \
-	libavformat56 \
-	libavcodec56 \
-	libavutil54 \
-	libffi-dev
-
-sudo add-apt-repository ppa:alex-p/tesseract-ocr -y
-
-sudo apt-get update
-sudo apt-get autoremove -y
-sudo apt-get install -y --no-install-recommends \
-		tesseract-ocr \
-		tesseract-ocr-eng \
-		tesseract-ocr-fra \
-		tesseract-ocr-deu
-
-pip install --upgrade pip
-mkdir -p packages
-wget -q 'https://www.dropbox.com/s/vaq0kbwi6e6au80/unpaper_6.1-1.deb?raw=1' -O packages/unpaper_6.1-1.deb
-sudo dpkg -i packages/unpaper_6.1-1.deb
-
-if [ ! -f /usr/local/bin/qpdf ]; then
-	export QPDF_RELEASE='https://github.com/qpdf/qpdf/releases/download/release-qpdf-8.0.2/qpdf-8.0.2.tar.gz'
-	mkdir qpdf
-    wget -q $QPDF_RELEASE -O - | tar xz -C qpdf --strip-components=1
-    cd qpdf/
-	export PATH="/usr/local/opt/ccache/libexec:$PATH"
-    ./configure --prefix=/usr
-    make -j 2
-    sudo make install
-    cd ..
-fi

.travis/osx_brew.sh

@@ -1,23 +0,0 @@
-#!/bin/bash
-# © 2017 James R. Barlow: github.com/jbarlow83
-set -uo pipefail
-set -x
-
-pip3 install homebrew-pypi-poet
-python3 .travis/autobrew.py
-cat ocrmypdf.rb
-
-# brew audit crashes Travis
-#brew audit ocrmypdf.rb
-
-# Important: disable debug output so token is hidden
-set +x
-git clone https://$HOMEBREW_OCRMYPDF_TOKEN@github.com/jbarlow83/homebrew-ocrmypdf.git
-set -x
-
-pushd homebrew-ocrmypdf
-cp ../ocrmypdf.rb Formula/ocrmypdf.rb
-git add Formula/ocrmypdf.rb
-git commit -m "homebrew-ocrmypdf: automatic release $TRAVIS_BUILD_NUMBER $TRAVIS_TAG"
-git push origin master
-popd

README.rst

@@ -4,10 +4,10 @@ OCRmyPDF
 .. image:: https://travis-ci.org/jbarlow83/OCRmyPDF.svg?branch=master
     :target: https://travis-ci.org/jbarlow83/OCRmyPDF
 
-.. image:: https://img.shields.io/pypi/v/ocrmypdf.svg   
+.. image:: https://img.shields.io/pypi/v/ocrmypdf.svg
     :target: https://pypi.org/project/ocrmypdf/
 
-.. image:: https://img.shields.io/homebrew/v/ocrmypdf.svg   
+.. image:: https://img.shields.io/homebrew/v/ocrmypdf.svg
     :alt: homebrew
     :target: http://brewformulas.org/Ocrmypdf
 
@@ -54,7 +54,7 @@ Motivation
 I searched the web for a free command line tool to OCR PDF files on
 Linux/UNIX: I found many, but none of them were really satisfying.
 
--  Either they produced PDF files with misplaced text under the image (making copy/paste impossible) 
+-  Either they produced PDF files with misplaced text under the image (making copy/paste impossible)
 -  Or they did not handle accents and multilingual characters
 -  Or they changed the resolution of the embedded images
 -  Or they generated ridiculously large PDF files
@@ -63,7 +63,7 @@ Linux/UNIX: I found many, but none of them were really satisfying.
 -  On top of that none of them produced PDF/A files (format dedicated for long time storage)
 
 ...so I decided to develop my own tool (using various existing scripts
-as an inspiration). 
+as an inspiration).
 
 Installation
 ------------
@@ -97,7 +97,7 @@ you can often find packages that provide language packs:
 
    # Debian/Ubuntu users
    apt-get install tesseract-ocr-chi-sim  # Example: Install Chinese Simplified language back
-   
+
 You can then pass the ``-l LANG`` argument to OCRmyPDF to give a hint as to what languages it should search for. Multiple
 languages can be requested.
 
@@ -126,9 +126,7 @@ If you detect an issue, please:
 Requirements
 ------------
 
-Runs on CPython 3.6, and requires external program installations of Ghostscript, Tesseract OCR, QPDF, and Leptonica. ocrmypdf is pure Python, but uses CFFI to portably generate library bindings.
-
-Python 3.5 is also supported.
+Runs on CPython 3.5, 3.6 and 3.7. Requires external program installations of Ghostscript, Tesseract OCR, QPDF, and Leptonica. ocrmypdf is pure Python, but uses CFFI to portably generate library bindings.
 
 Press & Media
 -------------
@@ -142,7 +140,7 @@ Press & Media
 License
 -------
 
-The OCRmyPDF software is licensed under the GNU GPLv3. Certain files are covered by other licenses, as noted in their source files. 
+The OCRmyPDF software is licensed under the GNU GPLv3. Certain files are covered by other licenses, as noted in their source files.
 
 The license for each test file varies, and is noted in tests/resources/README.rst. The documentation is licensed under Creative Commons Attribution-ShareAlike 4.0 (CC-BY-SA 4.0).
 

docs/cookbook.rst

@@ -65,8 +65,8 @@ By default OCRmyPDF assumes the document is English.
 
 .. code-block:: bash
 
-    ocrmypdf -l fre LeParisien.pdf LeParisien.pdf
-    ocrmypdf -l eng+fre Bilingual-English-French.pdf Bilingual-English-French.pdf
+    ocrmypdf -l fra LeParisien.pdf LeParisien.pdf
+    ocrmypdf -l eng+fra Bilingual-English-French.pdf Bilingual-English-French.pdf
 
 Language packs must be installed for all languages specified. See :ref:`Installing additional language packs <lang-packs>`.
 

docs/installation.rst

@@ -1,7 +1,7 @@
 Installation
 ============
 
-The easiest way to install OCRmyPDF to follow the steps for your operating system/platform. 
+The easiest way to install OCRmyPDF to follow the steps for your operating system/platform.
 
 If you want to use the latest version of OCRmyPDF, your best bet is to install the most recent version your platform provides, and then upgrade that version by installing the Python binary wheels.
 
@@ -23,7 +23,7 @@ To see what versions are available, check the `Debian Package Tracker <https://t
 Installing on macOS with Homebrew
 ---------------------------------
 
-.. image:: https://img.shields.io/homebrew/v/ocrmypdf.svg   
+.. image:: https://img.shields.io/homebrew/v/ocrmypdf.svg
     :alt: homebrew
     :target: http://brewformulas.org/Ocrmypdf
 
@@ -55,7 +55,7 @@ Follow the Docker installation instructions for your platform.  If you can run t
 .. code-block:: bash
 
     docker run hello-world
-   
+
 OCRmyPDF will use all available CPU cores.  By default, the VirtualBox machine instance on Windows and macOS has only a single CPU core enabled. Use the VirtualBox Manager to determine the name of your Docker engine host, and then follow these optional steps to enable multiple CPUs:
 
 .. code-block:: bash
@@ -77,25 +77,22 @@ Assuming you have a Docker engine running, you can download one of the three ava
         - Notes
     *   - ocrmypdf
         - ``docker pull jbarlow83/ocrmypdf``
-        - Latest ocrmypdf with Tesseract 3.x. Includes English, French, German, Spanish.
+        - Latest ocrmypdf with Tesseract 4.0.0-beta1 on Ubuntu 18.04. Includes English, French, German, Spanish, Portugeuse and Simplified Chinese.
     *   - ocrmypdf-polyglot
         - ``docker pull jbarlow83/ocrmypdf-polyglot``
         - As above, with all available language packs.
-    *   - ocrmypdf-tess4
-        - ``docker pull jbarlow83/ocrmypdf-tess4``
-        - Latest ocrmypdf with Tesseract 4.x and English, French, German, Spanish, Portuguese, Chinese Simplified, Arabic and Russian (the top 8).
 
 For example:
 
 .. code-block:: bash
 
-    docker pull jbarlow83/ocrmypdf-tess4
+    docker pull jbarlow83/ocrmypdf
 
 Then tag it to give a more convenient name, just ocrmypdf:
 
 .. code-block:: bash
 
-    docker tag jbarlow83/ocrmypdf-tess4 ocrmypdf
+    docker tag jbarlow83/ocrmypdf ocrmypdf
 
 .. _docker-polyglot:
 
@@ -106,14 +103,14 @@ You can then run ocrmypdf using the command:
 .. code-block:: bash
 
     docker run --rm ocrmypdf --help
-  
+
 To execute the OCRmyPDF on a local file, you must `provide a writable volume to the Docker image <https://docs.docker.com/userguide/dockervolumes/>`_, and both the input and output file must be inside the writable volume.  This example command uses the current working directory as the writable volume:
 
 .. code-block:: bash
 
     docker run --rm -v "$(pwd):/home/docker" <other docker arguments>   ocrmypdf <your arguments to ocrmypdf>
 
-In this worked example, the current working directory contains an input file called ``test.pdf`` and the output will go to ``output.pdf``: 
+In this worked example, the current working directory contains an input file called ``test.pdf`` and the output will go to ``output.pdf``:
 
 .. code-block:: bash
 
@@ -170,11 +167,11 @@ Install or upgrade the required Homebrew packages, if any are missing:
     brew install python3
     brew install libxml2 libffi leptonica
     brew install unpaper   # optional
-   
-Python 3.5 and 3.6 are supported.
+
+Python 3.5, 3.6 and 3.7 are supported.
 
 Install the required Tesseract OCR engine with the language packs you plan to use:
-   
+
 .. code-block:: bash
 
     brew install tesseract                       # Option 1: for English, French, German, Spanish
@@ -211,6 +208,26 @@ The command line program should now be available:
 
     ocrmypdf --help
 
+Installing the latest version on Ubuntu 18.04 LTS
+-------------------------------------------------
+
+Ubuntu 18.04 includes ocrmypdf 6.1.2. To install a more recent version, first
+install the system version to get all the dependencies:
+
+.. code-block:: bash
+
+    sudo apt-get update
+    sudo apt-get install \
+        ocrmypdf \
+        python3-pip
+
+Then install ocrmypdf 6.1.5 for the local user and set the user's ``PATH`` to check for the user's Python packages.
+
+.. code-block:: bash
+
+    export PATH=$HOME/.local/bin:$PATH
+    pip3 install --user ocrmypdf[fitz]
+
 
 Installing on Ubuntu 16.04 LTS
 ------------------------------
@@ -287,7 +304,7 @@ If you prefer to not modify your system in this matter, consider using a Docker
     sudo apt-get update
 
     sudo apt-get install \
-        python3.6 \
+        python3.6-dev \
         ghostscript \
         tesseract-ocr \
         tesseract-ocr-eng \
@@ -298,7 +315,7 @@ Now we need to install ``pip`` and let it install ocrmypdf:
 
 .. code-block:: bash
 
-    wget -O - -o /dev/null https://bootstrap.pypa.io/get-pip.py | python3.6
+    curl https://bootstrap.pypa.io/ez_setup.py -o - | python3.6 && python3.6 -m easy_install pip
     pip3.6 install ocrmypdf[fitz]
 
 The ``wget`` command will download a program and run it.
@@ -381,9 +398,9 @@ The following dependencies are recommended:
 - unpaper 6.1
 - PyMuPDF 1.12.5 or newer
 
-These are in addition to the Python packaging dependencies, meaning that unfortunately, the ``pip install`` command cannot satisfy all of them. 
+These are in addition to the Python packaging dependencies, meaning that unfortunately, the ``pip install`` command cannot satisfy all of them.
 
-Python 3.6 and Tesseract 4.x are recommended for best OCR results and best performance.
+Python 3.6 and Tesseract 4.0.0-beta.1 are recommended for best OCR results and best performance.
 
 The library PyMuPDF is not widely available in platform distributions, and it improves OCRmyPDF in certain conditions. Consider installing OCRmyPDF from the Python binary wheels, which include a precompiled version of this library.
 

docs/release_notes.rst

@@ -6,13 +6,49 @@ OCRmyPDF uses `semantic versioning <http://semver.org/>`_ for its command line i
 The OCRmyPDF package itself does not contain a public API, although it is fairly stable and breaking changes are usually timed with a major release. A future release will clearly define the stable public API.
 
 .. Issue regex
-   find:    [^`]\#([0-9]{1,3})[^0-9]  
+   find:    [^`]\#([0-9]{1,3})[^0-9]
    replace: `#$1 <https://github.com/jbarlow83/OCRmyPDF/issues/$1>`_
 
+
+v6.2.2
+------
+
+-   Backport compatibility fixes for Python 3.7 and ruffus 2.7.0 from v7.0.0
+-   Backport fix to ignore masks when deciding what colors are on a page
+-   Backport some minor improvements from v7.0.0: better argument validation and warnings about the Tesseract 4.0.0 ``--user-words`` regression
+
+v6.2.1
+------
+
+-   Fix recent versions of Tesseract (after 4.0.0-beta1) not being detected as supporting the ``sandwich`` renderer (`#271 <https://github.com/ppjbarlow83/OCRmyPDF/issues/271>`_).
+
+
+v6.2.0
+------
+
+-   **Docker**: The Docker image ``ocrmypdf-tess4`` has been removed. The main Docker images, ``ocrmypdf`` and ``ocrmypdf-polyglot`` now use Ubuntu 18.04 as a base image, and as such Tesseract 4.0.0-beta1 is now the Tesseract version they use. There is no Docker image based on Tesseract 3.05 anymore.
+
+-   Creation of PDF/A-3 is now supported. However, there is no ability to attach files to PDF/A-3.
+
+-   List more reasons why the file size might grow.
+
+-   Fix issue `#262 <https://github.com/ppjbarlow83/OCRmyPDF/issues/262>`_, ``--remove-background`` error on PDFs contained colormapped (paletted) images.
+
+-   Fix another XMP metadata validation issue, in cases where the input file's creation date has no timezone and the creation date is not overridden.
+
+
+v6.1.5
+------
+
+-   Fix issue `#253 <https://github.com/jbarlow83/OCRmyPDF/issues/253>`_, a possible division by zero when using the ``hocr`` renderer.
+
+-   Fix incorrectly formatted ``<xmp:ModifyDate>`` field inside XMP metadata for PDF/As.  veraPDF flags this as a PDF/A validation failure. The error is caused the timezone and final digit of the seconds of modified time to be omitted, so at worst the modification time stamp is rounded to the nearest 10 seconds.
+
+
 v6.1.4
 ------
 
--   Fix issue #248, ``--clean`` argument may remove OCR from left column of text on certain documents. We now set ``--layout none`` to suppress this.  
+-   Fix issue `#248 <https://github.com/jbarlow83/OCRmyPDF/issues/248>`_ ``--clean`` argument may remove OCR from left column of text on certain documents. We now set ``--layout none`` to suppress this.
 
 -   The test cache was updated to reflect the change above.
 
@@ -143,7 +179,7 @@ v5.5
 -   Add new argument ``--max-image-mpixels``. Pillow 5.0 now raises an exception when images may be decompression bombs. This argument can be used to override the limit Pillow sets.
 -   Fix output page cropped when using the sandwich renderer and OCR is skipped on a rotated and image-processed page
 -   A warning is now issued when old versions of Ghostscript are used in cases known to cause issues with non-Latin characters
--   Fix a few parameter validation checks for ``-output-type pdfa-1`` and ``pdfa-2`` 
+-   Fix a few parameter validation checks for ``-output-type pdfa-1`` and ``pdfa-2``
 
 
 v5.4.4
@@ -153,7 +189,7 @@ v5.4.4
 -   Fix issue `#200 <https://github.com/jbarlow83/OCRmyPDF/issues/200>`_: an uncommon syntax for formatting decimal numbers in a PDF would cause qpdf to issue a warning, which ocrmypdf treated as an error. Now this the warning is relayed.
 -   Fix an issue where intermediate PDFs would be created at version 1.3 instead of the version of the original file. It's possible but unlikely this had side effects.
 -   A warning is now issued when older versions of qpdf are used since issues like `#200 <https://github.com/jbarlow83/OCRmyPDF/issues/200>`_ cause qpdf to infinite-loop
--   Address issue `#140 <https://github.com/jbarlow83/OCRmyPDF/issues/140>`_: if Tesseract outputs invalid UTF-8, escape it and print its message instead of aborting with a Unicode error 
+-   Address issue `#140 <https://github.com/jbarlow83/OCRmyPDF/issues/140>`_: if Tesseract outputs invalid UTF-8, escape it and print its message instead of aborting with a Unicode error
 -   Adding previously unlisted setup requirement, pytest-runner
 -   Update documentation: fix an error in the example script for Synology with Docker images, improved security guidance, advised ``pip install --user``
 
@@ -322,7 +358,7 @@ v4.4.1
 
 -   To prevent a `TIFF output error <https://github.com/python-pillow/Pillow/issues/2206>`_ caused by img2pdf >= 0.2.1 and Pillow <= 3.4.2, dependencies have been tightened
 -   The Tesseract 4.00 simultaneous process limit was increased from 1 to 2, since it was observed that 1 lowers performance
--   Documentation improvements to describe the ``--tesseract-config`` feature 
+-   Documentation improvements to describe the ``--tesseract-config`` feature
 -   Added test cases and fixed error handling for ``--tesseract-config``
 -   Tweaks to setup.py to deal with issues in the v4.4 release
 
@@ -385,7 +421,7 @@ v4.3
 
 -   New feature ``--remove-background`` to detect and erase the background of color and grayscale images
 -   Better documentation
--   Fixed an issue with PDFs that draw images when the raster stack depth is zero 
+-   Fixed an issue with PDFs that draw images when the raster stack depth is zero
 -   ocrmypdf can now redirect its output to stdout for use in a shell pipeline
 
     +  This does not improve performance since temporary files are still used for buffering
@@ -453,7 +489,7 @@ v4.2
     +  fixes issue `#82 <https://github.com/jbarlow83/OCRmyPDF/issues/82>`_
 
 -   Fixes an issue where, with certain settings, monochrome images in PDFs would be converted to 8-bit grayscale, increasing file size (`#79 <https://github.com/jbarlow83/OCRmyPDF/issues/79>`_)
--   Support for Ubuntu 12.04 LTS "precise" has been dropped in favor of (roughly) Ubuntu 14.04 LTS "trusty" 
+-   Support for Ubuntu 12.04 LTS "precise" has been dropped in favor of (roughly) Ubuntu 14.04 LTS "trusty"
 
     +  Some Ubuntu "PPAs" (backports) are needed to make it work
 
@@ -568,7 +604,7 @@ New features
 
 -   Automatic page rotation (``-r``) is now available. It uses ignores any prior rotation information
     on PDFs and sets rotation based on the dominant orientation of detectable text. This feature is
-    fairly reliable but some false positives occur especially if there is not much text to work with. (`#4 <https://github.com/jbarlow83/OCRmyPDF/issues/4>`_) 
+    fairly reliable but some false positives occur especially if there is not much text to work with. (`#4 <https://github.com/jbarlow83/OCRmyPDF/issues/4>`_)
 -   Deskewing is now performed using Leptonica instead of unpaper. Leptonica is faster and more reliable
     at image deskewing than unpaper.
 
@@ -611,7 +647,7 @@ v3.2
 New features
 ^^^^^^^^^^^^
 
--   Lossless reconstruction: when possible, OCRmyPDF will inject text layers without 
+-   Lossless reconstruction: when possible, OCRmyPDF will inject text layers without
     otherwise manipulating the content and layout of a PDF page. For example, a PDF containing a mix
     of vector and raster content would see the vector content preserved. Images may still be transcoded
     during PDF/A conversion.  (``--deskew`` and ``--clean-final`` disable this mode, necessarily.)
@@ -646,10 +682,10 @@ Changes
     needed to implement support
 -   Improved some error messages related to missing input files
 -   Fixed issue `#20 <https://github.com/jbarlow83/OCRmyPDF/issues/20>`_ - uppercase .PDF extension not accepted
--   Fixed an issue where OCRmyPDF failed to text that certain pages contained previously OCR'ed text, 
+-   Fixed an issue where OCRmyPDF failed to text that certain pages contained previously OCR'ed text,
     such as OCR text produced by Tesseract 3.04
 -   Inserts /Creator tag into PDFs so that errors can be traced back to this project
--   Added new option ``--pdf-renderer=auto``, to let OCRmyPDF pick the best PDF renderer. 
+-   Added new option ``--pdf-renderer=auto``, to let OCRmyPDF pick the best PDF renderer.
     Currently it always chooses the 'hocrtransform' renderer but that behavior may change.
 -   Set up Travis CI automatic integration testing
 
@@ -659,19 +695,19 @@ v3.0
 New features
 ^^^^^^^^^^^^
 
--   Easier installation with a Docker container or Python's ``pip`` package manager 
+-   Easier installation with a Docker container or Python's ``pip`` package manager
 -   Eliminated many external dependencies, so it's easier to setup
 -   Now installs ``ocrmypdf`` to ``/usr/local/bin`` or equivalent for system-wide
     access and easier typing
 -   Improved command line syntax and usage help (``--help``)
 -   Tesseract 3.03+ PDF page rendering can be used instead for better positioning
     of recognized text (``--pdf-renderer tesseract``)
--   PDF metadata (title, author, keywords) are now transferred to the 
+-   PDF metadata (title, author, keywords) are now transferred to the
     output PDF
 -   PDF metadata can also be set from the command line (``--title``, etc.)
 -   Automatic repairs malformed input PDFs if possible
 -   Added test cases to confirm everything is working
--   Added option to skip extremely large pages that take too long to OCR and are 
+-   Added option to skip extremely large pages that take too long to OCR and are
     often not OCRable (e.g. large scanned maps or diagrams); other pages are still
     processed (``--skip-big``)
 -   Added option to kill Tesseract OCR process if it seems to be taking too long on
@@ -690,9 +726,9 @@ Changes
     available CPUs, increasing performance
 -   The ``-o DPI`` argument has been phased out, in favor of ``--oversample DPI``, in
     case we need ``-o OUTPUTFILE`` in the future
--   Removed several dependencies, so it's easier to install.  We no 
+-   Removed several dependencies, so it's easier to install.  We no
     longer use:
-    
+
     - GNU parallel_
     - ImageMagick_
     - Python 2.7
@@ -708,7 +744,7 @@ Changes
     - qpdf_ 5.0.0+
     - Unpaper_ 6.1 (optional)
     - some automatically managed Python packages
-  
+
 .. _ruffus: http://www.ruffus.org.uk/index.html
 .. _parallel: https://www.gnu.org/software/parallel/
 .. _ImageMagick: http://www.imagemagick.org/script/index.php
@@ -754,7 +790,7 @@ Release candidates
     - fixed some installer issues and errors in installation instructions
     - improve performance: run Ghostscript with multithreaded rendering
     - improve performance: use multiple cores by default
-    - bug fix: checking for wrong exception on process timeout 
+    - bug fix: checking for wrong exception on process timeout
 
 -   rc3: skipping version number intentionally to avoid confusion with Tesseract
 -   rc2: first release for public testing to test-PyPI, Github
@@ -777,11 +813,11 @@ where ``settings.txt`` contains *one argument per line*, for example:
 
 ::
 
-    -l 
-    deu 
-    --author 
-    A. Merkel 
-    --pdf-renderer 
+    -l
+    deu
+    --author
+    A. Merkel
+    --pdf-renderer
     tesseract
 
 

requirements.txt

@@ -1,10 +1,11 @@
 # requirements.txt can be used to replicate the developer's build environment
 # setup.py lists a separate set of requirements that are looser to simplify
 # installation
-ruffus == 2.6.3
-Pillow == 5.1.0
+ruffus == 2.7.0
+Pillow == 5.2.0
 reportlab == 3.4.0
 PyPDF2 == 1.26.0
 img2pdf == 0.2.4
 cffi == 1.11.5
-PyMuPDF == 1.12.5
+PyMuPDF == 1.12.5
+defusedxml == 0.5.0

setup.py

@@ -215,6 +215,7 @@ setup(
     classifiers=[
         "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
         "Development Status :: 5 - Production/Stable",
         "Environment :: Console",
         "Intended Audience :: End Users/Desktop",
@@ -243,12 +244,12 @@ setup(
         'cffi >= 1.9.1',          # must be a setup and install requirement
         'defusedxml >= 0.5.0',    # pure Python, so track HEAD closely
         'img2pdf >= 0.2.4',       # pure Python, so track HEAD closely
-        'Pillow >= 4.0.0, != 5.1.0 ; sys_platform == "darwin"',        
+        'Pillow >= 4.0.0, != 5.1.0 ; sys_platform == "darwin"',
                                   # Pillow < 4 has BytesIO/TIFF bug w/img2pdf 0.2.3
                                   # block 5.1.0, broken wheels
         'PyPDF2 >= 1.26',         # pure Python, so track HEAD closely
         'reportlab >= 3.3.0',     # oldest released version with sane image handling
-        'ruffus == 2.6.3',        # pinned - ocrmypdf implements a 2.6.3 workaround
+        'ruffus >= 2.7.0',
     ],
     extras_require={
         'fitz': ['PyMuPDF >= 1.12.5']     # for table of contents bug

src/ocrmypdf/__main__.py

@@ -70,7 +70,7 @@ def complain(message):
 if 'IDE_PROJECT_ROOTS' in os.environ:
     os.environ['PATH'] = '/usr/local/bin:' + os.environ['PATH']
 
-# -------- 
+# --------
 # Critical environment tests
 
 verify_python3_env()
@@ -85,6 +85,20 @@ if tesseract.version() < MINIMUM_TESS_VERSION:
 # -------------
 # Parser
 
+def numeric(basetype, min_=None, max_=None):
+    "Validator for numeric params"
+    min_ = basetype(min_) if min_ is not None else None
+    max_ = basetype(max_) if max_ is not None else None
+    def _numeric(string):
+        value = basetype(string)
+        if (min_ is not None and value < min_
+                or max_ is not None and value > max_):
+            msg = "%r not in valid range %r" % (string, (min_, max_))
+            raise argparse.ArgumentTypeError(msg)
+        return value
+    return _numeric
+
+
 parser = argparse.ArgumentParser(
     prog=PROGRAM_NAME,
     fromfile_prefix_chars='@',
@@ -156,14 +170,15 @@ parser.add_argument(
     '--image-dpi', metavar='DPI', type=int,
     help="For input image instead of PDF, use this DPI instead of file's.")
 parser.add_argument(
-    '--output-type', choices=['pdfa', 'pdf', 'pdfa-1', 'pdfa-2'], 
+    '--output-type', choices=['pdfa', 'pdf', 'pdfa-1', 'pdfa-2', 'pdfa-3'],
     default='pdfa',
     help="Choose output type. 'pdfa' creates a PDF/A-2b compliant file for "
          "long term archiving (default, recommended) but may not suitable "
          "for users who want their file altered as little as possible. 'pdfa' "
          "also has problems with full Unicode text. 'pdf' attempts to "
          "preserve file contents as much as possible. 'pdf-a1' creates a "
-         "PDF/A1-b file. 'pdf-a2' is equivalent to 'pdfa'."
+         "PDF/A1-b file. 'pdf-a2' is equivalent to 'pdfa'. 'pdf-a3' creates a "
+         "PDF/A3-b file."
          )
 
 # Use null string '\0' as sentinel to indicate the user supplied no argument,
@@ -232,7 +247,7 @@ preprocessing.add_argument(
     help="Clean page as above, and incorporate the cleaned image in the final "
          "PDF.  Might remove desired content.")
 preprocessing.add_argument(
-    '--oversample', metavar='DPI', type=int, default=0,
+    '--oversample', metavar='DPI', type=numeric(int, 0, 5000), default=0,
     help="Oversample images to at least the specified DPI, to improve OCR "
          "results slightly")
 
@@ -254,7 +269,7 @@ ocrsettings.add_argument(
 #          "pages")
 
 ocrsettings.add_argument(
-    '--skip-big', type=float, metavar='MPixels',
+    '--skip-big', type=numeric(float, 0, 5000), metavar='MPixels',
     help="Skip OCR on pages larger than the specified amount of megapixels, "
          "but include skipped pages in final output")
 
@@ -262,7 +277,7 @@ advanced = parser.add_argument_group(
     "Advanced",
     "Advanced options to control Tesseract's OCR behavior")
 advanced.add_argument(
-    '--max-image-mpixels', action='store', type=float, metavar='MPixels',
+    '--max-image-mpixels', action='store', type=numeric(float, 0), metavar='MPixels',
     help="Set maximum number of pixels to unpack before treating an image as a "
          "decompression bomb",
     default=128.0)
@@ -295,11 +310,11 @@ advanced.add_argument(
          " of Ghostscript; deprecated"
     )
 advanced.add_argument(
-    '--tesseract-timeout', default=180.0, type=float, metavar='SECONDS',
+    '--tesseract-timeout', default=180.0, type=numeric(float, 0), metavar='SECONDS',
     help='Give up on OCR after the timeout, but copy the preprocessed page '
          'into the final output')
 advanced.add_argument(
-    '--rotate-pages-threshold', default=14.0, type=float, metavar='CONFIDENCE',
+    '--rotate-pages-threshold', default=14.0, type=numeric(float, max_=1000), metavar='CONFIDENCE',
     help="Only rotate pages when confidence is above this value (arbitrary "
          "units reported by tesseract)")
 advanced.add_argument(
@@ -394,6 +409,14 @@ def check_options_output(options, log):
         log.info(
             "Ignoring --debug-rendering because it requires --pdf-renderer=hocr")
 
+    if options.output_type == 'pdfa':
+        options.output_type = 'pdfa-2'
+
+    if options.output_type == 'pdfa-3' and ghostscript.version() < '9.19':
+        raise MissingDependencyError(
+            "--output-type pdfa-3 requires Ghostscript 9.19 or later"
+        )
+
     lossless_reconstruction = False
     if options.pdf_renderer in ('hocr', 'sandwich'):
         if not any((options.deskew, options.clean_final, options.force_ocr,
@@ -482,6 +505,10 @@ def check_options_advanced(options, log):
             "--output-type is not 'pdfa', 'pdfa-1', or 'pdfa-2'"
         )
 
+    if tesseract.v4() and (options.user_words or options.user_patterns):
+        log.warning(
+            'Tesseract 4.x ignores --user-words, so this has no effect')
+
 
 def check_options_metadata(options, log):
     import unicodedata
@@ -565,7 +592,7 @@ def do_ruffus_exception(ruffus_five_tuple, options, log):
     if exc_name == 'builtins.SystemExit':
         match = re.search(r"\.(.+?)\)", exc_value)
         exit_code_name = match.groups()[0]
-        exit_code = getattr(ExitCode, exit_code_name, 'other_error')        
+        exit_code = getattr(ExitCode, exit_code_name, 'other_error')
     elif exc_name == 'ruffus.ruffus_exceptions.MissingInputFileError':
         log.error(cleanup_ruffus_error_message(exc_value))
         exit_code = ExitCode.input_file
@@ -589,7 +616,7 @@ def do_ruffus_exception(ruffus_five_tuple, options, log):
             (exc_name == 'ocrmypdf.exceptions.EncryptedPdfError'):
         log.error(textwrap.dedent("""\
             Input PDF is encrypted. The encryption must be removed to
-            perform OCR. 
+            perform OCR.
 
             For information about this PDF's security use
                 qpdf --show-encryption infilename
@@ -598,7 +625,7 @@ def do_ruffus_exception(ruffus_five_tuple, options, log):
                 qpdf --decrypt [--password=[password]] infilename
 
             """))
-        exit_code = ExitCode.encrypted_pdf        
+        exit_code = ExitCode.encrypted_pdf
     elif exc_name == 'ocrmypdf.exceptions.PdfMergeFailedError':
         log.error(textwrap.dedent("""\
             Failed to merge PDF image layer with OCR layer
@@ -629,33 +656,31 @@ def do_ruffus_exception(ruffus_five_tuple, options, log):
     return ExitCode.other_error
 
 
-def traverse_ruffus_exception(e_args, options, log):
-    """Walk through a RethrownJobError and find the first exception.
+def traverse_ruffus_exception(exceptions, options, log):
+    """Traverse a RethrownJobError and output the exceptions
 
-    Ruffus flattens exception to 5 element tuples. Because of a bug
-    in <= 2.6.3 it may present either the single:
-      (task, job, exc, value, stack)
-    or something like:
-      [[(task, job, exc, value, stack)]]
-    
-    Generally cross-process exception marshalling doesn't work well
-    and ruffus doesn't support because BaseException has its own
-    implementation of __reduce__ that attempts to reconstruct the
-    exception based on e.__init__(e.args).
-    
-    Attempting to log the exception directly marshalls it to the logger
-    which is probably in another process, so it's better to log only
-    data from the exception at this point.
+    Ruffus presents exceptions as 5 element tuples. The RethrownJobException
+    has a list of exceptions like
+        e.job_exceptions = [(5-tuple), (5-tuple), ...]
+
+    ruffus < 2.7.0 had a bug with exception marshalling that would give
+    different output whether the main or child process raised the exception.
+    We no longer support this.
+
+    Attempting to log the exception itself will re-marshall it to the logger
+    which is normally running in another process. It's better to avoid re-
+    marshalling.
 
     The exit code will be based on this, even if multiple exceptions occurred
     at the same time."""
 
-    if isinstance(e_args, Sequence) and isinstance(e_args[0], str) and \
-            len(e_args) == 5:
-        return do_ruffus_exception(e_args, options, log)
-    elif is_iterable_notstr(e_args):
-        for exc in e_args:
-            return traverse_ruffus_exception(exc, options, log)
+    exit_codes = []
+    for exc in exceptions:
+        exit_code = do_ruffus_exception(exc, options, log)
+        exit_codes.append(exit_code)
+
+    return exit_codes[0]  # Multiple codes are rare so take the first one
+
 
 
 def check_closed_streams(options):
@@ -740,7 +765,7 @@ def check_environ(options, _log):
     for k in old_envvars:
         if k in os.environ:
             _log.warning(textwrap.dedent("""\
-                OCRmyPDF no longer uses the environment variable {}. 
+                OCRmyPDF no longer uses the environment variable {}.
                 Change PATH to select alternate programs.""".format(k)))
 
 
@@ -783,12 +808,24 @@ def report_output_file_size(options, _log, input_file, output_file):
     ratio = output_size / input_size
     if ratio < 1.35 or input_size < 25000:
         return  # Seems fine
-    
+
     reasons = []
     if not fitz:
         reasons.append("The optional dependency PyMuPDF is not installed.")
-    if options.force_ocr:
-        reasons.append("The argument --force-ocr was issued.")
+    image_preproc = {
+        'deskew',
+        'clean_final',
+        'remove_background',
+        'oversample',
+        'force_ocr'
+    }
+    for arg in image_preproc:
+        attr = getattr(options, arg, None)
+        if not attr:
+            continue
+        reasons.append(
+                "The argument --{} was issued, causing transcoding.".format(
+                        arg.replace('_', '-')))
 
     if reasons:
         explanation = (
@@ -865,7 +902,8 @@ def run_pipeline():
     except ruffus_exceptions.RethrownJobError as e:
         if options.verbose:
             _log.debug(str(e))  # stringify exception so logger doesn't have to
-        exitcode = traverse_ruffus_exception(e.args, options, _log)
+        exceptions = e.job_exceptions
+        exitcode = traverse_ruffus_exception(exceptions, options, _log)
         if exitcode is None:
             _log.error("Unexpected ruffus exception: " + str(e))
             _log.error(repr(e))
@@ -898,7 +936,7 @@ def run_pipeline():
             _log.warning('Output file: The generated PDF is INVALID')
             return ExitCode.invalid_output_pdf
 
-        report_output_file_size(options, _log, start_input_file, 
+        report_output_file_size(options, _log, start_input_file,
                                 options.output_file)
 
     pdfinfo = context.get_pdfinfo()

src/ocrmypdf/_unicodefun.py

@@ -40,6 +40,11 @@ import codecs
 
 def verify_python3_env():
     """Ensures that the environment is good for unicode on Python 3."""
+
+    # PEP 538 changes in Python 3.7 should make this wrangling unnecessary
+    if sys.version_info[0:3] >= (3, 7, 0):
+        return
+
     try:
         import locale
         fs_enc = codecs.lookup(locale.getpreferredencoding()).name

src/ocrmypdf/exec/__init__.py

@@ -25,7 +25,7 @@ from ..exceptions import MissingDependencyError
 
 
 
-def get_version(program, *, 
+def get_version(program, *,
         version_arg='--version', regex=r'(\d+(\.\d+)*)'):
     "Get the version of the specified program"
     args_prog = [
@@ -37,6 +37,10 @@ def get_version(program, *,
             args_prog, close_fds=True, universal_newlines=True,
             stdout=PIPE, stderr=STDOUT, check=True)
         output = proc.stdout
+    except FileNotFoundError as e:
+        raise MissingDependencyError(
+            "Could not find program '{}' on the PATH".format(
+                program)) from e
     except CalledProcessError as e:
         if e.returncode < 0:
             raise MissingDependencyError(

src/ocrmypdf/exec/ghostscript.py

@@ -32,6 +32,18 @@ def version():
     return get_version('gs')
 
 
+def jpeg_passthrough_available():
+    """
+    Ghostscript 9.23 introduced JPEG passthrough but it seems to corrupt the
+    last two bytes of certain images, for now we disable it for 9.23 and
+    do not mention it for < 9.23.
+
+    https://bugs.ghostscript.com/show_bug.cgi?id=699216
+
+    """
+    return False
+
+
 def _gs_error_reported(stream):
     return re.search(r'error', stream, flags=re.IGNORECASE)
 
@@ -132,6 +144,12 @@ def generate_pdfa(pdf_pages, output_file, compression, log,
     # git commit fe1c025d.
     strategy = 'RGB' if version() >= '9.19' else '/RGB'
 
+    if version() == '9.23':
+        # 9.23: new feature JPEG passthrough is broken in some cases, best to
+        # disable it always
+        # https://bugs.ghostscript.com/show_bug.cgi?id=699216
+        compression_args.append('-dPassThroughJPEGImages=false')
+
     with NamedTemporaryFile(delete=True) as gs_pdf:
         args_gs = [
             "gs",

src/ocrmypdf/exec/tesseract.py

@@ -73,7 +73,8 @@ def has_textonly_pdf():
     """
     args_tess = [
         'tesseract',
-        '--print-parameters'
+        '--print-parameters',
+        'pdf'
     ]
     params = ''
     try:
@@ -159,7 +160,7 @@ def get_orientation(input_file, language: list, engine_mode, timeout: float,
             assert 'Rotate' not in osd
             angle = -angle % 360
         else:
-            # Tesseract == 3.04.01, hopefully also Tesseract > 3.04.01
+            # Tesseract >= 3.04.01
             # reports "Orientation in degrees" as a clockwise angle
             assert 'Rotate' in osd
 

src/ocrmypdf/helpers.py

@@ -22,6 +22,7 @@ from pathlib import Path
 import sys
 import os
 import multiprocessing
+import warnings
 
 
 def re_symlink(input_file, soft_link_name, log=None):
@@ -70,7 +71,7 @@ def is_iterable_notstr(thing):
 
 def page_number(input_file):
     "Get one-based page number implied by filename (000002.pdf -> 2)"
-    return int(os.path.basename(input_file)[0:6])
+    return int(os.path.basename(fspath(input_file))[0:6])
 
 
 def available_cpu_count():
@@ -85,7 +86,7 @@ def available_cpu_count():
     except (ImportError, AttributeError):
         pass
 
-    complain(
+    warnings.warn(
         "Could not get CPU count.  Assuming one (1) CPU."
         "Use -j N to set manually.")
     return 1

src/ocrmypdf/hocrtransform.py

@@ -322,8 +322,11 @@ class HocrTransform():
             dy = baseline_y2 - cursor[1]            
             text.moveCursor(dx, dy)
 
-            text.setHorizScale(100 * box_width / font_width)
-            text.textOut(elemtxt)
+            # If reportlab tells us this word is 0 units wide, our best seems
+            # to be to suppress this text
+            if font_width > 0:
+                text.setHorizScale(100 * box_width / font_width)
+                text.textOut(elemtxt)
         pdf.drawText(text)
 
 

src/ocrmypdf/leptonica.py

@@ -108,13 +108,6 @@ class LeptonicaIOError(LeptonicaError):
     pass
 
 
-class RemoveColormap(Enum):
-    to_binary = 0
-    to_grayscale = 1
-    to_full_color = 2
-    based_on_src = 3
-
-
 class Pix:
     """Wrapper around leptonica's PIX object.
 
@@ -141,9 +134,10 @@ class Pix:
 
     def __repr__(self):
         if self._pix:
-            s = "<leptonica.Pix image size={0}x{1} depth={2} at 0x{3:x}>"
+            s = "<leptonica.Pix image size={0}x{1} depth={2}{4} at 0x{3:x}>"
             return s.format(self._pix.w, self._pix.h, self._pix.d,
-                            int(ffi.cast("intptr_t", self._pix)))
+                            int(ffi.cast('intptr_t', self._pix)),
+                            '(colormapped)' if self._pix.colormap else '')
         else:
             return "<leptonica.Pix image NULL>"
 
@@ -328,13 +322,16 @@ class Pix:
             return None
 
     def remove_colormap(self, removal_type):
-        """Remove a palette
+        """Remove a palette (colormap); if no colormap, returns a copy of this 
+        image
+
+            removal_type - any of lept.REMOVE_CMAP_*
 
-            removal_type - RemovalColormap()
         """
-
-        with LeptonicaErrorTrap():
-            return Pix(lept.pixRemoveColormap(self._pix, removal_type))
+        return self
+        # with LeptonicaErrorTrap():
+        #     return Pix(lept.pixRemoveColormapGeneral(
+        #             self._pix, removal_type, lept.L_COPY))
 
     def otsu_adaptive_threshold(
             self, tile_size=(300, 300), kernel_size=(4, 4), scorefract=0.1):
@@ -426,9 +423,11 @@ class Pix:
     def background_norm(
             self, mask=None, grayscale=None, tile_size=(10, 15), fg_threshold=60,
             min_count=40, bg_val=200, smooth_kernel=(2, 1)):
+        # Background norm doesn't work on color mapped Pix, so remove colormap
+        target_pix = self.remove_colormap(lept.REMOVE_CMAP_BASED_ON_SRC)
         with LeptonicaErrorTrap():
             return Pix(lept.pixBackgroundNorm(
-                self._pix,
+                target_pix._pix,
                 mask or ffi.NULL,
                 grayscale or ffi.NULL,
                 tile_size[0],
@@ -450,8 +449,8 @@ class Pix:
         if get_leptonica_version() < 'leptonica-1.72':
             # Older versions of Leptonica (pre-1.72) have a buggy
             # implementation of pixCorrelationBinary that overflows on larger
-            # images.  Ubuntu trusty has 1.70. Ubuntu PPA
-            # ppa:rebuntu16/avidemux+unofficial has "leptonlib" 1.73.
+            # images.  Ubuntu 14.04/trusty has 1.70. Ubuntu PPA
+            # ppa:alex-p/tesseract-ocr has leptonlib 1.75.
             pix1_count = ffi.new('l_int32 *')
             pix2_count = ffi.new('l_int32 *')
             pixn_count = ffi.new('l_int32 *')

src/ocrmypdf/lib/_leptonica.py

@@ -3,8 +3,9 @@ import _cffi_backend
 
 ffi = _cffi_backend.FFI('ocrmypdf.lib._leptonica',
     _version = 0x2601,
-    _types = b'\x00\x00\x0F\x0D\x00\x00\xBB\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xBE\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\xBA\x03\x00\x00\x0F\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x0D\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x0D\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x0D\x01\x00\x00\x02\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x0D\x01\x00\x00\x0D\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x08\x11\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\xC7\x03\x00\x00\x1C\x01\x00\x00\x00\x0F\x00\x00\xBD\x0D\x00\x00\x00\x0F\x00\x00\x49\x0D\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x1F\x03\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x6E\x11\x00\x00\x6E\x11\x00\x00\x6E\x11\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x01\x11\x00\x00\x6E\x11\x00\x00\x6E\x11\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x01\x11\x00\x00\x49\x11\x00\x00\x49\x11\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x0D\x01\x00\x00\x07\x01\x00\x00\x49\x11\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x6E\x11\x00\x00\x6E\x11\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x0D\x01\x00\x00\x01\x03\x00\x00\x9D\x11\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x01\x11\x00\x00\xC6\x03\x00\x00\x65\x03\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x08\x11\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\xC4\x03\x00\x00\xA3\x11\x00\x00\x01\x11\x00\x00\x0D\x01\x00\x00\x00\x0F\x00\x00\xCA\x0D\x00\x00\x10\x11\x00\x00\x00\x0F\x00\x00\xCA\x0D\x00\x00\x9D\x11\x00\x00\x00\x0F\x00\x00\xCA\x0D\x00\x00\xCA\x03\x00\x00\x00\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x02\x09\x00\x00\xBE\x03\x00\x00\x02\x01\x00\x00\x0E\x01\x00\x00\x0B\x01\x00\x00\x05\x01\x00\x00\x03\x01\x00\x00\xBC\x03\x00\x00\xC5\x03\x00\x00\x04\x01\x00\x00\xC7\x03\x00\x00\x08\x01\x00\x00\x0C\x01\x00\x00\x06\x01\x00\x00\x00\x01',
-    _globals = (b'\x00\x00\xB1\x23boxDestroy',0,b'\x00\x00\x67\x23getLeptonicaVersion',0,b'\x00\x00\xB7\x23lept_free',0,b'\x00\x00\x69\x23makePixelSumTab8',0,b'\x00\x00\x16\x23pixAnd',0,b'\x00\x00\x23\x23pixBackgroundNorm',0,b'\x00\x00\x1B\x23pixCleanBackgroundToWhite',0,b'\x00\x00\x0D\x23pixClipRectangle',0,b'\x00\x00\x8D\x23pixColorFraction',0,b'\x00\x00\x59\x23pixColorMagnitude',0,b'\x00\x00\x0A\x23pixConvertRGBToLuminance',0,b'\x00\x00\x6B\x23pixCorrelationBinary',0,b'\x00\x00\x80\x23pixCountPixels',0,b'\x00\x00\x63\x23pixDeserializeFromMemory',0,b'\x00\x00\x50\x23pixDeskew',0,b'\x00\x00\xB4\x23pixDestroy',0,b'\x00\x00\x0A\x23pixEndianByteSwapNew',0,b'\x00\x00\x00\x23pixFindPageForeground',0,b'\x00\x00\x7B\x23pixFindSkew',0,b'\x00\x00\x2F\x23pixGammaTRC',0,b'\x00\x00\x70\x23pixGetAverageMaskedRGB',0,b'\x00\x00\x36\x23pixGlobalNormRGB',0,b'\x00\x00\x12\x23pixInvert',0,b'\x00\x00\x54\x23pixMaskOverColorPixels',0,b'\x00\x00\x85\x23pixNumSignificantGrayColors',0,b'\x00\x00\x96\x23pixOtsuAdaptiveThreshold',0,b'\x00\x00\x3E\x23pixOtsuThreshOnBackgroundNorm',0,b'\x00\x00\x60\x23pixRead',0,b'\x00\x00\x50\x23pixRemoveColormap',0,b'\x00\x00\x12\x23pixRotate180',0,b'\x00\x00\x50\x23pixRotateOrth',0,b'\x00\x00\x4B\x23pixScale',0,b'\x00\x00\xA0\x23pixSerializeToMemory',0,b'\x00\x00\xA5\x23pixWriteImpliedFormat',0,b'\x00\x00\xAB\x23pixWriteMemPng',0),
-    _struct_unions = ((b'\x00\x00\x00\xBA\x00\x00\x00\x02Box',b'\x00\x00\x02\x11x',b'\x00\x00\x02\x11y',b'\x00\x00\x02\x11w',b'\x00\x00\x02\x11h',b'\x00\x00\xC7\x11refcount'),(b'\x00\x00\x00\xBB\x00\x00\x00\x02Pix',b'\x00\x00\xC7\x11w',b'\x00\x00\xC7\x11h',b'\x00\x00\xC7\x11d',b'\x00\x00\xC7\x11spp',b'\x00\x00\xC7\x11wpl',b'\x00\x00\xC7\x11refcount',b'\x00\x00\x02\x11xres',b'\x00\x00\x02\x11yres',b'\x00\x00\x02\x11informat',b'\x00\x00\x02\x11special',b'\x00\x00\xBD\x11text',b'\x00\x00\xC3\x11colormap',b'\x00\x00\xC6\x11data'),(b'\x00\x00\x00\xBC\x00\x00\x00\x02PixColormap',b'\x00\x00\xB8\x11array',b'\x00\x00\x02\x11depth',b'\x00\x00\x02\x11nalloc',b'\x00\x00\x02\x11n')),
-    _typenames = (b'\x00\x00\x00\xBABOX',b'\x00\x00\x00\xBBPIX',b'\x00\x00\x00\xBCPIXCMAP',b'\x00\x00\x00\x1Fl_float32',b'\x00\x00\x00\xBFl_float64',b'\x00\x00\x00\xC1l_int16',b'\x00\x00\x00\x02l_int32',b'\x00\x00\x00\xC0l_int64',b'\x00\x00\x00\xC2l_int8',b'\x00\x00\x00\xC9l_uint16',b'\x00\x00\x00\xC7l_uint32',b'\x00\x00\x00\xC8l_uint64',b'\x00\x00\x00\xC5l_uint8'),
+    _types = b'\x00\x00\x0F\x0D\x00\x00\xBB\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xBE\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\xBA\x03\x00\x00\x0F\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x0D\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x0D\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x0D\x01\x00\x00\x02\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x0D\x01\x00\x00\x0D\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x08\x11\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\xC9\x03\x00\x00\x1C\x01\x00\x00\x00\x0F\x00\x00\xBD\x0D\x00\x00\x00\x0F\x00\x00\x49\x0D\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x1F\x03\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x01\x11\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x6E\x11\x00\x00\x6E\x11\x00\x00\x6E\x11\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x01\x11\x00\x00\x6E\x11\x00\x00\x6E\x11\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x01\x11\x00\x00\x49\x11\x00\x00\x49\x11\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x0D\x01\x00\x00\x07\x01\x00\x00\x49\x11\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x6E\x11\x00\x00\x6E\x11\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x0D\x01\x00\x00\x01\x03\x00\x00\x9D\x11\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x01\x11\x00\x00\xC8\x03\x00\x00\x65\x03\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\x08\x11\x00\x00\x01\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x02\x0D\x00\x00\xC6\x03\x00\x00\xA3\x11\x00\x00\x01\x11\x00\x00\x0D\x01\x00\x00\x00\x0F\x00\x00\xCC\x0D\x00\x00\x10\x11\x00\x00\x00\x0F\x00\x00\xCC\x0D\x00\x00\x9D\x11\x00\x00\x00\x0F\x00\x00\xCC\x0D\x00\x00\xCC\x03\x00\x00\x00\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x02\x09\x00\x00\xBE\x03\x00\x00\x02\x01\x00\x00\x0E\x01\x00\x00\x00\x0B\x00\x00\x01\x0B\x00\x00\x0B\x01\x00\x00\x05\x01\x00\x00\x03\x01\x00\x00\xBC\x03\x00\x00\xC7\x03\x00\x00\x04\x01\x00\x00\xC9\x03\x00\x00\x08\x01\x00\x00\x0C\x01\x00\x00\x06\x01\x00\x00\x00\x01',
+    _globals = (b'\xFF\xFF\xFF\x0BL_CLONE',2,b'\xFF\xFF\xFF\x0BL_COPY',1,b'\xFF\xFF\xFF\x0BL_COPY_CLONE',3,b'\xFF\xFF\xFF\x0BL_INSERT',0,b'\xFF\xFF\xFF\x0BL_NOCOPY',0,b'\xFF\xFF\xFF\x0BREMOVE_CMAP_BASED_ON_SRC',4,b'\xFF\xFF\xFF\x0BREMOVE_CMAP_TO_BINARY',0,b'\xFF\xFF\xFF\x0BREMOVE_CMAP_TO_FULL_COLOR',2,b'\xFF\xFF\xFF\x0BREMOVE_CMAP_TO_GRAYSCALE',1,b'\xFF\xFF\xFF\x0BREMOVE_CMAP_WITH_ALPHA',3,b'\x00\x00\xB1\x23boxDestroy',0,b'\x00\x00\x67\x23getLeptonicaVersion',0,b'\x00\x00\xB7\x23lept_free',0,b'\x00\x00\x69\x23makePixelSumTab8',0,b'\x00\x00\x16\x23pixAnd',0,b'\x00\x00\x23\x23pixBackgroundNorm',0,b'\x00\x00\x1B\x23pixCleanBackgroundToWhite',0,b'\x00\x00\x0D\x23pixClipRectangle',0,b'\x00\x00\x8D\x23pixColorFraction',0,b'\x00\x00\x59\x23pixColorMagnitude',0,b'\x00\x00\x0A\x23pixConvertRGBToLuminance',0,b'\x00\x00\x6B\x23pixCorrelationBinary',0,b'\x00\x00\x80\x23pixCountPixels',0,b'\x00\x00\x63\x23pixDeserializeFromMemory',0,b'\x00\x00\x50\x23pixDeskew',0,b'\x00\x00\xB4\x23pixDestroy',0,b'\x00\x00\x0A\x23pixEndianByteSwapNew',0,b'\x00\x00\x00\x23pixFindPageForeground',0,b'\x00\x00\x7B\x23pixFindSkew',0,b'\x00\x00\x2F\x23pixGammaTRC',0,b'\x00\x00\x70\x23pixGetAverageMaskedRGB',0,b'\x00\x00\x36\x23pixGlobalNormRGB',0,b'\x00\x00\x12\x23pixInvert',0,b'\x00\x00\x54\x23pixMaskOverColorPixels',0,b'\x00\x00\x85\x23pixNumSignificantGrayColors',0,b'\x00\x00\x96\x23pixOtsuAdaptiveThreshold',0,b'\x00\x00\x3E\x23pixOtsuThreshOnBackgroundNorm',0,b'\x00\x00\x60\x23pixRead',0,b'\x00\x00\x50\x23pixRemoveColormap',0,b'\x00\x00\x54\x23pixRemoveColormapGeneral',0,b'\x00\x00\x12\x23pixRotate180',0,b'\x00\x00\x50\x23pixRotateOrth',0,b'\x00\x00\x4B\x23pixScale',0,b'\x00\x00\xA0\x23pixSerializeToMemory',0,b'\x00\x00\xA5\x23pixWriteImpliedFormat',0,b'\x00\x00\xAB\x23pixWriteMemPng',0),
+    _struct_unions = ((b'\x00\x00\x00\xBA\x00\x00\x00\x02Box',b'\x00\x00\x02\x11x',b'\x00\x00\x02\x11y',b'\x00\x00\x02\x11w',b'\x00\x00\x02\x11h',b'\x00\x00\xC9\x11refcount'),(b'\x00\x00\x00\xBB\x00\x00\x00\x02Pix',b'\x00\x00\xC9\x11w',b'\x00\x00\xC9\x11h',b'\x00\x00\xC9\x11d',b'\x00\x00\xC9\x11spp',b'\x00\x00\xC9\x11wpl',b'\x00\x00\xC9\x11refcount',b'\x00\x00\x02\x11xres',b'\x00\x00\x02\x11yres',b'\x00\x00\x02\x11informat',b'\x00\x00\x02\x11special',b'\x00\x00\xBD\x11text',b'\x00\x00\xC5\x11colormap',b'\x00\x00\xC8\x11data'),(b'\x00\x00\x00\xBC\x00\x00\x00\x02PixColormap',b'\x00\x00\xB8\x11array',b'\x00\x00\x02\x11depth',b'\x00\x00\x02\x11nalloc',b'\x00\x00\x02\x11n')),
+    _enums = (b'\x00\x00\x00\xC0\x00\x00\x00\x16$1\x00REMOVE_CMAP_TO_BINARY,REMOVE_CMAP_TO_GRAYSCALE,REMOVE_CMAP_TO_FULL_COLOR,REMOVE_CMAP_WITH_ALPHA,REMOVE_CMAP_BASED_ON_SRC',b'\x00\x00\x00\xC1\x00\x00\x00\x16$2\x00L_NOCOPY,L_INSERT,L_COPY,L_CLONE,L_COPY_CLONE'),
+    _typenames = (b'\x00\x00\x00\xBABOX',b'\x00\x00\x00\xBBPIX',b'\x00\x00\x00\xBCPIXCMAP',b'\x00\x00\x00\x1Fl_float32',b'\x00\x00\x00\xBFl_float64',b'\x00\x00\x00\xC3l_int16',b'\x00\x00\x00\x02l_int32',b'\x00\x00\x00\xC2l_int64',b'\x00\x00\x00\xC4l_int8',b'\x00\x00\x00\xCBl_uint16',b'\x00\x00\x00\xC9l_uint32',b'\x00\x00\x00\xCAl_uint64',b'\x00\x00\x00\xC7l_uint8'),
 )

src/ocrmypdf/lib/compile_leptonica.py

@@ -72,6 +72,24 @@ struct Box
 };
 typedef struct Box    BOX;
 
+enum {
+    REMOVE_CMAP_TO_BINARY = 0,     /*!< remove colormap for conv to 1 bpp  */
+    REMOVE_CMAP_TO_GRAYSCALE = 1,  /*!< remove colormap for conv to 8 bpp  */
+    REMOVE_CMAP_TO_FULL_COLOR = 2, /*!< remove colormap for conv to 32 bpp */
+    REMOVE_CMAP_WITH_ALPHA = 3,    /*!< remove colormap and alpha          */
+    REMOVE_CMAP_BASED_ON_SRC = 4   /*!< remove depending on src format     */
+};
+
+/*! Access and storage flags */
+enum {
+    L_NOCOPY = 0,     /*!< do not copy the object; do not delete the ptr  */
+    L_INSERT = L_NOCOPY,    /*!< stuff it in; do not copy or clone        */
+    L_COPY = 1,       /*!< make/use a copy of the object                  */
+    L_CLONE = 2,      /*!< make/use clone (ref count) of the object       */
+    L_COPY_CLONE = 3  /*!< make a new array object (e.g., pixa) and fill  */
+                      /*!< the array with clones (e.g., pix)              */
+};
+
 """)
 
 ffibuilder.cdef("""
@@ -227,6 +245,11 @@ PIX *
 pixInvert(PIX * pixd,
           PIX * pixs);
 
+PIX *
+pixRemoveColormapGeneral(PIX     *pixs,
+                         l_int32  type,
+                         l_int32  ifnocmap);
+
 void                 
 boxDestroy(BOX  **pbox);
 

src/ocrmypdf/pdfa.py

@@ -125,28 +125,31 @@ def encode_pdf_date(d: datetime) -> str:
     s = d.strftime(pdfmark_date_fmt)
 
     tz = d.strftime('%z')
-    if tz == 'Z':
+    if tz == 'Z' or tz == '':
+        # Ghostscript <= 9.23 handles missing timezones incorrectly, so if
+        # timezone is missing, move it into GMT.
+        # https://bugs.ghostscript.com/show_bug.cgi?id=699182
         s += "+00'00'"
-    elif tz != '':
+    else:
         sign, tz_hours, tz_mins = tz[0], tz[1:3], tz[3:5]
-        s += "{}{}'{tz}'".format(sign, tz_hours, tz_mins)
+        s += "{}{}'{}'".format(sign, tz_hours, tz_mins)
     return s
 
 
 def decode_pdf_date(s: str) -> datetime:
-    pdfmark_date_fmts = (
-        r'%Y%m%d%H%M%S%z',  # +0430 etc
-        r'%Y%m%d%H%M%S',    # no time zone
-        r'%Y%m%d%H%M%SZ')   # trailing Z
-
     if s.startswith('D:'):
         s = s[2:]
-    for fmt in pdfmark_date_fmts:
-        try:
-            return datetime.strptime(s, fmt)
-        except ValueError:
-            continue
-    return None
+
+    # Literal Z00'00', is incorrect but found in the wild, 
+    # probably made by OS X Quartz -- standardize
+    if s.endswith("Z00'00'"):
+        s = s.replace("Z00'00'", '+0000')
+    elif s.endswith('Z'):
+        s = s.replace('Z', '+0000')
+    
+    s = s.replace("'", "")  # Remove apos from PDF time strings
+
+    return datetime.strptime(s, r'%Y%m%d%H%M%S%z')
 
 
 def _get_pdfmark_dates(pdfmark):
@@ -167,7 +170,12 @@ def _get_pdfmark_dates(pdfmark):
         date_str = pdfmark[key]
         if date_str.startswith('D:'):
             date_str = date_str[2:]
-        yield '  {} (D:{})'.format(key, date_str)
+        try:
+            yield '  {} (D:{})'.format(
+                    key, 
+                    encode_pdf_date(decode_pdf_date(date_str)))
+        except ValueError:
+            yield '  {} null'.format(key)
 
 
 def _get_pdfa_def(icc_profile, icc_identifier, pdfmark):

src/ocrmypdf/pipeline.py

@@ -18,7 +18,7 @@
 from contextlib import suppress
 from shutil import copyfileobj
 from pathlib import Path
-from datetime import datetime
+from datetime import datetime, timezone
 import sys
 import os
 import shutil
@@ -222,7 +222,7 @@ def repair_and_parse_pdf(
 
     pdfinfo = PdfInfo(output_file)
 
-    if pdfinfo.has_userunit and options.output_type == 'pdfa':
+    if pdfinfo.has_userunit and options.output_type.startswith('pdfa'):
         log.error(
             "This input file uses a PDF feature that is not supported "
             "by Ghostscript, so you cannot use --output-type=pdfa for this "
@@ -496,17 +496,23 @@ def rasterize_with_ghostscript(
     options = context.get_options()
     pageinfo = get_pageinfo(input_file, context)
 
-    device = 'png16m'  # 24-bit
-    if pageinfo.images:
-        if all(image.comp == 1 for image in pageinfo.images):
-            if all(image.bpc == 1 for image in pageinfo.images):
-                device = 'pngmono'
-            elif all(image.bpc > 1 and image.color == Colorspace.index
-                     for image in pageinfo.images):
-                device = 'png256'
-            elif all(image.bpc > 1 and image.color == Colorspace.gray
-                     for image in pageinfo.images):
-                device = 'pnggray'
+    colorspaces = ['pngmono', 'pnggray', 'png256', 'png16m']
+    device_idx = 0
+    def at_least(cs):
+        return max(device_idx, colorspaces.index(cs))
+
+    for image in pageinfo.images:
+        if image.type_ != 'image':
+            continue  # ignore masks
+        if image.bpc > 1:
+            if image.color == Colorspace.index:
+                device_idx = at_least('png256')
+            elif image.color == Colorspace.gray:
+                device_idx = at_least('pnggray')
+            else:
+                device_idx = at_least('png16m')
+
+    device = colorspaces[device_idx]
 
     log.debug("Rasterize {0} with {1}".format(
               os.path.basename(input_file), device))
@@ -900,7 +906,7 @@ def get_pdfmark(base_pdf, options):
         PROGRAM_NAME, VERSION,
         renderer_tag,
         tesseract.version())
-    pdfmark['/ModDate'] = encode_pdf_date(datetime.utcnow())
+    pdfmark['/ModDate'] = encode_pdf_date(datetime.now(timezone.utc))
     return pdfmark
 
 
@@ -970,6 +976,7 @@ def merge_pages_ghostscript(
     pdf_pages, _ = _merge_pages_common(
         input_files_groups, output_file, log, context)
     input_pdfinfo = context.get_pdfinfo()
+
     ghostscript.generate_pdfa(
         pdf_version=input_pdfinfo.min_version,
         pdf_pages=pdf_pages,
@@ -977,7 +984,7 @@ def merge_pages_ghostscript(
         compression=options.pdfa_image_compression,
         log=log,
         threads=options.jobs or 1,
-        pdfa_part=('1' if options.output_type == 'pdfa-1' else '2'))
+        pdfa_part=options.output_type[-1])  # is pdfa-1, pdfa-2, or pdfa-3
     if fitz:
         doc = fitz.Document(output_file + '_toc.pdf')
         doc.setToC(input_pdfinfo.table_of_contents)

test_requirements.txt

@@ -2,4 +2,4 @@ pytest >= 3.2
 pytest-helpers-namespace
 pytest-xdist
 pytest-cov
-pytest-timeout
+pytest-timeout == 1.2.1

tests/resources/README.rst

@@ -110,6 +110,7 @@ licensed under the specified license.
         - Wikipedia authors
         - CC-BY-SA 3.0
     *   - missing_docinfo.pdf
+        - @jbarlow83
         - @jbarlow83
         - PDF file with no /DocumentInfo section 
         - CC-BY-SA 4.0
@@ -117,7 +118,7 @@ licensed under the specified license.
         - PDF file generated by PDFPen pro that triggered content stream parse errors
         - @maxandersen
         - @maxandersen
-        - MIT
+        - CC-BY-SA 4.0
     *   - negzero.pdf
         - copy of formxobject.pdf with token that qpdf doesn't like
         - @jbarlow83
@@ -137,7 +138,7 @@ licensed under the specified license.
         - a PDF with vector art and text rendered as curves with no fonts
         - @Catscratch
         - @Catscratch
-        - MIT
+        - CC-BY-SA 4.0
 
 
 Assemblies
@@ -146,6 +147,7 @@ Assemblies
 These test resources are assemblies or derivatives from other previously mentioned files, released under the same license terms as their input files.
 
 - baiona_gray.png (from baiona.png)
+- baiona_colormapped.png (from baiona.png)
 - cardinal.pdf (four cardinal directions, baked-in rotated copies of LinnSequencer.jpg)
 - ccitt.pdf (LinnSequencer.jpg, converted to CCITT encoding)
 - encrypted_algo4.pdf (congress.jpg, encrypted with algorithm 4 - not supported by PyPDF2)

tests/spoof/tesseract_cache.py

@@ -106,6 +106,8 @@ def main():
     source = os.environ['_OCRMYPDF_TEST_INFILE']  # required
     args = parser.parse_args()
 
+    cache_disabled = os.environ.get('_OCRMYPDF_CACHE_DISABLED', False)
+
     if args.imagename == 'stdin':
         real_tesseract()
 
@@ -128,7 +130,7 @@ def main():
     print("Tesseract cache folder {} - ".format(cache_folder), end='', 
           file=sys.stderr)
 
-    if (cache_folder / 'stderr.bin').exists():
+    if (cache_folder / 'stderr.bin').exists() and not cache_disabled:
         # Cache hit
         print("HIT", file=sys.stderr)
 

tests/test_lept.py

@@ -0,0 +1,30 @@
+# © 2018 James R. Barlow: github.com/jbarlow83
+#
+# This file is part of OCRmyPDF.
+#
+# OCRmyPDF is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# OCRmyPDF is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import os
+import shutil
+import pytest
+import sys
+import ocrmypdf.leptonica as lept
+
+
+def test_colormap_backgroundnorm(resources):
+    # Issue #262 - unclear how to reproduce exactly, so just ensure leptonica
+    # can handle that case
+    pix = lept.Pix.read(resources / 'baiona-colormapped.png')
+    pix.background_norm()

tests/test_main.py

@@ -300,7 +300,8 @@ def test_autorotate_threshold(
 @pytest.mark.parametrize('renderer',RENDERERS)
 def test_ocr_timeout(renderer, resources, outpdf):
     out = check_ocrmypdf(resources / 'skew.pdf', outpdf,
-                         '--tesseract-timeout', '1.0')
+                         '--tesseract-timeout', '0.01',
+                         '--pdf-renderer', renderer)
     pdfinfo = PdfInfo(out)
     assert not pdfinfo[0].has_text
 
@@ -472,7 +473,7 @@ def test_tesseract_image_too_big(renderer, spoof_tesseract_big_image_error,
                                  resources, outpdf):
     check_ocrmypdf(
         resources / 'hugemono.pdf', outpdf, '-r',
-        '--pdf-renderer', renderer, 
+        '--pdf-renderer', renderer,
         '--max-image-mpixels', '0',
         env=spoof_tesseract_big_image_error)
 
@@ -610,10 +611,6 @@ def test_masks(spoof_tesseract_noop, resources, outpdf):
     p, out, err = run_ocrmypdf(
         resources / 'masks.pdf', outpdf, env=spoof_tesseract_noop)
 
-    if ghostscript.version() == '9.23' and \
-            p.returncode == ExitCode.invalid_output_pdf:
-        pytest.xfail('https://bugs.ghostscript.com/show_bug.cgi?id=699216')
-
     assert p.returncode == ExitCode.ok
 
 
@@ -722,7 +719,7 @@ def test_tesseract_config_notfound(renderer, resources, outdir):
         '--pdf-renderer', renderer,
         '--tesseract-config', cfg_file)
     assert "Can't open" in err, "No error message about missing config file"
-    assert p.returncode == ExitCode.ok
+    assert p.returncode == ExitCode.ok, err
 
 
 @pytest.mark.parametrize('renderer', RENDERERS)
@@ -906,7 +903,7 @@ def test_compression_changed(spoof_tesseract_noop, ocrmypdf_exec,
     if compression == "jpeg":
         assert pdfimage.enc == Encoding.jpeg
     else:
-        if ghostscript.version() >= '9.23':
+        if ghostscript.jpeg_passthrough_available():
             # Ghostscript 9.23 adds JPEG passthrough, which allows a JPEG to be
             # copied without transcoding - so report
             if image.endswith('jpg'):
@@ -955,17 +952,22 @@ def test_sidecar_nonempty(spoof_tesseract_cache, resources, outpdf):
     assert 'the' in ocr_text
 
 
-def test_pdfa_1(spoof_tesseract_cache, resources, outpdf):
+@pytest.mark.parametrize('pdfa_level', ['1', '2', '3'])
+def test_pdfa_n(spoof_tesseract_cache, pdfa_level, resources, outpdf):
+    if pdfa_level == '3' and ghostscript.version() < '9.19':
+        pytest.xfail(reason='Ghostscript >= 9.19 required')
+
     check_ocrmypdf(
         resources / 'ccitt.pdf', outpdf,
-        '--output-type', 'pdfa-1',
+        '--output-type', 'pdfa-' + pdfa_level,
         env=spoof_tesseract_cache
     )
 
     pdfa_info = file_claims_pdfa(outpdf)
-    assert pdfa_info['conformance'] == 'PDF/A-1B'
+    assert pdfa_info['conformance'] == 'PDF/A-{}B'.format(pdfa_level)
 
 
+@pytest.mark.skipif(sys.version_info >= (3, 7, 0), reason='fixed')
 def test_bad_locale():
     env = os.environ.copy()
     env['LC_ALL'] = 'C'
@@ -993,7 +995,7 @@ def test_bad_utf8(spoof_tess_bad_utf8, renderer, resources, no_outpdf):
 
 
 @pytest.mark.skipif(
-    not tesseract.has_textonly_pdf(), 
+    not tesseract.has_textonly_pdf(),
     reason="issue only affects sandwich")
 def test_rotate_deskew_timeout(resources, outdir):
     check_ocrmypdf(
@@ -1016,7 +1018,7 @@ def test_rotate_deskew_timeout(resources, outdir):
 
 
 @pytest.mark.skipif(
-    PIL.PILLOW_VERSION < '5.0.0',
+    PIL.__version__ < '5.0.0',
     reason="Pillow < 5.0.0 doesn't raise the exception")
 def test_decompression_bomb(resources, outpdf):
     p, out, err = run_ocrmypdf(
@@ -1043,10 +1045,10 @@ def test_text_curves(spoof_tesseract_noop, resources, outpdf):
     check_ocrmypdf(
         resources / 'vector.pdf', outpdf, '--force-ocr',
         env=spoof_tesseract_noop)
-    
+
     info = PdfInfo(outpdf)
     assert len(info.pages[0].images) != 0, "force did not rasterize"
-    
+
 
 def test_dev_null(spoof_tesseract_noop, resources):
     p, out, err = run_ocrmypdf(
@@ -1089,4 +1091,4 @@ def test_skip_repair(spoof_tesseract_noop, resources, outpdf):
         outpdf,
         '--skip-repair',
         env=spoof_tesseract_noop
-    )
+    )

tests/test_metadata.py

@@ -19,6 +19,7 @@
 import pytest
 import PyPDF2 as pypdf
 import datetime
+from datetime import timezone
 
 from ocrmypdf.pdfa import file_claims_pdfa, encode_pdf_date, decode_pdf_date
 from ocrmypdf.exceptions import ExitCode
@@ -72,11 +73,16 @@ def test_override_metadata(spoof_tesseract_noop, output_type, resources,
 
     assert p.returncode == ExitCode.ok, err
 
-    reader = pypdf.PdfFileReader(outpdf)
+    before = pypdf.PdfFileReader(str(input_file))
+    after = pypdf.PdfFileReader(outpdf)
 
-    assert reader.documentInfo['/Title'] == german
-    assert reader.documentInfo['/Author'] == chinese
-    assert reader.documentInfo.get('/Keywords', '') == ''
+    assert after.documentInfo['/Title'] == german
+    assert after.documentInfo['/Author'] == chinese
+    assert after.documentInfo.get('/Keywords', '') == ''
+
+    before_date = decode_pdf_date(before.documentInfo['/CreationDate'])
+    after_date = decode_pdf_date(after.documentInfo['/CreationDate'])
+    assert before_date == after_date
 
     pdfa_info = file_claims_pdfa(outpdf)
     assert pdfa_info['output'] == output_type
@@ -150,6 +156,6 @@ def test_creation_date_preserved(spoof_tesseract_noop, output_type, resources,
     # We expect that the modified date is quite recent
     date_after = decode_pdf_date(after['/ModDate'])
     assert seconds_between_dates(
-        date_after, datetime.datetime.utcnow()) < 1000
+        date_after, datetime.datetime.now(timezone.utc)) < 1000
 
 

tests/test_pageinfo.py

@@ -26,7 +26,6 @@ import os
 import shutil
 import pytest
 import img2pdf
-import pytest
 import sys
 import PyPDF2 as pypdf