Mention that v9.0.4 requires a source install for Py3.8 for now, due to lack of CI availability

Closes #337

.docker/alpine.dockerfile

@@ -14,7 +14,7 @@ RUN \
   && apk add --update \
   python3-dev \
   py3-setuptools \
-  jbig2enc@testing \
+  jbig2enc@community \
   ghostscript \
   qpdf@community \
   qpdf-dev@community \
@@ -56,7 +56,7 @@ RUN \
   # Add runtime dependencies
   && apk add --update \
   python3 \
-  jbig2enc@testing \
+  jbig2enc@community \
   ghostscript \
   qpdf@community \
   qpdf-dev@community \

.travis.yml

@@ -1,7 +1,7 @@
 cache:
   pip: true
   directories:
-  - $HOME/Library/Caches/Homebrew
+    - $HOME/Library/Caches/Homebrew
 
 matrix:
   include:
@@ -12,27 +12,58 @@ matrix:
       python: "3.6"
       env:
         - DIST=trusty
-      addons: &trusty_apt
+        - MINIMAL=true
+      addons:
         apt:
           update: true
           sources:
-          - sourceline: 'ppa:alex-p/tesseract-ocr'
+            - sourceline: "ppa:alex-p/tesseract-ocr"
-          - sourceline: 'ppa:heyarje/libav-11'
+            - sourceline: "ppa:vshn/ghostscript"
-          - sourceline: 'ppa:vshn/ghostscript'
           packages:
-          - ghostscript
+            - ghostscript
-          - libavcodec56
+            - libffi-dev
-          - libavformat56
+            - qpdf
-          - libavutil54
+            - tesseract-ocr
-          - libexempi3
+            - tesseract-ocr-deu
-          - libffi-dev
+            - tesseract-ocr-eng
-          - pngquant
+            - tesseract-ocr-fra
-          - poppler-utils
+      before_install: |
-          - qpdf
+        pip3 install --upgrade pip
-          - tesseract-ocr
+        pip3 install --upgrade wheel
-          - tesseract-ocr-deu
+    - os: linux
-          - tesseract-ocr-eng
+      dist: trusty
-          - tesseract-ocr-fra
+      sudo: required
+      language: python
+      python: "3.6"
+      env:
+        - DIST=trusty
+      addons:
+        apt:
+          update: true
+          sources:
+            - sourceline: "ppa:alex-p/tesseract-ocr"
+            - sourceline: "ppa:heyarje/libav-11"
+            - sourceline: "ppa:vshn/ghostscript"
+          packages:
+            - ghostscript
+            - libavcodec56
+            - libavformat56
+            - libavutil54
+            - libffi-dev
+            - qpdf
+            - tesseract-ocr
+            - tesseract-ocr-deu
+            - tesseract-ocr-eng
+            - tesseract-ocr-fra
+            - libexempi3 # --- optional extras from here ---
+            - pngquant
+            - poppler-utils
+      before_install: |
+        mkdir -p bin packages
+        pip3 install --upgrade pip
+        pip3 install --upgrade wheel
+        wget -q 'https://www.dropbox.com/s/vaq0kbwi6e6au80/unpaper_6.1-1.deb?raw=1' -O packages/unpaper_6.1-1.deb
+        sudo dpkg -i packages/unpaper_6.1-1.deb
     - os: linux
       dist: xenial
       sudo: required
@@ -44,21 +75,44 @@ matrix:
         apt:
           update: true
           sources:
-          - sourceline: 'ppa:alex-p/tesseract-ocr'
+            - sourceline: "ppa:alex-p/tesseract-ocr"
           packages:
-          - ghostscript
+            - ghostscript
-          - libexempi3
+            - libexempi3
-          - libffi-dev
+            - libffi-dev
-          - pngquant
+            - pngquant
-          - poppler-utils
+            - poppler-utils
-          - qpdf
+            - qpdf
-          - tesseract-ocr
+            - tesseract-ocr
-          - tesseract-ocr-deu
+            - tesseract-ocr-deu
-          - tesseract-ocr-eng
+            - tesseract-ocr-eng
-          - tesseract-ocr-fra
+            - tesseract-ocr-fra
-          - unpaper
+            - unpaper
+    # - os: linux
+    #   dist: xenial
+    #   sudo: required
+    #   language: python
+    #   python: "3.8"
+    #   env:
+    #     - DIST=xenial
+    #   addons:
+    #     apt:
+    #       update: true
+    #       sources:
+    #         - sourceline: "ppa:alex-p/tesseract-ocr"
+    #       packages:
+    #         - ghostscript
+    #         - libexempi3
+    #         - libffi-dev
+    #         - pngquant
+    #         - poppler-utils
+    #         - qpdf
+    #         - tesseract-ocr
+    #         - tesseract-ocr-deu
+    #         - tesseract-ocr-eng
+    #         - tesseract-ocr-fra
+    #         - unpaper
     - os: osx
-      osx_image: xcode9.2
       language: generic
       addons:
         homebrew:
@@ -74,73 +128,39 @@ matrix:
             - qpdf
             - tesseract
             - unpaper
-    - os: osx
+      before_install: |
-      osx_image: xcode9.2
+        pip3 install --upgrade pip
-      language: generic
+        pip3 install wheel
-      env:
-        - ADD_PDFMINER=1
-      addons:
-        homebrew:
-          update: true
-          packages:
-            - exempi
-            - ghostscript
-            - jbig2enc
-            - leptonica
-            - openjpeg
-            - pngquant
-            - python
-            - qpdf
-            - tesseract
-            - unpaper
 
 before_cache:
-- rm -f $HOME/.cache/pip/log/debug.log
+  - rm -f $HOME/.cache/pip/log/debug.log
-
-before_install: |
-  mkdir -p bin
-  if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
-    pip3 install --upgrade pip
-    pip3 install --upgrade wheel
-    if [[ "$DIST" == "trusty" ]]; then
-      mkdir -p packages
-      wget -q 'https://www.dropbox.com/s/vaq0kbwi6e6au80/unpaper_6.1-1.deb?raw=1' -O packages/unpaper_6.1-1.deb
-      sudo dpkg -i packages/unpaper_6.1-1.deb
-    fi
-  elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-    pip3 install --upgrade pip
-    pip3 install wheel
-  fi
 
 install:
-- export PATH=$PWD/bin:$PATH
+  - mkdir -p bin
-- pip3 install pycparser  # py3.7 workaround for https://github.com/eliben/pycparser/issues/251
+  - export PATH=$PWD/bin:$PATH
-- pip3 install -r requirements/main.txt
+  - pip3 install pycparser # py3.7 workaround for https://github.com/eliben/pycparser/issues/251
-- pip3 install --no-deps .
+  - pip3 install -r requirements/main.txt
-- |
+  - pip3 install --no-deps .
-  if [[ "$ADD_PDFMINER" == "1" ]]; then
+  - pip3 install -r requirements/test.txt
-    pip3 install --no-deps .[pdfminer]
-  fi
-- pip3 install -r requirements/test.txt
 
 script:
-- tesseract --version
+  - tesseract --version
-- qpdf --version
+  - qpdf --version
-- pytest -n auto
+  - pytest -n auto
 
 deploy:
   # release for main pypi
-  # 3.6 is considered the build leader and does the deploy, otherwise there is
+  # 3.7 is considered the build leader and does the deploy, otherwise there is
   # a race and all versions will try to deploy
   # OTOH if we ever need separate binary wheels then each version needs its
   # own deploy
-- provider: pypi
+  - provider: pypi
-  user: ocrmypdf-travis
+    user: ocrmypdf-travis
-  password:
+    password:
-    secure: "DTFOmmNL6olA0+yXvp4u9jXZlZeqrJsJ0526jzqf4a3gZ6jnGTq5UI6WzRsslSyoMMfXKtHQebqHM6ogSgCZinyZ3ufHJo8fn9brxbEc2gsiWkbj5o3bGwdWMT1vNNE7XW0VCpw87rZ1EEwjl4FJHFudMlPR1yfU5+uq0k0PACo="
+      secure: "DTFOmmNL6olA0+yXvp4u9jXZlZeqrJsJ0526jzqf4a3gZ6jnGTq5UI6WzRsslSyoMMfXKtHQebqHM6ogSgCZinyZ3ufHJo8fn9brxbEc2gsiWkbj5o3bGwdWMT1vNNE7XW0VCpw87rZ1EEwjl4FJHFudMlPR1yfU5+uq0k0PACo="
-  distributions: "sdist bdist_wheel"
+    distributions: "sdist bdist_wheel"
-  on:
+    on:
-    branch: master
+      branch: master
-    tags: true
+      tags: true
-    condition: $TRAVIS_PYTHON_VERSION == "3.6" && $TRAVIS_OS_NAME == "linux"
+      condition: $TRAVIS_PYTHON_VERSION == "3.7" && $TRAVIS_OS_NAME == "linux"
-  skip_upload_docs: true
+    skip_upload_docs: true

README.md

@@ -1,6 +1,6 @@
 <img src="docs/images/logo.svg" width="240" alt="OCRmyPDF">
 
-[![Travis build status][travis]](https://travis-ci.org/jbarlow83/OCRmyPDF) [![PyPI version][pypi]](https://pypi.org/project/ocrmypdf/) ![Homebrew version][homebrew] ![ReadTheDocs][docs]
+[![Travis build status][travis]](https://travis-ci.org/jbarlow83/OCRmyPDF) [![PyPI version][pypi]](https://pypi.org/project/ocrmypdf/) ![Homebrew version][homebrew] ![ReadTheDocs][docs] ![Python versions][pyversions]
 
 [travis]: https://travis-ci.org/jbarlow83/OCRmyPDF.svg?branch=master "Travis build status"
 
@@ -10,6 +10,8 @@
 
 [docs]: https://readthedocs.org/projects/ocrmypdf/badge/?version=latest "RTD"
 
+[pyversions]: https://img.shields.io/pypi/pyversions/ocrmypdf "Supported Python versions"
+
 OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched or copy-pasted.
 
 ```bash
@@ -120,7 +122,7 @@ If you detect an issue, please:
 Requirements
 ------------
 
-Runs on CPython 3.5, 3.6 and 3.7. Requires external program installations of Ghostscript, Tesseract OCR, QPDF, and Leptonica. ocrmypdf is pure Python, but uses CFFI to portably generate library bindings.
+In addition to the required Python version (3.6+), OCRmyPDF requires external program installations of Ghostscript, Tesseract OCR, QPDF, and Leptonica. ocrmypdf is pure Python, but uses CFFI to portably generate library bindings.
 
 Press & Media
 -------------

docs/advanced.rst

@@ -322,15 +322,23 @@ working files on a per page basis have the page number as a prefix
 (starting with page 1), an infix indicates the processing stage, and a
 suffix indicates the file type. Some important files include:
 
--  ``.page.png`` - what the input page looks like
+-  ``_rasterize.png`` - what the input page looks like
--  ``.image`` - the image we will show the user if we are in a mode that
+-  ``_ocr.png`` - the file that is sent to Tesseract for OCR; depending
-   changes the final appearance; may be in one of several image formats
--  ``.text.pdf`` - the OCR file; this will load as a blank page but
-   should have visible text if checked with a tool like pdftotext or
-   pdfminder.six
--  ``.ocr.png`` - the file that is sent to Tesseract for OCR; depending
    on arguments this may differ from the presentation image
--  ``layers.rendered.pdf`` - the composite PDF, before metadata repair
+-  ``_pp_deskew.png`` - the image, after deskewing
-   and optimization
+-  ``_pp_clean.png`` - the image, after cleaning with unpaper
+-  ``_ocr_tess.pdf`` - the OCR file; appears as a blank page with invisible
+   text embedded
+-  ``_ocr_tess.txt`` - the OCR text (not necessarily all text on the page,
+   if the page is mixed format)
+-  ``fix_docinfo.pdf`` - a temporary file created to fix the PDF DocumentInfo
+   data structure
+-  ``graft_layers.pdf`` - the rendered PDF with OCR layers grafted on
+-  ``pdfa.pdf`` - ``graft_layers.pdf`` after conversion to PDF/A
+-  ``pdfa.ps`` - a PostScript file used by Ghostscript for PDF/A conversion
+-  ``optimize.pdf`` - the PDF generated before optimization
+-  ``optimize.out.pdf`` - the PDF generated by optimization
+-  ``origin`` - the input file
+-  ``origin.pdf`` - the input file or the input image converted to PDF
 -  ``images/*`` - images extracted during the optimization process; here
    the prefix indicates a PDF object ID not a page number

docs/contributing.rst

@@ -0,0 +1,38 @@
+=======================
+Contributing guidelines
+=======================
+
+Contributions are welcome!
+
+Big changes
+===========
+
+Please open a new issue to discuss or propose a major change. Not only is it fun
+to discuss big ideas, but we might save each other's time too. Perhaps some of the
+work you're contemplating is already half-done in a development branch.
+
+Code style
+==========
+
+We use PEP8, ``black`` for code formatting and ``isort`` for import sorting. The
+settings for programs are in ``pyproject.toml`` and ``setup.cfg``.
+
+Tests
+=====
+
+New features should come with tests that confirm their correctness.
+
+New Python dependencies
+=======================
+
+If you are proposing a change that will require a new Python dependency, we
+prefer dependencies that are already packaged by Debian or Red Hat. This makes
+life much easier for our downstream package maintainers.
+
+Python dependencies must also be GPLv3 compatible.
+
+New non-Python dependencies
+===========================
+
+OCRmyPDF uses several external programs (Tesseract, Ghostscript and others) for
+its functionality. In general we prefer to avoid adding new external programs.

docs/cookbook.rst

@@ -165,8 +165,8 @@ might remove desirable content, especially from poor quality scans.
 -  ``--deskew`` will correct pages were scanned at a skewed angle by
    rotating them back into place. Skew determination and correction is
    performed using `Postl's variance of line
-   sums <http://www.leptonica.com/skew-measurement.html>`__ algorithm as
+   sums <http://www.leptonica.org/skew-measurement.html>`__ algorithm as
-   implemented in `Leptonica <http://www.leptonica.com/index.html>`__.
+   implemented in `Leptonica <http://www.leptonica.org/index.html>`__.
 -  ``--clean`` uses
    `unpaper <https://www.flameeyes.eu/projects/unpaper>`__ to clean up
    pages before OCR, but does not alter the final output. This makes it

docs/index.rst

@@ -22,11 +22,16 @@ PDF is the best format for storing and exchanging scanned documents.  Unfortunat
    cookbook
    docker
    advanced
-   api
    batch
    security
    errors
 
+.. toctree::
+   :caption: Developers
+   :maxdepth: 2
+
+   api
+   contributing
 
 Indices and tables
 ==================

docs/installation.rst

@@ -21,7 +21,7 @@ installing the Python binary wheels.
 Installing on Linux
 ===================
 
-Debian and Ubuntu 16.10 or newer
+Debian and Ubuntu 18.04 or newer
 --------------------------------
 
 .. |deb-stable| image:: https://repology.org/badge/version-for-repo/debian_stable/ocrmypdf.svg
@@ -33,27 +33,29 @@ Debian and Ubuntu 16.10 or newer
 .. |deb-unstable| image:: https://repology.org/badge/version-for-repo/debian_unstable/ocrmypdf.svg
     :alt: Debian unstable
 
-.. |ubu-1710| image:: https://repology.org/badge/version-for-repo/ubuntu_17_10/ocrmypdf.svg
-    :alt: Ubuntu 17.10
-
 .. |ubu-1804| image:: https://repology.org/badge/version-for-repo/ubuntu_18_04/ocrmypdf.svg
     :alt: Ubuntu 18.04 LTS
 
 .. |ubu-1810| image:: https://repology.org/badge/version-for-repo/ubuntu_18_10/ocrmypdf.svg
     :alt: Ubuntu 18.10
 
+.. |ubu-1904| image:: https://repology.org/badge/version-for-repo/ubuntu_19_04/ocrmypdf.svg
+    :alt: Ubuntu 19.04
 
-+-------------------------------------------+
+.. |ubu-1910| image:: https://repology.org/badge/version-for-repo/ubuntu_19_10/ocrmypdf.svg
-| **OCRmyPDF versions in Debian & Ubuntu**  |
+    :alt: Ubuntu 19.10
-+-------------------------------------------+
-| |latest|                                  |
-+-------------------------------------------+
-| |deb-stable| |deb-testing| |deb-unstable| |
-+-------------------------------------------+
-| |ubu-1710| |ubu-1804| |ubu-1810|          |
-+-------------------------------------------+
 
-Users of Debian 9 ("stretch") or later or Ubuntu 16.10 or later may
++-----------------------------------------------+
+| **OCRmyPDF versions in Debian & Ubuntu**      |
++-----------------------------------------------+
+| |latest|                                      |
++-----------------------------------------------+
+| |deb-stable| |deb-testing| |deb-unstable|     |
++-----------------------------------------------+
+| |ubu-1804| |ubu-1810| |ubu-1904| |ubu-1910|   |
++-----------------------------------------------+
+
+Users of Debian 9 ("stretch") or later or Ubuntu 18.04 or later may
 simply
 
 .. code-block:: bash
@@ -64,7 +66,8 @@ As indicated in the table above, Debian and Ubuntu releases may lag
 behind the latest version. If the version available for your platform is
 out of date, you could opt to install the latest version from source.
 See `Installing HEAD revision from
-sources <#installing-head-revision-from-sources>`__.
+sources <#installing-head-revision-from-sources>`__. Ubuntu 16.10 to 17.10
+inclusive also had ocrmypdf, but these versions are end of life.
 
 For full details on version availability for your platform, check the
 `Debian Package Tracker <https://tracker.debian.org/pkg/ocrmypdf>`__ or
@@ -81,20 +84,22 @@ For full details on version availability for your platform, check the
 Fedora 29 or newer
 ------------------
 
-.. |fedora-29| image:: https://repology.org/badge/version-for-repo/fedora29/ocrmypdf.svg
+.. |fedora-29| image:: https://repology.org/badge/version-for-repo/fedora_29/ocrmypdf.svg
     :alt: Fedora 29
 
+.. |fedora-30| image:: https://repology.org/badge/version-for-repo/fedora_30/ocrmypdf.svg
+    :alt: Fedora 30
+
 .. |fedora-rawhide| image:: https://repology.org/badge/version-for-repo/fedora_rawhide/ocrmypdf.svg
     :alt: Fedore Rawhide
 
-
++-----------------------------------------------+
-+------------------------------+
+| **OCRmyPDF version**                          |
-| **OCRmyPDF version**         |
++-----------------------------------------------+
-+------------------------------+
+| |latest|                                      |
-| |latest|                     |
++-----------------------------------------------+
-+------------------------------+
+| |fedora-29| |fedora-30| |fedora-rawhide|      |
-| |fedora-29| |fedora-rawhide| |
++-----------------------------------------------+
-+------------------------------+
 
 Users of Fedora 29 later may simply
 
@@ -403,14 +408,19 @@ The command line program should now be available:
 Installing on FreeBSD
 =====================
 
-FreeBSD 11.2 is known to work. Other versions likely work but have not
+.. image:: https://repology.org/badge/version-for-repo/freebsd/python:ocrmypdf.svg
-been tested.
+    :alt: FreeBSD
+    :target: https://repology.org/project/python:ocrmypdf/versions
 
-In general it should work to:
+FreeBSD 11.2, 11.3, 12.0-RELEASE and 13.0-CURRENT are supported. Other
+versions likely work but have not been tested.
 
-#. `Install and build
+.. code-block:: bash
-   pikepdf <https://pikepdf.readthedocs.io/en/latest/installation.html#installing-on-freebsd-11-2>`__.
+
-#. Install the equivalent list of dependencies for Linux.
+    pkg install py36-ocrmypdf
+
+To install a more recent version, you could attempt to first install the system
+version with ``pkg``, then use ``pip install --user ocrmypdf``.
 
 Installing the Docker image
 ===========================
@@ -503,11 +513,11 @@ manager. ``pip`` cannot provide them.
 -  Python 3.6 or newer
 -  Ghostscript 9.15 or newer
 -  qpdf 8.1.0 or newer
--  Tesseract 4.0.0-alpha or newer
+-  Tesseract 4.0.0-beta or newer
 
 As of ocrmypdf 7.2.1, the following versions are recommended:
 
--  Python 3.7
+-  Python 3.7 or 3.8
 -  Ghostscript 9.23 or newer
 -  qpdf 8.2.1
 -  Tesseract 4.0.0 or newer

docs/release_notes.rst

@@ -13,6 +13,54 @@ Note that it is licensed under GPLv3, so scripts that
 ``import ocrmypdf`` and are released publicly should probably also be
 licensed under GPLv3.
 
+v9.0.4
+======
+
+-  Fixed compatibility with Python 3.8 (but requires source install for the moment).
+-  Fixed Tesseract settings for ``--user-words`` and ``--user-patterns``.
+-  Changed to pikepdf 1.6.5 (for Python 3.8).
+-  Changed to Pillow 6.2.0 (to mitigate a security vulnerability in earlier Pillow).
+-  A debug message now mentions when English is automatically selected if the locale
+   is not English.
+
+v9.0.3
+======
+
+-  Embed an encoded version of the sRGB ICC profile in the intermediate
+   Postscript file (used for PDF/A conversion). Previously we included the
+   filename, which required Postscript to run with file access enabled. For
+   security, Ghostscript 9.28 enables ``-dSAFER`` and as such, no longer
+   permits access to any file by default. This fix is necessary for
+   compatibility with Ghostscript 9.28.
+-  Exclude a test that sometimes times out and fails in continuous integration
+   from the standard test suite.
+
+v9.0.2
+======
+
+-  The image optimizer now skips optimizing flate (PNG) encoded images in some
+   situations where the optimization effort was likely wasted.
+-  The image optimizer now ignores images that specify arbitrary decode arrays,
+   since these are rare.
+-  Fixed an issue that caused inversion of black and white in monochrome images.
+   We are not certain but the problem seems to be linked to Leptonica 1.76.0 and
+   older.
+-  Fixed some cases where the test suite failed if
+   English or German Tesseract language packs were not installed.
+-  Fixed a runtime error if the Tesseract English language is not installed.
+-  Improved explicit closing of Pillow images after use.
+-  Actually fixed of Alpine Docker image build.
+-  Changed to pikepdf 1.6.3.
+
+v9.0.1
+======
+
+-  Fixed test suite failing when either of optional dependencies unpaper and
+   pngquant were missing.
+-  Attempted fix of Alpine Docker image build.
+-  Documented that FreeBSD ports are now available.
+-  Changed to pikepdf 1.6.1.
+
 v9.0.0
 ======
 
@@ -59,6 +107,8 @@ v9.0.0
 -  Updated Docker images to use newer versions.
 -  Fixed images encoded as JBIG2 with a colorspace other than ``/DeviceGray``
    were not interpreted correctly.
+-  Fixed a OCR text-image registration (i.e. alignment) problem when the page
+   when MediaBox had a nonzero corner.
 
 v8.3.2
 ======

pyproject.toml

@@ -10,7 +10,7 @@ build-backend = "setuptools.build_meta"
 
 [tool.black]
 line-length = 88
-py36 = true
+target-version = ["py36", "py37", "py38"]
 skip-string-normalization = true
 include = '\.pyi?$'
 exclude = '''

requirements/main.txt

@@ -5,8 +5,8 @@ chardet == 3.0.4
 cffi == 1.12.2
 img2pdf == 0.3.3
 pdfminer.six == 20181108
-pikepdf == 1.6.0
+pikepdf == 1.6.5
-Pillow >= 5.0.0, != 5.1.0 ; sys_platform == "darwin"
+Pillow >= 6.2.0
 pycparser == 2.19
 python-xmp-toolkit == 2.0.1
 reportlab == 3.5.13

requirements/test.txt

@@ -1,6 +1,6 @@
-pytest >= 4.4.1, < 5
+pytest >= 5.0.0
 pytest-helpers-namespace >= 2019.1.8
-pytest-xdist == 1.28.0
+pytest-xdist >= 1.29.0  # For DumpError fix
 pytest-cov >= 2.6.1
 python-xmp-toolkit   # requires apt-get install libexempi3
                      # or brew install exempi

setup.py

@@ -68,6 +68,7 @@ setup(
     classifiers=[
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
         "Development Status :: 5 - Production/Stable",
         "Environment :: Console",
         "Intended Audience :: End Users/Desktop",
@@ -96,10 +97,8 @@ setup(
         'cffi >= 1.9.1',  # must be a setup and install requirement
         'img2pdf >= 0.3.0, < 0.4',  # pure Python, so track HEAD closely
         'pdfminer.six == 20181108',
-        'pikepdf >= 1.6.0, < 2',
+        'pikepdf >= 1.6.5, < 2',
-        'Pillow >= 4.0.0, != 5.1.0 ; sys_platform == "darwin"',
+        'Pillow >= 6.2.0',
-        # Pillow < 4 has BytesIO/TIFF bug w/img2pdf 0.2.3
-        # block 5.1.0, broken wheels
         'reportlab >= 3.3.0',  # oldest released version with sane image handling
         'tqdm >= 4',
     ],

src/ocrmypdf/_pipeline.py

@@ -54,9 +54,9 @@ def triage_image_file(input_file, output_file, options, log):
         # Recover the original filename
         log.error(str(e).replace(input_file, options.input_file))
         raise UnsupportedImageFormatError() from e
-    else:
-        log.info("Input file is an image")
 
+    with im:
+        log.info("Input file is an image")
         if 'dpi' in im.info:
             if im.info['dpi'] <= (96, 96) and not options.image_dpi:
                 log.info("Image size: (%d, %d)" % im.size)
@@ -89,7 +89,6 @@ def triage_image_file(input_file, output_file, options, log):
             elif im.mode == 'CMYK':
                 log.info('Input CMYK image has no ICC profile, not usable')
                 raise UnsupportedImageFormatError()
-        im.close()
 
     try:
         log.info("Image seems valid. Try converting to PDF...")
@@ -129,7 +128,7 @@ def triage(input_file, output_file, options, log):
         if _pdf_guess_version(input_file):
             if options.image_dpi:
                 log.warning(
-                    "Argument --image-dpi ignored because the "
+                    "Argument --image-dpi is being ignored because the "
                     "input file is a PDF, not an image."
                 )
             # Origin file is a pdf create a symlink with pdf extension

src/ocrmypdf/_sync.py

@@ -189,7 +189,7 @@ def worker_init(queue):
     root.addHandler(h)
 
 
-def worker_thread_init(queue):
+def worker_thread_init(_queue):
     pass
 
 
@@ -225,15 +225,15 @@ def exec_concurrent(context):
     if max_workers > 1:
         context.log.info("Start processing %d pages concurrent", max_workers)
 
-    # Tesseract 4.0 is multithreaded, and we also run multiple workers. We want to
+    # Tesseract 4.x can be multithreaded, and we also run multiple workers. We want
-    # avoid the situation where we end up trying to run NxN jobs on N CPU cores,
+    # to manage how many threads it uses to avoid creating total threads than cores.
-    # as that gives poor performance. Performance testing shows we're better off
+    # Performance testing shows we're better off
     # parallelizing ocrmypdf and forcing Tesseract to be single threaded, which we
     # get by setting the envvar OMP_THREAD_LIMIT to 1. But if the page count of the
     # input file is small, then we allow Tesseract to use threads, subject to the
-    # constraint: (ocrmypdf workers) * (tesseract threads) <= max_workers and limiting
+    # constraint: (ocrmypdf workers) * (tesseract threads) <= max_workers.
-    # Tesseract to 4 threads.
+    # As of Tesseract 4.1, 3 threads is the most effective on a 4 core/8 thread system.
-    tess_threads = min(4, context.options.jobs // max_workers)
+    tess_threads = min(3, context.options.jobs // max_workers)
     if context.options.tesseract_env is None:
         context.options.tesseract_env = os.environ.copy()
     context.options.tesseract_env.setdefault('OMP_THREAD_LIMIT', str(tess_threads))
@@ -301,7 +301,7 @@ def exec_concurrent(context):
 class NeverRaise(Exception):
     """An exception that is never raised"""
 
-    pass
+    pass  # pylint: disable=unnecessary-pass
 
 
 def run_pipeline(options, api=False):

src/ocrmypdf/_validation.py

@@ -17,6 +17,7 @@
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
 
+import locale
 import logging
 import os
 import sys
@@ -47,6 +48,7 @@ from .helpers import is_file_writable, is_iterable_notstr, monotonic, re_symlink
 # External dependencies
 
 HOCR_OK_LANGS = frozenset(['eng', 'deu', 'spa', 'ita', 'por'])
+DEFAULT_LANGUAGE = 'eng'  # Enforce English hegemony
 
 log = logging.getLogger(__name__)
 
@@ -58,7 +60,10 @@ verify_python3_env()
 
 def check_options_languages(options):
     if not options.language:
-        options.language = ['eng']  # Enforce English hegemony
+        options.language = [DEFAULT_LANGUAGE]
+        system_lang = locale.getlocale()[0]
+        if system_lang and not system_lang.startswith('en'):
+            log.debug("No language specified; assuming --language %s", DEFAULT_LANGUAGE)
 
     # Support v2.x "eng+deu" language syntax
     if '+' in options.language[0]:
@@ -107,7 +112,7 @@ def check_options_output(options):
         options.pdf_renderer = 'sandwich'
 
     if options.pdf_renderer == 'sandwich' and not tesseract.has_textonly_pdf(
-        options.tesseract_env
+        options.tesseract_env, languages
     ):
         raise MissingDependencyError(
             "You are using an alpha version of Tesseract 4.0 that does not support "

src/ocrmypdf/api.py

@@ -33,7 +33,7 @@ class TqdmConsole:
 
     def __init__(self, file):
         self.file = file
-        self.py36 = sys.version_info >= (3, 6)
+        self.py36 = sys.version_info[0:2] == (3, 6)
 
     def write(self, msg):
         # When no progress bar is active, tqdm.write() routes to print()
@@ -122,13 +122,23 @@ def create_options(*, input_file, output_file, **kwargs):
     for arg, val in kwargs.items():
         if val is None:
             continue
-        if arg == 'tesseract_env':
+
+        # These arguments with special handling for which we bypass
+        # argparse
+        if arg in {'tesseract_env', 'progress_bar'}:
             deferred.append((arg, val))
             continue
+
         cmd_style_arg = arg.replace('_', '-')
-        cmdline.append(f"--{cmd_style_arg}")
+
+        # Booleans are special: add only if True, omit for False
         if isinstance(val, bool):
+            if val:
+                cmdline.append(f"--{cmd_style_arg}")
             continue
+
+        # We have a parameter
+        cmdline.append(f"--{cmd_style_arg}")
         if isinstance(val, (int, float)):
             cmdline.append(str(val))
         elif isinstance(val, str):

src/ocrmypdf/exec/ghostscript.py

@@ -259,6 +259,7 @@ def generate_pdfa(
                 "-dQUIET",
                 "-dBATCH",
                 "-dNOPAUSE",
+                "-dSAFER",
                 "-dCompatibilityLevel=" + str(pdf_version),
                 "-sDEVICE=pdfwrite",
                 "-dAutoRotatePages=/None",

src/ocrmypdf/exec/pngquant.py

@@ -40,8 +40,7 @@ def available():
 
 def quantize(input_file, output_file, quality_min, quality_max):
     if input_file.endswith('.jpg'):
-        im = Image.open(input_file)
+        with Image.open(input_file) as im, NamedTemporaryFile(suffix='.png') as tmp:
-        with NamedTemporaryFile(suffix='.png') as tmp:
             im.save(tmp)
             args = [
                 'pngquant',

src/ocrmypdf/exec/tesseract.py

@@ -17,10 +17,8 @@
 
 import os
 import shutil
-import sys
 from collections import namedtuple
 from contextlib import suppress
-from functools import lru_cache
 from os import fspath
 from subprocess import PIPE, STDOUT, CalledProcessError, TimeoutExpired, run
 
@@ -61,13 +59,13 @@ def v4(tesseract_env=None):
     return version(tesseract_env) >= '4'
 
 
-def has_textonly_pdf(tesseract_env=None):
+def has_textonly_pdf(tesseract_env=None, langs=None):
     """Does Tesseract have textonly_pdf capability?
 
     Available in v4.00.00alpha since January 2017. Best to
-    parse the parameter list
+    parse the parameter list.
     """
-    args_tess = ['tesseract', '--print-parameters', 'pdf']
+    args_tess = tess_base_args(langs, engine_mode=None) + ['--print-parameters', 'pdf']
     params = ''
     try:
         proc = run(
@@ -233,8 +231,8 @@ def _generate_null_hocr(output_hocr, output_sidecar, image):
     the same size as the input image."""
     from PIL import Image
 
-    im = Image.open(image)
+    with Image.open(image) as im:
-    w, h = im.size
+        w, h = im.size
 
     with open(output_hocr, 'w', encoding="utf-8") as f:
         f.write(HOCR_TEMPLATE.format(w, h))
@@ -271,6 +269,9 @@ def generate_hocr(
     if user_patterns:
         args_tesseract.extend(['--user-patterns', user_patterns])
 
+    if user_words or user_patterns:
+        args_tesseract.extend(['-c', 'lstm_use_matrix=1'])
+
     # Reminder: test suite tesseract spoofers will break after any changes
     # to the number of order parameters here
     args_tesseract.extend([input_file, prefix, 'hocr', 'txt'] + tessconfig)
@@ -358,7 +359,7 @@ def generate_pdf(
     if pagesegmode is not None:
         args_tesseract.extend(['--psm', str(pagesegmode)])
 
-    if text_only and has_textonly_pdf(tesseract_env):
+    if text_only and has_textonly_pdf(tesseract_env, language):
         args_tesseract.extend(['-c', 'textonly_pdf=1'])
 
     if user_words:

src/ocrmypdf/exec/unpaper.py

@@ -42,33 +42,30 @@ def run(input_file, output_file, dpi, log, mode_args):
 
     SUFFIXES = {'1': '.pbm', 'L': '.pgm', 'RGB': '.ppm'}
 
-    im = Image.open(input_file)
+    with TemporaryDirectory() as tmpdir, Image.open(input_file) as im:
-    if im.mode not in SUFFIXES.keys():
+        if im.mode not in SUFFIXES.keys():
-        log.info("Converting image to other colorspace")
+            log.info("Converting image to other colorspace")
+            try:
+                if im.mode == 'P' and len(im.getcolors()) == 2:
+                    im = im.convert(mode='1')
+                else:
+                    im = im.convert(mode='RGB')
+            except IOError as e:
+                im.close()
+                raise MissingDependencyError(
+                    "Could not convert image with type " + im.mode
+                ) from e
+
         try:
-            if im.mode == 'P' and len(im.getcolors()) == 2:
+            suffix = SUFFIXES[im.mode]
-                im = im.convert(mode='1')
+        except KeyError:
-            else:
-                im = im.convert(mode='RGB')
-        except IOError as e:
-            im.close()
             raise MissingDependencyError(
-                "Could not convert image with type " + im.mode
+                "Failed to convert image to a supported format."
             ) from e
 
-    try:
-        suffix = SUFFIXES[im.mode]
-    except KeyError:
-        im.close()
-        raise MissingDependencyError(
-            "Failed to convert image to a supported format."
-        ) from e
-
-    with TemporaryDirectory() as tmpdir:
         input_pnm = os.path.join(tmpdir, f'input{suffix}')
         output_pnm = os.path.join(tmpdir, f'output{suffix}')
         im.save(input_pnm, format='PPM')
-        im.close()
 
         # To prevent any shenanigans from accepting arbitrary parameters in
         # --unpaper-args, we:
@@ -95,10 +92,12 @@ def run(input_file, output_file, dpi, log, mode_args):
             log.debug(proc.stdout)
             # unpaper sets dpi to 72; fix this
             try:
-                Image.open(output_pnm).save(output_file, dpi=(dpi, dpi))
+                with Image.open(output_pnm) as imout:
+                    imout.save(output_file, dpi=(dpi, dpi))
             except (FileNotFoundError, OSError):
                 raise SubprocessOutputError(
-                    "unpaper: failed to produce the expected output file. Called with: "
+                    "unpaper: failed to produce the expected output file. "
+                    + " Called with: "
                     + str(args_unpaper)
                 ) from None
 

src/ocrmypdf/optimize.py

@@ -73,6 +73,9 @@ def extract_image_filter(pike, root, log, image, xref):
     if filtdp[0] == Name.JPXDecode:
         return None  # Don't do JPEG2000
 
+    if Name.Decode in image:
+        return None  # Don't mess with custom Decode tables
+
     return pim, filtdp
 
 
@@ -104,6 +107,15 @@ def extract_image_generic(*, pike, root, log, image, xref, options):
         return None
     pim, filtdp = result
 
+    # Don't try to PNG-optimize 1bpp images, since JBIG2 does it better.
+    if pim.bits_per_component == 1:
+        return None
+
+    try:
+        pim.indexed  # pikepdf 1.6.3 can't handle [/Indexed [/Array...]]
+    except NotImplementedError:
+        return None
+
     if filtdp[0] == Name.DCTDecode and options.optimize >= 2:
         # This is a simple heuristic derived from some training data, that has
         # about a 70% chance of guessing whether the JPEG is high quality,
@@ -343,6 +355,7 @@ def transcode_jpegs(pike, jpegs, root, log, options):
 
 
 def transcode_pngs(pike, images, image_name_fn, root, log, options):
+    modified = set()
     if options.optimize >= 2:
         png_quality = (
             max(10, options.png_quality - 10),
@@ -363,6 +376,7 @@ def transcode_pngs(pike, images, image_name_fn, root, log, options):
                         png_quality[1],
                     )
                 )
+                modified.add(xref)
             with tqdm(
                 desc="PNGs",
                 total=len(futures),
@@ -372,10 +386,14 @@ def transcode_pngs(pike, images, image_name_fn, root, log, options):
                 for _future in concurrent.futures.as_completed(futures):
                     pbar.update()
 
-    for xref in images:
+    for xref in modified:
         im_obj = pike.get_object(xref, 0)
         try:
-            compdata = leptonica.CompressedData.open(png_name(root, xref))
+            pix = leptonica.Pix.open(png_name(root, xref))
+            if pix.mode == '1':
+                compdata = pix.generate_pdf_ci_data(leptonica.lept.L_G4_ENCODE, 0)
+            else:
+                compdata = leptonica.CompressedData.open(png_name(root, xref))
         except leptonica.LeptonicaError as e:
             # Most likely this means file not found, i.e. quantize did not
             # produce an improved version
@@ -391,62 +409,83 @@ def transcode_pngs(pike, images, image_name_fn, root, log, options):
                 f"{len(compdata)} > {int(im_obj.stream_dict.Length)}"
             )
             continue
+        if compdata.type == leptonica.lept.L_FLATE_ENCODE:
+            return rewrite_png(pike, im_obj, compdata, log)
+        elif compdata.type == leptonica.lept.L_G4_ENCODE:
+            return rewrite_png_as_g4(pike, im_obj, compdata, log)
 
-        # When a PNG is inserted into a PDF, we more or less copy the IDAT section from
-        # the PDF and transfer the rest of the PNG headers to PDF image metadata.
-        # One thing we have to do is tell the PDF reader whether a predictor was used
-        # on the image before Flate encoding. (Typically one is.)
-        # According to Leptonica source, PDF readers don't actually need us
-        # to specify the correct predictor, they just need a value of either:
-        #   1 - no predictor
-        #   10-14 - there is a predictor
-        # Leptonica's compdata->predictor only tells TRUE or FALSE
-        # From there the PNG decoder can infer the rest from the file.
-        # In practice the predictor should be Paeth, 14, so we'll use that.
-        # See:
-        #   - PDF RM 7.4.4.4 Table 10
-        #   - https://github.com/DanBloomberg/leptonica/blob/master/src/pdfio2.c#L757
-        predictor = 14 if compdata.predictor > 0 else 1
-        dparms = Dictionary(Predictor=predictor)
-        if predictor > 1:
-            dparms.BitsPerComponent = compdata.bps  # Yes, this is redundant
-            dparms.Colors = compdata.spp
-            dparms.Columns = compdata.w
 
-        im_obj.BitsPerComponent = compdata.bps
+def rewrite_png_as_g4(pike, im_obj, compdata, log):
-        im_obj.Width = compdata.w
+    im_obj.BitsPerComponent = 1
-        im_obj.Height = compdata.h
+    im_obj.Width = compdata.w
+    im_obj.Height = compdata.h
 
-        if compdata.ncolors > 0:
+    im_obj.write(compdata.read())
-            # .ncolors is the number of colors in the palette, not the number of
+
-            # colors used in a true color image
+    log.debug(f"PNG to G4 {im_obj.objgen}")
-            palette_pdf_string = compdata.get_palette_pdf_string()
+    if Name.Predictor in im_obj:
-            palette_data = pikepdf.Object.parse(palette_pdf_string)
+        del im_obj.Predictor
-            palette_stream = pikepdf.Stream(pike, bytes(palette_data))
+    if Name.DecodeParms in im_obj:
-            palette = [
+        del im_obj.DecodeParms
-                Name.Indexed,
+    im_obj.DecodeParms = Dictionary(
-                Name.DeviceRGB,
+        K=-1, BlackIs1=bool(compdata.minisblack), Columns=compdata.w
-                compdata.ncolors - 1,
+    )
-                palette_stream,
+
-            ]
+    im_obj.Filter = Name.CCITTFaxDecode
-            cs = palette
+    return
-        else:
+
-            if compdata.spp == 1:
+
-                # PDF interprets binary-1 as black in 1bpp, but PNG sets
+def rewrite_png(pike, im_obj, compdata, log):
-                # black to 0 for 1bpp. Create a palette that informs the PDF
+    # When a PNG is inserted into a PDF, we more or less copy the IDAT section from
-                # of the mapping - seems cleaner to go this way but pikepdf
+    # the PDF and transfer the rest of the PNG headers to PDF image metadata.
-                # needs to be patched to support it.
+    # One thing we have to do is tell the PDF reader whether a predictor was used
-                # palette = [Name.Indexed, Name.DeviceGray, 1, b"\xff\x00"]
+    # on the image before Flate encoding. (Typically one is.)
-                # cs = palette
+    # According to Leptonica source, PDF readers don't actually need us
-                cs = Name.DeviceGray
+    # to specify the correct predictor, they just need a value of either:
-            elif compdata.spp == 3:
+    #   1 - no predictor
-                cs = Name.DeviceRGB
+    #   10-14 - there is a predictor
-            elif compdata.spp == 4:
+    # Leptonica's compdata->predictor only tells TRUE or FALSE
-                cs = Name.DeviceCMYK
+    # 10-14 means the actual predictor is specified in the data, so for any
-        if compdata.bps == 1:
+    # number >= 10 the PDF reader will use whatever the PNG data specifies.
-            im_obj.Decode = [1, 0]  # Bit of a kludge but this inverts photometric too
+    # In practice Leptonica should use Paeth, 14, but 15 seems to be the
-        im_obj.ColorSpace = cs
+    # designated value for "optimal". So we will use 15.
-        im_obj.write(compdata.read(), filter=Name.FlateDecode, decode_parms=dparms)
+    # See:
+    #   - PDF RM 7.4.4.4 Table 10
+    #   - https://github.com/DanBloomberg/leptonica/blob/master/src/pdfio2.c#L757
+    predictor = 15 if compdata.predictor > 0 else 1
+    dparms = Dictionary(Predictor=predictor)
+    if predictor > 1:
+        dparms.BitsPerComponent = compdata.bps  # Yes, this is redundant
+        dparms.Colors = compdata.spp
+        dparms.Columns = compdata.w
+
+    im_obj.BitsPerComponent = compdata.bps
+    im_obj.Width = compdata.w
+    im_obj.Height = compdata.h
+
+    log.debug(
+        f"PNG {im_obj.objgen}: palette={compdata.ncolors} spp={compdata.spp} bps={compdata.bps}"
+    )
+    if compdata.ncolors > 0:
+        # .ncolors is the number of colors in the palette, not the number of
+        # colors used in a true color image. The palette string is always
+        # given as RGB tuples even when the image is grayscale; see
+        # https://github.com/DanBloomberg/leptonica/blob/master/src/colormap.c#L2067
+        palette_pdf_string = compdata.get_palette_pdf_string()
+        palette_data = pikepdf.Object.parse(palette_pdf_string)
+        palette_stream = pikepdf.Stream(pike, bytes(palette_data))
+        palette = [Name.Indexed, Name.DeviceRGB, compdata.ncolors - 1, palette_stream]
+        cs = palette
+    else:
+        # ncolors == 0 means we are using a colorspace without a palette
+        if compdata.spp == 1:
+            cs = Name.DeviceGray
+        elif compdata.spp == 3:
+            cs = Name.DeviceRGB
+        elif compdata.spp == 4:
+            cs = Name.DeviceCMYK
+    im_obj.ColorSpace = cs
+    im_obj.write(compdata.read(), filter=Name.FlateDecode, decode_parms=dparms)
 
 
 def optimize(input_file, output_file, context, save_settings):
@@ -510,11 +549,11 @@ def main(infile, outfile, level, jobs=1):
         """Emulate ocrmypdf's options"""
 
         def __init__(
-            self, input_file, jobs, optimize, jpeg_quality, png_quality, jb2lossy
+            self, input_file, jobs, optimize_, jpeg_quality, png_quality, jb2lossy
         ):
             self.input_file = input_file
             self.jobs = jobs
-            self.optimize = optimize
+            self.optimize = optimize_
             self.jpeg_quality = jpeg_quality
             self.png_quality = png_quality
             self.jbig2_page_group_size = 0
@@ -525,7 +564,7 @@ def main(infile, outfile, level, jobs=1):
     options = OptimizeOptions(
         input_file=infile,
         jobs=jobs,
-        optimize=int(level),
+        optimize_=int(level),
         jpeg_quality=0,  # Use default
         png_quality=0,
         jb2lossy=False,

src/ocrmypdf/pdfa.py

@@ -31,6 +31,7 @@ Ghostscript's handling of pdfmark.
 
 """
 
+import base64
 import os
 from binascii import hexlify
 from pathlib import Path
@@ -48,26 +49,13 @@ SRGB_ICC_PROFILE = pkg_resources.resource_filename('ocrmypdf', ICC_PROFILE_RELPA
 # files, from the Ghostscript documentation. Lines beginning with % are
 # comments. Python substitution variables have a '$' prefix.
 pdfa_def_template = u"""%!
-% Define entries in the document Info dictionary :
+% Define an ICC profile :
 /ICCProfile $icc_profile
 def
 
-% Define an ICC profile :
-
 [/_objdef {icc_PDFA} /type /stream /OBJ pdfmark
-[{icc_PDFA}
+[{icc_PDFA} << /N 3 >> /PUT pdfmark
-<<
+[{icc_PDFA} ICCProfile /PUT pdfmark
-  /N currentpagedevice /ProcessColorModel known {
-    currentpagedevice /ProcessColorModel get dup /DeviceGray eq
-    {pop 1} {
-      /DeviceRGB eq
-      {3}{4} ifelse
-    } ifelse
-  } {
-    (ERROR, unable to determine ProcessColorModel) == flush
-  } ifelse
->> /PUT pdfmark
-[{icc_PDFA} ICCProfile (r) file /PUT pdfmark
 
 % Define the output intent dictionary :
 
@@ -104,16 +92,10 @@ def generate_pdfa_ps(target_filename, icc='sRGB'):
     else:
         raise NotImplementedError("Only supporting sRGB")
 
-    # pdfmark must contain the full path to the ICC profile, and pdfmark must be
+    # Read the ICC profile, encode as ASCII85 and convert to a string which we
-    # also encoded in ASCII. ocrmypdf can be installed anywhere, including to
+    # will insert in the .ps file
-    # paths that have a non-ASCII character in the filename. Ghostscript
+    bytes_icc_profile = Path(icc_profile).read_bytes()
-    # accepts hex-encoded strings and converts them to byte strings, so
+    icc_profile = base64.a85encode(bytes_icc_profile, adobe=True).decode('ascii')
-    # we encode the path with fsencode() and use the hex representation.
-    # UTF-16 not accepted here. (Even though ASCII encodable is the usual case,
-    # do this always to avoid making it a rare conditional.)
-    bytes_icc_profile = os.fsencode(icc_profile)
-    hex_icc_profile = hexlify(bytes_icc_profile)
-    icc_profile = '<' + hex_icc_profile.decode('ascii') + '>'
 
     t = Template(pdfa_def_template)
     ps = t.substitute(icc_profile=icc_profile, icc_identifier=icc)

tests/conftest.py

@@ -176,10 +176,9 @@ def no_outpdf(tmp_path):
 def check_ocrmypdf(input_file, output_file, *args, env=None):
     """Run ocrmypdf and confirmed that a valid file was created"""
 
-    # p, out, err = run_ocrmypdf(input_file, output_file, *args, env=env)
-
     options = cli.parser.parse_args(
-        [str(input_file), str(output_file)] + [str(arg) for arg in args]
+        [str(input_file), str(output_file)]
+        + [str(arg) for arg in args if arg is not None]
     )
     api.check_options(options)
     if env:

tests/spoof/tesseract_badutf8.py

@@ -52,7 +52,7 @@ def main():
     elif sys.argv[1] == '--list-langs':
         print('List of available languages (1):\neng', file=sys.stderr)
         sys.exit(0)
-    elif sys.argv[1] == '--print-parameters':
+    elif sys.argv[-2] == '--print-parameters':
         print("Some parameters", file=sys.stderr)
         print("textonly_pdf\t1\tSome help text")
         sys.exit(0)

tests/spoof/tesseract_big_image_error.py

@@ -44,7 +44,7 @@ def main():
     elif sys.argv[1] == '--list-langs':
         print('List of available languages (1):\neng\n', file=sys.stderr)
         sys.exit(0)
-    elif sys.argv[1] == '--print-parameters':
+    elif sys.argv[-2] == '--print-parameters':
         print('A parameter list would go here\ntextonly_pdf 0\n', file=sys.stderr)
         sys.exit(0)
     elif sys.argv[-2] == 'hocr':

tests/spoof/tesseract_cache.py

@@ -100,6 +100,8 @@ def main():
     # Convert non-standard but supported -psm to --psm
     sys.argv = ['--psm' if arg == '-psm' else arg for arg in sys.argv]
 
+    if '_OCRMYPDF_TEST_INFILE' not in os.environ:
+        real_tesseract()  # test not properly set up
     source = os.environ['_OCRMYPDF_TEST_INFILE']  # required
     args = parser.parse_args()
 

tests/spoof/tesseract_crash.py

@@ -50,7 +50,7 @@ def main():
     elif sys.argv[1] == '--list-langs':
         print('List of available languages (1):\neng', file=sys.stderr)
         sys.exit(0)
-    elif sys.argv[1] == '--print-parameters':
+    elif sys.argv[-2] == '--print-parameters':
         print('A parameter list would go here\ntextonly_pdf 0\n', file=sys.stderr)
         sys.exit(0)
     elif sys.argv[-2] == 'hocr':

tests/spoof/tesseract_noop.py

@@ -76,7 +76,7 @@ def main():
     elif sys.argv[1] == '--list-langs':
         print('List of available languages (1):\neng', file=sys.stderr)
         sys.exit(0)
-    elif sys.argv[1] == '--print-parameters':
+    elif sys.argv[-2] == '--print-parameters':
         print("Some parameters", file=sys.stderr)
         print("textonly_pdf\t1\tSome help text")
         sys.exit(0)

tests/test_lept.py

@@ -38,7 +38,8 @@ def test_colormap_backgroundnorm(resources):
 def crom_pix(resources):
     pix = lept.Pix.open(resources / 'crom.png')
     im = Image.open(resources / 'crom.png')
-    return pix, im
+    yield pix, im
+    im.close()
 
 
 def test_pix_basic(crom_pix):

tests/test_main.py

@@ -1,4 +1,4 @@
-# © 2015-17 James R. Barlow: github.com/jbarlow83
+# © 2015-19 James R. Barlow: github.com/jbarlow83
 #
 # This file is part of OCRmyPDF.
 #
@@ -18,10 +18,9 @@
 import logging
 import os
 import shutil
-import sys
 from math import isclose
 from pathlib import Path
-from subprocess import DEVNULL, PIPE, run, Popen
+from subprocess import PIPE, run
 
 import PIL
 import pytest
@@ -84,11 +83,6 @@ def spoof_no_tess_gs_raster_fail(tmp_path_factory):
     )
 
 
-@pytest.fixture(scope='session')
-def spoof_tess_bad_utf8(tmp_path_factory):
-    return spoof(tmp_path_factory, tesseract='tesseract_badutf8.py')
-
-
 def test_quick(spoof_tesseract_cache, resources, outpdf):
     check_ocrmypdf(resources / 'ccitt.pdf', outpdf, env=spoof_tesseract_cache)
 
@@ -124,8 +118,8 @@ def test_deskew(spoof_tesseract_noop, resources, outdir):
 
 def test_remove_background(spoof_tesseract_noop, resources, outdir):
     # Ensure the input image does not contain pure white/black
-    im = Image.open(resources / 'congress.jpg')
+    with Image.open(resources / 'congress.jpg') as im:
-    assert im.getextrema() != ((0, 255), (0, 255), (0, 255))
+        assert im.getextrema() != ((0, 255), (0, 255), (0, 255))
 
     output_pdf = check_ocrmypdf(
         resources / 'congress.jpg',
@@ -151,8 +145,8 @@ def test_remove_background(spoof_tesseract_noop, resources, outdir):
     )
 
     # The output image should contain pure white and black
-    im = Image.open(output_png)
+    with Image.open(output_png) as im:
-    assert im.getextrema() == ((0, 255), (0, 255), (0, 255))
+        assert im.getextrema() == ((0, 255), (0, 255), (0, 255))
 
 
 # This will run 5 * 2 * 2 = 20 test cases
@@ -223,7 +217,7 @@ def test_skip_ocr(spoof_tesseract_cache, resources, outpdf):
     assert pdfinfo[0].has_text
 
 
-def test_redo_ocr(spoof_tesseract_cache, resources, outpdf):
+def test_redo_ocr(resources, outpdf):
     in_ = resources / 'graph_ocred.pdf'
     before = PdfInfo(in_, detailed_page_analysis=True)
     out = outpdf
@@ -355,10 +349,9 @@ def test_german(spoof_tesseract_cache, resources, outdir):
         sidecar,
         env=spoof_tesseract_cache,
     )
-    print(os.environ)
+    if 'deu' not in tesseract.languages():
-    assert (
+        pytest.xfail(reason="tesseract-deu language pack not installed")
-        p.returncode == ExitCode.ok
+    assert p.returncode == ExitCode.ok, "Requires tesseract deu language pack"
-    ), "This test may fail if Tesseract language packs are missing"
 
 
 def test_klingon(resources, outpdf):
@@ -550,68 +543,6 @@ def test_jbig2_passthrough(spoof_tesseract_cache, resources, outpdf):
     assert out_pageinfo[0].images[0].enc == Encoding.jbig2
 
 
-def test_stdin(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf):
-    input_file = str(resources / 'francais.pdf')
-    output_file = str(outpdf)
-
-    # Runs: ocrmypdf - output.pdf < testfile.pdf
-    with open(input_file, 'rb') as input_stream:
-        p_args = ocrmypdf_exec + ['-', output_file]
-        p = run(
-            p_args,
-            stdout=PIPE,
-            stderr=PIPE,
-            stdin=input_stream,
-            env=spoof_tesseract_noop,
-        )
-        assert p.returncode == ExitCode.ok
-
-
-def test_stdout(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf):
-    input_file = str(resources / 'francais.pdf')
-    output_file = str(outpdf)
-
-    # Runs: ocrmypdf francais.pdf - > test_stdout.pdf
-    with open(output_file, 'wb') as output_stream:
-        p_args = ocrmypdf_exec + [input_file, '-']
-        p = run(
-            p_args,
-            stdout=output_stream,
-            stderr=PIPE,
-            stdin=DEVNULL,
-            env=spoof_tesseract_noop,
-        )
-        assert p.returncode == ExitCode.ok
-
-    assert qpdf.check(output_file, log=None)
-
-
-@pytest.mark.skipif(
-    sys.version_info[0:3] >= (3, 6, 4), reason="issue fixed in Python 3.6.4"
-)
-def test_closed_streams(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf):
-    input_file = str(resources / 'francais.pdf')
-    output_file = str(outpdf)
-
-    def evil_closer():
-        os.close(0)
-        os.close(1)
-
-    p_args = ocrmypdf_exec + [input_file, output_file]
-    p = Popen(  # pylint: disable=subprocess-popen-preexec-fn
-        p_args,
-        close_fds=True,
-        stdout=None,
-        stderr=PIPE,
-        stdin=None,
-        env=spoof_tesseract_noop,
-        preexec_fn=evil_closer,
-    )
-    out, err = p.communicate()
-    print(err.decode())
-    assert p.returncode == ExitCode.ok
-
-
 def test_masks(spoof_tesseract_noop, resources, outpdf):
     assert (
         ocrmypdf.ocr(
@@ -682,6 +613,7 @@ language_model_penalty_non_freq_dict_word 0
     )
 
 
+@pytest.mark.slow  # This test sometimes times out in CI
 @pytest.mark.parametrize('renderer', RENDERERS)
 def test_tesseract_config_notfound(renderer, resources, outdir):
     cfg_file = outdir / 'nofile.cfg'
@@ -698,6 +630,7 @@ def test_tesseract_config_notfound(renderer, resources, outdir):
     assert p.returncode == ExitCode.ok, err
 
 
+@pytest.mark.slow  # This test sometimes times out in CI
 @pytest.mark.parametrize('renderer', RENDERERS)
 def test_tesseract_config_invalid(renderer, resources, outdir):
     cfg_file = outdir / 'test.cfg'
@@ -720,26 +653,11 @@ THIS FILE IS INVALID
     assert p.returncode == ExitCode.invalid_config
 
 
-@pytest.mark.skipif(tesseract.v4(), reason='arg has no effect in 4.0-beta1')
+@pytest.mark.skipif(not tesseract.has_user_words(), reason='not functional until 4.1.0')
-def test_user_words(resources, outdir):
+def test_user_words_ocr(resources, outdir):
+    # Does not actually test if --user-words causes output to differ
     word_list = outdir / 'wordlist.txt'
-    sidecar_before = outdir / 'sidecar_before.txt'
+    sidecar_after = outdir / 'sidecar.txt'
-    sidecar_after = outdir / 'sidecar_after.txt'
-
-    # Don't know how to make this test pass on various versions and platforms
-    # so weaken to merely testing that the argument is accepted
-    consistent = False
-
-    if consistent:
-        check_ocrmypdf(
-            resources / 'crom.png',
-            outdir / 'out.pdf',
-            '--image-dpi',
-            150,
-            '--sidecar',
-            sidecar_before,
-        )
-        assert 'cromulent' not in sidecar_before.open().read()
 
     with word_list.open('w') as f:
         f.write('cromulent\n')  # a perfectly cromulent word
@@ -755,9 +673,6 @@ def test_user_words(resources, outdir):
         word_list,
     )
 
-    if consistent:
-        assert 'cromulent' in sidecar_after.open().read()
-
 
 def test_form_xobject(spoof_tesseract_noop, resources, outpdf):
     check_ocrmypdf(
@@ -878,6 +793,7 @@ def test_compression_preserved(
         assert pdfimage.color == Colorspace.rgb, "Colorspace changed"
     elif im.mode.startswith('L'):
         assert pdfimage.color == Colorspace.gray, "Colorspace changed"
+    im.close()
 
 
 @pytest.mark.parametrize(
@@ -939,6 +855,7 @@ def test_compression_changed(
         assert pdfimage.color == Colorspace.rgb, "Colorspace changed"
     elif im.mode.startswith('L'):
         assert pdfimage.color == Colorspace.gray, "Colorspace changed"
+    im.close()
 
 
 def test_sidecar_pagecount(spoof_tesseract_cache, resources, outpdf):
@@ -993,36 +910,6 @@ def test_pdfa_n(spoof_tesseract_cache, pdfa_level, resources, outpdf):
     assert pdfa_info['conformance'] == f'PDF/A-{pdfa_level}B'
 
 
-@pytest.mark.skipif(sys.version_info >= (3, 7, 0), reason='better utf-8')
-@pytest.mark.skipif(
-    Path('/etc/alpine-release').exists(), reason="invalid test on alpine"
-)
-def test_bad_locale():
-    env = os.environ.copy()
-    env['LC_ALL'] = 'C'
-
-    p, out, err = run_ocrmypdf('a', 'b', env=env)
-    assert out == '', "stdout not clean"
-    assert p.returncode != 0
-    assert 'configured to use ASCII as encoding' in err, "should whine"
-
-
-@pytest.mark.parametrize('renderer', RENDERERS)
-def test_bad_utf8(spoof_tess_bad_utf8, renderer, resources, no_outpdf):
-    p, out, err = run_ocrmypdf(
-        resources / 'ccitt.pdf',
-        no_outpdf,
-        '--pdf-renderer',
-        renderer,
-        env=spoof_tess_bad_utf8,
-    )
-
-    assert out == '', "stdout not clean"
-    assert p.returncode != 0
-    assert 'not utf-8' in err, "should whine about utf-8"
-    assert '\\x96' in err, 'should repeat backslash encoded output'
-
-
 @pytest.mark.skipif(
     PIL.__version__ < '5.0.0', reason="Pillow < 5.0.0 doesn't raise the exception"
 )
@@ -1051,14 +938,6 @@ def test_text_curves(spoof_tesseract_noop, resources, outpdf):
     assert len(info.pages[0].images) != 0, "force did not rasterize"
 
 
-def test_dev_null(spoof_tesseract_noop, resources):
-    p, out, err = run_ocrmypdf(
-        resources / 'trivial.pdf', os.devnull, '--force-ocr', env=spoof_tesseract_noop
-    )
-    assert p.returncode == 0, "could not send output to /dev/null"
-    assert len(out) == 0, "wrote to stdout"
-
-
 def test_output_is_dir(spoof_tesseract_noop, resources, outdir):
     p, out, err = run_ocrmypdf(
         resources / 'trivial.pdf', outdir, '--force-ocr', env=spoof_tesseract_noop
@@ -1125,3 +1004,14 @@ def test_fast_web_view(
     )
     with pikepdf.open(outpdf) as pdf:
         assert pdf.is_linearized == expected
+
+
+def test_image_dpi_not_image(caplog, spoof_tesseract_noop, resources, outpdf):
+    check_ocrmypdf(
+        resources / 'trivial.pdf',
+        outpdf,
+        '--image-dpi',
+        '100',
+        env=spoof_tesseract_noop,
+    )
+    assert '--image-dpi is being ignored' in caplog.text

tests/test_optimize.py

@@ -48,13 +48,14 @@ def test_mono_not_inverted(resources, outdir):
         xres=10,
         yres=10,
         raster_device='pnggray',
-        log=logging.getLogger(name='test_mono_flip'),
+        log=logging.getLogger(name='test_mono_not_inverted'),
     )
 
-    im = Image.open(fspath(outdir / 'im.png'))
+    with Image.open(fspath(outdir / 'im.png')) as im:
-    assert im.getpixel((0, 0)) == 255, "Expected white background"
+        assert im.getpixel((0, 0)) == 255, "Expected white background"
 
 
+@pytest.mark.skipif(not pngquant.available(), reason='need pngquant')
 def test_jpg_png_params(resources, outpdf, spoof_tesseract_noop):
     check_ocrmypdf(
         resources / 'crom.png',
@@ -109,10 +110,10 @@ def test_flate_to_jbig2(resources, outdir, spoof_tesseract_noop):
     # This test requires an image that pngquant is capable of converting to
     # to 1bpp - so use an existing 1bpp image, convert up, confirm it can
     # convert down
-    im = Image.open(fspath(resources / 'typewriter.png'))
+    with Image.open(fspath(resources / 'typewriter.png')) as im:
-    assert im.mode in ('1', 'P')
+        assert im.mode in ('1', 'P')
-    im = im.convert('L')
+        im = im.convert('L')
-    im.save(fspath(outdir / 'type8.png'))
+        im.save(fspath(outdir / 'type8.png'))
 
     check_ocrmypdf(
         outdir / 'type8.png',

tests/test_rotation.py

@@ -224,12 +224,12 @@ def test_rotate_deskew_timeout(resources, outdir):
 @pytest.mark.parametrize('image_angle', (0, 90, 180, 270))
 def test_rotate_page_level(image_angle, page_angle, resources, outdir):
     def make_rotate_test(prefix, image_angle, page_angle):
-        im = Image.open(fspath(resources / 'typewriter.png'))
-        if image_angle != 0:
-            ccw_angle = -image_angle % 360
-            im = im.transpose(getattr(Image, f'ROTATE_{ccw_angle}'))
         memimg = BytesIO()
-        im.save(memimg, format='PNG')
+        with Image.open(fspath(resources / 'typewriter.png')) as im:
+            if image_angle != 0:
+                ccw_angle = -image_angle % 360
+                im = im.transpose(getattr(Image, f'ROTATE_{ccw_angle}'))
+            im.save(memimg, format='PNG')
         memimg.seek(0)
         mempdf = BytesIO()
         img2pdf.convert(

tests/test_stdio.py

@@ -0,0 +1,137 @@
+# © 2019 James R. Barlow: github.com/jbarlow83
+#
+# This file is part of OCRmyPDF.
+#
+# OCRmyPDF is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# OCRmyPDF is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import sys
+from pathlib import Path
+from subprocess import DEVNULL, PIPE, run, Popen
+
+import pytest
+
+from ocrmypdf.exceptions import ExitCode
+from ocrmypdf.exec import qpdf
+
+# pytest.helpers is dynamic
+# pylint: disable=no-member,redefined-outer-name
+
+run_ocrmypdf = pytest.helpers.run_ocrmypdf
+spoof = pytest.helpers.spoof
+
+
+@pytest.fixture(scope='session')
+def spoof_tess_bad_utf8(tmp_path_factory):
+    return spoof(tmp_path_factory, tesseract='tesseract_badutf8.py')
+
+
+def test_stdin(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf):
+    input_file = str(resources / 'francais.pdf')
+    output_file = str(outpdf)
+
+    # Runs: ocrmypdf - output.pdf < testfile.pdf
+    with open(input_file, 'rb') as input_stream:
+        p_args = ocrmypdf_exec + ['-', output_file]
+        p = run(
+            p_args,
+            stdout=PIPE,
+            stderr=PIPE,
+            stdin=input_stream,
+            env=spoof_tesseract_noop,
+        )
+        assert p.returncode == ExitCode.ok
+
+
+def test_stdout(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf):
+    input_file = str(resources / 'francais.pdf')
+    output_file = str(outpdf)
+
+    # Runs: ocrmypdf francais.pdf - > test_stdout.pdf
+    with open(output_file, 'wb') as output_stream:
+        p_args = ocrmypdf_exec + [input_file, '-']
+        p = run(
+            p_args,
+            stdout=output_stream,
+            stderr=PIPE,
+            stdin=DEVNULL,
+            env=spoof_tesseract_noop,
+        )
+        assert p.returncode == ExitCode.ok
+
+    assert qpdf.check(output_file, log=None)
+
+
+@pytest.mark.skipif(
+    sys.version_info[0:3] >= (3, 6, 4), reason="issue fixed in Python 3.6.4"
+)
+def test_closed_streams(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf):
+    input_file = str(resources / 'francais.pdf')
+    output_file = str(outpdf)
+
+    def evil_closer():
+        os.close(0)
+        os.close(1)
+
+    p_args = ocrmypdf_exec + [input_file, output_file]
+    p = Popen(  # pylint: disable=subprocess-popen-preexec-fn
+        p_args,
+        close_fds=True,
+        stdout=None,
+        stderr=PIPE,
+        stdin=None,
+        env=spoof_tesseract_noop,
+        preexec_fn=evil_closer,
+    )
+    out, err = p.communicate()
+    print(err.decode())
+    assert p.returncode == ExitCode.ok
+
+
+@pytest.mark.skipif(sys.version_info >= (3, 7, 0), reason='better utf-8')
+@pytest.mark.skipif(
+    Path('/etc/alpine-release').exists(), reason="invalid test on alpine"
+)
+def test_bad_locale():
+    env = os.environ.copy()
+    env['LC_ALL'] = 'C'
+
+    p, out, err = run_ocrmypdf('a', 'b', env=env)
+    assert out == '', "stdout not clean"
+    assert p.returncode != 0
+    assert 'configured to use ASCII as encoding' in err, "should whine"
+
+
+@pytest.mark.parametrize('renderer', ['hocr', 'sandwich'])
+def test_bad_utf8(spoof_tess_bad_utf8, renderer, resources, no_outpdf):
+    p, out, err = run_ocrmypdf(
+        resources / 'ccitt.pdf',
+        no_outpdf,
+        '--pdf-renderer',
+        renderer,
+        env=spoof_tess_bad_utf8,
+    )
+
+    assert out == '', "stdout not clean"
+    assert p.returncode != 0
+    assert 'not utf-8' in err, "should whine about utf-8"
+    assert '\\x96' in err, 'should repeat backslash encoded output'
+
+
+def test_dev_null(spoof_tesseract_noop, resources):
+    p, out, err = run_ocrmypdf(
+        resources / 'trivial.pdf', os.devnull, '--force-ocr', env=spoof_tesseract_noop
+    )
+    assert p.returncode == 0, "could not send output to /dev/null"
+    assert len(out) == 0, "wrote to stdout"

tests/test_validation.py

@@ -15,20 +15,23 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
+import locale
+import logging
 import os
-from unittest.mock import MagicMock, patch
+from unittest.mock import patch
 
 import pytest
 
 import ocrmypdf._validation as vd
 from ocrmypdf.api import create_options
 from ocrmypdf.exceptions import MissingDependencyError, BadArgsError
+from ocrmypdf.pdfinfo import PdfInfo
 
 
 def make_opts(input_file='a.pdf', output_file='b.pdf', language='eng', **kwargs):
-    return create_options(
+    if language is not None:
-        input_file=input_file, output_file=output_file, language=language, **kwargs
+        kwargs['language'] = language
-    )
+    return create_options(input_file=input_file, output_file=output_file, **kwargs)
 
 
 def test_hocr_notlatin_warning(caplog):
@@ -37,15 +40,21 @@ def test_hocr_notlatin_warning(caplog):
 
 
 def test_old_ghostscript(caplog):
-    with patch('ocrmypdf.exec.ghostscript.version', return_value='9.19'):
+    with patch('ocrmypdf.exec.ghostscript.version', return_value='9.19'), patch(
+        'ocrmypdf.exec.tesseract.has_textonly_pdf', return_value=True
+    ):
         vd.check_options_output(make_opts(language='chi_sim', output_type='pdfa'))
         assert 'Ghostscript does not work correctly' in caplog.text
 
-    with patch('ocrmypdf.exec.ghostscript.version', return_value='9.18'):
+    with patch('ocrmypdf.exec.ghostscript.version', return_value='9.18'), patch(
+        'ocrmypdf.exec.tesseract.has_textonly_pdf', return_value=True
+    ):
         with pytest.raises(MissingDependencyError):
             vd.check_options_output(make_opts(output_type='pdfa-3'))
 
-    with patch('ocrmypdf.exec.ghostscript.version', return_value='9.24'):
+    with patch('ocrmypdf.exec.ghostscript.version', return_value='9.24'), patch(
+        'ocrmypdf.exec.tesseract.has_textonly_pdf', return_value=True
+    ):
         with pytest.raises(MissingDependencyError):
             vd.check_dependency_versions(make_opts())
 
@@ -113,3 +122,41 @@ def test_report_file_size(tmp_path, caplog):
     os.truncate(out, 50000)
     vd.report_output_file_size(opts, in_, out)
     assert 'No reason' in caplog.text
+
+
+def test_false_action_store_true():
+    opts = make_opts(keep_temporary_files=True)
+    assert opts.keep_temporary_files == True
+    opts = make_opts(keep_temporary_files=False)
+    assert opts.keep_temporary_files == False
+
+
+@pytest.mark.parametrize('progress_bar', [True, False])
+def test_no_progress_bar(progress_bar, resources):
+    opts = make_opts(progress_bar=progress_bar, input_file=(resources / 'trivial.pdf'))
+    with patch('ocrmypdf.pdfinfo.info.tqdm', autospec=True) as tqdmpatch:
+        vd.check_options(opts)
+        pdfinfo = PdfInfo(opts.input_file, progbar=opts.progress_bar)
+        assert pdfinfo is not None
+        assert tqdmpatch.called
+        _args, kwargs = tqdmpatch.call_args
+        assert kwargs['disable'] != progress_bar
+
+
+def test_language_warning(caplog):
+    opts = make_opts(language=None)
+    caplog.set_level(logging.DEBUG)
+    with patch(
+        'ocrmypdf._validation.locale.getlocale', return_value=('en_US', 'UTF-8')
+    ):
+        vd.check_options_languages(opts)
+        assert opts.language == ['eng']
+        assert '' in caplog.text
+
+    opts = make_opts(language=None)
+    with patch(
+        'ocrmypdf._validation.locale.getlocale', return_value=('fr_FR', 'UTF-8')
+    ):
+        vd.check_options_languages(opts)
+        assert opts.language == ['eng']
+        assert 'assuming --language' in caplog.text