v9.1.1 release notes

Smaller download, needed for manylinux2010.

.docker/Dockerfile

@@ -1,6 +1,6 @@
 # OCRmyPDF
 #
-FROM ubuntu:19.04 as base
+FROM ubuntu:19.10 as base
 
 FROM base as builder
 
@@ -10,36 +10,36 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   build-essential autoconf automake libtool \
   libleptonica-dev \
   zlib1g-dev \
-  ocrmypdf \
+  python3 \
-  pngquant \
+  python3-distutils \
-  python3-pip \
+  ca-certificates \
-  python3-venv \
+  curl \
-  tesseract-ocr \
-  unpaper \
-  wget \
   git
 
+# Get the latest pip (Ubuntu version doesn't support manylinux2010)
+RUN \
+  curl https://bootstrap.pypa.io/get-pip.py | python3
 
 # Compile and install jbig2
 # Needs libleptonica-dev, zlib1g-dev
 RUN \
   mkdir jbig2 \
-  && wget -q https://github.com/agl/jbig2enc/archive/0.29.tar.gz -O - | \
+  && curl -L https://github.com/agl/jbig2enc/archive/0.29.tar.gz | \
   tar xz -C jbig2 --strip-components=1 \
   && cd jbig2 \
   && ./autogen.sh && ./configure && make && make install \
   && cd .. \
   && rm -rf jbig2
 
-RUN python3 -m venv /appenv
-
 COPY . /app
 
 WORKDIR /app
 
-RUN . /appenv/bin/activate; \
+RUN pip3 install --no-cache-dir \
-  pip install --upgrade pip \
+  -r requirements/main.txt \
-  && pip install .
+  -r requirements/webservice.txt \
+  -r requirements/test.txt \
+  .
 
 FROM base
 
@@ -53,7 +53,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   zlib1g \
   pngquant \
   python3 \
-  python3-venv \
   qpdf \
   tesseract-ocr \
   tesseract-ocr-chi-sim \
@@ -62,10 +61,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   tesseract-ocr-fra \
   tesseract-ocr-por \
   tesseract-ocr-spa \
-  unpaper \
+  unpaper
-  wget
+
+WORKDIR /app
+
+COPY --from=builder /usr/local/lib/ /usr/local/lib/
+COPY --from=builder /usr/local/bin/ /usr/local/bin/
 
-# Copy
 COPY --from=builder /app/misc/webservice.py /app/
 
 # Copy minimal project files to get the test suite.
@@ -74,7 +76,4 @@ COPY --from=builder /app/requirements /app/requirements
 COPY --from=builder /app/tests /app/tests
 COPY --from=builder /app/src /app/src
 
-COPY --from=builder /appenv /appenv
+ENTRYPOINT ["/usr/local/bin/ocrmypdf"]
-COPY --from=builder /usr/local /usr/local
-
-ENTRYPOINT ["/appenv/bin/ocrmypdf"]

.docker/alpine.dockerfile

@@ -1,91 +0,0 @@
-FROM alpine:3.9 as base
-
-FROM base as builder
-
-ENV LANG=C.UTF-8
-
-# Normally:
-# echo '@testing http://nl.alpinelinux.org/alpine/edge/testing' >> /etc/apk/repositories
-
-RUN \
-  echo -e '@testing http://nl.alpinelinux.org/alpine/edge/testing\n@community http://nl.alpinelinux.org/alpine/edge/community'\
-  >> /etc/apk/repositories \
-  # Add runtime dependencies
-  && apk add --update \
-  python3-dev \
-  py3-setuptools \
-  jbig2enc@testing \
-  ghostscript \
-  qpdf@community \
-  qpdf-dev@community \
-  tesseract-ocr \
-  unpaper \
-  pngquant \
-  libxml2-dev \
-  libxslt-dev \
-  zlib-dev \
-  libffi-dev \
-  leptonica-dev \
-  binutils \
-  && pip3 install --upgrade pip \
-  # Install pybind11 for pikepdf
-  && pip3 install pybind11 \
-  # Install flask for the webservice
-  && pip3 install flask \
-  # Add build dependencies
-  && apk add --virtual build-dependencies \
-  build-base \
-  git
-
-COPY . /app
-
-WORKDIR /app
-
-RUN pip3 install .
-
-FROM base
-
-ENV LANG=C.UTF-8
-
-# Normally:
-# echo '@testing http://nl.alpinelinux.org/alpine/edge/testing' >> /etc/apk/repositories
-
-RUN \
-  echo -e '@testing http://nl.alpinelinux.org/alpine/edge/testing\n@community http://nl.alpinelinux.org/alpine/edge/community'\
-  >> /etc/apk/repositories \
-  # Add runtime dependencies
-  && apk add --update \
-  python3 \
-  jbig2enc@community \
-  ghostscript \
-  qpdf@community \
-  qpdf-dev@community \
-  tesseract-ocr \
-  tesseract-ocr-data-deu \
-  tesseract-ocr-data-chi_sim \
-  unpaper \
-  pngquant \
-  libxml2 \
-  libxslt \
-  zlib \
-  libffi \
-  leptonica-dev \
-  binutils \
-  && mkdir /app
-
-WORKDIR /app
-
-# Copy build artifacts (python site-packages)
-COPY --from=builder /usr/lib/python3.6/site-packages /usr/lib/python3.6/site-packages
-COPY --from=builder /usr/bin/ocrmypdf /usr/bin/dumppdf.py /usr/bin/latin2ascii.py /usr/bin/pdf2txt.py /usr/bin/img2pdf /usr/bin/chardetect /usr/bin/
-
-# Copy
-COPY --from=builder /app/misc/webservice.py /app/
-
-# Copy minimal project files to get the test suite.
-COPY --from=builder /app/setup.cfg /app/setup.py /app/README.md /app/
-COPY --from=builder /app/requirements /app/requirements
-COPY --from=builder /app/tests /app/tests
-COPY --from=builder /app/src /app/src
-
-ENTRYPOINT ["/usr/bin/ocrmypdf"]

.travis.yml

@@ -62,8 +62,6 @@ matrix:
         mkdir -p bin packages
         pip3 install --upgrade pip
         pip3 install --upgrade wheel
-        wget -q 'https://www.dropbox.com/s/vaq0kbwi6e6au80/unpaper_6.1-1.deb?raw=1' -O packages/unpaper_6.1-1.deb
-        sudo dpkg -i packages/unpaper_6.1-1.deb
     - os: linux
       dist: xenial
       sudo: required
@@ -88,8 +86,31 @@ matrix:
             - tesseract-ocr-eng
             - tesseract-ocr-fra
             - unpaper
+    - os: linux
+      dist: bionic
+      sudo: required
+      language: python
+      python: "3.8"
+      env:
+        - DIST=bionic
+      addons:
+        apt:
+          update: true
+          sources:
+            - sourceline: "ppa:alex-p/tesseract-ocr"
+          packages:
+            - ghostscript
+            - libexempi3
+            - libffi-dev
+            - pngquant
+            - poppler-utils
+            - qpdf
+            - tesseract-ocr
+            - tesseract-ocr-deu
+            - tesseract-ocr-eng
+            - tesseract-ocr-fra
+            - unpaper
     - os: osx
-      osx_image: xcode9.2
       language: generic
       addons:
         homebrew:
@@ -115,10 +136,7 @@ before_cache:
 install:
   - mkdir -p bin
   - export PATH=$PWD/bin:$PATH
-  - pip3 install pycparser # py3.7 workaround for https://github.com/eliben/pycparser/issues/251
+  - pip3 install -r requirements/main.txt -r requirements/test.txt .
-  - pip3 install -r requirements/main.txt
-  - pip3 install --no-deps .
-  - pip3 install -r requirements/test.txt
 
 script:
   - tesseract --version
@@ -127,7 +145,7 @@ script:
 
 deploy:
   # release for main pypi
-  # 3.6 is considered the build leader and does the deploy, otherwise there is
+  # 3.7 is considered the build leader and does the deploy, otherwise there is
   # a race and all versions will try to deploy
   # OTOH if we ever need separate binary wheels then each version needs its
   # own deploy
@@ -139,5 +157,5 @@ deploy:
     on:
       branch: master
       tags: true
-      condition: $TRAVIS_PYTHON_VERSION == "3.6" && $TRAVIS_OS_NAME == "linux"
+      condition: $TRAVIS_PYTHON_VERSION == "3.7" && $TRAVIS_OS_NAME == "linux"
     skip_upload_docs: true

MANIFEST.in

@@ -1,43 +0,0 @@
-# requirements
-recursive-include requirements *
-
-# git
-include .git_archival.txt
-
-# docker
-include .dockerignore
-recursive-include .docker *
-
-# tests
-include .coveragerc
-recursive-include tests *.bin
-recursive-include tests *.jpg
-recursive-include tests *.jsonl
-recursive-include tests *.png
-recursive-include tests *.pdf
-recursive-include tests *.py
-recursive-include tests *.rst
-recursive-include tests *.txt
-recursive-exclude tests/resources/private *
-
-# documentation
-include LICENSE
-include *.rst
-recursive-exclude .github *
-recursive-include docs *.py
-recursive-include docs *.rst
-recursive-include docs *.svg
-recursive-exclude docs/_build *
-
-
-# support files
-recursive-include src/ocrmypdf/data *
-include *.py
-exclude tasks.py
-recursive-exclude .travis *
-exclude .travis*
-
-
-# code
-exclude src/ocrmypdf/lib/_leptonica.py
-exclude scratch.py

README.md

@@ -1,6 +1,6 @@
 <img src="docs/images/logo.svg" width="240" alt="OCRmyPDF">
 
-[![Travis build status][travis]](https://travis-ci.org/jbarlow83/OCRmyPDF) [![PyPI version][pypi]](https://pypi.org/project/ocrmypdf/) ![Homebrew version][homebrew] ![ReadTheDocs][docs]
+[![Travis build status][travis]](https://travis-ci.org/jbarlow83/OCRmyPDF) [![PyPI version][pypi]](https://pypi.org/project/ocrmypdf/) ![Homebrew version][homebrew] ![ReadTheDocs][docs] ![Python versions][pyversions]
 
 [travis]: https://travis-ci.org/jbarlow83/OCRmyPDF.svg?branch=master "Travis build status"
 
@@ -10,6 +10,8 @@
 
 [docs]: https://readthedocs.org/projects/ocrmypdf/badge/?version=latest "RTD"
 
+[pyversions]: https://img.shields.io/pypi/pyversions/ocrmypdf "Supported Python versions"
+
 OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched or copy-pasted.
 
 ```bash
@@ -120,7 +122,7 @@ If you detect an issue, please:
 Requirements
 ------------
 
-Runs on CPython 3.5, 3.6 and 3.7. Requires external program installations of Ghostscript, Tesseract OCR, QPDF, and Leptonica. ocrmypdf is pure Python, but uses CFFI to portably generate library bindings.
+In addition to the required Python version (3.6+), OCRmyPDF requires external program installations of Ghostscript, Tesseract OCR, QPDF, and Leptonica. ocrmypdf is pure Python, but uses CFFI to portably generate library bindings.
 
 Press & Media
 -------------

docs/advanced.rst

@@ -322,15 +322,23 @@ working files on a per page basis have the page number as a prefix
 (starting with page 1), an infix indicates the processing stage, and a
 suffix indicates the file type. Some important files include:
 
--  ``.page.png`` - what the input page looks like
+-  ``_rasterize.png`` - what the input page looks like
--  ``.image`` - the image we will show the user if we are in a mode that
+-  ``_ocr.png`` - the file that is sent to Tesseract for OCR; depending
-   changes the final appearance; may be in one of several image formats
--  ``.text.pdf`` - the OCR file; this will load as a blank page but
-   should have visible text if checked with a tool like pdftotext or
-   pdfminder.six
--  ``.ocr.png`` - the file that is sent to Tesseract for OCR; depending
    on arguments this may differ from the presentation image
--  ``layers.rendered.pdf`` - the composite PDF, before metadata repair
+-  ``_pp_deskew.png`` - the image, after deskewing
-   and optimization
+-  ``_pp_clean.png`` - the image, after cleaning with unpaper
+-  ``_ocr_tess.pdf`` - the OCR file; appears as a blank page with invisible
+   text embedded
+-  ``_ocr_tess.txt`` - the OCR text (not necessarily all text on the page,
+   if the page is mixed format)
+-  ``fix_docinfo.pdf`` - a temporary file created to fix the PDF DocumentInfo
+   data structure
+-  ``graft_layers.pdf`` - the rendered PDF with OCR layers grafted on
+-  ``pdfa.pdf`` - ``graft_layers.pdf`` after conversion to PDF/A
+-  ``pdfa.ps`` - a PostScript file used by Ghostscript for PDF/A conversion
+-  ``optimize.pdf`` - the PDF generated before optimization
+-  ``optimize.out.pdf`` - the PDF generated by optimization
+-  ``origin`` - the input file
+-  ``origin.pdf`` - the input file or the input image converted to PDF
 -  ``images/*`` - images extracted during the optimization process; here
    the prefix indicates a PDF object ID not a page number

docs/api.rst

@@ -71,7 +71,8 @@ Progress monitoring
 OCRmyPDF uses the ``tqdm`` package to implement its progress bars.
 :func:`ocrmypdf.configure_logging` will set up logging output to
 ``sys.stderr`` in a way that is compatible with the display of the
-progress bar.
+progress bar. Use ``ocrmypdf.ocr(...progress_bar=False)`` to disable
+the progress bar.
 
 Exceptions
 ----------

docs/batch.rst

@@ -47,7 +47,7 @@ where the PDFs are stored):
 
 .. code-block:: bash
 
-   find . -printf '%p' -name '*.pdf' -exec docker run --rm -v <host dir>:<container dir> jbarlow83/ocrmypdf-alpine '<container dir>/{}' '<container dir>/{}' \;
+   find . -printf '%p' -name '*.pdf' -exec docker run --rm -v <host dir>:<container dir> jbarlow83/ocrmypdf '<container dir>/{}' '<container dir>/{}' \;
 
 This only runs one ``ocrmypdf`` process at a time. This variation uses
 ``find`` to create a directory list and ``parallel`` to parallelize runs
@@ -244,6 +244,9 @@ Caveats
 Alternatives
 ------------
 
+-  `systemd user services <https://wiki.archlinux.org/index.php/Systemd/User>`__
+   can be configured to automatically perform OCR on a collection of files.
+
 -  `Watchman <https://facebook.github.io/watchman/>`__ is a more
    powerful alternative to ``watchmedo``.
 

docs/contributing.rst

@@ -0,0 +1,38 @@
+=======================
+Contributing guidelines
+=======================
+
+Contributions are welcome!
+
+Big changes
+===========
+
+Please open a new issue to discuss or propose a major change. Not only is it fun
+to discuss big ideas, but we might save each other's time too. Perhaps some of the
+work you're contemplating is already half-done in a development branch.
+
+Code style
+==========
+
+We use PEP8, ``black`` for code formatting and ``isort`` for import sorting. The
+settings for programs are in ``pyproject.toml`` and ``setup.cfg``.
+
+Tests
+=====
+
+New features should come with tests that confirm their correctness.
+
+New Python dependencies
+=======================
+
+If you are proposing a change that will require a new Python dependency, we
+prefer dependencies that are already packaged by Debian or Red Hat. This makes
+life much easier for our downstream package maintainers.
+
+Python dependencies must also be GPLv3 compatible.
+
+New non-Python dependencies
+===========================
+
+OCRmyPDF uses several external programs (Tesseract, Ghostscript and others) for
+its functionality. In general we prefer to avoid adding new external programs.

docs/cookbook.rst

@@ -165,8 +165,8 @@ might remove desirable content, especially from poor quality scans.
 -  ``--deskew`` will correct pages were scanned at a skewed angle by
    rotating them back into place. Skew determination and correction is
    performed using `Postl's variance of line
-   sums <http://www.leptonica.com/skew-measurement.html>`__ algorithm as
+   sums <http://www.leptonica.org/skew-measurement.html>`__ algorithm as
-   implemented in `Leptonica <http://www.leptonica.com/index.html>`__.
+   implemented in `Leptonica <http://www.leptonica.org/index.html>`__.
 -  ``--clean`` uses
    `unpaper <https://www.flameeyes.eu/projects/unpaper>`__ to clean up
    pages before OCR, but does not alter the final output. This makes it
@@ -216,6 +216,15 @@ processing or PDF/A conversion.
 
     ocrmypdf --tesseract-timeout=0 --remove-background input.pdf output.pdf
 
+Optimize images without performing OCR
+--------------------------------------
+
+You can also optimize all images without performing any OCR:
+
+.. code-block:: bash
+
+    ocrmypdf --tesseract-timeout=0 --optimize 3 --skip-text input.pdf output.pdf
+
 Redo existing OCR
 =================
 

docs/docker.rst

@@ -22,21 +22,20 @@ Installing the Docker image
 If you have `Docker <https://docs.docker.com/>`__ installed on your
 system, you can install a Docker image of the latest release.
 
-The recommended OCRmyPDF Docker image is currently named
+If you can run this command successfully, your system is ready to download and
-``ocrmypdf-alpine``:
-
-.. code-block:: bash
-
-   docker pull jbarlow83/ocrmypdf-alpine
-
-Follow the Docker installation instructions for your platform. If you
-can run this command successfully, your system is ready to download and
 execute the image:
 
 .. code-block:: bash
 
    docker run hello-world
 
+The recommended OCRmyPDF Docker image is currently named ``ocrmypdf``:
+
+.. code-block:: bash
+
+   docker pull jbarlow83/ocrmypdf
+
+
 OCRmyPDF will use all available CPU cores. By default, the VirtualBox
 machine instance on Windows and macOS has only a single CPU core
 enabled. Use the VirtualBox Manager to determine the name of your Docker
@@ -51,6 +50,9 @@ CPUs:
    docker-machine start "yourVM"
    eval $(docker-machine env "yourVM")
 
+See the Docker documentation for
+`adjusting memory and CPU on other platforms <https://docs.docker.com/config/containers/resource_constraints/>`__.
+
 Using the Docker image on the command line
 ==========================================
 
@@ -63,7 +65,7 @@ To start a Docker container (instance of the image):
 
 .. code-block:: bash
 
-   docker tag jbarlow83/ocrmypdf-alpine ocrmypdf
+   docker tag jbarlow83/ocrmypdf ocrmypdf
    docker run --rm -i ocrmypdf (... all other arguments here...)
 
 For convenience, create a shell alias to hide the Docker command. It is
@@ -103,12 +105,12 @@ on the public one:
 
 .. code-block:: dockerfile
 
-   FROM jbarlow83/ocrmypdf-alpine
+   FROM jbarlow83/ocrmypdf
 
    # Add French
-   RUN apk add tesseract-ocr-data-fra
+   RUN apt install tesseract-ocr-fra
 
-You can also copy training data to ``/usr/share/tessdata``.
+You can also copy training data to ``/usr/share/tesseract-ocr/<tesseract version>/tessdata``.
 
 Executing the test suite
 ========================
@@ -117,17 +119,16 @@ The OCRmyPDF test suite is installed with image. To run it:
 
 .. code-block:: bash
 
-   docker run --entrypoint python3 jbarlow83/ocrmypdf-alpine setup.py test
+   docker run --entrypoint python3  jbarlow83/ocrmypdf -m pytest
 
 Accessing the shell
 ===================
 
-``bash`` is not installed in the image. To use the busybox shell in the
+To use the bash shell in the Docker image:
-Docker image:
 
 .. code-block:: bash
 
-   docker run -it --entrypoint busybox  jbarlow83/ocrmypdf-alpine sh
+   docker run -it --entrypoint bash  jbarlow83/ocrmypdf
 
 Using the OCRmyPDF web service wrapper
 ======================================
@@ -137,7 +138,12 @@ service. The webservice may be launched as follows:
 
 .. code-block:: bash
 
-   docker run --entrypoint python3 -p 5000:5000 jbarlow83/ocrmypdf-alpine webservice.py
+   docker run --entrypoint python3 -p 5000:5000  jbarlow83/ocrmypdf webservice.py
+
+This will configure the machine to listen on port 5000. On Linux machines
+this is port 5000 of localhost. On macOS or Windows machines running
+Docker, this is port 5000 of the virtual machine that runs your Docker
+images. You can find its IP address using the command ``docker-machine ip``.
 
 Unlike command line usage this program will open a socket and wait for
 connections.
@@ -162,14 +168,3 @@ also licensed in this way.
 
 In addition to the above, please read our
 :ref:`general remarks on using OCRmyPDF as a service <ocr-service>`.
-
-Ubuntu-based Docker image
-=========================
-
-A Ubuntu-based OCRmyPDF image is also available. The main advantage this
-image offers is that it supports manylinux Python wheels (which are not
-supported on Alpine Linux). This may be useful for plugins.
-
-.. code-block:: bash
-
-   docker pull jbarlow83/ocrmypdf

docs/index.rst

@@ -12,6 +12,7 @@ PDF is the best format for storing and exchanging scanned documents.  Unfortunat
    introduction
    release_notes
    installation
+   optimizer
    languages
    jbig2
 
@@ -22,11 +23,16 @@ PDF is the best format for storing and exchanging scanned documents.  Unfortunat
    cookbook
    docker
    advanced
-   api
    batch
    security
    errors
 
+.. toctree::
+   :caption: Developers
+   :maxdepth: 2
+
+   api
+   contributing
 
 Indices and tables
 ==================

docs/installation.rst

@@ -21,7 +21,7 @@ installing the Python binary wheels.
 Installing on Linux
 ===================
 
-Debian and Ubuntu 16.10 or newer
+Debian and Ubuntu 18.04 or newer
 --------------------------------
 
 .. |deb-stable| image:: https://repology.org/badge/version-for-repo/debian_stable/ocrmypdf.svg
@@ -33,27 +33,29 @@ Debian and Ubuntu 16.10 or newer
 .. |deb-unstable| image:: https://repology.org/badge/version-for-repo/debian_unstable/ocrmypdf.svg
     :alt: Debian unstable
 
-.. |ubu-1710| image:: https://repology.org/badge/version-for-repo/ubuntu_17_10/ocrmypdf.svg
-    :alt: Ubuntu 17.10
-
 .. |ubu-1804| image:: https://repology.org/badge/version-for-repo/ubuntu_18_04/ocrmypdf.svg
     :alt: Ubuntu 18.04 LTS
 
 .. |ubu-1810| image:: https://repology.org/badge/version-for-repo/ubuntu_18_10/ocrmypdf.svg
     :alt: Ubuntu 18.10
 
+.. |ubu-1904| image:: https://repology.org/badge/version-for-repo/ubuntu_19_04/ocrmypdf.svg
+    :alt: Ubuntu 19.04
 
-+-------------------------------------------+
+.. |ubu-1910| image:: https://repology.org/badge/version-for-repo/ubuntu_19_10/ocrmypdf.svg
-| **OCRmyPDF versions in Debian & Ubuntu**  |
+    :alt: Ubuntu 19.10
-+-------------------------------------------+
-| |latest|                                  |
-+-------------------------------------------+
-| |deb-stable| |deb-testing| |deb-unstable| |
-+-------------------------------------------+
-| |ubu-1710| |ubu-1804| |ubu-1810|          |
-+-------------------------------------------+
 
-Users of Debian 9 ("stretch") or later or Ubuntu 16.10 or later may
++-----------------------------------------------+
+| **OCRmyPDF versions in Debian & Ubuntu**      |
++-----------------------------------------------+
+| |latest|                                      |
++-----------------------------------------------+
+| |deb-stable| |deb-testing| |deb-unstable|     |
++-----------------------------------------------+
+| |ubu-1804| |ubu-1810| |ubu-1904| |ubu-1910|   |
++-----------------------------------------------+
+
+Users of Debian 9 ("stretch") or later or Ubuntu 18.04 or later may
 simply
 
 .. code-block:: bash
@@ -64,7 +66,8 @@ As indicated in the table above, Debian and Ubuntu releases may lag
 behind the latest version. If the version available for your platform is
 out of date, you could opt to install the latest version from source.
 See `Installing HEAD revision from
-sources <#installing-head-revision-from-sources>`__.
+sources <#installing-head-revision-from-sources>`__. Ubuntu 16.10 to 17.10
+inclusive also had ocrmypdf, but these versions are end of life.
 
 For full details on version availability for your platform, check the
 `Debian Package Tracker <https://tracker.debian.org/pkg/ocrmypdf>`__ or
@@ -81,19 +84,22 @@ For full details on version availability for your platform, check the
 Fedora 29 or newer
 ------------------
 
-.. |fedora-29| image:: https://repology.org/badge/version-for-repo/fedora29/ocrmypdf.svg
+.. |fedora-29| image:: https://repology.org/badge/version-for-repo/fedora_29/ocrmypdf.svg
     :alt: Fedora 29
 
+.. |fedora-30| image:: https://repology.org/badge/version-for-repo/fedora_30/ocrmypdf.svg
+    :alt: Fedora 30
+
 .. |fedora-rawhide| image:: https://repology.org/badge/version-for-repo/fedora_rawhide/ocrmypdf.svg
     :alt: Fedore Rawhide
 
-+------------------------------+
++-----------------------------------------------+
-| **OCRmyPDF version**         |
+| **OCRmyPDF version**                          |
-+------------------------------+
++-----------------------------------------------+
-| |latest|                     |
+| |latest|                                      |
-+------------------------------+
++-----------------------------------------------+
-| |fedora-29| |fedora-rawhide| |
+| |fedora-29| |fedora-30| |fedora-rawhide|      |
-+------------------------------+
++-----------------------------------------------+
 
 Users of Fedora 29 later may simply
 
@@ -128,8 +134,7 @@ first install the system version to get most of the dependencies:
 
     sudo apt-get update
     sudo apt-get install \
-        ocrmypdf \
+        ocrmypdf
-        python3-pip
 
 There are a few system dependency changes since ocrmypdf 6.1.2. Let's
 get these, too.
@@ -140,13 +145,19 @@ get these, too.
         libxml2 \
         pngquant
 
+We will need a newer version of ``pip`` then was available for Ubuntu 18.04:
+
+.. code-block:: bash
+
+    wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py
+
 Then install the most recent ocrmypdf for the local user and set the
 user's ``PATH`` to check for the user's Python packages.
 
 .. code-block:: bash
 
     export PATH=$HOME/.local/bin:$PATH
-    pip3 install --user ocrmypdf
+    python3 -m pip install --user ocrmypdf
 
 To add JBIG2 encoding, see :ref:`jbig2`.
 
@@ -266,15 +277,11 @@ Now we need to install ``pip`` and let it install ocrmypdf:
     curl https://bootstrap.pypa.io/ez_setup.py -o - | python3.6 && python3.6 -m easy_install pip
     pip3.6 install ocrmypdf
 
-These installation instructions omit the optional dependency
+The optional dependency ``unpaper`` is only available at 0.4.2 in Ubuntu 14.04,
-``unpaper``, which is only available at version 0.4.2 in Ubuntu 14.04.
+and no backports are available. Previously the author maintained a backported
-The author could not find a backport of ``unpaper``, and created a .deb
+.deb package for unpaper 6.1, but since Ubuntu 14.04 is now end of life, this is
-package to do the job of installing unpaper 6.1 (for x86 64-bit only):
+not supported. As such, ``unpaper`` is not available on Ubuntu 14.04 or must by
-
+compiled by hand.
-.. code-block:: bash
-
-    wget -q 'https://www.dropbox.com/s/vaq0kbwi6e6au80/unpaper_6.1-1.deb?raw=1' -O unpaper_6.1-1.deb
-    sudo dpkg -i unpaper_6.1-1.deb
 
 To add JBIG2 encoding, see :ref:`jbig2`.
 
@@ -507,11 +514,11 @@ manager. ``pip`` cannot provide them.
 -  Python 3.6 or newer
 -  Ghostscript 9.15 or newer
 -  qpdf 8.1.0 or newer
--  Tesseract 4.0.0-alpha or newer
+-  Tesseract 4.0.0-beta or newer
 
 As of ocrmypdf 7.2.1, the following versions are recommended:
 
--  Python 3.7
+-  Python 3.7 or 3.8
 -  Ghostscript 9.23 or newer
 -  qpdf 8.2.1
 -  Tesseract 4.0.0 or newer

docs/introduction.rst

@@ -208,7 +208,7 @@ consider one of these similar open source programs:
 Web front-ends
 ==============
 
-The Docker image ``ocrmypdf-alpine`` provides a web service front-end
+The Docker image ``ocrmypdf`` provides a web service front-end
 that allows files to submitted over HTTP and the results "downloaded".
 This is an HTTP server intended to simplify web services deployments; it
 is not intended to be deployed on the public internet and no real

docs/optimizer.rst

@@ -0,0 +1,71 @@
+================
+PDF optimization
+================
+
+OCRmyPDF includes an image-oriented PDF optimizer. By default, the optimizer
+runs with safe settings with the goal of improving compression at no loss of
+quality. At higher optimization levels, lossy optimizations may be applied and
+tuned. Optimization occurs after OCR, and only if OCR succeeded.  It does not
+perform other possible optimizations such as deduplicating resources,
+consolidating fonts, simplifying vector drawings, or anything of that nature.
+
+Optimization ranges from ``-O0`` through ``-O3``, where ``0`` disables
+optimization and ``3`` implements all options. ``1``, the default, performs only
+safe and lossless optimizations. (This is similar to GCC's optimization
+parameter.) The exact type of optimizations performed will vary over time.
+
+Optimizations that always occurs
+================================
+
+OCRmyPDF will automatically replace obsolete or inferior compression schemes
+such as RLE or LZW with superior schemes such as Deflate and converting
+monochrome images to CCITT G4. Since this is harmless it always occurs and there
+is no way to disable it. Other non-image compressed objects are compressed as
+well.
+
+Fast web view
+=============
+
+OCRmyPDF automatically optimizes PDFs for "fast web view" in Adobe Acrobat's
+parlance, or equivalently, linearizes PDFs so that the resources they reference
+are presented in the order a viewer needs them for sequential display. This
+reduces the latency of viewing a PDF both online and from local storage. This
+actually slightly increases the file size.
+
+To disable this optimization and all others, use ``ocrmypdf --optimize 0 ...``
+or the shorthand ``-O0``.
+
+Lossless optimizations
+======================
+
+At optimization level ``-O1`` (the default), OCRmyPDF will also attempt lossless
+image optimization.
+
+If a JBIG2 encoder is available, then monochrome images will be converted to
+JBIG2, with the potential for huge savings on large black and white images,
+since JBIG2 is far more efficient than any other monochrome (bi-level)
+compression. (All known US patents related to JBIG2 have probably expired, but
+it remains the responsibility of the user to supply a JBIG2 encoder such as
+`jbig2enc <https://github.com/agl/jbig2enc>`__. OCRmyPDF does not implement
+JBIG2 encoding on its own.)
+
+OCRmyPDF currently does not attempt to recompress losslessly compressed objects
+more aggressively.
+
+Lossy optimizations
+===================
+
+At optimization level ``-O2`` and ``-O3``, OCRmyPDF will some attempt lossy
+image optimization.
+
+If ``pngquant`` is installed, OCRmyPDF will use it to perform quantize paletted
+images to reduce their size.
+
+The quality of JPEGs may be lowered, on the assumption that a lower quality
+image may be suitable for storage after OCR.
+
+It is not possible to optimize all image types. Uncommon image types may be
+skipped by the optimizer.
+
+OCRmyPDF provides :ref:`lossy mode JBIG2 <jbig2-lossy>` as an advanced feature
+that additional requires the argument ``--jbig2-lossy``.

docs/release_notes.rst

@@ -13,14 +13,78 @@ Note that it is licensed under GPLv3, so scripts that
 ``import ocrmypdf`` and are released publicly should probably also be
 licensed under GPLv3.
 
+v9.1.1
+======
+
+-  Expand the range of pdfminer.six versions that are supported.
+-  Fixed Docker build when using pikepdf 1.7.0.
+-  Fixed documentation to recommend using pip from get-pip.py.
+
+v9.1.0
+======
+
+-  Improved diagnostics when file size increases at output. Now warns if JBIG2
+   or pngquant were not available.
+-  pikepdf 1.7.0 is now required, to pick up changes that remove the need for
+   a source install on Linux systems running Python 3.8.
+
+v9.0.5
+======
+
+-  The Alpine Docker image (jbarlow83/ocrmypdf-alpine) has been dropped due to
+   the difficulties of supporting Alpine Linux.
+-  The primary Docker image (jbarlow83/ocrmypdf) has been improved to take on
+   the extra features that used to be exclusive to the Alpine image.
+-  No changes to application code.
+-  pdfminer.six version 20191020 is now supported.
+
+v9.0.4
+======
+
+-  Fixed compatibility with Python 3.8 (but requires source install for the moment).
+-  Fixed Tesseract settings for ``--user-words`` and ``--user-patterns``.
+-  Changed to pikepdf 1.6.5 (for Python 3.8).
+-  Changed to Pillow 6.2.0 (to mitigate a security vulnerability in earlier Pillow).
+-  A debug message now mentions when English is automatically selected if the locale
+   is not English.
+
+v9.0.3
+======
+
+-  Embed an encoded version of the sRGB ICC profile in the intermediate
+   Postscript file (used for PDF/A conversion). Previously we included the
+   filename, which required Postscript to run with file access enabled. For
+   security, Ghostscript 9.28 enables ``-dSAFER`` and as such, no longer
+   permits access to any file by default. This fix is necessary for
+   compatibility with Ghostscript 9.28.
+-  Exclude a test that sometimes times out and fails in continuous integration
+   from the standard test suite.
+
+v9.0.2
+======
+
+-  The image optimizer now skips optimizing flate (PNG) encoded images in some
+   situations where the optimization effort was likely wasted.
+-  The image optimizer now ignores images that specify arbitrary decode arrays,
+   since these are rare.
+-  Fixed an issue that caused inversion of black and white in monochrome images.
+   We are not certain but the problem seems to be linked to Leptonica 1.76.0 and
+   older.
+-  Fixed some cases where the test suite failed if
+   English or German Tesseract language packs were not installed.
+-  Fixed a runtime error if the Tesseract English language is not installed.
+-  Improved explicit closing of Pillow images after use.
+-  Actually fixed of Alpine Docker image build.
+-  Changed to pikepdf 1.6.3.
+
 v9.0.1
 ======
 
 -  Fixed test suite failing when either of optional dependencies unpaper and
    pngquant were missing.
--  Fixed Alpine Docker image build.
+-  Attempted fix of Alpine Docker image build.
 -  Documented that FreeBSD ports are now available.
--  Changed to pikepdf 1.6.1 (also for Alpine Docker).
+-  Changed to pikepdf 1.6.1.
 
 v9.0.0
 ======

pyproject.toml

@@ -10,7 +10,7 @@ build-backend = "setuptools.build_meta"
 
 [tool.black]
 line-length = 88
-py36 = true
+target-version = ["py36", "py37", "py38"]
 skip-string-normalization = true
 include = '\.pyi?$'
 exclude = '''

requirements/dev.txt

@@ -1,4 +1,2 @@
-check-manifest >= 0.35
 twine >= 1.8.1
 coverage >= 4.5
-GitPython == 2.1.3

requirements/main.txt

@@ -1,13 +1,10 @@
 # requirements.txt can be used to replicate the developer's build environment
 # setup.py lists a separate set of requirements that are looser to simplify
 # installation
-chardet == 3.0.4
+cffi == 1.13.2
-cffi == 1.12.2
 img2pdf == 0.3.3
-pdfminer.six == 20181108
+pdfminer.six == 20191110
-pikepdf == 1.6.1
+pikepdf == 1.7.0
-Pillow >= 5.0.0, != 5.1.0 ; sys_platform == "darwin"
+Pillow >= 6.2.0
-pycparser == 2.19
+reportlab == 3.5.32
-python-xmp-toolkit == 2.0.1
+tqdm == 4.37.0
-reportlab == 3.5.13
-tqdm == 4.32.1

requirements/test.txt

@@ -1,8 +1,8 @@
-pytest >= 4.4.1, < 5
+pytest >= 5.0.0
 pytest-helpers-namespace >= 2019.1.8
-pytest-xdist == 1.28.0
+pytest-xdist >= 1.29.0  # For DumpError fix
 pytest-cov >= 2.6.1
-python-xmp-toolkit   # requires apt-get install libexempi3
+python-xmp-toolkit == 2.0.1  # requires apt-get install libexempi3
                      # or brew install exempi
 PyPDF2 >= 1.26.0
 #PyMuPDF == 1.13.4    # optional

requirements/webservice.txt

@@ -0,0 +1 @@
+Flask >= 1, < 2

setup.py

@@ -68,6 +68,7 @@ setup(
     classifiers=[
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
         "Development Status :: 5 - Production/Stable",
         "Environment :: Console",
         "Intended Audience :: End Users/Desktop",
@@ -95,11 +96,9 @@ setup(
         'chardet >= 3.0.4, < 4',  # unlisted requirement of pdfminer.six 20181108
         'cffi >= 1.9.1',  # must be a setup and install requirement
         'img2pdf >= 0.3.0, < 0.4',  # pure Python, so track HEAD closely
-        'pdfminer.six == 20181108',
+        'pdfminer.six >= 20181108, <= 20191110',
-        'pikepdf >= 1.6.0, < 2',
+        'pikepdf >= 1.7.0, < 2',
-        'Pillow >= 4.0.0, != 5.1.0 ; sys_platform == "darwin"',
+        'Pillow >= 6.2.0',
-        # Pillow < 4 has BytesIO/TIFF bug w/img2pdf 0.2.3
-        # block 5.1.0, broken wheels
         'reportlab >= 3.3.0',  # oldest released version with sane image handling
         'tqdm >= 4',
     ],

src/ocrmypdf/_pipeline.py

@@ -54,9 +54,9 @@ def triage_image_file(input_file, output_file, options, log):
         # Recover the original filename
         log.error(str(e).replace(input_file, options.input_file))
         raise UnsupportedImageFormatError() from e
-    else:
-        log.info("Input file is an image")
 
+    with im:
+        log.info("Input file is an image")
         if 'dpi' in im.info:
             if im.info['dpi'] <= (96, 96) and not options.image_dpi:
                 log.info("Image size: (%d, %d)" % im.size)
@@ -89,7 +89,6 @@ def triage_image_file(input_file, output_file, options, log):
             elif im.mode == 'CMYK':
                 log.info('Input CMYK image has no ICC profile, not usable')
                 raise UnsupportedImageFormatError()
-        im.close()
 
     try:
         log.info("Image seems valid. Try converting to PDF...")

src/ocrmypdf/_sync.py

@@ -189,7 +189,7 @@ def worker_init(queue):
     root.addHandler(h)
 
 
-def worker_thread_init(queue):
+def worker_thread_init(_queue):
     pass
 
 
@@ -225,15 +225,15 @@ def exec_concurrent(context):
     if max_workers > 1:
         context.log.info("Start processing %d pages concurrent", max_workers)
 
-    # Tesseract 4.0 is multithreaded, and we also run multiple workers. We want to
+    # Tesseract 4.x can be multithreaded, and we also run multiple workers. We want
-    # avoid the situation where we end up trying to run NxN jobs on N CPU cores,
+    # to manage how many threads it uses to avoid creating total threads than cores.
-    # as that gives poor performance. Performance testing shows we're better off
+    # Performance testing shows we're better off
     # parallelizing ocrmypdf and forcing Tesseract to be single threaded, which we
     # get by setting the envvar OMP_THREAD_LIMIT to 1. But if the page count of the
     # input file is small, then we allow Tesseract to use threads, subject to the
-    # constraint: (ocrmypdf workers) * (tesseract threads) <= max_workers and limiting
+    # constraint: (ocrmypdf workers) * (tesseract threads) <= max_workers.
-    # Tesseract to 4 threads.
+    # As of Tesseract 4.1, 3 threads is the most effective on a 4 core/8 thread system.
-    tess_threads = min(4, context.options.jobs // max_workers)
+    tess_threads = min(3, context.options.jobs // max_workers)
     if context.options.tesseract_env is None:
         context.options.tesseract_env = os.environ.copy()
     context.options.tesseract_env.setdefault('OMP_THREAD_LIMIT', str(tess_threads))
@@ -301,7 +301,7 @@ def exec_concurrent(context):
 class NeverRaise(Exception):
     """An exception that is never raised"""
 
-    pass
+    pass  # pylint: disable=unnecessary-pass
 
 
 def run_pipeline(options, api=False):

src/ocrmypdf/_validation.py

@@ -17,6 +17,7 @@
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
 
+import locale
 import logging
 import os
 import sys
@@ -47,6 +48,7 @@ from .helpers import is_file_writable, is_iterable_notstr, monotonic, re_symlink
 # External dependencies
 
 HOCR_OK_LANGS = frozenset(['eng', 'deu', 'spa', 'ita', 'por'])
+DEFAULT_LANGUAGE = 'eng'  # Enforce English hegemony
 
 log = logging.getLogger(__name__)
 
@@ -58,7 +60,10 @@ verify_python3_env()
 
 def check_options_languages(options):
     if not options.language:
-        options.language = ['eng']  # Enforce English hegemony
+        options.language = [DEFAULT_LANGUAGE]
+        system_lang = locale.getlocale()[0]
+        if system_lang and not system_lang.startswith('en'):
+            log.debug("No language specified; assuming --language %s", DEFAULT_LANGUAGE)
 
     # Support v2.x "eng+deu" language syntax
     if '+' in options.language[0]:
@@ -107,7 +112,7 @@ def check_options_output(options):
         options.pdf_renderer = 'sandwich'
 
     if options.pdf_renderer == 'sandwich' and not tesseract.has_textonly_pdf(
-        options.tesseract_env
+        options.tesseract_env, languages
     ):
         raise MissingDependencyError(
             "You are using an alpha version of Tesseract 4.0 that does not support "
@@ -413,6 +418,20 @@ def report_output_file_size(options, input_file, output_file):
                 f"The argument --{arg.replace('_', '-')} was issued, causing transcoding."
             )
 
+    if options.optimize == 0:
+        reasons.append("Optimization was disabled.")
+    else:
+        image_optimizers = {
+            'jbig2': jbig2enc.available(),
+            'pngquant': pngquant.available(),
+        }
+        for name, available in image_optimizers.items():
+            if not available:
+                reasons.append(
+                    f"The optional dependency '{name}' was not found, so some image "
+                    f"optimizations could not be attempted."
+                )
+
     if reasons:
         explanation = "Possible reasons for this include:\n" + '\n'.join(reasons) + "\n"
     else:

src/ocrmypdf/api.py

@@ -33,7 +33,7 @@ class TqdmConsole:
 
     def __init__(self, file):
         self.file = file
-        self.py36 = sys.version_info >= (3, 6)
+        self.py36 = sys.version_info[0:2] == (3, 6)
 
     def write(self, msg):
         # When no progress bar is active, tqdm.write() routes to print()
@@ -122,13 +122,23 @@ def create_options(*, input_file, output_file, **kwargs):
     for arg, val in kwargs.items():
         if val is None:
             continue
-        if arg == 'tesseract_env':
+
+        # These arguments with special handling for which we bypass
+        # argparse
+        if arg in {'tesseract_env', 'progress_bar'}:
             deferred.append((arg, val))
             continue
+
         cmd_style_arg = arg.replace('_', '-')
-        cmdline.append(f"--{cmd_style_arg}")
+
+        # Booleans are special: add only if True, omit for False
         if isinstance(val, bool):
+            if val:
+                cmdline.append(f"--{cmd_style_arg}")
             continue
+
+        # We have a parameter
+        cmdline.append(f"--{cmd_style_arg}")
         if isinstance(val, (int, float)):
             cmdline.append(str(val))
         elif isinstance(val, str):

src/ocrmypdf/exec/ghostscript.py

@@ -15,8 +15,11 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
+"""Interface to Ghostscript executable"""
+
 import logging
 import re
+import warnings
 from functools import lru_cache
 from os import fspath
 from shutil import copy
@@ -193,7 +196,7 @@ def generate_pdfa(
     output_file,
     compression,
     log,
-    threads=1,
+    threads=None,  # deprecated parameter
     pdf_version='1.5',
     pdfa_part='2',
 ):
@@ -216,6 +219,10 @@ def generate_pdfa(
     """
     if not log:
         log = gslog
+    if threads is not None:
+        warnings.warn(
+            "use of deprecated parameter 'threads'", category=DeprecationWarning
+        )
 
     compression_args = []
     if compression == 'jpeg':
@@ -259,6 +266,7 @@ def generate_pdfa(
                 "-dQUIET",
                 "-dBATCH",
                 "-dNOPAUSE",
+                "-dSAFER",
                 "-dCompatibilityLevel=" + str(pdf_version),
                 "-sDEVICE=pdfwrite",
                 "-dAutoRotatePages=/None",

src/ocrmypdf/exec/jbig2enc.py

@@ -15,6 +15,8 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
+"""Interface to jbig2 executable"""
+
 from functools import lru_cache
 from subprocess import PIPE, run
 

src/ocrmypdf/exec/pngquant.py

@@ -15,6 +15,8 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
+"""Interface to pngquant executable"""
+
 from functools import lru_cache
 from subprocess import run
 from tempfile import NamedTemporaryFile
@@ -40,8 +42,7 @@ def available():
 
 def quantize(input_file, output_file, quality_min, quality_max):
     if input_file.endswith('.jpg'):
-        im = Image.open(input_file)
+        with Image.open(input_file) as im, NamedTemporaryFile(suffix='.png') as tmp:
-        with NamedTemporaryFile(suffix='.png') as tmp:
             im.save(tmp)
             args = [
                 'pngquant',

src/ocrmypdf/exec/qpdf.py

@@ -15,6 +15,8 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
+"""Interface to qpdf executable"""
+
 from functools import lru_cache
 from os import fspath
 from subprocess import PIPE, STDOUT, CalledProcessError, run

src/ocrmypdf/exec/tesseract.py

@@ -15,12 +15,13 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
+"""Interface to Tesseract executable"""
+
 import os
 import shutil
-import sys
 from collections import namedtuple
 from contextlib import suppress
-from functools import lru_cache
+import logging
 from os import fspath
 from subprocess import PIPE, STDOUT, CalledProcessError, TimeoutExpired, run
 
@@ -52,6 +53,11 @@ HOCR_TEMPLATE = """<?xml version="1.0" encoding="UTF-8"?>
 """
 
 
+class TesseractLoggerAdapter(logging.LoggerAdapter):
+    def process(self, msg, kwargs):
+        return '[tesseract] %s' % (msg), kwargs
+
+
 def version(tesseract_env=None):
     return get_version('tesseract', regex=r'tesseract\s(.+)', env=tesseract_env)
 
@@ -61,13 +67,13 @@ def v4(tesseract_env=None):
     return version(tesseract_env) >= '4'
 
 
-def has_textonly_pdf(tesseract_env=None):
+def has_textonly_pdf(tesseract_env=None, langs=None):
     """Does Tesseract have textonly_pdf capability?
 
     Available in v4.00.00alpha since January 2017. Best to
-    parse the parameter list
+    parse the parameter list.
     """
-    args_tess = ['tesseract', '--print-parameters', 'pdf']
+    args_tess = tess_base_args(langs, engine_mode=None) + ['--print-parameters', 'pdf']
     params = ''
     try:
         proc = run(
@@ -179,15 +185,16 @@ def get_orientation(input_file, engine_mode, timeout: float, log, tesseract_env=
         return oc
 
 
-def tesseract_log_output(log, stdout, input_file):
+def tesseract_log_output(mainlog, stdout, input_file):
-    prefix = "[tesseract] "
+    log = TesseractLoggerAdapter(
+        mainlog, extra=mainlog.extra if hasattr(mainlog, 'extra') else None
+    )
 
     try:
         text = stdout.decode()
     except UnicodeDecodeError:
         log.error(
-            prefix
+            "command line output was not utf-8. "
-            + "command line output was not utf-8. "
             + "This usually means Tesseract's language packs do not match "
             "the installed version of Tesseract."
         )
@@ -200,25 +207,25 @@ def tesseract_log_output(log, stdout, input_file):
         elif line.startswith("Warning in pixReadMem"):
             continue
         elif 'diacritics' in line:
-            log.warning(prefix + "lots of diacritics - possibly poor OCR")
+            log.warning("lots of diacritics - possibly poor OCR")
         elif line.startswith('OSD: Weak margin'):
-            log.warning(prefix + "unsure about page orientation")
+            log.warning("unsure about page orientation")
         elif 'Error in pixScanForForeground' in line:
             pass  # Appears to be spurious/problem with nonwhite borders
         elif 'Error in boxClipToRectangle' in line:
             pass  # Always appears with pixScanForForeground message
         elif 'parameter not found: ' in line.lower():
-            log.error(prefix + line.strip())
+            log.error(line.strip())
             problem = line.split('found: ')[1]
             raise TesseractConfigError(problem)
         elif 'error' in line.lower() or 'exception' in line.lower():
-            log.error(prefix + line.strip())
+            log.error(line.strip())
         elif 'warning' in line.lower():
-            log.warning(prefix + line.strip())
+            log.warning(line.strip())
         elif 'read_params_file' in line.lower():
-            log.error(prefix + line.strip())
+            log.error(line.strip())
         else:
-            log.info(prefix + line.strip())
+            log.info(line.strip())
 
 
 def page_timedout(log, input_file, timeout):
@@ -233,8 +240,8 @@ def _generate_null_hocr(output_hocr, output_sidecar, image):
     the same size as the input image."""
     from PIL import Image
 
-    im = Image.open(image)
+    with Image.open(image) as im:
-    w, h = im.size
+        w, h = im.size
 
     with open(output_hocr, 'w', encoding="utf-8") as f:
         f.write(HOCR_TEMPLATE.format(w, h))
@@ -271,6 +278,9 @@ def generate_hocr(
     if user_patterns:
         args_tesseract.extend(['--user-patterns', user_patterns])
 
+    if user_words or user_patterns:
+        args_tesseract.extend(['-c', 'lstm_use_matrix=1'])
+
     # Reminder: test suite tesseract spoofers will break after any changes
     # to the number of order parameters here
     args_tesseract.extend([input_file, prefix, 'hocr', 'txt'] + tessconfig)
@@ -358,7 +368,7 @@ def generate_pdf(
     if pagesegmode is not None:
         args_tesseract.extend(['--psm', str(pagesegmode)])
 
-    if text_only and has_textonly_pdf(tesseract_env):
+    if text_only and has_textonly_pdf(tesseract_env, language):
         args_tesseract.extend(['-c', 'textonly_pdf=1'])
 
     if user_words:

src/ocrmypdf/exec/unpaper.py

@@ -18,10 +18,11 @@
 # unpaper documentation:
 # https://github.com/Flameeyes/unpaper/blob/master/doc/basic-concepts.md
 
+"""Interface to unpaper executable"""
+
 import os
 import shlex
 import subprocess
-import sys
 from functools import lru_cache
 from subprocess import PIPE, STDOUT, CalledProcessError
 from tempfile import TemporaryDirectory
@@ -42,33 +43,30 @@ def run(input_file, output_file, dpi, log, mode_args):
 
     SUFFIXES = {'1': '.pbm', 'L': '.pgm', 'RGB': '.ppm'}
 
-    im = Image.open(input_file)
+    with TemporaryDirectory() as tmpdir, Image.open(input_file) as im:
-    if im.mode not in SUFFIXES.keys():
+        if im.mode not in SUFFIXES.keys():
-        log.info("Converting image to other colorspace")
+            log.info("Converting image to other colorspace")
+            try:
+                if im.mode == 'P' and len(im.getcolors()) == 2:
+                    im = im.convert(mode='1')
+                else:
+                    im = im.convert(mode='RGB')
+            except IOError as e:
+                im.close()
+                raise MissingDependencyError(
+                    "Could not convert image with type " + im.mode
+                ) from e
+
         try:
-            if im.mode == 'P' and len(im.getcolors()) == 2:
+            suffix = SUFFIXES[im.mode]
-                im = im.convert(mode='1')
+        except KeyError:
-            else:
-                im = im.convert(mode='RGB')
-        except IOError as e:
-            im.close()
             raise MissingDependencyError(
-                "Could not convert image with type " + im.mode
+                "Failed to convert image to a supported format."
             ) from e
 
-    try:
-        suffix = SUFFIXES[im.mode]
-    except KeyError:
-        im.close()
-        raise MissingDependencyError(
-            "Failed to convert image to a supported format."
-        ) from e
-
-    with TemporaryDirectory() as tmpdir:
         input_pnm = os.path.join(tmpdir, f'input{suffix}')
         output_pnm = os.path.join(tmpdir, f'output{suffix}')
         im.save(input_pnm, format='PPM')
-        im.close()
 
         # To prevent any shenanigans from accepting arbitrary parameters in
         # --unpaper-args, we:
@@ -95,10 +93,12 @@ def run(input_file, output_file, dpi, log, mode_args):
             log.debug(proc.stdout)
             # unpaper sets dpi to 72; fix this
             try:
-                Image.open(output_pnm).save(output_file, dpi=(dpi, dpi))
+                with Image.open(output_pnm) as imout:
+                    imout.save(output_file, dpi=(dpi, dpi))
             except (FileNotFoundError, OSError):
                 raise SubprocessOutputError(
-                    "unpaper: failed to produce the expected output file. Called with: "
+                    "unpaper: failed to produce the expected output file. "
+                    + " Called with: "
                     + str(args_unpaper)
                 ) from None
 

src/ocrmypdf/optimize.py

@@ -73,6 +73,9 @@ def extract_image_filter(pike, root, log, image, xref):
     if filtdp[0] == Name.JPXDecode:
         return None  # Don't do JPEG2000
 
+    if Name.Decode in image:
+        return None  # Don't mess with custom Decode tables
+
     return pim, filtdp
 
 
@@ -104,6 +107,15 @@ def extract_image_generic(*, pike, root, log, image, xref, options):
         return None
     pim, filtdp = result
 
+    # Don't try to PNG-optimize 1bpp images, since JBIG2 does it better.
+    if pim.bits_per_component == 1:
+        return None
+
+    try:
+        pim.indexed  # pikepdf 1.6.3 can't handle [/Indexed [/Array...]]
+    except NotImplementedError:
+        return None
+
     if filtdp[0] == Name.DCTDecode and options.optimize >= 2:
         # This is a simple heuristic derived from some training data, that has
         # about a 70% chance of guessing whether the JPEG is high quality,
@@ -343,6 +355,7 @@ def transcode_jpegs(pike, jpegs, root, log, options):
 
 
 def transcode_pngs(pike, images, image_name_fn, root, log, options):
+    modified = set()
     if options.optimize >= 2:
         png_quality = (
             max(10, options.png_quality - 10),
@@ -363,6 +376,7 @@ def transcode_pngs(pike, images, image_name_fn, root, log, options):
                         png_quality[1],
                     )
                 )
+                modified.add(xref)
             with tqdm(
                 desc="PNGs",
                 total=len(futures),
@@ -372,10 +386,14 @@ def transcode_pngs(pike, images, image_name_fn, root, log, options):
                 for _future in concurrent.futures.as_completed(futures):
                     pbar.update()
 
-    for xref in images:
+    for xref in modified:
         im_obj = pike.get_object(xref, 0)
         try:
-            compdata = leptonica.CompressedData.open(png_name(root, xref))
+            pix = leptonica.Pix.open(png_name(root, xref))
+            if pix.mode == '1':
+                compdata = pix.generate_pdf_ci_data(leptonica.lept.L_G4_ENCODE, 0)
+            else:
+                compdata = leptonica.CompressedData.open(png_name(root, xref))
         except leptonica.LeptonicaError as e:
             # Most likely this means file not found, i.e. quantize did not
             # produce an improved version
@@ -391,62 +409,83 @@ def transcode_pngs(pike, images, image_name_fn, root, log, options):
                 f"{len(compdata)} > {int(im_obj.stream_dict.Length)}"
             )
             continue
+        if compdata.type == leptonica.lept.L_FLATE_ENCODE:
+            return rewrite_png(pike, im_obj, compdata, log)
+        elif compdata.type == leptonica.lept.L_G4_ENCODE:
+            return rewrite_png_as_g4(pike, im_obj, compdata, log)
 
-        # When a PNG is inserted into a PDF, we more or less copy the IDAT section from
-        # the PDF and transfer the rest of the PNG headers to PDF image metadata.
-        # One thing we have to do is tell the PDF reader whether a predictor was used
-        # on the image before Flate encoding. (Typically one is.)
-        # According to Leptonica source, PDF readers don't actually need us
-        # to specify the correct predictor, they just need a value of either:
-        #   1 - no predictor
-        #   10-14 - there is a predictor
-        # Leptonica's compdata->predictor only tells TRUE or FALSE
-        # From there the PNG decoder can infer the rest from the file.
-        # In practice the predictor should be Paeth, 14, so we'll use that.
-        # See:
-        #   - PDF RM 7.4.4.4 Table 10
-        #   - https://github.com/DanBloomberg/leptonica/blob/master/src/pdfio2.c#L757
-        predictor = 14 if compdata.predictor > 0 else 1
-        dparms = Dictionary(Predictor=predictor)
-        if predictor > 1:
-            dparms.BitsPerComponent = compdata.bps  # Yes, this is redundant
-            dparms.Colors = compdata.spp
-            dparms.Columns = compdata.w
 
-        im_obj.BitsPerComponent = compdata.bps
+def rewrite_png_as_g4(pike, im_obj, compdata, log):
-        im_obj.Width = compdata.w
+    im_obj.BitsPerComponent = 1
-        im_obj.Height = compdata.h
+    im_obj.Width = compdata.w
+    im_obj.Height = compdata.h
 
-        if compdata.ncolors > 0:
+    im_obj.write(compdata.read())
-            # .ncolors is the number of colors in the palette, not the number of
+
-            # colors used in a true color image
+    log.debug(f"PNG to G4 {im_obj.objgen}")
-            palette_pdf_string = compdata.get_palette_pdf_string()
+    if Name.Predictor in im_obj:
-            palette_data = pikepdf.Object.parse(palette_pdf_string)
+        del im_obj.Predictor
-            palette_stream = pikepdf.Stream(pike, bytes(palette_data))
+    if Name.DecodeParms in im_obj:
-            palette = [
+        del im_obj.DecodeParms
-                Name.Indexed,
+    im_obj.DecodeParms = Dictionary(
-                Name.DeviceRGB,
+        K=-1, BlackIs1=bool(compdata.minisblack), Columns=compdata.w
-                compdata.ncolors - 1,
+    )
-                palette_stream,
+
-            ]
+    im_obj.Filter = Name.CCITTFaxDecode
-            cs = palette
+    return
-        else:
+
-            if compdata.spp == 1:
+
-                # PDF interprets binary-1 as black in 1bpp, but PNG sets
+def rewrite_png(pike, im_obj, compdata, log):
-                # black to 0 for 1bpp. Create a palette that informs the PDF
+    # When a PNG is inserted into a PDF, we more or less copy the IDAT section from
-                # of the mapping - seems cleaner to go this way but pikepdf
+    # the PDF and transfer the rest of the PNG headers to PDF image metadata.
-                # needs to be patched to support it.
+    # One thing we have to do is tell the PDF reader whether a predictor was used
-                # palette = [Name.Indexed, Name.DeviceGray, 1, b"\xff\x00"]
+    # on the image before Flate encoding. (Typically one is.)
-                # cs = palette
+    # According to Leptonica source, PDF readers don't actually need us
-                cs = Name.DeviceGray
+    # to specify the correct predictor, they just need a value of either:
-            elif compdata.spp == 3:
+    #   1 - no predictor
-                cs = Name.DeviceRGB
+    #   10-14 - there is a predictor
-            elif compdata.spp == 4:
+    # Leptonica's compdata->predictor only tells TRUE or FALSE
-                cs = Name.DeviceCMYK
+    # 10-14 means the actual predictor is specified in the data, so for any
-        if compdata.bps == 1:
+    # number >= 10 the PDF reader will use whatever the PNG data specifies.
-            im_obj.Decode = [1, 0]  # Bit of a kludge but this inverts photometric too
+    # In practice Leptonica should use Paeth, 14, but 15 seems to be the
-        im_obj.ColorSpace = cs
+    # designated value for "optimal". So we will use 15.
-        im_obj.write(compdata.read(), filter=Name.FlateDecode, decode_parms=dparms)
+    # See:
+    #   - PDF RM 7.4.4.4 Table 10
+    #   - https://github.com/DanBloomberg/leptonica/blob/master/src/pdfio2.c#L757
+    predictor = 15 if compdata.predictor > 0 else 1
+    dparms = Dictionary(Predictor=predictor)
+    if predictor > 1:
+        dparms.BitsPerComponent = compdata.bps  # Yes, this is redundant
+        dparms.Colors = compdata.spp
+        dparms.Columns = compdata.w
+
+    im_obj.BitsPerComponent = compdata.bps
+    im_obj.Width = compdata.w
+    im_obj.Height = compdata.h
+
+    log.debug(
+        f"PNG {im_obj.objgen}: palette={compdata.ncolors} spp={compdata.spp} bps={compdata.bps}"
+    )
+    if compdata.ncolors > 0:
+        # .ncolors is the number of colors in the palette, not the number of
+        # colors used in a true color image. The palette string is always
+        # given as RGB tuples even when the image is grayscale; see
+        # https://github.com/DanBloomberg/leptonica/blob/master/src/colormap.c#L2067
+        palette_pdf_string = compdata.get_palette_pdf_string()
+        palette_data = pikepdf.Object.parse(palette_pdf_string)
+        palette_stream = pikepdf.Stream(pike, bytes(palette_data))
+        palette = [Name.Indexed, Name.DeviceRGB, compdata.ncolors - 1, palette_stream]
+        cs = palette
+    else:
+        # ncolors == 0 means we are using a colorspace without a palette
+        if compdata.spp == 1:
+            cs = Name.DeviceGray
+        elif compdata.spp == 3:
+            cs = Name.DeviceRGB
+        elif compdata.spp == 4:
+            cs = Name.DeviceCMYK
+    im_obj.ColorSpace = cs
+    im_obj.write(compdata.read(), filter=Name.FlateDecode, decode_parms=dparms)
 
 
 def optimize(input_file, output_file, context, save_settings):
@@ -510,11 +549,11 @@ def main(infile, outfile, level, jobs=1):
         """Emulate ocrmypdf's options"""
 
         def __init__(
-            self, input_file, jobs, optimize, jpeg_quality, png_quality, jb2lossy
+            self, input_file, jobs, optimize_, jpeg_quality, png_quality, jb2lossy
         ):
             self.input_file = input_file
             self.jobs = jobs
-            self.optimize = optimize
+            self.optimize = optimize_
             self.jpeg_quality = jpeg_quality
             self.png_quality = png_quality
             self.jbig2_page_group_size = 0
@@ -525,7 +564,7 @@ def main(infile, outfile, level, jobs=1):
     options = OptimizeOptions(
         input_file=infile,
         jobs=jobs,
-        optimize=int(level),
+        optimize_=int(level),
         jpeg_quality=0,  # Use default
         png_quality=0,
         jb2lossy=False,

src/ocrmypdf/pdfa.py

@@ -31,8 +31,7 @@ Ghostscript's handling of pdfmark.
 
 """
 
-import os
+import base64
-from binascii import hexlify
 from pathlib import Path
 from string import Template
 
@@ -48,26 +47,13 @@ SRGB_ICC_PROFILE = pkg_resources.resource_filename('ocrmypdf', ICC_PROFILE_RELPA
 # files, from the Ghostscript documentation. Lines beginning with % are
 # comments. Python substitution variables have a '$' prefix.
 pdfa_def_template = u"""%!
-% Define entries in the document Info dictionary :
+% Define an ICC profile :
 /ICCProfile $icc_profile
 def
 
-% Define an ICC profile :
-
 [/_objdef {icc_PDFA} /type /stream /OBJ pdfmark
-[{icc_PDFA}
+[{icc_PDFA} << /N 3 >> /PUT pdfmark
-<<
+[{icc_PDFA} ICCProfile /PUT pdfmark
-  /N currentpagedevice /ProcessColorModel known {
-    currentpagedevice /ProcessColorModel get dup /DeviceGray eq
-    {pop 1} {
-      /DeviceRGB eq
-      {3}{4} ifelse
-    } ifelse
-  } {
-    (ERROR, unable to determine ProcessColorModel) == flush
-  } ifelse
->> /PUT pdfmark
-[{icc_PDFA} ICCProfile (r) file /PUT pdfmark
 
 % Define the output intent dictionary :
 
@@ -104,16 +90,10 @@ def generate_pdfa_ps(target_filename, icc='sRGB'):
     else:
         raise NotImplementedError("Only supporting sRGB")
 
-    # pdfmark must contain the full path to the ICC profile, and pdfmark must be
+    # Read the ICC profile, encode as ASCII85 and convert to a string which we
-    # also encoded in ASCII. ocrmypdf can be installed anywhere, including to
+    # will insert in the .ps file
-    # paths that have a non-ASCII character in the filename. Ghostscript
+    bytes_icc_profile = Path(icc_profile).read_bytes()
-    # accepts hex-encoded strings and converts them to byte strings, so
+    icc_profile = base64.a85encode(bytes_icc_profile, adobe=True).decode('ascii')
-    # we encode the path with fsencode() and use the hex representation.
-    # UTF-16 not accepted here. (Even though ASCII encodable is the usual case,
-    # do this always to avoid making it a rare conditional.)
-    bytes_icc_profile = os.fsencode(icc_profile)
-    hex_icc_profile = hexlify(bytes_icc_profile)
-    icc_profile = '<' + hex_icc_profile.decode('ascii') + '>'
 
     t = Template(pdfa_def_template)
     ps = t.substitute(icc_profile=icc_profile, icc_identifier=icc)

src/ocrmypdf/pdfinfo/layout.py

@@ -20,6 +20,7 @@ from math import copysign
 from pathlib import Path
 from unittest.mock import patch
 
+import pdfminer
 import pdfminer.encodingdb
 import pdfminer.pdfdevice
 import pdfminer.pdfinterp
@@ -36,51 +37,54 @@ from ..exceptions import EncryptedPdfError
 STRIP_NAME = re.compile(r'[0-9]+')
 
 #
-# Unconditional pdfminer patches
+#  pdfminer 20181108 patches
 #
 
+if pdfminer.__version__ == '20181108':
 
-def name2unicode(name):
+    def name2unicode(name):
-    """Fix pdfminer's name2unicode function
+        """Fix pdfminer's name2unicode function
 
-    Font cids that are mapped to names of the form /g123 seem to be, by convention
+        Font cids that are mapped to names of the form /g123 seem to be, by convention
-    characters with no corresponding Unicode entry. These can be subsetted fonts
+        characters with no corresponding Unicode entry. These can be subsetted fonts
-    or symbolic fonts. There seems to be no way to map /g123 fonts to Unicode,
+        or symbolic fonts. There seems to be no way to map /g123 fonts to Unicode,
-    barring a ToUnicode data structure.
+        barring a ToUnicode data structure.
-    """
+        """
-    if name in glyphname2unicode:
+        if name in glyphname2unicode:
-        return glyphname2unicode[name]
+            return glyphname2unicode[name]
-    if name.startswith('g') or name.startswith('a'):
+        if name.startswith('g') or name.startswith('a'):
-        raise KeyError(name)
-    if name.startswith('uni'):
-        try:
-            return chr(int(name[3:], 16))
-        except ValueError:  # Not hexadecimal
             raise KeyError(name)
-    m = STRIP_NAME.search(name)
+        if name.startswith('uni'):
-    if not m:
+            try:
-        raise KeyError(name)
+                return chr(int(name[3:], 16))
-    return chr(int(m.group(0)))
+            except ValueError:  # Not hexadecimal
+                raise KeyError(name)
+        m = STRIP_NAME.search(name)
+        if not m:
+            raise KeyError(name)
+        return chr(int(m.group(0)))
 
+    pdfminer.encodingdb.name2unicode = name2unicode
 
-pdfminer.encodingdb.name2unicode = name2unicode
+    original_PDFFont_init = PDFFont.__init__
 
-original_PDFFont_init = PDFFont.__init__
+    def PDFFont__init__(self, descriptor, widths, default_width=None):
+        original_PDFFont_init(self, descriptor, widths, default_width)
+        # PDF spec says descent should be negative
+        # A font with a positive descent implies it floats entirely above the
+        # baseline, i.e. it's not really a baseline anymore. I have fonts that
+        # claim a positive descent, but treating descent as positive always seems
+        # to misposition text.
+        if self.descent > 0:
+            self.descent = -self.descent
 
+    PDFFont.__init__ = PDFFont__init__
 
-def PDFFont__init__(self, descriptor, widths, default_width=None):
+#
-    original_PDFFont_init(self, descriptor, widths, default_width)
+# end of pdfminer 20181108 patches
-    # PDF spec says descent should be negative
+#
-    # A font with a positive descent implies it floats entirely above the
-    # baseline, i.e. it's not really a baseline anymore. I have fonts that
-    # claim a positive descent, but treating descent as positive always seems
-    # to misposition text.
-    if self.descent > 0:
-        self.descent = -self.descent
 
 
-PDFFont.__init__ = PDFFont__init__
-
 original_PDFSimpleFont_init = PDFSimpleFont.__init__
 
 
@@ -97,6 +101,7 @@ def PDFSimpleFont__init__(self, descriptor, widths, spec):
 
 
 PDFSimpleFont.__init__ = PDFSimpleFont__init__
+
 #
 # pdfminer patches when creator is PScript5.dll
 #
@@ -207,6 +212,7 @@ class TextPositionTracker(PDFLayoutAnalyzer):
         super().__init__(rsrcmgr, pageno, laparams)
         self.textstate = None
         self.result = None
+        self.cur_item = None  # not defined in pdfminer code as it should be
 
     def begin_page(self, page, ctm):
         super().begin_page(page, ctm)

tests/spoof/tesseract_badutf8.py

@@ -52,7 +52,7 @@ def main():
     elif sys.argv[1] == '--list-langs':
         print('List of available languages (1):\neng', file=sys.stderr)
         sys.exit(0)
-    elif sys.argv[1] == '--print-parameters':
+    elif sys.argv[-2] == '--print-parameters':
         print("Some parameters", file=sys.stderr)
         print("textonly_pdf\t1\tSome help text")
         sys.exit(0)

tests/spoof/tesseract_big_image_error.py

@@ -44,7 +44,7 @@ def main():
     elif sys.argv[1] == '--list-langs':
         print('List of available languages (1):\neng\n', file=sys.stderr)
         sys.exit(0)
-    elif sys.argv[1] == '--print-parameters':
+    elif sys.argv[-2] == '--print-parameters':
         print('A parameter list would go here\ntextonly_pdf 0\n', file=sys.stderr)
         sys.exit(0)
     elif sys.argv[-2] == 'hocr':

tests/spoof/tesseract_cache.py

@@ -100,6 +100,8 @@ def main():
     # Convert non-standard but supported -psm to --psm
     sys.argv = ['--psm' if arg == '-psm' else arg for arg in sys.argv]
 
+    if '_OCRMYPDF_TEST_INFILE' not in os.environ:
+        real_tesseract()  # test not properly set up
     source = os.environ['_OCRMYPDF_TEST_INFILE']  # required
     args = parser.parse_args()
 

tests/spoof/tesseract_crash.py

@@ -50,7 +50,7 @@ def main():
     elif sys.argv[1] == '--list-langs':
         print('List of available languages (1):\neng', file=sys.stderr)
         sys.exit(0)
-    elif sys.argv[1] == '--print-parameters':
+    elif sys.argv[-2] == '--print-parameters':
         print('A parameter list would go here\ntextonly_pdf 0\n', file=sys.stderr)
         sys.exit(0)
     elif sys.argv[-2] == 'hocr':

tests/spoof/tesseract_noop.py

@@ -76,7 +76,7 @@ def main():
     elif sys.argv[1] == '--list-langs':
         print('List of available languages (1):\neng', file=sys.stderr)
         sys.exit(0)
-    elif sys.argv[1] == '--print-parameters':
+    elif sys.argv[-2] == '--print-parameters':
         print("Some parameters", file=sys.stderr)
         print("textonly_pdf\t1\tSome help text")
         sys.exit(0)

tests/test_completion.py

@@ -19,6 +19,11 @@ from subprocess import run, PIPE
 
 import pytest
 
+pytestmark = pytest.mark.skipif(
+    pytest.helpers.running_in_docker(),  # pylint: disable=no-member
+    reason="docker can't complete",
+)
+
 
 def test_fish():
     try:

tests/test_lept.py

@@ -38,7 +38,8 @@ def test_colormap_backgroundnorm(resources):
 def crom_pix(resources):
     pix = lept.Pix.open(resources / 'crom.png')
     im = Image.open(resources / 'crom.png')
-    return pix, im
+    yield pix, im
+    im.close()
 
 
 def test_pix_basic(crom_pix):

tests/test_main.py

@@ -118,8 +118,8 @@ def test_deskew(spoof_tesseract_noop, resources, outdir):
 
 def test_remove_background(spoof_tesseract_noop, resources, outdir):
     # Ensure the input image does not contain pure white/black
-    im = Image.open(resources / 'congress.jpg')
+    with Image.open(resources / 'congress.jpg') as im:
-    assert im.getextrema() != ((0, 255), (0, 255), (0, 255))
+        assert im.getextrema() != ((0, 255), (0, 255), (0, 255))
 
     output_pdf = check_ocrmypdf(
         resources / 'congress.jpg',
@@ -145,8 +145,8 @@ def test_remove_background(spoof_tesseract_noop, resources, outdir):
     )
 
     # The output image should contain pure white and black
-    im = Image.open(output_png)
+    with Image.open(output_png) as im:
-    assert im.getextrema() == ((0, 255), (0, 255), (0, 255))
+        assert im.getextrema() == ((0, 255), (0, 255), (0, 255))
 
 
 # This will run 5 * 2 * 2 = 20 test cases
@@ -217,7 +217,7 @@ def test_skip_ocr(spoof_tesseract_cache, resources, outpdf):
     assert pdfinfo[0].has_text
 
 
-def test_redo_ocr(spoof_tesseract_cache, resources, outpdf):
+def test_redo_ocr(resources, outpdf):
     in_ = resources / 'graph_ocred.pdf'
     before = PdfInfo(in_, detailed_page_analysis=True)
     out = outpdf
@@ -349,10 +349,9 @@ def test_german(spoof_tesseract_cache, resources, outdir):
         sidecar,
         env=spoof_tesseract_cache,
     )
-    print(os.environ)
+    if 'deu' not in tesseract.languages():
-    assert (
+        pytest.xfail(reason="tesseract-deu language pack not installed")
-        p.returncode == ExitCode.ok
+    assert p.returncode == ExitCode.ok, "Requires tesseract deu language pack"
-    ), "This test may fail if Tesseract language packs are missing"
 
 
 def test_klingon(resources, outpdf):
@@ -614,6 +613,7 @@ language_model_penalty_non_freq_dict_word 0
     )
 
 
+@pytest.mark.slow  # This test sometimes times out in CI
 @pytest.mark.parametrize('renderer', RENDERERS)
 def test_tesseract_config_notfound(renderer, resources, outdir):
     cfg_file = outdir / 'nofile.cfg'
@@ -630,6 +630,7 @@ def test_tesseract_config_notfound(renderer, resources, outdir):
     assert p.returncode == ExitCode.ok, err
 
 
+@pytest.mark.slow  # This test sometimes times out in CI
 @pytest.mark.parametrize('renderer', RENDERERS)
 def test_tesseract_config_invalid(renderer, resources, outdir):
     cfg_file = outdir / 'test.cfg'
@@ -652,26 +653,11 @@ THIS FILE IS INVALID
     assert p.returncode == ExitCode.invalid_config
 
 
-@pytest.mark.skipif(tesseract.v4(), reason='arg has no effect in 4.0-beta1')
+@pytest.mark.skipif(not tesseract.has_user_words(), reason='not functional until 4.1.0')
-def test_user_words(resources, outdir):
+def test_user_words_ocr(resources, outdir):
+    # Does not actually test if --user-words causes output to differ
     word_list = outdir / 'wordlist.txt'
-    sidecar_before = outdir / 'sidecar_before.txt'
+    sidecar_after = outdir / 'sidecar.txt'
-    sidecar_after = outdir / 'sidecar_after.txt'
-
-    # Don't know how to make this test pass on various versions and platforms
-    # so weaken to merely testing that the argument is accepted
-    consistent = False
-
-    if consistent:
-        check_ocrmypdf(
-            resources / 'crom.png',
-            outdir / 'out.pdf',
-            '--image-dpi',
-            150,
-            '--sidecar',
-            sidecar_before,
-        )
-        assert 'cromulent' not in sidecar_before.open().read()
 
     with word_list.open('w') as f:
         f.write('cromulent\n')  # a perfectly cromulent word
@@ -687,9 +673,6 @@ def test_user_words(resources, outdir):
         word_list,
     )
 
-    if consistent:
-        assert 'cromulent' in sidecar_after.open().read()
-
 
 def test_form_xobject(spoof_tesseract_noop, resources, outpdf):
     check_ocrmypdf(
@@ -810,6 +793,7 @@ def test_compression_preserved(
         assert pdfimage.color == Colorspace.rgb, "Colorspace changed"
     elif im.mode.startswith('L'):
         assert pdfimage.color == Colorspace.gray, "Colorspace changed"
+    im.close()
 
 
 @pytest.mark.parametrize(
@@ -871,6 +855,7 @@ def test_compression_changed(
         assert pdfimage.color == Colorspace.rgb, "Colorspace changed"
     elif im.mode.startswith('L'):
         assert pdfimage.color == Colorspace.gray, "Colorspace changed"
+    im.close()
 
 
 def test_sidecar_pagecount(spoof_tesseract_cache, resources, outpdf):

tests/test_optimize.py

@@ -48,11 +48,11 @@ def test_mono_not_inverted(resources, outdir):
         xres=10,
         yres=10,
         raster_device='pnggray',
-        log=logging.getLogger(name='test_mono_flip'),
+        log=logging.getLogger(name='test_mono_not_inverted'),
     )
 
-    im = Image.open(fspath(outdir / 'im.png'))
+    with Image.open(fspath(outdir / 'im.png')) as im:
-    assert im.getpixel((0, 0)) == 255, "Expected white background"
+        assert im.getpixel((0, 0)) == 255, "Expected white background"
 
 
 @pytest.mark.skipif(not pngquant.available(), reason='need pngquant')
@@ -110,10 +110,10 @@ def test_flate_to_jbig2(resources, outdir, spoof_tesseract_noop):
     # This test requires an image that pngquant is capable of converting to
     # to 1bpp - so use an existing 1bpp image, convert up, confirm it can
     # convert down
-    im = Image.open(fspath(resources / 'typewriter.png'))
+    with Image.open(fspath(resources / 'typewriter.png')) as im:
-    assert im.mode in ('1', 'P')
+        assert im.mode in ('1', 'P')
-    im = im.convert('L')
+        im = im.convert('L')
-    im.save(fspath(outdir / 'type8.png'))
+        im.save(fspath(outdir / 'type8.png'))
 
     check_ocrmypdf(
         outdir / 'type8.png',

tests/test_rotation.py

@@ -224,12 +224,12 @@ def test_rotate_deskew_timeout(resources, outdir):
 @pytest.mark.parametrize('image_angle', (0, 90, 180, 270))
 def test_rotate_page_level(image_angle, page_angle, resources, outdir):
     def make_rotate_test(prefix, image_angle, page_angle):
-        im = Image.open(fspath(resources / 'typewriter.png'))
-        if image_angle != 0:
-            ccw_angle = -image_angle % 360
-            im = im.transpose(getattr(Image, f'ROTATE_{ccw_angle}'))
         memimg = BytesIO()
-        im.save(memimg, format='PNG')
+        with Image.open(fspath(resources / 'typewriter.png')) as im:
+            if image_angle != 0:
+                ccw_angle = -image_angle % 360
+                im = im.transpose(getattr(Image, f'ROTATE_{ccw_angle}'))
+            im.save(memimg, format='PNG')
         memimg.seek(0)
         mempdf = BytesIO()
         img2pdf.convert(

tests/test_validation.py

@@ -15,20 +15,22 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
+import logging
 import os
-from unittest.mock import MagicMock, patch
+from unittest.mock import patch
 
 import pytest
 
 import ocrmypdf._validation as vd
 from ocrmypdf.api import create_options
 from ocrmypdf.exceptions import MissingDependencyError, BadArgsError
+from ocrmypdf.pdfinfo import PdfInfo
 
 
 def make_opts(input_file='a.pdf', output_file='b.pdf', language='eng', **kwargs):
-    return create_options(
+    if language is not None:
-        input_file=input_file, output_file=output_file, language=language, **kwargs
+        kwargs['language'] = language
-    )
+    return create_options(input_file=input_file, output_file=output_file, **kwargs)
 
 
 def test_hocr_notlatin_warning(caplog):
@@ -37,15 +39,21 @@ def test_hocr_notlatin_warning(caplog):
 
 
 def test_old_ghostscript(caplog):
-    with patch('ocrmypdf.exec.ghostscript.version', return_value='9.19'):
+    with patch('ocrmypdf.exec.ghostscript.version', return_value='9.19'), patch(
+        'ocrmypdf.exec.tesseract.has_textonly_pdf', return_value=True
+    ):
         vd.check_options_output(make_opts(language='chi_sim', output_type='pdfa'))
         assert 'Ghostscript does not work correctly' in caplog.text
 
-    with patch('ocrmypdf.exec.ghostscript.version', return_value='9.18'):
+    with patch('ocrmypdf.exec.ghostscript.version', return_value='9.18'), patch(
+        'ocrmypdf.exec.tesseract.has_textonly_pdf', return_value=True
+    ):
         with pytest.raises(MissingDependencyError):
             vd.check_options_output(make_opts(output_type='pdfa-3'))
 
-    with patch('ocrmypdf.exec.ghostscript.version', return_value='9.24'):
+    with patch('ocrmypdf.exec.ghostscript.version', return_value='9.24'), patch(
+        'ocrmypdf.exec.tesseract.has_textonly_pdf', return_value=True
+    ):
         with pytest.raises(MissingDependencyError):
             vd.check_dependency_versions(make_opts())
 
@@ -108,8 +116,63 @@ def test_report_file_size(tmp_path, caplog):
     opts = make_opts()
     vd.report_output_file_size(opts, in_, out)
     assert caplog.text == ''
+    caplog.clear()
 
     os.truncate(in_, 25001)
     os.truncate(out, 50000)
+    with patch('ocrmypdf._validation.jbig2enc.available', return_value=True), patch(
+        'ocrmypdf._validation.pngquant.available', return_value=True
+    ):
+        vd.report_output_file_size(opts, in_, out)
+        assert 'No reason' in caplog.text
+    caplog.clear()
+
+    with patch('ocrmypdf._validation.jbig2enc.available', return_value=False), patch(
+        'ocrmypdf._validation.pngquant.available', return_value=True
+    ):
+        vd.report_output_file_size(opts, in_, out)
+        assert 'optional dependency' in caplog.text
+    caplog.clear()
+
+    opts = make_opts(in_, out, optimize=0)
     vd.report_output_file_size(opts, in_, out)
-    assert 'No reason' in caplog.text
+    assert 'disabled' in caplog.text
+    caplog.clear()
+
+
+def test_false_action_store_true():
+    opts = make_opts(keep_temporary_files=True)
+    assert opts.keep_temporary_files == True
+    opts = make_opts(keep_temporary_files=False)
+    assert opts.keep_temporary_files == False
+
+
+@pytest.mark.parametrize('progress_bar', [True, False])
+def test_no_progress_bar(progress_bar, resources):
+    opts = make_opts(progress_bar=progress_bar, input_file=(resources / 'trivial.pdf'))
+    with patch('ocrmypdf.pdfinfo.info.tqdm', autospec=True) as tqdmpatch:
+        vd.check_options(opts)
+        pdfinfo = PdfInfo(opts.input_file, progbar=opts.progress_bar)
+        assert pdfinfo is not None
+        assert tqdmpatch.called
+        _args, kwargs = tqdmpatch.call_args
+        assert kwargs['disable'] != progress_bar
+
+
+def test_language_warning(caplog):
+    opts = make_opts(language=None)
+    caplog.set_level(logging.DEBUG)
+    with patch(
+        'ocrmypdf._validation.locale.getlocale', return_value=('en_US', 'UTF-8')
+    ):
+        vd.check_options_languages(opts)
+        assert opts.language == ['eng']
+        assert '' in caplog.text
+
+    opts = make_opts(language=None)
+    with patch(
+        'ocrmypdf._validation.locale.getlocale', return_value=('fr_FR', 'UTF-8')
+    ):
+        vd.check_options_languages(opts)
+        assert opts.language == ['eng']
+        assert 'assuming --language' in caplog.text