v9.1.1 release notes

Smaller download, needed for manylinux2010.

.docker/Dockerfile

@@ -1,6 +1,6 @@
 # OCRmyPDF
 #
-FROM ubuntu:19.04 as base
+FROM ubuntu:19.10 as base
 
 FROM base as builder
 
@@ -10,36 +10,36 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   build-essential autoconf automake libtool \
   libleptonica-dev \
   zlib1g-dev \
-  ocrmypdf \
+  python3 \
-  pngquant \
+  python3-distutils \
-  python3-pip \
+  ca-certificates \
-  python3-venv \
+  curl \
-  tesseract-ocr \
-  unpaper \
-  wget \
   git
 
+# Get the latest pip (Ubuntu version doesn't support manylinux2010)
+RUN \
+  curl https://bootstrap.pypa.io/get-pip.py | python3
 
 # Compile and install jbig2
 # Needs libleptonica-dev, zlib1g-dev
 RUN \
   mkdir jbig2 \
-  && wget -q https://github.com/agl/jbig2enc/archive/0.29.tar.gz -O - | \
+  && curl -L https://github.com/agl/jbig2enc/archive/0.29.tar.gz | \
   tar xz -C jbig2 --strip-components=1 \
   && cd jbig2 \
   && ./autogen.sh && ./configure && make && make install \
   && cd .. \
   && rm -rf jbig2
 
-RUN python3 -m venv /appenv
-
 COPY . /app
 
 WORKDIR /app
 
-RUN . /appenv/bin/activate; \
+RUN pip3 install --no-cache-dir \
-  pip install --upgrade pip \
+  -r requirements/main.txt \
-  && pip install .
+  -r requirements/webservice.txt \
+  -r requirements/test.txt \
+  .
 
 FROM base
 
@@ -53,7 +53,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   zlib1g \
   pngquant \
   python3 \
-  python3-venv \
   qpdf \
   tesseract-ocr \
   tesseract-ocr-chi-sim \
@@ -62,10 +61,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   tesseract-ocr-fra \
   tesseract-ocr-por \
   tesseract-ocr-spa \
-  unpaper \
+  unpaper
-  wget
+
+WORKDIR /app
+
+COPY --from=builder /usr/local/lib/ /usr/local/lib/
+COPY --from=builder /usr/local/bin/ /usr/local/bin/
 
-# Copy
 COPY --from=builder /app/misc/webservice.py /app/
 
 # Copy minimal project files to get the test suite.
@@ -74,7 +76,4 @@ COPY --from=builder /app/requirements /app/requirements
 COPY --from=builder /app/tests /app/tests
 COPY --from=builder /app/src /app/src
 
-COPY --from=builder /appenv /appenv
+ENTRYPOINT ["/usr/local/bin/ocrmypdf"]
-COPY --from=builder /usr/local /usr/local
-
-ENTRYPOINT ["/appenv/bin/ocrmypdf"]

.docker/alpine.dockerfile

@@ -1,91 +0,0 @@
-FROM alpine:3.9 as base
-
-FROM base as builder
-
-ENV LANG=C.UTF-8
-
-# Normally:
-# echo '@testing http://nl.alpinelinux.org/alpine/edge/testing' >> /etc/apk/repositories
-
-RUN \
-  echo -e '@testing http://nl.alpinelinux.org/alpine/edge/testing\n@community http://nl.alpinelinux.org/alpine/edge/community'\
-  >> /etc/apk/repositories \
-  # Add runtime dependencies
-  && apk add --update \
-  python3-dev \
-  py3-setuptools \
-  jbig2enc@community \
-  ghostscript \
-  qpdf@community \
-  qpdf-dev@community \
-  tesseract-ocr \
-  unpaper \
-  pngquant \
-  libxml2-dev \
-  libxslt-dev \
-  zlib-dev \
-  libffi-dev \
-  leptonica-dev \
-  binutils \
-  && pip3 install --upgrade pip \
-  # Install pybind11 for pikepdf
-  && pip3 install pybind11 \
-  # Install flask for the webservice
-  && pip3 install flask \
-  # Add build dependencies
-  && apk add --virtual build-dependencies \
-  build-base \
-  git
-
-COPY . /app
-
-WORKDIR /app
-
-RUN pip3 install .
-
-FROM base
-
-ENV LANG=C.UTF-8
-
-# Normally:
-# echo '@testing http://nl.alpinelinux.org/alpine/edge/testing' >> /etc/apk/repositories
-
-RUN \
-  echo -e '@testing http://nl.alpinelinux.org/alpine/edge/testing\n@community http://nl.alpinelinux.org/alpine/edge/community'\
-  >> /etc/apk/repositories \
-  # Add runtime dependencies
-  && apk add --update \
-  python3 \
-  jbig2enc@community \
-  ghostscript \
-  qpdf@community \
-  qpdf-dev@community \
-  tesseract-ocr \
-  tesseract-ocr-data-deu \
-  tesseract-ocr-data-chi_sim \
-  unpaper \
-  pngquant \
-  libxml2 \
-  libxslt \
-  zlib \
-  libffi \
-  leptonica-dev \
-  binutils \
-  && mkdir /app
-
-WORKDIR /app
-
-# Copy build artifacts (python site-packages)
-COPY --from=builder /usr/lib/python3.6/site-packages /usr/lib/python3.6/site-packages
-COPY --from=builder /usr/bin/ocrmypdf /usr/bin/dumppdf.py /usr/bin/latin2ascii.py /usr/bin/pdf2txt.py /usr/bin/img2pdf /usr/bin/chardetect /usr/bin/
-
-# Copy
-COPY --from=builder /app/misc/webservice.py /app/
-
-# Copy minimal project files to get the test suite.
-COPY --from=builder /app/setup.cfg /app/setup.py /app/README.md /app/
-COPY --from=builder /app/requirements /app/requirements
-COPY --from=builder /app/tests /app/tests
-COPY --from=builder /app/src /app/src
-
-ENTRYPOINT ["/usr/bin/ocrmypdf"]

.travis.yml

@@ -62,8 +62,6 @@ matrix:
         mkdir -p bin packages
         pip3 install --upgrade pip
         pip3 install --upgrade wheel
-        wget -q 'https://www.dropbox.com/s/vaq0kbwi6e6au80/unpaper_6.1-1.deb?raw=1' -O packages/unpaper_6.1-1.deb
-        sudo dpkg -i packages/unpaper_6.1-1.deb
     - os: linux
       dist: xenial
       sudo: required
@@ -88,8 +86,31 @@ matrix:
             - tesseract-ocr-eng
             - tesseract-ocr-fra
             - unpaper
+    - os: linux
+      dist: bionic
+      sudo: required
+      language: python
+      python: "3.8"
+      env:
+        - DIST=bionic
+      addons:
+        apt:
+          update: true
+          sources:
+            - sourceline: "ppa:alex-p/tesseract-ocr"
+          packages:
+            - ghostscript
+            - libexempi3
+            - libffi-dev
+            - pngquant
+            - poppler-utils
+            - qpdf
+            - tesseract-ocr
+            - tesseract-ocr-deu
+            - tesseract-ocr-eng
+            - tesseract-ocr-fra
+            - unpaper
     - os: osx
-      osx_image: xcode9.2
       language: generic
       addons:
         homebrew:
@@ -115,10 +136,7 @@ before_cache:
 install:
   - mkdir -p bin
   - export PATH=$PWD/bin:$PATH
-  - pip3 install pycparser # py3.7 workaround for https://github.com/eliben/pycparser/issues/251
+  - pip3 install -r requirements/main.txt -r requirements/test.txt .
-  - pip3 install -r requirements/main.txt
-  - pip3 install --no-deps .
-  - pip3 install -r requirements/test.txt
 
 script:
   - tesseract --version

MANIFEST.in

@@ -1,43 +0,0 @@
-# requirements
-recursive-include requirements *
-
-# git
-include .git_archival.txt
-
-# docker
-include .dockerignore
-recursive-include .docker *
-
-# tests
-include .coveragerc
-recursive-include tests *.bin
-recursive-include tests *.jpg
-recursive-include tests *.jsonl
-recursive-include tests *.png
-recursive-include tests *.pdf
-recursive-include tests *.py
-recursive-include tests *.rst
-recursive-include tests *.txt
-recursive-exclude tests/resources/private *
-
-# documentation
-include LICENSE
-include *.rst
-recursive-exclude .github *
-recursive-include docs *.py
-recursive-include docs *.rst
-recursive-include docs *.svg
-recursive-exclude docs/_build *
-
-
-# support files
-recursive-include src/ocrmypdf/data *
-include *.py
-exclude tasks.py
-recursive-exclude .travis *
-exclude .travis*
-
-
-# code
-exclude src/ocrmypdf/lib/_leptonica.py
-exclude scratch.py

README.md

@@ -1,6 +1,6 @@
 <img src="docs/images/logo.svg" width="240" alt="OCRmyPDF">
 
-[![Travis build status][travis]](https://travis-ci.org/jbarlow83/OCRmyPDF) [![PyPI version][pypi]](https://pypi.org/project/ocrmypdf/) ![Homebrew version][homebrew] ![ReadTheDocs][docs]
+[![Travis build status][travis]](https://travis-ci.org/jbarlow83/OCRmyPDF) [![PyPI version][pypi]](https://pypi.org/project/ocrmypdf/) ![Homebrew version][homebrew] ![ReadTheDocs][docs] ![Python versions][pyversions]
 
 [travis]: https://travis-ci.org/jbarlow83/OCRmyPDF.svg?branch=master "Travis build status"
 
@@ -10,6 +10,8 @@
 
 [docs]: https://readthedocs.org/projects/ocrmypdf/badge/?version=latest "RTD"
 
+[pyversions]: https://img.shields.io/pypi/pyversions/ocrmypdf "Supported Python versions"
+
 OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched or copy-pasted.
 
 ```bash
@@ -120,7 +122,7 @@ If you detect an issue, please:
 Requirements
 ------------
 
-Runs on CPython 3.5, 3.6 and 3.7. Requires external program installations of Ghostscript, Tesseract OCR, QPDF, and Leptonica. ocrmypdf is pure Python, but uses CFFI to portably generate library bindings.
+In addition to the required Python version (3.6+), OCRmyPDF requires external program installations of Ghostscript, Tesseract OCR, QPDF, and Leptonica. ocrmypdf is pure Python, but uses CFFI to portably generate library bindings.
 
 Press & Media
 -------------

docs/advanced.rst

@@ -322,15 +322,23 @@ working files on a per page basis have the page number as a prefix
 (starting with page 1), an infix indicates the processing stage, and a
 suffix indicates the file type. Some important files include:
 
--  ``.page.png`` - what the input page looks like
+-  ``_rasterize.png`` - what the input page looks like
--  ``.image`` - the image we will show the user if we are in a mode that
+-  ``_ocr.png`` - the file that is sent to Tesseract for OCR; depending
-   changes the final appearance; may be in one of several image formats
--  ``.text.pdf`` - the OCR file; this will load as a blank page but
-   should have visible text if checked with a tool like pdftotext or
-   pdfminder.six
--  ``.ocr.png`` - the file that is sent to Tesseract for OCR; depending
    on arguments this may differ from the presentation image
--  ``layers.rendered.pdf`` - the composite PDF, before metadata repair
+-  ``_pp_deskew.png`` - the image, after deskewing
-   and optimization
+-  ``_pp_clean.png`` - the image, after cleaning with unpaper
+-  ``_ocr_tess.pdf`` - the OCR file; appears as a blank page with invisible
+   text embedded
+-  ``_ocr_tess.txt`` - the OCR text (not necessarily all text on the page,
+   if the page is mixed format)
+-  ``fix_docinfo.pdf`` - a temporary file created to fix the PDF DocumentInfo
+   data structure
+-  ``graft_layers.pdf`` - the rendered PDF with OCR layers grafted on
+-  ``pdfa.pdf`` - ``graft_layers.pdf`` after conversion to PDF/A
+-  ``pdfa.ps`` - a PostScript file used by Ghostscript for PDF/A conversion
+-  ``optimize.pdf`` - the PDF generated before optimization
+-  ``optimize.out.pdf`` - the PDF generated by optimization
+-  ``origin`` - the input file
+-  ``origin.pdf`` - the input file or the input image converted to PDF
 -  ``images/*`` - images extracted during the optimization process; here
    the prefix indicates a PDF object ID not a page number

docs/api.rst

@@ -71,7 +71,8 @@ Progress monitoring
 OCRmyPDF uses the ``tqdm`` package to implement its progress bars.
 :func:`ocrmypdf.configure_logging` will set up logging output to
 ``sys.stderr`` in a way that is compatible with the display of the
-progress bar.
+progress bar. Use ``ocrmypdf.ocr(...progress_bar=False)`` to disable
+the progress bar.
 
 Exceptions
 ----------

docs/batch.rst

@@ -47,7 +47,7 @@ where the PDFs are stored):
 
 .. code-block:: bash
 
-   find . -printf '%p' -name '*.pdf' -exec docker run --rm -v <host dir>:<container dir> jbarlow83/ocrmypdf-alpine '<container dir>/{}' '<container dir>/{}' \;
+   find . -printf '%p' -name '*.pdf' -exec docker run --rm -v <host dir>:<container dir> jbarlow83/ocrmypdf '<container dir>/{}' '<container dir>/{}' \;
 
 This only runs one ``ocrmypdf`` process at a time. This variation uses
 ``find`` to create a directory list and ``parallel`` to parallelize runs
@@ -244,6 +244,9 @@ Caveats
 Alternatives
 ------------
 
+-  `systemd user services <https://wiki.archlinux.org/index.php/Systemd/User>`__
+   can be configured to automatically perform OCR on a collection of files.
+
 -  `Watchman <https://facebook.github.io/watchman/>`__ is a more
    powerful alternative to ``watchmedo``.
 

docs/contributing.rst

@@ -0,0 +1,38 @@
+=======================
+Contributing guidelines
+=======================
+
+Contributions are welcome!
+
+Big changes
+===========
+
+Please open a new issue to discuss or propose a major change. Not only is it fun
+to discuss big ideas, but we might save each other's time too. Perhaps some of the
+work you're contemplating is already half-done in a development branch.
+
+Code style
+==========
+
+We use PEP8, ``black`` for code formatting and ``isort`` for import sorting. The
+settings for programs are in ``pyproject.toml`` and ``setup.cfg``.
+
+Tests
+=====
+
+New features should come with tests that confirm their correctness.
+
+New Python dependencies
+=======================
+
+If you are proposing a change that will require a new Python dependency, we
+prefer dependencies that are already packaged by Debian or Red Hat. This makes
+life much easier for our downstream package maintainers.
+
+Python dependencies must also be GPLv3 compatible.
+
+New non-Python dependencies
+===========================
+
+OCRmyPDF uses several external programs (Tesseract, Ghostscript and others) for
+its functionality. In general we prefer to avoid adding new external programs.

docs/cookbook.rst

@@ -216,6 +216,15 @@ processing or PDF/A conversion.
 
     ocrmypdf --tesseract-timeout=0 --remove-background input.pdf output.pdf
 
+Optimize images without performing OCR
+--------------------------------------
+
+You can also optimize all images without performing any OCR:
+
+.. code-block:: bash
+
+    ocrmypdf --tesseract-timeout=0 --optimize 3 --skip-text input.pdf output.pdf
+
 Redo existing OCR
 =================
 

docs/docker.rst

@@ -22,21 +22,20 @@ Installing the Docker image
 If you have `Docker <https://docs.docker.com/>`__ installed on your
 system, you can install a Docker image of the latest release.
 
-The recommended OCRmyPDF Docker image is currently named
+If you can run this command successfully, your system is ready to download and
-``ocrmypdf-alpine``:
-
-.. code-block:: bash
-
-   docker pull jbarlow83/ocrmypdf-alpine
-
-Follow the Docker installation instructions for your platform. If you
-can run this command successfully, your system is ready to download and
 execute the image:
 
 .. code-block:: bash
 
    docker run hello-world
 
+The recommended OCRmyPDF Docker image is currently named ``ocrmypdf``:
+
+.. code-block:: bash
+
+   docker pull jbarlow83/ocrmypdf
+
+
 OCRmyPDF will use all available CPU cores. By default, the VirtualBox
 machine instance on Windows and macOS has only a single CPU core
 enabled. Use the VirtualBox Manager to determine the name of your Docker
@@ -51,6 +50,9 @@ CPUs:
    docker-machine start "yourVM"
    eval $(docker-machine env "yourVM")
 
+See the Docker documentation for
+`adjusting memory and CPU on other platforms <https://docs.docker.com/config/containers/resource_constraints/>`__.
+
 Using the Docker image on the command line
 ==========================================
 
@@ -63,7 +65,7 @@ To start a Docker container (instance of the image):
 
 .. code-block:: bash
 
-   docker tag jbarlow83/ocrmypdf-alpine ocrmypdf
+   docker tag jbarlow83/ocrmypdf ocrmypdf
    docker run --rm -i ocrmypdf (... all other arguments here...)
 
 For convenience, create a shell alias to hide the Docker command. It is
@@ -103,12 +105,12 @@ on the public one:
 
 .. code-block:: dockerfile
 
-   FROM jbarlow83/ocrmypdf-alpine
+   FROM jbarlow83/ocrmypdf
 
    # Add French
-   RUN apk add tesseract-ocr-data-fra
+   RUN apt install tesseract-ocr-fra
 
-You can also copy training data to ``/usr/share/tessdata``.
+You can also copy training data to ``/usr/share/tesseract-ocr/<tesseract version>/tessdata``.
 
 Executing the test suite
 ========================
@@ -117,17 +119,16 @@ The OCRmyPDF test suite is installed with image. To run it:
 
 .. code-block:: bash
 
-   docker run --entrypoint python3 jbarlow83/ocrmypdf-alpine setup.py test
+   docker run --entrypoint python3  jbarlow83/ocrmypdf -m pytest
 
 Accessing the shell
 ===================
 
-``bash`` is not installed in the image. To use the busybox shell in the
+To use the bash shell in the Docker image:
-Docker image:
 
 .. code-block:: bash
 
-   docker run -it --entrypoint busybox  jbarlow83/ocrmypdf-alpine sh
+   docker run -it --entrypoint bash  jbarlow83/ocrmypdf
 
 Using the OCRmyPDF web service wrapper
 ======================================
@@ -137,7 +138,12 @@ service. The webservice may be launched as follows:
 
 .. code-block:: bash
 
-   docker run --entrypoint python3 -p 5000:5000 jbarlow83/ocrmypdf-alpine webservice.py
+   docker run --entrypoint python3 -p 5000:5000  jbarlow83/ocrmypdf webservice.py
+
+This will configure the machine to listen on port 5000. On Linux machines
+this is port 5000 of localhost. On macOS or Windows machines running
+Docker, this is port 5000 of the virtual machine that runs your Docker
+images. You can find its IP address using the command ``docker-machine ip``.
 
 Unlike command line usage this program will open a socket and wait for
 connections.
@@ -162,14 +168,3 @@ also licensed in this way.
 
 In addition to the above, please read our
 :ref:`general remarks on using OCRmyPDF as a service <ocr-service>`.
-
-Ubuntu-based Docker image
-=========================
-
-A Ubuntu-based OCRmyPDF image is also available. The main advantage this
-image offers is that it supports manylinux Python wheels (which are not
-supported on Alpine Linux). This may be useful for plugins.
-
-.. code-block:: bash
-
-   docker pull jbarlow83/ocrmypdf

docs/index.rst

@@ -12,6 +12,7 @@ PDF is the best format for storing and exchanging scanned documents.  Unfortunat
    introduction
    release_notes
    installation
+   optimizer
    languages
    jbig2
 
@@ -22,11 +23,16 @@ PDF is the best format for storing and exchanging scanned documents.  Unfortunat
    cookbook
    docker
    advanced
-   api
    batch
    security
    errors
 
+.. toctree::
+   :caption: Developers
+   :maxdepth: 2
+
+   api
+   contributing
 
 Indices and tables
 ==================

docs/installation.rst

@@ -134,8 +134,7 @@ first install the system version to get most of the dependencies:
 
     sudo apt-get update
     sudo apt-get install \
-        ocrmypdf \
+        ocrmypdf
-        python3-pip
 
 There are a few system dependency changes since ocrmypdf 6.1.2. Let's
 get these, too.
@@ -146,13 +145,19 @@ get these, too.
         libxml2 \
         pngquant
 
+We will need a newer version of ``pip`` then was available for Ubuntu 18.04:
+
+.. code-block:: bash
+
+    wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py
+
 Then install the most recent ocrmypdf for the local user and set the
 user's ``PATH`` to check for the user's Python packages.
 
 .. code-block:: bash
 
     export PATH=$HOME/.local/bin:$PATH
-    pip3 install --user ocrmypdf
+    python3 -m pip install --user ocrmypdf
 
 To add JBIG2 encoding, see :ref:`jbig2`.
 
@@ -272,15 +277,11 @@ Now we need to install ``pip`` and let it install ocrmypdf:
     curl https://bootstrap.pypa.io/ez_setup.py -o - | python3.6 && python3.6 -m easy_install pip
     pip3.6 install ocrmypdf
 
-These installation instructions omit the optional dependency
+The optional dependency ``unpaper`` is only available at 0.4.2 in Ubuntu 14.04,
-``unpaper``, which is only available at version 0.4.2 in Ubuntu 14.04.
+and no backports are available. Previously the author maintained a backported
-The author could not find a backport of ``unpaper``, and created a .deb
+.deb package for unpaper 6.1, but since Ubuntu 14.04 is now end of life, this is
-package to do the job of installing unpaper 6.1 (for x86 64-bit only):
+not supported. As such, ``unpaper`` is not available on Ubuntu 14.04 or must by
-
+compiled by hand.
-.. code-block:: bash
-
-    wget -q 'https://www.dropbox.com/s/vaq0kbwi6e6au80/unpaper_6.1-1.deb?raw=1' -O unpaper_6.1-1.deb
-    sudo dpkg -i unpaper_6.1-1.deb
 
 To add JBIG2 encoding, see :ref:`jbig2`.
 
@@ -517,7 +518,7 @@ manager. ``pip`` cannot provide them.
 
 As of ocrmypdf 7.2.1, the following versions are recommended:
 
--  Python 3.7
+-  Python 3.7 or 3.8
 -  Ghostscript 9.23 or newer
 -  qpdf 8.2.1
 -  Tesseract 4.0.0 or newer

docs/introduction.rst

@@ -208,7 +208,7 @@ consider one of these similar open source programs:
 Web front-ends
 ==============
 
-The Docker image ``ocrmypdf-alpine`` provides a web service front-end
+The Docker image ``ocrmypdf`` provides a web service front-end
 that allows files to submitted over HTTP and the results "downloaded".
 This is an HTTP server intended to simplify web services deployments; it
 is not intended to be deployed on the public internet and no real

docs/optimizer.rst

@@ -0,0 +1,71 @@
+================
+PDF optimization
+================
+
+OCRmyPDF includes an image-oriented PDF optimizer. By default, the optimizer
+runs with safe settings with the goal of improving compression at no loss of
+quality. At higher optimization levels, lossy optimizations may be applied and
+tuned. Optimization occurs after OCR, and only if OCR succeeded.  It does not
+perform other possible optimizations such as deduplicating resources,
+consolidating fonts, simplifying vector drawings, or anything of that nature.
+
+Optimization ranges from ``-O0`` through ``-O3``, where ``0`` disables
+optimization and ``3`` implements all options. ``1``, the default, performs only
+safe and lossless optimizations. (This is similar to GCC's optimization
+parameter.) The exact type of optimizations performed will vary over time.
+
+Optimizations that always occurs
+================================
+
+OCRmyPDF will automatically replace obsolete or inferior compression schemes
+such as RLE or LZW with superior schemes such as Deflate and converting
+monochrome images to CCITT G4. Since this is harmless it always occurs and there
+is no way to disable it. Other non-image compressed objects are compressed as
+well.
+
+Fast web view
+=============
+
+OCRmyPDF automatically optimizes PDFs for "fast web view" in Adobe Acrobat's
+parlance, or equivalently, linearizes PDFs so that the resources they reference
+are presented in the order a viewer needs them for sequential display. This
+reduces the latency of viewing a PDF both online and from local storage. This
+actually slightly increases the file size.
+
+To disable this optimization and all others, use ``ocrmypdf --optimize 0 ...``
+or the shorthand ``-O0``.
+
+Lossless optimizations
+======================
+
+At optimization level ``-O1`` (the default), OCRmyPDF will also attempt lossless
+image optimization.
+
+If a JBIG2 encoder is available, then monochrome images will be converted to
+JBIG2, with the potential for huge savings on large black and white images,
+since JBIG2 is far more efficient than any other monochrome (bi-level)
+compression. (All known US patents related to JBIG2 have probably expired, but
+it remains the responsibility of the user to supply a JBIG2 encoder such as
+`jbig2enc <https://github.com/agl/jbig2enc>`__. OCRmyPDF does not implement
+JBIG2 encoding on its own.)
+
+OCRmyPDF currently does not attempt to recompress losslessly compressed objects
+more aggressively.
+
+Lossy optimizations
+===================
+
+At optimization level ``-O2`` and ``-O3``, OCRmyPDF will some attempt lossy
+image optimization.
+
+If ``pngquant`` is installed, OCRmyPDF will use it to perform quantize paletted
+images to reduce their size.
+
+The quality of JPEGs may be lowered, on the assumption that a lower quality
+image may be suitable for storage after OCR.
+
+It is not possible to optimize all image types. Uncommon image types may be
+skipped by the optimizer.
+
+OCRmyPDF provides :ref:`lossy mode JBIG2 <jbig2-lossy>` as an advanced feature
+that additional requires the argument ``--jbig2-lossy``.

docs/release_notes.rst

@@ -13,6 +13,53 @@ Note that it is licensed under GPLv3, so scripts that
 ``import ocrmypdf`` and are released publicly should probably also be
 licensed under GPLv3.
 
+v9.1.1
+======
+
+-  Expand the range of pdfminer.six versions that are supported.
+-  Fixed Docker build when using pikepdf 1.7.0.
+-  Fixed documentation to recommend using pip from get-pip.py.
+
+v9.1.0
+======
+
+-  Improved diagnostics when file size increases at output. Now warns if JBIG2
+   or pngquant were not available.
+-  pikepdf 1.7.0 is now required, to pick up changes that remove the need for
+   a source install on Linux systems running Python 3.8.
+
+v9.0.5
+======
+
+-  The Alpine Docker image (jbarlow83/ocrmypdf-alpine) has been dropped due to
+   the difficulties of supporting Alpine Linux.
+-  The primary Docker image (jbarlow83/ocrmypdf) has been improved to take on
+   the extra features that used to be exclusive to the Alpine image.
+-  No changes to application code.
+-  pdfminer.six version 20191020 is now supported.
+
+v9.0.4
+======
+
+-  Fixed compatibility with Python 3.8 (but requires source install for the moment).
+-  Fixed Tesseract settings for ``--user-words`` and ``--user-patterns``.
+-  Changed to pikepdf 1.6.5 (for Python 3.8).
+-  Changed to Pillow 6.2.0 (to mitigate a security vulnerability in earlier Pillow).
+-  A debug message now mentions when English is automatically selected if the locale
+   is not English.
+
+v9.0.3
+======
+
+-  Embed an encoded version of the sRGB ICC profile in the intermediate
+   Postscript file (used for PDF/A conversion). Previously we included the
+   filename, which required Postscript to run with file access enabled. For
+   security, Ghostscript 9.28 enables ``-dSAFER`` and as such, no longer
+   permits access to any file by default. This fix is necessary for
+   compatibility with Ghostscript 9.28.
+-  Exclude a test that sometimes times out and fails in continuous integration
+   from the standard test suite.
+
 v9.0.2
 ======
 
@@ -23,7 +70,7 @@ v9.0.2
 -  Fixed an issue that caused inversion of black and white in monochrome images.
    We are not certain but the problem seems to be linked to Leptonica 1.76.0 and
    older.
--  Fixed some cases where the test suite failed or produced unexpected if
+-  Fixed some cases where the test suite failed if
    English or German Tesseract language packs were not installed.
 -  Fixed a runtime error if the Tesseract English language is not installed.
 -  Improved explicit closing of Pillow images after use.

pyproject.toml

@@ -10,7 +10,7 @@ build-backend = "setuptools.build_meta"
 
 [tool.black]
 line-length = 88
-py36 = true
+target-version = ["py36", "py37", "py38"]
 skip-string-normalization = true
 include = '\.pyi?$'
 exclude = '''

requirements/dev.txt

@@ -1,4 +1,2 @@
-check-manifest >= 0.35
 twine >= 1.8.1
 coverage >= 4.5
-GitPython == 2.1.3

requirements/main.txt

@@ -1,13 +1,10 @@
 # requirements.txt can be used to replicate the developer's build environment
 # setup.py lists a separate set of requirements that are looser to simplify
 # installation
-chardet == 3.0.4
+cffi == 1.13.2
-cffi == 1.12.2
 img2pdf == 0.3.3
-pdfminer.six == 20181108
+pdfminer.six == 20191110
-pikepdf == 1.6.1
+pikepdf == 1.7.0
-Pillow >= 5.0.0, != 5.1.0 ; sys_platform == "darwin"
+Pillow >= 6.2.0
-pycparser == 2.19
+reportlab == 3.5.32
-python-xmp-toolkit == 2.0.1
+tqdm == 4.37.0
-reportlab == 3.5.13
-tqdm == 4.32.1

requirements/test.txt

@@ -2,7 +2,7 @@ pytest >= 5.0.0
 pytest-helpers-namespace >= 2019.1.8
 pytest-xdist >= 1.29.0  # For DumpError fix
 pytest-cov >= 2.6.1
-python-xmp-toolkit   # requires apt-get install libexempi3
+python-xmp-toolkit == 2.0.1  # requires apt-get install libexempi3
                      # or brew install exempi
 PyPDF2 >= 1.26.0
 #PyMuPDF == 1.13.4    # optional

requirements/webservice.txt

@@ -0,0 +1 @@
+Flask >= 1, < 2

setup.py

@@ -68,6 +68,7 @@ setup(
     classifiers=[
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
         "Development Status :: 5 - Production/Stable",
         "Environment :: Console",
         "Intended Audience :: End Users/Desktop",
@@ -95,11 +96,9 @@ setup(
         'chardet >= 3.0.4, < 4',  # unlisted requirement of pdfminer.six 20181108
         'cffi >= 1.9.1',  # must be a setup and install requirement
         'img2pdf >= 0.3.0, < 0.4',  # pure Python, so track HEAD closely
-        'pdfminer.six == 20181108',
+        'pdfminer.six >= 20181108, <= 20191110',
-        'pikepdf >= 1.6.0, < 2',
+        'pikepdf >= 1.7.0, < 2',
-        'Pillow >= 4.0.0, != 5.1.0 ; sys_platform == "darwin"',
+        'Pillow >= 6.2.0',
-        # Pillow < 4 has BytesIO/TIFF bug w/img2pdf 0.2.3
-        # block 5.1.0, broken wheels
         'reportlab >= 3.3.0',  # oldest released version with sane image handling
         'tqdm >= 4',
     ],

src/ocrmypdf/_sync.py

@@ -189,7 +189,7 @@ def worker_init(queue):
     root.addHandler(h)
 
 
-def worker_thread_init(queue):
+def worker_thread_init(_queue):
     pass
 
 
@@ -225,15 +225,15 @@ def exec_concurrent(context):
     if max_workers > 1:
         context.log.info("Start processing %d pages concurrent", max_workers)
 
-    # Tesseract 4.0 is multithreaded, and we also run multiple workers. We want to
+    # Tesseract 4.x can be multithreaded, and we also run multiple workers. We want
-    # avoid the situation where we end up trying to run NxN jobs on N CPU cores,
+    # to manage how many threads it uses to avoid creating total threads than cores.
-    # as that gives poor performance. Performance testing shows we're better off
+    # Performance testing shows we're better off
     # parallelizing ocrmypdf and forcing Tesseract to be single threaded, which we
     # get by setting the envvar OMP_THREAD_LIMIT to 1. But if the page count of the
     # input file is small, then we allow Tesseract to use threads, subject to the
-    # constraint: (ocrmypdf workers) * (tesseract threads) <= max_workers and limiting
+    # constraint: (ocrmypdf workers) * (tesseract threads) <= max_workers.
-    # Tesseract to 4 threads.
+    # As of Tesseract 4.1, 3 threads is the most effective on a 4 core/8 thread system.
-    tess_threads = min(4, context.options.jobs // max_workers)
+    tess_threads = min(3, context.options.jobs // max_workers)
     if context.options.tesseract_env is None:
         context.options.tesseract_env = os.environ.copy()
     context.options.tesseract_env.setdefault('OMP_THREAD_LIMIT', str(tess_threads))
@@ -301,7 +301,7 @@ def exec_concurrent(context):
 class NeverRaise(Exception):
     """An exception that is never raised"""
 
-    pass
+    pass  # pylint: disable=unnecessary-pass
 
 
 def run_pipeline(options, api=False):

src/ocrmypdf/_validation.py

@@ -17,6 +17,7 @@
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
 
+import locale
 import logging
 import os
 import sys
@@ -47,6 +48,7 @@ from .helpers import is_file_writable, is_iterable_notstr, monotonic, re_symlink
 # External dependencies
 
 HOCR_OK_LANGS = frozenset(['eng', 'deu', 'spa', 'ita', 'por'])
+DEFAULT_LANGUAGE = 'eng'  # Enforce English hegemony
 
 log = logging.getLogger(__name__)
 
@@ -58,7 +60,10 @@ verify_python3_env()
 
 def check_options_languages(options):
     if not options.language:
-        options.language = ['eng']  # Enforce English hegemony
+        options.language = [DEFAULT_LANGUAGE]
+        system_lang = locale.getlocale()[0]
+        if system_lang and not system_lang.startswith('en'):
+            log.debug("No language specified; assuming --language %s", DEFAULT_LANGUAGE)
 
     # Support v2.x "eng+deu" language syntax
     if '+' in options.language[0]:
@@ -413,6 +418,20 @@ def report_output_file_size(options, input_file, output_file):
                 f"The argument --{arg.replace('_', '-')} was issued, causing transcoding."
             )
 
+    if options.optimize == 0:
+        reasons.append("Optimization was disabled.")
+    else:
+        image_optimizers = {
+            'jbig2': jbig2enc.available(),
+            'pngquant': pngquant.available(),
+        }
+        for name, available in image_optimizers.items():
+            if not available:
+                reasons.append(
+                    f"The optional dependency '{name}' was not found, so some image "
+                    f"optimizations could not be attempted."
+                )
+
     if reasons:
         explanation = "Possible reasons for this include:\n" + '\n'.join(reasons) + "\n"
     else:

src/ocrmypdf/api.py

@@ -33,7 +33,7 @@ class TqdmConsole:
 
     def __init__(self, file):
         self.file = file
-        self.py36 = sys.version_info >= (3, 6)
+        self.py36 = sys.version_info[0:2] == (3, 6)
 
     def write(self, msg):
         # When no progress bar is active, tqdm.write() routes to print()
@@ -122,13 +122,23 @@ def create_options(*, input_file, output_file, **kwargs):
     for arg, val in kwargs.items():
         if val is None:
             continue
-        if arg == 'tesseract_env':
+
+        # These arguments with special handling for which we bypass
+        # argparse
+        if arg in {'tesseract_env', 'progress_bar'}:
             deferred.append((arg, val))
             continue
+
         cmd_style_arg = arg.replace('_', '-')
-        cmdline.append(f"--{cmd_style_arg}")
+
+        # Booleans are special: add only if True, omit for False
         if isinstance(val, bool):
+            if val:
+                cmdline.append(f"--{cmd_style_arg}")
             continue
+
+        # We have a parameter
+        cmdline.append(f"--{cmd_style_arg}")
         if isinstance(val, (int, float)):
             cmdline.append(str(val))
         elif isinstance(val, str):

src/ocrmypdf/exec/ghostscript.py

@@ -15,8 +15,11 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
+"""Interface to Ghostscript executable"""
+
 import logging
 import re
+import warnings
 from functools import lru_cache
 from os import fspath
 from shutil import copy
@@ -193,7 +196,7 @@ def generate_pdfa(
     output_file,
     compression,
     log,
-    threads=1,
+    threads=None,  # deprecated parameter
     pdf_version='1.5',
     pdfa_part='2',
 ):
@@ -216,6 +219,10 @@ def generate_pdfa(
     """
     if not log:
         log = gslog
+    if threads is not None:
+        warnings.warn(
+            "use of deprecated parameter 'threads'", category=DeprecationWarning
+        )
 
     compression_args = []
     if compression == 'jpeg':
@@ -259,6 +266,7 @@ def generate_pdfa(
                 "-dQUIET",
                 "-dBATCH",
                 "-dNOPAUSE",
+                "-dSAFER",
                 "-dCompatibilityLevel=" + str(pdf_version),
                 "-sDEVICE=pdfwrite",
                 "-dAutoRotatePages=/None",

src/ocrmypdf/exec/jbig2enc.py

@@ -15,6 +15,8 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
+"""Interface to jbig2 executable"""
+
 from functools import lru_cache
 from subprocess import PIPE, run
 

src/ocrmypdf/exec/pngquant.py

@@ -15,6 +15,8 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
+"""Interface to pngquant executable"""
+
 from functools import lru_cache
 from subprocess import run
 from tempfile import NamedTemporaryFile

src/ocrmypdf/exec/qpdf.py

@@ -15,6 +15,8 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
+"""Interface to qpdf executable"""
+
 from functools import lru_cache
 from os import fspath
 from subprocess import PIPE, STDOUT, CalledProcessError, run

src/ocrmypdf/exec/tesseract.py

@@ -15,12 +15,13 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
+"""Interface to Tesseract executable"""
+
 import os
 import shutil
-import sys
 from collections import namedtuple
 from contextlib import suppress
-from functools import lru_cache
+import logging
 from os import fspath
 from subprocess import PIPE, STDOUT, CalledProcessError, TimeoutExpired, run
 
@@ -52,6 +53,11 @@ HOCR_TEMPLATE = """<?xml version="1.0" encoding="UTF-8"?>
 """
 
 
+class TesseractLoggerAdapter(logging.LoggerAdapter):
+    def process(self, msg, kwargs):
+        return '[tesseract] %s' % (msg), kwargs
+
+
 def version(tesseract_env=None):
     return get_version('tesseract', regex=r'tesseract\s(.+)', env=tesseract_env)
 
@@ -179,15 +185,16 @@ def get_orientation(input_file, engine_mode, timeout: float, log, tesseract_env=
         return oc
 
 
-def tesseract_log_output(log, stdout, input_file):
+def tesseract_log_output(mainlog, stdout, input_file):
-    prefix = "[tesseract] "
+    log = TesseractLoggerAdapter(
+        mainlog, extra=mainlog.extra if hasattr(mainlog, 'extra') else None
+    )
 
     try:
         text = stdout.decode()
     except UnicodeDecodeError:
         log.error(
-            prefix
+            "command line output was not utf-8. "
-            + "command line output was not utf-8. "
             + "This usually means Tesseract's language packs do not match "
             "the installed version of Tesseract."
         )
@@ -200,25 +207,25 @@ def tesseract_log_output(log, stdout, input_file):
         elif line.startswith("Warning in pixReadMem"):
             continue
         elif 'diacritics' in line:
-            log.warning(prefix + "lots of diacritics - possibly poor OCR")
+            log.warning("lots of diacritics - possibly poor OCR")
         elif line.startswith('OSD: Weak margin'):
-            log.warning(prefix + "unsure about page orientation")
+            log.warning("unsure about page orientation")
         elif 'Error in pixScanForForeground' in line:
             pass  # Appears to be spurious/problem with nonwhite borders
         elif 'Error in boxClipToRectangle' in line:
             pass  # Always appears with pixScanForForeground message
         elif 'parameter not found: ' in line.lower():
-            log.error(prefix + line.strip())
+            log.error(line.strip())
             problem = line.split('found: ')[1]
             raise TesseractConfigError(problem)
         elif 'error' in line.lower() or 'exception' in line.lower():
-            log.error(prefix + line.strip())
+            log.error(line.strip())
         elif 'warning' in line.lower():
-            log.warning(prefix + line.strip())
+            log.warning(line.strip())
         elif 'read_params_file' in line.lower():
-            log.error(prefix + line.strip())
+            log.error(line.strip())
         else:
-            log.info(prefix + line.strip())
+            log.info(line.strip())
 
 
 def page_timedout(log, input_file, timeout):
@@ -271,6 +278,9 @@ def generate_hocr(
     if user_patterns:
         args_tesseract.extend(['--user-patterns', user_patterns])
 
+    if user_words or user_patterns:
+        args_tesseract.extend(['-c', 'lstm_use_matrix=1'])
+
     # Reminder: test suite tesseract spoofers will break after any changes
     # to the number of order parameters here
     args_tesseract.extend([input_file, prefix, 'hocr', 'txt'] + tessconfig)

src/ocrmypdf/exec/unpaper.py

@@ -18,10 +18,11 @@
 # unpaper documentation:
 # https://github.com/Flameeyes/unpaper/blob/master/doc/basic-concepts.md
 
+"""Interface to unpaper executable"""
+
 import os
 import shlex
 import subprocess
-import sys
 from functools import lru_cache
 from subprocess import PIPE, STDOUT, CalledProcessError
 from tempfile import TemporaryDirectory

src/ocrmypdf/optimize.py

@@ -111,6 +111,11 @@ def extract_image_generic(*, pike, root, log, image, xref, options):
     if pim.bits_per_component == 1:
         return None
 
+    try:
+        pim.indexed  # pikepdf 1.6.3 can't handle [/Indexed [/Array...]]
+    except NotImplementedError:
+        return None
+
     if filtdp[0] == Name.DCTDecode and options.optimize >= 2:
         # This is a simple heuristic derived from some training data, that has
         # about a 70% chance of guessing whether the JPEG is high quality,
@@ -544,11 +549,11 @@ def main(infile, outfile, level, jobs=1):
         """Emulate ocrmypdf's options"""
 
         def __init__(
-            self, input_file, jobs, optimize, jpeg_quality, png_quality, jb2lossy
+            self, input_file, jobs, optimize_, jpeg_quality, png_quality, jb2lossy
         ):
             self.input_file = input_file
             self.jobs = jobs
-            self.optimize = optimize
+            self.optimize = optimize_
             self.jpeg_quality = jpeg_quality
             self.png_quality = png_quality
             self.jbig2_page_group_size = 0
@@ -559,7 +564,7 @@ def main(infile, outfile, level, jobs=1):
     options = OptimizeOptions(
         input_file=infile,
         jobs=jobs,
-        optimize=int(level),
+        optimize_=int(level),
         jpeg_quality=0,  # Use default
         png_quality=0,
         jb2lossy=False,

src/ocrmypdf/pdfa.py

@@ -31,8 +31,7 @@ Ghostscript's handling of pdfmark.
 
 """
 
-import os
+import base64
-from binascii import hexlify
 from pathlib import Path
 from string import Template
 
@@ -48,26 +47,13 @@ SRGB_ICC_PROFILE = pkg_resources.resource_filename('ocrmypdf', ICC_PROFILE_RELPA
 # files, from the Ghostscript documentation. Lines beginning with % are
 # comments. Python substitution variables have a '$' prefix.
 pdfa_def_template = u"""%!
-% Define entries in the document Info dictionary :
+% Define an ICC profile :
 /ICCProfile $icc_profile
 def
 
-% Define an ICC profile :
-
 [/_objdef {icc_PDFA} /type /stream /OBJ pdfmark
-[{icc_PDFA}
+[{icc_PDFA} << /N 3 >> /PUT pdfmark
-<<
+[{icc_PDFA} ICCProfile /PUT pdfmark
-  /N currentpagedevice /ProcessColorModel known {
-    currentpagedevice /ProcessColorModel get dup /DeviceGray eq
-    {pop 1} {
-      /DeviceRGB eq
-      {3}{4} ifelse
-    } ifelse
-  } {
-    (ERROR, unable to determine ProcessColorModel) == flush
-  } ifelse
->> /PUT pdfmark
-[{icc_PDFA} ICCProfile (r) file /PUT pdfmark
 
 % Define the output intent dictionary :
 
@@ -104,16 +90,10 @@ def generate_pdfa_ps(target_filename, icc='sRGB'):
     else:
         raise NotImplementedError("Only supporting sRGB")
 
-    # pdfmark must contain the full path to the ICC profile, and pdfmark must be
+    # Read the ICC profile, encode as ASCII85 and convert to a string which we
-    # also encoded in ASCII. ocrmypdf can be installed anywhere, including to
+    # will insert in the .ps file
-    # paths that have a non-ASCII character in the filename. Ghostscript
+    bytes_icc_profile = Path(icc_profile).read_bytes()
-    # accepts hex-encoded strings and converts them to byte strings, so
+    icc_profile = base64.a85encode(bytes_icc_profile, adobe=True).decode('ascii')
-    # we encode the path with fsencode() and use the hex representation.
-    # UTF-16 not accepted here. (Even though ASCII encodable is the usual case,
-    # do this always to avoid making it a rare conditional.)
-    bytes_icc_profile = os.fsencode(icc_profile)
-    hex_icc_profile = hexlify(bytes_icc_profile)
-    icc_profile = '<' + hex_icc_profile.decode('ascii') + '>'
 
     t = Template(pdfa_def_template)
     ps = t.substitute(icc_profile=icc_profile, icc_identifier=icc)

src/ocrmypdf/pdfinfo/layout.py

@@ -20,6 +20,7 @@ from math import copysign
 from pathlib import Path
 from unittest.mock import patch
 
+import pdfminer
 import pdfminer.encodingdb
 import pdfminer.pdfdevice
 import pdfminer.pdfinterp
@@ -36,51 +37,54 @@ from ..exceptions import EncryptedPdfError
 STRIP_NAME = re.compile(r'[0-9]+')
 
 #
-# Unconditional pdfminer patches
+#  pdfminer 20181108 patches
 #
 
+if pdfminer.__version__ == '20181108':
 
-def name2unicode(name):
+    def name2unicode(name):
-    """Fix pdfminer's name2unicode function
+        """Fix pdfminer's name2unicode function
 
-    Font cids that are mapped to names of the form /g123 seem to be, by convention
+        Font cids that are mapped to names of the form /g123 seem to be, by convention
-    characters with no corresponding Unicode entry. These can be subsetted fonts
+        characters with no corresponding Unicode entry. These can be subsetted fonts
-    or symbolic fonts. There seems to be no way to map /g123 fonts to Unicode,
+        or symbolic fonts. There seems to be no way to map /g123 fonts to Unicode,
-    barring a ToUnicode data structure.
+        barring a ToUnicode data structure.
-    """
+        """
-    if name in glyphname2unicode:
+        if name in glyphname2unicode:
-        return glyphname2unicode[name]
+            return glyphname2unicode[name]
-    if name.startswith('g') or name.startswith('a'):
+        if name.startswith('g') or name.startswith('a'):
-        raise KeyError(name)
-    if name.startswith('uni'):
-        try:
-            return chr(int(name[3:], 16))
-        except ValueError:  # Not hexadecimal
             raise KeyError(name)
-    m = STRIP_NAME.search(name)
+        if name.startswith('uni'):
-    if not m:
+            try:
-        raise KeyError(name)
+                return chr(int(name[3:], 16))
-    return chr(int(m.group(0)))
+            except ValueError:  # Not hexadecimal
+                raise KeyError(name)
+        m = STRIP_NAME.search(name)
+        if not m:
+            raise KeyError(name)
+        return chr(int(m.group(0)))
 
+    pdfminer.encodingdb.name2unicode = name2unicode
 
-pdfminer.encodingdb.name2unicode = name2unicode
+    original_PDFFont_init = PDFFont.__init__
 
-original_PDFFont_init = PDFFont.__init__
+    def PDFFont__init__(self, descriptor, widths, default_width=None):
+        original_PDFFont_init(self, descriptor, widths, default_width)
+        # PDF spec says descent should be negative
+        # A font with a positive descent implies it floats entirely above the
+        # baseline, i.e. it's not really a baseline anymore. I have fonts that
+        # claim a positive descent, but treating descent as positive always seems
+        # to misposition text.
+        if self.descent > 0:
+            self.descent = -self.descent
 
+    PDFFont.__init__ = PDFFont__init__
 
-def PDFFont__init__(self, descriptor, widths, default_width=None):
+#
-    original_PDFFont_init(self, descriptor, widths, default_width)
+# end of pdfminer 20181108 patches
-    # PDF spec says descent should be negative
+#
-    # A font with a positive descent implies it floats entirely above the
-    # baseline, i.e. it's not really a baseline anymore. I have fonts that
-    # claim a positive descent, but treating descent as positive always seems
-    # to misposition text.
-    if self.descent > 0:
-        self.descent = -self.descent
 
 
-PDFFont.__init__ = PDFFont__init__
-
 original_PDFSimpleFont_init = PDFSimpleFont.__init__
 
 
@@ -97,6 +101,7 @@ def PDFSimpleFont__init__(self, descriptor, widths, spec):
 
 
 PDFSimpleFont.__init__ = PDFSimpleFont__init__
+
 #
 # pdfminer patches when creator is PScript5.dll
 #
@@ -207,6 +212,7 @@ class TextPositionTracker(PDFLayoutAnalyzer):
         super().__init__(rsrcmgr, pageno, laparams)
         self.textstate = None
         self.result = None
+        self.cur_item = None  # not defined in pdfminer code as it should be
 
     def begin_page(self, page, ctm):
         super().begin_page(page, ctm)

tests/test_completion.py

@@ -19,6 +19,11 @@ from subprocess import run, PIPE
 
 import pytest
 
+pytestmark = pytest.mark.skipif(
+    pytest.helpers.running_in_docker(),  # pylint: disable=no-member
+    reason="docker can't complete",
+)
+
 
 def test_fish():
     try:

tests/test_main.py

@@ -217,7 +217,7 @@ def test_skip_ocr(spoof_tesseract_cache, resources, outpdf):
     assert pdfinfo[0].has_text
 
 
-def test_redo_ocr(spoof_tesseract_cache, resources, outpdf):
+def test_redo_ocr(resources, outpdf):
     in_ = resources / 'graph_ocred.pdf'
     before = PdfInfo(in_, detailed_page_analysis=True)
     out = outpdf
@@ -613,6 +613,7 @@ language_model_penalty_non_freq_dict_word 0
     )
 
 
+@pytest.mark.slow  # This test sometimes times out in CI
 @pytest.mark.parametrize('renderer', RENDERERS)
 def test_tesseract_config_notfound(renderer, resources, outdir):
     cfg_file = outdir / 'nofile.cfg'
@@ -629,6 +630,7 @@ def test_tesseract_config_notfound(renderer, resources, outdir):
     assert p.returncode == ExitCode.ok, err
 
 
+@pytest.mark.slow  # This test sometimes times out in CI
 @pytest.mark.parametrize('renderer', RENDERERS)
 def test_tesseract_config_invalid(renderer, resources, outdir):
     cfg_file = outdir / 'test.cfg'

tests/test_validation.py

@@ -15,20 +15,22 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
+import logging
 import os
-from unittest.mock import MagicMock, patch
+from unittest.mock import patch
 
 import pytest
 
 import ocrmypdf._validation as vd
 from ocrmypdf.api import create_options
 from ocrmypdf.exceptions import MissingDependencyError, BadArgsError
+from ocrmypdf.pdfinfo import PdfInfo
 
 
 def make_opts(input_file='a.pdf', output_file='b.pdf', language='eng', **kwargs):
-    return create_options(
+    if language is not None:
-        input_file=input_file, output_file=output_file, language=language, **kwargs
+        kwargs['language'] = language
-    )
+    return create_options(input_file=input_file, output_file=output_file, **kwargs)
 
 
 def test_hocr_notlatin_warning(caplog):
@@ -114,8 +116,63 @@ def test_report_file_size(tmp_path, caplog):
     opts = make_opts()
     vd.report_output_file_size(opts, in_, out)
     assert caplog.text == ''
+    caplog.clear()
 
     os.truncate(in_, 25001)
     os.truncate(out, 50000)
+    with patch('ocrmypdf._validation.jbig2enc.available', return_value=True), patch(
+        'ocrmypdf._validation.pngquant.available', return_value=True
+    ):
+        vd.report_output_file_size(opts, in_, out)
+        assert 'No reason' in caplog.text
+    caplog.clear()
+
+    with patch('ocrmypdf._validation.jbig2enc.available', return_value=False), patch(
+        'ocrmypdf._validation.pngquant.available', return_value=True
+    ):
+        vd.report_output_file_size(opts, in_, out)
+        assert 'optional dependency' in caplog.text
+    caplog.clear()
+
+    opts = make_opts(in_, out, optimize=0)
     vd.report_output_file_size(opts, in_, out)
-    assert 'No reason' in caplog.text
+    assert 'disabled' in caplog.text
+    caplog.clear()
+
+
+def test_false_action_store_true():
+    opts = make_opts(keep_temporary_files=True)
+    assert opts.keep_temporary_files == True
+    opts = make_opts(keep_temporary_files=False)
+    assert opts.keep_temporary_files == False
+
+
+@pytest.mark.parametrize('progress_bar', [True, False])
+def test_no_progress_bar(progress_bar, resources):
+    opts = make_opts(progress_bar=progress_bar, input_file=(resources / 'trivial.pdf'))
+    with patch('ocrmypdf.pdfinfo.info.tqdm', autospec=True) as tqdmpatch:
+        vd.check_options(opts)
+        pdfinfo = PdfInfo(opts.input_file, progbar=opts.progress_bar)
+        assert pdfinfo is not None
+        assert tqdmpatch.called
+        _args, kwargs = tqdmpatch.call_args
+        assert kwargs['disable'] != progress_bar
+
+
+def test_language_warning(caplog):
+    opts = make_opts(language=None)
+    caplog.set_level(logging.DEBUG)
+    with patch(
+        'ocrmypdf._validation.locale.getlocale', return_value=('en_US', 'UTF-8')
+    ):
+        vd.check_options_languages(opts)
+        assert opts.language == ['eng']
+        assert '' in caplog.text
+
+    opts = make_opts(language=None)
+    with patch(
+        'ocrmypdf._validation.locale.getlocale', return_value=('fr_FR', 'UTF-8')
+    ):
+        vd.check_options_languages(opts)
+        assert opts.language == ['eng']
+        assert 'assuming --language' in caplog.text