test: further fixes to test_report_file_size

[ci skip]

.docker/Dockerfile

@@ -1,6 +1,6 @@
 # OCRmyPDF
 #
-FROM ubuntu:19.04 as base
+FROM ubuntu:19.10 as base
 
 FROM base as builder
 
@@ -10,16 +10,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   build-essential autoconf automake libtool \
   libleptonica-dev \
   zlib1g-dev \
-  ocrmypdf \
-  pngquant \
+  python3-setuptools \
   python3-pip \
-  python3-venv \
-  tesseract-ocr \
-  unpaper \
   wget \
   git
 
-
 # Compile and install jbig2
 # Needs libleptonica-dev, zlib1g-dev
 RUN \
@@ -31,15 +26,15 @@ RUN \
   && cd .. \
   && rm -rf jbig2
 
-RUN python3 -m venv /appenv
-
 COPY . /app
 
 WORKDIR /app
 
-RUN . /appenv/bin/activate; \
-  pip install --upgrade pip \
-  && pip install .
+RUN pip3 install --no-cache-dir \
+  -r requirements/main.txt \
+  -r requirements/webservice.txt \
+  -r requirements/test.txt \
+  .
 
 FROM base
 
@@ -53,7 +48,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   zlib1g \
   pngquant \
   python3 \
-  python3-venv \
   qpdf \
   tesseract-ocr \
   tesseract-ocr-chi-sim \
@@ -62,10 +56,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   tesseract-ocr-fra \
   tesseract-ocr-por \
   tesseract-ocr-spa \
-  unpaper \
-  wget
+  unpaper
+
+WORKDIR /app
+
+COPY --from=builder /usr/local/lib/ /usr/local/lib/
+COPY --from=builder /usr/local/bin/ /usr/local/bin/
 
-# Copy
 COPY --from=builder /app/misc/webservice.py /app/
 
 # Copy minimal project files to get the test suite.
@@ -74,7 +71,4 @@ COPY --from=builder /app/requirements /app/requirements
 COPY --from=builder /app/tests /app/tests
 COPY --from=builder /app/src /app/src
 
-COPY --from=builder /appenv /appenv
-COPY --from=builder /usr/local /usr/local
-
-ENTRYPOINT ["/appenv/bin/ocrmypdf"]
+ENTRYPOINT ["/usr/local/bin/ocrmypdf"]

.docker/alpine.dockerfile

@@ -1,91 +0,0 @@
-FROM alpine:3.9 as base
-
-FROM base as builder
-
-ENV LANG=C.UTF-8
-
-# Normally:
-# echo '@testing http://nl.alpinelinux.org/alpine/edge/testing' >> /etc/apk/repositories
-
-RUN \
-  echo -e '@testing http://nl.alpinelinux.org/alpine/edge/testing\n@community http://nl.alpinelinux.org/alpine/edge/community'\
-  >> /etc/apk/repositories \
-  # Add runtime dependencies
-  && apk add --update \
-  python3-dev \
-  py3-setuptools \
-  jbig2enc@community \
-  ghostscript \
-  qpdf@community \
-  qpdf-dev@community \
-  tesseract-ocr \
-  unpaper \
-  pngquant \
-  libxml2-dev \
-  libxslt-dev \
-  zlib-dev \
-  libffi-dev \
-  leptonica-dev \
-  binutils \
-  && pip3 install --upgrade pip \
-  # Install pybind11 for pikepdf
-  && pip3 install pybind11 \
-  # Install flask for the webservice
-  && pip3 install flask \
-  # Add build dependencies
-  && apk add --virtual build-dependencies \
-  build-base \
-  git
-
-COPY . /app
-
-WORKDIR /app
-
-RUN pip3 install .
-
-FROM base
-
-ENV LANG=C.UTF-8
-
-# Normally:
-# echo '@testing http://nl.alpinelinux.org/alpine/edge/testing' >> /etc/apk/repositories
-
-RUN \
-  echo -e '@testing http://nl.alpinelinux.org/alpine/edge/testing\n@community http://nl.alpinelinux.org/alpine/edge/community'\
-  >> /etc/apk/repositories \
-  # Add runtime dependencies
-  && apk add --update \
-  python3 \
-  jbig2enc@community \
-  ghostscript \
-  qpdf@community \
-  qpdf-dev@community \
-  tesseract-ocr \
-  tesseract-ocr-data-deu \
-  tesseract-ocr-data-chi_sim \
-  unpaper \
-  pngquant \
-  libxml2 \
-  libxslt \
-  zlib \
-  libffi \
-  leptonica-dev \
-  binutils \
-  && mkdir /app
-
-WORKDIR /app
-
-# Copy build artifacts (python site-packages)
-COPY --from=builder /usr/lib/python3.6/site-packages /usr/lib/python3.6/site-packages
-COPY --from=builder /usr/bin/ocrmypdf /usr/bin/dumppdf.py /usr/bin/latin2ascii.py /usr/bin/pdf2txt.py /usr/bin/img2pdf /usr/bin/chardetect /usr/bin/
-
-# Copy
-COPY --from=builder /app/misc/webservice.py /app/
-
-# Copy minimal project files to get the test suite.
-COPY --from=builder /app/setup.cfg /app/setup.py /app/README.md /app/
-COPY --from=builder /app/requirements /app/requirements
-COPY --from=builder /app/tests /app/tests
-COPY --from=builder /app/src /app/src
-
-ENTRYPOINT ["/usr/bin/ocrmypdf"]

.travis.yml

@@ -62,8 +62,6 @@ matrix:
         mkdir -p bin packages
         pip3 install --upgrade pip
         pip3 install --upgrade wheel
-        wget -q 'https://www.dropbox.com/s/vaq0kbwi6e6au80/unpaper_6.1-1.deb?raw=1' -O packages/unpaper_6.1-1.deb
-        sudo dpkg -i packages/unpaper_6.1-1.deb
     - os: linux
       dist: xenial
       sudo: required
@@ -88,30 +86,30 @@ matrix:
             - tesseract-ocr-eng
             - tesseract-ocr-fra
             - unpaper
-    # - os: linux
-    #   dist: xenial
-    #   sudo: required
-    #   language: python
-    #   python: "3.8"
-    #   env:
-    #     - DIST=xenial
-    #   addons:
-    #     apt:
-    #       update: true
-    #       sources:
-    #         - sourceline: "ppa:alex-p/tesseract-ocr"
-    #       packages:
-    #         - ghostscript
-    #         - libexempi3
-    #         - libffi-dev
-    #         - pngquant
-    #         - poppler-utils
-    #         - qpdf
-    #         - tesseract-ocr
-    #         - tesseract-ocr-deu
-    #         - tesseract-ocr-eng
-    #         - tesseract-ocr-fra
-    #         - unpaper
+    - os: linux
+      dist: bionic
+      sudo: required
+      language: python
+      python: "3.8"
+      env:
+        - DIST=bionic
+      addons:
+        apt:
+          update: true
+          sources:
+            - sourceline: "ppa:alex-p/tesseract-ocr"
+          packages:
+            - ghostscript
+            - libexempi3
+            - libffi-dev
+            - pngquant
+            - poppler-utils
+            - qpdf
+            - tesseract-ocr
+            - tesseract-ocr-deu
+            - tesseract-ocr-eng
+            - tesseract-ocr-fra
+            - unpaper
     - os: osx
       language: generic
       addons:
@@ -138,10 +136,7 @@ before_cache:
 install:
   - mkdir -p bin
   - export PATH=$PWD/bin:$PATH
-  - pip3 install pycparser # py3.7 workaround for https://github.com/eliben/pycparser/issues/251
-  - pip3 install -r requirements/main.txt
-  - pip3 install --no-deps .
-  - pip3 install -r requirements/test.txt
+  - pip3 install -r requirements/main.txt -r requirements/test.txt .
 
 script:
   - tesseract --version

MANIFEST.in

@@ -1,43 +0,0 @@
-# requirements
-recursive-include requirements *
-
-# git
-include .git_archival.txt
-
-# docker
-include .dockerignore
-recursive-include .docker *
-
-# tests
-include .coveragerc
-recursive-include tests *.bin
-recursive-include tests *.jpg
-recursive-include tests *.jsonl
-recursive-include tests *.png
-recursive-include tests *.pdf
-recursive-include tests *.py
-recursive-include tests *.rst
-recursive-include tests *.txt
-recursive-exclude tests/resources/private *
-
-# documentation
-include LICENSE
-include *.rst
-recursive-exclude .github *
-recursive-include docs *.py
-recursive-include docs *.rst
-recursive-include docs *.svg
-recursive-exclude docs/_build *
-
-
-# support files
-recursive-include src/ocrmypdf/data *
-include *.py
-exclude tasks.py
-recursive-exclude .travis *
-exclude .travis*
-
-
-# code
-exclude src/ocrmypdf/lib/_leptonica.py
-exclude scratch.py

docs/api.rst

@@ -71,7 +71,8 @@ Progress monitoring
 OCRmyPDF uses the ``tqdm`` package to implement its progress bars.
 :func:`ocrmypdf.configure_logging` will set up logging output to
 ``sys.stderr`` in a way that is compatible with the display of the
-progress bar.
+progress bar. Use ``ocrmypdf.ocr(...progress_bar=False)`` to disable
+the progress bar.
 
 Exceptions
 ----------

docs/batch.rst

@@ -47,7 +47,7 @@ where the PDFs are stored):
 
 .. code-block:: bash
 
-   find . -printf '%p' -name '*.pdf' -exec docker run --rm -v <host dir>:<container dir> jbarlow83/ocrmypdf-alpine '<container dir>/{}' '<container dir>/{}' \;
+   find . -printf '%p' -name '*.pdf' -exec docker run --rm -v <host dir>:<container dir> jbarlow83/ocrmypdf '<container dir>/{}' '<container dir>/{}' \;
 
 This only runs one ``ocrmypdf`` process at a time. This variation uses
 ``find`` to create a directory list and ``parallel`` to parallelize runs
@@ -244,6 +244,9 @@ Caveats
 Alternatives
 ------------
 
+-  `systemd user services <https://wiki.archlinux.org/index.php/Systemd/User>`__
+   can be configured to automatically perform OCR on a collection of files.
+
 -  `Watchman <https://facebook.github.io/watchman/>`__ is a more
    powerful alternative to ``watchmedo``.
 

docs/cookbook.rst

@@ -216,6 +216,15 @@ processing or PDF/A conversion.
 
     ocrmypdf --tesseract-timeout=0 --remove-background input.pdf output.pdf
 
+Optimize images without performing OCR
+--------------------------------------
+
+You can also optimize all images without performing any OCR:
+
+.. code-block:: bash
+
+    ocrmypdf --tesseract-timeout=0 --optimize 3 --skip-text input.pdf output.pdf
+
 Redo existing OCR
 =================
 

docs/docker.rst

@@ -22,21 +22,20 @@ Installing the Docker image
 If you have `Docker <https://docs.docker.com/>`__ installed on your
 system, you can install a Docker image of the latest release.
 
-The recommended OCRmyPDF Docker image is currently named
-``ocrmypdf-alpine``:
-
-.. code-block:: bash
-
-   docker pull jbarlow83/ocrmypdf-alpine
-
-Follow the Docker installation instructions for your platform. If you
-can run this command successfully, your system is ready to download and
+If you can run this command successfully, your system is ready to download and
 execute the image:
 
 .. code-block:: bash
 
    docker run hello-world
 
+The recommended OCRmyPDF Docker image is currently named ``ocrmypdf``:
+
+.. code-block:: bash
+
+   docker pull jbarlow83/ocrmypdf
+
+
 OCRmyPDF will use all available CPU cores. By default, the VirtualBox
 machine instance on Windows and macOS has only a single CPU core
 enabled. Use the VirtualBox Manager to determine the name of your Docker
@@ -51,6 +50,9 @@ CPUs:
    docker-machine start "yourVM"
    eval $(docker-machine env "yourVM")
 
+See the Docker documentation for
+`adjusting memory and CPU on other platforms <https://docs.docker.com/config/containers/resource_constraints/>`__.
+
 Using the Docker image on the command line
 ==========================================
 
@@ -63,7 +65,7 @@ To start a Docker container (instance of the image):
 
 .. code-block:: bash
 
-   docker tag jbarlow83/ocrmypdf-alpine ocrmypdf
+   docker tag jbarlow83/ocrmypdf ocrmypdf
    docker run --rm -i ocrmypdf (... all other arguments here...)
 
 For convenience, create a shell alias to hide the Docker command. It is
@@ -103,7 +105,7 @@ on the public one:
 
 .. code-block:: dockerfile
 
-   FROM jbarlow83/ocrmypdf-alpine
+   FROM jbarlow83/ocrmypdf
 
    # Add French
    RUN apk add tesseract-ocr-data-fra
@@ -117,17 +119,16 @@ The OCRmyPDF test suite is installed with image. To run it:
 
 .. code-block:: bash
 
-   docker run --entrypoint python3 jbarlow83/ocrmypdf-alpine setup.py test
+   docker run --entrypoint python3  jbarlow83/ocrmypdf -m pytest
 
 Accessing the shell
 ===================
 
-``bash`` is not installed in the image. To use the busybox shell in the
-Docker image:
+To use the bash shell in the Docker image:
 
 .. code-block:: bash
 
-   docker run -it --entrypoint busybox  jbarlow83/ocrmypdf-alpine sh
+   docker run -it --entrypoint bash  jbarlow83/ocrmypdf
 
 Using the OCRmyPDF web service wrapper
 ======================================
@@ -137,7 +138,12 @@ service. The webservice may be launched as follows:
 
 .. code-block:: bash
 
-   docker run --entrypoint python3 -p 5000:5000 jbarlow83/ocrmypdf-alpine webservice.py
+   docker run --entrypoint python3 -p 5000:5000  jbarlow83/ocrmypdf webservice.py
+
+This will configure the machine to listen on port 5000. On Linux machines
+this is port 5000 of localhost. On macOS or Windows machines running
+Docker, this is port 5000 of the virtual machine that runs your Docker
+images. You can find its IP address using the command ``docker-machine ip``.
 
 Unlike command line usage this program will open a socket and wait for
 connections.
@@ -162,14 +168,3 @@ also licensed in this way.
 
 In addition to the above, please read our
 :ref:`general remarks on using OCRmyPDF as a service <ocr-service>`.
-
-Ubuntu-based Docker image
-=========================
-
-A Ubuntu-based OCRmyPDF image is also available. The main advantage this
-image offers is that it supports manylinux Python wheels (which are not
-supported on Alpine Linux). This may be useful for plugins.
-
-.. code-block:: bash
-
-   docker pull jbarlow83/ocrmypdf

docs/index.rst

@@ -12,6 +12,7 @@ PDF is the best format for storing and exchanging scanned documents.  Unfortunat
    introduction
    release_notes
    installation
+   optimizer
    languages
    jbig2
 

docs/installation.rst

@@ -152,7 +152,8 @@ user's ``PATH`` to check for the user's Python packages.
 .. code-block:: bash
 
     export PATH=$HOME/.local/bin:$PATH
-    pip3 install --user ocrmypdf
+    python3 -m pip install --user --upgrade pip
+    python3 -m pip install --user ocrmypdf
 
 To add JBIG2 encoding, see :ref:`jbig2`.
 
@@ -272,15 +273,11 @@ Now we need to install ``pip`` and let it install ocrmypdf:
     curl https://bootstrap.pypa.io/ez_setup.py -o - | python3.6 && python3.6 -m easy_install pip
     pip3.6 install ocrmypdf
 
-These installation instructions omit the optional dependency
-``unpaper``, which is only available at version 0.4.2 in Ubuntu 14.04.
-The author could not find a backport of ``unpaper``, and created a .deb
-package to do the job of installing unpaper 6.1 (for x86 64-bit only):
-
-.. code-block:: bash
-
-    wget -q 'https://www.dropbox.com/s/vaq0kbwi6e6au80/unpaper_6.1-1.deb?raw=1' -O unpaper_6.1-1.deb
-    sudo dpkg -i unpaper_6.1-1.deb
+The optional dependency ``unpaper`` is only available at 0.4.2 in Ubuntu 14.04,
+and no backports are available. Previously the author maintained a backported
+.deb package for unpaper 6.1, but since Ubuntu 14.04 is now end of life, this is
+not supported. As such, ``unpaper`` is not available on Ubuntu 14.04 or must by
+compiled by hand.
 
 To add JBIG2 encoding, see :ref:`jbig2`.
 

docs/introduction.rst

@@ -208,7 +208,7 @@ consider one of these similar open source programs:
 Web front-ends
 ==============
 
-The Docker image ``ocrmypdf-alpine`` provides a web service front-end
+The Docker image ``ocrmypdf`` provides a web service front-end
 that allows files to submitted over HTTP and the results "downloaded".
 This is an HTTP server intended to simplify web services deployments; it
 is not intended to be deployed on the public internet and no real

docs/optimizer.rst

@@ -0,0 +1,71 @@
+================
+PDF optimization
+================
+
+OCRmyPDF includes an image-oriented PDF optimizer. By default, the optimizer
+runs with safe settings with the goal of improving compression at no loss of
+quality. At higher optimization levels, lossy optimizations may be applied and
+tuned. Optimization occurs after OCR, and only if OCR succeeded.  It does not
+perform other possible optimizations such as deduplicating resources,
+consolidating fonts, simplifying vector drawings, or anything of that nature.
+
+Optimization ranges from ``-O0`` through ``-O3``, where ``0`` disables
+optimization and ``3`` implements all options. ``1``, the default, performs only
+safe and lossless optimizations. (This is similar to GCC's optimization
+parameter.) The exact type of optimizations performed will vary over time.
+
+Optimizations that always occurs
+================================
+
+OCRmyPDF will automatically replace obsolete or inferior compression schemes
+such as RLE or LZW with superior schemes such as Deflate and converting
+monochrome images to CCITT G4. Since this is harmless it always occurs and there
+is no way to disable it. Other non-image compressed objects are compressed as
+well.
+
+Fast web view
+=============
+
+OCRmyPDF automatically optimizes PDFs for "fast web view" in Adobe Acrobat's
+parlance, or equivalently, linearizes PDFs so that the resources they reference
+are presented in the order a viewer needs them for sequential display. This
+reduces the latency of viewing a PDF both online and from local storage. This
+actually slightly increases the file size.
+
+To disable this optimization and all others, use ``ocrmypdf --optimize 0 ...``
+or the shorthand ``-O0``.
+
+Lossless optimizations
+======================
+
+At optimization level ``-O1`` (the default), OCRmyPDF will also attempt lossless
+image optimization.
+
+If a JBIG2 encoder is available, then monochrome images will be converted to
+JBIG2, with the potential for huge savings on large black and white images,
+since JBIG2 is far more efficient than any other monochrome (bi-level)
+compression. (All known US patents related to JBIG2 have probably expired, but
+it remains the responsibility of the user to supply a JBIG2 encoder such as
+`jbig2enc <https://github.com/agl/jbig2enc>`__. OCRmyPDF does not implement
+JBIG2 encoding on its own.)
+
+OCRmyPDF currently does not attempt to recompress losslessly compressed objects
+more aggressively.
+
+Lossy optimizations
+===================
+
+At optimization level ``-O2`` and ``-O3``, OCRmyPDF will some attempt lossy
+image optimization.
+
+If ``pngquant`` is installed, OCRmyPDF will use it to perform quantize paletted
+images to reduce their size.
+
+The quality of JPEGs may be lowered, on the assumption that a lower quality
+image may be suitable for storage after OCR.
+
+It is not possible to optimize all image types. Uncommon image types may be
+skipped by the optimizer.
+
+OCRmyPDF provides :ref:`lossy mode JBIG2 <jbig2-lossy>` as an advanced feature
+that additional requires the argument ``--jbig2-lossy``.

docs/release_notes.rst

@@ -13,6 +13,24 @@ Note that it is licensed under GPLv3, so scripts that
 ``import ocrmypdf`` and are released publicly should probably also be
 licensed under GPLv3.
 
+v9.1.0
+======
+
+-  Improved diagnostics when file size increases at output. Now warns if JBIG2
+   or pngquant were not available.
+-  pikepdf 1.7.0 is now required, to pick up changes that remove the need for
+   a source install on Linux systems running Python 3.8.
+
+v9.0.5
+======
+
+-  The Alpine Docker image (jbarlow83/ocrmypdf-alpine) has been dropped due to
+   the difficulties of supporting Alpine Linux.
+-  The primary Docker image (jbarlow83/ocrmypdf) has been improved to take on
+   the extra features that used to be exclusive to the Alpine image.
+-  No changes to application code.
+-  pdfminer.six version 20191020 is now supported.
+
 v9.0.4
 ======
 

requirements/main.txt

@@ -1,13 +1,10 @@
 # requirements.txt can be used to replicate the developer's build environment
 # setup.py lists a separate set of requirements that are looser to simplify
 # installation
-chardet == 3.0.4
-cffi == 1.12.2
+cffi == 1.13.2
 img2pdf == 0.3.3
-pdfminer.six == 20181108
-pikepdf == 1.6.5
+pdfminer.six == 20191020
+pikepdf == 1.7.0
 Pillow >= 6.2.0
-pycparser == 2.19
-python-xmp-toolkit == 2.0.1
-reportlab == 3.5.13
-tqdm == 4.32.1
+reportlab == 3.5.32
+tqdm == 4.37.0

requirements/test.txt

@@ -2,7 +2,7 @@ pytest >= 5.0.0
 pytest-helpers-namespace >= 2019.1.8
 pytest-xdist >= 1.29.0  # For DumpError fix
 pytest-cov >= 2.6.1
-python-xmp-toolkit   # requires apt-get install libexempi3
+python-xmp-toolkit == 2.0.1  # requires apt-get install libexempi3
                      # or brew install exempi
 PyPDF2 >= 1.26.0
 #PyMuPDF == 1.13.4    # optional

requirements/webservice.txt

@@ -0,0 +1 @@
+Flask >= 1, < 2

setup.py

@@ -96,8 +96,8 @@ setup(
         'chardet >= 3.0.4, < 4',  # unlisted requirement of pdfminer.six 20181108
         'cffi >= 1.9.1',  # must be a setup and install requirement
         'img2pdf >= 0.3.0, < 0.4',  # pure Python, so track HEAD closely
-        'pdfminer.six == 20181108',
-        'pikepdf >= 1.6.5, < 2',
+        'pdfminer.six >= 20181108, <= 20191020',
+        'pikepdf >= 1.7.0, < 2',
         'Pillow >= 6.2.0',
         'reportlab >= 3.3.0',  # oldest released version with sane image handling
         'tqdm >= 4',

src/ocrmypdf/_validation.py

@@ -418,6 +418,20 @@ def report_output_file_size(options, input_file, output_file):
                 f"The argument --{arg.replace('_', '-')} was issued, causing transcoding."
             )
 
+    if options.optimize == 0:
+        reasons.append("Optimization was disabled.")
+    else:
+        image_optimizers = {
+            'jbig2': jbig2enc.available(),
+            'pngquant': pngquant.available(),
+        }
+        for name, available in image_optimizers.items():
+            if not available:
+                reasons.append(
+                    f"The optional dependency '{name}' was not found, so some image "
+                    f"optimizations could not be attempted."
+                )
+
     if reasons:
         explanation = "Possible reasons for this include:\n" + '\n'.join(reasons) + "\n"
     else:

src/ocrmypdf/exec/ghostscript.py

@@ -15,8 +15,11 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
+"""Interface to Ghostscript executable"""
+
 import logging
 import re
+import warnings
 from functools import lru_cache
 from os import fspath
 from shutil import copy
@@ -193,7 +196,7 @@ def generate_pdfa(
     output_file,
     compression,
     log,
-    threads=1,
+    threads=None,  # deprecated parameter
     pdf_version='1.5',
     pdfa_part='2',
 ):
@@ -216,6 +219,10 @@ def generate_pdfa(
     """
     if not log:
         log = gslog
+    if threads is not None:
+        warnings.warn(
+            "use of deprecated parameter 'threads'", category=DeprecationWarning
+        )
 
     compression_args = []
     if compression == 'jpeg':

src/ocrmypdf/exec/jbig2enc.py

@@ -15,6 +15,8 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
+"""Interface to jbig2 executable"""
+
 from functools import lru_cache
 from subprocess import PIPE, run
 

src/ocrmypdf/exec/pngquant.py

@@ -15,6 +15,8 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
+"""Interface to pngquant executable"""
+
 from functools import lru_cache
 from subprocess import run
 from tempfile import NamedTemporaryFile

src/ocrmypdf/exec/qpdf.py

@@ -15,6 +15,8 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
+"""Interface to qpdf executable"""
+
 from functools import lru_cache
 from os import fspath
 from subprocess import PIPE, STDOUT, CalledProcessError, run

src/ocrmypdf/exec/tesseract.py

@@ -15,10 +15,13 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
+"""Interface to Tesseract executable"""
+
 import os
 import shutil
 from collections import namedtuple
 from contextlib import suppress
+import logging
 from os import fspath
 from subprocess import PIPE, STDOUT, CalledProcessError, TimeoutExpired, run
 
@@ -50,6 +53,11 @@ HOCR_TEMPLATE = """<?xml version="1.0" encoding="UTF-8"?>
 """
 
 
+class TesseractLoggerAdapter(logging.LoggerAdapter):
+    def process(self, msg, kwargs):
+        return '[tesseract] %s' % (msg), kwargs
+
+
 def version(tesseract_env=None):
     return get_version('tesseract', regex=r'tesseract\s(.+)', env=tesseract_env)
 
@@ -177,15 +185,16 @@ def get_orientation(input_file, engine_mode, timeout: float, log, tesseract_env=
         return oc
 
 
-def tesseract_log_output(log, stdout, input_file):
-    prefix = "[tesseract] "
+def tesseract_log_output(mainlog, stdout, input_file):
+    log = TesseractLoggerAdapter(
+        mainlog, extra=mainlog.extra if hasattr(mainlog, 'extra') else None
+    )
 
     try:
         text = stdout.decode()
     except UnicodeDecodeError:
         log.error(
-            prefix
-            + "command line output was not utf-8. "
+            "command line output was not utf-8. "
             + "This usually means Tesseract's language packs do not match "
             "the installed version of Tesseract."
         )
@@ -198,25 +207,25 @@ def tesseract_log_output(log, stdout, input_file):
         elif line.startswith("Warning in pixReadMem"):
             continue
         elif 'diacritics' in line:
-            log.warning(prefix + "lots of diacritics - possibly poor OCR")
+            log.warning("lots of diacritics - possibly poor OCR")
         elif line.startswith('OSD: Weak margin'):
-            log.warning(prefix + "unsure about page orientation")
+            log.warning("unsure about page orientation")
         elif 'Error in pixScanForForeground' in line:
             pass  # Appears to be spurious/problem with nonwhite borders
         elif 'Error in boxClipToRectangle' in line:
             pass  # Always appears with pixScanForForeground message
         elif 'parameter not found: ' in line.lower():
-            log.error(prefix + line.strip())
+            log.error(line.strip())
             problem = line.split('found: ')[1]
             raise TesseractConfigError(problem)
         elif 'error' in line.lower() or 'exception' in line.lower():
-            log.error(prefix + line.strip())
+            log.error(line.strip())
         elif 'warning' in line.lower():
-            log.warning(prefix + line.strip())
+            log.warning(line.strip())
         elif 'read_params_file' in line.lower():
-            log.error(prefix + line.strip())
+            log.error(line.strip())
         else:
-            log.info(prefix + line.strip())
+            log.info(line.strip())
 
 
 def page_timedout(log, input_file, timeout):

src/ocrmypdf/exec/unpaper.py

@@ -18,10 +18,11 @@
 # unpaper documentation:
 # https://github.com/Flameeyes/unpaper/blob/master/doc/basic-concepts.md
 
+"""Interface to unpaper executable"""
+
 import os
 import shlex
 import subprocess
-import sys
 from functools import lru_cache
 from subprocess import PIPE, STDOUT, CalledProcessError
 from tempfile import TemporaryDirectory

src/ocrmypdf/pdfa.py

@@ -32,8 +32,6 @@ Ghostscript's handling of pdfmark.
 """
 
 import base64
-import os
-from binascii import hexlify
 from pathlib import Path
 from string import Template
 

src/ocrmypdf/pdfinfo/layout.py

@@ -20,6 +20,7 @@ from math import copysign
 from pathlib import Path
 from unittest.mock import patch
 
+import pdfminer
 import pdfminer.encodingdb
 import pdfminer.pdfdevice
 import pdfminer.pdfinterp
@@ -36,51 +37,54 @@ from ..exceptions import EncryptedPdfError
 STRIP_NAME = re.compile(r'[0-9]+')
 
 #
-# Unconditional pdfminer patches
+#  pdfminer 20181108 patches
 #
 
+if pdfminer.__version__ == '20181108':
 
-def name2unicode(name):
-    """Fix pdfminer's name2unicode function
+    def name2unicode(name):
+        """Fix pdfminer's name2unicode function
 
-    Font cids that are mapped to names of the form /g123 seem to be, by convention
-    characters with no corresponding Unicode entry. These can be subsetted fonts
-    or symbolic fonts. There seems to be no way to map /g123 fonts to Unicode,
-    barring a ToUnicode data structure.
-    """
-    if name in glyphname2unicode:
-        return glyphname2unicode[name]
-    if name.startswith('g') or name.startswith('a'):
-        raise KeyError(name)
-    if name.startswith('uni'):
-        try:
-            return chr(int(name[3:], 16))
-        except ValueError:  # Not hexadecimal
+        Font cids that are mapped to names of the form /g123 seem to be, by convention
+        characters with no corresponding Unicode entry. These can be subsetted fonts
+        or symbolic fonts. There seems to be no way to map /g123 fonts to Unicode,
+        barring a ToUnicode data structure.
+        """
+        if name in glyphname2unicode:
+            return glyphname2unicode[name]
+        if name.startswith('g') or name.startswith('a'):
             raise KeyError(name)
-    m = STRIP_NAME.search(name)
-    if not m:
-        raise KeyError(name)
-    return chr(int(m.group(0)))
+        if name.startswith('uni'):
+            try:
+                return chr(int(name[3:], 16))
+            except ValueError:  # Not hexadecimal
+                raise KeyError(name)
+        m = STRIP_NAME.search(name)
+        if not m:
+            raise KeyError(name)
+        return chr(int(m.group(0)))
 
+    pdfminer.encodingdb.name2unicode = name2unicode
 
-pdfminer.encodingdb.name2unicode = name2unicode
+    original_PDFFont_init = PDFFont.__init__
 
-original_PDFFont_init = PDFFont.__init__
+    def PDFFont__init__(self, descriptor, widths, default_width=None):
+        original_PDFFont_init(self, descriptor, widths, default_width)
+        # PDF spec says descent should be negative
+        # A font with a positive descent implies it floats entirely above the
+        # baseline, i.e. it's not really a baseline anymore. I have fonts that
+        # claim a positive descent, but treating descent as positive always seems
+        # to misposition text.
+        if self.descent > 0:
+            self.descent = -self.descent
 
+    PDFFont.__init__ = PDFFont__init__
 
-def PDFFont__init__(self, descriptor, widths, default_width=None):
-    original_PDFFont_init(self, descriptor, widths, default_width)
-    # PDF spec says descent should be negative
-    # A font with a positive descent implies it floats entirely above the
-    # baseline, i.e. it's not really a baseline anymore. I have fonts that
-    # claim a positive descent, but treating descent as positive always seems
-    # to misposition text.
-    if self.descent > 0:
-        self.descent = -self.descent
+#
+# end of pdfminer 20181108 patches
+#
 
 
-PDFFont.__init__ = PDFFont__init__
-
 original_PDFSimpleFont_init = PDFSimpleFont.__init__
 
 
@@ -97,6 +101,7 @@ def PDFSimpleFont__init__(self, descriptor, widths, spec):
 
 
 PDFSimpleFont.__init__ = PDFSimpleFont__init__
+
 #
 # pdfminer patches when creator is PScript5.dll
 #
@@ -207,6 +212,7 @@ class TextPositionTracker(PDFLayoutAnalyzer):
         super().__init__(rsrcmgr, pageno, laparams)
         self.textstate = None
         self.result = None
+        self.cur_item = None  # not defined in pdfminer code as it should be
 
     def begin_page(self, page, ctm):
         super().begin_page(page, ctm)

tests/test_completion.py

@@ -19,6 +19,11 @@ from subprocess import run, PIPE
 
 import pytest
 
+pytestmark = pytest.mark.skipif(
+    pytest.helpers.running_in_docker(),  # pylint: disable=no-member
+    reason="docker can't complete",
+)
+
 
 def test_fish():
     try:

tests/test_validation.py

@@ -15,7 +15,6 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
-import locale
 import logging
 import os
 from unittest.mock import patch
@@ -117,11 +116,28 @@ def test_report_file_size(tmp_path, caplog):
     opts = make_opts()
     vd.report_output_file_size(opts, in_, out)
     assert caplog.text == ''
+    caplog.clear()
 
     os.truncate(in_, 25001)
     os.truncate(out, 50000)
+    with patch('ocrmypdf._validation.jbig2enc.available', return_value=True), patch(
+        'ocrmypdf._validation.pngquant.available', return_value=True
+    ):
+        vd.report_output_file_size(opts, in_, out)
+        assert 'No reason' in caplog.text
+    caplog.clear()
+
+    with patch('ocrmypdf._validation.jbig2enc.available', return_value=False), patch(
+        'ocrmypdf._validation.pngquant.available', return_value=True
+    ):
+        vd.report_output_file_size(opts, in_, out)
+        assert 'optional dependency' in caplog.text
+    caplog.clear()
+
+    opts = make_opts(in_, out, optimize=0)
     vd.report_output_file_size(opts, in_, out)
-    assert 'No reason' in caplog.text
+    assert 'disabled' in caplog.text
+    caplog.clear()
 
 
 def test_false_action_store_true():