Support pdfminer.six 20191020

[ci skip]

.docker/Dockerfile

@@ -1,6 +1,6 @@
 # OCRmyPDF
 #
-FROM ubuntu:19.04 as base
+FROM ubuntu:19.10 as base
 
 FROM base as builder
 
@@ -10,16 +10,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   build-essential autoconf automake libtool \
   libleptonica-dev \
   zlib1g-dev \
-  ocrmypdf \
-  pngquant \
+  python3-setuptools \
   python3-pip \
-  python3-venv \
-  tesseract-ocr \
-  unpaper \
   wget \
   git
 
-
 # Compile and install jbig2
 # Needs libleptonica-dev, zlib1g-dev
 RUN \
@@ -31,15 +26,15 @@ RUN \
   && cd .. \
   && rm -rf jbig2
 
-RUN python3 -m venv /appenv
-
 COPY . /app
 
 WORKDIR /app
 
-RUN . /appenv/bin/activate; \
-  pip install --upgrade pip \
-  && pip install .
+RUN pip3 install --no-cache-dir \
+  -r requirements/main.txt \
+  -r requirements/webservice.txt \
+  -r requirements/test.txt \
+  .
 
 FROM base
 
@@ -53,7 +48,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   zlib1g \
   pngquant \
   python3 \
-  python3-venv \
   qpdf \
   tesseract-ocr \
   tesseract-ocr-chi-sim \
@@ -62,10 +56,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   tesseract-ocr-fra \
   tesseract-ocr-por \
   tesseract-ocr-spa \
-  unpaper \
-  wget
+  unpaper
+
+WORKDIR /app
+
+COPY --from=builder /usr/local/lib/ /usr/local/lib/
+COPY --from=builder /usr/local/bin/ /usr/local/bin/
 
-# Copy
 COPY --from=builder /app/misc/webservice.py /app/
 
 # Copy minimal project files to get the test suite.
@@ -74,7 +71,4 @@ COPY --from=builder /app/requirements /app/requirements
 COPY --from=builder /app/tests /app/tests
 COPY --from=builder /app/src /app/src
 
-COPY --from=builder /appenv /appenv
-COPY --from=builder /usr/local /usr/local
-
-ENTRYPOINT ["/appenv/bin/ocrmypdf"]
+ENTRYPOINT ["/usr/local/bin/ocrmypdf"]

.docker/alpine.dockerfile

@@ -1,91 +0,0 @@
-FROM alpine:3.9 as base
-
-FROM base as builder
-
-ENV LANG=C.UTF-8
-
-# Normally:
-# echo '@testing http://nl.alpinelinux.org/alpine/edge/testing' >> /etc/apk/repositories
-
-RUN \
-  echo -e '@testing http://nl.alpinelinux.org/alpine/edge/testing\n@community http://nl.alpinelinux.org/alpine/edge/community'\
-  >> /etc/apk/repositories \
-  # Add runtime dependencies
-  && apk add --update \
-  python3-dev \
-  py3-setuptools \
-  jbig2enc@community \
-  ghostscript \
-  qpdf@community \
-  qpdf-dev@community \
-  tesseract-ocr \
-  unpaper \
-  pngquant \
-  libxml2-dev \
-  libxslt-dev \
-  zlib-dev \
-  libffi-dev \
-  leptonica-dev \
-  binutils \
-  && pip3 install --upgrade pip \
-  # Install pybind11 for pikepdf
-  && pip3 install pybind11 \
-  # Install flask for the webservice
-  && pip3 install flask \
-  # Add build dependencies
-  && apk add --virtual build-dependencies \
-  build-base \
-  git
-
-COPY . /app
-
-WORKDIR /app
-
-RUN pip3 install .
-
-FROM base
-
-ENV LANG=C.UTF-8
-
-# Normally:
-# echo '@testing http://nl.alpinelinux.org/alpine/edge/testing' >> /etc/apk/repositories
-
-RUN \
-  echo -e '@testing http://nl.alpinelinux.org/alpine/edge/testing\n@community http://nl.alpinelinux.org/alpine/edge/community'\
-  >> /etc/apk/repositories \
-  # Add runtime dependencies
-  && apk add --update \
-  python3 \
-  jbig2enc@community \
-  ghostscript \
-  qpdf@community \
-  qpdf-dev@community \
-  tesseract-ocr \
-  tesseract-ocr-data-deu \
-  tesseract-ocr-data-chi_sim \
-  unpaper \
-  pngquant \
-  libxml2 \
-  libxslt \
-  zlib \
-  libffi \
-  leptonica-dev \
-  binutils \
-  && mkdir /app
-
-WORKDIR /app
-
-# Copy build artifacts (python site-packages)
-COPY --from=builder /usr/lib/python3.6/site-packages /usr/lib/python3.6/site-packages
-COPY --from=builder /usr/bin/ocrmypdf /usr/bin/dumppdf.py /usr/bin/latin2ascii.py /usr/bin/pdf2txt.py /usr/bin/img2pdf /usr/bin/chardetect /usr/bin/
-
-# Copy
-COPY --from=builder /app/misc/webservice.py /app/
-
-# Copy minimal project files to get the test suite.
-COPY --from=builder /app/setup.cfg /app/setup.py /app/README.md /app/
-COPY --from=builder /app/requirements /app/requirements
-COPY --from=builder /app/tests /app/tests
-COPY --from=builder /app/src /app/src
-
-ENTRYPOINT ["/usr/bin/ocrmypdf"]

.travis.yml

@@ -62,8 +62,6 @@ matrix:
         mkdir -p bin packages
         pip3 install --upgrade pip
         pip3 install --upgrade wheel
-        wget -q 'https://www.dropbox.com/s/vaq0kbwi6e6au80/unpaper_6.1-1.deb?raw=1' -O packages/unpaper_6.1-1.deb
-        sudo dpkg -i packages/unpaper_6.1-1.deb
     - os: linux
       dist: xenial
       sudo: required
@@ -88,8 +86,31 @@ matrix:
             - tesseract-ocr-eng
             - tesseract-ocr-fra
             - unpaper
+    # - os: linux
+    #   dist: bionic
+    #   sudo: required
+    #   language: python
+    #   python: "3.8"
+    #   env:
+    #     - DIST=bionic
+    #   addons:
+    #     apt:
+    #       update: true
+    #       sources:
+    #         - sourceline: "ppa:alex-p/tesseract-ocr"
+    #       packages:
+    #         - ghostscript
+    #         - libexempi3
+    #         - libffi-dev
+    #         - pngquant
+    #         - poppler-utils
+    #         - qpdf
+    #         - tesseract-ocr
+    #         - tesseract-ocr-deu
+    #         - tesseract-ocr-eng
+    #         - tesseract-ocr-fra
+    #         - unpaper
     - os: osx
-      osx_image: xcode9.2
       language: generic
       addons:
         homebrew:
@@ -115,10 +136,7 @@ before_cache:
 install:
   - mkdir -p bin
   - export PATH=$PWD/bin:$PATH
-  - pip3 install pycparser # py3.7 workaround for https://github.com/eliben/pycparser/issues/251
-  - pip3 install -r requirements/main.txt
-  - pip3 install --no-deps .
-  - pip3 install -r requirements/test.txt
+  - pip3 install -r requirements/main.txt -r requirements/test.txt .
 
 script:
   - tesseract --version

MANIFEST.in

@@ -1,43 +0,0 @@
-# requirements
-recursive-include requirements *
-
-# git
-include .git_archival.txt
-
-# docker
-include .dockerignore
-recursive-include .docker *
-
-# tests
-include .coveragerc
-recursive-include tests *.bin
-recursive-include tests *.jpg
-recursive-include tests *.jsonl
-recursive-include tests *.png
-recursive-include tests *.pdf
-recursive-include tests *.py
-recursive-include tests *.rst
-recursive-include tests *.txt
-recursive-exclude tests/resources/private *
-
-# documentation
-include LICENSE
-include *.rst
-recursive-exclude .github *
-recursive-include docs *.py
-recursive-include docs *.rst
-recursive-include docs *.svg
-recursive-exclude docs/_build *
-
-
-# support files
-recursive-include src/ocrmypdf/data *
-include *.py
-exclude tasks.py
-recursive-exclude .travis *
-exclude .travis*
-
-
-# code
-exclude src/ocrmypdf/lib/_leptonica.py
-exclude scratch.py

README.md

@@ -1,6 +1,6 @@
 <img src="docs/images/logo.svg" width="240" alt="OCRmyPDF">
 
-[![Travis build status][travis]](https://travis-ci.org/jbarlow83/OCRmyPDF) [![PyPI version][pypi]](https://pypi.org/project/ocrmypdf/) ![Homebrew version][homebrew] ![ReadTheDocs][docs]
+[![Travis build status][travis]](https://travis-ci.org/jbarlow83/OCRmyPDF) [![PyPI version][pypi]](https://pypi.org/project/ocrmypdf/) ![Homebrew version][homebrew] ![ReadTheDocs][docs] ![Python versions][pyversions]
 
 [travis]: https://travis-ci.org/jbarlow83/OCRmyPDF.svg?branch=master "Travis build status"
 
@@ -10,6 +10,8 @@
 
 [docs]: https://readthedocs.org/projects/ocrmypdf/badge/?version=latest "RTD"
 
+[pyversions]: https://img.shields.io/pypi/pyversions/ocrmypdf "Supported Python versions"
+
 OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched or copy-pasted.
 
 ```bash
@@ -120,7 +122,7 @@ If you detect an issue, please:
 Requirements
 ------------
 
-Runs on CPython 3.5, 3.6 and 3.7. Requires external program installations of Ghostscript, Tesseract OCR, QPDF, and Leptonica. ocrmypdf is pure Python, but uses CFFI to portably generate library bindings.
+In addition to the required Python version (3.6+), OCRmyPDF requires external program installations of Ghostscript, Tesseract OCR, QPDF, and Leptonica. ocrmypdf is pure Python, but uses CFFI to portably generate library bindings.
 
 Press & Media
 -------------

docs/advanced.rst

@@ -322,15 +322,23 @@ working files on a per page basis have the page number as a prefix
 (starting with page 1), an infix indicates the processing stage, and a
 suffix indicates the file type. Some important files include:
 
--  ``.page.png`` - what the input page looks like
--  ``.image`` - the image we will show the user if we are in a mode that
-   changes the final appearance; may be in one of several image formats
--  ``.text.pdf`` - the OCR file; this will load as a blank page but
-   should have visible text if checked with a tool like pdftotext or
-   pdfminder.six
--  ``.ocr.png`` - the file that is sent to Tesseract for OCR; depending
+-  ``_rasterize.png`` - what the input page looks like
+-  ``_ocr.png`` - the file that is sent to Tesseract for OCR; depending
    on arguments this may differ from the presentation image
--  ``layers.rendered.pdf`` - the composite PDF, before metadata repair
-   and optimization
+-  ``_pp_deskew.png`` - the image, after deskewing
+-  ``_pp_clean.png`` - the image, after cleaning with unpaper
+-  ``_ocr_tess.pdf`` - the OCR file; appears as a blank page with invisible
+   text embedded
+-  ``_ocr_tess.txt`` - the OCR text (not necessarily all text on the page,
+   if the page is mixed format)
+-  ``fix_docinfo.pdf`` - a temporary file created to fix the PDF DocumentInfo
+   data structure
+-  ``graft_layers.pdf`` - the rendered PDF with OCR layers grafted on
+-  ``pdfa.pdf`` - ``graft_layers.pdf`` after conversion to PDF/A
+-  ``pdfa.ps`` - a PostScript file used by Ghostscript for PDF/A conversion
+-  ``optimize.pdf`` - the PDF generated before optimization
+-  ``optimize.out.pdf`` - the PDF generated by optimization
+-  ``origin`` - the input file
+-  ``origin.pdf`` - the input file or the input image converted to PDF
 -  ``images/*`` - images extracted during the optimization process; here
    the prefix indicates a PDF object ID not a page number

docs/batch.rst

@@ -47,7 +47,7 @@ where the PDFs are stored):
 
 .. code-block:: bash
 
-   find . -printf '%p' -name '*.pdf' -exec docker run --rm -v <host dir>:<container dir> jbarlow83/ocrmypdf-alpine '<container dir>/{}' '<container dir>/{}' \;
+   find . -printf '%p' -name '*.pdf' -exec docker run --rm -v <host dir>:<container dir> jbarlow83/ocrmypdf '<container dir>/{}' '<container dir>/{}' \;
 
 This only runs one ``ocrmypdf`` process at a time. This variation uses
 ``find`` to create a directory list and ``parallel`` to parallelize runs

docs/contributing.rst

@@ -0,0 +1,38 @@
+=======================
+Contributing guidelines
+=======================
+
+Contributions are welcome!
+
+Big changes
+===========
+
+Please open a new issue to discuss or propose a major change. Not only is it fun
+to discuss big ideas, but we might save each other's time too. Perhaps some of the
+work you're contemplating is already half-done in a development branch.
+
+Code style
+==========
+
+We use PEP8, ``black`` for code formatting and ``isort`` for import sorting. The
+settings for programs are in ``pyproject.toml`` and ``setup.cfg``.
+
+Tests
+=====
+
+New features should come with tests that confirm their correctness.
+
+New Python dependencies
+=======================
+
+If you are proposing a change that will require a new Python dependency, we
+prefer dependencies that are already packaged by Debian or Red Hat. This makes
+life much easier for our downstream package maintainers.
+
+Python dependencies must also be GPLv3 compatible.
+
+New non-Python dependencies
+===========================
+
+OCRmyPDF uses several external programs (Tesseract, Ghostscript and others) for
+its functionality. In general we prefer to avoid adding new external programs.

docs/cookbook.rst

@@ -216,6 +216,15 @@ processing or PDF/A conversion.
 
     ocrmypdf --tesseract-timeout=0 --remove-background input.pdf output.pdf
 
+Optimize images without performing OCR
+--------------------------------------
+
+You can also optimize all images without performing any OCR:
+
+.. code-block:: bash
+
+    ocrmypdf --tesseract-timeout=0 --optimize 3 --skip-text input.pdf output.pdf
+
 Redo existing OCR
 =================
 

docs/docker.rst

@@ -22,21 +22,20 @@ Installing the Docker image
 If you have `Docker <https://docs.docker.com/>`__ installed on your
 system, you can install a Docker image of the latest release.
 
-The recommended OCRmyPDF Docker image is currently named
-``ocrmypdf-alpine``:
-
-.. code-block:: bash
-
-   docker pull jbarlow83/ocrmypdf-alpine
-
-Follow the Docker installation instructions for your platform. If you
-can run this command successfully, your system is ready to download and
+If you can run this command successfully, your system is ready to download and
 execute the image:
 
 .. code-block:: bash
 
    docker run hello-world
 
+The recommended OCRmyPDF Docker image is currently named ``ocrmypdf``:
+
+.. code-block:: bash
+
+   docker pull jbarlow83/ocrmypdf
+
+
 OCRmyPDF will use all available CPU cores. By default, the VirtualBox
 machine instance on Windows and macOS has only a single CPU core
 enabled. Use the VirtualBox Manager to determine the name of your Docker
@@ -51,6 +50,9 @@ CPUs:
    docker-machine start "yourVM"
    eval $(docker-machine env "yourVM")
 
+See the Docker documentation for
+`adjusting memory and CPU on other platforms <https://docs.docker.com/config/containers/resource_constraints/>`__.
+
 Using the Docker image on the command line
 ==========================================
 
@@ -63,7 +65,7 @@ To start a Docker container (instance of the image):
 
 .. code-block:: bash
 
-   docker tag jbarlow83/ocrmypdf-alpine ocrmypdf
+   docker tag jbarlow83/ocrmypdf ocrmypdf
    docker run --rm -i ocrmypdf (... all other arguments here...)
 
 For convenience, create a shell alias to hide the Docker command. It is
@@ -103,7 +105,7 @@ on the public one:
 
 .. code-block:: dockerfile
 
-   FROM jbarlow83/ocrmypdf-alpine
+   FROM jbarlow83/ocrmypdf
 
    # Add French
    RUN apk add tesseract-ocr-data-fra
@@ -117,17 +119,16 @@ The OCRmyPDF test suite is installed with image. To run it:
 
 .. code-block:: bash
 
-   docker run --entrypoint python3 jbarlow83/ocrmypdf-alpine setup.py test
+   docker run --entrypoint python3  jbarlow83/ocrmypdf -m pytest
 
 Accessing the shell
 ===================
 
-``bash`` is not installed in the image. To use the busybox shell in the
-Docker image:
+To use the bash shell in the Docker image:
 
 .. code-block:: bash
 
-   docker run -it --entrypoint busybox  jbarlow83/ocrmypdf-alpine sh
+   docker run -it --entrypoint bash  jbarlow83/ocrmypdf
 
 Using the OCRmyPDF web service wrapper
 ======================================
@@ -137,7 +138,12 @@ service. The webservice may be launched as follows:
 
 .. code-block:: bash
 
-   docker run --entrypoint python3 -p 5000:5000 jbarlow83/ocrmypdf-alpine webservice.py
+   docker run --entrypoint python3 -p 5000:5000  jbarlow83/ocrmypdf webservice.py
+
+This will configure the machine to listen on port 5000. On Linux machines
+this is port 5000 of localhost. On macOS or Windows machines running
+Docker, this is port 5000 of the virtual machine that runs your Docker
+images. You can find its IP address using the command ``docker-machine ip``.
 
 Unlike command line usage this program will open a socket and wait for
 connections.
@@ -162,14 +168,3 @@ also licensed in this way.
 
 In addition to the above, please read our
 :ref:`general remarks on using OCRmyPDF as a service <ocr-service>`.
-
-Ubuntu-based Docker image
-=========================
-
-A Ubuntu-based OCRmyPDF image is also available. The main advantage this
-image offers is that it supports manylinux Python wheels (which are not
-supported on Alpine Linux). This may be useful for plugins.
-
-.. code-block:: bash
-
-   docker pull jbarlow83/ocrmypdf

docs/index.rst

@@ -22,11 +22,16 @@ PDF is the best format for storing and exchanging scanned documents.  Unfortunat
    cookbook
    docker
    advanced
-   api
    batch
    security
    errors
 
+.. toctree::
+   :caption: Developers
+   :maxdepth: 2
+
+   api
+   contributing
 
 Indices and tables
 ==================

docs/installation.rst

@@ -272,15 +272,11 @@ Now we need to install ``pip`` and let it install ocrmypdf:
     curl https://bootstrap.pypa.io/ez_setup.py -o - | python3.6 && python3.6 -m easy_install pip
     pip3.6 install ocrmypdf
 
-These installation instructions omit the optional dependency
-``unpaper``, which is only available at version 0.4.2 in Ubuntu 14.04.
-The author could not find a backport of ``unpaper``, and created a .deb
-package to do the job of installing unpaper 6.1 (for x86 64-bit only):
-
-.. code-block:: bash
-
-    wget -q 'https://www.dropbox.com/s/vaq0kbwi6e6au80/unpaper_6.1-1.deb?raw=1' -O unpaper_6.1-1.deb
-    sudo dpkg -i unpaper_6.1-1.deb
+The optional dependency ``unpaper`` is only available at 0.4.2 in Ubuntu 14.04,
+and no backports are available. Previously the author maintained a backported
+.deb package for unpaper 6.1, but since Ubuntu 14.04 is now end of life, this is
+not supported. As such, ``unpaper`` is not available on Ubuntu 14.04 or must by
+compiled by hand.
 
 To add JBIG2 encoding, see :ref:`jbig2`.
 
@@ -517,7 +513,7 @@ manager. ``pip`` cannot provide them.
 
 As of ocrmypdf 7.2.1, the following versions are recommended:
 
--  Python 3.7
+-  Python 3.7 or 3.8
 -  Ghostscript 9.23 or newer
 -  qpdf 8.2.1
 -  Tesseract 4.0.0 or newer

docs/introduction.rst

@@ -208,7 +208,7 @@ consider one of these similar open source programs:
 Web front-ends
 ==============
 
-The Docker image ``ocrmypdf-alpine`` provides a web service front-end
+The Docker image ``ocrmypdf`` provides a web service front-end
 that allows files to submitted over HTTP and the results "downloaded".
 This is an HTTP server intended to simplify web services deployments; it
 is not intended to be deployed on the public internet and no real

docs/release_notes.rst

@@ -13,6 +13,38 @@ Note that it is licensed under GPLv3, so scripts that
 ``import ocrmypdf`` and are released publicly should probably also be
 licensed under GPLv3.
 
+v9.0.5
+======
+
+-  The Alpine Docker image (jbarlow83/ocrmypdf-alpine) has been dropped due to
+   the difficulties of supporting Alpine Linux.
+-  The primary Docker image (jbarlow83/ocrmypdf) has been improved to take on
+   the extra features that used to be exclusive to the Alpine image.
+-  No changes to application code.
+-  pdfminer.six version 20191020 is now supported.
+
+v9.0.4
+======
+
+-  Fixed compatibility with Python 3.8 (but requires source install for the moment).
+-  Fixed Tesseract settings for ``--user-words`` and ``--user-patterns``.
+-  Changed to pikepdf 1.6.5 (for Python 3.8).
+-  Changed to Pillow 6.2.0 (to mitigate a security vulnerability in earlier Pillow).
+-  A debug message now mentions when English is automatically selected if the locale
+   is not English.
+
+v9.0.3
+======
+
+-  Embed an encoded version of the sRGB ICC profile in the intermediate
+   Postscript file (used for PDF/A conversion). Previously we included the
+   filename, which required Postscript to run with file access enabled. For
+   security, Ghostscript 9.28 enables ``-dSAFER`` and as such, no longer
+   permits access to any file by default. This fix is necessary for
+   compatibility with Ghostscript 9.28.
+-  Exclude a test that sometimes times out and fails in continuous integration
+   from the standard test suite.
+
 v9.0.2
 ======
 
@@ -23,7 +55,7 @@ v9.0.2
 -  Fixed an issue that caused inversion of black and white in monochrome images.
    We are not certain but the problem seems to be linked to Leptonica 1.76.0 and
    older.
--  Fixed some cases where the test suite failed or produced unexpected if
+-  Fixed some cases where the test suite failed if
    English or German Tesseract language packs were not installed.
 -  Fixed a runtime error if the Tesseract English language is not installed.
 -  Improved explicit closing of Pillow images after use.

pyproject.toml

@@ -10,7 +10,7 @@ build-backend = "setuptools.build_meta"
 
 [tool.black]
 line-length = 88
-py36 = true
+target-version = ["py36", "py37", "py38"]
 skip-string-normalization = true
 include = '\.pyi?$'
 exclude = '''

requirements/main.txt

@@ -1,13 +1,10 @@
 # requirements.txt can be used to replicate the developer's build environment
 # setup.py lists a separate set of requirements that are looser to simplify
 # installation
-chardet == 3.0.4
-cffi == 1.12.2
+cffi == 1.13.2
 img2pdf == 0.3.3
-pdfminer.six == 20181108
-pikepdf == 1.6.1
-Pillow >= 5.0.0, != 5.1.0 ; sys_platform == "darwin"
-pycparser == 2.19
-python-xmp-toolkit == 2.0.1
-reportlab == 3.5.13
-tqdm == 4.32.1
+pdfminer.six == 20191020
+pikepdf == 1.6.5
+Pillow >= 6.2.0
+reportlab == 3.5.32
+tqdm == 4.37.0

requirements/test.txt

@@ -2,7 +2,7 @@ pytest >= 5.0.0
 pytest-helpers-namespace >= 2019.1.8
 pytest-xdist >= 1.29.0  # For DumpError fix
 pytest-cov >= 2.6.1
-python-xmp-toolkit   # requires apt-get install libexempi3
+python-xmp-toolkit == 2.0.1  # requires apt-get install libexempi3
                      # or brew install exempi
 PyPDF2 >= 1.26.0
 #PyMuPDF == 1.13.4    # optional

requirements/webservice.txt

@@ -0,0 +1 @@
+Flask >= 1, < 2

setup.py

@@ -68,6 +68,7 @@ setup(
     classifiers=[
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
         "Development Status :: 5 - Production/Stable",
         "Environment :: Console",
         "Intended Audience :: End Users/Desktop",
@@ -95,11 +96,9 @@ setup(
         'chardet >= 3.0.4, < 4',  # unlisted requirement of pdfminer.six 20181108
         'cffi >= 1.9.1',  # must be a setup and install requirement
         'img2pdf >= 0.3.0, < 0.4',  # pure Python, so track HEAD closely
-        'pdfminer.six == 20181108',
-        'pikepdf >= 1.6.0, < 2',
-        'Pillow >= 4.0.0, != 5.1.0 ; sys_platform == "darwin"',
-        # Pillow < 4 has BytesIO/TIFF bug w/img2pdf 0.2.3
-        # block 5.1.0, broken wheels
+        'pdfminer.six >= 20181108, <= 20191020',
+        'pikepdf >= 1.6.5, < 2',
+        'Pillow >= 6.2.0',
         'reportlab >= 3.3.0',  # oldest released version with sane image handling
         'tqdm >= 4',
     ],

src/ocrmypdf/_sync.py

@@ -189,7 +189,7 @@ def worker_init(queue):
     root.addHandler(h)
 
 
-def worker_thread_init(queue):
+def worker_thread_init(_queue):
     pass
 
 
@@ -225,15 +225,15 @@ def exec_concurrent(context):
     if max_workers > 1:
         context.log.info("Start processing %d pages concurrent", max_workers)
 
-    # Tesseract 4.0 is multithreaded, and we also run multiple workers. We want to
-    # avoid the situation where we end up trying to run NxN jobs on N CPU cores,
-    # as that gives poor performance. Performance testing shows we're better off
+    # Tesseract 4.x can be multithreaded, and we also run multiple workers. We want
+    # to manage how many threads it uses to avoid creating total threads than cores.
+    # Performance testing shows we're better off
     # parallelizing ocrmypdf and forcing Tesseract to be single threaded, which we
     # get by setting the envvar OMP_THREAD_LIMIT to 1. But if the page count of the
     # input file is small, then we allow Tesseract to use threads, subject to the
-    # constraint: (ocrmypdf workers) * (tesseract threads) <= max_workers and limiting
-    # Tesseract to 4 threads.
-    tess_threads = min(4, context.options.jobs // max_workers)
+    # constraint: (ocrmypdf workers) * (tesseract threads) <= max_workers.
+    # As of Tesseract 4.1, 3 threads is the most effective on a 4 core/8 thread system.
+    tess_threads = min(3, context.options.jobs // max_workers)
     if context.options.tesseract_env is None:
         context.options.tesseract_env = os.environ.copy()
     context.options.tesseract_env.setdefault('OMP_THREAD_LIMIT', str(tess_threads))
@@ -301,7 +301,7 @@ def exec_concurrent(context):
 class NeverRaise(Exception):
     """An exception that is never raised"""
 
-    pass
+    pass  # pylint: disable=unnecessary-pass
 
 
 def run_pipeline(options, api=False):

src/ocrmypdf/_validation.py

@@ -17,6 +17,7 @@
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
 
+import locale
 import logging
 import os
 import sys
@@ -47,6 +48,7 @@ from .helpers import is_file_writable, is_iterable_notstr, monotonic, re_symlink
 # External dependencies
 
 HOCR_OK_LANGS = frozenset(['eng', 'deu', 'spa', 'ita', 'por'])
+DEFAULT_LANGUAGE = 'eng'  # Enforce English hegemony
 
 log = logging.getLogger(__name__)
 
@@ -58,7 +60,10 @@ verify_python3_env()
 
 def check_options_languages(options):
     if not options.language:
-        options.language = ['eng']  # Enforce English hegemony
+        options.language = [DEFAULT_LANGUAGE]
+        system_lang = locale.getlocale()[0]
+        if system_lang and not system_lang.startswith('en'):
+            log.debug("No language specified; assuming --language %s", DEFAULT_LANGUAGE)
 
     # Support v2.x "eng+deu" language syntax
     if '+' in options.language[0]:

src/ocrmypdf/api.py

@@ -33,7 +33,7 @@ class TqdmConsole:
 
     def __init__(self, file):
         self.file = file
-        self.py36 = sys.version_info >= (3, 6)
+        self.py36 = sys.version_info[0:2] == (3, 6)
 
     def write(self, msg):
         # When no progress bar is active, tqdm.write() routes to print()
@@ -122,13 +122,23 @@ def create_options(*, input_file, output_file, **kwargs):
     for arg, val in kwargs.items():
         if val is None:
             continue
-        if arg == 'tesseract_env':
+
+        # These arguments with special handling for which we bypass
+        # argparse
+        if arg in {'tesseract_env', 'progress_bar'}:
             deferred.append((arg, val))
             continue
+
         cmd_style_arg = arg.replace('_', '-')
-        cmdline.append(f"--{cmd_style_arg}")
+
+        # Booleans are special: add only if True, omit for False
         if isinstance(val, bool):
+            if val:
+                cmdline.append(f"--{cmd_style_arg}")
             continue
+
+        # We have a parameter
+        cmdline.append(f"--{cmd_style_arg}")
         if isinstance(val, (int, float)):
             cmdline.append(str(val))
         elif isinstance(val, str):

src/ocrmypdf/exec/ghostscript.py

@@ -259,6 +259,7 @@ def generate_pdfa(
                 "-dQUIET",
                 "-dBATCH",
                 "-dNOPAUSE",
+                "-dSAFER",
                 "-dCompatibilityLevel=" + str(pdf_version),
                 "-sDEVICE=pdfwrite",
                 "-dAutoRotatePages=/None",

src/ocrmypdf/exec/tesseract.py

@@ -17,10 +17,8 @@
 
 import os
 import shutil
-import sys
 from collections import namedtuple
 from contextlib import suppress
-from functools import lru_cache
 from os import fspath
 from subprocess import PIPE, STDOUT, CalledProcessError, TimeoutExpired, run
 
@@ -271,6 +269,9 @@ def generate_hocr(
     if user_patterns:
         args_tesseract.extend(['--user-patterns', user_patterns])
 
+    if user_words or user_patterns:
+        args_tesseract.extend(['-c', 'lstm_use_matrix=1'])
+
     # Reminder: test suite tesseract spoofers will break after any changes
     # to the number of order parameters here
     args_tesseract.extend([input_file, prefix, 'hocr', 'txt'] + tessconfig)

src/ocrmypdf/optimize.py

@@ -111,6 +111,11 @@ def extract_image_generic(*, pike, root, log, image, xref, options):
     if pim.bits_per_component == 1:
         return None
 
+    try:
+        pim.indexed  # pikepdf 1.6.3 can't handle [/Indexed [/Array...]]
+    except NotImplementedError:
+        return None
+
     if filtdp[0] == Name.DCTDecode and options.optimize >= 2:
         # This is a simple heuristic derived from some training data, that has
         # about a 70% chance of guessing whether the JPEG is high quality,
@@ -544,11 +549,11 @@ def main(infile, outfile, level, jobs=1):
         """Emulate ocrmypdf's options"""
 
         def __init__(
-            self, input_file, jobs, optimize, jpeg_quality, png_quality, jb2lossy
+            self, input_file, jobs, optimize_, jpeg_quality, png_quality, jb2lossy
         ):
             self.input_file = input_file
             self.jobs = jobs
-            self.optimize = optimize
+            self.optimize = optimize_
             self.jpeg_quality = jpeg_quality
             self.png_quality = png_quality
             self.jbig2_page_group_size = 0
@@ -559,7 +564,7 @@ def main(infile, outfile, level, jobs=1):
     options = OptimizeOptions(
         input_file=infile,
         jobs=jobs,
-        optimize=int(level),
+        optimize_=int(level),
         jpeg_quality=0,  # Use default
         png_quality=0,
         jb2lossy=False,

src/ocrmypdf/pdfa.py

@@ -31,6 +31,7 @@ Ghostscript's handling of pdfmark.
 
 """
 
+import base64
 import os
 from binascii import hexlify
 from pathlib import Path
@@ -48,26 +49,13 @@ SRGB_ICC_PROFILE = pkg_resources.resource_filename('ocrmypdf', ICC_PROFILE_RELPA
 # files, from the Ghostscript documentation. Lines beginning with % are
 # comments. Python substitution variables have a '$' prefix.
 pdfa_def_template = u"""%!
-% Define entries in the document Info dictionary :
+% Define an ICC profile :
 /ICCProfile $icc_profile
 def
 
-% Define an ICC profile :
-
 [/_objdef {icc_PDFA} /type /stream /OBJ pdfmark
-[{icc_PDFA}
-<<
-  /N currentpagedevice /ProcessColorModel known {
-    currentpagedevice /ProcessColorModel get dup /DeviceGray eq
-    {pop 1} {
-      /DeviceRGB eq
-      {3}{4} ifelse
-    } ifelse
-  } {
-    (ERROR, unable to determine ProcessColorModel) == flush
-  } ifelse
->> /PUT pdfmark
-[{icc_PDFA} ICCProfile (r) file /PUT pdfmark
+[{icc_PDFA} << /N 3 >> /PUT pdfmark
+[{icc_PDFA} ICCProfile /PUT pdfmark
 
 % Define the output intent dictionary :
 
@@ -104,16 +92,10 @@ def generate_pdfa_ps(target_filename, icc='sRGB'):
     else:
         raise NotImplementedError("Only supporting sRGB")
 
-    # pdfmark must contain the full path to the ICC profile, and pdfmark must be
-    # also encoded in ASCII. ocrmypdf can be installed anywhere, including to
-    # paths that have a non-ASCII character in the filename. Ghostscript
-    # accepts hex-encoded strings and converts them to byte strings, so
-    # we encode the path with fsencode() and use the hex representation.
-    # UTF-16 not accepted here. (Even though ASCII encodable is the usual case,
-    # do this always to avoid making it a rare conditional.)
-    bytes_icc_profile = os.fsencode(icc_profile)
-    hex_icc_profile = hexlify(bytes_icc_profile)
-    icc_profile = '<' + hex_icc_profile.decode('ascii') + '>'
+    # Read the ICC profile, encode as ASCII85 and convert to a string which we
+    # will insert in the .ps file
+    bytes_icc_profile = Path(icc_profile).read_bytes()
+    icc_profile = base64.a85encode(bytes_icc_profile, adobe=True).decode('ascii')
 
     t = Template(pdfa_def_template)
     ps = t.substitute(icc_profile=icc_profile, icc_identifier=icc)

src/ocrmypdf/pdfinfo/layout.py

@@ -20,6 +20,7 @@ from math import copysign
 from pathlib import Path
 from unittest.mock import patch
 
+import pdfminer
 import pdfminer.encodingdb
 import pdfminer.pdfdevice
 import pdfminer.pdfinterp
@@ -36,51 +37,54 @@ from ..exceptions import EncryptedPdfError
 STRIP_NAME = re.compile(r'[0-9]+')
 
 #
-# Unconditional pdfminer patches
+#  pdfminer 20181108 patches
 #
 
+if pdfminer.__version__ == '20181108':
 
-def name2unicode(name):
-    """Fix pdfminer's name2unicode function
+    def name2unicode(name):
+        """Fix pdfminer's name2unicode function
 
-    Font cids that are mapped to names of the form /g123 seem to be, by convention
-    characters with no corresponding Unicode entry. These can be subsetted fonts
-    or symbolic fonts. There seems to be no way to map /g123 fonts to Unicode,
-    barring a ToUnicode data structure.
-    """
-    if name in glyphname2unicode:
-        return glyphname2unicode[name]
-    if name.startswith('g') or name.startswith('a'):
-        raise KeyError(name)
-    if name.startswith('uni'):
-        try:
-            return chr(int(name[3:], 16))
-        except ValueError:  # Not hexadecimal
+        Font cids that are mapped to names of the form /g123 seem to be, by convention
+        characters with no corresponding Unicode entry. These can be subsetted fonts
+        or symbolic fonts. There seems to be no way to map /g123 fonts to Unicode,
+        barring a ToUnicode data structure.
+        """
+        if name in glyphname2unicode:
+            return glyphname2unicode[name]
+        if name.startswith('g') or name.startswith('a'):
             raise KeyError(name)
-    m = STRIP_NAME.search(name)
-    if not m:
-        raise KeyError(name)
-    return chr(int(m.group(0)))
+        if name.startswith('uni'):
+            try:
+                return chr(int(name[3:], 16))
+            except ValueError:  # Not hexadecimal
+                raise KeyError(name)
+        m = STRIP_NAME.search(name)
+        if not m:
+            raise KeyError(name)
+        return chr(int(m.group(0)))
 
+    pdfminer.encodingdb.name2unicode = name2unicode
 
-pdfminer.encodingdb.name2unicode = name2unicode
+    original_PDFFont_init = PDFFont.__init__
 
-original_PDFFont_init = PDFFont.__init__
+    def PDFFont__init__(self, descriptor, widths, default_width=None):
+        original_PDFFont_init(self, descriptor, widths, default_width)
+        # PDF spec says descent should be negative
+        # A font with a positive descent implies it floats entirely above the
+        # baseline, i.e. it's not really a baseline anymore. I have fonts that
+        # claim a positive descent, but treating descent as positive always seems
+        # to misposition text.
+        if self.descent > 0:
+            self.descent = -self.descent
 
+    PDFFont.__init__ = PDFFont__init__
 
-def PDFFont__init__(self, descriptor, widths, default_width=None):
-    original_PDFFont_init(self, descriptor, widths, default_width)
-    # PDF spec says descent should be negative
-    # A font with a positive descent implies it floats entirely above the
-    # baseline, i.e. it's not really a baseline anymore. I have fonts that
-    # claim a positive descent, but treating descent as positive always seems
-    # to misposition text.
-    if self.descent > 0:
-        self.descent = -self.descent
+#
+# end of pdfminer 20181108 patches
+#
 
 
-PDFFont.__init__ = PDFFont__init__
-
 original_PDFSimpleFont_init = PDFSimpleFont.__init__
 
 
@@ -97,6 +101,7 @@ def PDFSimpleFont__init__(self, descriptor, widths, spec):
 
 
 PDFSimpleFont.__init__ = PDFSimpleFont__init__
+
 #
 # pdfminer patches when creator is PScript5.dll
 #

tests/test_completion.py

@@ -19,6 +19,11 @@ from subprocess import run, PIPE
 
 import pytest
 
+pytestmark = pytest.mark.skipif(
+    pytest.helpers.running_in_docker(),  # pylint: disable=no-member
+    reason="docker can't complete",
+)
+
 
 def test_fish():
     try:

tests/test_main.py

@@ -217,7 +217,7 @@ def test_skip_ocr(spoof_tesseract_cache, resources, outpdf):
     assert pdfinfo[0].has_text
 
 
-def test_redo_ocr(spoof_tesseract_cache, resources, outpdf):
+def test_redo_ocr(resources, outpdf):
     in_ = resources / 'graph_ocred.pdf'
     before = PdfInfo(in_, detailed_page_analysis=True)
     out = outpdf
@@ -613,6 +613,7 @@ language_model_penalty_non_freq_dict_word 0
     )
 
 
+@pytest.mark.slow  # This test sometimes times out in CI
 @pytest.mark.parametrize('renderer', RENDERERS)
 def test_tesseract_config_notfound(renderer, resources, outdir):
     cfg_file = outdir / 'nofile.cfg'
@@ -629,6 +630,7 @@ def test_tesseract_config_notfound(renderer, resources, outdir):
     assert p.returncode == ExitCode.ok, err
 
 
+@pytest.mark.slow  # This test sometimes times out in CI
 @pytest.mark.parametrize('renderer', RENDERERS)
 def test_tesseract_config_invalid(renderer, resources, outdir):
     cfg_file = outdir / 'test.cfg'

tests/test_validation.py

@@ -15,20 +15,23 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
+import locale
+import logging
 import os
-from unittest.mock import MagicMock, patch
+from unittest.mock import patch
 
 import pytest
 
 import ocrmypdf._validation as vd
 from ocrmypdf.api import create_options
 from ocrmypdf.exceptions import MissingDependencyError, BadArgsError
+from ocrmypdf.pdfinfo import PdfInfo
 
 
 def make_opts(input_file='a.pdf', output_file='b.pdf', language='eng', **kwargs):
-    return create_options(
-        input_file=input_file, output_file=output_file, language=language, **kwargs
-    )
+    if language is not None:
+        kwargs['language'] = language
+    return create_options(input_file=input_file, output_file=output_file, **kwargs)
 
 
 def test_hocr_notlatin_warning(caplog):
@@ -119,3 +122,41 @@ def test_report_file_size(tmp_path, caplog):
     os.truncate(out, 50000)
     vd.report_output_file_size(opts, in_, out)
     assert 'No reason' in caplog.text
+
+
+def test_false_action_store_true():
+    opts = make_opts(keep_temporary_files=True)
+    assert opts.keep_temporary_files == True
+    opts = make_opts(keep_temporary_files=False)
+    assert opts.keep_temporary_files == False
+
+
+@pytest.mark.parametrize('progress_bar', [True, False])
+def test_no_progress_bar(progress_bar, resources):
+    opts = make_opts(progress_bar=progress_bar, input_file=(resources / 'trivial.pdf'))
+    with patch('ocrmypdf.pdfinfo.info.tqdm', autospec=True) as tqdmpatch:
+        vd.check_options(opts)
+        pdfinfo = PdfInfo(opts.input_file, progbar=opts.progress_bar)
+        assert pdfinfo is not None
+        assert tqdmpatch.called
+        _args, kwargs = tqdmpatch.call_args
+        assert kwargs['disable'] != progress_bar
+
+
+def test_language_warning(caplog):
+    opts = make_opts(language=None)
+    caplog.set_level(logging.DEBUG)
+    with patch(
+        'ocrmypdf._validation.locale.getlocale', return_value=('en_US', 'UTF-8')
+    ):
+        vd.check_options_languages(opts)
+        assert opts.language == ['eng']
+        assert '' in caplog.text
+
+    opts = make_opts(language=None)
+    with patch(
+        'ocrmypdf._validation.locale.getlocale', return_value=('fr_FR', 'UTF-8')
+    ):
+        vd.check_options_languages(opts)
+        assert opts.language == ['eng']
+        assert 'assuming --language' in caplog.text