Make the go/no-go decision pluggable

It got dropped during the change. This feature has also been altered so that
the final visual appearance of the file is not affected, only the OCR image.

.coveragerc

@@ -1,9 +1,18 @@
 # Coverage isn't really compatible with subprocesses so results are unreliable
 
+[paths]
+source =
+	src
+	*/site-packages
+
 [run]
-branch = True
-#concurrency = multiprocessing
-source = ocrmypdf/
+branch = true
+parallel = true
+source =
+	src/ocrmypdf
+	tests
+omit =
+	tests/spoof/*
 
 [report]
 exclude_lines =

.docker/Dockerfile

@@ -1,17 +1,60 @@
 # OCRmyPDF
 #
-FROM      ubuntu:18.04
+FROM ubuntu:19.04 as base
+
+FROM base as builder
+
+ENV LANG=C.UTF-8
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
   build-essential autoconf automake libtool \
   libleptonica-dev \
   zlib1g-dev \
-  libexempi3 \
   ocrmypdf \
   pngquant \
   python3-pip \
   python3-venv \
   tesseract-ocr \
+  unpaper \
+  wget \
+  git
+
+
+# Compile and install jbig2
+# Needs libleptonica-dev, zlib1g-dev
+RUN \
+  mkdir jbig2 \
+  && wget -q https://github.com/agl/jbig2enc/archive/0.29.tar.gz -O - | \
+  tar xz -C jbig2 --strip-components=1 \
+  && cd jbig2 \
+  && ./autogen.sh && ./configure && make && make install \
+  && cd .. \
+  && rm -rf jbig2
+
+RUN python3 -m venv /appenv
+
+COPY . /app
+
+WORKDIR /app
+
+RUN . /appenv/bin/activate; \
+  pip install --upgrade pip \
+  && pip install .
+
+FROM base
+
+ENV LANG=C.UTF-8
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  ghostscript \
+  img2pdf \
+  liblept5 \
+  zlib1g \
+  pngquant \
+  python3 \
+  python3-venv \
+  qpdf \
+  tesseract-ocr \
   tesseract-ocr-chi-sim \
   tesseract-ocr-deu \
   tesseract-ocr-eng \
@@ -21,52 +64,16 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   unpaper \
   wget
 
+# Copy
+COPY --from=builder /app/misc/webservice.py /app/
 
-ENV LANG=C.UTF-8
+# Copy minimal project files to get the test suite.
+COPY --from=builder /app/setup.cfg /app/setup.py /app/README.md /app/
+COPY --from=builder /app/requirements /app/requirements
+COPY --from=builder /app/tests /app/tests
+COPY --from=builder /app/src /app/src
 
-# Compile and install jbig2
-# Needs libleptonica-dev, zlib1g-dev
-RUN \
-  mkdir jbig2 \
-  && wget -q https://github.com/agl/jbig2enc/archive/0.29.tar.gz -O - | \
-      tar xz -C jbig2 --strip-components=1 \
-  && cd jbig2 \
-  && ./autogen.sh && ./configure && make && make install \
-  && cd .. \
-  && rm -rf jbig2
+COPY --from=builder /appenv /appenv
+COPY --from=builder /usr/local /usr/local
 
-RUN apt-get remove -y autoconf automake libtool
-
-RUN python3 -m venv --system-site-packages /appenv
-
-# This installs the latest binary wheel instead of the code in the current
-# folder. Installing from source will fail, apparently because cffi needs
-# build-essentials (gcc) to do a source installation
-# (i.e. "pip install ."). It's unclear to me why this is the case.
-RUN . /appenv/bin/activate; \
-  pip install --upgrade pip \
-  && pip install --upgrade ocrmypdf
-
-# Now copy the application in, mainly to get the test suite.
-# Do this now to make the best use of Docker cache.
-COPY . /application
-RUN . /appenv/bin/activate; \
-  pip install -r /application/requirements/test.txt
-
-# Remove the junk, including the source version of application since it was
-# already installed
-RUN rm -rf /tmp/* /var/tmp/* /root/* /application/ocrmypdf \
-  && apt-get remove -y build-essential \
-  && apt-get autoremove -y \
-  && apt-get autoclean -y
-
-RUN useradd docker \
-  && mkdir /home/docker \
-  && chown docker:docker /home/docker
-
-USER docker
-WORKDIR /home/docker
-
-# Must use array form of ENTRYPOINT
-# Non-array form does not append other arguments, because that is "intuitive"
-ENTRYPOINT ["/application/.docker/docker-wrapper.sh"]
+ENTRYPOINT ["/appenv/bin/ocrmypdf"]

.docker/alpine.dockerfile

@@ -4,33 +4,38 @@ FROM base as builder
 
 ENV LANG=C.UTF-8
 
+# Normally:
+# echo '@testing http://nl.alpinelinux.org/alpine/edge/testing' >> /etc/apk/repositories
+
 RUN \
-  echo '@testing http://nl.alpinelinux.org/alpine/edge/testing' >> /etc/apk/repositories \
+  echo -e '@testing http://nl.alpinelinux.org/alpine/edge/testing\n@community http://nl.alpinelinux.org/alpine/edge/community'\
+  >> /etc/apk/repositories \
   # Add runtime dependencies
   && apk add --update \
-    python3-dev \
-    py3-setuptools \
-    jbig2enc@testing \
-    ghostscript \
-    qpdf \
-    tesseract-ocr \
-    unpaper \
-    pngquant \
-    libxml2-dev \
-    libxslt-dev \
-    zlib-dev \
-    qpdf-dev \
-    libffi-dev \
-    leptonica-dev \
-    binutils \
+  python3-dev \
+  py3-setuptools \
+  jbig2enc@testing \
+  ghostscript \
+  qpdf@community \
+  qpdf-dev@community \
+  tesseract-ocr \
+  unpaper \
+  pngquant \
+  libxml2-dev \
+  libxslt-dev \
+  zlib-dev \
+  libffi-dev \
+  leptonica-dev \
+  binutils \
+  && pip3 install --upgrade pip \
   # Install pybind11 for pikepdf
   && pip3 install pybind11 \
   # Install flask for the webservice
   && pip3 install flask \
   # Add build dependencies
   && apk add --virtual build-dependencies \
-    build-base \
-    git
+  build-base \
+  git
 
 COPY . /app
 
@@ -42,43 +47,45 @@ FROM base
 
 ENV LANG=C.UTF-8
 
+# Normally:
+# echo '@testing http://nl.alpinelinux.org/alpine/edge/testing' >> /etc/apk/repositories
+
 RUN \
-  echo '@testing http://nl.alpinelinux.org/alpine/edge/testing' >> /etc/apk/repositories \
+  echo -e '@testing http://nl.alpinelinux.org/alpine/edge/testing\n@community http://nl.alpinelinux.org/alpine/edge/community'\
+  >> /etc/apk/repositories \
   # Add runtime dependencies
   && apk add --update \
-    python3 \
-    jbig2enc@testing \
-    ghostscript \
-    qpdf \
-    tesseract-ocr \
-    tesseract-ocr-data-deu \
-    tesseract-ocr-data-chi_sim \
-    unpaper \
-    pngquant \
-    libxml2 \
-    libxslt \
-    zlib \
-    qpdf \
-    libffi \
-    leptonica-dev \
-    binutils \
+  python3 \
+  jbig2enc@testing \
+  ghostscript \
+  qpdf@community \
+  qpdf-dev@community \
+  tesseract-ocr \
+  tesseract-ocr-data-deu \
+  tesseract-ocr-data-chi_sim \
+  unpaper \
+  pngquant \
+  libxml2 \
+  libxslt \
+  zlib \
+  libffi \
+  leptonica-dev \
+  binutils \
   && mkdir /app
 
 WORKDIR /app
 
-# Copy build artifacts (python site-packages9
+# Copy build artifacts (python site-packages)
 COPY --from=builder /usr/lib/python3.6/site-packages /usr/lib/python3.6/site-packages
 COPY --from=builder /usr/bin/ocrmypdf /usr/bin/dumppdf.py /usr/bin/latin2ascii.py /usr/bin/pdf2txt.py /usr/bin/img2pdf /usr/bin/chardetect /usr/bin/
 
 # Copy
-COPY --from=builder /app/.docker/webservice.py /app/
+COPY --from=builder /app/misc/webservice.py /app/
 
 # Copy minimal project files to get the test suite.
 COPY --from=builder /app/setup.cfg /app/setup.py /app/README.md /app/
 COPY --from=builder /app/requirements /app/requirements
 COPY --from=builder /app/tests /app/tests
 COPY --from=builder /app/src /app/src
-# Copy PKG-INFO from build artifact in app dir to make setuptools-scm happy
-RUN cp /usr/lib/python3.6/site-packages/ocrmypdf-*.egg-info/PKG-INFO /app
 
 ENTRYPOINT ["/usr/bin/ocrmypdf"]

.docker/docker-wrapper.sh

@@ -1,5 +0,0 @@
-#!/bin/bash
-
-. /appenv/bin/activate
-cd /home/docker
-exec ocrmypdf "$@"

.docker/polyglot.dockerfile

@@ -1,17 +0,0 @@
-# OCRmyPDF polyglot
-#
-FROM      jbarlow83/ocrmypdf:latest
-
-USER root
-
-# Update system and install our dependencies
-RUN apt-get update && apt-get install -y --no-install-recommends \
-  tesseract-ocr-all 
-
-RUN apt-get autoremove -y && apt-get clean -y
-
-USER docker
-
-# Must use array form of ENTRYPOINT
-# Non-array form does not append other arguments, because that is "intuitive"
-ENTRYPOINT ["/application/.docker/docker-wrapper.sh"]

.docker/webservice.dockerfile

@@ -1,24 +0,0 @@
-# OCRmyPDF webservice
-#
-FROM      jbarlow83/ocrmypdf-polyglot:latest
-
-USER root
-
-# Update system and install our dependencies
-RUN apt-get update && apt-get install -y --no-install-recommends \
-  python3-flask
-
-RUN apt-get autoremove -y && apt-get clean -y
-
-EXPOSE 5000
-
-COPY .docker/webservice.py /application
-
-USER docker
-
-VOLUME ["/config"]
-
-# This config file is optional
-ENV OCRMYPDF_WEBSERVICE_SETTINGS "/config/config.py"
-
-ENTRYPOINT ["python3", "/application/webservice.py"]

.dockerignore

@@ -6,7 +6,6 @@
 **/*.pyc
 .*/
 !.git/
-!.docker/
 .ruffus_history.sqlite
 bin/
 build/

.gitignore

@@ -4,8 +4,10 @@
 .pytest_cache/
 .ruffus_history.sqlite
 .venv/
+.venv*/
 *.pyc
 *.sublime-*
+*.DS_Store
 
 # Package building
 .eggs/
@@ -13,6 +15,7 @@
 build/
 dist/
 wheelhouse/
+pip-wheel-metadata/
 
 # Automatically generated files
 docs/_build/
@@ -40,4 +43,4 @@ tests/output/
 tests/resources/private/
 tmp/
 /debug_tests.py
-*.traineddata
+private/

.pre-commit-config.yaml

@@ -3,4 +3,4 @@ repos:
     rev: stable
     hooks:
     - id: black
-      language_version: python3.6
+      language_version: python3.7

docs/api.rst

@@ -0,0 +1,72 @@
+Using the OCRmyPDF API
+======================
+
+OCRmyPDF originated as a command line program and continues to have this legacy, but parts of it can be imported and used in other Python applications.
+
+Some applications may want to consider running ocrmypdf from a subprocess call anyway, as this provides isolation of its activities.
+
+Example
+-------
+
+OCRmyPDF one high-level function to run its main engine from an application. The parameters are symmetric to the command line arguments and largely have the same functions.
+
+.. code-block:: python
+
+    from ocrmypdf import ocrmypdf
+
+    ocrmypdf('input.pdf', 'output.pdf', deskew=True)
+
+With a few exceptions, all of the command line arguments are available and may be passed as equivalent keywords.
+
+A few differences are that ``verbose`` and ``quiet`` are not available. Instead, output should be managed by configuring logging.
+
+Parent process requirements
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The :func:`ocrmypdf.ocrmypdf` function runs OCRmyPDF similar to command line execution. To do this, it will:
+- create a monitoring thread
+- create worker processes (forking itself)
+- manage the signal flags of worker processes
+0 execute other subprocesses (forking and executing other programs)
+
+The Python process that calls ``ocrmypdf()`` must be sufficiently privileged to perform these actions. If it is not, ``ocrmypdf()`` will fail.
+
+There is no currently no option to manage how jobs are scheduled other than the argument ``jobs=`` which will limit the number of worker processes.
+
+Forking a child process to call ``ocrmypdf()`` is suggested. That way your application will survive even if OCRmyPDF does not.
+
+Logging
+^^^^^^^
+
+OCRmyPDF will log under loggers named ``ocrmypdf``. In addition, it imports ``pdfminer`` and ``PIL``, both of which post log messages under those logging namespaces.
+
+You can configure the logging as desired for your application or call :func:`ocrmypdf.configure_logging` to configure logging the same way OCRmyPDF itself does. The command line parameters such as ``--quiet`` and ``--verbose`` have no equivalents in the API; you must configure logging.
+
+Progress monitoring
+^^^^^^^^^^^^^^^^^^^
+
+OCRmyPDF uses the ``tqdm`` package to implement its progress bars. :func:`ocrmypdf.configure_logging` will set up logging output to ``sys.stderr`` in a way that is compatible with the display of the progress bar.
+
+Exceptions
+^^^^^^^^^^
+
+OCRmyPDF may throw standard Python exceptions, ``ocrmypdf.exceptions.*`` exceptions, some exceptions related to multiprocessing, and ``KeyboardInterrupt``. The parent process should provide an exception handler. OCRmyPDF will clean up its temporary files and worker processes automatically when an exception occurs.
+
+Programs that call OCRmyPDF should consider trapping KeyboardInterrupt so that they allow OCR to terminate with the whole program terminating.
+
+When OCRmyPDF succeeds conditionally, it may return an integer exit code.
+
+Reference
+---------
+
+.. autofunction:: ocrmypdf.ocrmypdf
+
+.. autoclass:: ocrmypdf.Verbosity
+    :members:
+    :undoc-members:
+
+.. autoclass:: ocrmypdf.ExitCode
+    :members:
+    :undoc-members:
+
+.. autofunction:: ocrmypdf.configure_logging

docs/batch.rst

@@ -28,7 +28,7 @@ This will walk through a directory tree and run OCR on all files in place, print
 .. code-block:: bash
 
 	find . -printf '%p' -name '*.pdf' -exec ocrmypdf '{}' '{}' \;
-	
+
 Alternatively, with a docker container (mounts a volume to the container where the PDFs are stored):
 
 .. code-block:: bash
@@ -95,16 +95,6 @@ This user contributed script also provides an example of batch processing.
 	                print("OCR complete")
 	            logging.info(result)
 
-API
-"""
-
-OCRmyPDF is currently supported as a command line interface. This means that even if you are using OCRmyPDF in a Python script, you should run it in a subprocess rather importing the ocrmypdf package.
-
-The reason for this limitation is that the `ruffus <https://github.com/bunbun/ruffus/>`_ library that OCRmyPDF depends on is unfortunately not reentrant. OCRmyPDF works by defining each operation it does as a ruffus task that takes one or more files as input and generates one or more files as output. As such ruffus is fairly fundamental.
-
-(If you find individual functions implemented in OCRmyPDF useful (such as ``ocrmypdf.pdfinfo``), you can use these if you wish to.)
-
-
 Synology DiskStations
 """""""""""""""""""""
 

docs/conf.py

@@ -30,9 +30,7 @@
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = [
-    # 'sphinx.ext.mathjax',
-]
+extensions = ['sphinx.ext.napoleon']
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']

docs/docker.rst

@@ -5,7 +5,7 @@ OCRmyPDF is also available in a Docker image that packages recent versions of al
 
 For users who already have Docker installed this may be an easy and convenient option. However, it is less performant than a system installation and may require Docker engine configuration.
 
-OCRmyPDF needs a generous amount of RAM, CPU cores, and temporary storage space.
+OCRmyPDF needs a generous amount of RAM, CPU cores, temporary storage space, whether running in a Docker container or on its own. It may be necessary to ensure the container is provisioned with additional resources.
 
 .. _docker-install:
 
@@ -46,14 +46,15 @@ To start a Docker container (instance of the image):
 .. code-block:: bash
 
     docker tag jbarlow83/ocrmypdf-alpine ocrmypdf
-    docker run --rm ocrmypdf (... all other arguments here...)
+    docker run --rm -i ocrmypdf (... all other arguments here...)
 
-For convenience, create a shell alias to hide the Docker command:
+For convenience, create a shell alias to hide the Docker command. It is easier to send the input file to file stdin and read the output from stdout – this avoids the occasionally messy permission issues with Docker entirely.
 
 .. code-block:: bash
 
-    alias ocrmypdf='docker run --rm -v "$(pwd):/home/docker" ocrmypdf'
+    alias ocrmypdf='docker run --rm -i ocrmypdf'
     ocrmypdf --version  # runs docker version
+    ocrmypdf <input.pdf >output.pdf
 
 Or in the wonderful `fish shell <https://fishshell.com/>`_:
 
@@ -62,6 +63,12 @@ Or in the wonderful `fish shell <https://fishshell.com/>`_:
     alias ocrmypdf 'docker run --rm ocrmypdf'
     funcsave ocrmypdf
 
+Alternately, you could mount the local current working directory as a Docker volume:
+
+.. code-block:: bash
+
+    docker run --rm -v $(pwd):/data ocrmypdf /data/input.pdf /data/output.pdf
+
 .. _docker-lang-packs:
 
 Adding languages to the Docker image
@@ -76,6 +83,8 @@ By default the Docker image includes English, German and Simplified Chinese, the
     # Add French
     RUN apk add tesseract-ocr-data-fra
 
+You can also copy training data to ``/usr/share/tessdata``.
+
 Executing the test suite
 ------------------------
 
@@ -85,6 +94,15 @@ The OCRmyPDF test suite is installed with image.  To run it:
 
     docker run --entrypoint python3 jbarlow83/ocrmypdf-alpine setup.py test
 
+Accessing the shell
+-------------------
+
+``bash`` is not installed in the image. To use the busybox shell in the Docker image:
+
+.. code-block:: bash
+
+    docker run -it --entrypoint busybox  jbarlow83/ocrmypdf-alpine sh
+
 Using the OCRmyPDF web service wrapper
 --------------------------------------
 
@@ -98,7 +116,7 @@ Unlike command line usage this program will open a socket and wait for connectio
 
 .. warning::
 
-    The OCRmyPDF web service wrapper is intended for demonstration or development. It provides no security, no authentication, no protection against denial of service attacks, and no load balancing. The default Flask WSGI server is used, which is intended for development only. The server is single-threaded and so can respond to only one client at a time. It cannot respond to clients while busy with OCR.
+    The OCRmyPDF web service wrapper is intended for demonstration or development. It provides no security, no authentication, no protection against denial of service attacks, and no load balancing. The default Flask WSGI server is used, which is intended for development only. The server is single-threaded and so can respond to only one client at a time. While running OCR, it cannot respond to any other clients.
 
 Clients must keep their open connection while waiting for OCR to complete. This may entail setting a long timeout; this interface is more useful for internal HTTP API calls.
 
@@ -106,50 +124,11 @@ Unlike the rest of OCRmyPDF, this web service is licensed under the Affero GPLv3
 
 In addition to the above, please read our :ref:`general remarks on using OCRmyPDF as a service <ocr-service>`.
 
-Legacy Ubuntu Docker images
----------------------------
+Ubuntu-based Docker image
+-------------------------
 
-Previously OCRmyPDF was delivered in several Docker images for different purposes, based on Ubuntu.
-
-The Ubuntu-based images will be maintained for some time but should not be used for new deployments. They are as follows:
-
-.. list-table::
-    :widths: auto
-    :header-rows: 1
-
-    *   - Image name
-        - Download command
-        - Notes
-    *   - ocrmypdf
-        - ``docker pull jbarlow83/ocrmypdf``
-        - Latest ocrmypdf with Tesseract 4.0.0-beta1 on Ubuntu 18.04. Includes English, French, German, Spanish, Portugeuse and Simplified Chinese.
-    *   - ocrmypdf-polyglot
-        - ``docker pull jbarlow83/ocrmypdf-polyglot``
-        - As above, with all available language packs.
-    *   - ocrmypdf-webservice
-        - ``docker pull jbarlow83/ocrmypdf-webservice``
-        - All language packs, and a simple HTTP wrapper allowing OCRmyPDF to be used as a web service. Note that this component is licensed under AGPLv3.
-
-To execute the Ubuntu-based OCRmyPDF on a local file, you must `provide a writable volume to the Docker image <https://docs.docker.com/userguide/dockervolumes/>`_, and both the input and output file must be inside the writable volume. This limitation applies only to the legacy images.
-
-This example command uses the current working directory as the writable volume:
+A Ubuntu-based OCRmyPDF image is also available. The main advantage this image offers is that it supports manylinux Python wheels (which are not supported on Alpine Linux). This may be useful for plugins.
 
 .. code-block:: bash
 
-    docker run --rm -v "$(pwd):/home/docker" <other docker arguments>   ocrmypdf <your arguments to ocrmypdf>
-
-In this worked example, the current working directory contains an input file called ``test.pdf`` and the output will go to ``output.pdf``:
-
-.. code-block:: bash
-
-    docker run --rm -v "$(pwd):/home/docker"   ocrmypdf --skip-text test.pdf output.pdf
-
-.. note:: The working directory should be a writable local volume or Docker may not have permission to access it.
-
-Note that ``ocrmypdf`` has its own separate ``-v VERBOSITYLEVEL`` argument to control debug verbosity. All Docker arguments should before the ``ocrmypdf`` image name and all arguments to ``ocrmypdf`` should be listed after.
-
-In some environments the permissions associated with Docker can be complex to configure. The process that executes Docker may end up not having the permissions to write the specified file system. In that case one can stream the file into and out of the Docker process and avoid all permission hassles, using ``-`` as the input and output filename:
-
-.. code-block:: bash
-
-    docker run --rm -i   ocrmypdf <other arguments to ocrmypdf> - - <input.pdf >output.pdf
+    docker pull jbarlow83/ocrmypdf

docs/index.rst

@@ -27,6 +27,7 @@ PDF is the best format for storing and exchanging scanned documents.  Unfortunat
    cookbook
    docker
    advanced
+   api
    batch
    security
    errors

docs/installation.rst

@@ -118,13 +118,13 @@ install the system version to get most of the dependencies:
         ocrmypdf \
         python3-pip
 
-There are a few dependency changes between ocrmypdf 6.1.2 and 7.x. Let's get
+There are a few system dependency changes since ocrmypdf 6.1.2. Let's get
 these, too.
 
 .. code-block:: bash
 
     sudo apt-get install \
-        libexempi3 \
+        libxml2 \
         pngquant
 
 Then install the most recent ocrmypdf for the local user and set the user's ``PATH`` to check for the user's Python packages.
@@ -364,6 +364,16 @@ The command line program should now be available:
 
     ocrmypdf --help
 
+Installing on FreeBSD
+---------------------
+
+FreeBSD 11.2 is known to work. Other versions likely work but have not been tested.
+
+In general it should work to:
+
+#. `Install and build pikepdf <https://pikepdf.readthedocs.io/en/latest/installation.html#installing-on-freebsd-11-2>`_.
+#. Install the equivalent list of dependencies for Linux.
+
 Installing the Docker image
 ---------------------------
 
@@ -490,3 +500,17 @@ To install all of the development and test requirements:
     pip install -r requirements/dev.txt -r requirements/test.txt
 
 To add JBIG2 encoding, see :ref:`jbig2`.
+
+Shell completions
+-----------------
+
+Completions for ``bash`` and ``fish`` are available in the project's
+``misc/completion`` folder. The ``bash`` completions are likely ``zsh``
+compatible but this has not been confirmed. Package maintainers, please install
+these at the appropriate locations for your system.
+
+To manually install the ``bash`` completion, copy ``misc/completion/ocrmypdf.bash`` to
+``/etc/bash_completion.d/ocrmypdf`` (rename the file).
+
+To manually install the ``fish`` completion, copy ``misc/completion/ocrmypdf.fish`` to
+``~/.config/fish/completions/ocrmypdf.fish``.

docs/introduction.rst

@@ -95,7 +95,6 @@ Ghostscript also imposes some limitations:
 Regarding OCRmyPDF itself:
 
 * PDFs that use transparency are not currently represented in the test suite
-* The Python API exported by ``import ocrmypdf`` is design to help scripts that use OCRmyPDF but is not currently capable of running OCRmyPDF jobs due to limitations in an underlying library.
 
 Similar programs
 ----------------

docs/release_notes.rst

@@ -5,14 +5,38 @@ OCRmyPDF uses `semantic versioning <http://semver.org/>`_ for its command line i
 
 The ``ocrmypdf`` package may now be imported. The public API may be useful in scripts that launch OCRmyPDF processes or that wish to use some of its features for working with PDFs.
 
-Unfortunately, the public API does **not** expose the ability to actually OCR a PDF. This is due to a limitation in an underlying library (ruffus) that makes OCRmyPDF non-reentrant.
-
 Note that it is licensed under GPLv3, so scripts that ``import ocrmypdf`` and are released publicly should probably also be licensed under GPLv3.
 
 .. Issue regex
    find:    [^`]\#([0-9]{1,3})[^0-9]
    replace: `#$1 <https://github.com/jbarlow83/OCRmyPDF/issues/$1>`_
 
+v8.3.0
+------
+
+-   Improved the strategy for updating pages when a new image of the page was produced. We now attempt to preserve more content from the original file, for annotations in particular.
+
+-   For PDFs with more than 100 pages and a sequence where one PDF page was replaced and one or more subsequent ones were skipped, an intermediate file would be corrupted while grafting OCR text, causing processing to fail. This is a regression, likely introduced in v8.2.4.
+
+-   Previously, we resized the images produced by Ghostscript by a small number of pixels to ensure the output image size was an exactly what we wanted. Having discovered a way to get Ghostscript to produce the exact image sizes we require, we eliminated the resizing step.
+
+-   Command line completions for ``bash`` are now available, in addition to ``fish``, both in ``misc/completion``. Package maintainers, please install these so users can take advantage.
+
+-   Updated requirements.
+
+-   pikepdf 1.3.0 is now required.
+
+v8.2.4
+------
+
+-   Fixed a false positive while checking for a certain type of PDF that only Acrobat can read. We now more accurately detect Acrobat-only PDFs.
+
+-   OCRmyPDF holds fewer open file handles and is more prompt about releasing those it no longer needs.
+
+-   Minor optimization: we no longer traverse the table of contents to ensure all references in it are resolved, as changes to libqpdf have made this unnecessary.
+
+-   pikepdf 1.2.0 is now required.
+
 v8.2.3
 ------
 

misc/completion/ocrmypdf.bash

@@ -0,0 +1,87 @@
+# ocrmypdf completion                                     -*- shell-script -*-
+
+_ocrmypdf()
+{
+    local cur prev cword words split
+    _init_completion -s || return
+
+    case $prev in
+        --version|-h|--help)
+            return
+            ;;
+        --user-words|--user-patterns|--tesseract-config)
+            _filedir
+            return
+            ;;
+        --output-type)
+            COMPREPLY=( $( compgen -W 'pdfa pdf pdfa-1 pdfa-2 pdfa-3' -- \
+                "$cur" ) )
+            return
+            ;;
+        --pdf-renderer)
+            COMPREPLY=( $( compgen -W 'auto hocr sandwich' -- "$cur" ) )
+            return
+            ;;
+        --pdfa-image-compression)
+            COMPREPLY=( $( compgen -W 'auto jpeg lossless' -- "$cur" ) )
+            return
+            ;;
+        -O|--optimize|--tesseract-oem)
+            COMPREPLY=( $( compgen -W '{0..3}' -- "$cur" ) )
+            return
+            ;;
+        --jpeg-quality|--png-quality)
+            COMPREPLY=( $( compgen -W '{0..100}' -- "$cur" ) )
+            return
+            ;;
+        -l|--language)
+            COMPREPLY=$( command tesseract --list-langs 2>/dev/null )
+            COMPREPLY=( $( compgen -W '${COMPREPLY[@]##*:}' -- "$cur" ) )
+            return
+            ;;
+        --image-dpi|--oversample|--skip-big|--max-image-mpixels|\
+        --tesseract-timeout|--rotate-pages-threshold)
+            COMPREPLY=( $( compgen -P "$cur" -W '{0..9}' ) )
+            return
+            ;;
+        -j|--jobs)
+            COMPREPLY=( $( compgen -W '{1..'$( _ncpus )'}' -- "$cur" ) )
+            return
+            ;;
+        -v|--verbose)
+            COMPREPLY=( $( compgen -W '{1..9}' -- "$cur" ) ) # max level ?
+            return
+            ;;
+        --tesseract-pagesegmode)
+            COMPREPLY=( $( compgen -W '{1..13}' -- "$cur" ) )
+            return
+            ;;
+        --sidecar|--title|--author|--subject|--keywords|--unpaper-args)
+            # argument required but no completions available
+            return
+            ;;
+    esac
+
+    $split && return
+
+    if [[ $cur == -* ]]; then
+        COMPREPLY=( $( compgen -W '--language --image-dpi --output-type
+            --sidecar --version --jobs --quiet --verbose --title --author
+            --subject --keywords --rotate-pages --remove-background --deskew
+            --clean --clean-final --unpaper-args --oversample --remove-vectors
+            --mask-barcodes --threshold --force-ocr --skip-text --redo-ocr
+            --skip-big --jpeg-quality --png-quality --jbig2-lossy
+            --max-image-mpixels --tesseract-config --tesseract-pagesegmode
+            --help --tesseract-oem --pdf-renderer --tesseract-timeout
+            --rotate-pages-threshold --pdfa-image-compression --user-words
+            --user-patterns --keep-temporary-files --flowchart --output-type' \
+            --  "$cur" ) )
+        return
+    else
+        _filedir
+        return
+    fi
+} &&
+complete -F _ocrmypdf ocrmypdf
+
+# ex: filetype=sh

requirements/main.txt

@@ -5,9 +5,9 @@ chardet == 3.0.4
 cffi == 1.12.2
 img2pdf == 0.3.3
 pdfminer.six == 20181108
-pikepdf == 1.1.0
+pikepdf == 1.3.0
 Pillow >= 5.0.0, != 5.1.0 ; sys_platform == "darwin"
 pycparser == 2.19
 python-xmp-toolkit == 2.0.1
 reportlab == 3.5.13
-ruffus == 2.8.1
+tqdm == 4.32.1

requirements/test.txt

@@ -1,6 +1,6 @@
-pytest == 4.3.0
+pytest >= 4.4.1, < 5
 pytest-helpers-namespace >= 2019.1.8
-pytest-xdist
+pytest-xdist == 1.28.0
 pytest-cov >= 2.6.1
 python-xmp-toolkit   # requires apt-get install libexempi3
                      # or brew install exempi

setup.cfg

@@ -13,6 +13,8 @@ norecursedirs = lib .pc .git output cache resources
 testpaths = tests
 filterwarnings =
 	ignore:.*XMLParser.*:DeprecationWarning
+markers =
+	slow
 
 [isort]
 multi_line_output=3

setup.py

@@ -26,9 +26,6 @@ if sys.version_info < (3, 6):
     sys.exit(1)
 
 from setuptools import setup, find_packages
-from subprocess import STDOUT, check_output, CalledProcessError
-from collections.abc import Mapping
-import re
 
 # pylint: disable=w0613
 
@@ -99,16 +96,16 @@ setup(
         'cffi >= 1.9.1',  # must be a setup and install requirement
         'img2pdf >= 0.3.0, < 0.4',  # pure Python, so track HEAD closely
         'pdfminer.six == 20181108 ; sys_platform != "darwin"',
-        'pikepdf >= 1.1.0, < 2',
+        'pikepdf >= 1.3.0, < 2',
         'Pillow >= 4.0.0, != 5.1.0 ; sys_platform == "darwin"',
         # Pillow < 4 has BytesIO/TIFF bug w/img2pdf 0.2.3
         # block 5.1.0, broken wheels
         'reportlab >= 3.3.0',  # oldest released version with sane image handling
-        'ruffus >= 2.7.0',
+        'tqdm >= 4',
     ],
     extras_require={'pdfminer': ['pdfminer.six == 20181108']},
     tests_require=tests_require,
-    entry_points={'console_scripts': ['ocrmypdf = ocrmypdf.__main__:run_pipeline']},
+    entry_points={'console_scripts': ['ocrmypdf = ocrmypdf.__main__:run']},
     package_data={'ocrmypdf': ['data/sRGB.icc']},
     include_package_data=True,
     zip_safe=False,

src/ocrmypdf/__init__.py

@@ -44,3 +44,4 @@ from . import hocrtransform
 from . import leptonica
 from . import pdfa
 from . import pdfinfo
+from .api import ocrmypdf, configure_logging, Verbosity

src/ocrmypdf/_jobcontext.py

@@ -15,69 +15,110 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
+import logging
 import shutil
 import sys
-from contextlib import suppress
-from multiprocessing.managers import SyncManager
-
-from .pdfinfo import PdfInfo
+import os
 
 
-class JobContext:
-    """Holds our context for a particular run of the pipeline
+class PicklableLoggerMixin:
+    def __init__(self):
+        self._log = None
 
-    A multiprocessing manager effectively creates a separate process
-    that keeps the master job context object.  Other threads access
-    job context via multiprocessing proxy objects.
+    @property
+    def log(self):
+        if not self._log:
+            self._log = self.get_logger()
+        return self._log
 
-    While this would naturally lend itself @property's it seems to make
-    a little more sense to use functions to make it explicitly that the
-    invocation requires marshalling data across a process boundary.
+    def __getstate__(self):
+        # Python 3.6 is incapable of pickling a logger and marshalling it to another
+        # process (threading._RLock error), so we disconnect it before pickling,
+        # and create a new logger in the worker process.
+        state = self.__dict__.copy()
+        state['_log'] = None
+        return state
 
+
+class PDFContext(PicklableLoggerMixin):
+    """Holds our context for a particular run of the pipeline"""
+
+    def __init__(self, options, work_folder, origin, pdfinfo):
+        PicklableLoggerMixin.__init__(self)
+        self.options = options
+        self.work_folder = work_folder
+        self.origin = origin
+        self.pdfinfo = pdfinfo
+        if options:
+            self.name = os.path.basename(options.input_file)
+        else:
+            self.name = 'origin.pdf'
+        if self.name == '-':
+            self.name = 'stdin'
+
+    def get_logger(self):
+        return make_logger(self.options, filename=self.name)
+
+    def get_path(self, name):
+        return os.path.join(self.work_folder, name)
+
+    def get_page_contexts(self):
+        npages = len(self.pdfinfo)
+        for n in range(npages):
+            yield PageContext(self, n)
+
+
+class PageContext(PicklableLoggerMixin):
+    """Holds our context for a page
+
+    Must be pickable, so only store intrinsic/simple data elements
     """
 
-    def __init__(self):
-        self.pdfinfo = None
-        self.options = None
-        self.work_folder = None
-        self.rotations = {}
+    def __init__(self, pdf_context, pageno):
+        PicklableLoggerMixin.__init__(self)
+        self.work_folder = pdf_context.work_folder
+        self.origin = pdf_context.origin
+        self.options = pdf_context.options
+        self.name = pdf_context.name
+        self.pageno = pageno
+        self.pageinfo = pdf_context.pdfinfo[pageno]
+        self._log = None
 
-    def generate_pdfinfo(self, infile):
-        self.pdfinfo = PdfInfo(infile)
+    def get_logger(self):
+        return make_logger(self.options, filename=self.name, page=self.pageno + 1)
 
-    def get_pdfinfo(self):
-        "What we know about the input PDF"
-        return self.pdfinfo
-
-    def set_pdfinfo(self, pdfinfo):
-        self.pdfinfo = pdfinfo
-
-    def get_options(self):
-        return self.options
-
-    def set_options(self, options):
-        self.options = options
-
-    def get_work_folder(self):
-        return self.work_folder
-
-    def set_work_folder(self, work_folder):
-        self.work_folder = work_folder
-
-    def get_rotation(self, pageno):
-        return self.rotations.get(pageno, 0)
-
-    def set_rotation(self, pageno, value):
-        self.rotations[pageno] = value
-
-
-class JobContextManager(SyncManager):
-    pass
+    def get_path(self, name):
+        return os.path.join(self.work_folder, "%06d_%s" % (self.pageno + 1, name))
 
 
 def cleanup_working_files(work_folder, options):
     if options.keep_temporary_files:
         print(f"Temporary working files saved at:\n{work_folder}", file=sys.stderr)
     else:
-        with suppress(FileNotFoundError):
-            shutil.rmtree(work_folder)
+        shutil.rmtree(work_folder, ignore_errors=True)
+
+
+class LogNameAdapter(logging.LoggerAdapter):
+    def process(self, msg, kwargs):
+        # return '[%s] %s' % (self.extra['filename'], msg), kwargs
+        return '%s' % (msg,), kwargs
+
+
+class LogNamePageAdapter(logging.LoggerAdapter):
+    def process(self, msg, kwargs):
+        return (
+            #'[%s:%05u] %s' % (self.extra['filename'], self.extra['page'], msg),
+            '%4u: %s' % (self.extra['page'], msg),
+            kwargs,
+        )
+
+
+def make_logger(options=None, prefix='ocrmypdf', filename=None, page=None):
+    log = logging.getLogger(prefix)
+    if filename and page:
+        adapter = LogNamePageAdapter(log, dict(filename=filename, page=page))
+    elif filename:
+        adapter = LogNameAdapter(log, dict(filename=filename))
+    else:
+        adapter = log
+    return adapter

src/ocrmypdf/_plugins.py

@@ -0,0 +1,79 @@
+# © 2019 James R. Barlow: github.com/jbarlow83
+#
+# This file is part of OCRmyPDF.
+#
+# OCRmyPDF is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# OCRmyPDF is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+import importlib
+import os
+import sys
+from pathlib import Path
+
+log = logging.getLogger(__name__)
+
+
+def _load_function_from_module(location):
+    """Load a function given a module location
+
+    For location=a.b.c, will effectively run "from a.b import c"
+
+    Example:
+        _load_function_from_module("a.b.c")
+
+    """
+    module_parts = location.split('.')
+    module_name = '.'.join(module_parts[:-1])
+    object_name = module_parts[-1]
+    module = importlib.import_module(module_name)
+    fn = getattr(module, object_name)
+    log.debug(f"Loaded function: from {module_name} import {object_name}")
+    return fn
+
+
+def _load_function_from_pyfile(location):
+    """Load a function from a file
+
+    Example:
+        _load_function_from_pyfile("test.py::blur_filter")
+    """
+    filename, object_name = location.split('::', maxsplit=1)
+    log.debug(f"Loading function {object_name} from {filename}")
+
+    module_name = Path(filename).stem
+    spec = importlib.util.spec_from_file_location(module_name, filename)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    fn = getattr(module, object_name)
+    return fn
+
+
+def load_plugin(plugin):
+    if callable(plugin):
+        return plugin
+
+    if not isinstance(plugin, str):
+        raise TypeError()
+
+    if '::' not in plugin:
+        plugin = _load_function_from_module(plugin)
+    else:
+        plugin = _load_function_from_pyfile(plugin)
+
+    return plugin
+
+
+def check_plugin_loadable(plugin):
+    load_plugin(plugin)
+    return plugin

src/ocrmypdf/_sync.py

@@ -0,0 +1,370 @@
+# © 2016 James R. Barlow: github.com/jbarlow83
+#
+# This file is part of OCRmyPDF.
+#
+# OCRmyPDF is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# OCRmyPDF is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+import logging.handlers
+import multiprocessing
+import os
+import signal
+import sys
+import threading
+from collections import namedtuple
+from tempfile import mkdtemp
+
+from tqdm import tqdm
+
+from . import __version__
+from ._jobcontext import PDFContext, cleanup_working_files, make_logger
+from ._pipeline import (
+    convert_to_pdfa,
+    copy_final,
+    create_ocr_image,
+    create_pdf_page_from_image,
+    create_visible_page_jpg,
+    generate_postscript_stub,
+    get_orientation_correction,
+    get_pdfinfo,
+    is_ocr_required,
+    merge_sidecars,
+    metadata_fixup,
+    ocr_tesseract_hocr,
+    ocr_tesseract_textonly_pdf,
+    optimize_pdf,
+    preprocess_clean,
+    preprocess_deskew,
+    preprocess_remove_background,
+    rasterize,
+    rasterize_preview,
+    render_hocr_page,
+    should_visible_page_image_use_jpg,
+    triage,
+    validate_pdfinfo_options,
+)
+from ._validation import (
+    check_requested_output_file,
+    create_input_file,
+    report_output_file_size,
+)
+from ._weave import OcrGrafter
+from .exceptions import ExitCode, ExitCodeException
+from .exec import qpdf
+from .helpers import available_cpu_count
+from .pdfa import file_claims_pdfa
+
+PageResult = namedtuple(
+    'PageResult', 'pageno, pdf_page_from_image, ocr, text, orientation_correction'
+)
+
+
+def preprocess(page_context, image, remove_background, deskew, clean):
+    if remove_background:
+        image = preprocess_remove_background(image, page_context)
+    if deskew:
+        image = preprocess_deskew(image, page_context)
+    if clean:
+        image = preprocess_clean(image, page_context)
+    return image
+
+
+def exec_page_sync(page_context):
+    options = page_context.options
+    orientation_correction = 0
+    pdf_page_from_image_out = None
+    ocr_out = None
+    text_out = None
+    if is_ocr_required(page_context):
+        if options.rotate_pages:
+            # Rasterize
+            rasterize_preview_out = rasterize_preview(page_context.origin, page_context)
+            orientation_correction = get_orientation_correction(
+                rasterize_preview_out, page_context
+            )
+
+        rasterize_out = rasterize(
+            page_context.origin,
+            page_context,
+            correction=orientation_correction,
+            remove_vectors=False,
+        )
+
+        if not any([options.clean, options.clean_final, options.remove_vectors]):
+            ocr_image = preprocess_out = preprocess(
+                page_context,
+                rasterize_out,
+                options.remove_background,
+                options.deskew,
+                clean=False,
+            )
+        else:
+            if not options.lossless_reconstruction:
+                preprocess_out = preprocess(
+                    page_context,
+                    rasterize_out,
+                    options.remove_background,
+                    options.deskew,
+                    clean=options.clean_final,
+                )
+            if options.remove_vectors:
+                rasterize_ocr_out = rasterize(
+                    page_context.origin,
+                    page_context,
+                    correction=orientation_correction,
+                    remove_vectors=True,
+                    output_tag='_ocr',
+                )
+            else:
+                rasterize_ocr_out = rasterize_out
+            ocr_image = preprocess(
+                page_context,
+                rasterize_ocr_out,
+                options.remove_background,
+                options.deskew,
+                clean=options.clean,
+            )
+
+        ocr_image_out = create_ocr_image(ocr_image, page_context)
+
+        pdf_page_from_image_out = None
+        if not options.lossless_reconstruction:
+            visible_image_out = preprocess_out
+            if should_visible_page_image_use_jpg(page_context.pageinfo):
+                visible_image_out = create_visible_page_jpg(
+                    visible_image_out, page_context
+                )
+            pdf_page_from_image_out = create_pdf_page_from_image(
+                visible_image_out, page_context
+            )
+
+        if options.pdf_renderer == 'hocr':
+            (hocr_out, text_out) = ocr_tesseract_hocr(ocr_image_out, page_context)
+            ocr_out = render_hocr_page(hocr_out, page_context)
+
+        if options.pdf_renderer == 'sandwich':
+            (ocr_out, text_out) = ocr_tesseract_textonly_pdf(
+                ocr_image_out, page_context
+            )
+
+    return PageResult(
+        pageno=page_context.pageno,
+        pdf_page_from_image=pdf_page_from_image_out,
+        ocr=ocr_out,
+        text=text_out,
+        orientation_correction=orientation_correction,
+    )
+
+
+def post_process(pdf_file, context):
+    pdf_out = pdf_file
+    if context.options.output_type.startswith('pdfa'):
+        ps_stub_out = generate_postscript_stub(context)
+        pdf_out = convert_to_pdfa(pdf_out, ps_stub_out, context)
+
+    pdf_out = metadata_fixup(pdf_out, context)
+    return optimize_pdf(pdf_out, context)
+
+
+def worker_init(queue):
+    """Initialize a process pool worker"""
+
+    # Ignore SIGINT (our parent process will kill us gracefully)
+    signal.signal(signal.SIGINT, signal.SIG_IGN)
+
+    # Reconfigure the root logger for this process to send all messages to a queue
+    h = logging.handlers.QueueHandler(queue)
+    root = logging.getLogger()
+    root.handlers = []
+    root.addHandler(h)
+
+
+def worker_thread_init(queue):
+    pass
+
+
+def log_listener(queue):
+    """Listen to the worker processes and forward the messages to logging
+
+    For simplicity this is a thread rather than a process. Only one process
+    should actually write to sys.stderr or whatever we're using, so if this is
+    made into a process the main application needs to be directed to it.
+
+    See https://docs.python.org/3/howto/logging-cookbook.html#logging-to-a-single-file-from-multiple-processes
+    """
+
+    while True:
+        try:
+            record = queue.get()
+            if record is None:
+                break
+            logger = logging.getLogger(record.name)
+            logger.handle(record)
+        except Exception:
+            import traceback
+
+            print("Logging problem", file=sys.stderr)
+            traceback.print_exc(file=sys.stderr)
+
+
+def exec_concurrent(context):
+    """Execute the pipeline concurrently"""
+
+    # Run exec_page_sync on every page context
+    max_workers = min(len(context.pdfinfo), context.options.jobs)
+    if max_workers > 1:
+        context.log.info("Start processing %d pages concurrent" % max_workers)
+
+    if context.options.use_threads:
+        from multiprocessing.dummy import Pool
+
+        initializer = worker_thread_init
+    else:
+        Pool = multiprocessing.Pool
+        initializer = worker_init
+
+    sidecars = [None] * len(context.pdfinfo)
+    ocrgraft = OcrGrafter(context)
+
+    log_queue = multiprocessing.Queue(-1)
+    listener = threading.Thread(target=log_listener, args=(log_queue,))
+    listener.start()
+    with tqdm(
+        total=(2 * len(context.pdfinfo)),
+        desc='OCR',
+        unit='page',
+        unit_scale=0.5,
+        disable=not context.options.progress_bar,
+    ) as pbar, Pool(
+        processes=max_workers, initializer=initializer, initargs=(log_queue,)
+    ) as pool:
+        results = pool.imap_unordered(exec_page_sync, context.get_page_contexts())
+        while True:
+            try:
+                page_result = results.next()
+                sidecars[page_result.pageno] = page_result.text
+                pbar.update()
+                ocrgraft.graft_page(page_result)
+                pbar.update()
+            except StopIteration:
+                break
+            except (Exception, KeyboardInterrupt):
+                pool.terminate()
+                log_queue.put_nowait(None)  # Terminate log listener
+                # Don't try listener.join() here, will deadlock
+                raise
+
+    log_queue.put_nowait(None)
+    listener.join()
+
+    # Output sidecar text
+    if context.options.sidecar:
+        text = merge_sidecars(sidecars, context)
+        # Copy text file to destination
+        copy_final(text, context.options.sidecar, context)
+
+    # Merge layers to one single pdf
+    # pdf = weave_layers(layers, context)
+    pdf = ocrgraft.finalize()
+
+    # PDF/A and metadata
+    pdf = post_process(pdf, context)
+
+    # Copy PDF file to destination
+    copy_final(pdf, context.options.output_file, context)
+
+
+class NeverRaise(Exception):
+    """An exception that is never raised"""
+
+    pass
+
+
+def run_pipeline(options, api=False):
+    log = make_logger(options, __name__)
+
+    # Any changes to options will not take effect for options that are already
+    # bound to function parameters in the pipeline. (For example
+    # options.input_file, options.pdf_renderer are already bound.)
+    if not options.jobs:
+        options.jobs = available_cpu_count()
+
+    # Performance is improved by setting Tesseract to single threaded. In tests
+    # this gives better throughput than letting a smaller number of Tesseract
+    # jobs run multithreaded. Same story for pngquant. Tess <4 ignores this
+    # variable, but harmless to set if ignored.
+    os.environ.setdefault('OMP_THREAD_LIMIT', '1')
+
+    work_folder = mkdtemp(prefix="com.github.ocrmypdf.")
+    try:
+        check_requested_output_file(options)
+        start_input_file = create_input_file(options, work_folder)
+
+        # Triage image or pdf
+        origin_pdf = triage(
+            start_input_file, os.path.join(work_folder, 'origin.pdf'), options, log
+        )
+
+        # Gather pdfinfo and create context
+        pdfinfo = get_pdfinfo(
+            origin_pdf,
+            detailed_page_analysis=options.redo_ocr,
+            progbar=options.progress_bar,
+        )
+        context = PDFContext(options, work_folder, origin_pdf, pdfinfo)
+
+        # Validate options are okay for this pdf
+        validate_pdfinfo_options(context)
+
+        # Execute the pipeline
+        exec_concurrent(context)
+    except (KeyboardInterrupt if not api else NeverRaise) as e:
+        log.error("KeyboardInterrupt")
+        return ExitCode.ctrl_c
+    except (ExitCodeException if not api else NeverRaise) as e:
+        if str(e):
+            log.error("%s: %s", type(e).__name__, str(e))
+        else:
+            log.error(type(e).__name__)
+        return e.exit_code
+    except (Exception if not api else NeverRaise) as e:
+        log.exception("An exception occurred while executing the pipeline")
+        return ExitCode.other_error
+    finally:
+        cleanup_working_files(work_folder, options)
+
+    if options.output_file == '-':
+        log.info("Output sent to stdout")
+    elif os.path.samefile(options.output_file, os.devnull):
+        pass  # Say nothing when sending to dev null
+    else:
+        if options.output_type.startswith('pdfa'):
+            pdfa_info = file_claims_pdfa(options.output_file)
+            if pdfa_info['pass']:
+                msg = f"Output file is a {pdfa_info['conformance']} (as expected)"
+                log.info(msg)
+            else:
+                msg = (
+                    f"Output file is okay but is not PDF/A "
+                    f"(seems to be {pdfa_info['conformance']})"
+                )
+                log.warning(msg)
+                return ExitCode.pdfa_conversion_failed
+        if not qpdf.check(options.output_file, log):
+            log.warning('Output file: The generated PDF is INVALID')
+            return ExitCode.invalid_output_pdf
+
+        report_output_file_size(options, start_input_file, options.output_file)
+
+    return ExitCode.ok

src/ocrmypdf/_validation.py

@@ -0,0 +1,434 @@
+#!/usr/bin/env python3
+# © 2015-17 James R. Barlow: github.com/jbarlow83
+#
+# This file is part of OCRmyPDF.
+#
+# OCRmyPDF is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# OCRmyPDF is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import logging
+import os
+import sys
+from pathlib import Path
+from shutil import copyfileobj
+
+import PIL
+
+from ._unicodefun import verify_python3_env
+from .exceptions import (
+    BadArgsError,
+    InputFileError,
+    MissingDependencyError,
+    OutputFileAccessError,
+)
+from .exec import (
+    check_external_program,
+    ghostscript,
+    jbig2enc,
+    pngquant,
+    qpdf,
+    tesseract,
+    unpaper,
+)
+from .helpers import is_file_writable, re_symlink
+
+# -------------
+# External dependencies
+
+HOCR_OK_LANGS = frozenset(['eng', 'deu', 'spa', 'ita', 'por'])
+
+log = logging.getLogger(__name__)
+
+
+# --------
+# Critical environment tests
+verify_python3_env()
+
+
+def check_options_languages(options):
+    if not options.language:
+        options.language = ['eng']  # Enforce English hegemony
+
+    # Support v2.x "eng+deu" language syntax
+    if '+' in options.language[0]:
+        options.language = options.language[0].split('+')
+
+    languages = set(options.language)
+    if not languages.issubset(tesseract.languages()):
+        msg = (
+            "The installed version of tesseract does not have language "
+            "data for the following requested languages: \n"
+        )
+        for lang in languages - tesseract.languages():
+            msg += lang + '\n'
+        raise MissingDependencyError(msg)
+
+
+def check_options_output(options):
+    # We have these constraints to check for.
+    # 1. Ghostscript < 9.20 mangles multibyte Unicode
+    # 2. hocr doesn't work on non-Latin languages (so don't select it)
+
+    languages = set(options.language)
+    is_latin = languages.issubset(HOCR_OK_LANGS)
+
+    if options.pdf_renderer == 'hocr' and not is_latin:
+        msg = (
+            "The 'hocr' PDF renderer is known to cause problems with one "
+            "or more of the languages in your document.  Use "
+            "--pdf-renderer auto (the default) to avoid this issue."
+        )
+        log.warning(msg)
+
+    if ghostscript.version() < '9.20' and options.output_type != 'pdf' and not is_latin:
+        # https://bugs.ghostscript.com/show_bug.cgi?id=696874
+        # Ghostscript < 9.20 fails to encode multibyte characters properly
+        msg = (
+            "The installed version of Ghostscript does not work correctly "
+            "with the OCR languages you specified. Use --output-type pdf or "
+            "upgrade to Ghostscript 9.20 or later to avoid this issue."
+        )
+        msg += f"Found Ghostscript {ghostscript.version()}"
+        log.warning(msg)
+
+    # Decide on what renderer to use
+    if options.pdf_renderer == 'auto':
+        options.pdf_renderer = 'sandwich'
+
+    if options.pdf_renderer == 'sandwich' and not tesseract.has_textonly_pdf(
+        options.tesseract_env
+    ):
+        raise MissingDependencyError(
+            "You are using an alpha version of Tesseract 4.0 that does not support "
+            "the textonly_pdf parameter. We don't support versions this old."
+        )
+
+    if options.output_type == 'pdfa':
+        options.output_type = 'pdfa-2'
+
+    if options.output_type == 'pdfa-3' and ghostscript.version() < '9.19':
+        raise MissingDependencyError(
+            "--output-type pdfa-3 requires Ghostscript 9.19 or later"
+        )
+
+    lossless_reconstruction = False
+    if not any(
+        (
+            options.deskew,
+            options.clean_final,
+            options.force_ocr,
+            options.remove_background,
+        )
+    ):
+        lossless_reconstruction = True
+    options.lossless_reconstruction = lossless_reconstruction
+
+    if not options.lossless_reconstruction and options.redo_ocr:
+        raise BadArgsError(
+            "--redo-ocr is not currently compatible with --deskew, "
+            "--clean-final, and --remove-background"
+        )
+
+
+def check_options_sidecar(options):
+    if options.sidecar == '\0':
+        if options.output_file == '-':
+            raise BadArgsError(
+                "--sidecar filename must be specified when output file is stdout."
+            )
+        options.sidecar = options.output_file + '.txt'
+
+
+def check_options_preprocessing(options):
+    if options.clean_final:
+        options.clean = True
+    if options.unpaper_args and not options.clean:
+        raise BadArgsError("--clean is required for --unpaper-args")
+    if options.clean:
+        check_external_program(
+            program='unpaper',
+            package='unpaper',
+            version_checker=unpaper.version,
+            need_version='6.1',
+            required_for=['--clean, --clean-final'],
+        )
+        try:
+            if options.unpaper_args:
+                options.unpaper_args = unpaper.validate_custom_args(
+                    options.unpaper_args
+                )
+        except Exception as e:
+            raise BadArgsError(str(e))
+
+
+def check_options_ocr_behavior(options):
+    exclusive_options = sum(
+        [
+            (1 if opt else 0)
+            for opt in (options.force_ocr, options.skip_text, options.redo_ocr)
+        ]
+    )
+    if exclusive_options >= 2:
+        raise BadArgsError(
+            "Error: choose only one of --force-ocr, --skip-text, --redo-ocr."
+        )
+
+
+def check_options_optimizing(options):
+    if options.optimize >= 2:
+        check_external_program(
+            program='pngquant',
+            package='pngquant',
+            version_checker=pngquant.version,
+            need_version='2.0.1',
+            required_for='--optimize {2,3}',
+        )
+
+    if options.optimize >= 2:
+        # Although we use JBIG2 for optimize=1, don't nag about it unless the
+        # user is asking for more optimization
+        check_external_program(
+            program='jbig2',
+            package='jbig2enc',
+            version_checker=jbig2enc.version,
+            need_version='0.28',
+            required_for='--optimize {2,3} | --jbig2-lossy',
+            recommended=True if not options.jbig2_lossy else False,
+        )
+
+    if options.optimize == 0 and any(
+        [options.jbig2_lossy, options.png_quality, options.jpeg_quality]
+    ):
+        log.warning(
+            "The arguments --jbig2-lossy, --png-quality, and --jpeg-quality "
+            "will be ignored because --optimize=0."
+        )
+
+
+def check_options_advanced(options):
+    if options.pdfa_image_compression != 'auto' and options.output_type.startswith(
+        'pdfa'
+    ):
+        log.warning(
+            "--pdfa-image-compression argument has no effect when "
+            "--output-type is not 'pdfa', 'pdfa-1', or 'pdfa-2'"
+        )
+    if tesseract.v4(options.tesseract_env) and (
+        options.user_words or options.user_patterns
+    ):
+        log.warning('Tesseract 4.x ignores --user-words, so this has no effect')
+
+
+def check_options_metadata(options):
+    import unicodedata
+
+    docinfo = [options.title, options.author, options.keywords, options.subject]
+    for s in (m for m in docinfo if m):
+        for c in s:
+            if unicodedata.category(c) == 'Co' or ord(c) >= 0x10000:
+                raise ValueError(
+                    "One of the metadata strings contains "
+                    "an unsupported Unicode character: '{}' (U+{})".format(
+                        c, hex(ord(c))[2:].upper()
+                    )
+                )
+
+
+def check_options_pillow(options):
+    PIL.Image.MAX_IMAGE_PIXELS = int(options.max_image_mpixels * 1_000_000)
+    if PIL.Image.MAX_IMAGE_PIXELS == 0:
+        PIL.Image.MAX_IMAGE_PIXELS = None
+
+
+def check_options(options):
+    check_options_languages(options)
+    check_options_metadata(options)
+    check_options_output(options)
+    check_options_sidecar(options)
+    check_options_preprocessing(options)
+    check_options_ocr_behavior(options)
+    check_options_optimizing(options)
+    check_options_advanced(options)
+    check_options_pillow(options)
+    check_dependency_versions(options)
+    check_environ(options)
+
+
+def check_closed_streams(options):
+    """Work around Python issue with multiprocessing forking on closed streams
+
+    https://bugs.python.org/issue28326
+
+    Attempting to a fork/exec a new Python process when any of std{in,out,err}
+    are closed or not flushable for some reason may raise an exception.
+    Fix this by opening devnull if the handle seems to be closed.  Do this
+    globally to avoid tracking places all places that fork.
+
+    Seems to be specific to multiprocessing.Process not all Python process
+    forkers.
+
+    The error actually occurs when the stream object is not flushable,
+    but replacing an open stream object that is not flushable with
+    /dev/null is a bad idea since it will create a silent failure.  Replacing
+    a closed handle with /dev/null seems safe.
+
+    """
+
+    if sys.version_info[0:3] >= (3, 6, 4):
+        return True  # Issued fixed in Python 3.6.4+
+
+    if sys.stderr is None:
+        sys.stderr = open(os.devnull, 'w')
+
+    if sys.stdin is None:
+        if options.input_file == '-':
+            log.error("Trying to read from stdin but stdin seems closed")
+            return False
+        sys.stdin = open(os.devnull, 'r')
+
+    if sys.stdout is None:
+        if options.output_file == '-':
+            # Can't replace stdout if the user is piping
+            # If this case can even happen, it must be some kind of weird
+            # stream.
+            log.error(
+                "Output was set to stdout '-' but the stream attached to "
+                "stdout does not support the flush() system call.  This "
+                "will fail."
+            )
+            return False
+        sys.stdout = open(os.devnull, 'w')
+
+    return True
+
+
+def log_page_orientations(pdfinfo):
+    direction = {0: 'n', 90: 'e', 180: 's', 270: 'w'}
+    orientations = []
+    for n, page in enumerate(pdfinfo):
+        angle = page.rotation or 0
+        if angle != 0:
+            orientations.append('{0}{1}'.format(n + 1, direction.get(angle, '')))
+    if orientations:
+        log.info('Page orientations detected: %s', ' '.join(orientations))
+
+
+def check_environ(options):
+    old_envvars = (
+        'OCRMYPDF_TESSERACT',
+        'OCRMYPDF_QPDF',
+        'OCRMYPDF_GS',
+        'OCRMYPDF_UNPAPER',
+    )
+    for k in old_envvars:
+        if k in os.environ:
+            log.warning(
+                "OCRmyPDF no longer uses the environment variable {k}."
+                "Change PATH to select alternate programs."
+            )
+
+
+def create_input_file(options, work_folder):
+    if options.input_file == '-':
+        # stdin
+        log.info('reading file from standard input')
+        target = os.path.join(work_folder, 'stdin')
+        with open(target, 'wb') as stream_buffer:
+            copyfileobj(sys.stdin.buffer, stream_buffer)
+        return target
+    else:
+        try:
+            target = os.path.join(work_folder, 'origin')
+            re_symlink(options.input_file, target)
+            return target
+        except FileNotFoundError:
+            raise InputFileError(f"File not found - {options.input_file}")
+
+
+def check_requested_output_file(options):
+    if options.output_file == '-':
+        if sys.stdout.isatty():
+            raise BadArgsError(
+                "Output was set to stdout '-' but it looks like stdout "
+                "is connected to a terminal.  Please redirect stdout to a "
+                "file."
+            )
+    elif not is_file_writable(options.output_file):
+        raise OutputFileAccessError(
+            f"Output file location ({options.output_file}) is not a writable file."
+        )
+
+
+def report_output_file_size(options, input_file, output_file):
+    try:
+        output_size = Path(output_file).stat().st_size
+        input_size = Path(input_file).stat().st_size
+    except FileNotFoundError:
+        return  # Outputting to stream or something
+    ratio = output_size / input_size
+    if ratio < 1.35 or input_size < 25000:
+        return  # Seems fine
+
+    reasons = []
+    image_preproc = {
+        'deskew',
+        'clean_final',
+        'remove_background',
+        'oversample',
+        'force_ocr',
+    }
+    for arg in image_preproc:
+        attr = getattr(options, arg, None)
+        if not attr:
+            continue
+        reasons.append(
+            f"The argument --{arg.replace('_', '-')} was issued, causing transcoding."
+        )
+
+    if reasons:
+        explanation = "Possible reasons for this include:\n" + '\n'.join(reasons) + "\n"
+    else:
+        explanation = "No reason for this increase is known.  Please report this issue."
+
+    log.warning(
+        f"The output file size is {ratio:.2f}× larger than the input file.\n"
+        f"{explanation}"
+    )
+
+
+def check_dependency_versions(options):
+    check_external_program(
+        program='tesseract',
+        package={'darwin': 'tesseract', 'linux': 'tesseract-ocr'},
+        version_checker=tesseract.version,
+        need_version='4.0.0',  # using backport for Travis CI
+    )
+    check_external_program(
+        program='gs',
+        package='ghostscript',
+        version_checker=ghostscript.version,
+        need_version='9.15',  # limited by Travis CI / Ubuntu 14.04 backports
+    )
+    if ghostscript.version() == '9.24':
+        raise MissingDependencyError(
+            "Ghostscript 9.24 contains serious regressions and is not "
+            "supported. Please upgrade to Ghostscript 9.25 or use an older "
+            "version."
+        )
+    check_external_program(
+        program='qpdf',
+        package='qpdf',
+        version_checker=qpdf.version,
+        need_version='8.0.2',
+    )

src/ocrmypdf/_weave.py

@@ -15,17 +15,15 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
-from itertools import groupby
-from pathlib import Path
 import os
+from contextlib import suppress
+from pathlib import Path
 
 import pikepdf
 
 from .exec import tesseract
-from .helpers import flatten_groups, page_number
 
-
-MAX_OPEN_PAGE_PDFS = int(os.environ.get('_OCRMYPDF_MAX_OPEN_PAGE_PDFS', 100))
+MAX_REPLACE_PAGES = int(os.environ.get('_OCRMYPDF_MAX_REPLACE_PAGES', 100))
 
 
 def _update_page_resources(*, page, font, font_key, procset):
@@ -47,7 +45,7 @@ def _update_page_resources(*, page, font, font_key, procset):
     resources['/ProcSet'] = procset
 
 
-def strip_invisible_text(pdf, page, log):
+def strip_invisible_text(pdf, page):
     stream = []
     in_text_obj = False
     render_mode = 0
@@ -105,16 +103,6 @@ def _weave_layers_graft(
     pdf_text = pikepdf.open(text)
     pdf_text_contents = pdf_text.pages[0].Contents.read_bytes()
 
-    if not tesseract.has_textonly_pdf():
-        # If we don't have textonly_pdf, edit the stream to delete the
-        # instruction to draw the image Tesseract generated, which we do not
-        # use.
-        stream = bytearray(pdf_text_contents)
-        pattern = b'/Im1 Do'
-        idx = stream.find(pattern)
-        stream[idx : (idx + len(pattern))] = b' ' * len(pattern)
-        pdf_text_contents = bytes(stream)
-
     base_page = pdf_base.pages.p(page_num)
 
     # The text page always will be oriented up by this stage but the original
@@ -142,7 +130,7 @@ def _weave_layers_graft(
     scale_x = wp / wt
     scale_y = hp / ht
 
-    log.debug('%r', (scale_x, scale_y))
+    # log.debug('%r', scale_x, scale_y)
     scale = pikepdf.PdfMatrix().scaled(scale_x, scale_y)
 
     # Translate the text so it is centered at (0, 0), rotate it there, adjust
@@ -154,13 +142,14 @@ def _weave_layers_graft(
     new_text_layer = pikepdf.Stream(pdf_base, pdf_text_contents)
 
     if strip_old_text:
-        strip_invisible_text(pdf_base, base_page, log)
+        strip_invisible_text(pdf_base, base_page)
 
     base_page.page_contents_add(new_text_layer, prepend=True)
 
     _update_page_resources(
         page=base_page, font=font, font_key=font_key, procset=procset
     )
+    pdf_text.close()
 
 
 def _find_font(text, pdf_base):
@@ -169,251 +158,126 @@ def _find_font(text, pdf_base):
     font, font_key = None, None
     possible_font_names = ('/f-0-0', '/F1')
     try:
-        pdf_text = pikepdf.open(text)
-        pdf_text_fonts = pdf_text.pages[0].Resources.get('/Font', {})
-    except Exception:
+        with pikepdf.open(text) as pdf_text:
+            try:
+                pdf_text_fonts = pdf_text.pages[0].Resources.get('/Font', {})
+            except (AttributeError, IndexError, KeyError):
+                return None, None
+            for f in possible_font_names:
+                pdf_text_font = pdf_text_fonts.get(f, None)
+                if pdf_text_font is not None:
+                    font_key = f
+                    break
+            if pdf_text_font:
+                font = pdf_base.copy_foreign(pdf_text_font)
+            return font, font_key
+    except (FileNotFoundError, pikepdf.PdfError):
+        # PdfError occurs if a 0-length file is written e.g. due to OCR timeout
         return None, None
 
-    for f in possible_font_names:
-        pdf_text_font = pdf_text_fonts.get(f, None)
-        if pdf_text_font is not None:
-            font_key = f
-            break
-    if pdf_text_font:
-        font = pdf_base.copy_foreign(pdf_text_font)
-    return font, font_key
 
+class OcrGrafter:
+    def __init__(self, context):
+        self.context = context
+        self.log = context.log
+        self.path_base = Path(context.origin).resolve()
 
-def _traverse_toc(pdf_base, visitor_fn, log):
-    """
-    Walk the table of contents, calling visitor_fn() at each node
+        self.pdf_base = pikepdf.open(self.path_base)
+        self.font, self.font_key = None, None
 
-    The /Outlines data structure is a messy data structure, but rather than
-    navigating hierarchically we just track unique nodes.  Enqueue nodes when
-    we find them, and never visit them again.  set() is awesome.  We look for
-    the two types of object in the table of contents that can be page bookmarks
-    and update the page entry.
+        self.pdfinfo = context.pdfinfo
+        self.output_file = context.get_path('weave_layers.pdf')
 
-    """
+        self.procset = self.pdf_base.make_indirect(
+            pikepdf.Object.parse(b'[ /PDF /Text /ImageB /ImageC /ImageI ]')
+        )
 
-    visited = set()
-    queue = set()
-    link_keys = ('/Parent', '/First', '/Last', '/Prev', '/Next')
+        self.emplacements = 1
+        self.interim_count = 0
 
-    if not '/Outlines' in pdf_base.root:
-        return
-
-    queue.add(pdf_base.root.Outlines.objgen)
-    while queue:
-        objgen = queue.pop()
-        visited.add(objgen)
-        node = pdf_base.get_object(objgen)
-        log.debug('fix toc: exploring outline entries at %r', objgen)
-
-        # Enumerate other nodes we could visit from here
-        for key in link_keys:
-            if key not in node:
-                continue
-            item = node[key]
-            if not item.is_indirect:
-                # Direct references are not allowed here, but it's not clear
-                # what we should do if we find any. Removing them is an option:
-                # node[key] = pdf_base.make_indirect(None)
-                continue
-            objgen = item.objgen
-            if objgen not in visited:
-                queue.add(objgen)
-
-        if visitor_fn:
-            visitor_fn(pdf_base, node, log)
-
-
-def _fix_toc(pdf_base, pageref_remap, log):
-    """Repair the table of contents
-
-    Whenever we replace a page wholesale, it gets assigned a new objgen number
-    and other references to it within the PDF become invalid, most notably in
-    the table of contents (/Outlines in PDF-speak).  In weave_layers we collect
-    pageref_remap, a mapping that describes the new objgen number given an old
-    one.  (objgen is a tuple, and the gen is almost always zero.)
-
-    It may ultimately be better to find a way to rebuild a page in place.
-
-    """
-
-    if not pageref_remap:
-        return
-
-    def remap_dest(dest_node):
-        """
-        Inner helper function: change the objgen for any page from the old we
-        invalidated to its new one.
-        """
-        try:
-            pageref = dest_node[0]
-            if pageref['/Type'] == '/Page' and pageref.objgen in pageref_remap:
-                new_objgen = pageref_remap[pageref.objgen]
-                dest_node[0] = pdf_base.get_object(new_objgen)
-        except (IndexError, TypeError) as e:
-            log.warning("This file may contain invalid table of contents entries")
-            log.debug(e)
-
-    def visit_remap_dest(pdf_base, node, log):
-        """
-        Visitor function to fix ToC entries
-
-        Test for the two types of references to pages that can occur in ToCs.
-        Both types have the same final format (an indirect reference to the
-        target page).
-        """
-        if '/Dest' in node:
-            # /Dest reference to another page (old method)
-            remap_dest(node['/Dest'])
-        elif '/A' in node:
-            # /A (action) command set to "GoTo" (newer method)
-            if '/S' in node['/A'] and node['/A']['/S'] == '/GoTo':
-                remap_dest(node['/A']['/D'])
-
-    _traverse_toc(pdf_base, visit_remap_dest, log)
-
-
-def weave_layers(infiles, output_file, log, context):
-    """Apply text layer and/or image layer changes to baseline file
-
-    This is where the magic happens. infiles will be the main PDF to modify,
-    and optional .text.pdf and .image-layer.pdf files, organized however ruffus
-    organizes them.
-
-    From .text.pdf, we copy the content stream (which contains the Tesseract
-    OCR results), and rotate it into place. The first time we do this, we also
-    copy the GlyphlessFont, and then reference that font again.
-
-    For .image-layer.pdf, we check if this is a "pointer" to the original file,
-    or a new file. If a new file, we replace the page and remember that we
-    replaced this page.
-
-    Every 100 open files, we save intermediate results, to avoid any resource
-    limits, since pikepdf/qpdf need to keep a lot of open file handles in the
-    background. When objects are copied from one file to another qpdf, qpdf
-    doesn't actually copy the data until asked to write, so all the resources
-    it may need to remain available.
-
-    For completeness, we set up a /ProcSet on every page, although it's
-    unlikely any PDF viewer cares about this anymore.
-
-    """
-
-    def input_sorter(key):
-        try:
-            return page_number(key)
-        except ValueError:
-            return -1
-
-    flat_inputs = sorted(flatten_groups(infiles), key=input_sorter)
-    groups = groupby(flat_inputs, key=input_sorter)
-
-    # Extract first item
-    _, basegroup = next(groups)
-    base = list(basegroup)[0]
-    path_base = Path(base).resolve()
-    pdf_base = pikepdf.open(path_base)
-    keep_open = []
-    font, font_key, procset = None, None, None
-    pdfinfo = context.get_pdfinfo()
-    pagerefs = {}
-
-    # Walk the table of contents first, to trigger pikepdf/qpdf to resolve all
-    # page references in the table of contents. Some PDF generators put invalid
-    # references in the ToC, so we want to resolve them to null before we
-    # create any references, or the ToC will be corrupted
-    _traverse_toc(pdf_base, None, log)
-
-    procset = pdf_base.make_indirect(
-        pikepdf.Object.parse(b'[ /PDF /Text /ImageB /ImageC /ImageI ]')
-    )
-
-    # Iterate rest
-    for page_num, layers in groups:
-        layers = list(layers)
-        log.debug(page_num)
-        log.debug(layers)
-
-        text = next((ii for ii in layers if ii.endswith('.text.pdf')), None)
-        image = next((ii for ii in layers if ii.endswith('.image-layer.pdf')), None)
-
-        if text and not font:
-            font, font_key = _find_font(text, pdf_base)
-
-        replacing = False
-        content_rotation = pdfinfo[page_num - 1].rotation
+    def graft_page(self, page_result):
+        pageno, image, text, _sidecar, autorotate_correction = page_result
+        if text and not self.font:
+            self.font, self.font_key = _find_font(text, self.pdf_base)
 
+        emplaced_page = False
+        content_rotation = self.pdfinfo[pageno].rotation
         path_image = Path(image).resolve() if image else None
-        if path_image is not None and path_image != path_base:
-            # We are replacing the old page with a rasterized PDF of the new
-            # page
-            log.debug("Replace")
-            old_objgen = pdf_base.pages[page_num - 1].objgen
+        if path_image is not None and path_image != self.path_base:
+            # We are updating the old page with a rasterized PDF of the new
+            # page (without changing objgen, to preserve references)
+            self.log.debug("Emplacement update")
+            with pikepdf.open(image) as pdf_image:
+                self.emplacements += 1
+                foreign_image_page = pdf_image.pages[0]
+                self.pdf_base.pages.append(foreign_image_page)
+                local_image_page = self.pdf_base.pages[-1]
+                self.pdf_base.pages[pageno].emplace(local_image_page)
+                del self.pdf_base.pages[-1]
+            emplaced_page = True
 
-            pdf_image = pikepdf.open(image)
-            keep_open.append(pdf_image)
-            image_page = pdf_image.pages[0]
-            pdf_base.pages[page_num - 1] = image_page
-
-            # We're adding a new page, which will get a new objgen number pair,
-            # so we need to update any references to it.  qpdf did not like
-            # my attempt to update the old object in place, but that is an
-            # option to consider
-            pagerefs[old_objgen] = pdf_base.pages[page_num - 1].objgen
-            replacing = True
-
-        autorotate_correction = context.get_rotation(page_num - 1)
-        if replacing:
+        if emplaced_page:
             content_rotation = autorotate_correction
         text_rotation = autorotate_correction
         text_misaligned = (text_rotation - content_rotation) % 360
-        log.debug(
+        self.log.debug(
             '%r',
             [text_rotation, autorotate_correction, text_misaligned, content_rotation],
         )
 
-        if text and font:
+        if text and self.font:
             # Graft the text layer onto this page, whether new or old
-            strip_old = context.get_options().redo_ocr
+            strip_old = self.context.options.redo_ocr
             _weave_layers_graft(
-                pdf_base=pdf_base,
-                page_num=page_num,
+                pdf_base=self.pdf_base,
+                page_num=pageno + 1,
                 text=text,
-                font=font,
-                font_key=font_key,
+                font=self.font,
+                font_key=self.font_key,
                 rotation=text_misaligned,
-                procset=procset,
+                procset=self.procset,
                 strip_old_text=strip_old,
-                log=log,
+                log=self.log,
             )
 
         # Correct the rotation if applicable
-        pdf_base.pages[page_num - 1].Rotate = (
+        self.pdf_base.pages[pageno].Rotate = (
             content_rotation - autorotate_correction
         ) % 360
 
-        if len(keep_open) > MAX_OPEN_PAGE_PDFS:
-            # qpdf limitations require us to keep files open when we intend
-            # to copy content from them before saving. However, we want to keep
-            # a lid on file handles and memory usage, so for big files we're
-            # going to stop and save periodically. Attach the font to page 1
-            # even if page 1 doesn't use it, so we have a way to get it back.
-            page0 = pdf_base.pages[0]
-            _update_page_resources(
-                page=page0, font=font, font_key=font_key, procset=procset
-            )
-            interim = output_file + f'_working{page_num}.pdf'
-            pdf_base.save(interim)
-            del pdf_base
-            keep_open = []
+        if self.emplacements % MAX_REPLACE_PAGES == 0:
+            self.save_and_reload()
 
-            pdf_base = pikepdf.open(interim)
-            procset = pdf_base.pages[0].Resources.ProcSet
-            font, font_key = None, None  # Reacquire this information
+    def save_and_reload(self):
+        # Periodically save and reload the Pdf object. This will keep a
+        # lid on our memory usage for very large files. Attach the font to
+        # page 1 even if page 1 doesn't use it, so we have a way to get it
+        # back.
+        page0 = self.pdf_base.pages[0]
+        _update_page_resources(
+            page=page0, font=self.font, font_key=self.font_key, procset=self.procset
+        )
 
-    _fix_toc(pdf_base, pagerefs, log)
-    pdf_base.save(output_file)
+        # We cannot read and write the same file, that will corrupt it
+        # but we don't to keep more copies than we need to. Delete intermediates.
+        # {interim_count} is the opened file we were updateing
+        # {interim_count - 1} can be deleted
+        # {interim_count + 1} is the new file will produce and open
+        old_file = self.output_file + f'_working{self.interim_count - 1}.pdf'
+        if not self.context.options.keep_temporary_files:
+            with suppress(FileNotFoundError):
+                os.unlink(old_file)
+
+        next_file = self.output_file + f'_working{self.interim_count + 1}.pdf'
+        self.pdf_base.save(next_file)
+        self.pdf_base.close()
+
+        self.pdf_base = pikepdf.open(next_file)
+        self.procset = self.pdf_base.pages[0].Resources.ProcSet
+        self.font, self.font_key = None, None  # Ensure we reacquire this information
+        self.interim_count += 1
+
+    def finalize(self):
+        self.pdf_base.save(self.output_file)
+        self.pdf_base.close()
+        return self.output_file

src/ocrmypdf/api.py

@@ -0,0 +1,243 @@
+# © 2019 James R. Barlow: github.com/jbarlow83
+#
+# This file is part of OCRmyPDF.
+#
+# OCRmyPDF is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# OCRmyPDF is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+import os
+import sys
+from enum import IntEnum
+from pathlib import Path
+
+from tqdm import tqdm
+
+from .cli import parser
+from ._sync import run_pipeline
+from ._validation import check_options
+
+
+class TqdmConsole:
+    """Wrapper to log messages in a way that is compatible with tqdm progress bar"""
+
+    def __init__(self, file):
+        self.file = file
+        self.py36 = sys.version_info >= (3, 6)
+
+    def write(self, msg):
+        # When no progress bar is active, tqdm.write() routes to print()
+        if self.py36:
+            if msg.strip() != '':
+                tqdm.write(msg.rstrip(), end='\n', file=self.file)
+        else:
+            tqdm.write(msg.rstrip(), end='\n', file=self.file)
+
+    def flush(self):
+        if hasattr(self.file, "flush"):
+            self.file.flush()
+
+
+class Verbosity(IntEnum):
+    """Verbosity level for configure_logging."""
+
+    quiet = -1  #: Suppress most messages
+    default = 0  #: Default level of logging
+    debug = 1  #: Output ocrmypdf debug messages
+    debug_all = 2  #: More detailed debugging from ocrmypdf and dependent modules
+
+
+def configure_logging(verbosity, progress_bar_friendly=True, manage_root_logger=False):
+    """Set up logging.
+
+    Library users may wish to use this function if they want their log output to be
+    similar to ocrmypdf command line interface. If not used, the external application
+    should configure logging on its own.
+
+    ocrmypdf will perform all of its logging under the `"ocrmypdf"` logging namespace.
+    In addition, ocrmypdf imports pdfminer, which logs under `"pdfminer"`. A library
+    user may wish to configure both; note that pdfminer is extremely chatty at the log
+    level logging.INFO.
+
+    Library users may perform additional configuration afterwards.
+
+    Args:
+        verbosity (Verbosity): Verbosity level.
+        progress_bar_friendly (bool): Install the TqdmConsole log handler, which is
+            compatible with the tqdm progress bar; without this log messages will
+            overwrite the progress bar
+        manage_root_logger (bool): Configure the process's root logger, to ensure
+            all log output is sent through
+    """
+
+    prefix = '' if manage_root_logger else 'ocrmypdf'
+    log = logging.getLogger(prefix)
+    log.setLevel(logging.INFO)
+
+    if progress_bar_friendly:
+        console = logging.StreamHandler(stream=TqdmConsole(sys.stderr))
+    else:
+        console = logging.StreamHandler(stream=sys.stderr)
+
+    if verbosity < 0:
+        console.setLevel(logging.ERROR)
+    elif verbosity >= 1:
+        console.setLevel(logging.DEBUG)
+    else:
+        console.setLevel(logging.INFO)
+
+    formatter = logging.Formatter('%(levelname)7s - %(message)s')
+    if verbosity >= 1:
+        log.setLevel(logging.DEBUG)
+    if verbosity >= 2:
+        formatter = logging.Formatter('%(name)s - %(levelname)7s - %(message)s')
+
+    console.setFormatter(formatter)
+    log.addHandler(console)
+
+    if verbosity <= 1:
+        pdfminer_log = logging.getLogger('pdfminer')
+        pdfminer_log.setLevel(logging.ERROR)
+        pil_log = logging.getLogger('PIL')
+        pil_log.setLevel(logging.INFO)
+
+
+def create_options(*, input_file, output_file, **kwargs):
+    cmdline = []
+    deferred = []
+
+    for arg, val in kwargs.items():
+        if val is None:
+            continue
+        if (arg.startswith('plugin') or arg.startswith('filter')) and (
+            callable(val) or isinstance(val, str)
+        ):
+            deferred.append((arg, val))
+            continue
+        elif arg == 'tesseract_env':
+            deferred.append((arg, val))
+            continue
+        cmd_style_arg = arg.replace('_', '-')
+        cmdline.append(f"--{cmd_style_arg}")
+        if isinstance(val, bool):
+            continue
+        if isinstance(val, (int, float)):
+            cmdline.append(str(val))
+        elif isinstance(val, str):
+            cmdline.append(val)
+        elif isinstance(val, Path):
+            cmdline.append(str(val))
+        else:
+            raise TypeError(f"{arg}: {val} ({type(val)})")
+
+    cmdline.append(str(input_file))
+    cmdline.append(str(output_file))
+
+    parser.api_mode = True
+    options = parser.parse_args(cmdline)
+    for keyword, val in deferred:
+        setattr(options, keyword, val)
+
+    # If we are running a Tesseract spoof, ensure it knows what the input file is
+    if os.environ.get('PYTEST_CURRENT_TEST') and options.tesseract_env:
+        options.tesseract_env['_OCRMYPDF_TEST_INFILE'] = input_file
+
+    return options
+
+
+def ocrmypdf(  # pylint: disable=unused-argument
+    input_file,
+    output_file,
+    *,
+    language=None,
+    image_dpi=None,
+    output_type=None,
+    sidecar=None,
+    jobs=None,
+    use_threads=None,
+    title=None,
+    author=None,
+    subject=None,
+    keywords=None,
+    rotate_pages=None,
+    remove_background=None,
+    deskew=None,
+    clean=None,
+    clean_final=None,
+    unpaper_args=None,
+    oversample=None,
+    remove_vectors=None,
+    mask_barcodes=None,
+    threshold=None,
+    force_ocr=None,
+    skip_text=None,
+    redo_ocr=None,
+    skip_big=None,
+    optimize=None,
+    jpg_quality=None,
+    png_quality=None,
+    jbig2_lossy=None,
+    jbig2_page_group_size=None,
+    max_image_mpixels=None,
+    tesseract_config=None,
+    tesseract_pagesegmode=None,
+    tesseract_oem=None,
+    pdf_renderer=None,
+    tesseract_timeout=None,
+    rotate_pages_threshold=None,
+    pdfa_image_compression=None,
+    user_words=None,
+    user_patterns=None,
+    keep_temporary_files=None,
+    progress_bar=None,
+    filter_ocr_image=None,
+    plugin_validation=None,
+    tesseract_env=None,
+):
+    """Run OCRmyPDF on one PDF or image.
+
+    For most arguments, see documentation for the equivalent command line parameter.
+    A few specific arguments are discussed here:
+
+    Args:
+        use_threads (bool): Use worker threads instead of processes. This reduces
+            performance but may make debugging easier since it is easier to set
+            breakpoints.
+        tesseract_env (dict): Override environment variables for Tesseract
+    Raises:
+        ocrmypdf.PdfMergeFailedError: If the input PDF is malformed, preventing merging
+            with the OCR layer.
+        ocrmypdf.MissingDependencyError: If a required dependency program is missing or
+            was not found on PATH.
+        ocrmypdf.UnsupportedImageFormatError: If the input file type was an image that
+            could not be read, or some other file type that is not a PDF.
+        ocrmypdf.DpiError: If the input file is an image, but the resolution of the
+            image is not credible (allowing it to proceed would cause poor OCR).
+        ocrmypdf.OutputFileAccessError: If an attempt to write to the intended output
+            file failed.
+        ocrmypdf.PriorOcrFoundError: If the input PDF seems to have OCR or digital
+            text already, and settings did not tell us to proceed.
+        ocrmypdf.InputFileError: Any other problem with the input file.
+        ocrmypdf.SubprocessOutputError: Any error related to executing a subprocess.
+        ocrmypdf.EncryptedPdfERror: If the input PDF is encrypted (password protected).
+            OCRmyPDF does not remove passwords.
+        ocrmypdf.TesseractConfigError: If Tesseract reported its configuration was not
+            valid.
+
+    Returns:
+        :class:`ocrmypdf.ExitCode`
+    """
+
+    options = create_options(**locals())
+    check_options(options)
+    return run_pipeline(options, api=True)

src/ocrmypdf/cli.py

@@ -0,0 +1,488 @@
+# © 2015-19 James R. Barlow: github.com/jbarlow83
+#
+# This file is part of OCRmyPDF.
+#
+# OCRmyPDF is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# OCRmyPDF is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
+
+import argparse
+
+from . import PROGRAM_NAME, VERSION
+from ._plugins import check_plugin_loadable
+
+
+def numeric(basetype, min_=None, max_=None):
+    """Validator for numeric params"""
+    min_ = basetype(min_) if min_ is not None else None
+    max_ = basetype(max_) if max_ is not None else None
+
+    def _numeric(string):
+        value = basetype(string)
+        if (min_ is not None and value < min_) or (max_ is not None and value > max_):
+            msg = "%r not in valid range %r" % (string, (min_, max_))
+            raise argparse.ArgumentTypeError(msg)
+        return value
+
+    _numeric.__name__ = basetype.__name__
+    return _numeric
+
+
+class ArgumentParser(argparse.ArgumentParser):
+    """Override parser's default behavior of calling sys.exit()
+
+    https://stackoverflow.com/questions/5943249/python-argparse-and-controlling-overriding-the-exit-status-code
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.api_mode = False
+
+    def error(self, message):
+        if not self.api_mode:
+            super().error(message)
+            return
+        raise ValueError(message)
+
+
+parser = ArgumentParser(
+    prog=PROGRAM_NAME,
+    fromfile_prefix_chars='@',
+    formatter_class=argparse.RawDescriptionHelpFormatter,
+    description="""\
+Generates a searchable PDF or PDF/A from a regular PDF.
+
+OCRmyPDF rasterizes each page of the input PDF, optionally corrects page
+rotation and performs image processing, runs the Tesseract OCR engine on the
+image, and then creates a PDF from the OCR information.
+""",
+    epilog="""\
+OCRmyPDF attempts to keep the output file at about the same size.  If a file
+contains losslessly compressed images, and output file will be losslessly
+compressed as well.
+
+PDF is a page description file that attempts to preserve a layout exactly.
+A PDF can contain vector objects (such as text or lines) and raster objects
+(images).  A page might have multiple images.  OCRmyPDF is prepared to deal
+with the wide variety of PDFs that exist in the wild.
+
+When a PDF page contains text, OCRmyPDF assumes that the page has already
+been OCRed or is a "born digital" page that should not be OCRed.  The default
+behavior is to exit in this case without producing a file.  You can use the
+option --skip-text to ignore pages with text, or --force-ocr to rasterize
+all objects on the page and produce an image-only PDF as output.
+
+    ocrmypdf --skip-text file_with_some_text_pages.pdf output.pdf
+
+    ocrmypdf --force-ocr word_document.pdf output.pdf
+
+If you are concerned about long-term archiving of PDFs, use the default option
+--output-type pdfa which converts the PDF to a standardized PDF/A-2b.  This
+converts images to sRGB colorspace, removes some features from the PDF such
+as Javascript or forms. If you want to minimize the number of changes made to
+your PDF, use --output-type pdf.
+
+If OCRmyPDF is given an image file as input, it will attempt to convert the
+image to a PDF before processing.  For more control over the conversion of
+images to PDF, use the Python package img2pdf or other image to PDF software.
+
+For example, this command uses img2pdf to convert all .png files beginning
+with the 'page' prefix to a PDF, fitting each image on A4-sized paper, and
+sending the result to OCRmyPDF through a pipe.  img2pdf is a dependency of
+ocrmypdf so it is already installed.
+
+    img2pdf --pagesize A4 page*.png | ocrmypdf - myfile.pdf
+
+Online documentation is located at:
+    https://ocrmypdf.readthedocs.io/en/latest/introduction.html
+
+""",
+)
+
+parser.add_argument(
+    'input_file',
+    metavar="input_pdf_or_image",
+    help="PDF file containing the images to be OCRed (or '-' to read from "
+    "standard input)",
+)
+parser.add_argument(
+    'output_file',
+    metavar="output_pdf",
+    help="Output searchable PDF file (or '-' to write to standard output). "
+    "Existing files will be ovewritten. If same as input file, the "
+    "input file will be updated only if processing is successful.",
+)
+parser.add_argument(
+    '-l',
+    '--language',
+    action='append',
+    help="Language(s) of the file to be OCRed (see tesseract --list-langs for "
+    "all language packs installed in your system). Use -l eng+deu for "
+    "multiple languages.",
+)
+parser.add_argument(
+    '--image-dpi',
+    metavar='DPI',
+    type=int,
+    help="For input image instead of PDF, use this DPI instead of file's.",
+)
+parser.add_argument(
+    '--output-type',
+    choices=['pdfa', 'pdf', 'pdfa-1', 'pdfa-2', 'pdfa-3'],
+    default='pdfa',
+    help="Choose output type. 'pdfa' creates a PDF/A-2b compliant file for "
+    "long term archiving (default, recommended) but may not suitable "
+    "for users who want their file altered as little as possible. 'pdfa' "
+    "also has problems with full Unicode text. 'pdf' attempts to "
+    "preserve file contents as much as possible. 'pdf-a1' creates a "
+    "PDF/A1-b file. 'pdf-a2' is equivalent to 'pdfa'. 'pdf-a3' creates a "
+    "PDF/A3-b file.",
+)
+
+# Use null string '\0' as sentinel to indicate the user supplied no argument,
+# since that is the only invalid character for filepaths on all platforms
+# bool('\0') is True in Python
+parser.add_argument(
+    '--sidecar',
+    nargs='?',
+    const='\0',
+    default=None,
+    metavar='FILE',
+    help="Generate sidecar text files that contain the same text recognized "
+    "by Tesseract. This may be useful for building a OCR text database. "
+    "If FILE is omitted, the sidecar file be named {output_file}.txt "
+    "If FILE is set to '-', the sidecar is written to stdout (a "
+    "convenient way to preview OCR quality). The output file and sidecar "
+    "may not both use stdout at the same time.",
+)
+
+parser.add_argument(
+    '--version',
+    action='version',
+    version=VERSION,
+    help="Print program version and exit",
+)
+
+jobcontrol = parser.add_argument_group("Job control options")
+jobcontrol.add_argument(
+    '-j',
+    '--jobs',
+    metavar='N',
+    type=numeric(int, 0, 256),
+    help="Use up to N CPU cores simultaneously (default: use all).",
+)
+jobcontrol.add_argument(
+    '-q', '--quiet', action='store_true', help="Suppress INFO messages"
+)
+jobcontrol.add_argument(
+    '-v',
+    '--verbose',
+    type=int,
+    default=0,
+    nargs='?',
+    help="Print more verbose messages for each additional verbose level. Use "
+    "`-v 1` typically for much more detailed logging. Higher numbers "
+    "are probably only useful in debugging.",
+)
+jobcontrol.add_argument(
+    '--no-progress-bar',
+    action='store_false',
+    dest='progress_bar',
+    help=argparse.SUPPRESS,
+)
+jobcontrol.add_argument('--use-threads', action='store_true', help=argparse.SUPPRESS)
+
+metadata = parser.add_argument_group(
+    "Metadata options",
+    "Set output PDF/A metadata (default: copy input document's metadata)",
+)
+metadata.add_argument(
+    '--title', type=str, help="Set document title (place multiple words in quotes)"
+)
+metadata.add_argument('--author', type=str, help="Set document author")
+metadata.add_argument('--subject', type=str, help="Set document subject description")
+metadata.add_argument('--keywords', type=str, help="Set document keywords")
+
+preprocessing = parser.add_argument_group(
+    "Image preprocessing options",
+    "Options to improve the quality of the final PDF and OCR",
+)
+preprocessing.add_argument(
+    '-r',
+    '--rotate-pages',
+    action='store_true',
+    help="Automatically rotate pages based on detected text orientation",
+)
+preprocessing.add_argument(
+    '--remove-background',
+    action='store_true',
+    help="Attempt to remove background from gray or color pages, setting it "
+    "to white ",
+)
+preprocessing.add_argument(
+    '-d', '--deskew', action='store_true', help="Deskew each page before performing OCR"
+)
+preprocessing.add_argument(
+    '-c',
+    '--clean',
+    action='store_true',
+    help="Clean pages from scanning artifacts before performing OCR, and send "
+    "the cleaned page to OCR, but do not include the cleaned page in "
+    "the output",
+)
+preprocessing.add_argument(
+    '-i',
+    '--clean-final',
+    action='store_true',
+    help="Clean page as above, and incorporate the cleaned image in the final "
+    "PDF.  Might remove desired content.",
+)
+preprocessing.add_argument(
+    '--unpaper-args',
+    type=str,
+    default=None,
+    help="A quoted string of arguments to pass to unpaper. Requires --clean. "
+    "Example: --unpaper-args '--layout double'.",
+)
+preprocessing.add_argument(
+    '--oversample',
+    metavar='DPI',
+    type=numeric(int, 0, 5000),
+    default=0,
+    help="Oversample images to at least the specified DPI, to improve OCR "
+    "results slightly",
+)
+preprocessing.add_argument(
+    '--remove-vectors',
+    action='store_true',
+    help="EXPERIMENTAL. Mask out any vector objects in the PDF so that they "
+    "will not be included in OCR. This can eliminate false characters.",
+)
+preprocessing.add_argument(
+    '--mask-barcodes',
+    action='store_true',
+    help="EXPERIMENTAL. Mask out any barcodes that appear in the PDF so they are not "
+    "considered during OCR. Barcodes can introduce false characters into "
+    "OCR.",
+)
+preprocessing.add_argument(
+    '--threshold',
+    action='store_true',
+    help="EXPERIMENTAL. Threshold image to 1bpp before sending it to Tesseract for OCR. Can "
+    "improve OCR quality compared to Tesseract's thresholder.",
+)
+
+ocrsettings = parser.add_argument_group("OCR options", "Control how OCR is applied")
+ocrsettings.add_argument(
+    '-f',
+    '--force-ocr',
+    action='store_true',
+    help="Rasterize any text or vector objects on each page, apply OCR, and "
+    "save the rastered output (this rewrites the PDF)",
+)
+ocrsettings.add_argument(
+    '-s',
+    '--skip-text',
+    action='store_true',
+    help="Skip OCR on any pages that already contain text, but include the "
+    "page in final output; useful for PDFs that contain a mix of "
+    "images, text pages, and/or previously OCRed pages",
+)
+ocrsettings.add_argument(
+    '--redo-ocr',
+    action='store_true',
+    help="Attempt to detect and remove the hidden OCR layer from files that "
+    "were previously OCRed with OCRmyPDF or another program. Apply OCR "
+    "to text found in raster images. Existing visible text objects will "
+    "not be changed. If there is no existing OCR, OCR will be added.",
+)
+ocrsettings.add_argument(
+    '--skip-big',
+    type=numeric(float, 0, 5000),
+    metavar='MPixels',
+    help="Skip OCR on pages larger than the specified amount of megapixels, "
+    "but include skipped pages in final output",
+)
+
+optimizing = parser.add_argument_group(
+    "Optimization options", "Control how the PDF is optimized after OCR"
+)
+optimizing.add_argument(
+    '-O',
+    '--optimize',
+    type=int,
+    choices=range(0, 4),
+    default=1,
+    help=(
+        "Control how PDF is optimized after processing:"
+        "0 - do not optimize; "
+        "1 - do safe, lossless optimizations (default); "
+        "2 - do some lossy optimizations; "
+        "3 - do aggressive lossy optimizations (including lossy JBIG2)"
+    ),
+)
+optimizing.add_argument(
+    '--jpeg-quality',
+    type=numeric(int, 0, 100),
+    default=0,
+    metavar='Q',
+    help=(
+        "Adjust JPEG quality level for JPEG optimization. "
+        "100 is best quality and largest output size; "
+        "1 is lowest quality and smallest output; "
+        "0 uses the default."
+    ),
+)
+optimizing.add_argument(
+    '--jpg-quality',
+    type=numeric(int, 0, 100),
+    default=0,
+    metavar='Q',
+    dest='jpeg_quality',
+    help=argparse.SUPPRESS,  # Alias for --jpeg-quality
+)
+optimizing.add_argument(
+    '--png-quality',
+    type=numeric(int, 0, 100),
+    default=0,
+    metavar='Q',
+    help=(
+        "Adjust PNG quality level to use when quantizing PNGs. "
+        "Values have same meaning as with --jpeg-quality"
+    ),
+)
+optimizing.add_argument(
+    '--jbig2-lossy',
+    action='store_true',
+    help=(
+        "Enable JBIG2 lossy mode (better compression, not suitable for some "
+        "use cases - see documentation)."
+    ),
+)
+optimizing.add_argument(
+    '--jbig2-page-group-size',
+    type=numeric(int, 1, 10000),
+    default=0,
+    metavar='N',
+    # Adjust number of pages to consider at once for JBIG2 compression
+    help=argparse.SUPPRESS,
+)
+
+advanced = parser.add_argument_group(
+    "Advanced", "Advanced options to control Tesseract's OCR behavior"
+)
+advanced.add_argument(
+    '--max-image-mpixels',
+    action='store',
+    type=numeric(float, 0),
+    metavar='MPixels',
+    help="Set maximum number of pixels to unpack before treating an image as a "
+    "decompression bomb",
+    default=128.0,
+)
+advanced.add_argument(
+    '--tesseract-config',
+    action='append',
+    metavar='CFG',
+    default=[],
+    help="Additional Tesseract configuration files -- see documentation",
+)
+advanced.add_argument(
+    '--tesseract-pagesegmode',
+    action='store',
+    type=int,
+    metavar='PSM',
+    choices=range(0, 14),
+    help="Set Tesseract page segmentation mode (see tesseract --help)",
+)
+advanced.add_argument(
+    '--tesseract-oem',
+    action='store',
+    type=int,
+    metavar='MODE',
+    choices=range(0, 4),
+    help=(
+        "Set Tesseract 4.0 OCR engine mode: "
+        "0 - original Tesseract only; "
+        "1 - neural nets LSTM only; "
+        "2 - Tesseract + LSTM; "
+        "3 - default."
+    ),
+)
+advanced.add_argument(
+    '--pdf-renderer',
+    choices=['auto', 'hocr', 'sandwich'],
+    default='auto',
+    help="Choose OCR PDF renderer - the default option is to let OCRmyPDF "
+    "choose.  See documentation for discussion.",
+)
+advanced.add_argument(
+    '--tesseract-timeout',
+    default=180.0,
+    type=numeric(float, 0),
+    metavar='SECONDS',
+    help='Give up on OCR after the timeout, but copy the preprocessed page '
+    'into the final output',
+)
+advanced.add_argument(
+    '--rotate-pages-threshold',
+    default=14.0,
+    type=numeric(float, 0, 1000),
+    metavar='CONFIDENCE',
+    help="Only rotate pages when confidence is above this value (arbitrary "
+    "units reported by tesseract)",
+)
+advanced.add_argument(
+    '--pdfa-image-compression',
+    choices=['auto', 'jpeg', 'lossless'],
+    default='auto',
+    help="Specify how to compress images in the output PDF/A. 'auto' lets "
+    "OCRmyPDF decide.  'jpeg' changes all grayscale and color images to "
+    "JPEG compression.  'lossless' uses PNG-style lossless compression "
+    "for all images.  Monochrome images are always compressed using a "
+    "lossless codec.  Compression settings "
+    "are applied to all pages, including those for which OCR was "
+    "skipped.  Not supported for --output-type=pdf ; that setting "
+    "preserves the original compression of all images.",
+)
+advanced.add_argument(
+    '--user-words',
+    metavar='FILE',
+    help="Specify the location of the Tesseract user words file. This is a "
+    "list of words Tesseract should consider while performing OCR in "
+    "addition to its standard language dictionaries. This can improve "
+    "OCR quality especially for specialized and technical documents.",
+)
+advanced.add_argument(
+    '--user-patterns',
+    metavar='FILE',
+    help="Specify the location of the Tesseract user patterns file.",
+)
+
+plugins = parser.add_argument_group("Filters and Plugins", argparse.SUPPRESS)
+plugins.add_argument(
+    '--filter-ocr-image', help=argparse.SUPPRESS, type=check_plugin_loadable
+)
+plugins.add_argument(
+    '--plugin-validation', help=argparse.SUPPRESS, type=check_plugin_loadable
+)
+
+debugging = parser.add_argument_group(
+    "Debugging", "Arguments to help with troubleshooting and debugging"
+)
+debugging.add_argument(
+    '-k',
+    '--keep-temporary-files',
+    action='store_true',
+    help="Keep temporary files (helpful for debugging)",
+)
+debugging.add_argument('--tesseract-env', type=str, help=argparse.SUPPRESS)

src/ocrmypdf/exec/__init__.py

@@ -17,6 +17,7 @@
 
 """Wrappers to manage subprocess calls"""
 
+import logging
 import os
 import re
 import sys
@@ -24,8 +25,10 @@ from subprocess import run, STDOUT, PIPE, CalledProcessError
 from ..exceptions import MissingDependencyError, ExitCode
 from collections.abc import Mapping
 
+log = logging.Logger(__name__)
 
-def get_version(program, *, version_arg='--version', regex=r'(\d+(\.\d+)*)'):
+
+def get_version(program, *, version_arg='--version', regex=r'(\d+(\.\d+)*)', env=None):
     "Get the version of the specified program"
     args_prog = [program, version_arg]
     try:
@@ -36,6 +39,7 @@ def get_version(program, *, version_arg='--version', regex=r'(\d+(\.\d+)*)'):
             stdout=PIPE,
             stderr=STDOUT,
             check=True,
+            env=env,
         )
         output = proc.stdout
     except FileNotFoundError as e:
@@ -115,7 +119,7 @@ def _get_platform():
     return sys.platform
 
 
-def _error_trailer(log, program, package, **kwargs):
+def _error_trailer(program, package, **kwargs):
     if isinstance(package, Mapping):
         package = package[_get_platform()]
 
@@ -125,7 +129,7 @@ def _error_trailer(log, program, package, **kwargs):
         log.info(linux_install_advice.format(**locals()))
 
 
-def _error_missing_program(log, program, package, required_for, recommended):
+def _error_missing_program(program, package, required_for, recommended):
     if required_for:
         log.error(missing_optional_program.format(**locals()))
     elif recommended:
@@ -135,9 +139,7 @@ def _error_missing_program(log, program, package, required_for, recommended):
     _error_trailer(**locals())
 
 
-def _error_old_version(
-    log, program, package, need_version, found_version, required_for
-):
+def _error_old_version(program, package, need_version, found_version, required_for):
     if required_for:
         log.error(old_version_required_for.format(**locals()))
     else:
@@ -147,27 +149,28 @@ def _error_old_version(
 
 def check_external_program(
     *,
-    log,
     program,
     package,
     version_checker,
     need_version,
     required_for=None,
     recommended=False,
+    **kwargs,  # To consume log parameter
 ):
+    if kwargs:
+        if not 'log' in kwargs:
+            log.warning('check_external_program(log=...) is deprecated')
     try:
         found_version = version_checker()
     except (CalledProcessError, FileNotFoundError, MissingDependencyError):
-        _error_missing_program(log, program, package, required_for, recommended)
+        _error_missing_program(program, package, required_for, recommended)
         if not recommended:
-            sys.exit(ExitCode.missing_dependency)
+            raise MissingDependencyError()
         return
 
     if found_version < need_version:
-        _error_old_version(
-            log, program, package, need_version, found_version, required_for
-        )
+        _error_old_version(program, package, need_version, found_version, required_for)
         if not recommended:
-            sys.exit(ExitCode.missing_dependency)
+            raise MissingDependencyError()
 
-    log.debug(f'Found {program} {found_version}')
+    log.debug('Found %s %s', program, found_version)

src/ocrmypdf/exec/ghostscript.py

@@ -15,6 +15,7 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
+import logging
 import re
 from functools import lru_cache
 from os import fspath
@@ -24,8 +25,11 @@ from tempfile import NamedTemporaryFile
 
 from PIL import Image
 
-from . import get_version
 from ..exceptions import SubprocessOutputError
+from . import get_version
+
+
+gslog = logging.getLogger()
 
 
 @lru_cache(maxsize=1)
@@ -129,10 +133,11 @@ def rasterize_pdf(
     :param filter_vector: if True, remove vector graphics objects
     :return:
     """
-    res = xres, yres
-    int_res = round(xres), round(yres)
+    res = round(xres, 6), round(yres, 6)
     if not page_dpi:
         page_dpi = res
+    if not log:
+        log = gslog
 
     with NamedTemporaryFile(delete=True) as tmp:
         args_gs = (
@@ -145,7 +150,7 @@ def rasterize_pdf(
                 f'-sDEVICE={raster_device}',
                 f'-dFirstPage={pageno}',
                 f'-dLastPage={pageno}',
-                f'-r{str(int_res[0])}x{str(int_res[1])}',
+                f'-r{res[0]:f}x{res[1]:f}',
             ]
             + (['-dFILTERVECTOR'] if filter_vector else [])
             + [
@@ -165,26 +170,10 @@ def rasterize_pdf(
             log.debug(p.stdout)
 
         if p.returncode != 0:
-            log.error('Ghostscript rasterizing failed')
-            raise SubprocessOutputError()
-
-        # Ghostscript only accepts integers for output resolution
-        # if the resolution happens to be fractional, then the discrepancy
-        # would change the size of the output page, especially if the DPI
-        # is quite low. Resize the image to the expected size
+            raise SubprocessOutputError('Ghostscript rasterizing failed')
 
         tmp.seek(0)
         with Image.open(tmp) as im:
-            expected_size = (
-                round(im.size[0] / int_res[0] * res[0]),
-                round(im.size[1] / int_res[1] * res[1]),
-            )
-            if expected_size != im.size or page_dpi != (xres, yres):
-                log.debug(
-                    f"Ghostscript: resize output image {im.size} -> {expected_size}"
-                )
-                im = im.resize(expected_size)
-
             if rotation is not None:
                 log.debug("Rotating output by %i", rotation)
                 # rotation is a clockwise angle and Image.ROTATE_* is
@@ -226,6 +215,9 @@ def generate_pdfa(
     images entirely. (The feature was added in 9.23 but broken, and the 9.24
     release of Ghostscript had regressions, so we don't support it until 9.25.)
     """
+    if not log:
+        log = gslog
+
     compression_args = []
     if compression == 'jpeg':
         compression_args = [
@@ -269,7 +261,6 @@ def generate_pdfa(
                 "-dBATCH",
                 "-dNOPAUSE",
                 "-dCompatibilityLevel=" + str(pdf_version),
-                "-dNumRenderingThreads=" + str(threads),
                 "-sDEVICE=pdfwrite",
                 "-dAutoRotatePages=/None",
                 "-sColorConversionStrategy=" + strategy,
@@ -304,5 +295,4 @@ def generate_pdfa(
             # PDF/A - check PDF/A status elsewhere
             copy(gs_pdf.name, fspath(output_file))
         else:
-            log.error('Ghostscript PDF/A rendering failed')
-            raise SubprocessOutputError()
+            raise SubprocessOutputError('Ghostscript PDF/A rendering failed')

src/ocrmypdf/exec/tesseract.py

@@ -22,18 +22,14 @@ from collections import namedtuple
 from contextlib import suppress
 from functools import lru_cache
 from os import fspath
-from subprocess import (
-    PIPE,
-    STDOUT,
-    CalledProcessError,
-    TimeoutExpired,
-    check_output,
-    run,
-)
-from textwrap import dedent
+from subprocess import PIPE, STDOUT, CalledProcessError, TimeoutExpired, run
 
 from . import get_version
-from ..exceptions import MissingDependencyError, TesseractConfigError
+from ..exceptions import (
+    MissingDependencyError,
+    TesseractConfigError,
+    SubprocessOutputError,
+)
 from ..helpers import page_number
 
 OrientationConfidence = namedtuple('OrientationConfidence', ('angle', 'confidence'))
@@ -56,18 +52,16 @@ HOCR_TEMPLATE = """<?xml version="1.0" encoding="UTF-8"?>
 """
 
 
-@lru_cache(maxsize=1)
-def version():
-    return get_version('tesseract', regex=r'tesseract\s(.+)')
+def version(tesseract_env=None):
+    return get_version('tesseract', regex=r'tesseract\s(.+)', env=tesseract_env)
 
 
-def v4():
+def v4(tesseract_env=None):
     "Is this Tesseract v4.0?"
-    return version() >= '4'
+    return version(tesseract_env) >= '4'
 
 
-@lru_cache(maxsize=1)
-def has_textonly_pdf():
+def has_textonly_pdf(tesseract_env=None):
     """Does Tesseract have textonly_pdf capability?
 
     Available in v4.00.00alpha since January 2017. Best to
@@ -76,41 +70,51 @@ def has_textonly_pdf():
     args_tess = ['tesseract', '--print-parameters', 'pdf']
     params = ''
     try:
-        params = check_output(args_tess, universal_newlines=True, stderr=STDOUT)
+        proc = run(
+            args_tess,
+            check=True,
+            universal_newlines=True,
+            stdout=PIPE,
+            stderr=STDOUT,
+            env=tesseract_env,
+        )
+        params = proc.stdout
     except CalledProcessError as e:
-        print("Could not --print-parameters from tesseract", file=sys.stderr)
-        raise MissingDependencyError from e
+        raise MissingDependencyError(
+            "Could not --print-parameters from tesseract"
+        ) from e
     if 'textonly_pdf' in params:
         return True
     return False
 
 
-@lru_cache(maxsize=1)
-def languages():
+def languages(tesseract_env=None):
     def lang_error(output):
-        msg = dedent(
-            """Tesseract failed to report available languages.
-        Output from Tesseract:
-        -----------
-        """
+        msg = (
+            "Tesseract failed to report available languages.\n"
+            "Output from Tesseract:\n"
+            "-----------\n"
         )
         msg += output
-        print(msg, file=sys.stderr)
+        return msg
 
     args_tess = ['tesseract', '--list-langs']
     try:
         proc = run(
-            args_tess, universal_newlines=True, stdout=PIPE, stderr=STDOUT, check=True
+            args_tess,
+            universal_newlines=True,
+            stdout=PIPE,
+            stderr=STDOUT,
+            check=True,
+            env=tesseract_env,
         )
         output = proc.stdout
     except CalledProcessError as e:
-        lang_error(e.output)
-        raise MissingDependencyError from e
+        raise MissingDependencyError(lang_error(e.output)) from e
 
     header, *rest = output.splitlines()
     if not header.startswith('List of available languages'):
-        lang_error(output)
-        raise MissingDependencyError
+        raise MissingDependencyError(lang_error(output))
     return set(lang.strip() for lang in rest)
 
 
@@ -123,7 +127,7 @@ def tess_base_args(langs, engine_mode):
     return args
 
 
-def get_orientation(input_file, engine_mode, timeout: float, log):
+def get_orientation(input_file, engine_mode, timeout: float, log, tesseract_env=None):
     args_tesseract = tess_base_args(['osd'], engine_mode) + [
         '--psm',
         '0',
@@ -132,7 +136,15 @@ def get_orientation(input_file, engine_mode, timeout: float, log):
     ]
 
     try:
-        stdout = check_output(args_tesseract, stderr=STDOUT, timeout=timeout)
+        p = run(
+            args_tesseract,
+            stdout=PIPE,
+            stderr=STDOUT,
+            timeout=timeout,
+            check=True,
+            env=tesseract_env,
+        )
+        stdout = p.stdout
     except TimeoutExpired:
         return OrientationConfidence(angle=0, confidence=0.0)
     except CalledProcessError as e:
@@ -142,7 +154,7 @@ def get_orientation(input_file, engine_mode, timeout: float, log):
             or b'Image too large' in e.output
         ):
             return OrientationConfidence(0, 0)
-        raise e from e
+        raise SubprocessOutputError() from e
     else:
         osd = {}
         for line in stdout.decode().splitlines():
@@ -159,7 +171,7 @@ def get_orientation(input_file, engine_mode, timeout: float, log):
 
 
 def tesseract_log_output(log, stdout, input_file):
-    prefix = f"{(page_number(input_file)):4d}: [tesseract] "
+    prefix = "[tesseract] "
 
     try:
         text = stdout.decode()
@@ -231,6 +243,7 @@ def generate_hocr(
     pagesegmode: int,
     user_words,
     user_patterns,
+    tesseract_env,
     log,
 ):
 
@@ -254,7 +267,15 @@ def generate_hocr(
     args_tesseract.extend([input_file, prefix, 'hocr', 'txt'] + tessconfig)
     try:
         log.debug(args_tesseract)
-        stdout = check_output(args_tesseract, stderr=STDOUT, timeout=timeout)
+        p = run(
+            args_tesseract,
+            stdout=PIPE,
+            stderr=STDOUT,
+            timeout=timeout,
+            check=True,
+            env=tesseract_env,
+        )
+        stdout = p.stdout
     except TimeoutExpired:
         # Generate a HOCR file with no recognized text if tesseract times out
         # Temporary workaround to hocrTransform not being able to function if
@@ -267,7 +288,7 @@ def generate_hocr(
             _generate_null_hocr(output_hocr, output_sidecar, input_file)
             return
 
-        raise e from e
+        raise SubprocessOutputError() from e
     else:
         tesseract_log_output(log, stdout, input_file)
         # The sidecar text file will get the suffix .txt; rename it to
@@ -306,9 +327,10 @@ def generate_pdf(
     pagesegmode: int,
     user_words,
     user_patterns,
+    tesseract_env,
     log,
 ):
-    '''Use Tesseract to render a PDF.
+    """Use Tesseract to render a PDF.
 
     input_image -- image to analyze
     skip_pdf -- if we time out, use this file as output
@@ -320,14 +342,14 @@ def generate_pdf(
     tessconfig -- tesseract configuration
     timeout -- timeout (seconds)
     log -- logger object
-    '''
+    """
 
     args_tesseract = tess_base_args(language, engine_mode)
 
     if pagesegmode is not None:
         args_tesseract.extend(['--psm', str(pagesegmode)])
 
-    if text_only and has_textonly_pdf():
+    if text_only and has_textonly_pdf(tesseract_env):
         args_tesseract.extend(['-c', 'textonly_pdf=1'])
 
     if user_words:
@@ -342,10 +364,17 @@ def generate_pdf(
     # to the number of order parameters here
 
     args_tesseract.extend([input_image, prefix, 'pdf', 'txt'] + tessconfig)
-
     try:
         log.debug(args_tesseract)
-        stdout = check_output(args_tesseract, stderr=STDOUT, timeout=timeout)
+        p = run(
+            args_tesseract,
+            stdout=PIPE,
+            stderr=STDOUT,
+            timeout=timeout,
+            check=True,
+            env=tesseract_env,
+        )
+        stdout = p.stdout
         if os.path.exists(prefix + '.txt'):
             shutil.move(prefix + '.txt', output_text)
     except TimeoutExpired:
@@ -356,6 +385,6 @@ def generate_pdf(
         if b'Image too large' in e.output:
             use_skip_page(text_only, skip_pdf, output_pdf, output_text)
             return
-        raise e from e
+        raise SubprocessOutputError() from e
     else:
         tesseract_log_output(log, stdout, input_image)

src/ocrmypdf/exec/unpaper.py

@@ -29,11 +29,7 @@ from tempfile import TemporaryDirectory
 from . import get_version
 from ..exceptions import MissingDependencyError, SubprocessOutputError
 
-try:
-    from PIL import Image
-except ImportError:
-    print("Could not find Python3 imaging library", file=sys.stderr)
-    raise
+from PIL import Image
 
 
 @lru_cache(maxsize=1)
@@ -55,16 +51,18 @@ def run(input_file, output_file, dpi, log, mode_args):
             else:
                 im = im.convert(mode='RGB')
         except IOError as e:
-            log.error("Could not convert image with type " + im.mode)
             im.close()
-            raise MissingDependencyError() from e
+            raise MissingDependencyError(
+                "Could not convert image with type " + im.mode
+            ) from e
 
     try:
         suffix = SUFFIXES[im.mode]
     except KeyError:
-        log.error("Failed to convert image to a supported format.")
         im.close()
-        raise MissingDependencyError() from e
+        raise MissingDependencyError(
+            "Failed to convert image to a supported format."
+        ) from e
 
     with TemporaryDirectory() as tmpdir:
         input_pnm = os.path.join(tmpdir, f'input{suffix}')

src/ocrmypdf/filters.py

@@ -0,0 +1,10 @@
+from PIL import Image
+import PIL.ImageOps
+
+
+def invert(im):
+    return PIL.ImageOps.invert(im.convert('L'))
+
+
+def whiteout(im):
+    return Image.new(im.mode, im.size)

src/ocrmypdf/helpers.py

@@ -15,32 +15,35 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
+import logging
 import multiprocessing
 import os
-import sys
 import warnings
 from collections.abc import Iterable
 from contextlib import suppress
-from functools import partial, wraps
+from functools import wraps
 from pathlib import Path
 
+log = logging.getLogger(__name__)
 
-def re_symlink(input_file, soft_link_name, log=None):
+
+def re_symlink(input_file, soft_link_name, *args, **kwargs):
     """
     Helper function: relinks soft symbolic link if necessary
     """
+    if len(args) == 1 and isinstance(args[0], logging.Logger):
+        log.warning("Deprecated: re_symlink(,log)")
+    if 'log' in kwargs:
+        log.warning('Deprecated: re_symlink(...log=)')
+
     input_file = os.fspath(input_file)
     soft_link_name = os.fspath(soft_link_name)
-    if log is None:
-        prdebug = partial(print, file=sys.stderr)
-    else:
-        prdebug = log.debug
 
     # Guard against soft linking to oneself
     if input_file == soft_link_name:
-        prdebug(
-            "Warning: No symbolic link made. You are using "
-            + "the original data directory as the working directory."
+        log.warning(
+            "No symbolic link made. You are using "
+            "the original data directory as the working directory."
         )
         return
 
@@ -48,25 +51,21 @@ def re_symlink(input_file, soft_link_name, log=None):
     if os.path.lexists(soft_link_name):
         # do not delete or overwrite real (non-soft link) file
         if not os.path.islink(soft_link_name):
-            raise FileExistsError("%s exists and is not a link" % soft_link_name)
+            raise FileExistsError(f"{soft_link_name} exists and is not a link")
         try:
             os.unlink(soft_link_name)
         except OSError:
-            prdebug("Can't unlink %s" % (soft_link_name))
+            log.debug("Can't unlink %s", soft_link_name)
 
     if not os.path.exists(input_file):
-        raise FileNotFoundError("trying to create a broken symlink to %s" % input_file)
+        raise FileNotFoundError(f"trying to create a broken symlink to {input_file}")
 
-    prdebug("os.symlink(%s, %s)" % (input_file, soft_link_name))
+    log.debug("os.symlink(%s, %s)", input_file, soft_link_name)
 
     # Create symbolic link using absolute path
     os.symlink(os.path.abspath(input_file), soft_link_name)
 
 
-def is_iterable_notstr(thing):
-    return isinstance(thing, Iterable) and not isinstance(thing, str)
-
-
 def page_number(input_file):
     """Get one-based page number implied by filename (000002.pdf -> 2)"""
     return int(os.path.basename(os.fspath(input_file))[0:6])
@@ -122,14 +121,6 @@ def is_file_writable(test_file):
         return True
 
 
-def flatten_groups(groups):
-    for obj in groups:
-        if is_iterable_notstr(obj):
-            yield from obj
-        else:
-            yield obj
-
-
 def deprecated(func):
     """Warn that function is deprecated"""
 

src/ocrmypdf/optimize.py

@@ -16,20 +16,21 @@
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
 import concurrent.futures
-import logging
 import sys
+import tempfile
 from collections import defaultdict
 from os import fspath
 from pathlib import Path
 
 from PIL import Image
-
+from tqdm import tqdm
 import pikepdf
-from pikepdf import Name, Dictionary, Array
+from pikepdf import Name, Dictionary
 
 from . import leptonica
-from ._jobcontext import JobContext
+from ._jobcontext import PDFContext
 from .exec import jbig2enc, pngquant
+from .exceptions import OutputFileAccessError
 from .helpers import re_symlink
 
 DEFAULT_JPEG_QUALITY = 75
@@ -267,9 +268,16 @@ def _produce_jbig2_images(jbig2_groups, root, log, options):
 
     with concurrent.futures.ThreadPoolExecutor(max_workers=options.jobs) as executor:
         futures = jbig2_futures(executor, root, jbig2_groups)
-        for future in concurrent.futures.as_completed(futures):
-            proc = future.result()
-            log.debug(proc.stderr.decode())
+        with tqdm(
+            total=len(jbig2_groups),
+            desc="JBIG2",
+            unit='item',
+            disable=not options.progress_bar,
+        ) as pbar:
+            for future in concurrent.futures.as_completed(futures):
+                proc = future.result()
+                log.debug(proc.stderr.decode())
+                pbar.update()
 
 
 def convert_to_jbig2(pike, jbig2_groups, root, log, options):
@@ -311,7 +319,9 @@ def convert_to_jbig2(pike, jbig2_groups, root, log, options):
 
 
 def transcode_jpegs(pike, jpegs, root, log, options):
-    for xref in jpegs:
+    for xref in tqdm(
+        jpegs, desc="JPEGs", unit='image', disable=not options.progress_bar
+    ):
         in_jpg = Path(jpg_name(root, xref))
         opt_jpg = in_jpg.with_suffix('.opt.jpg')
 
@@ -340,15 +350,26 @@ def transcode_pngs(pike, images, image_name_fn, root, log, options):
         with concurrent.futures.ThreadPoolExecutor(
             max_workers=options.jobs
         ) as executor:
+            futures = []
             for xref in images:
                 log.debug(image_name_fn(root, xref))
-                executor.submit(
-                    pngquant.quantize,
-                    image_name_fn(root, xref),
-                    png_name(root, xref),
-                    png_quality[0],
-                    png_quality[1],
+                futures.append(
+                    executor.submit(
+                        pngquant.quantize,
+                        image_name_fn(root, xref),
+                        png_name(root, xref),
+                        png_quality[0],
+                        png_quality[1],
+                    )
                 )
+            with tqdm(
+                desc="PNGs",
+                total=len(futures),
+                unit='image',
+                disable=not options.progress_bar,
+            ) as pbar:
+                for _future in concurrent.futures.as_completed(futures):
+                    pbar.update()
 
     for xref in images:
         im_obj = pike.get_object(xref, 0)
@@ -427,11 +448,11 @@ def transcode_pngs(pike, images, image_name_fn, root, log, options):
         im_obj.write(compdata.read(), filter=Name.FlateDecode, decode_parms=dparms)
 
 
-def optimize(input_file, output_file, log, context):
-
-    options = context.get_options()
+def optimize(input_file, output_file, context):
+    log = context.log
+    options = context.options
     if options.optimize == 0:
-        re_symlink(input_file, output_file, log)
+        re_symlink(input_file, output_file)
         return
 
     if options.jpeg_quality == 0:
@@ -441,40 +462,44 @@ def optimize(input_file, output_file, log, context):
     if options.jbig2_page_group_size == 0:
         options.jbig2_page_group_size = 10 if options.jbig2_lossy else 1
 
-    pike = pikepdf.Pdf.open(input_file)
+    with pikepdf.Pdf.open(input_file) as pike:
+        root = Path(output_file).parent / 'images'
+        root.mkdir(exist_ok=True)
 
-    root = Path(output_file).parent / 'images'
-    root.mkdir(exist_ok=True)
+        jpegs, pngs = extract_images_generic(pike, root, log, options)
+        transcode_jpegs(pike, jpegs, root, log, options)
+        # if options.optimize >= 2:
+        # Try pngifying the jpegs
+        #    transcode_pngs(pike, jpegs, jpg_name, root, log, options)
+        transcode_pngs(pike, pngs, png_name, root, log, options)
 
-    jpegs, pngs = extract_images_generic(pike, root, log, options)
-    transcode_jpegs(pike, jpegs, root, log, options)
-    # if options.optimize >= 2:
-    # Try pngifying the jpegs
-    #    transcode_pngs(pike, jpegs, jpg_name, root, log, options)
-    transcode_pngs(pike, pngs, png_name, root, log, options)
+        jbig2_groups = extract_images_jbig2(pike, root, log, options)
+        convert_to_jbig2(pike, jbig2_groups, root, log, options)
 
-    jbig2_groups = extract_images_jbig2(pike, root, log, options)
-    convert_to_jbig2(pike, jbig2_groups, root, log, options)
-
-    target_file = Path(output_file).with_suffix('.opt.pdf')
-    pike.remove_unreferenced_resources()
-    pike.save(
-        target_file,
-        preserve_pdfa=True,
-        object_stream_mode=pikepdf.ObjectStreamMode.generate,
-    )
+        target_file = Path(output_file).with_suffix('.opt.pdf')
+        pike.remove_unreferenced_resources()
+        pike.save(
+            target_file,
+            preserve_pdfa=True,
+            object_stream_mode=pikepdf.ObjectStreamMode.generate,
+        )
 
     input_size = Path(input_file).stat().st_size
     output_size = Path(target_file).stat().st_size
+    if output_size == 0:
+        raise OutputFileAccessError(
+            f"Output file not created after optimizing. We probably ran "
+            f"out of disk space in the temporary folder: {tempfile.gettempdir()}."
+        )
     ratio = input_size / output_size
     savings = 1 - output_size / input_size
     log.info(f"Optimize ratio: {ratio:.2f} savings: {(100 * savings):.1f}%")
 
     if savings < 0:
         log.info("Optimize did not improve the file - discarded")
-        re_symlink(input_file, output_file, log)
+        re_symlink(input_file, output_file)
     else:
-        re_symlink(target_file, output_file, log)
+        re_symlink(target_file, output_file)
 
 
 def main(infile, outfile, level, jobs=1):
@@ -484,30 +509,32 @@ def main(infile, outfile, level, jobs=1):
     class OptimizeOptions:
         """Emulate ocrmypdf's options"""
 
-        def __init__(self, jobs, optimize, jpeg_quality, png_quality, jb2lossy):
+        def __init__(
+            self, input_file, jobs, optimize, jpeg_quality, png_quality, jb2lossy
+        ):
+            self.input_file = input_file
             self.jobs = jobs
             self.optimize = optimize
             self.jpeg_quality = jpeg_quality
             self.png_quality = png_quality
             self.jbig2_page_group_size = 0
             self.jbig2_lossy = jb2lossy
+            self.quiet = True
+            self.progress_bar = False
 
-    logging.basicConfig(level=logging.DEBUG)
-    log = logging.getLogger()
-
-    ctx = JobContext()
     options = OptimizeOptions(
+        input_file=infile,
         jobs=jobs,
         optimize=int(level),
         jpeg_quality=0,  # Use default
         png_quality=0,
         jb2lossy=False,
     )
-    ctx.set_options(options)
 
     with TemporaryDirectory() as td:
+        context = PDFContext(options, td, infile, None)
         tmpout = Path(td) / 'out.pdf'
-        optimize(infile, tmpout, log, ctx)
+        optimize(infile, tmpout, context)
         copy(fspath(tmpout), fspath(outfile))
 
 

src/ocrmypdf/pdfa.py

@@ -131,19 +131,19 @@ def file_claims_pdfa(filename):
     do full PDF/A validation.
     """
 
-    pdf = pikepdf.open(filename)
-    pdfmeta = pdf.open_metadata()
-    if not pdfmeta.pdfa_status:
-        return {
-            'pass': False,
-            'output': 'pdf',
-            'conformance': 'No PDF/A metadata in XMP',
-        }
-    valid_part_conforms = {'1A', '1B', '2A', '2B', '2U', '3A', '3B', '3U'}
-    conformance = f'PDF/A-{pdfmeta.pdfa_status}'
-    pdfa_dict = {}
-    if pdfmeta.pdfa_status in valid_part_conforms:
-        pdfa_dict['pass'] = True
-        pdfa_dict['output'] = 'pdfa'
-    pdfa_dict['conformance'] = conformance
+    with pikepdf.open(filename) as pdf:
+        pdfmeta = pdf.open_metadata()
+        if not pdfmeta.pdfa_status:
+            return {
+                'pass': False,
+                'output': 'pdf',
+                'conformance': 'No PDF/A metadata in XMP',
+            }
+        valid_part_conforms = {'1A', '1B', '2A', '2B', '2U', '3A', '3B', '3U'}
+        conformance = f'PDF/A-{pdfmeta.pdfa_status}'
+        pdfa_dict = {}
+        if pdfmeta.pdfa_status in valid_part_conforms:
+            pdfa_dict['pass'] = True
+            pdfa_dict['output'] = 'pdfa'
+        pdfa_dict['conformance'] = conformance
     return pdfa_dict

src/ocrmypdf/pdfinfo/__init__.py

@@ -19,21 +19,24 @@
 from collections import namedtuple
 from decimal import Decimal
 from enum import Enum
+import logging
 from math import hypot, isclose
 from os import fspath
 from pathlib import Path
-from unittest.mock import Mock
 from warnings import warn
 import re
 
 from pikepdf import PdfMatrix
 import pikepdf
+from tqdm import tqdm
 
 from . import ghosttext
 
 from ..exceptions import EncryptedPdfError, MissingDependencyError
 
 
+logger = logging.getLogger()
+
 Colorspace = Enum('Colorspace', 'gray rgb cmyk lab icc index sep devn pattern jpeg2000')
 
 Encoding = Enum(
@@ -614,12 +617,10 @@ def _pdf_get_pageinfo(pdf, pageno: int, infile, xmltext):
     return pageinfo
 
 
-def _pdf_get_all_pageinfo(infile, detailed_analysis=False, log=None):
-    if not log:
-        log = Mock()
-
-    pdf = pikepdf.open(infile)
+def _pdf_get_all_pageinfo(infile, detailed_analysis=False, log=None, progbar=False):
+    pdf = pikepdf.open(infile)  # Do not close in this function
     if pdf.is_encrypted:
+        pdf.close()
         raise EncryptedPdfError()  # Triggered by encryption with empty passwd
     if detailed_analysis:
         pages_xml = None
@@ -627,7 +628,13 @@ def _pdf_get_all_pageinfo(infile, detailed_analysis=False, log=None):
         pages_xml = ghosttext.extract_text_xml(infile, pdf, pageno=None, log=log)
 
     pages = []
-    for n in range(len(pdf.pages)):
+    for n, _ in tqdm(
+        enumerate(pdf.pages),
+        total=len(pdf.pages),
+        desc="Scan",
+        unit='page',
+        disable=not progbar,
+    ):
         page_xml = pages_xml[n] if pages_xml else None
         page = PageInfo(pdf, n, infile, page_xml, detailed_analysis)
         pages.append(page)
@@ -749,13 +756,19 @@ class PageInfo:
 class PdfInfo:
     """Get summary information about a PDF"""
 
-    def __init__(self, infile, detailed_page_analysis=False, log=None):
+    def __init__(self, infile, detailed_page_analysis=False, log=logger, progbar=False):
         self._infile = infile
         self._pages, pdf = _pdf_get_all_pageinfo(
-            infile, detailed_page_analysis, log=log
+            infile, detailed_page_analysis, log=log, progbar=progbar
         )
         self._needs_rendering = pdf.root.get('/NeedsRendering', False)
-        self._has_acroform = '/AcroForm' in pdf.root
+        self._has_acroform = False
+        if '/AcroForm' in pdf.root:
+            if len(pdf.root.AcroForm.get('/Fields', [])) > 0:
+                self._has_acroform = True
+            elif '/XFA' in pdf.root.AcroForm:
+                self._has_acroform = True
+        pdf.close()
 
     @property
     def pages(self):

src/ocrmypdf/pdfinfo/ghosttext.py

@@ -15,11 +15,14 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
+import logging
 import re
 import xml.etree.ElementTree as ET
 
 from ..exec import ghostscript
 
+gslog = logging.getLogger()
+
 # Forgive me for I have sinned
 # I am using regular expressions to parse XML. However the XML in this case,
 # generated by Ghostscript, is self-consistent enough to be parseable.
@@ -74,7 +77,7 @@ def page_get_textblocks(infile, pageno, xmltext, height):
     return [block for block in joined_blocks()]
 
 
-def extract_text_xml(infile, pdf, pageno=None, log=None):
+def extract_text_xml(infile, pdf, pageno=None, log=gslog):
     existing_text = ghostscript.extract_text(infile, pageno=None)
     existing_text = regex_remove_char_tags.sub(b' ', existing_text)
 

src/ocrmypdf/pdfinfo/layout.py

@@ -172,6 +172,7 @@ class LTStateAwareChar(LTChar):
             - the Unicode mapping is known, and both have the same render mode
             - the Unicode mapping is unknown but both are part of the same font
         """
+        # pylint: disable=protected-access
         both_unicode_mapped = isinstance(self._text, str) and isinstance(obj._text, str)
         try:
             if both_unicode_mapped:
@@ -184,7 +185,7 @@ class LTStateAwareChar(LTChar):
 
     def get_text(self):
         if isinstance(self._text, tuple):
-            return '�'
+            return '\ufffd'  # standard 'Unknown symbol'
         return self._text
 
     def __repr__(self):

tests/cache/aspect/__-l__eng__000001_ocr.png__000001_ocr_hocr__hocr__txt/hocr.bin

@@ -9,23 +9,23 @@
   <meta name='ocr-capabilities' content='ocr_page ocr_carea ocr_par ocr_line ocrx_word ocrp_wconf'/>
 </head>
 <body>
-  <div class='ocr_page' id='page_1' title='image "/var/folders/37/78_114p552q16vv6vmgm5kr00000gn/T/com.github.ocrmypdf.jz4b2s0s/000001.ocr.png"; bbox 0 0 1000 800; ppageno 0'>
+  <div class='ocr_page' id='page_1' title='image "/var/folders/2s/7t022mgj0h5cprbq0dtb1ksm0000gn/T/com.github.ocrmypdf.xam82ph5/000001_ocr.png"; bbox 0 0 1000 800; ppageno 0'>
    <div class='ocr_carea' id='block_1_1' title="bbox 296 96 704 504">
     <p class='ocr_par' id='par_1_1' lang='eng' title="bbox 296 96 704 504">
      <span class='ocr_line' id='line_1_1' title="bbox 296 96 704 504; baseline 0 296; x_size 169.33333; x_descenders 42.333332; x_ascenders 42.333336">
-      <span class='ocrx_word' id='word_1_1' title='bbox 296 96 704 504; x_wconf 95'><strong><em> </em></strong></span>
+      <span class='ocrx_word' id='word_1_1' title='bbox 296 96 704 504; x_wconf 95'> </span>
      </span>
     </p>
    </div>
    <div class='ocr_carea' id='block_1_2' title="bbox 150 592 841 622">
     <p class='ocr_par' id='par_1_2' lang='eng' title="bbox 150 592 841 622">
      <span class='ocr_line' id='line_1_2' title="bbox 150 592 841 622; baseline 0 -6; x_size 30; x_descenders 6; x_ascenders 8">
-      <span class='ocrx_word' id='word_1_2' title='bbox 150 592 230 616; x_wconf 96'><strong><em>This</em></strong></span>
-      <span class='ocrx_word' id='word_1_3' title='bbox 260 592 384 616; x_wconf 95'><strong><em>should</em></strong></span>
-      <span class='ocrx_word' id='word_1_4' title='bbox 413 592 449 616; x_wconf 95'><strong><em>be</em></strong></span>
-      <span class='ocrx_word' id='word_1_5' title='bbox 479 600 493 616; x_wconf 95'><strong><em>a</em></strong></span>
-      <span class='ocrx_word' id='word_1_6' title='bbox 523 592 668 622; x_wconf 95'><strong><em>perfect</em></strong></span>
-      <span class='ocrx_word' id='word_1_7' title='bbox 698 592 841 616; x_wconf 55'><strong><em>circle:</em></strong></span>
+      <span class='ocrx_word' id='word_1_2' title='bbox 150 592 230 616; x_wconf 96'>This</span>
+      <span class='ocrx_word' id='word_1_3' title='bbox 260 592 384 616; x_wconf 95'>should</span>
+      <span class='ocrx_word' id='word_1_4' title='bbox 413 592 449 616; x_wconf 95'>be</span>
+      <span class='ocrx_word' id='word_1_5' title='bbox 479 600 493 616; x_wconf 95'>a</span>
+      <span class='ocrx_word' id='word_1_6' title='bbox 523 592 668 622; x_wconf 95'>perfect</span>
+      <span class='ocrx_word' id='word_1_7' title='bbox 698 592 841 616; x_wconf 55'>circle:</span>
      </span>
     </p>
    </div>

tests/cache/cardinal/__-l__eng__000002_ocr.png__000002_ocr_tess__pdf__txt/txt.bin

@@ -103,7 +103,7 @@ ERASE, REPEAT, PLAY/STOP, or LOCATE.
 
 © Will sync to standard LinnDrum or Linn 9000 sync tone.
 
-© Utilizes ultra high-speed, 8 MHz 80186 16 bit computer internally for FAST operation.
+® Utilizes ultra high-speed, 8 MHz 80186 16 bit computer internally for FAST operation.
 * TEMPO may be specified in BEATS-PER-MINUTE or FRAMES-PER-BEAT at 24, 25, or 30 frames per second,
 
 (even drop frame!)
@@ -115,9 +115,9 @@ on the TAP TEMPO button.
 ¢ TEMPO CHANGES may be programmed into a sequence, with smooth transitions if desired.
 ¢ Any TIME SIGNATURE may be used, and may be changed within a song.
 
-linn
-Linn Electronics, Inc.
+nn
 
+Linn Electronics, Inc.
 18720 Oxnard Street, Tarzana, CA 91356
 (818) 708-8131 TELEX #298949 LINN UR
 

tests/cache/cardinal/__-l__eng__000003_ocr.png__000003_ocr_tess__pdf__txt/txt.bin

@@ -0,0 +1,128 @@
+2A NNI‘I 6F6867# XATALL IE18-80L (818)
+
+9SEI6 VO “BUBZIRY, “J0aNS PIPUXO OZLEI
+“Uy ‘soTUOMOI,q UUrT
+
+uut]
+
+“‘SUOS B UIJIM pasueyo oq ABU pue ‘posn oq AWW AYN IVNOIS AWLL AUV
+“parlsop Jr SUOTIISUBI} YIOOUIS YIM “BoueNbas eB OJUI pourtueIZOId 9q ABU SFONWHO OdINAL e
+
+‘uonng OdNAL dV L 9) uO
+
+sojou Jayienb Suiddy} Aq 10 ‘syUSTIOIOUI oINUTIAI-J8g-Jesg & JO sys} UL ofquisn(pe ‘ATTeouIAUINU paiajus oq ABU OdINALL e
+
+(jouer doup u3a9)
+
+“puooes Jed souely O€ 10 “SZ “pz 18 [LVAG-MAd-SHN VU 10 ALOANIWAAd-SLVAd U! patyoeds aq kewl OAL «
+‘uoTe1odo [SVx JO} Aj[eusoyUT JoyndUIOd 11g 9] 98108 ZHI 8g ‘poeds-ysry Bann soz] e
+
+"9U0} DUAS 0006 UUL] Jo wNIqUUr] prepue}s 0} OUAS [ITAA ©
+
+“ONYBA 9}OU poloapes Aue Je sas—nd jndyno 07 pewureigold 3q ACW SL Ad LNO YADONAL OML
+
+"ALVOOT 10 GOLS/AV 1d ‘LWddad “ASV
+
+SUIpNpoUr ‘suOTIOUN] posn A[UOUILUOS 94] JO AUBUT [O1]UOD AJ9]OWIAI 0} PousIsse oq ACUI ST AdNI HOLIMSLOO OME «
+“SUIPIONAI I[IYM P2sesd JOU Iv $3}OU BUTISIXO—ZUIPIOIA SATON.ASOP-UON
+
+‘suoneurldxa peuoyippe sdeydsip uowng g1TqH
+
+oy] ‘pepsau JI ‘suoneiodo [ye yYsnosy] NOA sapins ApIespo Avfdsip QO] Joey Z7¢ 9y3—uoeIodo Urea] 0} Ased ‘aus «
+
+jUorel]suowtap & IO} Aepol Jayeap uur’] INOA dag ‘dISHUL
+INOA 0} UONUS}]¥ PaplAIPUN INOA SUTJOASp ITY ps pue
+p1osai ‘asoduod no Jay 0} pausisap st 1s0uenbesuur’] oy)
+Aum Aposiooid $,Jeu], ‘SS9d0Id SATTBS1D OY} YIM SOIOJIOIUT
+yey) xo]dwWI0d Os dq JOA9U P[NoUsS osn NOA AZopOuYdE} oy
+
+ISTUMOIAUIO?) NOAA UOHISOdWIO)
+
+"NOSpr] B Oy ‘AONUTJUT yada 0} seq Maz Se] BY] Jas UdAd
+uvd NOA ‘palisap JJ ‘souanbes Mou ¥B OVUT sjied ou] [Te Adoo
+ATesrewO Ne WI) [IM ONOS ALVAAO JeyIe80} wey}
+,deyd,, 0} UOTOUNJ ONOS ALVA ou] asn usy] ‘saouanbes
+JENPIAIpUt UI (“949 ‘snJOYD ‘aS1OA) UOTIDIS JIseq Yes
+Pl0da1 OF ST ABM JOuIOUY “(812g 666 01 dn) ysnory) ABM
+
+dU} [fe YORI] YORs p10991 0} ST SUOS B 9789I9 0} ABM SUG,
+
+SUOS & SUTVAID
+
+*suoT}oes poJUBMUN
+
+SAOUIOI 0} ABM SWS dU} SoyeIodo SUV ALATAaG
+
+“OBPLIq dy} PUB SNIOY PUOdAS dT]] Ud9MIAQ SIDA ISI
+
+ay) Jo Adoo B JJasuT WYSE NOAA ‘afdwexs 10.f ‘UO JUSIN]JIP
+
+B IO aouaNbas sues OY} UI—JOY OUP 0} UOTIEIO] 9UO WOT]
+$1Bq JAOUI OF NOA sMOTIe WOTIOUNS AdOO/IMASNI OULL
+
+‘SUIPIONAI JIVIS Udy) “OQuINU eq porisop ay} puy
+
+0} CNIMAY 10 ‘CYVM Od LSWA “AEVOOT esn Apduns
+
+sainjeay [PUOHIPPY
+
+‘gouanbas & UTYIIM s]UTOd a1y1dads 3¥ $9100 QnPIOAO OL "UOT}
+-ouns dALLS ATONIS 24) Suisn pasueyo Jo ‘pasesa ‘pappe
+aq osye ABUT S9]ON ‘U0 9q ]IIM 1 “yoeq podeyd uayM
+—aouanbas oy] ul skeyd 71 a10J9q Isnf posers oq 0} d]0U ayy
+ssaid pue ASvwug ploy Aydunis ‘jou Suomm & aseso OL
+
+sunipa
+
+jsesdueyo ureisoid pue ‘fepod ureysns
+‘yonoplalje ‘AWOOTOA ‘UOTyeTNpow ‘pusg youd Surpnyour
+pep10del are $199JJ2 TCTIN [WV iPeqqnpseao aq Aeur syoen
+Ze 07 dn ‘Kem sie Uy *(foeI} JOyOUR OJOS 10 ALLAN
+NOA ssofum) duAS yOaysod ul Avy [[IM Yow] ISI 93 “prooar
+NOA 3[IYM—SUIPIOIA LIBIS PU YORI) TUdIOTJIP B JOaTas
+*y1ed MOU B QNPIsA0 OL, “SuIps0daJ-jods 10} aouanbes mno0k
+UI UOHBIO] Aue ssad0e ATYOIND 0} owt} Aue ye pasn aq AvUE
+SJONUOD FLIVOOT pur ‘ANIMA ‘CYVMaYOd LSVd
+{SUIPIOSAI {IY posesa JOU se So]OU SuTsTXO—
+yous} 3U} OUT poppe aq JIM poteyd sajou yeuonippe Auy
+
+*(povesjap 10 poysn{pe oq ABW UOTIIII0D BUTUTT]) j{paqoeLI09
+
+2q ][IM S1OLIe Sur [fe ATUO—patey]d nod Jey Jedy ]],NOA
+
+‘] req 0] punose yoeq sdoo] sduanbas ay] Udy AA “YOu Yor
+
+§,sa0uaNbas at} O] SUIT) UI preogday [IW] INO Avy usy3
+AV'1d pue (YOON ssoid Ayduus ‘aousnbes & p1o09es OF,
+
+g0uaNbas & SUIP10I0y]
+
+‘JONWOD s}JouNaI TeuONdGO e
+
+"UOTJEZIUOIYUAS OPOS UIT} FLAWS [euondo e
+
+‘sou .sulddoys, noyyM sayelodo pue yoegdvyd ZuLINp S¥IOM NOLLOANNYOO ONIWILL e
+
+‘onqea ory AY
+
+pojoojes-oid & ye sajou pyoy Aue syeadas ATTeONewWO Ne UOTOUNS [WAdAY OAISNOX e
+‘LSVJ SUnIpS soyeu UOTOUN ASV UA OUlN-[eal SAISNIOXY e
+‘Koy B JO YONO} 941 12 CASOdSNVALL 0g ABU Syde] [Te 10 9UC e
+
+i ASIP Jed
+
+S9}0U OOO‘OTT JOA SpfOy puv SpUOdeS UT SBUOS Xa[AUIOD So10}S DALIP YSIP , 74 € ISCJ-CNIN
+
+jSIOZISOUJUAS
+
+stuoydAjod of 0} dn skeyd A[snoourynuls ‘spouueYd [IW 9T JO duo 0} pousisse oq
+ABUL YORI] YOR ‘syous) oruoydAjod ‘snoouelnurs 7¢ SuTeJUOS ssouUaNbas QO] OY} JO YORA e
+
+‘SJONUOS ATWOOT pur ‘GNIMAY ‘GaVM OA
+LSVd ‘GYOOde AOLS ‘AV Td YIM Jopsocas ade} Yowsj-N[NU O} eps st UOTLISdO @
+LOPNOUT SaINjeoy s[quyIeUlss AUB S.JJ ‘OSN pue UIes] 0} o[duns A[suIzeUe JOA ‘PnJsomod APOUIOITXO
+St 1] “UeIOIsNUL feUOIssajoid oY} 10 JOO} soUBULIOJIJAd pue UOTIsOduIOS 11e-dY1-JO-9}e)s B SI IONUANbDaguUT] ay
+
+JOps1odady soUINbIS [GTI YVAL ZE
+Jgouanbaguury oy
+

tests/cache/cardinal/__-l__eng__000004_ocr.png__000004_ocr_tess__pdf__txt/txt.bin

@@ -0,0 +1,124 @@
+2A NNI‘I 6F6867# XATALL IE18-80L (818)
+
+9SEI6 VO “BUBZIRY, “J0aNS PIPUXO OZLEI
+“Uy ‘soTUOMOI,q UUrT
+
+uu
+
+“‘SUOS B UIJIM pasueyo oq ABU pue ‘posn oq AWW AYN IVNOIS AWLL AUV
+“parlsop Jr SUOTIISUBI} YIOOUIS YIM “BoueNbas eB OJUI pourtueIZOId 9q ABU SFONWHO OdINAL e
+
+‘uonng OdNAL dV L 9) uO
+
+sojou Jayienb Suiddy} Aq 10 ‘syUSTIOIOUI oINUTIAI-J8g-Jesg & JO sys} UL ofquisn(pe ‘ATTeouIAUINU paiajus oq ABU OdINALL e
+
+(jouer doup u3a9)
+
+“puooes Jed souely O€ 10 “SZ “pz 18 [LVAG-MAd-SHN VU 10 ALOANIWAAd-SLVAd U! patyoeds aq kewl OAL «
+‘uoTe1odo [SVx JO} Aj[eusoyUT JoyndUIOd 11g 9] 98108 ZHI 8g ‘poeds-ysry Bann soz] e
+
+"9U0} DUAS 0006 UUL] Jo wNIqUUr] prepue}s 0} OUAS [ITAA ©
+
+“ONYBA 9}OU poloapes Aue Je sas—nd jndyno 07 pewureigold 3q ACW SL Ad LNO YADONAL OML
+
+"ALVOOT 10 GOLS/AV 1d ‘LWddad “ASV
+
+SUIpNpoUr ‘suOTIOUN] posn A[UOUILUOS 94] JO AUBUT [O1]UOD AJ9]OWIAI 0} PousIsse oq ACUI ST AdNI HOLIMSLOO OME «
+“SUIPIONAI I[IYM P2sesd JOU Iv $3}OU BUTISIXO—ZUIPIOIA SATON.ASOP-UON
+
+‘suoneurldxa peuoyippe sdeydsip uowng g1TqH
+
+oy] ‘pepsau JI ‘suoneiodo [ye yYsnosy] NOA sapins ApIespo Avfdsip QO] Joey Z7¢ 9y3—uoeIodo Urea] 0} Ased ‘aus «
+
+jUorel]suowtap & IO} Aepol Jayeap uur’] INOA dag ‘dISHUL
+INOA 0} UONUS}]¥ PaplAIPUN INOA SUTJOASp ITY ps pue
+p1osai ‘asoduod no Jay 0} pausisap st 1s0uenbesuur’] oy)
+Aum Aposiooid $,Jeu], ‘SS9d0Id SATTBS1D OY} YIM SOIOJIOIUT
+yey) xo]dwWI0d Os dq JOA9U P[NoUsS osn NOA AZopOuYdE} oy
+
+ISTUMOIAUIO?) NOAA UOHISOdWIO)
+
+"NOSpr] B Oy ‘AONUTJUT yada 0} seq Maz Se] BY] Jas UdAd
+uvd NOA ‘palisap JJ ‘souanbes Mou ¥B OVUT sjied ou] [Te Adoo
+ATesrewO Ne WI) [IM ONOS ALVAAO JeyIe80} wey}
+,deyd,, 0} UOTOUNJ ONOS ALVA ou] asn usy] ‘saouanbes
+JENPIAIpUt UI (“949 ‘snJOYD ‘aS1OA) UOTIDIS JIseq Yes
+Pl0da1 OF ST ABM JOuIOUY “(812g 666 01 dn) ysnory) ABM
+
+dU} [fe YORI] YORs p10991 0} ST SUOS B 9789I9 0} ABM SUG,
+
+SUOS & SUTVAID
+
+*suoT}oes poJUBMUN
+
+SAOUIOI 0} ABM SWS dU} SoyeIodo SUV ALATAaG
+
+“OBPLIq dy} PUB SNIOY PUOdAS dT]] Ud9MIAQ SIDA ISI
+
+ay) Jo Adoo B JJasuT WYSE NOAA ‘afdwexs 10.f ‘UO JUSIN]JIP
+
+B IO aouaNbas sues OY} UI—JOY OUP 0} UOTIEIO] 9UO WOT]
+$1Bq JAOUI OF NOA sMOTIe WOTIOUNS AdOO/IMASNI OULL
+
+‘SUIPIONAI JIVIS Udy) “OQuINU eq porisop ay} puy
+
+0} CNIMAY 10 ‘CYVM Od LSWA “AEVOOT esn Apduns
+
+sainjeay [PUOHIPPY
+
+‘gouanbas & UTYIIM s]UTOd a1y1dads 3¥ $9100 QnPIOAO OL "UOT}
+-ouns dALLS ATONIS 24) Suisn pasueyo Jo ‘pasesa ‘pappe
+aq osye ABUT S9]ON ‘U0 9q ]IIM 1 “yoeq podeyd uayM
+—aouanbas oy] ul skeyd 71 a10J9q Isnf posers oq 0} d]0U ayy
+ssaid pue ASvwug ploy Aydunis ‘jou Suomm & aseso OL
+
+sunipa
+
+jsesdueyo ureisoid pue ‘fepod ureysns
+‘yonoplalje ‘AWOOTOA ‘UOTyeTNpow ‘pusg youd Surpnyour
+pep10del are $199JJ2 TCTIN [WV iPeqqnpseao aq Aeur syoen
+Ze 07 dn ‘Kem sie Uy *(foeI} JOyOUR OJOS 10 ALLAN
+NOA ssofum) duAS yOaysod ul Avy [[IM Yow] ISI 93 “prooar
+NOA 3[IYM—SUIPIOIA LIBIS PU YORI) TUdIOTJIP B JOaTas
+*y1ed MOU B QNPIsA0 OL, “SuIps0daJ-jods 10} aouanbes mno0k
+UI UOHBIO] Aue ssad0e ATYOIND 0} owt} Aue ye pasn aq AvUE
+SJONUOD FLIVOOT pur ‘ANIMA ‘CYVMaYOd LSVd
+{SUIPIOSAI {IY posesa JOU se So]OU SuTsTXO—
+yous} 3U} OUT poppe aq JIM poteyd sajou yeuonippe Auy
+*(povesjap 10 poysn{pe oq ABW UOTIIII0D BUTUTT]) j{paqoeLI09
+2q ][IM S1OLIe Sur [fe ATUO—patey]d nod Jey Jedy ]],NOA
+‘] req 0] punose yoeq sdoo] sduanbas ay] Udy AA “YOu Yor
+§,sa0uaNbas at} O] SUIT) UI preogday [IW] INO Avy usy3
+AV'1d pue (YOON ssoid Ayduus ‘aousnbes & p1o09es OF,
+
+g0uaNbas & SUIP10I0y]
+
+‘JONWOD s}JouNaI TeuONdGO e
+
+"UOTJEZIUOIYUAS OPOS UIT} FLAWS [euondo e
+
+‘sou .sulddoys, noyyM sayelodo pue yoegdvyd ZuLINp S¥IOM NOLLOANNYOO ONIWILL e
+
+‘onqea ory AY
+
+pojoojes-oid & ye sajou pyoy Aue syeadas ATTeONewWO Ne UOTOUNS [WAdAY OAISNOX e
+‘LSVJ SUnIpS soyeu UOTOUN ASV UA OUlN-[eal SAISNIOXY e
+‘Koy B JO YONO} 941 12 CASOdSNVALL 0g ABU Syde] [Te 10 9UC e
+
+i ASIP Jed
+
+S9}0U OOO‘OTT JOA SpfOy puv SpUOdeS UT SBUOS Xa[AUIOD So10}S DALIP YSIP , 74 € ISCJ-CNIN
+
+jSIOZISOUJUAS
+
+stuoydAjod of 0} dn skeyd A[snoourynuls ‘spouueYd [IW 9T JO duo 0} pousisse oq
+ABUL YORI] YOR ‘syous) oruoydAjod ‘snoouelnurs 7¢ SuTeJUOS ssouUaNbas QO] OY} JO YORA e
+
+‘SJONUOS ATWOOT pur ‘GNIMAY ‘GaVM OA
+LSVd ‘GYOOde AOLS ‘AV Td YIM Jopsocas ade} Yowsj-N[NU O} eps st UOTLISdO @
+LOPNOUT SaINjeoy s[quyIeUlss AUB S.JJ ‘OSN pue UIes] 0} o[duns A[suIzeUe JOA ‘PnJsomod APOUIOITXO
+St 1] “UeIOIsNUL feUOIssajoid oY} 10 JOO} soUBULIOJIJAd pue UOTIsOduIOS 11e-dY1-JO-9}e)s B SI IONUANbDaguUT] ay
+
+JOps1odady soUINbIS [GTI YVAL ZE
+Jgouanbaguury oy
+