Fix pikepdf PdfMatrix deprecation warning; v15.4.3 release notes

Fixes #1187

.docker/Dockerfile.alpine

@@ -0,0 +1,83 @@
+# SPDX-FileCopyrightText: 2023 James R. Barlow
+# SPDX-License-Identifier: MPL-2.0
+
+FROM alpine:3.18 as base
+
+ENV LANG=C.UTF-8
+ENV TZ=UTC
+
+RUN apk add --no-cache \
+    python3 \
+    zlib
+
+FROM base as builder
+
+RUN apk add --no-cache \
+    ca-certificates \
+    git \
+    python3-dev \
+    py3-pip
+
+# On arm64, we need to build cffi from source.
+ARG TARGETPLATFORM
+
+RUN if [ "${TARGETPLATFORM}" == "linux/arm64" ]; then \
+    apk add --no-cache \
+    build-base \
+    autoconf \
+    automake \
+    libtool \
+    zlib-dev \
+    libffi-dev \
+    cairo-dev \
+    pkgconfig \
+    ;  \
+    fi
+
+COPY . /app
+
+WORKDIR /app
+
+RUN python3 -m venv .venv
+
+RUN source .venv/bin/activate \
+    && python3 -m pip install --no-cache-dir --upgrade pip \
+    && python3 -m pip install --no-cache-dir wheel \
+    && python3 -m pip install --no-cache-dir .[test,webservice,watcher]
+
+FROM base
+
+RUN apk add --no-cache \
+    ghostscript \
+    jbig2dec \
+    jbig2enc \
+    pngquant \
+    tesseract-ocr \
+    tesseract-ocr-data-chi_sim \
+    tesseract-ocr-data-deu \
+    tesseract-ocr-data-eng \
+    tesseract-ocr-data-fra \
+    tesseract-ocr-data-osd \
+    tesseract-ocr-data-por \
+    tesseract-ocr-data-spa \
+    ttf-droid \
+    unpaper \
+    && rm -rf /var/cache/apk/*
+
+WORKDIR /app
+
+COPY --from=builder /usr/local/lib/ /usr/local/lib/
+COPY --from=builder /usr/local/bin/ /usr/local/bin/
+
+COPY --from=builder /app/.venv/ /app/.venv/
+
+COPY --from=builder /app/misc/webservice.py /app/
+COPY --from=builder /app/misc/watcher.py /app/
+
+# Copy minimal project files to get the test suite.
+COPY --from=builder /app/pyproject.toml /app/README.md /app/
+COPY --from=builder /app/tests /app/tests
+
+ENV PATH="/app/.venv/bin:${PATH}"
+
+ENTRYPOINT ["/app/.venv/bin/ocrmypdf"]

.github/workflows/build.yml

@@ -28,11 +28,14 @@ jobs:
             python: "3.10"
           - os: ubuntu-22.04
             python: "3.11"
-          #- os: ubuntu-latest
-          #  python: "pypy3.9"
           - os: ubuntu-22.04
             python: "3.9"
             tesseract5: true
+          - os: ubuntu-latest
+            python: "3.12"
+            tesseract5: true
+            #- os: ubuntu-latest
+            #  python: "pypy3.9"
 
     env:
       OS: ${{ matrix.os }}
@@ -44,14 +47,15 @@ jobs:
           fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags
 
       - uses: actions/setup-python@v4
-        name: Install Python
+        name: Setup Python
         with:
           python-version: ${{ matrix.python }}
+          cache: "pip"
 
       - name: Install Tesseract 5
         if: matrix.tesseract5
         run: |
-          sudo add-apt-repository ppa:alex-p/tesseract-ocr-devel
+          sudo add-apt-repository -y ppa:alex-p/tesseract-ocr-devel
 
       - name: Install common packages
         run: |
@@ -109,7 +113,7 @@ jobs:
     strategy:
       matrix:
         os: [macos-latest]
-        python: ["3.10", "3.11"]
+        python: ["3.10", "3.11", "3.12"]
 
     env:
       OS: ${{ matrix.os }}
@@ -129,13 +133,15 @@ jobs:
             ghostscript \
             jbig2enc \
             openjpeg \
+            openssl \
             pngquant \
             tesseract
 
       - uses: actions/setup-python@v4
-        name: Install Python
+        name: Setup Python
         with:
           python-version: ${{ matrix.python }}
+          cache: "pip"
 
       - name: Install Python packages
         run: |
@@ -165,7 +171,7 @@ jobs:
     strategy:
       matrix:
         os: [windows-latest]
-        python: ["3.10", "3.11"]
+        python: ["3.10", "3.11", "3.12"]
 
     env:
       OS: ${{ matrix.os }}
@@ -177,9 +183,10 @@ jobs:
           fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags
 
       - uses: actions/setup-python@v4
-        name: Install Python
+        name: Setup Python
         with:
           python-version: ${{ matrix.python }}
+          cache: "pip"
 
       - name: Install system packages
         run: |
@@ -210,9 +217,10 @@ jobs:
           fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags
 
       - uses: actions/setup-python@v4
-        name: Install Python
+        name: Setup Python
         with:
           python-version: "3.9"
+          cache: "pip"
 
       - name: Make wheels and sdist
         run: |
@@ -268,8 +276,8 @@ jobs:
             ./dist/*.whl
             ./dist/*.tar.gz
 
-  docker:
+  docker_ubuntu:
-    name: Build Docker images
+    name: Build Ubuntu-based Docker image
     needs: [wheel_sdist_linux, test_linux, test_macos, test_windows]
     runs-on: ubuntu-latest
     if: github.event_name != 'pull_request'
@@ -313,4 +321,52 @@ jobs:
             --push \
             --platform linux/arm64/v8,linux/amd64  \
             --tag "${DOCKER_REPOSITORY}/${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}" \
+            --tag "${DOCKER_REPOSITORY}/${DOCKER_IMAGE_NAME}-ubuntu:${DOCKER_IMAGE_TAG}" \
             --file .docker/Dockerfile .
+
+  docker_alpine:
+    name: Build Alpine-based Docker images
+    needs: [wheel_sdist_linux, test_linux, test_macos, test_windows]
+    runs-on: ubuntu-latest
+    if: github.event_name != 'pull_request'
+    steps:
+      - name: Set image tag to release or branch
+        run: echo "DOCKER_IMAGE_TAG=${GITHUB_REF##*/}" >> $GITHUB_ENV
+
+      - name: If main, set to latest
+        run: echo 'DOCKER_IMAGE_TAG=latest' >> $GITHUB_ENV
+        if: env.DOCKER_IMAGE_TAG == 'main'
+
+      - name: Set Docker Hub repository to username
+        run: echo "DOCKER_REPOSITORY=jbarlow83" >> $GITHUB_ENV
+
+      - name: Set image name
+        run: echo "DOCKER_IMAGE_NAME=ocrmypdf-alpine" >> $GITHUB_ENV
+
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: jbarlow83
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Print image tag
+        run: echo "Building image ${DOCKER_REPOSITORY}/${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}"
+
+      - name: Build
+        run: |
+          docker buildx build \
+            --push \
+            --platform linux/amd64  \
+            --tag "${DOCKER_REPOSITORY}/${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}" \
+            --file .docker/Dockerfile.alpine .

.reuse/dep5

@@ -3,7 +3,6 @@ Upstream-Name: OCRmyPDF
 Upstream-Contact: James R. Barlow <james@purplerock.ca>
 Source: https://github.com/ocrmypdf/OCRmyPDF
 
-
 Files:
  .git_archival.txt
  docs/images/logo-social.png
@@ -11,14 +10,13 @@ Files:
  docs/images/logo-square.png
  docs/images/logo-square.svg
  docs/images/logo.svg
- setup.cfg
 Copyright: (C) 2022 James R. Barlow
 License: MPL-2.0
 
 Files:
- .github/ISSUE_TEMPLATE/*.md
+ .github/ISSUE_TEMPLATE/*.yml
  docs/images/macos-workflow.png
-Copyright: (C) 2022 James R. Barlow
+Copyright: (C) 2023 James R. Barlow
 License: CC-BY-SA-4.0
 
 Files:
@@ -34,15 +32,16 @@ Files:
  tests/resources/invalid.pdf
  tests/resources/kcs.pdf
  tests/resources/livecycle.pdf
+ tests/resources/meta.pdf
  tests/resources/missing_docinfo.pdf
  tests/resources/negzero.pdf
  tests/resources/no_contents.pdf
+ tests/resources/tagged*
  tests/resources/toc.pdf
  tests/resources/trivial.pdf
  tests/resources/truetype_font_nomapping.pdf
  tests/resources/type3_font_nomapping.pdf
- misc/screencast/*
+Copyright: (C) 2023 James R. Barlow
-Copyright: (C) 2022 James R. Barlow
 License: CC-BY-SA-4.0
 
 Files:
@@ -52,7 +51,6 @@ Copyright: (C) 2012 SmokeyJoe
 License: GFDL-1.2-or-later or CC-BY-SA-3.0
 
 Files: tests/resources/c02-22.pdf
- tests/resources/congress.jpg
  tests/resources/multipage.pdf
 Copyright: Public domain
 License: public-domain
@@ -72,6 +70,7 @@ Files: tests/resources/linn.png
  tests/resources/ccitt.pdf
  tests/resources/cardinal.pdf
  tests/resources/jbig2.pdf
+ tests/resources/jbig2_baddevicen.pdf
  tests/resources/skew.pdf
  tests/resources/rotated_skew.pdf
  tests/resources/poster.pdf

README.md

@@ -39,8 +39,8 @@ ocrmypdf                      # it's a scriptable command line program
 - Distributes work across all available CPU cores
 - Uses [Tesseract OCR](https://github.com/tesseract-ocr/tesseract) engine to recognize more than [100 languages](https://github.com/tesseract-ocr/tessdata)
 - Keeps your private data private.
-- Scales properly to handle files with thousands of pages
+- Scales properly to handle files with thousands of pages.
-- Battle-tested on millions of PDFs
+- Battle-tested on millions of PDFs.
 
 <img src="misc/screencast/demo.svg" alt="Demo of OCRmyPDF in a terminal session">
 

docs/advanced.rst

@@ -117,14 +117,14 @@ exceed a certain number of megapixels with ``--skip-big``. (A 300 DPI,
 OCR for huge images
 -------------------
 
-Separate from these settings, Tesseract has internal limits on the size
+Tesseract has internal limits on the size
 of images it will process. If you issue
 ``--tesseract-downsample-large-images``, OCRmyPDF will downsample images
 to fit Tesseract limits. (The limits are usually entered only for scanned
 images of oversized media, such as large maps or blueprints exceeding
 110 cm or 43 inches in either dimension, and at high DPI.)
 
-``--tesseract-downsample-above`` adjusts the threshold at which images
+``--tesseract-downsample-above Npixels`` adjusts the threshold at which images
 will be downsampled. By default, only images that exceed any of Tesseract's
 internal limits are downsampled.
 
@@ -195,10 +195,10 @@ In each case OCRmyPDF will search the ``PATH`` environment variable to
 locate the binaries. By modifying the ``PATH`` environment variable, you
 can override the binaries that OCRmyPDF uses.
 
-Changing tesseract configuration variables
+Changing Tesseract configuration variables
 ------------------------------------------
 
-You can override tesseract's default `control
+You can override Tesseract's default `control
 parameters <https://tesseract-ocr.github.io/tessdoc/tess3/ControlParams.html>`__
 with a configuration file.
 
@@ -273,7 +273,7 @@ Unlike ``sandwich`` this renderer is implemented within OCRmyPDF; anyone
 looking to customize how OCR is presented should look here. A major
 disadvantage of this renderer is it not capable of correctly handling
 text outside the Latin alphabet (specifically, it supports the ISO 8859-1
-character). Pull requests to improve the situation are welcome.
+character set). Pull requests to improve the situation are welcome.
 
 Currently, this renderer has the best compatibility with Mozilla's
 PDF.js viewer.
@@ -286,11 +286,31 @@ Rendering and rasterizing options
 .. versionadded:: 14.3.0
 
 The ``--continue-on-soft-render-error`` option allows OCRmyPDF to
-proceed if a page cannot be rasterized rendered. This is useful if you are
+proceed if a page cannot be rasterized/rendered. This is useful if you are
 trying to get the best possible OCR from a PDF that is not well-formed,
 and you are willing to accept some pages that may not visually match the
 input, and that may not OCR well.
 
+Color conversion strategy
+=========================
+
+.. versionadded:: 15.0.0
+
+OCRmyPDF uses Ghostscript to convert PDF to PDF/A. In some cases, this
+conversion requires color conversion. The default strategy is to convert
+using the ``LeaveColorUnchanged`` strategy, which preserves the original
+color space wherever possible (some rare color spaces might still be
+converted).
+
+Usually document scanners produce PDFs in the sRGB color space, and do
+not need to be converted, so the default strategy is appropriate.
+
+Suppose that you have a document that was prepared for professional
+printing in a Separation or CMYK color space, and text was converted to
+curves. In this case, you may want to use a different color conversion
+strategy. The ``--color-conversion-strategy`` option allows you to select a
+different strategy, such as ``RGB``.
+
 Return code policy
 ==================
 

docs/api.rst

@@ -39,9 +39,7 @@ Parent process requirements
 The :func:`ocrmypdf.ocr` function runs OCRmyPDF similar to command line
 execution. To do this, it will:
 
-- create a monitoring thread
+- create worker processes or threads
-- create worker processes (on Linux, forking itself; on Windows and macOS, by
-  spawning)
 - manage the signal flags of its worker processes
 - execute other subprocesses (forking and executing other programs)
 
@@ -54,7 +52,19 @@ processes.
 
 Creating a child process to call :func:`ocrmypdf.ocr()` is suggested. That
 way your application will survive and remain interactive even if
-OCRmyPDF fails for any reason.
+OCRmyPDF fails for any reason. For example:
+
+.. code-block:: python
+
+    from multiprocessing import Process
+
+    def ocrmypdf_process():
+        ocrmypdf.ocr('input.pdf', 'output.pdf')
+
+    def call_ocrmypdf_from_my_app():
+        p = Process(target=ocrmypdf_process)
+        p.start()
+        p.join()
 
 Programs that call :func:`ocrmypdf.ocr()` should also install a SIGBUS signal
 handler (except on Windows), to raise an exception if access to a memory
@@ -89,12 +99,21 @@ your use case.
 Progress monitoring
 -------------------
 
-OCRmyPDF uses the ``tqdm`` package to implement its progress bars.
+OCRmyPDF uses the ``rich`` package to implement its progress bars.
 :func:`ocrmypdf.configure_logging` will set up logging output to
 ``sys.stderr`` in a way that is compatible with the display of the
 progress bar. Use ``ocrmypdf.ocr(...progress_bar=False)`` to disable
 the progress bar.
 
+Standard output
+---------------
+
+OCRmyPDF is strict about not writing to standard output so that
+users can safely use it in a pipeline and produce a valid output
+file. A caller application will have to ensure it does not write to
+standard output either, if it wants to be compatible with this
+behavior and support piping to a file.
+
 Exceptions
 ----------
 
@@ -104,18 +123,4 @@ exceptions, some exceptions related to multiprocessing, and
 handler. OCRmyPDF will clean up its temporary files and worker processes
 automatically when an exception occurs.
 
-Programs that call OCRmyPDF should consider trapping KeyboardInterrupt
-so that they allow OCR to terminate with the whole program terminating.
-
 When OCRmyPDF succeeds conditionally, it returns an integer exit code.
-
-Reference
----------
-
-.. autofunction:: ocrmypdf.ocr
-
-.. autoclass:: ocrmypdf.Verbosity
-    :members:
-    :undoc-members:
-
-.. autofunction:: ocrmypdf.configure_logging

docs/apiref.rst

@@ -3,11 +3,11 @@
 .. SPDX-License-Identifier: CC-BY-SA-4.0
 
 =============
-API Reference
+API reference
 =============
 
 This page summarizes the rest of the public API. Generally speaking this
-should mainly of interest to plugin developers.
+should be mainly of interest to plugin developers.
 
 ocrmypdf
 ========
@@ -18,6 +18,18 @@ ocrmypdf
 .. autoclass:: ocrmypdf.PdfContext
     :members:
 
+.. autoclass:: ocrmypdf.Verbosity
+    :members:
+    :undoc-members:
+
+.. autofunction:: ocrmypdf.configure_logging
+
+.. autofunction:: ocrmypdf.ocr
+
+.. autofunction:: ocrmypdf.pdf_to_hocr
+
+.. autofunction:: ocrmypdf.hocr_to_ocr_pdf
+
 ocrmypdf.exceptions
 ===================
 

docs/batch.rst

@@ -152,7 +152,7 @@ The watcher service is included in the OCRmyPDF Docker image. To run it:
     docker run \
         --volume <path to files to convert>:/input \
         --volume <path to store results>:/output \
-        --volume <path to store processed originals>:/archive \
+        --volume <path to store processed originals>:/processed \
         --env OCR_OUTPUT_DIRECTORY_YEAR_MONTH=1 \
         --env OCR_ON_SUCCESS_ARCHIVE=1 \
         --env OCR_DESKEW=1 \
@@ -163,7 +163,7 @@ The watcher service is included in the OCRmyPDF Docker image. To run it:
 
 This service will watch for a file that matches ``/input/\*.pdf``,
 convert it to a OCRed PDF in ``/output/``, and move the processed
-original to ``/archive``. The parameters to this image are:
+original to ``/processed``. The parameters to this image are:
 
 .. csv-table:: watcher.py parameters for Docker
     :header: "Parameter", "Description"
@@ -171,7 +171,7 @@ original to ``/archive``. The parameters to this image are:
 
     "``--volume <path to files to convert>:/input``", "Files placed in this location will be OCRed"
     "``--volume <path to store results>:/output``", "This is where OCRed files will be stored"
-    "``--volume <path to store processed originals>:/archive``", "Archive processed originals here"
+    "``--volume <path to store processed originals>:/processed``", "Archive processed originals here"
     "``--env OCR_OUTPUT_DIRECTORY_YEAR_MONTH=1``", "Define environment variable ``OCR_OUTPUT_DIRECTORY_YEAR_MONTH=1`` to place files in the output in ``{output}/{year}/{month}/{filename}``"
     "``--env OCR_ON_SUCCESS_ARCHIVE=1``", "Define environment variable ``OCR_ON_SUCCESS_ARCHIVE`` to move processed originals"
     "``--env OCR_DESKEW=1``", "Define environment variable ``OCR_DESKEW``  to apply deskew to crooked input PDFs"

docs/cloud.rst

@@ -1,3 +1,8 @@
+.. SPDX-FileCopyrightText: 2023 James R. Barlow
+..
+.. SPDX-License-Identifier: CC-BY-SA-4.0
+
+
 .. _ocr-service:
 
 ==================

docs/contributing.rst

@@ -19,7 +19,7 @@ Code style
 ==========
 
 We use PEP8, ``black`` for code formatting and ``ruff`` for everything else. The
-settings for these programs are in ``pyproject.toml`` and ``setup.cfg``. Pull
+settings for these programs are in ``pyproject.toml``. Pull
 requests should follow the style guide. One difference we use from "black" style
 is that strings shown to the user are always in double quotes (``"``) and strings
 for internal uses are in single quotes (``'``).

docs/docker.rst

@@ -8,17 +8,16 @@
 OCRmyPDF Docker image
 =====================
 
-OCRmyPDF is also available in a Docker image that packages recent
+OCRmyPDF is also available in Docker images that packages recent
 versions of all dependencies.
 
 For users who already have Docker installed this may be an easy and
-convenient option. However, it is less performant than a system
+convenient option.
-installation and may require Docker engine configuration.
 
-OCRmyPDF needs a generous amount of RAM, CPU cores, temporary storage
+On platforms other than Linux, Docker runs in a virtual machine, and so may
-space, whether running in a Docker container or on its own. It may be
+be less performant. You may also want to adjust the Docker virtual machine's
-necessary to ensure the container is provisioned with additional
+memory and CPU allocation. On Linux, the Docker image runs natively and
-resources.
+performance is comparable to a system installation.
 
 .. _docker-install:
 
@@ -35,28 +34,37 @@ execute the image:
 
    docker run hello-world
 
-The recommended OCRmyPDF Docker image is currently named ``ocrmypdf``:
+.. list-table:: Docker images
+   :width: 30 20 50
+   :header-rows: 1
+
+   * - Image
+     - Architecture
+     - Description
+   * - ``jbarlow83/ocrmypdf-alpine``
+     - x86_64 only
+     - Recommended image, based on Alpine Linux.
+   * - ``jbarlow83/ocrmypdf-ubuntu``
+     - x86_64 and arm64
+     - Alternate image, based on Ubuntu. When the Alpine image is considered
+       stable and available for arm64, this image will be deprecated.
+   * - ``jbarlow83/ocrmypdf``
+     - x86_64 and arm64
+     - Currently an alias for ocrmypdf-ubuntu. When the Alpine image is
+       considered stable and available for arm64, this name point to the
+       Alpine image. If you don't about the difference between Alpine and
+       Ubuntu, use this image.
+
+To install:
 
 .. code-block:: bash
 
-   docker pull jbarlow83/ocrmypdf
+   docker pull jbarlow83/ocrmypdf-alpine
 
+The ``ocrmypdf`` image is also available, but is deprecated and will be removed
+in the future.
 
-OCRmyPDF will use all available CPU cores. By default, the VirtualBox
+OCRmyPDF will use all available CPU cores. See the Docker documentation for
-machine instance on Windows and macOS has only a single CPU core
-enabled. Use the VirtualBox Manager to determine the name of your Docker
-engine host, and then follow these optional steps to enable multiple
-CPUs:
-
-.. code-block:: bash
-
-   # Optional step for Mac OS X users
-   docker-machine stop "yourVM"
-   VBoxManage modifyvm "yourVM" --cpus 2  # or whatever number of core is desired
-   docker-machine start "yourVM"
-   eval $(docker-machine env "yourVM")
-
-See the Docker documentation for
 `adjusting memory and CPU on other platforms <https://docs.docker.com/config/containers/resource_constraints/>`__.
 
 Using the Docker image on the command line
@@ -66,6 +74,8 @@ Using the Docker image on the command line
 container is ephemeral – it runs for one OCR job and terminates, just like a
 command line program. We are using Docker to deliver an application (as opposed
 to the more conventional case, where a Docker container runs as a server).
+For that reason we usually use the ``--rm`` argument to delete the container
+when it exits.
 
 To start a Docker container (instance of the image):
 
@@ -132,17 +142,35 @@ You can then add new data with either a Dockerfile:
 
 .. code-block:: dockerfile
 
-   FROM jbarlow83/ocrmypdf
+   FROM jbarlow83/ocrmypdf:{TAG}
 
    # Example: add a tessdata_best file
    COPY chi_tra_vert.traineddata /usr/share/tesseract-ocr/<data version>/tessdata/
 
+When creating your own image, you should always pin a specific version of the
+OCRmyPDF Docker image. This ensures that your image will not break when a new
+version of OCRmyPDF is released.
+
 Alternately, you can copy training data into a Docker container as follows:
 
 .. code-block:: bash
 
    docker cp mycustomtraining.traineddata name_of_container:/usr/share/tesseract-ocr/<tesseract version>/tessdata/
 
+Extending the Docker image
+==========================
+
+You can extend the Docker image with your own customizations, similar to the way
+it is extended to add language packs.
+
+Note that the Docker image is subject to change at any time. For example, the base
+image may be updated to a newer version of Ubuntu or Debian. Such changes will be
+noted in the release notes but might occur at minor versions releases, unless the
+way a "casual" user of the Docker image is affected.
+
+If you extend the Docker image, you should pin a specific version of the OCRmyPDF
+Docker image.
+
 Executing the test suite
 ========================
 
@@ -150,16 +178,16 @@ The OCRmyPDF test suite is installed with image. To run it:
 
 .. code-block:: bash
 
-   docker run --entrypoint python3  jbarlow83/ocrmypdf -m pytest
+   docker run --rm --entrypoint python  jbarlow83/ocrmypdf -m pytest
 
 Accessing the shell
 ===================
 
-To use the bash shell in the Docker image:
+To use the shell in the Docker image:
 
 .. code-block:: bash
 
-   docker run -it --entrypoint bash  jbarlow83/ocrmypdf
+   docker run -it --entrypoint sh  jbarlow83/ocrmypdf
 
 Using the OCRmyPDF web service wrapper
 ======================================
@@ -169,7 +197,10 @@ service. The webservice may be launched as follows:
 
 .. code-block:: bash
 
-   docker run --entrypoint python3 -p 5000:5000  jbarlow83/ocrmypdf webservice.py
+   docker run --entrypoint python -p 5000:5000  jbarlow83/ocrmypdf webservice.py
+
+We omit the ``--rm`` parameter so that the container will not be
+automatically deleted when it exits.
 
 This will configure the machine to listen on port 5000. On Linux machines
 this is port 5000 of localhost. On macOS or Windows machines running

docs/index.rst

@@ -12,7 +12,7 @@ files, allowing them to be searched.
 
 PDF is the best format for storing and exchanging scanned documents.
 Unfortunately, PDFs can be difficult to modify. OCRmyPDF makes it easy to apply
-image processing and OCR to existing PDFs.
+image processing and OCR (recognized, searchable text) to existing PDFs.
 
 .. toctree::
    :maxdepth: 1

docs/installation.rst

@@ -72,7 +72,7 @@ Debian and Ubuntu 20.04 or newer
 | |ubu-2004| |ubu-2204|                         |
 +-----------------------------------------------+
 
-Users of Debian 11, or Ubuntu 20.04 LTS, or newer may simply
+Users of Debian or Ubuntu may simply
 
 .. code-block:: bash
 
@@ -99,11 +99,11 @@ For full details on version availability for your platform, check the
 Fedora
 ------
 
-.. |fedora-35| image:: https://repology.org/badge/version-for-repo/fedora_35/ocrmypdf.svg
+.. |fedora-37| image:: https://repology.org/badge/version-for-repo/fedora_37/ocrmypdf.svg
-    :alt: Fedora 35
+    :alt: Fedora 37
 
-.. |fedora-36| image:: https://repology.org/badge/version-for-repo/fedora_36/ocrmypdf.svg
+.. |fedora-38| image:: https://repology.org/badge/version-for-repo/fedora_38/ocrmypdf.svg
-    :alt: Fedora 36
+    :alt: Fedora 38
 
 .. |fedora-rawhide| image:: https://repology.org/badge/version-for-repo/fedora_rawhide/ocrmypdf.svg
     :alt: Fedore Rawhide
@@ -113,10 +113,10 @@ Fedora
 +-----------------------------------------------+
 | |latest|                                      |
 +-----------------------------------------------+
-| |fedora-35| |fedora-36| |fedora-rawhide|      |
+| |fedora-37| |fedora-38| |fedora-rawhide|      |
 +-----------------------------------------------+
 
-Users of Fedora 29 or later may simply
+Users of Fedora may simply
 
 .. code-block:: bash
 
@@ -155,7 +155,7 @@ user, follow these steps:
 
 If you get the message ``WARNING: The script ocrmypdf is installed in
 '/home/$USER/.local/bin' which is not on PATH.``, you may need to re-login
-or open a new shell, or manually add this to your user's PATH.
+or open a new shell, or manually adjust your PATH.
 
 To add JBIG2 encoding, see :ref:`jbig2`.
 
@@ -382,29 +382,33 @@ Native Windows
 
 You must install the following for Windows:
 
-* Python 3.9 (64-bit) or later
+* Python 64-bit
-* Tesseract 4.1.1 (64-bit) or later
+* Tesseract 64-bit
-* Ghostscript 9.50 (64-bit) or later
+* Ghostscript 64-bit
 
-Using the `Chocolatey <https://chocolatey.org/>`_ package manager, install the
+Using the `winget <https://docs.microsoft.com/en-us/windows/package-manager/winget/>`_
-following when running in an Administrator command prompt:
+package manager:
+
+* ``winget install -e --id Python.Python.3.11``
+* ``winget install -e --id UB-Mannheim.TesseractOCR``
+* ``winget install -e --id ArtifexSoftware.GhostScript``
+
+
+(Or alternately, using the `Chocolatey <https://chocolatey.org/>`_ package manager, install
+the following when running in an Administrator command prompt):
 
 * ``choco install python3``
 * ``choco install --pre tesseract``
 * ``choco install ghostscript``
 * ``choco install pngquant`` (optional)
 
-The commands above will install Python 3.x (latest version), Tesseract, Ghostscript
+Either set of commands will install the required software. At the mmoment there is no
-and pngquant. Chocolatey may also need to install the Windows Visual C++ Runtime
+single command to install Windows.
-DLLs or other Windows patches, and may require a reboot.
 
 You may then use ``pip`` to install ocrmypdf. (This can performed by a user or
 Administrator.):
 
-* ``pip install ocrmypdf``
+* ``python3 -m pip install ocrmypdf``
-
-Chocolatey automatically selects appropriate versions of these applications. Please make sure
-you are installing the 64-bit versions.
 
 OCRmyPDF will check the Windows Registry and standard locations in your Program Files
 for third party software it needs (specifically, Tesseract and Ghostscript). To
@@ -416,12 +420,12 @@ to change the PATH.
 
     As of early 2021, users have reported problems with the Microsoft Store version of
     Python and OCRmyPDF. These issues affect many other third party Python packages.
-    Please download Python from Python.org or Chocolatey instead, and do not use the
+    Please download Python from Python.org or a package manager instead of the
     Microsoft Store version.
 
 .. warning::
 
-    32-bit Windows might work, but is not supported.
+    32-bit Windows is not supported.
 
 Windows Subsystem for Linux
 ---------------------------
@@ -448,7 +452,7 @@ Cygwin64
 
 First install the the following prerequisite Cygwin packages using ``setup-x86_64.exe``::
 
-    python38 (or later)
+    python39 (or later)
     python3?-devel
     python3?-pip
     python3?-lxml
@@ -457,7 +461,7 @@ First install the the following prerequisite Cygwin packages using ``setup-x86_6
        (where 3? means match the version of python3 you installed)
 
     gcc-g++
-    ghostscript (<=9.50 or >=9.52-2 see note below)
+    ghostscript
     libexempi3
     libexempi-devel
     libffi6
@@ -468,13 +472,6 @@ First install the the following prerequisite Cygwin packages using ``setup-x86_6
     tesseract-ocr
     tesseract-ocr-devel
 
-.. note::
-
-    The Cygwin package for Ghostscript in versions 9.52 and
-    9.52-1 contained a bug that caused an exception to occur when
-    ocrmypdf invoked gs.  Make sure you have either 9.50 (or earlier)
-    or 9.52-2 (or later).
-
 Then open a Cygwin terminal (i.e. ``mintty``), run the following commands. Note
 that if you are using the version of ``pip`` that was installed with the Cygwin
 Python package, the command name will be ``pip3``.  If you have since updated
@@ -554,12 +551,15 @@ manager. ``pip`` cannot provide them.
 The following versions are required:
 
 -  Python 3.9 or newer
--  Ghostscript 9.50 or newer
+-  Ghostscript 9.55 or newer
 -  Tesseract 4.1.1 or newer
 -  jbig2enc 0.29 or newer
 -  pngquant 2.5 or newer
 -  unpaper 6.1
 
+We recommend 64-bit versions of all software. (32-bit versions are not
+supported, although on Linux, they may still work.)
+
 jbig2enc, pngquant, and unpaper are optional. If missing certain
 features are disabled. OCRmyPDF will discover them as soon as they are
 available.
@@ -665,3 +665,18 @@ To manually install the ``bash`` completion, copy
 To manually install the ``fish`` completion, copy
 ``misc/completion/ocrmypdf.fish`` to
 ``~/.config/fish/completions/ocrmypdf.fish``.
+
+Note on 32-bit support
+======================
+
+Many Python libraries no longer 32-bit binary wheels for Linux. This
+includes many of the libraries that OCRmyPDF depends on, such as
+Pillow. The easiest way to express this to end users is to say we don't
+support 32-bit Linux.
+
+However, if your Linux distribution still supports 32-bit binaries, you
+can still install and use OCRmyPDF. A warning message will appear.
+In practice, OCRmyPDF may need more than 32-bit memory space to run when
+large documents are processed, so there are practical limitations to what
+users can accomplish with it. Still, for the common use case of an 32-bit
+ARM NAS or Raspberry Pi processing small documents, it should work.

docs/introduction.rst

@@ -6,23 +6,23 @@
 Introduction
 ============
 
-OCRmyPDF is an application and library that adds text "layers" to images
+OCRmyPDF is a Python application and library that adds text "layers" to images in
-in PDFs, making scanned image PDFs searchable. It uses OCR to guess what text
+PDFs, making scanned image PDFs searchable. It uses OCR to guess the text
-is contained in images. It is written in Python. OCRmyPDF supports plugins
+contained in images. OCRmyPDF also supports plugins
-that allow customization of its processing steps, and is very tolerant of
+that enable customization of its processing steps, and it is highly tolerant
-PDFs that contain scanned images and "born digital" content that needs no
+of PDFs containing scanned images and "born digital" content that doesn't
-text recognition.
+require text recognition.
 
 About OCR
 =========
 
 `Optical character
 recognition <https://en.wikipedia.org/wiki/Optical_character_recognition>`__
-is technology that converts images of typed or handwritten text, such as
+is a technology that converts images of typed or handwritten text, such as
-in a scanned document, to computer text that can be selected, searched and copied.
+in a scanned document, into computer text that can be selected, searched and copied.
 
 OCRmyPDF uses
-`Tesseract <https://github.com/tesseract-ocr/tesseract>`__, the best
+`Tesseract <https://github.com/tesseract-ocr/tesseract>`__, a widely
 available open source OCR engine, to perform OCR.
 
 .. _raster-vector:
@@ -30,19 +30,19 @@ available open source OCR engine, to perform OCR.
 About PDFs
 ==========
 
-PDFs are page description files that attempts to preserve a layout
+PDFs are page description files that attempt to preserve a layout
 exactly. They contain `vector
 graphics <http://vector-conversions.com/vectorizing/raster_vs_vector.html>`__
-that can contain raster objects such as scanned images. Because PDFs can
+that can contain raster objects, such as scanned images. Because PDFs can
 contain multiple pages (unlike many image formats) and can contain fonts
-and text, it is a good format for exchanging scanned documents.
+and text, they are a suitable format for exchanging scanned documents.
 
 |image|
 
-A PDF page might contain multiple images, even if it only appears to
+A PDF page may contain multiple images, even if it appears to have only
-have one image. Some scanners or scanning software will segment pages
+one image. Some scanners or scanning software may segment pages into
-into monochromatic text and color regions for example, to improve the
+monochromatic text and color regions, for example, to enhance the compression
-compression ratio and appearance of the page.
+ratio and appearance of the page.
 
 Rasterizing a PDF is the process of generating corresponding raster images.
 OCR engines like Tesseract work with images, not scalable vector graphics
@@ -54,147 +54,131 @@ About PDF/A
 `PDF/A <https://en.wikipedia.org/wiki/PDF/A>`__ is an ISO-standardized
 subset of the full PDF specification that is designed for archiving (the
 'A' stands for Archive). PDF/A differs from PDF primarily by omitting
-features that would make it difficult to read the file in the future,
+features that could complicate future file readability,
 such as embedded Javascript, video, audio and references to external
 fonts. All fonts and resources needed to interpret the PDF must be
 contained within it. Because PDF/A disables Javascript and other types
-of embedded content, it is probably more secure.
+of embedded content, it is likely more secure.
 
 There are various conformance levels and versions, such as "PDF/A-2b".
 
-Generally speaking, the best format for scanned documents is PDF/A. Some
+In general, the preferred format for scanned documents is PDF/A. Some
 governments and jurisdictions, US Courts in particular, `mandate the use
 of PDF/A <https://pdfblog.com/2012/02/13/what-is-pdfa/>`__ for scanned
 documents.
 
-Since most people who scan documents are interested in reading them
+Since most individuals scanning documents aim for long-term readability,
-indefinitely into the future, OCRmyPDF generates PDF/A-2b by default.
+OCRmyPDF defaults to generating PDF/A-2b.
 
-PDF/A has a few drawbacks. Some PDF viewers include an alert that the
+PDF/A does have a few drawbacks. Some PDF viewers display an alert
-file is a PDF/A, which may confuse some users. It also tends to produce
+indicating that the file is in PDF/A format, which may confuse some users.
-larger files than PDF, because it embeds certain resources even if they
+Additionally, it tends to result in larger files than standard PDFs because
-are commonly available. PDF/A files can be digitally signed, but may not
+it embeds certain resources, even if they are widely available. PDF/A
-be encrypted, to ensure they can be read in the future. Fortunately,
+files can be digitally signed but may not be encrypted to ensure future
-converting from PDF/A to a regular PDF is trivial, and any PDF viewer
+readability. Fortunately, converting from PDF/A to a regular PDF is
-can view PDF/A.
+straightforward, and any PDF viewer can handle PDF/A files.
 
 What OCRmyPDF does
 ==================
 
-OCRmyPDF analyzes each page of a PDF to determine the colorspace and
+OCRmyPDF analyzes each page of a PDF to determine the required colorspace
-resolution (DPI) needed to capture all of the information on that page
+and resolution (DPI) for capturing all the information on that page without
-without losing content. It uses
+losing content. It uses
-`Ghostscript <http://ghostscript.com/>`__ to rasterize the page, and
+`Ghostscript <http://ghostscript.com/>`__ to rasterize each page and subsequently
-then performs OCR on the rasterized image to create an OCR "layer".
+performs OCR on the rasterized image to generate an OCR "layer." This layer
-The layer is then grafted back onto the original PDF.
+is then integrated back into the original PDF.
 
-While one can use a program like Ghostscript or ImageMagick to get an
+While it is possible to use a program like Ghostscript or ImageMagick to
-image and put the image through Tesseract, that actually creates a new
+obtain an image and then run that image through Tesseract OCR, this process
-PDF and many details may be lost. OCRmyPDF can produce a minimally
+actually generates a new PDF, potentially resulting in the loss of various
-changed PDF as output.
+details (such as the document's metadata). In contrast, OCRmyPDF can produce
+a minimally altered PDF as the output.
 
-OCRmyPDF also provides some image processing options, like deskew, which
+OCRmyPDF also offers several image processing options, such as deskew, which
-improves the appearance of files and quality of OCR. When these are used,
+enhances the visual quality of files and the accuracy of OCR. When these
-the OCR layer is grafted onto the processed image instead.
+options are utilized, the OCR layer is integrated into the processed image.
 
-By default, OCRmyPDF produces archival PDFs – PDF/A, which are a
+By default, OCRmyPDF generates archival PDFs in the PDF/A format, which is
-stricter subset of PDF features designed for long term archives. If
+a more rigid subset of PDF features designed for long-term archives. If you
-regular PDFs are desired, this can be disabled with
+prefer regular PDFs, you can disable this feature using the
-``--output-type pdf``.
+``--output-type pdf`` option.
 
 Why you shouldn't do this manually
 ==================================
 
 A PDF is similar to an HTML file, in that it contains document structure
-along with images. Sometimes a PDF does nothing more than present a full
+along with images. While some PDFs may solely display a full-page image,
-page image, but often there is additional content that would be lost.
+they often contain additional content that would be forfeited if not preserved.
 
-A manual process could work like either of these:
+A manual process could take one of these approaches:
 
-1. Rasterize each page as an image, OCR the images, and combine the
+1. Rasterize each page as an image, perform OCR on the images, and then merge the
-   output into a PDF. This preserves the layout of each page, but
+   output into a PDF. This method preserves the layout of each page, but
-   resamples all images (possibly losing quality, increasing file size,
+   resamples all images potentially leading to quality loss, increased file size,
-   introducing compression artifacts, etc.).
+   and the introduction of compression artifacts, among other issues.
-2. Extract each image, OCR, and combine the output into a PDF. This
+2. Extract each image, OCR, and combine the output into a PDF. This approach
-   loses the context in which images are used in the PDF, meaning that
+   loses the context in which images are used in the PDF, potentially resulting
-   cropping, rotation and scaling of pages may be lost. Some scanned
+   in loss of information related to scaling and position of images. Some scanned
-   PDFs use multiple images segmented into black and white, grayscale
+   PDFs contain multiple images segmented into black and white, grayscale
    and color regions, with stencil masks to prevent overlap, as this can
-   enhance the appearance of a file while reducing file size. Clearly,
+   enhance the appearance of a file while reducing file size.
-   reassembling these images will be easy. This also loses and text or
+   Reassembling these images can be challenging, and risks losing vector art
-   vector art on any pages in a PDF with both scanned and pure digital
+   or text that is not part of an image.
-   content.
 
-In the case of a PDF that is nothing other than a container of images
+In cases where a PDF solely serves as a container for images without any
-(no rotation, scaling, cropping, one image per page), the second
+rotation, scaling, or cropping, the second approach can be lossless.
-approach can be lossless.
 
-OCRmyPDF uses several strategies depending on input options and the
+OCRmyPDF uses various strategies depending on input options and the input PDF
-input PDF itself, but generally speaking it rasterizes a page for OCR
+itself. Generally, it rasterizes a page for OCR and then integrates the OCR
-and then grafts the OCR back onto the original. As such it can handle
+data back into the original PDF. This approach allows it to handle complex
-complex PDFs and still preserve their contents as much as possible.
+PDFs and preserve their content as much as possible.
 
-OCRmyPDF also supports a many, many edge cases that have cropped over
+Furthermore, OCRmyPDF supports a wide range of edge cases that have emerged
-several years of development. We support PDF features like images inside
+during several years of development. It accommodates PDF features like
-of Form XObjects, and pages with UserUnit scaling. We support rare image
+images within Form XObjects and pages with UserUnit scaling. It also
-formats like non-monochrome 1-bit images. We warn about files you may
+supports less common image formats like non-monochrome 1-bit images and
-not to OCR. Thanks to pikepdf and QPDF, we auto-repair PDFs that are
+provides warnings about files you may not want to OCR. Thanks to tools
-damaged. (Not that you need to know what any of these are! You should be
+like pikepdf and QPDF, it can auto-repair damaged PDFs. You don't need to
-able to throw any PDF at it.)
+understand the intricacies of these issues; you should be able to use
+OCRmyPDF with any PDF file, and expect reasonable results.
 
 Limitations
 ===========
 
-OCRmyPDF is limited by the Tesseract OCR engine. As such it experiences
+OCRmyPDF is subject to limitations imposed by the Tesseract OCR engine.
-these limitations, as do any other programs that rely on Tesseract:
+These limitations are inherent to any software relying on Tesseract:
 
--  The OCR is not as accurate as commercial OCR solutions.
+-  The OCR accuracy may not match that of commercial OCR solutions.
--  It is not capable of recognizing handwriting.
+-  It is incapable of recognizing handwriting.
--  It may find gibberish and report this as OCR output.
+-  It may detect gibberish and report it as OCR output.
--  If a document contains languages outside of those given in the
+-  Results may be subpar when a document contains languages not specified
-   ``-l LANG`` arguments, results may be poor.
+   in the ``-l LANG`` argument.
--  It is not always good at analyzing the natural reading order of
+-  Tesseract may struggle to analyze the natural reading order of documents.
-   documents. For example, it may fail to recognize that a document
+   For instance, it might fail to recognize two columns in a document and
-   contains two columns, and may try to join text across columns.
+   attempt to join text across columns.
--  Poor quality scans may produce poor quality OCR. Garbage in, garbage
+-  Poor quality scans can result in subpar OCR quality. In other words, the
-   out.
+   quality of the OCR output depends on the quality of the input.
--  It does not expose information about what font family text belongs
+-  Tesseract does not provide information about the font family to which text
-   to.
+   belongs.
-
+-  Tesseract does not divide text into paragraphs or headings. It only provides
-OCRmyPDF is also limited by the PDF specification:
+   the text and its bounding box. As such, the generated PDF does not
-
+   contain any information about the document's structure.
--  PDF encodes the position of text glyphs but does not encode document
-   structure. There is no markup that divides a document in sections,
-   paragraphs, sentences, or even words (since blank spaces are not
-   represented). As such all elements of document structure including
-   the spaces between words must be derived heuristically. Some PDF
-   viewers do a better job of this than others.
--  Because some popular open source PDF viewers have a particularly hard
-   time with spaces between words, OCRmyPDF appends a space to each text
-   element as a workaround (when using ``--pdf-renderer hocr``). While
-   this mixes document structure with graphical information that ideally
-   should be left to the PDF viewer to interpret, it improves
-   compatibility with some viewers and does not cause problems for
-   better ones.
 
 Ghostscript also imposes some limitations:
 
--  PDFs containing JBIG2-encoded content will be converted to CCITT
+-  PDFs containing JPEG 2000-encoded content may be converted to JPEG
-   Group4 encoding, which has lower compression ratios, if Ghostscript
-   PDF/A is enabled.
--  PDFs containing JPEG 2000-encoded content will be converted to JPEG
    encoding, which may introduce compression artifacts, if Ghostscript
    PDF/A is enabled.
--  Ghostscript may transcode grayscale and color images, either lossy to
+-  Ghostscript may transcode grayscale and color images, potentially
-   lossless or lossless to lossy, based on an internal algorithm. This
+   lossily, based on an internal algorithm. This
    behavior can be suppressed by setting ``--pdfa-image-compression`` to
    ``jpeg`` or ``lossless`` to set all images to one type or the other.
-   Ghostscript has no option to maintain the input image's format.
+   Ghostscript lacks an option to maintain the input image's format.
    (Modern Ghostscript can copy JPEG images without transcoding them.)
 -  Ghostscript's PDF/A conversion removes any XMP metadata that is not
    one of the standard XMP metadata namespaces for PDFs. In particular,
    PRISM Metadata is removed.
--  Ghostscript's PDF/A conversion seems to remove or deactivate
+-  Ghostscript's PDF/A conversion may remove or deactivate
    hyperlinks and other active content.
 
 You can use ``--output-type pdf`` to disable PDF/A conversion and produce
@@ -202,7 +186,7 @@ a standard, non-archival PDF.
 
 Regarding OCRmyPDF itself:
 
--  PDFs that use transparency are not currently represented in the test
+-  PDFs using transparency are not currently represented in the test
    suite
 
 Similar programs
@@ -210,11 +194,7 @@ Similar programs
 
 To the author's knowledge, OCRmyPDF is the most feature-rich and
 thoroughly tested command line OCR PDF conversion tool. If it does not
-meet your needs, contributions and suggestions are welcome. If not,
+meet your needs, contributions and suggestions are welcome.
-consider one of these similar open source programs:
-
--  pdf2pdfocr
--  pdfsandwich
 
 Ghostscript recently added three "pdfocr" output devices. They work by
 rasterizing all content and converting all pages to a single colour space.
@@ -222,16 +202,19 @@ rasterizing all content and converting all pages to a single colour space.
 Web front-ends
 ==============
 
-The Docker image ``ocrmypdf`` provides a web service front-end
+The Docker image of OCRmyPDF provides a web service front-end
-that allows files to submitted over HTTP and the results "downloaded".
+that allows files to submitted over HTTP, and the results can be downloaded.
-This is an HTTP server intended to simplify web services deployments; it
+This is an HTTP server intended to demonstrate how OCRmyPDF can be
-is not intended to be deployed on the public internet and no real
+integrated into a web service. It is not intended to be deployed on the
-security measures to speak of.
+public internet and does not provide any security measures.
 
 In addition, the following third-party integrations are available:
 
+-  `Paperless-ngx <https://docs.paperless-ngx.com/>`__ is a free software
+   document management system that uses OCRmyPDF to perform OCR on
+   uploaded documents.
 -  `Nextcloud OCR <https://github.com/janis91/ocr>`__ is a free software
-   plugin for the Nextcloud private cloud software
+   plugin for the Nextcloud private cloud software.
 
 OCRmyPDF is not designed to be secure against malware-bearing PDFs (see
 `Using OCRmyPDF online <ocr-service>`__). Users should ensure they

docs/jbig2.rst

@@ -14,17 +14,20 @@ expired as of 2017, but it is possible that unknown patents exist.
 
 JBIG2 encoding is recommended for OCRmyPDF and is used to losslessly
 create smaller PDFs. If JBIG2 encoding is not available, lower quality
-encodings will be used.
+CCITT encoding will be used for monochrome images.
 
 JBIG2 decoding is not patented and is performed automatically by most
 PDF viewers. It is widely supported and has been part of the PDF
 specification since 2001.
 
-On macOS, Homebrew packages jbig2enc and OCRmyPDF includes it by
+JBIG encoding is automatically provided by these OCRmyPDF packages:
-default. The Docker image for OCRmyPDF also builds its own JBIG2 encoder
+- Docker image (both Ubuntu and Alpine)
-from source.
+- Snap package
+- ArchLinux AUR package
+- Alpine Linux package
+- Homebrew on macOS
 
-For all other Linux, you must build a JBIG2 encoder from source:
+For all other platforms, you would need to build the JBIG2 encoder from source:
 
 .. code-block:: bash
 
@@ -43,21 +46,30 @@ as libtool and leptonica-devel.
 Lossy mode JBIG2
 ================
 
-OCRmyPDF provides lossy mode JBIG2 as an advanced feature. Users should
+OCRmyPDF provides lossy mode JBIG2 as an advanced and potentially dangerous
+feature. Users should
 `review the technical concerns with JBIG2 in lossy
 mode <https://en.wikipedia.org/wiki/JBIG2#Disadvantages>`__
-and decide if this feature is acceptable for their use case.
+and decide if this feature is acceptable for their use case. In general,
+this mode should not be used for archival purposes, should not be used when
+the original document is not available or will be destroyed, and should
+not be used when numbers present in the document are important, because
+there is a risk of 6/8 and 8/6 substitution errors.
 
 JBIG2 lossy mode does achieve higher compression ratios than any other
 monochrome (bitonal) compression technology; for large text documents
 the savings are considerable. JBIG2 lossless still gives great
 compression ratios and is a major improvement over the older CCITT G4
-standard. As explained above, there is some risk of substitution errors.
+standard.
 
 To turn on JBIG2 lossy mode, add the argument ``--jbig2-lossy``.
 ``--optimize {1,2,3}`` are necessary for the argument to take effect
 also required. Also, a JBIG2 encoder must be installed as described in
 the previous section.
 
+You can adjust the threshold for JBIG2 compression with the
+``--jbig2-threshold``. The default is 0.85, meaning that if two symbols
+are 85% similar, they will be compressed together.
+
 *Due to an oversight, ocrmypdf v7.0 and v7.1 used lossy mode by
 default.*

docs/maintainers.rst

@@ -56,4 +56,12 @@ improve OCRmyPDF's compression.
 Command line completions
 ------------------------
 
-Please ensure that command line completions are installed.
+Please ensure that command line completions are installed, as described in the
+installation documentation.
+
+32-bit Linux support
+--------------------
+
+If you maintain a Linux distribution that supports 32-bit x86 or ARM, OCRmyPDF
+should continue to work as long as all of its dependencies continue to be
+available in 32-bit form. Please note we do not test on 32-bit platforms.

docs/optimizer.rst

@@ -45,8 +45,8 @@ Optimizations that always occurs
 ================================
 
 OCRmyPDF will automatically replace obsolete or inferior compression schemes
-such as RLE or LZW with superior schemes such as Deflate and converting
+such as RLE or LZW with superior schemes such as Deflate, and convert
-monochrome images to CCITT G4. Since this is harmless it always occurs and there
+monochrome images to CCITT G4. Since this is lossless, it always occurs and there
 is no way to disable it. Other non-image compressed objects are compressed as
 well.
 

docs/plugins.rst

@@ -177,8 +177,13 @@ Custom command line arguments
 Execution and progress reporting
 --------------------------------
 
+.. autoclass:: ocrmypdf.pluginspec.ProgressBar
+    :members:
+    :special-members: __init__, __enter__, __exit__
+
 .. autoclass:: ocrmypdf.pluginspec.Executor
     :members:
+    :special-members: __call__
 
 .. autofunction:: ocrmypdf.pluginspec.get_logging_console
 

docs/release_notes.rst

@@ -28,31 +28,143 @@ tagged yet.
 
 .. |OCRmyPDF PyPI| image:: https://img.shields.io/pypi/v/ocrmypdf.svg
 
+v15.4.3
+=======
+
+-  Fixed deprecation warning in pikepdf older than 8.7.1; pikepdf >= 8.7.1 is
+   now required.
+
+v15.4.2
+=======
+
+-  We now raise an exception on a certain class of PDFs that likely need an
+   explicit color conversion strategy selected to display correctly
+   for PDF/A conversion.
+-  Fixed an error that occurred while trying to write a log message after the
+   debug log handler was removed.
+
+v15.4.1
+=======
+
+-  Fixed misc/watcher.py regressions: accept ``--ocr-json-settings`` as either
+   filename or JSON string, as previously; and argument count mismatch.
+   :issue:`1183,1185`
+-  We no longer attempt to set /ProcSet in the PDF output, since this is an
+   obsolete PDF feature.
+-  Documentation improvements.
+
+v15.4.0
+=======
+
+-  Added new experimental APIs to support offline editing of the final text.
+   Specifically, one can now generate hOCR files with OCRmyPDF, edit them with
+   some other tool, and then finalize the PDF. They are experimental and
+   subject to change, including details of how the working folder is used.
+   There is no command line interface.
+-  Code reorganization: executors, progress bars, initialization and setup.
+-  Fixed test coverage in cases where the coverage tool did not properly trace
+   into threads or subprocesses. This code was still being tested but appeared
+   as not covered.
+-  In the test suite, reduced use of subprocesses and other techniques that
+   interfere with coverage measurement.
+-  Improved error check for when we appear to be running inside a snap container
+   and files are not available.
+-  Plugin specification now properly defines progress bars as a protocol rather
+   than defining them as "tqdm-like".
+-  We now default to using "forkserver" process creation on POSIX platforms
+   rather than fork, since this is method is more robust and avoids some
+   issues when threads are present.
+-  Fixed an instance where the user's request to ``--no-use-threads`` was ignored.
+-  If a PDF does not have language metadata on its top level object, we add
+   the OCR language.
+-  Replace some cryptic test error messages with more helpful ones.
+-  Debug messages for how OCRmyPDF picks the colorspace for a page are now
+   more descriptive.
+
+v15.3.1
+=======
+
+-  Fixed an issue with logging settings for misc/watcher.py introduced in the
+   previous release. :issue:`1180`
+-  We now attempt to preserve the input's extended attributes when creating
+   the output file.
+-  For some reason, the macOS build now needs OpenSSL explicitly installed.
+-  Updated documentation on Docker performance concerns.
+
+v15.3.0
+=======
+
+-  Update misc/watcher.py to improve command line interface using Typer, and
+   support ``.env`` specification of environment variables. Improved error
+   messages. Thanks to @mflagg2814 for the PR that prompted this improvement.
+-  Improved error message when a file cannot be read because we are running in
+   a snap container.
+
+v15.2.0
+=======
+
+-  Added a Docker image based on Alpine Linux. This image is smaller than the
+   Ubuntu-based image and may be useful in some situations. Currently hosted at
+   jbarlow83/ocrmypdf-alpine. Currently not available in ARM flavor.
+-  The Ubuntu Docker is now aliased to jbarlow83/ocrmypdf-ubuntu.
+-  Updated Docker documentation.
+
+v15.1.0
+=======
+
+-  We now require Pillow 10.0.1, due a serious security vulnerability in all earlier
+   versions of that dependency. The vulnerability concerns WebP images and could
+   be triggered in OCRmyPDF when creating a PDF from a malicious WebP image.
+-  Added some keyword arguments to ``ocrmypdf.ocr`` that were previously accepted
+   but undocumented.
+-  Documentation updates and typing improvements.
+
+v15.0.2
+=======
+
+-  Added Python 3.12 to test matrix.
+-  Updated documentation for notes on Python 3.12, 32-bit support and some new
+   features in v15.
+
+v15.0.1
+=======
+
+-  Wheels Python tag changed to py39.
+-  Marked as a expected fail a test that fails on recent Ghostscript versions.
+-  Clarified documentation and release notes around the extent of 32-bit support.
+-  Updated installation documentation to changes in v15.
+
 v15.0.0
 =======
 
 -  Dropped support for Python 3.8.
--  Dropped support many older dependencies - see ``pyproject.toml`` for details.
+-  Dropped support some older dependencies, specifically ``coloredlogs`` and
-   Generally speaking, Ubuntu 22.04 is our baseline system.
+   ``tqdm`` in favor of rich - see ``pyproject.toml`` for details.
--  Dropped support 32-bit Windows and Linux. You must use a 64-bit operating system,
+   Generally speaking, Ubuntu 22.04 is our new baseline system.
-   and 64-bit versions of Python, Tesseract and Ghostscript to use OCRmyPDF. Many of
+-  Tightened version requirements for some dependencies.
-   our dependencies are dropping 32-bit support (e.g. Pillow), and we are following
+-  Dropped support for 32-bit Linux wheels. We strongly recommend a 64-bit operating
-   suit.
+   system, and 64-bit versions of Python, Tesseract and Ghostscript to use OCRmyPDF.
+   Many of our dependencies are dropping 32-bit builds (e.g. Pillow), and we are
+   following suit. (Maintainers may still build 32-bit versions from source.)
 -  Changed to trusted release for PyPI publishing.
--  pikepdf memory mapping is enabled again for improved performance, now an issue
+-  pikepdf memory mapping is enabled again for improved performance, now that an
-   with pikepdf has been fixed.
+   issue with feature in pikepdf is fixed.
 -  ``ocrmypdf.helpers.calculate_downsample`` previously had two variants, one
    that took a ``PIL.Image`` and one that took a ``tuple[int, int]``. The latter
    was removed.
 -  The snap version of ocrmypdf is now based on Ubuntu core22.
--  We now account situations where a small portion of an image on a page reports a
+-  We now account for situations where a small portion of an image on a page is drawn
-   high DPI (resolution). Previously, the entire page would be rasterized at the
+   at high DPI (resolution). Previously, the entire page would be rasterized at the
-   highest resolution, which caused performance problems. Now, the page is rasterized
+   highest resolution of any feature, which caused performance problems. Now,
+   the page is rasterized
    at a resolution based on the average DPI of the page, weighted by the area that
    each feature occupies. Typically, small areas of high resolution in PDFs are
    errors or quirks from the repeated use of assets and high resolution is not
    beneficial. :issue:`1010,1104,1004,1079,1010`
--  Ghostscript color conversion strategy is now configurable. :issue:`1143`
+-  Ghostscript color conversion strategy is now configurable using
+   ``--color-conversion-strategy``. :issue:`1143`
+-  JBIG2 threshold for optimization is now configurable using
+   ``--jbig2-threshold``. :issue:`1133`
 
 v14.4.0
 =======

misc/watcher.py

@@ -5,107 +5,125 @@
 
 """Watch a directory for new PDFs and OCR them."""
 
-from __future__ import annotations
+# Do not enable annotations!
+# https://github.com/tiangolo/typer/discussions/598
+# from __future__ import annotations
 
 import json
 import logging
-import os
 import shutil
 import sys
 import time
 from datetime import datetime
+from enum import Enum
 from pathlib import Path
+from typing import Annotated, Any
 
 import pikepdf
+import typer
+from dotenv import load_dotenv
 from watchdog.events import PatternMatchingEventHandler
 from watchdog.observers import Observer
 from watchdog.observers.polling import PollingObserver
 
 import ocrmypdf
 
+load_dotenv()
+
+
 # pylint: disable=logging-format-interpolation
-
+app = typer.Typer(name="ocrmypdf-watcher")
-
-def getenv_bool(name: str, default: str = 'False'):
-    return os.getenv(name, default).lower() in ('true', 'yes', 'y', '1')
-
-
-INPUT_DIRECTORY = os.getenv('OCR_INPUT_DIRECTORY', '/input')
-OUTPUT_DIRECTORY = os.getenv('OCR_OUTPUT_DIRECTORY', '/output')
-ARCHIVE_DIRECTORY = os.getenv('OCR_ARCHIVE_DIRECTORY', '/processed')
-OUTPUT_DIRECTORY_YEAR_MONTH = getenv_bool('OCR_OUTPUT_DIRECTORY_YEAR_MONTH')
-ON_SUCCESS_DELETE = getenv_bool('OCR_ON_SUCCESS_DELETE')
-ON_SUCCESS_ARCHIVE = getenv_bool('OCR_ON_SUCCESS_ARCHIVE')
-DESKEW = getenv_bool('OCR_DESKEW')
-OCR_JSON_SETTINGS = json.loads(os.getenv('OCR_JSON_SETTINGS', '{}'))
-POLL_NEW_FILE_SECONDS = int(os.getenv('OCR_POLL_NEW_FILE_SECONDS', '1'))
-USE_POLLING = getenv_bool('OCR_USE_POLLING')
-RETRIES_LOADING_FILE = int(os.getenv('OCR_RETRIES_LOADING_FILE', '5'))
-LOGLEVEL = os.getenv('OCR_LOGLEVEL', 'INFO')
-PATTERNS = ['*.pdf', '*.PDF']
 
 log = logging.getLogger('ocrmypdf-watcher')
 
 
-def get_output_dir(root, basename):
+class LoggingLevelEnum(str, Enum):
-    if OUTPUT_DIRECTORY_YEAR_MONTH:
+    """Enum for logging levels."""
+
+    DEBUG = "DEBUG"
+    INFO = "INFO"
+    WARNING = "WARNING"
+    ERROR = "ERROR"
+    CRITICAL = "CRITICAL"
+
+
+def get_output_dir(root: Path, basename: str, output_dir_year_month: bool) -> Path:
+    if output_dir_year_month:
         today = datetime.today()
-        output_directory_year_month = (
+        output_directory_year_month = root / str(today.year) / f'{today.month:02d}'
-            Path(root) / str(today.year) / f'{today.month:02d}'
-        )
         if not output_directory_year_month.exists():
             output_directory_year_month.mkdir(parents=True, exist_ok=True)
         output_path = Path(output_directory_year_month) / basename
     else:
-        output_path = Path(OUTPUT_DIRECTORY) / basename
+        output_path = root / basename
     return output_path
 
 
-def wait_for_file_ready(file_path):
+def wait_for_file_ready(
+    file_path: Path, poll_new_file_seconds: int, retries_loading_file: int
+):
     # This loop waits to make sure that the file is completely loaded on
     # disk before attempting to read. Docker sometimes will publish the
     # watchdog event before the file is actually fully on disk, causing
     # pikepdf to fail.
 
-    retries = RETRIES_LOADING_FILE
+    tries = retries_loading_file + 1
-    while retries:
+    while tries:
         try:
-            pdf = pikepdf.open(file_path)
+            with pikepdf.Pdf.open(file_path) as pdf:
-        except (FileNotFoundError, pikepdf.PdfError) as e:
+                log.debug(f"{file_path} ready with {pdf.pages} pages")
+                return True
+        except (FileNotFoundError, OSError) as e:
             log.info(f"File {file_path} is not ready yet")
             log.debug("Exception was", exc_info=e)
-            time.sleep(POLL_NEW_FILE_SECONDS)
+            time.sleep(poll_new_file_seconds)
-            retries -= 1
+            tries -= 1
-        else:
+        except pikepdf.PdfError as e:
-            pdf.close()
+            log.info(f"File {file_path} is not full written yet")
-            return True
+            log.debug("Exception was", exc_info=e)
+            time.sleep(poll_new_file_seconds)
+            tries -= 1
 
     return False
 
 
-def execute_ocrmypdf(file_path):
+def execute_ocrmypdf(
-    file_path = Path(file_path)
+    *,
-    output_path = get_output_dir(OUTPUT_DIRECTORY, file_path.name)
+    file_path: Path,
+    archive_dir: Path,
+    output_dir: Path,
+    ocrmypdf_kwargs: dict[str, Any],
+    on_success_delete: bool,
+    on_success_archive: bool,
+    poll_new_file_seconds: int,
+    retries_loading_file: int,
+    output_dir_year_month: bool,
+):
+    output_path = get_output_dir(output_dir, file_path.name, output_dir_year_month)
 
     log.info("-" * 20)
-    log.info(f'New file: {file_path}. Waiting until fully loaded...')
+    log.info(f'New file: {file_path}. Waiting until fully written...')
-    if not wait_for_file_ready(file_path):
+    if not wait_for_file_ready(file_path, poll_new_file_seconds, retries_loading_file):
         log.info(f"Gave up waiting for {file_path} to become ready")
         return
     log.info(f'Attempting to OCRmyPDF to: {output_path}')
+
+    log.debug(
+        f'OCRmyPDF input_file={file_path} output_file={output_path} '
+        f'kwargs: {ocrmypdf_kwargs}'
+    )
     exit_code = ocrmypdf.ocr(
         input_file=file_path,
         output_file=output_path,
-        deskew=DESKEW,
+        **ocrmypdf_kwargs,
-        **OCR_JSON_SETTINGS,
     )
     if exit_code == 0:
-        if ON_SUCCESS_DELETE:
+        if on_success_delete:
             log.info(f'OCR is done. Deleting: {file_path}')
             file_path.unlink()
-        elif ON_SUCCESS_ARCHIVE:
+        elif on_success_archive:
-            log.info(f'OCR is done. Archiving {file_path.name} to {ARCHIVE_DIRECTORY}')
+            log.info(f'OCR is done. Archiving {file_path.name} to {archive_dir}')
-            shutil.move(file_path, f'{ARCHIVE_DIRECTORY}/{file_path.name}')
+            shutil.move(file_path, f'{archive_dir}/{file_path.name}')
         else:
             log.info('OCR is done')
     else:
@@ -113,61 +131,204 @@ def execute_ocrmypdf(file_path):
 
 
 class HandleObserverEvent(PatternMatchingEventHandler):
+    def __init__(
+        self,
+        patterns=None,
+        ignore_patterns=None,
+        ignore_directories=False,
+        case_sensitive=False,
+        settings={},
+    ):
+        super().__init__(
+            patterns=patterns,
+            ignore_patterns=ignore_patterns,
+            ignore_directories=ignore_directories,
+            case_sensitive=case_sensitive,
+        )
+        self._settings = settings
+
     def on_any_event(self, event):
         if event.event_type in ['created']:
-            execute_ocrmypdf(event.src_path)
+            execute_ocrmypdf(file_path=Path(event.src_path), **self._settings)
 
 
-def main():
+@app.command()
+def main(
+    input_dir: Annotated[
+        Path,
+        typer.Argument(
+            envvar='OCR_INPUT_DIRECTORY',
+            exists=True,
+            file_okay=False,
+            dir_okay=True,
+            readable=True,
+            resolve_path=True,
+        ),
+    ] = '/input',
+    output_dir: Annotated[
+        Path,
+        typer.Argument(
+            envvar='OCR_OUTPUT_DIRECTORY',
+            exists=True,
+            file_okay=False,
+            dir_okay=True,
+            writable=True,
+            resolve_path=True,
+        ),
+    ] = '/output',
+    archive_dir: Annotated[
+        Path,
+        typer.Argument(
+            envvar='OCR_ARCHIVE_DIRECTORY',
+            exists=True,
+            file_okay=False,
+            dir_okay=True,
+            writable=True,
+            resolve_path=True,
+        ),
+    ] = '/processed',
+    output_dir_year_month: Annotated[
+        bool,
+        typer.Option(
+            envvar='OCR_OUTPUT_DIRECTORY_YEAR_MONTH',
+            help='Create a subdirectory in the output directory for each year and month',
+        ),
+    ] = False,
+    on_success_delete: Annotated[
+        bool,
+        typer.Option(
+            envvar='OCR_ON_SUCCESS_DELETE',
+            help='Delete the input file after successful OCR',
+        ),
+    ] = False,
+    on_success_archive: Annotated[
+        bool,
+        typer.Option(
+            envvar='OCR_ON_SUCCESS_ARCHIVE',
+            help='Archive the input file after successful OCR',
+        ),
+    ] = False,
+    deskew: Annotated[
+        bool,
+        typer.Option(
+            envvar='OCR_DESKEW',
+            help='Deskew the input file before OCR',
+        ),
+    ] = False,
+    ocr_json_settings: Annotated[
+        str,
+        typer.Option(
+            envvar='OCR_JSON_SETTINGS',
+            help='JSON settings to pass to OCRmyPDF (JSON string or file path)',
+        ),
+    ] = None,
+    poll_new_file_seconds: Annotated[
+        int,
+        typer.Option(
+            envvar='OCR_POLL_NEW_FILE_SECONDS',
+            help='Seconds to wait before polling a new file',
+            min=0,
+        ),
+    ] = 1,
+    use_polling: Annotated[
+        bool,
+        typer.Option(
+            envvar='OCR_USE_POLLING',
+            help='Use polling instead of filesystem events',
+        ),
+    ] = False,
+    retries_loading_file: Annotated[
+        int,
+        typer.Option(
+            envvar='OCR_RETRIES_LOADING_FILE',
+            help='Number of times to retry loading a file before giving up',
+            min=0,
+        ),
+    ] = 5,
+    loglevel: Annotated[
+        LoggingLevelEnum,
+        typer.Option(
+            envvar='OCR_LOGLEVEL',
+            help='Logging level',
+        ),
+    ] = LoggingLevelEnum.INFO,
+    patterns: Annotated[
+        str,
+        typer.Option(
+            envvar='OCR_PATTERNS',
+            help='File patterns to watch',
+        ),
+    ] = '*.pdf,*.PDF',
+):
     ocrmypdf.configure_logging(
         verbosity=(
             ocrmypdf.Verbosity.default
-            if LOGLEVEL != 'DEBUG'
+            if loglevel != LoggingLevelEnum.DEBUG
             else ocrmypdf.Verbosity.debug
         ),
         manage_root_logger=True,
     )
-    log.setLevel(LOGLEVEL)
+    log.setLevel(loglevel.value)
     log.info(
         f"Starting OCRmyPDF watcher with config:\n"
-        f"Input Directory: {INPUT_DIRECTORY}\n"
+        f"Input Directory: {input_dir}\n"
-        f"Output Directory: {OUTPUT_DIRECTORY}\n"
+        f"Output Directory: {output_dir}\n"
-        f"Output Directory Year & Month: {OUTPUT_DIRECTORY_YEAR_MONTH}\n"
+        f"Output Directory Year & Month: {output_dir_year_month}\n"
-        f"Archive Directory: {ARCHIVE_DIRECTORY}"
+        f"Archive Directory: {archive_dir}"
     )
     log.debug(
-        f"INPUT_DIRECTORY: {INPUT_DIRECTORY}\n"
+        f"INPUT_DIRECTORY: {input_dir}\n"
-        f"OUTPUT_DIRECTORY: {OUTPUT_DIRECTORY}\n"
+        f"OUTPUT_DIRECTORY: {output_dir}\n"
-        f"OUTPUT_DIRECTORY_YEAR_MONTH: {OUTPUT_DIRECTORY_YEAR_MONTH}\n"
+        f"OUTPUT_DIRECTORY_YEAR_MONTH: {output_dir_year_month}\n"
-        f"ARCHIVE_DIRECTORY: {ARCHIVE_DIRECTORY}\n"
+        f"ARCHIVE_DIRECTORY: {archive_dir}\n"
-        f"ON_SUCCESS_DELETE: {ON_SUCCESS_DELETE}\n"
+        f"ON_SUCCESS_DELETE: {on_success_delete}\n"
-        f"ON_SUCCESS_ARCHIVE: {ON_SUCCESS_ARCHIVE}\n"
+        f"ON_SUCCESS_ARCHIVE: {on_success_archive}\n"
-        f"DESKEW: {DESKEW}\n"
+        f"DESKEW: {deskew}\n"
-        f"ARGS: {OCR_JSON_SETTINGS}\n"
+        f"ARGS: {ocr_json_settings}\n"
-        f"POLL_NEW_FILE_SECONDS: {POLL_NEW_FILE_SECONDS}\n"
+        f"POLL_NEW_FILE_SECONDS: {poll_new_file_seconds}\n"
-        f"RETRIES_LOADING_FILE: {RETRIES_LOADING_FILE}\n"
+        f"RETRIES_LOADING_FILE: {retries_loading_file}\n"
-        f"USE_POLLING: {USE_POLLING}\n"
+        f"USE_POLLING: {use_polling}\n"
-        f"LOGLEVEL: {LOGLEVEL}"
+        f"LOGLEVEL: {loglevel.value}"
     )
 
-    if 'input_file' in OCR_JSON_SETTINGS or 'output_file' in OCR_JSON_SETTINGS:
+    if ocr_json_settings and Path(ocr_json_settings).exists():
-        log.error('OCR_JSON_SETTINGS should not specify input file or output file')
+        json_settings = json.loads(Path(ocr_json_settings).read_text())
+    else:
+        json_settings = json.loads(ocr_json_settings or '{}')
+
+    if 'input_file' in json_settings or 'output_file' in json_settings:
+        log.error(
+            'OCR_JSON_SETTINGS (--ocr-json-settings) may not specify input/output file'
+        )
         sys.exit(1)
 
-    handler = HandleObserverEvent(patterns=PATTERNS)
+    handler = HandleObserverEvent(
-    if USE_POLLING:
+        patterns=patterns.split(','),
+        settings={
+            'archive_dir': archive_dir,
+            'output_dir': output_dir,
+            'ocrmypdf_kwargs': json_settings | {'deskew': deskew},
+            'on_success_delete': on_success_delete,
+            'on_success_archive': on_success_archive,
+            'poll_new_file_seconds': poll_new_file_seconds,
+            'retries_loading_file': retries_loading_file,
+            'output_dir_year_month': output_dir_year_month,
+        },
+    )
+    if use_polling:
         observer = PollingObserver()
     else:
         observer = Observer()
-    observer.schedule(handler, INPUT_DIRECTORY, recursive=True)
+    observer.schedule(handler, input_dir, recursive=True)
     observer.start()
+    typer.echo(f"Watching {input_dir} for new PDFs. Press Ctrl+C to exit.")
     try:
         while True:
-            time.sleep(1)
+            time.sleep(30)
     except KeyboardInterrupt:
         observer.stop()
     observer.join()
 
 
 if __name__ == "__main__":
-    main()
+    app()

pyproject.toml

@@ -12,12 +12,12 @@ readme = "README.md"
 license = { text = "MPL-2.0" }
 requires-python = ">=3.9"
 dependencies = [
-  "Pillow>=9.0.1",
+  "Pillow>=10.0.1",
   "deprecation>=2.1.0",
   "img2pdf>=0.4.4",
   "packaging>=20",
   "pdfminer.six>=20220319",
-  "pikepdf>=8",
+  "pikepdf>=8.7.1",
   "pluggy>=0.13.0",
   "reportlab>=3.6.8",
   "rich>=13",
@@ -50,7 +50,7 @@ Tracker = "https://github.com/ocrmypdf/OCRmyPDF/issues"
 
 [project.optional-dependencies]
 docs = ["sphinx", "sphinx-issues", "sphinx-rtd-theme"]
-extended_test = ["PyMuPDF==1.19.1"]
+extended_test = ["PyMuPDF>=1.19.1"]
 test = [
   "coverage[toml]>=6.2",
   "hypothesis>=6.36.0",
@@ -61,7 +61,7 @@ test = [
   "types-Pillow",
   "types-humanfriendly",
 ]
-watcher = ["watchdog>=1.0.2"]
+watcher = ["watchdog>=1.0.2", "typer[all]", "python-dotenv"]
 webservice = ["Flask>=2.0.1"]
 
 [project.scripts]
@@ -77,11 +77,11 @@ namespaces = false
 [tool.setuptools_scm]
 
 [tool.distutils.bdist_wheel]
-python-tag = "py38"
+python-tag = "py39"
 
 [tool.black]
 line-length = 88
-target-version = ["py38", "py39", "py310", "py311"]
+target-version = ["py39", "py310", "py311"]
 skip-string-normalization = true
 include = '\.pyi?$'
 exclude = '''
@@ -105,7 +105,8 @@ exclude = '''
 [tool.coverage.run]
 branch = true
 parallel = true
-concurrency = ["multiprocessing"]
+concurrency = ["multiprocessing", "thread"]
+sigterm = true
 
 [tool.coverage.paths]
 source = ["src/ocrmypdf"]
@@ -157,7 +158,7 @@ select = [
   "I001", # isort
   "UP",   # pyupgrade
 ]
-target-version = "py38"
+target-version = "py39"
 
 [tool.ruff.isort]
 known-first-party = ["ocrmypdf"]

src/ocrmypdf/__init__.py

@@ -10,8 +10,15 @@ from pluggy import HookimplMarker as _HookimplMarker
 from ocrmypdf import helpers, hocrtransform, pdfa, pdfinfo
 from ocrmypdf._concurrent import Executor
 from ocrmypdf._jobcontext import PageContext, PdfContext
+from ocrmypdf._pipelines._common import (
+    configure_debug_logging,
+)
 from ocrmypdf._version import PROGRAM_NAME, __version__
-from ocrmypdf.api import Verbosity, configure_logging, ocr
+from ocrmypdf.api import (
+    Verbosity,
+    configure_logging,
+    ocr,
+)
 from ocrmypdf.exceptions import (
     BadArgsError,
     DpiError,
@@ -30,9 +37,11 @@ from ocrmypdf.pluginspec import OcrEngine, OrientationConfidence
 
 hookimpl = _HookimplMarker('ocrmypdf')
 
+
 __all__ = [
     '__version__',
     'BadArgsError',
+    'configure_debug_logging',
     'configure_logging',
     'DpiError',
     'EncryptedPdfError',

src/ocrmypdf/__main__.py

@@ -7,14 +7,15 @@
 from __future__ import annotations
 
 import logging
+import multiprocessing
 import os
 import signal
 import sys
 from contextlib import suppress
 
 from ocrmypdf import __version__
+from ocrmypdf._pipelines.ocr import run_pipeline_cli
 from ocrmypdf._plugin_manager import get_parser_options_plugins
-from ocrmypdf._sync import run_pipeline
 from ocrmypdf._validation import check_options
 from ocrmypdf.api import Verbosity, configure_logging
 from ocrmypdf.exceptions import (
@@ -71,9 +72,12 @@ def run(args=None):
     with suppress(AttributeError, OSError):
         signal.signal(signal.SIGBUS, sigbus)
 
-    result = run_pipeline(options=options, plugin_manager=plugin_manager)
+    result = run_pipeline_cli(options=options, plugin_manager=plugin_manager)
     return result
 
 
 if __name__ == '__main__':
+    multiprocessing.freeze_support()
+    if os.name == 'posix':
+        multiprocessing.set_start_method('forkserver')
     sys.exit(run())

src/ocrmypdf/_concurrent.py

@@ -7,29 +7,18 @@ from __future__ import annotations
 
 import threading
 from abc import ABC, abstractmethod
-from typing import Callable, Iterable
+from collections.abc import Iterable
+from typing import Callable, TypeVar
+
+from ocrmypdf._progressbar import NullProgressBar, ProgressBar
+
+T = TypeVar('T')
 
 
 def _task_noop(*_args, **_kwargs):
     return
 
 
-class NullProgressBar:
-    """Progress bar API that takes no actions."""
-
-    def __init__(self, **kwargs):
-        pass
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, exc_type, exc_value, traceback):
-        return False
-
-    def update(self, _arg=None):
-        return
-
-
 class Executor(ABC):
     """Abstract concurrent executor."""
 
@@ -45,11 +34,11 @@ class Executor(ABC):
         *,
         use_threads: bool,
         max_workers: int,
-        tqdm_kwargs: dict,
+        progress_kwargs: dict,
         worker_initializer: Callable | None = None,
-        task: Callable | None = None,
+        task: Callable[..., T] | None = None,
         task_arguments: Iterable | None = None,
-        task_finished: Callable | None = None,
+        task_finished: Callable[[T, ProgressBar], None] | None = None,
     ) -> None:
         """Set up parallel execution and progress reporting.
 
@@ -59,7 +48,7 @@ class Executor(ABC):
                 heavily, and parallelizing it with threads is not expected to be
                 performant).
             max_workers: The maximum number of workers that should be run.
-            tqdm_kwargs: Arguments to set up the progress bar.
+            progress_kwargs: Arguments to set up the progress bar.
             worker_initializer: Called when a worker is initialized, in the worker's
                 execution context. If the child workers are processes, it must be
                 possible to marshall/pickle the worker initializer.
@@ -85,7 +74,7 @@ class Executor(ABC):
             self._execute(
                 use_threads=use_threads,
                 max_workers=max_workers,
-                tqdm_kwargs=tqdm_kwargs,
+                progress_kwargs=progress_kwargs,
                 worker_initializer=worker_initializer,
                 task=task,
                 task_arguments=task_arguments,
@@ -98,7 +87,7 @@ class Executor(ABC):
         *,
         use_threads: bool,
         max_workers: int,
-        tqdm_kwargs: dict,
+        progress_kwargs: dict,
         worker_initializer: Callable,
         task: Callable,
         task_arguments: Iterable,
@@ -124,13 +113,13 @@ class SerialExecutor(Executor):
         *,
         use_threads: bool,
         max_workers: int,
-        tqdm_kwargs: dict,
+        progress_kwargs: dict,
         worker_initializer: Callable,
         task: Callable,
         task_arguments: Iterable,
         task_finished: Callable,
     ):  # pylint: disable=unused-argument
-        with self.pbar_class(**tqdm_kwargs) as pbar:
+        with self.pbar_class(**progress_kwargs) as pbar:
             for args in task_arguments:
-                result = task(args)
+                result = task(*args)
                 task_finished(result, pbar)

src/ocrmypdf/_exec/ghostscript.py

@@ -8,7 +8,7 @@ from __future__ import annotations
 import logging
 import os
 import re
-import sys
+from collections import deque
 from io import BytesIO
 from os import fspath
 from pathlib import Path
@@ -17,18 +17,10 @@ from subprocess import PIPE, CalledProcessError
 from packaging.version import Version
 from PIL import Image, UnidentifiedImageError
 
-from ocrmypdf.exceptions import SubprocessOutputError
+from ocrmypdf.exceptions import ColorConversionNeededError, SubprocessOutputError
 from ocrmypdf.helpers import Resolution
 from ocrmypdf.subprocess import get_version, run, run_polling_stderr
 
-# Remove this workaround when we require Pillow >= 10
-try:
-    Transpose = Image.Transpose  # type: ignore
-except AttributeError:
-    # Pillow 9 shim
-    Transpose = Image  # type: ignore
-
-
 COLOR_CONVERSION_STRATEGIES = frozenset(
     [
         'CMYK',
@@ -38,38 +30,44 @@ COLOR_CONVERSION_STRATEGIES = frozenset(
         'UseDeviceIndependentColor',
     ]
 )
+# Ghostscript executable - gswin32c is not supported
+GS = 'gswin64c' if os.name == 'nt' else 'gs'
+
 
 log = logging.getLogger(__name__)
 
 
 class DuplicateFilter(logging.Filter):
-    """Filter out duplicate log messages."""
+    """Filter out duplicate log messages.
 
-    def __init__(self, logger: logging.Logger):
+    A context window of default 5 messages is used to determine if a message is a
-        self.last: logging.LogRecord | None = None
+    duplicate. This is because some Ghostscript messages are word wrapped.
-        self.count = 0
+    """
+
+    def __init__(self, logger: logging.Logger, context_window=5):
+        self.window: deque[str] = deque([], maxlen=context_window)
         self.logger = logger
+        self.levelno = logging.DEBUG
+        self.count = 0
 
     def filter(self, record):
-        if self.last and record.msg == self.last.msg:
+        if record.msg in self.window:
             self.count += 1
+            self.levelno = record.levelno
             return False
         else:
             if self.count >= 1:
-                rep_msg = f"(previous message repeated {self.count} times)"
+                rep_msg = f"(suppressed {self.count} repeated lines)"
                 self.count = 0  # Avoid infinite recursion
-                self.logger.log(self.last.levelno, rep_msg)
+                self.logger.log(self.levelno, rep_msg)
-            self.last = record
+                self.window.clear()
+            self.window.append(record.msg)
             return True
 
 
 log.addFilter(DuplicateFilter(log))
 
 
-# Ghostscript executable - gswin32c is not supported
-GS = 'gswin64c' if os.name == 'nt' else 'gs'
-
-
 def version() -> Version:
     return Version(get_version(GS))
 
@@ -79,6 +77,20 @@ def _gs_error_reported(stream) -> bool:
     return bool(match)
 
 
+def _gs_devicen_reported(stream) -> bool:
+    """Did Ghostscript warn about a DeviceN with inappropriate alternate?
+
+    If so, we need the user to select a color conversion, or the resulting PDF will
+    not present correctly in some PDF viewers.
+    """
+    match = re.search(
+        r'DeviceN.*inappropriate alternate',
+        stream,
+        flags=re.IGNORECASE | re.MULTILINE,
+    )
+    return bool(match)
+
+
 def rasterize_pdf(
     input_file: os.PathLike,
     output_file: os.PathLike,
@@ -138,11 +150,11 @@ def rasterize_pdf(
                 # rotation is a clockwise angle and Image.ROTATE_* is
                 # counterclockwise so this cancels out the rotation
                 if rotation == 90:
-                    im = im.transpose(Transpose.ROTATE_90)
+                    im = im.transpose(Image.Transpose.ROTATE_90)
                 elif rotation == 180:
-                    im = im.transpose(Transpose.ROTATE_180)
+                    im = im.transpose(Image.Transpose.ROTATE_180)
                 elif rotation == 270:
-                    im = im.transpose(Transpose.ROTATE_270)
+                    im = im.transpose(Image.Transpose.ROTATE_270)
                 if rotation % 180 == 90:
                     page_dpi = page_dpi.flip_axis()
             im.save(fspath(output_file), dpi=page_dpi)
@@ -252,7 +264,6 @@ def generate_pdfa(
         ]
     )
     args_gs.extend(fspath(s) for s in pdf_pages)  # Stringify Path objs
-
     try:
         with Path(output_file).open('wb') as output:
             p = run_polling_stderr(
@@ -281,3 +292,5 @@ def generate_pdfa(
             # the **** pattern to split the stderr into parts.
             for part in stderr.split('****'):
                 log.error(part)
+        if _gs_devicen_reported(stderr):
+            raise ColorConversionNeededError()

src/ocrmypdf/_exec/jbig2enc.py

@@ -25,7 +25,7 @@ def available():
     return True
 
 
-def convert_group(*, cwd, infiles, out_prefix, threshold):
+def convert_group(cwd, infiles, out_prefix, threshold):
     args = [
         'jbig2',
         '-b',
@@ -43,21 +43,9 @@ def convert_group(*, cwd, infiles, out_prefix, threshold):
     return proc
 
 
-def convert_group_mp(args):
+def convert_single(cwd, infile, outfile, threshold):
-    return convert_group(
-        cwd=args[0], infiles=args[1], out_prefix=args[2], threshold=args[3]
-    )
-
-
-def convert_single(*, cwd, infile, outfile, threshold):
     args = ['jbig2', '--pdf', '-t', str(threshold), infile]
     with open(outfile, 'wb') as fstdout:
         proc = run(args, cwd=cwd, stdout=fstdout, stderr=PIPE)
     proc.check_returncode()
     return proc
-
-
-def convert_single_mp(args):
-    return convert_single(
-        cwd=args[0], infile=args[1], outfile=args[2], threshold=args[3]
-    )

src/ocrmypdf/_exec/pngquant.py

@@ -5,13 +5,10 @@
 
 from __future__ import annotations
 
-from contextlib import contextmanager
-from io import BytesIO
 from pathlib import Path
 from subprocess import PIPE
 
 from packaging.version import Version
-from PIL import Image
 
 from ocrmypdf.exceptions import MissingDependencyError
 from ocrmypdf.subprocess import get_version, run
@@ -29,21 +26,16 @@ def available():
     return True
 
 
-@contextmanager
-def input_as_png(input_file: Path):
-    if not input_file.name.endswith('.png'):
-        with Image.open(input_file) as im:
-            bio = BytesIO()
-            im.save(bio, format='png')
-            bio.seek(0)
-            yield bio
-    else:
-        with open(input_file, 'rb') as f:
-            yield f
-
-
 def quantize(input_file: Path, output_file: Path, quality_min: int, quality_max: int):
-    with input_as_png(input_file) as input_stream:
+    """Quantize a PNG image using pngquant.
+
+    Args:
+        input_file: Input PNG image
+        output_file: Output PNG image
+        quality_min: Minimum quality to use
+        quality_max: Maximum quality to use
+    """
+    with open(input_file, 'rb') as input_stream:
         args = [
             'pngquant',
             '--force',
@@ -58,7 +50,3 @@ def quantize(input_file: Path, output_file: Path, quality_min: int, quality_max:
     if result.returncode == 0:
         # input_file could be the same as output_file, so we defer the write
         output_file.write_bytes(result.stdout)
-
-
-def quantize_mp(args):
-    return quantize(*args)

src/ocrmypdf/_exec/tesseract.py

@@ -7,13 +7,13 @@ from __future__ import annotations
 
 import logging
 import re
+from contextlib import suppress
 from math import pi
 from os import fspath
 from pathlib import Path
 from subprocess import PIPE, STDOUT, CalledProcessError, TimeoutExpired
 
 from packaging.version import Version
-from PIL import Image
 
 from ocrmypdf.exceptions import (
     MissingDependencyError,
@@ -26,25 +26,6 @@ from ocrmypdf.subprocess import get_version, run
 log = logging.getLogger(__name__)
 
 
-HOCR_TEMPLATE = """<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
-    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
- <head>
-  <title></title>
-<meta http-equiv="Content-Type" content="text/html;charset=utf-8" />
-  <meta name='ocr-system' content='tesseract 4.1.1' />
-  <meta name='ocr-capabilities'
-    content='ocr_page ocr_carea ocr_par ocr_line ocrx_word ocrp_wconf'/>
-</head>
-<body>
-  <div class='ocr_page' id='page_1'
-    title='image "_blank.png"; bbox 0 0 {0} {1}; ppageno 0'>
-  </div>
- </body>
-</html>
-"""
-
 TESSERACT_THRESHOLDING_METHODS: dict[str, int] = {
     'auto': 0,
     'otsu': 0,
@@ -284,14 +265,11 @@ def page_timedout(timeout: float) -> None:
 
 
 def _generate_null_hocr(output_hocr: Path, output_text: Path, image: Path) -> None:
-    """Produce a .hocr file that reports no text detected.
+    """Produce an empty .hocr file.
 
     Ensures page is the same size as the input image.
     """
-    with Image.open(image) as im:
+    output_hocr.write_text('', encoding='utf-8')
-        w, h = im.size
-
-    output_hocr.write_text(HOCR_TEMPLATE.format(w, h), encoding='utf-8')
     output_text.write_text('[skipped page]', encoding='utf-8')
 
 
@@ -350,7 +328,7 @@ def generate_hocr(
         tesseract_log_output(stdout)
         # The sidecar text file will get the suffix .txt; rename it to
         # whatever caller wants it named
-        if prefix.with_suffix('.txt').exists():
+        with suppress(FileNotFoundError):
             prefix.with_suffix('.txt').replace(output_text)
 
 
@@ -406,7 +384,7 @@ def generate_pdf(
     try:
         p = run(args_tesseract, stdout=PIPE, stderr=STDOUT, timeout=timeout, check=True)
         stdout = p.stdout
-        if prefix.with_suffix('.txt').exists():
+        with suppress(FileNotFoundError):
             prefix.with_suffix('.txt').replace(output_text)
     except TimeoutExpired:
         page_timedout(timeout)

src/ocrmypdf/_exec/unpaper.py

@@ -9,11 +9,12 @@ import logging
 import os
 import shlex
 import sys
+from collections.abc import Iterator
 from contextlib import contextmanager
 from decimal import Decimal
 from pathlib import Path
 from subprocess import PIPE, STDOUT
-from typing import Iterator, Union
+from typing import Union
 
 from packaging.version import Version
 from PIL import Image
@@ -72,52 +73,20 @@ def version() -> Version:
     return Version(get_version('unpaper'))
 
 
-SUPPORTED_MODES = {'1', 'L', 'RGB'}
-
-
-def _convert_image(im: Image.Image) -> tuple[Image.Image, bool]:
-    im_modified = False
-
-    if im.mode not in SUPPORTED_MODES:
-        log.info("Converting image to other colorspace")
-        try:
-            if im.mode == 'P' and len(im.getcolors()) == 2:
-                im = im.convert(mode='1')
-            else:
-                im = im.convert(mode='RGB')
-        except OSError as e:
-            raise MissingDependencyError(
-                "Could not convert image with type " + im.mode
-            ) from e
-        else:
-            im_modified = True
-        if im.mode not in SUPPORTED_MODES:
-            raise MissingDependencyError(
-                "Failed to convert image to a supported format."
-            ) from None
-    return im, im_modified
-
-
 @contextmanager
 def _setup_unpaper_io(input_file: Path) -> Iterator[tuple[Path, Path, Path]]:
     with Image.open(input_file) as im:
         if im.width * im.height >= UNPAPER_IMAGE_PIXEL_LIMIT:
             raise UnpaperImageTooLargeError(w=im.width, h=im.height)
-        im, im_modified = _convert_image(im)
 
-        with TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
+    with TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
-            tmppath = Path(tmpdir)
+        tmppath = Path(tmpdir)
-            if im_modified or input_file.suffix != '.png':
+        # No changes, PNG input, just use the file we already have
-                input_png = tmppath / 'input.png'
+        input_png = input_file
-                im.save(input_png, format='PNG')
+        # unpaper can write .png too, but it seems to write them slowly
-            else:
+        # adds a few seconds to test suite - so just use pnm
-                # No changes, PNG input, just use the file we already have
+        output_pnm = tmppath / 'output.pnm'
-                input_png = input_file
+        yield input_png, output_pnm, tmppath
-
-            # unpaper can write .png too, but it seems to write them slowly
-            # adds a few seconds to test suite - so just use pnm
-            output_pnm = tmppath / 'output.pnm'
-            yield input_png, output_pnm, tmppath
 
 
 def run_unpaper(

src/ocrmypdf/_graft.py

@@ -7,32 +7,46 @@ from __future__ import annotations
 
 import logging
 from contextlib import suppress
+from enum import Enum
 from pathlib import Path
 
 from pikepdf import (
     Dictionary,
+    Matrix,
     Name,
-    Object,
     Operator,
+    Page,
     Pdf,
     PdfError,
-    PdfMatrix,
     Stream,
     parse_content_stream,
     unparse_content_stream,
 )
 
+from ocrmypdf._jobcontext import PdfContext
+
+
+class RenderMode(Enum):
+    ON_TOP = 0
+    UNDERNEATH = 1
+
+
 log = logging.getLogger(__name__)
 MAX_REPLACE_PAGES = 100
 
 
-def _ensure_dictionary(obj, name):
+def _ensure_dictionary(obj: Dictionary | Stream, name: Name):
     if name not in obj:
         obj[name] = Dictionary({})
     return obj[name]
 
 
-def _update_resources(*, obj, font, font_key, procset):
+def _update_resources(
+    *,
+    obj: Dictionary | Stream,
+    font: Dictionary | None,
+    font_key: Name | None,
+):
     """Update this obj's fonts with a reference to the Glyphless font.
 
     obj can be a page or Form XObject.
@@ -42,13 +56,8 @@ def _update_resources(*, obj, font, font_key, procset):
     if font_key is not None and font_key not in fonts:
         fonts[font_key] = font
 
-    # Reassign /ProcSet to one that just lists everything - ProcSet is
-    # obsolete and doesn't matter but recommended for old viewer support
-    if procset:
-        resources['/ProcSet'] = procset
 
-
+def strip_invisible_text(pdf: Pdf, page: Page):
-def strip_invisible_text(pdf, page):
     stream = []
     in_text_obj = False
     render_mode = 0
@@ -79,22 +88,20 @@ def strip_invisible_text(pdf, page):
 class OcrGrafter:
     """Manages grafting text-only PDFs onto regular PDFs."""
 
-    def __init__(self, context):
+    def __init__(self, context: PdfContext):
         self.context = context
         self.path_base = context.origin
 
         self.pdf_base = Pdf.open(self.path_base)
-        self.font, self.font_key = None, None
+        self.font: Dictionary | None = None
+        self.font_key: Name | None = None
 
         self.pdfinfo = context.pdfinfo
         self.output_file = context.get_path('graft_layers.pdf')
 
-        self.procset = self.pdf_base.make_indirect(
-            Object.parse(b'[ /PDF /Text /ImageB /ImageC /ImageI ]')
-        )
-
         self.emplacements = 1
         self.interim_count = 0
+        self.render_mode = RenderMode.UNDERNEATH
 
     def graft_page(
         self,
@@ -119,7 +126,9 @@ class OcrGrafter:
                 foreign_image_page = pdf_image.pages[0]
                 self.pdf_base.pages.append(foreign_image_page)
                 local_image_page = self.pdf_base.pages[-1]
-                self.pdf_base.pages[pageno].emplace(local_image_page)
+                self.pdf_base.pages[pageno].emplace(
+                    local_image_page, retain=(Name.Parent,)
+                )
                 del self.pdf_base.pages[-1]
             emplaced_page = True
 
@@ -135,6 +144,8 @@ class OcrGrafter:
         )
 
         if textpdf and self.font:
+            if self.font_key is None:
+                raise ValueError("Font key is not set")
             # Graft the text layer onto this page, whether new or old, possibly
             # rotating the text layer by the amount is misaligned.
             strip_old = self.context.options.redo_ocr
@@ -144,7 +155,6 @@ class OcrGrafter:
                 font=self.font,
                 font_key=self.font_key,
                 text_rotation=text_misaligned,
-                procset=self.procset,
                 strip_old_text=strip_old,
             )
 
@@ -159,7 +169,7 @@ class OcrGrafter:
         if self.emplacements % MAX_REPLACE_PAGES == 0:
             self.save_and_reload()
 
-    def save_and_reload(self):
+    def save_and_reload(self) -> None:
         """Save and reload the Pdf.
 
         This will keep a lid on our memory usage for very large files. Attach
@@ -167,9 +177,7 @@ class OcrGrafter:
         back.
         """
         page0 = self.pdf_base.pages[0]
-        _update_resources(
+        _update_resources(obj=page0.obj, font=self.font, font_key=self.font_key)
-            obj=page0, font=self.font, font_key=self.font_key, procset=self.procset
-        )
 
         # We cannot read and write the same file, that will corrupt it
         # but we don't to keep more copies than we need to. Delete intermediates.
@@ -188,7 +196,6 @@ class OcrGrafter:
         self.pdf_base.close()
 
         self.pdf_base = Pdf.open(next_file)
-        self.procset = self.pdf_base.pages[0].Resources.ProcSet
         self.font, self.font_key = None, None  # Ensure we reacquire this information
         self.interim_count += 1
 
@@ -197,24 +204,32 @@ class OcrGrafter:
         self.pdf_base.close()
         return self.output_file
 
-    def _find_font(self, text):
+    def _find_font(self, text: Path) -> tuple[Dictionary | None, Name | None]:
         """Copy a font from the filename text into pdf_base."""
         font, font_key = None, None
         possible_font_names = ('/f-0-0', '/F1')
         try:
             with Pdf.open(text) as pdf_text:
                 try:
-                    pdf_text_fonts = pdf_text.pages[0].Resources.get('/Font', {})
+                    pdf_text_fonts = pdf_text.pages[0].Resources.get(
+                        Name.Font, Dictionary()
+                    )
                 except (AttributeError, IndexError, KeyError):
                     return None, None
+                if not isinstance(pdf_text_fonts, Dictionary):
+                    log.warning("Page fonts are not stored in a dictionary")
+                    return None, None
                 pdf_text_font = None
                 for f in possible_font_names:
                     pdf_text_font = pdf_text_fonts.get(f, None)
                     if pdf_text_font is not None:
-                        font_key = f
+                        font_key = Name(f)
                         break
                 if pdf_text_font:
                     font = self.pdf_base.copy_foreign(pdf_text_font)
+                if not isinstance(font, Dictionary):
+                    log.warning("Font is not a dictionary")
+                    font, font_key = None, None
                 return font, font_key
         except (FileNotFoundError, PdfError):
             # PdfError occurs if a 0-length file is written e.g. due to OCR timeout
@@ -225,9 +240,8 @@ class OcrGrafter:
         *,
         page_num: int,
         textpdf: Path,
-        font: Object,
+        font: Dictionary,
-        font_key: Object,
+        font_key: Name,
-        procset: Object,
         text_rotation: int,
         strip_old_text: bool,
     ):
@@ -248,19 +262,19 @@ class OcrGrafter:
             # content may have a rotation applied. Wrap the text stream with a rotation
             # so it will be oriented the same way as the rest of the page content.
             # (Previous versions OCRmyPDF rotated the content layer to match the text.)
-            mediabox = [float(pdf_text.pages[0].MediaBox[v]) for v in range(4)]
+            mediabox = pdf_text.pages[0].mediabox
             wt, ht = mediabox[2] - mediabox[0], mediabox[3] - mediabox[1]
 
-            mediabox = [float(base_page.MediaBox[v]) for v in range(4)]
+            mediabox = base_page.mediabox
             wp, hp = mediabox[2] - mediabox[0], mediabox[3] - mediabox[1]
 
-            translate = PdfMatrix().translated(-wt / 2, -ht / 2)
+            translate = Matrix().translated(-wt / 2, -ht / 2)
-            untranslate = PdfMatrix().translated(wp / 2, hp / 2)
+            untranslate = Matrix().translated(wp / 2, hp / 2)
-            corner = PdfMatrix().translated(mediabox[0], mediabox[1])
+            corner = Matrix().translated(mediabox[0], mediabox[1])
             # -rotation because the input is a clockwise angle and this formula
             # uses CCW
             text_rotation = -text_rotation % 360
-            rotate = PdfMatrix().rotated(text_rotation)
+            rotate = Matrix().rotated(text_rotation)
 
             # Because of rounding of DPI, we might get a text layer that is not
             # identically sized to the target page. Scale to adjust. Normally this
@@ -271,14 +285,15 @@ class OcrGrafter:
             scale_y = hp / ht
 
             # log.debug('%r', scale_x, scale_y)
-            scale = PdfMatrix().scaled(scale_x, scale_y)
+            scale = Matrix().scaled(scale_x, scale_y)
 
             # Translate the text so it is centered at (0, 0), rotate it there, adjust
             # for a size different between initial and text PDF, then untranslate, and
-            # finally move the lower left corner to match the mediabox
+            # finally move the lower left corner to match the mediabox. All transforms
-            ctm = translate @ rotate @ scale @ untranslate @ corner
+            # must be premultiplied so they are applied in reverse order here.
+            ctm = corner @ untranslate @ scale @ rotate @ translate
 
-            base_resources = _ensure_dictionary(base_page, Name.Resources)
+            base_resources = _ensure_dictionary(base_page.obj, Name.Resources)
             base_xobjs = _ensure_dictionary(base_resources, Name.XObject)
             text_xobj_name = Name.random(prefix="OCR-")
             xobj = self.pdf_base.make_stream(pdf_text_contents)
@@ -287,9 +302,7 @@ class OcrGrafter:
             xobj.Subtype = Name.Form
             xobj.FormType = 1
             xobj.BBox = mediabox
-            _update_resources(
+            _update_resources(obj=xobj, font=font, font_key=font_key)
-                obj=xobj, font=font, font_key=font_key, procset=[Name.PDF]
-            )
 
             pdf_draw_xobj = (
                 (b'q %s cm\n' % ctm.encode()) + (b'%s Do\n' % text_xobj_name) + b'\nQ\n'
@@ -299,8 +312,8 @@ class OcrGrafter:
             if strip_old_text:
                 strip_invisible_text(self.pdf_base, base_page)
 
-            base_page.contents_add(new_text_layer, prepend=True)
+            base_page.contents_add(
-
+                new_text_layer, prepend=self.render_mode == RenderMode.UNDERNEATH
-            _update_resources(
-                obj=base_page, font=font, font_key=font_key, procset=procset
             )
+
+            _update_resources(obj=base_page.obj, font=font, font_key=font_key)

src/ocrmypdf/_jobcontext.py

@@ -6,12 +6,10 @@
 from __future__ import annotations
 
 import os
-import shutil
-import sys
 from argparse import Namespace
+from collections.abc import Iterator
 from copy import copy
 from pathlib import Path
-from typing import Iterator
 
 from pluggy import PluginManager
 
@@ -55,6 +53,12 @@ class PdfContext:
         for n in range(npages):
             yield PageContext(self, n)
 
+    def get_page_context_args(self) -> Iterator[tuple[PageContext]]:
+        """Get all ``PageContext`` for this PDF packaged in tuple for args-splatting."""
+        npages = len(self.pdfinfo)
+        for n in range(npages):
+            yield (PageContext(self, n),)
+
 
 class PageContext:
     """Holds our context for a page.
@@ -94,10 +98,3 @@ class PageContext:
         if not isinstance(state['options'].output_file, (str, bytes, os.PathLike)):
             state['options'].output_file = 'stream'
         return state
-
-
-def cleanup_working_files(work_folder: Path, options: Namespace):
-    if options.keep_temporary_files:
-        print(f"Temporary working files retained at:\n{work_folder}", file=sys.stderr)
-    else:
-        shutil.rmtree(work_folder, ignore_errors=True)

src/ocrmypdf/_logging.py

@@ -9,15 +9,6 @@ import logging
 
 from rich.console import Console
 from rich.logging import RichHandler
-from rich.progress import (
-    BarColumn,
-    MofNCompleteColumn,
-    Progress,
-    TaskProgressColumn,
-    TextColumn,
-    TimeRemainingColumn,
-)
-from rich.table import Column
 
 
 class PageNumberFilter(logging.Filter):
@@ -37,56 +28,3 @@ class RichLoggingHandler(RichHandler):
         super().__init__(
             console=console, show_level=False, show_time=False, markup=True, **kwargs
         )
-
-
-class RichTqdmProgressAdapter:
-    """Adapt tqdm API to rich progress bar."""
-
-    def __init__(
-        self,
-        *,
-        console: Console,
-        desc: str,
-        total: float | None = None,
-        unit: str | None = None,
-        unit_scale: float | None = 1.0,
-        disable: bool = False,
-        **kwargs,
-    ):
-        self.progress = Progress(
-            TextColumn(
-                "[progress.description]{task.description}",
-                table_column=Column(min_width=20),
-            ),
-            BarColumn(),
-            TaskProgressColumn(),
-            MofNCompleteColumn(),
-            TimeRemainingColumn(),
-            console=console,
-            auto_refresh=True,
-            redirect_stderr=True,
-            redirect_stdout=False,
-            disable=disable,
-            **kwargs,
-        )
-        self.unit_scale = unit_scale
-        self.progress_bar = self.progress.add_task(
-            desc,
-            total=total * self.unit_scale
-            if total is not None and self.unit_scale is not None
-            else None,
-            unit=unit,
-        )
-
-    def __enter__(self):
-        self.progress.start()
-        return self
-
-    def __exit__(self, exc_type, exc_value, traceback):
-        self.progress.refresh()
-        self.progress.stop()
-        return False
-
-    def update(self, value=None):
-        advance = self.unit_scale if value is None else value
-        self.progress.update(self.progress_bar, advance=advance)

src/ocrmypdf/_metadata.py

@@ -0,0 +1,184 @@
+# SPDX-FileCopyrightText: 2023 James R. Barlow
+# SPDX-License-Identifier: MPL-2.0
+
+"""OCRmyPDF page processing pipeline functions."""
+
+from __future__ import annotations
+
+import logging
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from pikepdf import Dictionary, Name, Pdf
+from pikepdf import __version__ as PIKEPDF_VERSION
+from pikepdf.models.metadata import PdfMetadata, encode_pdf_date
+
+from ocrmypdf._jobcontext import PdfContext
+from ocrmypdf._version import PROGRAM_NAME
+from ocrmypdf._version import __version__ as OCRMYPF_VERSION
+from ocrmypdf.languages import iso_639_2_from_3
+
+log = logging.getLogger(__name__)
+
+
+def get_docinfo(base_pdf: Pdf, context: PdfContext) -> dict[str, str]:
+    """Read the document info and store it in a dictionary."""
+    options = context.options
+
+    def from_document_info(key):
+        try:
+            s = base_pdf.docinfo[key]
+            return str(s)
+        except (KeyError, TypeError):
+            return ''
+
+    pdfmark = {
+        k: from_document_info(k)
+        for k in ('/Title', '/Author', '/Keywords', '/Subject', '/CreationDate')
+    }
+    if options.title:
+        pdfmark['/Title'] = options.title
+    if options.author:
+        pdfmark['/Author'] = options.author
+    if options.keywords:
+        pdfmark['/Keywords'] = options.keywords
+    if options.subject:
+        pdfmark['/Subject'] = options.subject
+
+    creator_tag = context.plugin_manager.hook.get_ocr_engine().creator_tag(options)
+
+    pdfmark['/Creator'] = f'{PROGRAM_NAME} {OCRMYPF_VERSION} / {creator_tag}'
+    pdfmark['/Producer'] = f'pikepdf {PIKEPDF_VERSION}'
+    pdfmark['/ModDate'] = encode_pdf_date(datetime.now(timezone.utc))
+    return pdfmark
+
+
+def report_on_metadata(options, missing):
+    if not missing:
+        return
+    if options.output_type.startswith('pdfa'):
+        log.warning(
+            "Some input metadata could not be copied because it is not "
+            "permitted in PDF/A. You may wish to examine the output "
+            "PDF's XMP metadata."
+        )
+        log.debug("The following metadata fields were not copied: %r", missing)
+    else:
+        log.error(
+            "Some input metadata could not be copied."
+            "You may wish to examine the output PDF's XMP metadata."
+        )
+        log.info("The following metadata fields were not copied: %r", missing)
+
+
+def repair_docinfo_nuls(pdf):
+    """If the DocumentInfo block contains NUL characters, remove them.
+
+    If the DocumentInfo block is malformed, log an error and continue.
+    """
+    modified = False
+    try:
+        if not isinstance(pdf.docinfo, Dictionary):
+            raise TypeError("DocumentInfo is not a dictionary")
+        for k, v in pdf.docinfo.items():
+            if isinstance(v, str) and b'\x00' in bytes(v):
+                pdf.docinfo[k] = bytes(v).replace(b'\x00', b'')
+                modified = True
+    except TypeError:
+        # TypeError can also be raised if dictionary items are unexpected types
+        log.error("File contains a malformed DocumentInfo block - continuing anyway.")
+    return modified
+
+
+def should_linearize(working_file: Path, context: PdfContext) -> bool:
+    """Determine whether the PDF should be linearized.
+
+    For smaller files, linearization is not worth the effort.
+    """
+    filesize = os.stat(working_file).st_size
+    if filesize > (context.options.fast_web_view * 1_000_000):
+        return True
+    return False
+
+
+def _fix_metadata(meta_original: PdfMetadata, meta_pdf: PdfMetadata):
+    # If xmp:CreateDate is missing, set it to the modify date to
+    # ensure consistency with Ghostscript.
+    if 'xmp:CreateDate' not in meta_pdf:
+        meta_pdf['xmp:CreateDate'] = meta_pdf.get('xmp:ModifyDate', '')
+    if meta_pdf.get('dc:title') == 'Untitled':
+        # Ghostscript likes to set title to Untitled if omitted from input.
+        # Reverse this, because PDF/A TechNote 0003:Metadata in PDF/A-1
+        # and the XMP Spec do not make this recommendation.
+        if 'dc:title' not in meta_original:
+            del meta_pdf['dc:title']
+
+
+def _unset_empty_metadata(meta: PdfMetadata, options):
+    """Unset metadata fields that were explicitly set to empty strings.
+
+    If the user explicitly specified an empty string for any of the
+    following, they should be unset and not reported as missing in
+    the output pdf. Note that some metadata fields use differing names
+    between PDF/A and PDF.
+    """
+    if options.title == '' and 'dc:title' in meta:
+        del meta['dc:title']  # PDF/A and PDF
+    if options.author == '':
+        if 'dc:creator' in meta:
+            del meta['dc:creator']  # PDF/A (Not xmp:CreatorTool)
+        if 'pdf:Author' in meta:
+            del meta['pdf:Author']  # PDF
+    if options.subject == '':
+        if 'dc:description' in meta:
+            del meta['dc:description']  # PDF/A
+        if 'dc:subject' in meta:
+            del meta['dc:subject']  # PDF
+    if options.keywords == '' and 'pdf:Keywords' in meta:
+        del meta['pdf:Keywords']  # PDF/A and PDF
+
+
+def _set_language(pdf: Pdf, languages: list[str]):
+    """Set the language of the PDF."""
+    if Name.Lang in pdf.Root or not languages:
+        return  # Already set or can't change
+    primary_language_iso639_3 = languages[0]
+    if not primary_language_iso639_3:
+        return
+    iso639_2 = iso_639_2_from_3(primary_language_iso639_3)
+    if not iso639_2:
+        return
+    pdf.Root.Lang = iso639_2
+
+
+def metadata_fixup(
+    working_file: Path, context: PdfContext, pdf_save_settings: dict[str, Any]
+) -> Path:
+    """Fix certain metadata fields after Ghostscript PDF/A conversion.
+
+    Also report on metadata in the input file that was not retained during
+    PDF/A conversion.
+    """
+    output_file = context.get_path('metafix.pdf')
+    options = context.options
+
+    with Pdf.open(context.origin) as original, Pdf.open(working_file) as pdf:
+        docinfo = get_docinfo(original, context)
+        with original.open_metadata(
+            set_pikepdf_as_editor=False, update_docinfo=False, strict=False
+        ) as meta_original, pdf.open_metadata() as meta_pdf:
+            meta_pdf.load_from_docinfo(
+                docinfo, delete_missing=False, raise_failure=False
+            )
+            _fix_metadata(meta_original, meta_pdf)
+            _unset_empty_metadata(meta_original, options)
+            _unset_empty_metadata(meta_pdf, options)
+            meta_missing = set(meta_original.keys()) - set(meta_pdf.keys())
+            report_on_metadata(options, meta_missing)
+
+        _set_language(pdf, options.languages)
+        pdf.save(output_file, **pdf_save_settings)
+
+    return output_file

src/ocrmypdf/_pipeline.py

@@ -10,28 +10,27 @@ import logging
 import os
 import re
 import sys
+from collections.abc import Iterable, Iterator, Sequence
 from contextlib import suppress
-from datetime import datetime, timezone
 from pathlib import Path
-from shutil import copyfileobj
+from shutil import copyfileobj, copystat
-from typing import Any, BinaryIO, Iterable, Iterator, Sequence, cast
+from typing import Any, BinaryIO, TypeVar, cast
 
 import img2pdf
 import pikepdf
-from pikepdf.models.metadata import encode_pdf_date
 from PIL import Image, ImageColor, ImageDraw
 
 from ocrmypdf._concurrent import Executor
 from ocrmypdf._exec import unpaper
 from ocrmypdf._jobcontext import PageContext, PdfContext
-from ocrmypdf._version import PROGRAM_NAME
+from ocrmypdf._metadata import repair_docinfo_nuls
-from ocrmypdf._version import __version__ as VERSION
 from ocrmypdf.exceptions import (
     DigitalSignatureError,
     DpiError,
     EncryptedPdfError,
     InputFileError,
     PriorOcrFoundError,
+    TaggedPDFError,
     UnsupportedImageFormatError,
 )
 from ocrmypdf.helpers import IMG2PDF_KWARGS, Resolution, safe_symlink
@@ -40,13 +39,7 @@ from ocrmypdf.pdfa import generate_pdfa_ps
 from ocrmypdf.pdfinfo import Colorspace, Encoding, PageInfo, PdfInfo
 from ocrmypdf.pluginspec import OrientationConfidence
 
-# Remove this workaround when we require Pillow >= 10
+T = TypeVar("T")
-try:
-    BICUBIC = Image.Resampling.BICUBIC  # type: ignore
-except AttributeError:  # pragma: no cover
-    # Pillow 9 shim
-    BICUBIC = Image.BICUBIC  # type: ignore
-
 log = logging.getLogger(__name__)
 
 VECTOR_PAGE_DPI = 400
@@ -145,6 +138,7 @@ def _pdf_guess_version(input_file: Path, search_window=1024) -> str:
 def triage(
     original_filename: str, input_file: Path, output_file: Path, options
 ) -> Path:
+    """Triage the input file. We can handle PDFs and images."""
     try:
         if _pdf_guess_version(input_file):
             if options.image_dpi:
@@ -171,14 +165,17 @@ def get_pdfinfo(
     detailed_analysis: bool = False,
     progbar: bool = False,
     max_workers: int | None = None,
+    use_threads: bool = True,
     check_pages=None,
 ) -> PdfInfo:
+    """Get the PDF info."""
     try:
         return PdfInfo(
             input_file,
             detailed_analysis=detailed_analysis,
             progbar=progbar,
             max_workers=max_workers,
+            use_threads=use_threads,
             check_pages=check_pages,
             executor=executor,
         )
@@ -189,6 +186,7 @@ def get_pdfinfo(
 
 
 def validate_pdfinfo_options(context: PdfContext) -> None:
+    """Validate the PDF info options."""
     pdfinfo = context.pdfinfo
     options = context.options
 
@@ -220,6 +218,16 @@ def validate_pdfinfo_options(context: PdfContext) -> None:
                     "form and all filled form fields. The output PDF will be "
                     "'flattened' and will no longer be fillable."
                 )
+    if pdfinfo.is_tagged:
+        if options.force_ocr or options.skip_text or options.redo_ocr:
+            log.warning(
+                "This PDF is marked as a Tagged PDF. This often indicates "
+                "that the PDF was generated from an office document and does "
+                "not need OCR. PDF pages processed by OCRmyPDF may not be "
+                "tagged correctly."
+            )
+        else:
+            raise TaggedPDFError()
     context.plugin_manager.hook.validate(pdfinfo=pdfinfo, options=options)
 
 
@@ -434,6 +442,7 @@ def get_orientation_correction(preview: Path, page_context: PageContext) -> int:
 
 
 def calculate_image_dpi(page_context: PageContext) -> Resolution:
+    """Calculate the DPI for the page image."""
     pageinfo = page_context.pageinfo
     dpi_profile = pageinfo.page_dpi_profile()
     if dpi_profile and dpi_profile.average_to_max_dpi_ratio < 0.8:
@@ -509,6 +518,7 @@ def rasterize(
                 device_idx = at_least('png16m')
 
     if pageinfo.has_vector:
+        log.debug("Page has vector content, using png16m")
         device_idx = at_least('png16m')
 
     device = colorspaces[device_idx]
@@ -532,6 +542,7 @@ def rasterize(
 
 
 def preprocess_remove_background(input_file: Path, page_context: PageContext) -> Path:
+    """Remove the background from the input image (temporarily disabled)."""
     if any(image.bpc > 1 for image in page_context.pageinfo.images):
         raise NotImplementedError("--remove-background is temporarily not implemented")
         # output_file = page_context.get_path('pp_rm_bg.png')
@@ -562,7 +573,7 @@ def preprocess_deskew(input_file: Path, page_context: PageContext) -> Path:
         # resampling if image is mode '1' or 'P'
         deskewed = im.rotate(
             deskew_angle_degrees,
-            resample=BICUBIC,
+            resample=Image.Resampling.BICUBIC,
             fillcolor=ImageColor.getcolor('white', mode=im.mode),  # type: ignore
         )
         deskewed.save(output_file, dpi=dpi)
@@ -571,6 +582,7 @@ def preprocess_deskew(input_file: Path, page_context: PageContext) -> Path:
 
 
 def preprocess_clean(input_file: Path, page_context: PageContext) -> Path:
+    """Clean the input image using unpaper."""
     output_file = page_context.get_path('pp_clean.png')
     dpi = get_page_square_dpi(page_context, calculate_image_dpi(page_context))
     return unpaper.clean(
@@ -631,6 +643,7 @@ def create_ocr_image(image: Path, page_context: PageContext) -> Path:
 
 
 def ocr_engine_hocr(input_file: Path, page_context: PageContext) -> tuple[Path, Path]:
+    """Run the OCR engine and generate hOCR output."""
     hocr_out = page_context.get_path('ocr_hocr.hocr')
     hocr_text_out = page_context.get_path('ocr_hocr.txt')
     options = page_context.options
@@ -642,7 +655,7 @@ def ocr_engine_hocr(input_file: Path, page_context: PageContext) -> tuple[Path,
         output_text=hocr_text_out,
         options=options,
     )
-    return (hocr_out, hocr_text_out)
+    return hocr_out, hocr_text_out
 
 
 def should_visible_page_image_use_jpg(pageinfo: PageInfo) -> bool:
@@ -662,6 +675,10 @@ def should_visible_page_image_use_jpg(pageinfo: PageInfo) -> bool:
 
 
 def create_visible_page_jpg(image: Path, page_context: PageContext) -> Path:
+    """Create a visible page image in JPEG format.
+
+    This is intended to be used when all images on the page were originally JPEGs.
+    """
     output_file = page_context.get_path('visible.jpg')
     with Image.open(image) as im:
         # At this point the image should be a .png, but deskew, unpaper
@@ -683,6 +700,7 @@ def create_visible_page_jpg(image: Path, page_context: PageContext) -> Path:
 def create_pdf_page_from_image(
     image: Path, page_context: PageContext, orientation_correction: int
 ) -> Path:
+    """Create a PDF page from a page image."""
     # We rasterize a square DPI version of each page because most image
     # processing tools don't support rectangular DPI. Use the square DPI as it
     # accurately describes the image. It would be possible to resample the image
@@ -714,12 +732,18 @@ def create_pdf_page_from_image(
 
 
 def render_hocr_page(hocr: Path, page_context: PageContext) -> Path:
+    """Render the hOCR page to a PDF."""
     options = page_context.options
     output_file = page_context.get_path('ocr_hocr.pdf')
+    if hocr.stat().st_size == 0:
+        # If hOCR file is empty (skipped page marker), create an empty PDF file
+        output_file.touch()
+        return output_file
+
     dpi = get_page_square_dpi(page_context, calculate_image_dpi(page_context))
     debug_mode = options.pdf_renderer == 'hocrdebug'
 
-    hocrtransform = HocrTransform(hocr_filename=hocr, dpi=dpi.x)  # square
+    hocrtransform = HocrTransform(hocr_filename=hocr, dpi=dpi.to_scalar())  # square
     hocrtransform.to_pdf(
         out_filename=output_file,
         image_filename=None,
@@ -733,6 +757,7 @@ def render_hocr_page(hocr: Path, page_context: PageContext) -> Path:
 def ocr_engine_textonly_pdf(
     input_image: Path, page_context: PageContext
 ) -> tuple[Path, Path]:
+    """Run the OCR engine and generate a text-only PDF (will look blank)."""
     output_pdf = page_context.get_path('ocr_tess.pdf')
     output_text = page_context.get_path('ocr_tess.txt')
     options = page_context.options
@@ -747,37 +772,6 @@ def ocr_engine_textonly_pdf(
     return (output_pdf, output_text)
 
 
-def get_docinfo(base_pdf: pikepdf.Pdf, context: PdfContext) -> dict[str, str]:
-    options = context.options
-
-    def from_document_info(key):
-        try:
-            s = base_pdf.docinfo[key]
-            return str(s)
-        except (KeyError, TypeError):
-            return ''
-
-    pdfmark = {
-        k: from_document_info(k)
-        for k in ('/Title', '/Author', '/Keywords', '/Subject', '/CreationDate')
-    }
-    if options.title:
-        pdfmark['/Title'] = options.title
-    if options.author:
-        pdfmark['/Author'] = options.author
-    if options.keywords:
-        pdfmark['/Keywords'] = options.keywords
-    if options.subject:
-        pdfmark['/Subject'] = options.subject
-
-    creator_tag = context.plugin_manager.hook.get_ocr_engine().creator_tag(options)
-
-    pdfmark['/Creator'] = f'{PROGRAM_NAME} {VERSION} / {creator_tag}'
-    pdfmark['/Producer'] = f'pikepdf {pikepdf.__version__}'
-    pdfmark['/ModDate'] = encode_pdf_date(datetime.now(timezone.utc))
-    return pdfmark
-
-
 def generate_postscript_stub(context: PdfContext) -> Path:
     """Generates a PostScript file stub for the given PDF context.
 
@@ -793,6 +787,14 @@ def generate_postscript_stub(context: PdfContext) -> Path:
 
 
 def convert_to_pdfa(input_pdf: Path, input_ps_stub: Path, context: PdfContext) -> Path:
+    """Converts the given PDF to PDF/A.
+
+    Args:
+        input_pdf: The input PDF file path (presumably not PDF/A).
+        input_ps_stub: The input PostScript file path, containing instructions
+            for the PDF/A generator to use.
+        context: The PDF context.
+    """
     options = context.options
     input_pdfinfo = context.pdfinfo
     fix_docinfo_file = context.get_path('fix_docinfo.pdf')
@@ -804,7 +806,7 @@ def convert_to_pdfa(input_pdf: Path, input_ps_stub: Path, context: PdfContext) -
     # pikepdf can deal with this, but we make the world a better place by
     # stamping them out as soon as possible.
     with pikepdf.open(input_pdf) as pdf_file:
-        if _repair_docinfo_nuls(pdf_file):
+        if repair_docinfo_nuls(pdf_file):
             pdf_file.save(fix_docinfo_file)
         else:
             safe_symlink(input_pdf, fix_docinfo_file)
@@ -827,26 +829,11 @@ def convert_to_pdfa(input_pdf: Path, input_ps_stub: Path, context: PdfContext) -
     return output_file
 
 
-def _repair_docinfo_nuls(pdf):
-    """If the DocumentInfo block contains NUL characters, remove them.
-
-    If the DocumentInfo block is malformed, log an error and continue.
-    """
-    modified = False
-    try:
-        if not isinstance(pdf.docinfo, pikepdf.Dictionary):
-            raise TypeError("DocumentInfo is not a dictionary")
-        for k, v in pdf.docinfo.items():
-            if isinstance(v, str) and b'\x00' in bytes(v):
-                pdf.docinfo[k] = bytes(v).replace(b'\x00', b'')
-                modified = True
-    except TypeError:
-        # TypeError can also be raised if dictionary items are unexpected types
-        log.error("File contains a malformed DocumentInfo block - continuing anyway.")
-    return modified
-
-
 def should_linearize(working_file: Path, context: PdfContext) -> bool:
+    """Determine whether the PDF should be linearized.
+
+    For smaller files, linearization is not worth the effort.
+    """
     filesize = os.stat(working_file).st_size
     if filesize > (context.options.fast_web_view * 1_000_000):
         return True
@@ -854,6 +841,11 @@ def should_linearize(working_file: Path, context: PdfContext) -> bool:
 
 
 def get_pdf_save_settings(output_type: str) -> dict[str, Any]:
+    """Get pikepdf.Pdf.save settings for the given output type.
+
+    Essentially, don't use features that are incompatible with a given
+    PDF/A specification.
+    """
     if output_type == 'pdfa-1':
         # Trigger recompression to ensure object streams are removed, because
         # Acrobat complains about them in PDF/A-1b validation.
@@ -871,83 +863,6 @@ def get_pdf_save_settings(output_type: str) -> dict[str, Any]:
         )
 
 
-def metadata_fixup(working_file: Path, context: PdfContext) -> Path:
-    output_file = context.get_path('metafix.pdf')
-    options = context.options
-
-    def report_on_metadata(missing):
-        if not missing:
-            return
-        if options.output_type.startswith('pdfa'):
-            log.warning(
-                "Some input metadata could not be copied because it is not "
-                "permitted in PDF/A. You may wish to examine the output "
-                "PDF's XMP metadata."
-            )
-            log.debug("The following metadata fields were not copied: %r", missing)
-        else:
-            log.error(
-                "Some input metadata could not be copied."
-                "You may wish to examine the output PDF's XMP metadata."
-            )
-            log.info("The following metadata fields were not copied: %r", missing)
-
-    with pikepdf.open(context.origin) as original, pikepdf.open(working_file) as pdf:
-        docinfo = get_docinfo(original, context)
-        with pdf.open_metadata() as meta_pdf:
-            meta_pdf.load_from_docinfo(
-                docinfo, delete_missing=False, raise_failure=False
-            )
-            # If xmp:CreateDate is missing, set it to the modify date to
-            # ensure consistency with Ghostscript.
-            if 'xmp:CreateDate' not in meta_pdf:
-                meta_pdf['xmp:CreateDate'] = meta_pdf.get('xmp:ModifyDate', '')
-
-            with original.open_metadata(
-                set_pikepdf_as_editor=False, update_docinfo=False, strict=False
-            ) as meta_original:
-                if meta_pdf.get('dc:title') == 'Untitled':
-                    # Ghostscript likes to set title to Untitled if omitted from input.
-                    # Reverse this, because PDF/A TechNote 0003:Metadata in PDF/A-1
-                    # and the XMP Spec do not make this recommendation.
-                    if 'dc:title' not in meta_original:
-                        del meta_pdf['dc:title']
-                # If the user explicitly specified an empty string for any of the
-                # following, they should be unset and not reported as missing in
-                # the output pdf. Note that some metadata fields use differing names
-                # between PDF-A and PDF.
-                for meta in [meta_pdf, meta_original]:
-                    if options.title == '' and 'dc:title' in meta:
-                        del meta['dc:title']  # PDF-A and PDF
-                    if options.author == '':
-                        if 'dc:creator' in meta:
-                            del meta['dc:creator']  # PDF-A (Not xmp:CreatorTool)
-                        if 'pdf:Author' in meta:
-                            del meta['pdf:Author']  # PDF
-                    if options.subject == '':
-                        if 'dc:description' in meta:
-                            del meta['dc:description']  # PDF-A
-                        if 'dc:subject' in meta:
-                            del meta['dc:subject']  # PDF
-                    if options.keywords == '' and 'pdf:Keywords' in meta:
-                        del meta['pdf:Keywords']  # PDF-A and PDF
-                meta_missing = set(meta_original.keys()) - set(meta_pdf.keys())
-                report_on_metadata(meta_missing)
-
-        optimizing = context.plugin_manager.hook.is_optimization_enabled(
-            context=context
-        )
-        pdf.save(
-            output_file,
-            **get_pdf_save_settings(options.output_type),
-            linearize=(  # Don't linearize if optimize() will be linearizing too
-                not optimizing and should_linearize(working_file, context)
-            ),
-        )
-
-    return output_file
-
-
 def _file_size_ratio(
     input_file: Path, output_file: Path
 ) -> tuple[float | None, float | None]:
@@ -974,6 +889,7 @@ def _file_size_ratio(
 def optimize_pdf(
     input_file: Path, context: PdfContext, executor: Executor
 ) -> tuple[Path, Sequence[str]]:
+    """Optimize the given PDF file."""
     output_file = context.get_path('optimize.pdf')
     output_pdf, messages = context.plugin_manager.hook.optimize_pdf(
         input_pdf=input_file,
@@ -993,8 +909,8 @@ def optimize_pdf(
 
 
 def enumerate_compress_ranges(
-    iterable: Iterable,
+    iterable: Iterable[T],
-) -> Iterator[tuple[tuple[int, int], Any]]:
+) -> Iterator[tuple[tuple[int, int], T | None]]:
     """Enumerate the ranges of non-empty elements in an iterable.
 
     Compresses consecutive ranges of length 1 into single elements.
@@ -1022,21 +938,22 @@ def enumerate_compress_ranges(
 
 
 def merge_sidecars(txt_files: Iterable[Path | None], context: PdfContext) -> Path:
+    """Merge the page sidecar files into a single file.
+
+    Sidecar files are created by the OCR engine and contain the text for each
+    page in the PDF. This function merges the sidecar files into a single file
+    and returns the path to the merged file.
+    """
     output_file = context.get_path('sidecar.txt')
     with open(output_file, 'w', encoding="utf-8") as stream:
         for (from_, to_), txt_file in enumerate_compress_ranges(txt_files):
             if from_ != 1:
-                stream.write('\f')  # Form feed between pages
+                stream.write('\f')  # Form feed between pages for all pages after first
             if txt_file:
-                with open(txt_file, encoding="utf-8") as in_:
+                txt = txt_file.read_text(encoding="utf-8")
-                    txt = in_.read()
+                # Some versions of Tesseract add a form feed at the end and
-                    # Some OCR engines (e.g. Tesseract v4 alpha) add form feeds
+                # others don't. Remove it if it exists, since we add one manually.
-                    # between pages, and some do not. For consistency, we ignore
+                stream.write(txt.removesuffix('\f'))
-                    # any added by the OCR engine and them on our own.
-                    if txt.endswith('\f'):
-                        stream.write(txt[:-1])
-                    else:
-                        stream.write(txt)
             else:
                 if from_ != to_:
                     pages = f'{from_}-{to_}'
@@ -1047,14 +964,14 @@ def merge_sidecars(txt_files: Iterable[Path | None], context: PdfContext) -> Pat
 
 
 def copy_final(
-    input_file: Path, output_file: str | Path | BinaryIO, _context: PdfContext
+    input_file: Path, output_file: str | Path | BinaryIO, original_file: Path | None
 ) -> None:
     """Copy the final temporary file to the output destination.
 
     Args:
-        input_file (Path): The input file to copy.
+        input_file (Path): The intermediate input file to copy.
         output_file (str | Path | BinaryIO): The output file to copy to.
-        _context (PdfContext): The PDF context.
+        original_file: The original file to copy attributes from.
 
     Returns:
         None
@@ -1073,5 +990,12 @@ def copy_final(
             # At this point we overwrite the output_file specified by the user
             # use copyfileobj because then we use open() to create the file and
             # get the appropriate umask, ownership, etc.
-            with open(output_file, 'wb') as output_stream:
+            with open(output_file, 'w+b') as output_stream:
                 copyfileobj(input_stream, output_stream)
+            # Attempt to copy file attributes from input to output
+            if original_file:
+                with suppress(OSError):
+                    # Copy original file's permissions, ownership, etc. if possible
+                    copystat(original_file, output_file)
+                    # Set output file's modification time to now
+                    Path(output_file).touch(exist_ok=True)

src/ocrmypdf/_pipelines/__init__.py

@@ -0,0 +1,5 @@
+# SPDX-FileCopyrightText: 2023 James R. Barlow
+# SPDX-License-Identifier: MPL-2.0
+
+
+from __future__ import annotations

src/ocrmypdf/_pipelines/_common.py

@@ -1,65 +1,53 @@
-# SPDX-FileCopyrightText: 2019-2022 James R. Barlow
+# SPDX-FileCopyrightText: 2023 James R. Barlow
-# SPDX-FileCopyrightText: 2019 Martin Wind
 # SPDX-License-Identifier: MPL-2.0
 
-"""Implements the concurrent and page synchronous parts of the pipeline."""
-
-
 from __future__ import annotations
 
 import argparse
+import json
 import logging
 import logging.handlers
 import os
+import shutil
 import sys
 import threading
+from collections.abc import Sequence
 from concurrent.futures.process import BrokenProcessPool
 from concurrent.futures.thread import BrokenThreadPool
-from functools import partial
+from contextlib import contextmanager
+from dataclasses import dataclass
 from pathlib import Path
-from tempfile import mkdtemp
+from typing import Callable, NamedTuple, cast
-from typing import NamedTuple, Sequence, cast
 
 import PIL
 
 from ocrmypdf._concurrent import Executor, setup_executor
-from ocrmypdf._graft import OcrGrafter
+from ocrmypdf._jobcontext import PageContext, PdfContext
-from ocrmypdf._jobcontext import PageContext, PdfContext, cleanup_working_files
 from ocrmypdf._logging import PageNumberFilter
+from ocrmypdf._metadata import metadata_fixup
 from ocrmypdf._pipeline import (
     convert_to_pdfa,
-    copy_final,
     create_ocr_image,
     create_pdf_page_from_image,
     create_visible_page_jpg,
     generate_postscript_stub,
     get_orientation_correction,
-    get_pdfinfo,
+    get_pdf_save_settings,
-    is_ocr_required,
-    merge_sidecars,
-    metadata_fixup,
-    ocr_engine_hocr,
-    ocr_engine_textonly_pdf,
     optimize_pdf,
     preprocess_clean,
     preprocess_deskew,
     preprocess_remove_background,
     rasterize,
     rasterize_preview,
-    render_hocr_page,
+    should_linearize,
     should_visible_page_image_use_jpg,
-    triage,
-    validate_pdfinfo_options,
 )
-from ocrmypdf._plugin_manager import OcrmypdfPluginManager, get_plugin_manager
+from ocrmypdf._plugin_manager import OcrmypdfPluginManager
 from ocrmypdf._validation import (
-    check_requested_output_file,
-    create_input_file,
     report_output_file_size,
 )
 from ocrmypdf.exceptions import ExitCode, ExitCodeException
 from ocrmypdf.helpers import (
-    NeverRaise,
     available_cpu_count,
     check_pdf,
     pikepdf_enable_mmap,
@@ -68,33 +56,248 @@ from ocrmypdf.helpers import (
 from ocrmypdf.pdfa import file_claims_pdfa
 
 log = logging.getLogger(__name__)
+tls = threading.local()
+tls.pageno = None
+
+
+def _set_logging_tls(tls):
+    """Inject current page number (when available) into log records."""
+    old_factory = logging.getLogRecordFactory()
+
+    def wrapper(*args, **kwargs):
+        record = old_factory(*args, **kwargs)
+        if hasattr(tls, 'pageno'):
+            record.pageno = tls.pageno
+        return record
+
+    logging.setLogRecordFactory(wrapper)
+
+
+_set_logging_tls(tls)
+
+
+def set_thread_pageno(pageno: int | None):
+    """Set page number (1-based) that the current thread is processing."""
+    tls.pageno = pageno
 
 
 class PageResult(NamedTuple):
     """Result when a page is finished processing."""
 
     pageno: int
-    pdf_page_from_image: Path | None
+    """Page number, 0-based."""
-    ocr: Path | None
+
-    text: Path | None
+    pdf_page_from_image: Path | None = None
-    orientation_correction: int
+    """Single page PDF from image."""
+
+    ocr: Path | None = None
+    """Single page OCR PDF."""
+
+    text: Path | None = None
+    """Single page text file."""
+
+    orientation_correction: int = 0
+    """Orientation correction in degrees."""
 
 
-tls = threading.local()
+@dataclass
-tls.pageno = None
+class HOCRResult:
+    """Result when hOCR is finished processing."""
+
+    pageno: int
+    """Page number, 0-based."""
+
+    pdf_page_from_image: Path | None = None
+    """Single page PDF from image."""
+
+    hocr: Path | None = None
+    """Single page hOCR file."""
+
+    textpdf: Path | None = None
+    """hOCR file after conversion to PDF."""
+
+    orientation_correction: int = 0
+    """Orientation correction in degrees."""
+
+    def __getstate__(self):
+        """Return state values to be pickled."""
+        return {
+            k: (
+                ('Path://' + str(v))
+                if k in ('pdf_page_from_image', 'hocr', 'textpdf') and v is not None
+                else v
+            )
+            for k, v in self.__dict__.items()
+        }
+
+    def __setstate__(self, state):
+        """Restore state from the unpickled state values."""
+        self.__dict__.update(
+            {
+                k: (
+                    Path(v.removeprefix('Path://'))
+                    if k in ('pdf_page_from_image', 'hocr', 'textpdf') and v is not None
+                    else v
+                )
+                for k, v in state.items()
+            }
+        )
+
+    @classmethod
+    def from_json(cls, json_str: str) -> HOCRResult:
+        """Create an instance from a dict."""
+        return cls(**json.loads(json_str))
+
+    def to_json(self) -> str:
+        """Serialize to a JSON string."""
+        return json.dumps(self.__getstate__())
 
 
-old_factory = logging.getLogRecordFactory()
+def configure_debug_logging(
+    log_filename: Path, prefix: str = ''
+) -> tuple[logging.FileHandler, Callable[[], None]]:
+    """Create a debug log file at a specified location.
+
+    Returns the log handler, and a function to remove the handler.
+
+    Args:
+        log_filename: Where to the put the log file.
+        prefix: The logging domain prefix that should be sent to the log.
+    """
+    log_file_handler = logging.FileHandler(log_filename, delay=True)
+    log_file_handler.setLevel(logging.DEBUG)
+    formatter = logging.Formatter(
+        '[%(asctime)s] - %(name)s - %(levelname)7s -%(pageno)s %(message)s'
+    )
+    log_file_handler.setFormatter(formatter)
+    log_file_handler.addFilter(PageNumberFilter())
+    logging.getLogger(prefix).addHandler(log_file_handler)
+
+    def remover():
+        try:
+            logging.getLogger(prefix).removeHandler(log_file_handler)
+            log_file_handler.close()
+        except OSError as e:
+            print(e, file=sys.stderr)
+
+    return log_file_handler, remover
 
 
-def record_factory(*args, **kwargs):
+def worker_init(max_pixels: int) -> None:
-    record = old_factory(*args, **kwargs)
+    """Initialize a worker thread or process."""
-    if hasattr(tls, 'pageno'):
+    # In Windows, child process will not inherit our change to this value in
-        record.pageno = tls.pageno
+    # the parent process, so ensure workers get it set. Not needed when running
-    return record
+    # threaded, but harmless to set again.
+    PIL.Image.MAX_IMAGE_PIXELS = max_pixels
+    pikepdf_enable_mmap()
 
 
-logging.setLogRecordFactory(record_factory)
+@contextmanager
+def manage_debug_log_handler(
+    *,
+    options: argparse.Namespace,
+    work_folder: Path,
+):
+    remover = None
+    if (options.keep_temporary_files or options.verbose >= 1) and not os.environ.get(
+        'PYTEST_CURRENT_TEST', ''
+    ):
+        # Debug log for command line interface only with verbose output
+        # See https://github.com/pytest-dev/pytest/issues/5502 for why we skip this
+        # when pytest is running
+        _debug_log_handler, remover = configure_debug_logging(
+            work_folder / "debug.log", prefix=""
+        )  # pragma: no cover
+    try:
+        yield
+    finally:
+        if remover:
+            remover()
+
+
+@contextmanager
+def manage_work_folder(*, work_folder: Path, retain: bool, print_location: bool):
+    try:
+        yield work_folder
+    finally:
+        if retain:
+            if print_location:
+                print(
+                    f"Temporary working files retained at:\n{work_folder}",
+                    file=sys.stderr,
+                )
+        else:
+            shutil.rmtree(work_folder, ignore_errors=True)
+
+
+def cli_exception_handler(
+    fn: Callable[[argparse.Namespace, OcrmypdfPluginManager], ExitCode],
+    options: argparse.Namespace,
+    plugin_manager: OcrmypdfPluginManager,
+) -> ExitCode:
+    """Convert exceptions into command line error messages and exit codes.
+
+    When known exceptions are raised, the exception message is printed to stderr
+    and the program exits with a non-zero exit code. When unknown exceptions are
+    raised, the exception traceback is printed to stderr and the program exits
+    with a non-zero exit code.
+    """
+    try:
+        # We cannot use a generator and yield here, as would be the usual pattern
+        # for exception handling context managers, because we need to return an exit
+        # code.
+        return fn(options, plugin_manager)
+    except KeyboardInterrupt:
+        if options.verbose >= 1:
+            log.exception("KeyboardInterrupt")
+        else:
+            log.error("KeyboardInterrupt")
+        return ExitCode.ctrl_c
+    except ExitCodeException as e:
+        e = cast(ExitCodeException, e)
+        if options.verbose >= 1:
+            log.exception("ExitCodeException")
+        elif str(e):
+            log.error("%s: %s", type(e).__name__, str(e))
+        else:
+            log.error(type(e).__name__)
+        return e.exit_code
+    except PIL.Image.DecompressionBombError:
+        log.exception(
+            "A decompression bomb error was encountered while executing the "
+            "pipeline. Use the argument --max-image-mpixels to raise the maximum "
+            "image pixel limit."
+        )
+        return ExitCode.other_error
+    except (
+        BrokenProcessPool,
+        BrokenThreadPool,
+    ):
+        log.exception(
+            "A worker process was terminated unexpectedly. This is known to occur if "
+            "processing your file takes all available swap space and RAM. It may "
+            "help to try again with a smaller number of jobs, using the --jobs "
+            "argument."
+        )
+        return ExitCode.child_process_error
+    except Exception:  # pylint: disable=broad-except
+        log.exception("An exception occurred while executing the pipeline")
+        return ExitCode.other_error
+
+
+def setup_pipeline(
+    options: argparse.Namespace,
+    plugin_manager: OcrmypdfPluginManager,
+) -> Executor:
+    # Any changes to options will not take effect for options that are already
+    # bound to function parameters in the pipeline. (For example
+    # options.input_file, options.pdf_renderer are already bound.)
+    if not options.jobs:
+        options.jobs = available_cpu_count()
+
+    pikepdf_enable_mmap()
+    executor = setup_executor(plugin_manager)
+    return executor
 
 
 def preprocess(
@@ -104,6 +307,7 @@ def preprocess(
     deskew: bool,
     clean: bool,
 ) -> Path:
+    """Preprocess an image."""
     if remove_background:
         image = preprocess_remove_background(image, page_context)
     if deskew:
@@ -116,6 +320,7 @@ def preprocess(
 def make_intermediate_images(
     page_context: PageContext, orientation_correction: int
 ) -> tuple[Path, Path | None]:
+    """Create intermediate and preprocessed images for OCR."""
     options = page_context.options
 
     ocr_image = preprocess_out = None
@@ -172,19 +377,9 @@ def make_intermediate_images(
     return ocr_image, preprocess_out
 
 
-def exec_page_sync(page_context: PageContext) -> PageResult:
+def process_page(page_context: PageContext) -> tuple[Path, Path | None, int]:
+    """Process page to create OCR image, visible page image and orientation."""
     options = page_context.options
-    tls.pageno = page_context.pageno + 1
-
-    if not is_ocr_required(page_context):
-        return PageResult(
-            pageno=page_context.pageno,
-            pdf_page_from_image=None,
-            ocr=None,
-            text=None,
-            orientation_correction=0,
-        )
-
     orientation_correction = 0
     if options.rotate_pages:
         # Rasterize
@@ -212,255 +407,48 @@ def exec_page_sync(page_context: PageContext) -> PageResult:
         pdf_page_from_image_out = create_pdf_page_from_image(
             visible_image_out, page_context, orientation_correction
         )
-
+    return ocr_image_out, pdf_page_from_image_out, orientation_correction
-    if options.pdf_renderer.startswith('hocr'):
-        (hocr_out, text_out) = ocr_engine_hocr(ocr_image_out, page_context)
-        ocr_out = render_hocr_page(hocr_out, page_context)
-    elif options.pdf_renderer == 'sandwich':
-        (ocr_out, text_out) = ocr_engine_textonly_pdf(ocr_image_out, page_context)
-    else:
-        raise NotImplementedError(f"pdf_renderer {options.pdf_renderer}")
-
-    return PageResult(
-        pageno=page_context.pageno,
-        pdf_page_from_image=pdf_page_from_image_out,
-        ocr=ocr_out,
-        text=text_out,
-        orientation_correction=orientation_correction,
-    )
 
 
-def post_process(
+def postprocess(
     pdf_file: Path, context: PdfContext, executor: Executor
 ) -> tuple[Path, Sequence[str]]:
+    """Postprocess the PDF file."""
     pdf_out = pdf_file
     if context.options.output_type.startswith('pdfa'):
         ps_stub_out = generate_postscript_stub(context)
         pdf_out = convert_to_pdfa(pdf_out, ps_stub_out, context)
 
-    pdf_out = metadata_fixup(pdf_out, context)
+    optimizing = context.plugin_manager.hook.is_optimization_enabled(context=context)
+    save_settings = get_pdf_save_settings(context.options.output_type)
+    save_settings['linearize'] = not optimizing and should_linearize(pdf_out, context)
+
+    pdf_out = metadata_fixup(pdf_out, context, pdf_save_settings=save_settings)
     return optimize_pdf(pdf_out, context, executor)
 
 
-def worker_init(max_pixels: int) -> None:
+def report_output_pdf(options, start_input_file, optimize_messages) -> ExitCode:
-    # In Windows, child process will not inherit our change to this value in
+    if options.output_file == '-':
-    # the parent process, so ensure workers get it set. Not needed when running
+        log.info("Output sent to stdout")
-    # threaded, but harmless to set again.
+    elif hasattr(options.output_file, 'writable') and options.output_file.writable():
-    PIL.Image.MAX_IMAGE_PIXELS = max_pixels
+        log.info("Output written to stream")
-    pikepdf_enable_mmap()
+    elif samefile(options.output_file, Path(os.devnull)):
-
+        pass  # Say nothing when sending to dev null
-
+    else:
-def exec_concurrent(context: PdfContext, executor: Executor) -> Sequence[str]:
+        if options.output_type.startswith('pdfa'):
-    """Execute the pipeline concurrently."""
+            pdfa_info = file_claims_pdfa(options.output_file)
-    # Run exec_page_sync on every page context
+            if pdfa_info['pass']:
-    options = context.options
+                log.info("Output file is a %s (as expected)", pdfa_info['conformance'])
-    max_workers = min(len(context.pdfinfo), options.jobs)
+            else:
-    if max_workers > 1:
+                log.warning(
-        log.info("Start processing %d pages concurrently", max_workers)
+                    "Output file is okay but is not PDF/A (seems to be %s)",
-
+                    pdfa_info['conformance'],
-    sidecars: list[Path | None] = [None] * len(context.pdfinfo)
+                )
-    ocrgraft = OcrGrafter(context)
+                return ExitCode.pdfa_conversion_failed
-
+        if not check_pdf(options.output_file):
-    def update_page(result: PageResult, pbar):
+            log.warning('Output file: The generated PDF is INVALID')
-        try:
+            return ExitCode.invalid_output_pdf
-            tls.pageno = result.pageno + 1
+        report_output_file_size(
-            sidecars[result.pageno] = result.text
+            options, start_input_file, options.output_file, optimize_messages
-            pbar.update()
-            ocrgraft.graft_page(
-                pageno=result.pageno,
-                image=result.pdf_page_from_image,
-                textpdf=result.ocr,
-                autorotate_correction=result.orientation_correction,
-            )
-            pbar.update()
-        finally:
-            tls.pageno = None
-
-    executor(
-        use_threads=options.use_threads,
-        max_workers=max_workers,
-        tqdm_kwargs=dict(
-            total=(2 * len(context.pdfinfo)),
-            desc='OCR' if options.tesseract_timeout > 0 else 'Image processing',
-            unit='page',
-            unit_scale=0.5,
-            disable=not options.progress_bar,
-        ),
-        worker_initializer=partial(worker_init, PIL.Image.MAX_IMAGE_PIXELS),
-        task=exec_page_sync,
-        task_arguments=context.get_page_contexts(),
-        task_finished=update_page,
-    )
-
-    # Output sidecar text
-    if options.sidecar:
-        text = merge_sidecars(sidecars, context)
-        # Copy text file to destination
-        copy_final(text, options.sidecar, context)
-
-    # Merge layers to one single pdf
-    pdf = ocrgraft.finalize()
-
-    messages: Sequence[str] = []
-    if options.output_type != 'none':
-        # PDF/A and metadata
-        log.info("Postprocessing...")
-        pdf, messages = post_process(pdf, context, executor)
-
-        # Copy PDF file to destination
-        copy_final(pdf, options.output_file, context)
-    return messages
-
-
-def configure_debug_logging(
-    log_filename: Path, prefix: str = ''
-) -> logging.FileHandler:
-    """Create a debug log file at a specified location.
-
-    Arguments:
-        log_filename: Where to the put the log file.
-        prefix: The logging domain prefix that should be sent to the log.
-    """
-    log_file_handler = logging.FileHandler(log_filename, delay=True)
-    log_file_handler.setLevel(logging.DEBUG)
-    formatter = logging.Formatter(
-        '[%(asctime)s] - %(name)s - %(levelname)7s -%(pageno)s %(message)s'
-    )
-    log_file_handler.setFormatter(formatter)
-    log_file_handler.addFilter(PageNumberFilter())
-    logging.getLogger(prefix).addHandler(log_file_handler)
-    return log_file_handler
-
-
-def run_pipeline(
-    options: argparse.Namespace,
-    *,
-    plugin_manager: OcrmypdfPluginManager | None,
-    api: bool = False,
-) -> ExitCode:
-    # Any changes to options will not take effect for options that are already
-    # bound to function parameters in the pipeline. (For example
-    # options.input_file, options.pdf_renderer are already bound.)
-    if not options.jobs:
-        options.jobs = available_cpu_count()
-    if not plugin_manager:
-        plugin_manager = get_plugin_manager(options.plugins)
-
-    work_folder = Path(mkdtemp(prefix="ocrmypdf.io."))
-    debug_log_handler = None
-    if (
-        (options.keep_temporary_files or options.verbose >= 1)
-        and not os.environ.get('PYTEST_CURRENT_TEST', '')
-        and not api
-    ):
-        # Debug log for command line interface only with verbose output
-        # See https://github.com/pytest-dev/pytest/issues/5502 for why we skip this
-        # when pytest is running
-        debug_log_handler = configure_debug_logging(
-            Path(work_folder) / "debug.log"
-        )  # pragma: no cover
-
-    pikepdf_enable_mmap()
-
-    executor = setup_executor(plugin_manager)
-    try:
-        check_requested_output_file(options)
-        start_input_file, original_filename = create_input_file(options, work_folder)
-
-        # Triage image or pdf
-        origin_pdf = triage(
-            original_filename, start_input_file, work_folder / 'origin.pdf', options
         )
-
-        # Gather pdfinfo and create context
-        pdfinfo = get_pdfinfo(
-            origin_pdf,
-            executor=executor,
-            detailed_analysis=options.redo_ocr,
-            progbar=options.progress_bar,
-            max_workers=options.jobs if not options.use_threads else 1,  # To help debug
-            check_pages=options.pages,
-        )
-
-        context = PdfContext(options, work_folder, origin_pdf, pdfinfo, plugin_manager)
-
-        # Validate options are okay for this pdf
-        validate_pdfinfo_options(context)
-
-        # Execute the pipeline
-        optimize_messages = exec_concurrent(context, executor)
-
-        if options.output_file == '-':
-            log.info("Output sent to stdout")
-        elif (
-            hasattr(options.output_file, 'writable') and options.output_file.writable()
-        ):
-            log.info("Output written to stream")
-        elif samefile(options.output_file, Path(os.devnull)):
-            pass  # Say nothing when sending to dev null
-        else:
-            if options.output_type.startswith('pdfa'):
-                pdfa_info = file_claims_pdfa(options.output_file)
-                if pdfa_info['pass']:
-                    log.info(
-                        "Output file is a %s (as expected)", pdfa_info['conformance']
-                    )
-                else:
-                    log.warning(
-                        "Output file is okay but is not PDF/A (seems to be %s)",
-                        pdfa_info['conformance'],
-                    )
-                    return ExitCode.pdfa_conversion_failed
-            if not check_pdf(options.output_file):
-                log.warning('Output file: The generated PDF is INVALID')
-                return ExitCode.invalid_output_pdf
-            report_output_file_size(
-                options, start_input_file, options.output_file, optimize_messages
-            )
-
-    except KeyboardInterrupt if not api else NeverRaise:
-        if options.verbose >= 1:
-            log.exception("KeyboardInterrupt")
-        else:
-            log.error("KeyboardInterrupt")
-        return ExitCode.ctrl_c
-    except ExitCodeException if not api else NeverRaise as e:
-        e = cast(ExitCodeException, e)
-        if options.verbose >= 1:
-            log.exception("ExitCodeException")
-        elif str(e):
-            log.error("%s: %s", type(e).__name__, str(e))
-        else:
-            log.error(type(e).__name__)
-        return e.exit_code
-    except PIL.Image.DecompressionBombError if not api else NeverRaise:
-        log.exception(
-            "A decompression bomb error was encountered while executing the "
-            "pipeline. Use the argument --max-image-mpixels to raise the maximum "
-            "image pixel limit."
-        )
-        return ExitCode.other_error
-    except (
-        BrokenProcessPool if not api else NeverRaise,
-        BrokenThreadPool if not api else NeverRaise,
-    ):
-        log.exception(
-            "A worker process was terminated unexpectedly. This is known to occur if "
-            "processing your file takes all available swap space and RAM. It may "
-            "help to try again with a smaller number of jobs, using the --jobs "
-            "argument."
-        )
-        return ExitCode.child_process_error
-    except Exception if not api else NeverRaise:  # pylint: disable=broad-except
-        log.exception("An exception occurred while executing the pipeline")
-        return ExitCode.other_error
-    finally:
-        if debug_log_handler:
-            try:
-                debug_log_handler.close()
-                log.removeHandler(debug_log_handler)
-            except OSError as e:
-                print(e, file=sys.stderr)
-        cleanup_working_files(work_folder, options)
-
     return ExitCode.ok

src/ocrmypdf/_pipelines/hocr_to_ocr_pdf.py

@@ -0,0 +1,134 @@
+# SPDX-FileCopyrightText: 2019-2023 James R. Barlow
+# SPDX-FileCopyrightText: 2019 Martin Wind
+# SPDX-License-Identifier: MPL-2.0
+
+"""Implements the concurrent and page synchronous parts of the pipeline."""
+
+
+from __future__ import annotations
+
+import argparse
+import logging
+import logging.handlers
+from collections.abc import Sequence
+from functools import partial
+
+import PIL
+
+from ocrmypdf._concurrent import Executor
+from ocrmypdf._graft import OcrGrafter
+from ocrmypdf._jobcontext import PageContext, PdfContext
+from ocrmypdf._pipeline import (
+    copy_final,
+    get_pdfinfo,
+    render_hocr_page,
+)
+from ocrmypdf._pipelines._common import (
+    HOCRResult,
+    manage_work_folder,
+    postprocess,
+    report_output_pdf,
+    set_thread_pageno,
+    setup_pipeline,
+    worker_init,
+)
+from ocrmypdf._plugin_manager import OcrmypdfPluginManager
+from ocrmypdf._progressbar import ProgressBar
+from ocrmypdf.exceptions import ExitCode
+
+log = logging.getLogger(__name__)
+
+
+def _exec_hocrtransform_sync(page_context: PageContext) -> HOCRResult:
+    """Process each page."""
+    hocr_json = page_context.get_path('hocr.json')
+    if not hocr_json.exists():
+        # No hOCR file, so no OCR was performed on this page.
+        return HOCRResult(pageno=page_context.pageno)
+    hocr_result = HOCRResult.from_json(hocr_json.read_text())
+    hocr_result.textpdf = render_hocr_page(
+        page_context.get_path('ocr_hocr.hocr'), page_context
+    )
+    return hocr_result
+
+
+def exec_hocr_to_ocr_pdf(context: PdfContext, executor: Executor) -> Sequence[str]:
+    """Convert hOCR files to OCR PDF."""
+    # Run exec_page_sync on every page
+    options = context.options
+    max_workers = min(len(context.pdfinfo), options.jobs)
+    if max_workers > 1:
+        log.info("Continue processing %d pages concurrently", max_workers)
+
+    ocrgraft = OcrGrafter(context)
+
+    def graft_page(result: HOCRResult, pbar: ProgressBar):
+        """Graft text only PDF on to main PDF's page."""
+        try:
+            set_thread_pageno(result.pageno + 1)
+            pbar.update()
+            ocrgraft.graft_page(
+                pageno=result.pageno,
+                image=result.pdf_page_from_image,
+                textpdf=result.textpdf,
+                autorotate_correction=result.orientation_correction,
+            )
+            pbar.update()
+        finally:
+            set_thread_pageno(None)
+
+    executor(
+        use_threads=options.use_threads,
+        max_workers=max_workers,
+        progress_kwargs=dict(
+            total=(2 * len(context.pdfinfo)),
+            desc='Grafting hOCR to PDF',
+            unit='page',
+            unit_scale=0.5,
+            disable=not options.progress_bar,
+        ),
+        worker_initializer=partial(worker_init, PIL.Image.MAX_IMAGE_PIXELS),
+        task=_exec_hocrtransform_sync,
+        task_arguments=context.get_page_context_args(),
+        task_finished=graft_page,
+    )
+
+    pdf = ocrgraft.finalize()
+    messages: Sequence[str] = []
+    if options.output_type != 'none':
+        # PDF/A and metadata
+        log.info("Postprocessing...")
+        pdf, messages = postprocess(pdf, context, executor)
+
+        # Copy PDF file to destination (we don't know the input PDF file name)
+        copy_final(pdf, options.output_file, None)
+    return messages
+
+
+def run_hocr_to_ocr_pdf_pipeline(
+    options: argparse.Namespace,
+    *,
+    plugin_manager: OcrmypdfPluginManager,
+) -> ExitCode:
+    """Run pipeline to convert hOCR to final output PDF."""
+    with manage_work_folder(
+        work_folder=options.work_folder, retain=True, print_location=False
+    ) as work_folder:
+        executor = setup_pipeline(options, plugin_manager)
+        origin_pdf = work_folder / 'origin.pdf'
+
+        # Gather pdfinfo and create context
+        pdfinfo = get_pdfinfo(
+            origin_pdf,
+            executor=executor,
+            detailed_analysis=options.redo_ocr,
+            progbar=options.progress_bar,
+            max_workers=options.jobs,
+            use_threads=options.use_threads,
+            check_pages=options.pages,
+        )
+        context = PdfContext(options, work_folder, origin_pdf, pdfinfo, plugin_manager)
+        plugin_manager.hook.check_options(options=options)
+        optimize_messages = exec_hocr_to_ocr_pdf(context, executor)
+
+        return report_output_pdf(options, origin_pdf, optimize_messages)

src/ocrmypdf/_pipelines/ocr.py

@@ -0,0 +1,225 @@
+# SPDX-FileCopyrightText: 2019-2023 James R. Barlow
+# SPDX-FileCopyrightText: 2019 Martin Wind
+# SPDX-License-Identifier: MPL-2.0
+
+"""Implements the concurrent and page synchronous parts of the pipeline."""
+
+
+from __future__ import annotations
+
+import argparse
+import logging
+import logging.handlers
+from collections.abc import Sequence
+from functools import partial
+from pathlib import Path
+from tempfile import mkdtemp
+
+import PIL
+
+from ocrmypdf._concurrent import Executor
+from ocrmypdf._graft import OcrGrafter
+from ocrmypdf._jobcontext import PageContext, PdfContext
+from ocrmypdf._pipeline import (
+    copy_final,
+    get_pdfinfo,
+    is_ocr_required,
+    merge_sidecars,
+    ocr_engine_hocr,
+    ocr_engine_textonly_pdf,
+    render_hocr_page,
+    triage,
+    validate_pdfinfo_options,
+)
+from ocrmypdf._pipelines._common import (
+    PageResult,
+    cli_exception_handler,
+    manage_debug_log_handler,
+    manage_work_folder,
+    postprocess,
+    process_page,
+    report_output_pdf,
+    set_thread_pageno,
+    setup_pipeline,
+    worker_init,
+)
+from ocrmypdf._plugin_manager import OcrmypdfPluginManager
+from ocrmypdf._progressbar import ProgressBar
+from ocrmypdf._validation import (
+    check_requested_output_file,
+    create_input_file,
+)
+from ocrmypdf.exceptions import ExitCode
+
+log = logging.getLogger(__name__)
+
+
+def _image_to_ocr_text(
+    page_context: PageContext, ocr_image_out: Path
+) -> tuple[Path, Path]:
+    """Run OCR engine on image to create OCR PDF and text file."""
+    options = page_context.options
+    if options.pdf_renderer.startswith('hocr'):
+        hocr_out, text_out = ocr_engine_hocr(ocr_image_out, page_context)
+        ocr_out = render_hocr_page(hocr_out, page_context)
+    elif options.pdf_renderer == 'sandwich':
+        ocr_out, text_out = ocr_engine_textonly_pdf(ocr_image_out, page_context)
+    else:
+        raise NotImplementedError(f"pdf_renderer {options.pdf_renderer}")
+    return ocr_out, text_out
+
+
+def _exec_page_sync(page_context: PageContext) -> PageResult:
+    """Execute a pipeline for a single page synchronously."""
+    set_thread_pageno(page_context.pageno + 1)
+
+    if not is_ocr_required(page_context):
+        return PageResult(pageno=page_context.pageno)
+
+    ocr_image_out, pdf_page_from_image_out, orientation_correction = process_page(
+        page_context
+    )
+    ocr_out, text_out = _image_to_ocr_text(page_context, ocr_image_out)
+
+    return PageResult(
+        pageno=page_context.pageno,
+        pdf_page_from_image=pdf_page_from_image_out,
+        ocr=ocr_out,
+        text=text_out,
+        orientation_correction=orientation_correction,
+    )
+
+
+def exec_concurrent(context: PdfContext, executor: Executor) -> Sequence[str]:
+    """Execute the OCR pipeline concurrently."""
+    options = context.options
+    max_workers = min(len(context.pdfinfo), options.jobs)
+    if max_workers > 1:
+        log.info("Start processing %d pages concurrently", max_workers)
+
+    sidecars: list[Path | None] = [None] * len(context.pdfinfo)
+    ocrgraft = OcrGrafter(context)
+
+    def update_page(result: PageResult, pbar: ProgressBar):
+        """After OCR is complete for a page, update the PDF."""
+        try:
+            set_thread_pageno(result.pageno + 1)
+            sidecars[result.pageno] = result.text
+            pbar.update()
+            ocrgraft.graft_page(
+                pageno=result.pageno,
+                image=result.pdf_page_from_image,
+                textpdf=result.ocr,
+                autorotate_correction=result.orientation_correction,
+            )
+            pbar.update()
+        finally:
+            set_thread_pageno(None)
+
+    executor(
+        use_threads=options.use_threads,
+        max_workers=max_workers,
+        progress_kwargs=dict(
+            total=(2 * len(context.pdfinfo)),
+            desc='OCR' if options.tesseract_timeout > 0 else 'Image processing',
+            unit='page',
+            unit_scale=0.5,
+            disable=not options.progress_bar,
+        ),
+        worker_initializer=partial(worker_init, PIL.Image.MAX_IMAGE_PIXELS),
+        task=_exec_page_sync,
+        task_arguments=context.get_page_context_args(),
+        task_finished=update_page,
+    )
+
+    # Output sidecar text
+    if options.sidecar:
+        text = merge_sidecars(sidecars, context)
+        # Copy text file to destination
+        copy_final(text, options.sidecar, options.input_file)
+
+    # Merge layers to one single pdf
+    pdf = ocrgraft.finalize()
+
+    messages: Sequence[str] = []
+    if options.output_type != 'none':
+        # PDF/A and metadata
+        log.info("Postprocessing...")
+        pdf, messages = postprocess(pdf, context, executor)
+
+        # Copy PDF file to destination
+        copy_final(pdf, options.output_file, options.input_file)
+    return messages
+
+
+def _run_pipeline(
+    options: argparse.Namespace,
+    plugin_manager: OcrmypdfPluginManager,
+) -> ExitCode:
+    with manage_work_folder(
+        work_folder=Path(mkdtemp(prefix="ocrmypdf.io.")),
+        retain=options.keep_temporary_files,
+        print_location=options.keep_temporary_files,
+    ) as work_folder, manage_debug_log_handler(
+        options=options, work_folder=work_folder
+    ):
+        executor = setup_pipeline(options, plugin_manager)
+        check_requested_output_file(options)
+        start_input_file, original_filename = create_input_file(options, work_folder)
+
+        # Triage image or pdf
+        origin_pdf = triage(
+            original_filename, start_input_file, work_folder / 'origin.pdf', options
+        )
+
+        # Gather pdfinfo and create context
+        pdfinfo = get_pdfinfo(
+            origin_pdf,
+            executor=executor,
+            detailed_analysis=options.redo_ocr,
+            progbar=options.progress_bar,
+            max_workers=options.jobs,
+            use_threads=options.use_threads,
+            check_pages=options.pages,
+        )
+
+        context = PdfContext(options, work_folder, origin_pdf, pdfinfo, plugin_manager)
+
+        # Validate options are okay for this pdf
+        validate_pdfinfo_options(context)
+
+        # Execute the pipeline
+        optimize_messages = exec_concurrent(context, executor)
+
+        exitcode = report_output_pdf(options, start_input_file, optimize_messages)
+        return exitcode
+
+
+def run_pipeline_cli(
+    options: argparse.Namespace,
+    *,
+    plugin_manager: OcrmypdfPluginManager,
+) -> ExitCode:
+    """Run the OCR pipeline with command line exception handling.
+
+    Args:
+        options: The parsed command line options.
+        plugin_manager: The plugin manager to use. If not provided, one will be
+            created.
+    """
+    return cli_exception_handler(_run_pipeline, options, plugin_manager)
+
+
+def run_pipeline(
+    options: argparse.Namespace,
+    *,
+    plugin_manager: OcrmypdfPluginManager,
+) -> ExitCode:
+    """Run the OCR pipeline without command line exception handling.
+
+    Args:
+        options: The parsed command line options.
+        plugin_manager: The plugin manager to use. If not provided, one will be
+            created.
+    """
+    return _run_pipeline(options, plugin_manager)

src/ocrmypdf/_pipelines/pdf_to_hocr.py

@@ -0,0 +1,116 @@
+# SPDX-FileCopyrightText: 2019-2023 James R. Barlow
+# SPDX-FileCopyrightText: 2019 Martin Wind
+# SPDX-License-Identifier: MPL-2.0
+
+"""Implements the concurrent and page synchronous parts of the pipeline."""
+
+
+from __future__ import annotations
+
+import argparse
+import logging
+import logging.handlers
+import shutil
+from functools import partial
+
+import PIL
+
+from ocrmypdf._concurrent import Executor
+from ocrmypdf._jobcontext import PageContext, PdfContext
+from ocrmypdf._pipeline import (
+    get_pdfinfo,
+    is_ocr_required,
+    ocr_engine_hocr,
+    validate_pdfinfo_options,
+)
+from ocrmypdf._pipelines._common import (
+    HOCRResult,
+    manage_work_folder,
+    process_page,
+    set_thread_pageno,
+    setup_pipeline,
+    worker_init,
+)
+from ocrmypdf._plugin_manager import OcrmypdfPluginManager
+from ocrmypdf._validation import (
+    set_lossless_reconstruction,
+)
+
+log = logging.getLogger(__name__)
+
+
+def _exec_page_hocr_sync(page_context: PageContext) -> HOCRResult:
+    """Execute a pipeline for a single page hOCR."""
+    set_thread_pageno(page_context.pageno + 1)
+
+    if not is_ocr_required(page_context):
+        return HOCRResult(pageno=page_context.pageno)
+
+    ocr_image_out, pdf_page_from_image_out, orientation_correction = process_page(
+        page_context
+    )
+    hocr_out, _ = ocr_engine_hocr(ocr_image_out, page_context)
+
+    result = HOCRResult(
+        pageno=page_context.pageno,
+        pdf_page_from_image=pdf_page_from_image_out,
+        hocr=hocr_out,
+        orientation_correction=orientation_correction,
+    )
+    page_context.get_path('hocr.json').write_text(result.to_json())
+    return result
+
+
+def exec_pdf_to_hocr(context: PdfContext, executor: Executor) -> None:
+    """Execute the OCR pipeline concurrently and output hOCR."""
+    # Run exec_page_sync on every page
+    options = context.options
+    max_workers = min(len(context.pdfinfo), options.jobs)
+    if max_workers > 1:
+        log.info("Start processing %d pages concurrently", max_workers)
+
+    executor(
+        use_threads=options.use_threads,
+        max_workers=max_workers,
+        progress_kwargs=dict(
+            total=(2 * len(context.pdfinfo)),
+            desc='hOCR',
+            unit='page',
+            unit_scale=0.5,
+            disable=not options.progress_bar,
+        ),
+        worker_initializer=partial(worker_init, PIL.Image.MAX_IMAGE_PIXELS),
+        task=_exec_page_hocr_sync,
+        task_arguments=context.get_page_context_args(),
+    )
+
+
+def run_hocr_pipeline(
+    options: argparse.Namespace,
+    *,
+    plugin_manager: OcrmypdfPluginManager,
+) -> None:
+    """Run pipeline to output hOCR."""
+    with manage_work_folder(
+        work_folder=options.output_folder, retain=True, print_location=False
+    ) as work_folder:
+        executor = setup_pipeline(options, plugin_manager)
+        shutil.copy2(options.input_file, work_folder / 'origin.pdf')
+
+        # Gather pdfinfo and create context
+        pdfinfo = get_pdfinfo(
+            options.input_file,
+            executor=executor,
+            detailed_analysis=options.redo_ocr,
+            progbar=options.progress_bar,
+            max_workers=options.jobs,
+            use_threads=options.use_threads,
+            check_pages=options.pages,
+        )
+        context = PdfContext(
+            options, work_folder, options.input_file, pdfinfo, plugin_manager
+        )
+        # Validate options are okay for this pdf
+        set_lossless_reconstruction(options)
+        validate_pdfinfo_options(context)
+        exec_pdf_to_hocr(context, executor)

src/ocrmypdf/_plugin_manager.py

@@ -10,8 +10,8 @@ import importlib
 import importlib.util
 import pkgutil
 import sys
+from collections.abc import Sequence
 from pathlib import Path
-from typing import Sequence
 
 import pluggy
 
@@ -33,7 +33,7 @@ class OcrmypdfPluginManager(pluggy.PluginManager):
     def __init__(
         self,
         *args,
-        plugins: list[str | Path],
+        plugins: Sequence[str | Path],
         builtins: bool = True,
         **kwargs,
     ):
@@ -101,11 +101,11 @@ class OcrmypdfPluginManager(pluggy.PluginManager):
 
 
 def get_plugin_manager(
-    plugins: list[str | Path], builtins=True
+    plugins: Sequence[str | Path] | None = None, builtins=True
 ) -> OcrmypdfPluginManager:
     return OcrmypdfPluginManager(
         project_name='ocrmypdf',
-        plugins=plugins,
+        plugins=plugins if plugins is not None else [],
         builtins=builtins,
     )
 

src/ocrmypdf/_progressbar.py

@@ -0,0 +1,142 @@
+# SPDX-FileCopyrightText: 2023 James R. Barlow
+# SPDX-License-Identifier: MPL-2.0
+
+"""Defines progress bar API."""
+
+from __future__ import annotations
+
+from typing import Protocol
+
+from rich.console import Console
+from rich.progress import (
+    BarColumn,
+    MofNCompleteColumn,
+    Progress,
+    TaskProgressColumn,
+    TextColumn,
+    TimeRemainingColumn,
+)
+from rich.table import Column
+
+
+class ProgressBar(Protocol):
+    """The protocol that OCRmyPDF expects progress bar classes to be compatible with.
+
+    In practice this could be used for any time of monitoring, not just a progress bar.
+
+    Calling the class should return a new progress bar object, which is activated
+    with ``__enter__`` and terminated with ``__exit__``. An update method is called
+    whenever the progress bar is updated. Progress bar objects will not be reused;
+    a new one will be created for each group of tasks.
+
+    The progress bar is held in the main process/thread and not updated by child
+    process/threads. When a child notifies the parent of completed work, the
+    parent updates the progress bar.
+
+    Progress bars should never write to ``sys.stdout``, or they will corrupt the
+    output if OCRmyPDF writes a PDF to standard output.
+
+    The type of events that OCRmyPDF reports to a progress bar may change in
+    minor releases.
+    """
+
+    def __init__(
+        self,
+        *,
+        total: int | float | None,
+        desc: str | None,
+        unit: str | None,
+        disable: bool = False,
+        **kwargs,
+    ):
+        """Initialize a progress bar.
+
+        *total* indicates the total number of work units. If None, the total
+        number of work units is unknown. If *disable* is True, the progress bar
+        should be disabled. *unit* is a description of the work unit.
+        *desc* is a description of the overall task to be performed.
+
+        Unrecognized keyword arguments must be ignored, as the list of keyword
+        arguments may grow with time.
+        """
+
+    def __enter__(self):
+        """Enter a progress bar context."""
+
+    def __exit__(self, *args):
+        """Exit a progress bar context."""
+
+    def update(self, n=1):
+        """Update the progress bar by an increment.
+
+        For use within a progress bar context.
+        """
+
+
+class NullProgressBar:
+    """Progress bar API that takes no actions."""
+
+    def __init__(self, **kwargs):
+        pass
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        return False
+
+    def update(self, _arg=None):
+        return
+
+
+class RichProgressBar:
+    """Display progress bar using rich."""
+
+    def __init__(
+        self,
+        *,
+        console: Console,
+        desc: str,
+        total: float | None = None,
+        unit: str | None = None,
+        unit_scale: float | None = 1.0,
+        disable: bool = False,
+        **kwargs,
+    ):
+        self.progress = Progress(
+            TextColumn(
+                "[progress.description]{task.description}",
+                table_column=Column(min_width=20),
+            ),
+            BarColumn(),
+            TaskProgressColumn(),
+            MofNCompleteColumn(),
+            TimeRemainingColumn(),
+            console=console,
+            auto_refresh=True,
+            redirect_stderr=True,
+            redirect_stdout=False,
+            disable=disable,
+            **kwargs,
+        )
+        self.unit_scale = unit_scale
+        self.progress_bar = self.progress.add_task(
+            desc,
+            total=total * self.unit_scale
+            if total is not None and self.unit_scale is not None
+            else None,
+            unit=unit,
+        )
+
+    def __enter__(self):
+        self.progress.start()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.progress.refresh()
+        self.progress.stop()
+        return False
+
+    def update(self, value=None):
+        advance = self.unit_scale if value is None else value
+        self.progress.update(self.progress_bar, advance=advance)

src/ocrmypdf/_validation.py

@@ -12,9 +12,9 @@ import os
 import sys
 import unicodedata
 from argparse import Namespace
+from collections.abc import Sequence
 from pathlib import Path
 from shutil import copyfileobj
-from typing import Sequence
 
 import pikepdf
 import PIL
@@ -44,7 +44,7 @@ log = logging.getLogger(__name__)
 
 def check_platform() -> None:
     if sys.maxsize <= 2**32:  # pragma: no cover
-        log.error(
+        log.warning(
             "You are running OCRmyPDF in a 32-bit (x86) Python interpreter. "
             "This is not supported. 32-bit does not have enough address space "
             "to process large files. "
@@ -99,6 +99,8 @@ def check_options_output(options: Namespace) -> None:
             f"`-` to suppress this message."
         )
 
+
+def set_lossless_reconstruction(options: Namespace) -> None:
     lossless_reconstruction = False
     if not any(
         (
@@ -228,6 +230,7 @@ def _check_plugin_invariant_options(options: Namespace) -> None:
     check_platform()
     check_options_metadata(options)
     check_options_output(options)
+    set_lossless_reconstruction(options)
     check_options_sidecar(options)
     check_options_preprocessing(options)
     check_options_ocr_behavior(options)
@@ -245,6 +248,18 @@ def check_options(options: Namespace, plugin_manager: PluginManager) -> None:
     _check_plugin_options(options, plugin_manager)
 
 
+def _in_docker():
+    return Path('/.dockerenv').exists()
+
+
+def _in_snap():
+    try:
+        cgroup_text = Path('/proc/self/cgroup').read_text()
+        return 'snap.ocrmypdf' in cgroup_text
+    except FileNotFoundError:
+        return False
+
+
 def create_input_file(options: Namespace, work_folder: Path) -> tuple[Path, str]:
     if options.input_file == '-':
         # stdin
@@ -268,9 +283,9 @@ def create_input_file(options: Namespace, work_folder: Path) -> tuple[Path, str]
             return target, os.fspath(options.input_file)
         except FileNotFoundError as e:
             msg = f"File not found - {options.input_file}"
-            if Path('/.dockerenv').exists():  # pragma: no cover
+            if _in_docker():  # pragma: no cover
                 msg += (
-                    "\nDocker cannot your working directory unless you "
+                    "\nDocker cannot access your working directory unless you "
                     "explicitly share it with the Docker container and set up"
                     "permissions correctly.\n"
                     "You may find it easier to use stdin/stdout:"
@@ -278,6 +293,15 @@ def create_input_file(options: Namespace, work_folder: Path) -> tuple[Path, str]
                     "\tdocker run -i --rm jbarlow83/ocrmypdf - - <input.pdf >output.pdf"
                     "\n"
                 )
+            elif _in_snap():  # pragma: no cover
+                msg += (
+                    "\nSnap applications cannot access files outside of "
+                    "your home directory unless you explicitly allow it. "
+                    "You may find it easier to use stdin/stdout:"
+                    "\n"
+                    "\tsnap run ocrmypdf - - <input.pdf >output.pdf"
+                    "\n"
+                )
             raise InputFileError(msg) from e
 
 

src/ocrmypdf/api.py

@@ -10,17 +10,20 @@ import os
 import sys
 import threading
 from argparse import Namespace
+from collections.abc import Iterable, Sequence
 from enum import IntEnum
 from io import IOBase
 from pathlib import Path
-from typing import AnyStr, BinaryIO, Iterable, Union
+from typing import AnyStr, BinaryIO, Union
 from warnings import warn
 
 import pluggy
 
 from ocrmypdf._logging import PageNumberFilter
+from ocrmypdf._pipelines.hocr_to_ocr_pdf import run_hocr_to_ocr_pdf_pipeline
+from ocrmypdf._pipelines.ocr import run_pipeline, run_pipeline_cli
+from ocrmypdf._pipelines.pdf_to_hocr import run_hocr_pipeline
 from ocrmypdf._plugin_manager import get_plugin_manager
-from ocrmypdf._sync import run_pipeline
 from ocrmypdf._validation import check_options
 from ocrmypdf.cli import ArgumentParser, get_parser
 from ocrmypdf.helpers import is_iterable_notstr
@@ -28,6 +31,9 @@ from ocrmypdf.helpers import is_iterable_notstr
 StrPath = Union[Path, AnyStr]
 PathOrIO = Union[BinaryIO, StrPath]
 
+# Installing plugins affects the global state of the Python interpreter,
+# so we need to use a lock to prevent multiple threads from installing
+# plugins at the same time.
 _api_lock = threading.Lock()
 
 
@@ -132,34 +138,19 @@ def configure_logging(
     return log
 
 
-def create_options(
+def _kwargs_to_cmdline(
-    *, input_file: PathOrIO, output_file: PathOrIO, parser: ArgumentParser, **kwargs
+    *, defer_kwargs: set[str], **kwargs
-) -> Namespace:
+) -> tuple[list[str], dict[str, AnyStr]]:
-    """Construct an options object from the input/output files and keyword arguments.
+    """Convert kwargs to command line arguments."""
-
-    Args:
-        input_file: Input file path or file object.
-        output_file: Output file path or file object.
-        parser: ArgumentParser object.
-        **kwargs: Keyword arguments.
-
-    Returns:
-        argparse.Namespace: A Namespace object containing the parsed arguments.
-
-    Raises:
-        TypeError: If the type of a keyword argument is not supported.
-    """
     cmdline = []
-    deferred = []
+    deferred = {}
-
     for arg, val in kwargs.items():
         if val is None:
             continue
 
-        # These arguments with special handling for which we bypass
+        # Skip arguments that are handled elsewhere
-        # argparse
+        if arg in defer_kwargs:
-        if arg in {'progress_bar', 'plugins'}:
+            deferred[arg] = val
-            deferred.append((arg, val))
             continue
 
         cmd_style_arg = arg.replace('_', '-')
@@ -186,7 +177,30 @@ def create_options(
             cmdline.append(str(val))
         else:
             raise TypeError(f"{arg}: {val} ({type(val)})")
+    return cmdline, deferred
 
+
+def create_options(
+    *, input_file: PathOrIO, output_file: PathOrIO, parser: ArgumentParser, **kwargs
+) -> Namespace:
+    """Construct an options object from the input/output files and keyword arguments.
+
+    Args:
+        input_file: Input file path or file object.
+        output_file: Output file path or file object.
+        parser: ArgumentParser object.
+        **kwargs: Keyword arguments.
+
+    Returns:
+        argparse.Namespace: A Namespace object containing the parsed arguments.
+
+    Raises:
+        TypeError: If the type of a keyword argument is not supported.
+    """
+    cmdline, deferred = _kwargs_to_cmdline(
+        defer_kwargs={'progress_bar', 'plugins', 'parser', 'input_file', 'output_file'},
+        **kwargs,
+    )
     if isinstance(input_file, (BinaryIO, IOBase)):
         cmdline.append('stream://input_file')
     else:
@@ -198,7 +212,7 @@ def create_options(
 
     parser.enable_api_mode()
     options = parser.parse_args(cmdline)
-    for keyword, val in deferred:
+    for keyword, val in deferred.items():
         setattr(options, keyword, val)
 
     if options.input_file == 'stream://input_file':
@@ -209,7 +223,7 @@ def create_options(
     return options
 
 
-def ocr(  # noqa: ruff: disable=D417
+def ocr(  # noqa: D417
     input_file: PathOrIO,
     output_file: PathOrIO,
     *,
@@ -240,6 +254,7 @@ def ocr(  # noqa: ruff: disable=D417
     png_quality: int | None = None,
     jbig2_lossy: bool | None = None,
     jbig2_page_group_size: int | None = None,
+    jbig2_threshold: float | None = None,
     pages: str | None = None,
     max_image_mpixels: float | None = None,
     tesseract_config: Iterable[str] | None = None,
@@ -249,12 +264,16 @@ def ocr(  # noqa: ruff: disable=D417
     pdf_renderer: str | None = None,
     tesseract_timeout: float | None = None,
     tesseract_non_ocr_timeout: float | None = None,
+    tesseract_downsample_above: int | None = None,
+    tesseract_downsample_large_images: bool | None = None,
     rotate_pages_threshold: float | None = None,
     pdfa_image_compression: str | None = None,
+    color_conversion_strategy: str | None = None,
     user_words: os.PathLike | None = None,
     user_patterns: os.PathLike | None = None,
     fast_web_view: float | None = None,
     continue_on_soft_render_error: bool | None = None,
+    invalidate_digital_signatures: bool | None = None,
     plugins: Iterable[StrPath] | None = None,
     plugin_manager=None,
     keep_temporary_files: bool | None = None,
@@ -330,17 +349,15 @@ def ocr(  # noqa: ruff: disable=D417
         plugins = list(plugins)
 
     # No new variable names should be assigned until these two steps are run
-    create_options_kwargs = {k: v for k, v in locals().items() if k != 'kwargs'}
+    create_options_kwargs = {
+        k: v
+        for k, v in locals().items()
+        if k not in {'input_file', 'output_file', 'kwargs'}
+    }
     create_options_kwargs.update(kwargs)
 
     parser = get_parser()
-    create_options_kwargs['parser'] = parser
-
     with _api_lock:
-        # We can't allow multiple ocrmypdf.ocr() threads to run in parallel, because
-        # they might install different plugins, and generally speaking we have areas
-        # of code that use global state.
-
         if not plugin_manager:
             plugin_manager = get_plugin_manager(plugins)
         plugin_manager.hook.add_options(parser=parser)  # pylint: disable=no-member
@@ -348,9 +365,177 @@ def ocr(  # noqa: ruff: disable=D417
         if 'verbose' in kwargs:
             warn("ocrmypdf.ocr(verbose=) is ignored. Use ocrmypdf.configure_logging().")
 
-        options = create_options(**create_options_kwargs)
+        options = create_options(
+            input_file=input_file,
+            output_file=output_file,
+            parser=parser,
+            **create_options_kwargs,
+        )
         check_options(options, plugin_manager)
-        return run_pipeline(options=options, plugin_manager=plugin_manager, api=True)
+        return run_pipeline(options=options, plugin_manager=plugin_manager)
+
+
+def _pdf_to_hocr(  # noqa: D417
+    input_pdf: Path,
+    output_folder: Path,
+    *,
+    language: Iterable[str] | None = None,
+    image_dpi: int | None = None,
+    jobs: int | None = None,
+    use_threads: bool | None = None,
+    title: str | None = None,
+    author: str | None = None,
+    subject: str | None = None,
+    keywords: str | None = None,
+    rotate_pages: bool | None = None,
+    remove_background: bool | None = None,
+    deskew: bool | None = None,
+    clean: bool | None = None,
+    clean_final: bool | None = None,
+    unpaper_args: str | None = None,
+    oversample: int | None = None,
+    remove_vectors: bool | None = None,
+    force_ocr: bool | None = None,
+    skip_text: bool | None = None,
+    redo_ocr: bool | None = None,
+    skip_big: float | None = None,
+    pages: str | None = None,
+    max_image_mpixels: float | None = None,
+    tesseract_config: Iterable[str] | None = None,
+    tesseract_pagesegmode: int | None = None,
+    tesseract_oem: int | None = None,
+    tesseract_thresholding: int | None = None,
+    tesseract_timeout: float | None = None,
+    tesseract_non_ocr_timeout: float | None = None,
+    tesseract_downsample_above: int | None = None,
+    tesseract_downsample_large_images: bool | None = None,
+    rotate_pages_threshold: float | None = None,
+    user_words: os.PathLike | None = None,
+    user_patterns: os.PathLike | None = None,
+    continue_on_soft_render_error: bool | None = None,
+    invalidate_digital_signatures: bool | None = None,
+    plugin_manager=None,
+    plugins: Sequence[StrPath] | None = None,
+    keep_temporary_files: bool | None = None,
+    **kwargs,
+):
+    """Partially run OCRmyPDF and produces an output folder containing hOCR files.
+
+    Given a PDF file, this function will run OCRmyPDF up to the point where
+    the PDF is rasterized to images, OCRed, and the hOCR files are produced,
+    all of which are saved to the output folder. This is useful for applications
+    that want to provide an interface for users to edit the text before
+    rendering the final PDF.
+
+    Use :func:`hocr_to_ocr_pdf` to produce the final PDF.
+
+    For arguments not explicitly documented here, see documentation for the
+    equivalent command line parameter.
+
+    This API is **experimental** and subject to change.
+
+    Args:
+        input_pdf: Input PDF file path.
+        output_folder: Output folder path.
+        **kwargs: Keyword arguments.
+    """
+    # No new variable names should be assigned until these two steps are run
+    create_options_kwargs = {
+        k: v
+        for k, v in locals().items()
+        if k not in {'input_pdf', 'output_folder', 'kwargs'}
+    }
+    create_options_kwargs.update(kwargs)
+
+    parser = get_parser()
+
+    with _api_lock:
+        if not plugin_manager:
+            plugin_manager = get_plugin_manager(plugins)
+        plugin_manager.hook.add_options(parser=parser)  # pylint: disable=no-member
+
+        cmdline, deferred = _kwargs_to_cmdline(
+            defer_kwargs={'input_pdf', 'output_folder', 'plugins'},
+            **create_options_kwargs,
+        )
+        cmdline.append(str(input_pdf))
+        cmdline.append(str(output_folder))
+        parser.enable_api_mode()
+        options = parser.parse_args(cmdline)
+        for keyword, val in deferred.items():
+            setattr(options, keyword, val)
+        delattr(options, 'output_file')
+        setattr(options, 'output_folder', output_folder)
+
+        return run_hocr_pipeline(options=options, plugin_manager=plugin_manager)
+
+
+def _hocr_to_ocr_pdf(  # noqa: D417
+    work_folder: Path,
+    output_file: Path,
+    *,
+    jobs: int | None = None,
+    use_threads: bool | None = None,
+    optimize: int | None = None,
+    jpg_quality: int | None = None,
+    png_quality: int | None = None,
+    jbig2_lossy: bool | None = None,
+    jbig2_page_group_size: int | None = None,
+    jbig2_threshold: float | None = None,
+    pdfa_image_compression: str | None = None,
+    color_conversion_strategy: str | None = None,
+    fast_web_view: float | None = None,
+    plugin_manager=None,
+    plugins: Sequence[StrPath] | None = None,
+    **kwargs,
+):
+    """Run OCRmyPDF on a work folder and produce an output PDF.
+
+    After running :func:`pdf_to_hocr`, this function will run OCRmyPDF on the work
+    folder to produce an output PDF. This function consolidates any changes made
+    to the hOCR files in the work folder and produces a final PDF.
+
+    For arguments not explicitly documented here, see documentation for the
+    equivalent command line parameter.
+
+    This API is **experimental** and subject to change.
+
+    Args:
+        work_folder: Work folder path, as generated by :func:`pdf_to_hocr`.
+        output_file: Output PDF file path.
+        **kwargs: Keyword arguments.
+    """
+    # No new variable names should be assigned until these two steps are run
+    create_options_kwargs = {
+        k: v
+        for k, v in locals().items()
+        if k not in {'work_folder', 'output_pdf', 'kwargs'}
+    }
+    create_options_kwargs.update(kwargs)
+
+    parser = get_parser()
+
+    with _api_lock:
+        if not plugin_manager:
+            plugin_manager = get_plugin_manager(plugins)
+        plugin_manager.hook.add_options(parser=parser)  # pylint: disable=no-member
+
+        cmdline, deferred = _kwargs_to_cmdline(
+            defer_kwargs={'work_folder', 'output_file', 'plugins'},
+            **create_options_kwargs,
+        )
+        cmdline.append(str(work_folder))
+        cmdline.append(str(output_file))
+        parser.enable_api_mode()
+        options = parser.parse_args(cmdline)
+        for keyword, val in deferred.items():
+            setattr(options, keyword, val)
+        delattr(options, 'input_file')
+        setattr(options, 'work_folder', work_folder)
+
+        return run_hocr_to_ocr_pdf_pipeline(
+            options=options, plugin_manager=plugin_manager
+        )
 
 
 __all__ = [
@@ -363,4 +548,5 @@ __all__ = [
     'get_plugin_manager',
     'ocr',
     'run_pipeline',
+    'run_pipeline_cli',
 ]

src/ocrmypdf/builtin_plugins/concurrency.py

@@ -12,22 +12,26 @@ import queue
 import signal
 import sys
 import threading
+from collections.abc import Iterable
 from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed
 from contextlib import suppress
-from typing import Callable, Iterable, Type, Union
+from typing import Callable, Union
 
 from rich.console import Console as RichConsole
 
 from ocrmypdf import Executor, hookimpl
-from ocrmypdf._logging import RichLoggingHandler, RichTqdmProgressAdapter
+from ocrmypdf._logging import RichLoggingHandler
+from ocrmypdf._progressbar import RichProgressBar
 from ocrmypdf.exceptions import InputFileError
 from ocrmypdf.helpers import remove_all_log_handlers
 
-FuturesExecutorClass = Union[Type[ThreadPoolExecutor], Type[ProcessPoolExecutor]]
+FuturesExecutorClass = Union[type[ThreadPoolExecutor], type[ProcessPoolExecutor]]
 Queue = Union[multiprocessing.Queue, queue.Queue]
 UserInit = Callable[[], None]
 WorkerInit = Callable[[Queue, UserInit, int], None]
 
+RichTqdmProgressAdapter = RichProgressBar  # Deprecated shim; remove in OCRmyPDF 16
+
 
 def log_listener(q: Queue):
     """Listen to the worker processes and forward the messages to logging.
@@ -100,7 +104,7 @@ class StandardExecutor(Executor):
         *,
         use_threads: bool,
         max_workers: int,
-        tqdm_kwargs: dict,
+        progress_kwargs: dict,
         worker_initializer: Callable,
         task: Callable,
         task_arguments: Iterable,
@@ -118,15 +122,20 @@ class StandardExecutor(Executor):
         # Regardless of whether we use_threads for worker processes, the log_listener
         # must be a thread. Make sure we create the listener after the worker pool,
         # so that it does not get forked into the workers.
+        # If use_threads is False, we are currently guilty of creating a thread before
+        # forking on Linux, which is not recommended. However, we take a big
+        # performance hit in pdfinfo if we can't fork. Long term solution is to
+        # replace most of this with an asyncio implementation, and probably to
+        # migrate some of pdfinfo into C++ or Rust.
         listener = threading.Thread(target=log_listener, args=(log_queue,))
         listener.start()
 
-        with self.pbar_class(**tqdm_kwargs) as pbar, executor_class(
+        with self.pbar_class(**progress_kwargs) as pbar, executor_class(
             max_workers=max_workers,
             initializer=initializer,
             initargs=(log_queue, worker_initializer, logging.getLogger("").level),
         ) as executor:
-            futures = [executor.submit(task, args) for args in task_arguments]
+            futures = [executor.submit(task, *args) for args in task_arguments]
             try:
                 for future in as_completed(futures):
                     result = future.result()
@@ -166,10 +175,10 @@ RICH_CONSOLE = RichConsole(stderr=True)
 def get_progressbar_class():
     """Return the default progress bar class."""
 
-    def partial_RichTqdmProgressAdapter(*args, **kwargs):
+    def partial_RichProgressBar(*args, **kwargs):
-        return RichTqdmProgressAdapter(*args, **kwargs, console=RICH_CONSOLE)
+        return RichProgressBar(*args, **kwargs, console=RICH_CONSOLE)
 
-    return partial_RichTqdmProgressAdapter
+    return partial_RichProgressBar
 
 
 @hookimpl

src/ocrmypdf/builtin_plugins/optimize.py

@@ -6,8 +6,8 @@ from __future__ import annotations
 
 import argparse
 import logging
+from collections.abc import Sequence
 from pathlib import Path
-from typing import Sequence
 
 from ocrmypdf import Executor, PdfContext, hookimpl
 from ocrmypdf._exec import jbig2enc, pngquant

src/ocrmypdf/cli.py

@@ -6,7 +6,8 @@
 from __future__ import annotations
 
 import argparse
-from typing import Any, Callable, Mapping, TypeVar
+from collections.abc import Mapping
+from typing import Any, Callable, TypeVar
 
 from ocrmypdf._version import PROGRAM_NAME as _PROGRAM_NAME
 from ocrmypdf._version import __version__ as _VERSION
@@ -192,8 +193,8 @@ Online documentation is located at:
         help="Choose output type. 'pdfa' creates a PDF/A-2b compliant file for "
         "long term archiving (default, recommended) but may not suitable "
         "for users who want their file altered as little as possible. 'pdfa' "
-        "also has problems with full Unicode text. 'pdf' attempts to "
+        "also has problems with full Unicode text. 'pdf' minimizes changes "
-        "preserve file contents as much as possible. 'pdf-a1' creates a "
+        "to the input file. 'pdf-a1' creates a "
         "PDF/A1-b file. 'pdf-a2' is equivalent to 'pdfa'. 'pdf-a3' creates a "
         "PDF/A3-b file. 'none' will produce no output, which may be helpful if "
         "only the --sidecar is desired.",

src/ocrmypdf/exceptions.py

@@ -108,10 +108,16 @@ class EncryptedPdfError(ExitCodeException):
     )
 
 
-class DigitalSignatureError(ExitCodeException):
+class TesseractConfigError(ExitCodeException):
+    """Tesseract config can't be parsed."""
+
+    exit_code = ExitCode.invalid_config
+    message = "Error occurred while parsing a Tesseract configuration file"
+
+
+class DigitalSignatureError(InputFileError):
     """PDF has a digital signature."""
 
-    exit_code = ExitCode.input_file
     message = dedent(
         """\
         Input PDF has a digital signature. OCR would alter the document,
@@ -120,8 +126,27 @@ class DigitalSignatureError(ExitCodeException):
     )
 
 
-class TesseractConfigError(ExitCodeException):
+class TaggedPDFError(InputFileError):
-    """Tesseract config can't be parsed."""
+    """PDF is tagged."""
 
-    exit_code = ExitCode.invalid_config
+    message = dedent(
-    message = "Error occurred while parsing a Tesseract configuration file"
+        """\
+        This PDF is marked as a Tagged PDF. This often indicates
+        that the PDF was generated from an office document and does
+        not need OCR. Use --force-ocr, --skip-text or --redo-ocr to
+        override this error.
+        """
+    )
+
+
+class ColorConversionNeededError(BadArgsError):
+    """PDF needs color conversion."""
+
+    message = dedent(
+        """\
+        The input PDF has an unusual color space. Use
+        --color-conversion-strategy to convert to a common color space
+        such as RGB, or use --output-type pdf to skip PDF/A conversion
+        and retain the original color space.
+        """
+    )

src/ocrmypdf/extra_plugins/semfree.py

@@ -20,12 +20,13 @@ from __future__ import annotations
 import logging
 import logging.handlers
 import signal
+from collections.abc import Iterable, Iterator
 from contextlib import suppress
 from enum import Enum, auto
 from itertools import islice, repeat, takewhile, zip_longest
 from multiprocessing import Pipe, Process
 from multiprocessing.connection import Connection, wait
-from typing import Callable, Iterable, Iterator
+from typing import Callable
 
 from ocrmypdf import Executor, hookimpl
 from ocrmypdf._concurrent import NullProgressBar
@@ -93,7 +94,7 @@ def process_loop(
 
     for args in task_args:
         try:
-            result = task(args)
+            result = task(*args)
         except Exception as e:  # pylint: disable=broad-except
             conn.send((MessageType.exception, e))
             break
@@ -113,16 +114,16 @@ class LambdaExecutor(Executor):
         *,
         use_threads: bool,
         max_workers: int,
-        tqdm_kwargs: dict,
+        progress_kwargs: dict,
         worker_initializer: Callable,
         task: Callable,
         task_arguments: Iterable,
         task_finished: Callable,
     ):
         if use_threads and max_workers == 1:
-            with self.pbar_class(**tqdm_kwargs) as pbar:
+            with self.pbar_class(**progress_kwargs) as pbar:
                 for args in task_arguments:
-                    result = task(args)
+                    result = task(*args)
                     task_finished(result, pbar)
             return
 
@@ -156,7 +157,7 @@ class LambdaExecutor(Executor):
         for process in processes:
             process.start()
 
-        with self.pbar_class(**tqdm_kwargs) as pbar:
+        with self.pbar_class(**progress_kwargs) as pbar:
             while connections:
                 for result in wait(connections):
                     if not isinstance(result, Connection):

src/ocrmypdf/helpers.py

@@ -10,7 +10,7 @@ import multiprocessing
 import os
 import shutil
 import warnings
-from collections.abc import Iterable
+from collections.abc import Iterable, Sequence
 from contextlib import suppress
 from decimal import Decimal
 from io import StringIO
@@ -21,9 +21,6 @@ from typing import (
     Any,
     Callable,
     Generic,
-    Sequence,
-    SupportsFloat,
-    SupportsRound,
     TypeVar,
 )
 
@@ -322,10 +319,16 @@ def remove_all_log_handlers(logger: logging.Logger) -> None:
 
 
 def pikepdf_enable_mmap() -> None:
-    """Enable pikepdf mmap."""
+    """Enable pikepdf memory mapping."""
     try:
-        if pikepdf._core.set_access_default_mmap(True):
+        pikepdf._core.set_access_default_mmap(True)
-            log.debug("pikepdf mmap enabled")
+        log.debug(
+            "pikepdf mmap "
+            + (
+                'enabled'
+                if pikepdf._core.get_access_default_mmap()  # type: ignore[attr-defined]
+                else 'disabled'
+            )
+        )
     except AttributeError:
         log.debug("pikepdf mmap not available")
-    log.debug("pikepdf mmap disabled")

src/ocrmypdf/imageops.py

@@ -7,18 +7,10 @@ from __future__ import annotations
 
 import logging
 from math import floor, sqrt
-from typing import Optional, Tuple
+from typing import Optional
 
 from PIL import Image
 
-# Remove this workaround when we require Pillow >= 9.1.0
-try:
-    Resampling = Image.Resampling  # type: ignore
-except AttributeError:
-    # Pillow 9 shim
-    Resampling = Image  # type: ignore
-
-
 # While from __future__ import annotations, we use singledispatch here, which
 # does not support annotations. Disable check about using old-style typing
 # until Python 3.10, OR when drop singledispatch in ocrmypdf 15.
@@ -43,13 +35,13 @@ def bytes_per_pixel(mode: str) -> int:
 
 
 def _calculate_downsample(
-    image_size: Tuple[int, int],
+    image_size: tuple[int, int],
     bytes_per_pixel: int,
     *,
-    max_size: Optional[Tuple[int, int]] = None,
+    max_size: Optional[tuple[int, int]] = None,
     max_pixels: Optional[int] = None,
     max_bytes: Optional[int] = None,
-) -> Tuple[int, int]:
+) -> tuple[int, int]:
     """Calculate image size required to downsample an image to fit limits.
 
     If no limit is exceeded, the input image's size is returned.
@@ -106,10 +98,10 @@ def _calculate_downsample(
 def calculate_downsample(
     image: Image.Image,
     *,
-    max_size: Optional[Tuple[int, int]] = None,
+    max_size: Optional[tuple[int, int]] = None,
     max_pixels: Optional[int] = None,
     max_bytes: Optional[int] = None,
-) -> Tuple[int, int]:
+) -> tuple[int, int]:
     """Calculate image size required to downsample an image to fit limits.
 
     If no limit is exceeded, the input image's size is returned.
@@ -135,7 +127,7 @@ def downsample_image(
     image: Image.Image,
     new_size: tuple[int, int],
     *,
-    resample_mode: Image.Resampling = Resampling.BICUBIC,
+    resample_mode: Image.Resampling = Image.Resampling.BICUBIC,
     reducing_gap: int = 3,
 ) -> Image.Image:
     """Downsample an image to fit within the given limits.

src/ocrmypdf/languages.py

@@ -0,0 +1,849 @@
+# SPDX-FileCopyrightText: 2023 James R. Barlow
+# SPDX-License-Identifier: MPL-2.0
+
+"""Language codes and names from ISO 639.
+
+Derived from
+https://www.loc.gov/standards/iso639-2/ascii_8bits.html
+"""
+
+
+from typing import NamedTuple
+
+
+class ISOCodeData(NamedTuple):
+    """Data for a single ISO 639 code."""
+    alt: str
+    alpha_2: str
+    english: str
+    french: str
+
+
+ISO_639_3 = {
+    'aar': ISOCodeData('', 'aa', 'Afar', 'afar'),
+    'abk': ISOCodeData('', 'ab', 'Abkhazian', 'abkhaze'),
+    'ace': ISOCodeData('', '', 'Achinese', 'aceh'),
+    'ach': ISOCodeData('', '', 'Acoli', 'acoli'),
+    'ada': ISOCodeData('', '', 'Adangme', 'adangme'),
+    'ady': ISOCodeData('', '', 'Adyghe; Adygei', 'adyghé'),
+    'afa': ISOCodeData(
+        '',
+        '',
+        'Afro-Asiatic languages',
+        'afro-asiatiques, langues',
+    ),
+    'afh': ISOCodeData('', '', 'Afrihili', 'afrihili'),
+    'afr': ISOCodeData('', 'af', 'Afrikaans', 'afrikaans'),
+    'ain': ISOCodeData('', '', 'Ainu', 'aïnou'),
+    'aka': ISOCodeData('', 'ak', 'Akan', 'akan'),
+    'akk': ISOCodeData('', '', 'Akkadian', 'akkadien'),
+    'alb': ISOCodeData('sqi', 'sq', 'Albanian', 'albanais'),
+    'ale': ISOCodeData('', '', 'Aleut', 'aléoute'),
+    'alg': ISOCodeData(
+        '',
+        '',
+        'Algonquian languages',
+        'algonquines, langues',
+    ),
+    'alt': ISOCodeData('', '', 'Southern Altai', 'altai du Sud'),
+    'amh': ISOCodeData('', 'am', 'Amharic', 'amharique'),
+    'ang': ISOCodeData(
+        '',
+        '',
+        'English, Old (ca.450-1100)',
+        'anglo-saxon (ca.450-1100)',
+    ),
+    'anp': ISOCodeData('', '', 'Angika', 'angika'),
+    'apa': ISOCodeData('', '', 'Apache languages', 'apaches, langues'),
+    'ara': ISOCodeData('', 'ar', 'Arabic', 'arabe'),
+    'arc': ISOCodeData(
+        '',
+        '',
+        'Official Aramaic (700-300 BCE); Imperial Aramaic (700-300 BCE)',
+        "araméen d'empire (700-300 BCE)",
+    ),
+    'arg': ISOCodeData('', 'an', 'Aragonese', 'aragonais'),
+    'arm': ISOCodeData('hye', 'hy', 'Armenian', 'arménien'),
+    'arn': ISOCodeData(
+        '',
+        '',
+        'Mapudungun; Mapuche',
+        'mapudungun; mapuche; mapuce',
+    ),
+    'arp': ISOCodeData('', '', 'Arapaho', 'arapaho'),
+    'art': ISOCodeData(
+        '',
+        '',
+        'Artificial languages',
+        'artificielles, langues',
+    ),
+    'arw': ISOCodeData('', '', 'Arawak', 'arawak'),
+    'asm': ISOCodeData('', 'as', 'Assamese', 'assamais'),
+    'ast': ISOCodeData(
+        '',
+        '',
+        'Asturian; Bable; Leonese; Asturleonese',
+        'asturien; bable; léonais; asturoléonais',
+    ),
+    'ath': ISOCodeData(
+        '',
+        '',
+        'Athapascan languages',
+        'athapascanes, langues',
+    ),
+    'aus': ISOCodeData(
+        '',
+        '',
+        'Australian languages',
+        'australiennes, langues',
+    ),
+    'ava': ISOCodeData('', 'av', 'Avaric', 'avar'),
+    'ave': ISOCodeData('', 'ae', 'Avestan', 'avestique'),
+    'awa': ISOCodeData('', '', 'Awadhi', 'awadhi'),
+    'aym': ISOCodeData('', 'ay', 'Aymara', 'aymara'),
+    'aze': ISOCodeData('', 'az', 'Azerbaijani', 'azéri'),
+    'bad': ISOCodeData('', '', 'Banda languages', 'banda, langues'),
+    'bai': ISOCodeData('', '', 'Bamileke languages', 'bamiléké, langues'),
+    'bak': ISOCodeData('', 'ba', 'Bashkir', 'bachkir'),
+    'bal': ISOCodeData('', '', 'Baluchi', 'baloutchi'),
+    'bam': ISOCodeData('', 'bm', 'Bambara', 'bambara'),
+    'ban': ISOCodeData('', '', 'Balinese', 'balinais'),
+    'baq': ISOCodeData('eus', 'eu', 'Basque', 'basque'),
+    'bas': ISOCodeData('', '', 'Basa', 'basa'),
+    'bat': ISOCodeData('', '', 'Baltic languages', 'baltes, langues'),
+    'bej': ISOCodeData('', '', 'Beja; Bedawiyet', 'bedja'),
+    'bel': ISOCodeData('', 'be', 'Belarusian', 'biélorusse'),
+    'bem': ISOCodeData('', '', 'Bemba', 'bemba'),
+    'ben': ISOCodeData('', 'bn', 'Bengali', 'bengali'),
+    'ber': ISOCodeData('', '', 'Berber languages', 'berbères, langues'),
+    'bho': ISOCodeData('', '', 'Bhojpuri', 'bhojpuri'),
+    'bih': ISOCodeData('', 'bh', 'Bihari languages', 'langues biharis'),
+    'bik': ISOCodeData('', '', 'Bikol', 'bikol'),
+    'bin': ISOCodeData('', '', 'Bini; Edo', 'bini; edo'),
+    'bis': ISOCodeData('', 'bi', 'Bislama', 'bichlamar'),
+    'bla': ISOCodeData('', '', 'Siksika', 'blackfoot'),
+    'bnt': ISOCodeData('', '', 'Bantu languages', 'bantou, langues'),
+    'bos': ISOCodeData('', 'bs', 'Bosnian', 'bosniaque'),
+    'bra': ISOCodeData('', '', 'Braj', 'braj'),
+    'bre': ISOCodeData('', 'br', 'Breton', 'breton'),
+    'btk': ISOCodeData('', '', 'Batak languages', 'batak, langues'),
+    'bua': ISOCodeData('', '', 'Buriat', 'bouriate'),
+    'bug': ISOCodeData('', '', 'Buginese', 'bugi'),
+    'bul': ISOCodeData('', 'bg', 'Bulgarian', 'bulgare'),
+    'bur': ISOCodeData('mya', 'my', 'Burmese', 'birman'),
+    'byn': ISOCodeData('', '', 'Blin; Bilin', 'blin; bilen'),
+    'cad': ISOCodeData('', '', 'Caddo', 'caddo'),
+    'cai': ISOCodeData(
+        '',
+        '',
+        'Central American Indian languages',
+        "amérindiennes de L'Amérique centrale, langues",
+    ),
+    'car': ISOCodeData('', '', 'Galibi Carib', 'karib; galibi; carib'),
+    'cat': ISOCodeData('', 'ca', 'Catalan; Valencian', 'catalan; valencien'),
+    'cau': ISOCodeData(
+        '',
+        '',
+        'Caucasian languages',
+        'caucasiennes, langues',
+    ),
+    'ceb': ISOCodeData('', '', 'Cebuano', 'cebuano'),
+    'cel': ISOCodeData(
+        '',
+        '',
+        'Celtic languages',
+        'celtiques, langues; celtes, langues',
+    ),
+    'cha': ISOCodeData('', 'ch', 'Chamorro', 'chamorro'),
+    'chb': ISOCodeData('', '', 'Chibcha', 'chibcha'),
+    'che': ISOCodeData('', 'ce', 'Chechen', 'tchétchène'),
+    'chg': ISOCodeData('', '', 'Chagatai', 'djaghataï'),
+    'chi': ISOCodeData('zho', 'zh', 'Chinese', 'chinois'),
+    'chk': ISOCodeData('', '', 'Chuukese', 'chuuk'),
+    'chm': ISOCodeData('', '', 'Mari', 'mari'),
+    'chn': ISOCodeData('', '', 'Chinook jargon', 'chinook, jargon'),
+    'cho': ISOCodeData('', '', 'Choctaw', 'choctaw'),
+    'chp': ISOCodeData('', '', 'Chipewyan; Dene Suline', 'chipewyan'),
+    'chr': ISOCodeData('', '', 'Cherokee', 'cherokee'),
+    'chu': ISOCodeData(
+        '',
+        'cu',
+        ('Church Slavic; Old Slavonic; Church Slavonic;'
+         ' Old Bulgarian; Old Church Slavonic'),
+        "slavon d'église; vieux slave; slavon liturgique; vieux bulgare",
+    ),
+    'chv': ISOCodeData('', 'cv', 'Chuvash', 'tchouvache'),
+    'chy': ISOCodeData('', '', 'Cheyenne', 'cheyenne'),
+    'cmc': ISOCodeData('', '', 'Chamic languages', 'chames, langues'),
+    'cnr': ISOCodeData('', '', 'Montenegrin', 'monténégrin'),
+    'cop': ISOCodeData('', '', 'Coptic', 'copte'),
+    'cor': ISOCodeData('', 'kw', 'Cornish', 'cornique'),
+    'cos': ISOCodeData('', 'co', 'Corsican', 'corse'),
+    'cpe': ISOCodeData(
+        '',
+        '',
+        'Creoles and pidgins, English based',
+        "créoles et pidgins basés sur l'anglais",
+    ),
+    'cpf': ISOCodeData(
+        '',
+        '',
+        'Creoles and pidgins, French-based',
+        'créoles et pidgins basés sur le français',
+    ),
+    'cpp': ISOCodeData(
+        '',
+        '',
+        'Creoles and pidgins, Portuguese-based',
+        'créoles et pidgins basés sur le portugais',
+    ),
+    'cre': ISOCodeData('', 'cr', 'Cree', 'cree'),
+    'crh': ISOCodeData(
+        '',
+        '',
+        'Crimean Tatar; Crimean Turkish',
+        'tatar de Crimé',
+    ),
+    'crp': ISOCodeData('', '', 'Creoles and pidgins', 'créoles et pidgins'),
+    'csb': ISOCodeData('', '', 'Kashubian', 'kachoube'),
+    'cus': ISOCodeData('', '', 'Cushitic languages', 'couchitiques, langues'),
+    'cze': ISOCodeData('ces', 'cs', 'Czech', 'tchèque'),
+    'dak': ISOCodeData('', '', 'Dakota', 'dakota'),
+    'dan': ISOCodeData('', 'da', 'Danish', 'danois'),
+    'dar': ISOCodeData('', '', 'Dargwa', 'dargwa'),
+    'day': ISOCodeData('', '', 'Land Dayak languages', 'dayak, langues'),
+    'del': ISOCodeData('', '', 'Delaware', 'delaware'),
+    'den': ISOCodeData('', '', 'Slave (Athapascan)', 'esclave (athapascan)'),
+    'dgr': ISOCodeData('', '', 'Dogrib', 'dogrib'),
+    'din': ISOCodeData('', '', 'Dinka', 'dinka'),
+    'div': ISOCodeData('', 'dv', 'Divehi; Dhivehi; Maldivian', 'maldivien'),
+    'doi': ISOCodeData('', '', 'Dogri', 'dogri'),
+    'dra': ISOCodeData(
+        '',
+        '',
+        'Dravidian languages',
+        'dravidiennes, langues',
+    ),
+    'dsb': ISOCodeData('', '', 'Lower Sorbian', 'bas-sorabe'),
+    'dua': ISOCodeData('', '', 'Duala', 'douala'),
+    'dum': ISOCodeData(
+        '',
+        '',
+        'Dutch, Middle (ca.1050-1350)',
+        'néerlandais moyen (ca. 1050-1350)',
+    ),
+    'dut': ISOCodeData('nld', 'nl', 'Dutch; Flemish', 'néerlandais; flamand'),
+    'dyu': ISOCodeData('', '', 'Dyula', 'dioula'),
+    'dzo': ISOCodeData('', 'dz', 'Dzongkha', 'dzongkha'),
+    'efi': ISOCodeData('', '', 'Efik', 'efik'),
+    'egy': ISOCodeData('', '', 'Egyptian (Ancient)', 'égyptien'),
+    'eka': ISOCodeData('', '', 'Ekajuk', 'ekajuk'),
+    'elx': ISOCodeData('', '', 'Elamite', 'élamite'),
+    'eng': ISOCodeData('', 'en', 'English', 'anglais'),
+    'enm': ISOCodeData(
+        '',
+        '',
+        'English, Middle (1100-1500)',
+        'anglais moyen (1100-1500)',
+    ),
+    'epo': ISOCodeData('', 'eo', 'Esperanto', 'espéranto'),
+    'est': ISOCodeData('', 'et', 'Estonian', 'estonien'),
+    'ewe': ISOCodeData('', 'ee', 'Ewe', 'éwé'),
+    'ewo': ISOCodeData('', '', 'Ewondo', 'éwondo'),
+    'fan': ISOCodeData('', '', 'Fang', 'fang'),
+    'fao': ISOCodeData('', 'fo', 'Faroese', 'féroïen'),
+    'fat': ISOCodeData('', '', 'Fanti', 'fanti'),
+    'fij': ISOCodeData('', 'fj', 'Fijian', 'fidjien'),
+    'fil': ISOCodeData('', '', 'Filipino; Pilipino', 'filipino; pilipino'),
+    'fin': ISOCodeData('', 'fi', 'Finnish', 'finnois'),
+    'fiu': ISOCodeData(
+        '',
+        '',
+        'Finno-Ugrian languages',
+        'finno-ougriennes, langues',
+    ),
+    'fon': ISOCodeData('', '', 'Fon', 'fon'),
+    'fre': ISOCodeData('fra', 'fr', 'French', 'français'),
+    'frm': ISOCodeData(
+        '',
+        '',
+        'French, Middle (ca.1400-1600)',
+        'français moyen (1400-1600)',
+    ),
+    'fro': ISOCodeData(
+        '',
+        '',
+        'French, Old (842-ca.1400)',
+        'français ancien (842-ca.1400)',
+    ),
+    'frr': ISOCodeData('', '', 'Northern Frisian', 'frison septentrional'),
+    'frs': ISOCodeData('', '', 'Eastern Frisian', 'frison oriental'),
+    'fry': ISOCodeData('', 'fy', 'Western Frisian', 'frison occidental'),
+    'ful': ISOCodeData('', 'ff', 'Fulah', 'peul'),
+    'fur': ISOCodeData('', '', 'Friulian', 'frioulan'),
+    'gaa': ISOCodeData('', '', 'Ga', 'ga'),
+    'gay': ISOCodeData('', '', 'Gayo', 'gayo'),
+    'gba': ISOCodeData('', '', 'Gbaya', 'gbaya'),
+    'gem': ISOCodeData('', '', 'Germanic languages', 'germaniques, langues'),
+    'geo': ISOCodeData('kat', 'ka', 'Georgian', 'géorgien'),
+    'ger': ISOCodeData('deu', 'de', 'German', 'allemand'),
+    'gez': ISOCodeData('', '', 'Geez', 'guèze'),
+    'gil': ISOCodeData('', '', 'Gilbertese', 'kiribati'),
+    'gla': ISOCodeData(
+        '',
+        'gd',
+        'Gaelic; Scottish Gaelic',
+        'gaélique; gaélique écossais',
+    ),
+    'gle': ISOCodeData('', 'ga', 'Irish', 'irlandais'),
+    'glg': ISOCodeData('', 'gl', 'Galician', 'galicien'),
+    'glv': ISOCodeData('', 'gv', 'Manx', 'manx; mannois'),
+    'gmh': ISOCodeData(
+        '',
+        '',
+        'German, Middle High (ca.1050-1500)',
+        'allemand, moyen haut (ca. 1050-1500)',
+    ),
+    'goh': ISOCodeData(
+        '',
+        '',
+        'German, Old High (ca.750-1050)',
+        'allemand, vieux haut (ca. 750-1050)',
+    ),
+    'gon': ISOCodeData('', '', 'Gondi', 'gond'),
+    'gor': ISOCodeData('', '', 'Gorontalo', 'gorontalo'),
+    'got': ISOCodeData('', '', 'Gothic', 'gothique'),
+    'grb': ISOCodeData('', '', 'Grebo', 'grebo'),
+    'grc': ISOCodeData(
+        '',
+        '',
+        'Greek, Ancient (to 1453)',
+        "grec ancien (jusqu'à 1453)",
+    ),
+    'gre': ISOCodeData(
+        'ell',
+        'el',
+        'Greek, Modern (1453-)',
+        'grec moderne (après 1453)',
+    ),
+    'grn': ISOCodeData('', 'gn', 'Guarani', 'guarani'),
+    'gsw': ISOCodeData(
+        '',
+        '',
+        'Swiss German; Alemannic; Alsatian',
+        'suisse alémanique; alémanique; alsacien',
+    ),
+    'guj': ISOCodeData('', 'gu', 'Gujarati', 'goudjrati'),
+    'gwi': ISOCodeData('', '', "Gwich'in", "gwich'in"),
+    'hai': ISOCodeData('', '', 'Haida', 'haida'),
+    'hat': ISOCodeData(
+        '',
+        'ht',
+        'Haitian; Haitian Creole',
+        'haïtien; créole haïtien',
+    ),
+    'hau': ISOCodeData('', 'ha', 'Hausa', 'haoussa'),
+    'haw': ISOCodeData('', '', 'Hawaiian', 'hawaïen'),
+    'heb': ISOCodeData('', 'he', 'Hebrew', 'hébreu'),
+    'her': ISOCodeData('', 'hz', 'Herero', 'herero'),
+    'hil': ISOCodeData('', '', 'Hiligaynon', 'hiligaynon'),
+    'him': ISOCodeData(
+        '',
+        '',
+        'Himachali languages; Western Pahari languages',
+        'langues himachalis; langues paharis occidentales',
+    ),
+    'hin': ISOCodeData('', 'hi', 'Hindi', 'hindi'),
+    'hit': ISOCodeData('', '', 'Hittite', 'hittite'),
+    'hmn': ISOCodeData('', '', 'Hmong; Mong', 'hmong'),
+    'hmo': ISOCodeData('', 'ho', 'Hiri Motu', 'hiri motu'),
+    'hrv': ISOCodeData('', 'hr', 'Croatian', 'croate'),
+    'hsb': ISOCodeData('', '', 'Upper Sorbian', 'haut-sorabe'),
+    'hun': ISOCodeData('', 'hu', 'Hungarian', 'hongrois'),
+    'hup': ISOCodeData('', '', 'Hupa', 'hupa'),
+    'iba': ISOCodeData('', '', 'Iban', 'iban'),
+    'ibo': ISOCodeData('', 'ig', 'Igbo', 'igbo'),
+    'ice': ISOCodeData('isl', 'is', 'Icelandic', 'islandais'),
+    'ido': ISOCodeData('', 'io', 'Ido', 'ido'),
+    'iii': ISOCodeData('', 'ii', 'Sichuan Yi; Nuosu', 'yi de Sichuan'),
+    'ijo': ISOCodeData('', '', 'Ijo languages', 'ijo, langues'),
+    'iku': ISOCodeData('', 'iu', 'Inuktitut', 'inuktitut'),
+    'ile': ISOCodeData('', 'ie', 'Interlingue; Occidental', 'interlingue'),
+    'ilo': ISOCodeData('', '', 'Iloko', 'ilocano'),
+    'ina': ISOCodeData(
+        '',
+        'ia',
+        'Interlingua (International Auxiliary Language Association)',
+        'interlingua (langue auxiliaire internationale)',
+    ),
+    'inc': ISOCodeData('', '', 'Indic languages', 'indo-aryennes, langues'),
+    'ind': ISOCodeData('', 'id', 'Indonesian', 'indonésien'),
+    'ine': ISOCodeData(
+        '',
+        '',
+        'Indo-European languages',
+        'indo-européennes, langues',
+    ),
+    'inh': ISOCodeData('', '', 'Ingush', 'ingouche'),
+    'ipk': ISOCodeData('', 'ik', 'Inupiaq', 'inupiaq'),
+    'ira': ISOCodeData('', '', 'Iranian languages', 'iraniennes, langues'),
+    'iro': ISOCodeData('', '', 'Iroquoian languages', 'iroquoises, langues'),
+    'ita': ISOCodeData('', 'it', 'Italian', 'italien'),
+    'jav': ISOCodeData('', 'jv', 'Javanese', 'javanais'),
+    'jbo': ISOCodeData('', '', 'Lojban', 'lojban'),
+    'jpn': ISOCodeData('', 'ja', 'Japanese', 'japonais'),
+    'jpr': ISOCodeData('', '', 'Judeo-Persian', 'judéo-persan'),
+    'jrb': ISOCodeData('', '', 'Judeo-Arabic', 'judéo-arabe'),
+    'kaa': ISOCodeData('', '', 'Kara-Kalpak', 'karakalpak'),
+    'kab': ISOCodeData('', '', 'Kabyle', 'kabyle'),
+    'kac': ISOCodeData('', '', 'Kachin; Jingpho', 'kachin; jingpho'),
+    'kal': ISOCodeData('', 'kl', 'Kalaallisut; Greenlandic', 'groenlandais'),
+    'kam': ISOCodeData('', '', 'Kamba', 'kamba'),
+    'kan': ISOCodeData('', 'kn', 'Kannada', 'kannada'),
+    'kar': ISOCodeData('', '', 'Karen languages', 'karen, langues'),
+    'kas': ISOCodeData('', 'ks', 'Kashmiri', 'kashmiri'),
+    'kau': ISOCodeData('', 'kr', 'Kanuri', 'kanouri'),
+    'kaw': ISOCodeData('', '', 'Kawi', 'kawi'),
+    'kaz': ISOCodeData('', 'kk', 'Kazakh', 'kazakh'),
+    'kbd': ISOCodeData('', '', 'Kabardian', 'kabardien'),
+    'kha': ISOCodeData('', '', 'Khasi', 'khasi'),
+    'khi': ISOCodeData('', '', 'Khoisan languages', 'khoïsan, langues'),
+    'khm': ISOCodeData('', 'km', 'Central Khmer', 'khmer central'),
+    'kho': ISOCodeData('', '', 'Khotanese; Sakan', 'khotanais; sakan'),
+    'kik': ISOCodeData('', 'ki', 'Kikuyu; Gikuyu', 'kikuyu'),
+    'kin': ISOCodeData('', 'rw', 'Kinyarwanda', 'rwanda'),
+    'kir': ISOCodeData('', 'ky', 'Kirghiz; Kyrgyz', 'kirghiz'),
+    'kmb': ISOCodeData('', '', 'Kimbundu', 'kimbundu'),
+    'kok': ISOCodeData('', '', 'Konkani', 'konkani'),
+    'kom': ISOCodeData('', 'kv', 'Komi', 'kom'),
+    'kon': ISOCodeData('', 'kg', 'Kongo', 'kongo'),
+    'kor': ISOCodeData('', 'ko', 'Korean', 'coréen'),
+    'kos': ISOCodeData('', '', 'Kosraean', 'kosrae'),
+    'kpe': ISOCodeData('', '', 'Kpelle', 'kpellé'),
+    'krc': ISOCodeData('', '', 'Karachay-Balkar', 'karatchai balkar'),
+    'krl': ISOCodeData('', '', 'Karelian', 'carélien'),
+    'kro': ISOCodeData('', '', 'Kru languages', 'krou, langues'),
+    'kru': ISOCodeData('', '', 'Kurukh', 'kurukh'),
+    'kua': ISOCodeData('', 'kj', 'Kuanyama; Kwanyama', 'kuanyama; kwanyama'),
+    'kum': ISOCodeData('', '', 'Kumyk', 'koumyk'),
+    'kur': ISOCodeData('', 'ku', 'Kurdish', 'kurde'),
+    'kut': ISOCodeData('', '', 'Kutenai', 'kutenai'),
+    'lad': ISOCodeData('', '', 'Ladino', 'judéo-espagnol'),
+    'lah': ISOCodeData('', '', 'Lahnda', 'lahnda'),
+    'lam': ISOCodeData('', '', 'Lamba', 'lamba'),
+    'lao': ISOCodeData('', 'lo', 'Lao', 'lao'),
+    'lat': ISOCodeData('', 'la', 'Latin', 'latin'),
+    'lav': ISOCodeData('', 'lv', 'Latvian', 'letton'),
+    'lez': ISOCodeData('', '', 'Lezghian', 'lezghien'),
+    'lim': ISOCodeData(
+        '',
+        'li',
+        'Limburgan; Limburger; Limburgish',
+        'limbourgeois',
+    ),
+    'lin': ISOCodeData('', 'ln', 'Lingala', 'lingala'),
+    'lit': ISOCodeData('', 'lt', 'Lithuanian', 'lituanien'),
+    'lol': ISOCodeData('', '', 'Mongo', 'mongo'),
+    'loz': ISOCodeData('', '', 'Lozi', 'lozi'),
+    'ltz': ISOCodeData(
+        '',
+        'lb',
+        'Luxembourgish; Letzeburgesch',
+        'luxembourgeois',
+    ),
+    'lua': ISOCodeData('', '', 'Luba-Lulua', 'luba-lulua'),
+    'lub': ISOCodeData('', 'lu', 'Luba-Katanga', 'luba-katanga'),
+    'lug': ISOCodeData('', 'lg', 'Ganda', 'ganda'),
+    'lui': ISOCodeData('', '', 'Luiseno', 'luiseno'),
+    'lun': ISOCodeData('', '', 'Lunda', 'lunda'),
+    'luo': ISOCodeData(
+        '',
+        '',
+        'Luo (Kenya and Tanzania)',
+        'luo (Kenya et Tanzanie)',
+    ),
+    'lus': ISOCodeData('', '', 'Lushai', 'lushai'),
+    'mac': ISOCodeData('mkd', 'mk', 'Macedonian', 'macédonien'),
+    'mad': ISOCodeData('', '', 'Madurese', 'madourais'),
+    'mag': ISOCodeData('', '', 'Magahi', 'magahi'),
+    'mah': ISOCodeData('', 'mh', 'Marshallese', 'marshall'),
+    'mai': ISOCodeData('', '', 'Maithili', 'maithili'),
+    'mak': ISOCodeData('', '', 'Makasar', 'makassar'),
+    'mal': ISOCodeData('', 'ml', 'Malayalam', 'malayalam'),
+    'man': ISOCodeData('', '', 'Mandingo', 'mandingue'),
+    'mao': ISOCodeData('mri', 'mi', 'Maori', 'maori'),
+    'map': ISOCodeData(
+        '',
+        '',
+        'Austronesian languages',
+        'austronésiennes, langues',
+    ),
+    'mar': ISOCodeData('', 'mr', 'Marathi', 'marathe'),
+    'mas': ISOCodeData('', '', 'Masai', 'massaï'),
+    'may': ISOCodeData('msa', 'ms', 'Malay', 'malais'),
+    'mdf': ISOCodeData('', '', 'Moksha', 'moksa'),
+    'mdr': ISOCodeData('', '', 'Mandar', 'mandar'),
+    'men': ISOCodeData('', '', 'Mende', 'mendé'),
+    'mga': ISOCodeData(
+        '',
+        '',
+        'Irish, Middle (900-1200)',
+        'irlandais moyen (900-1200)',
+    ),
+    'mic': ISOCodeData('', '', "Mi'kmaq; Micmac", "mi'kmaq; micmac"),
+    'min': ISOCodeData('', '', 'Minangkabau', 'minangkabau'),
+    'mis': ISOCodeData('', '', 'Uncoded languages', 'langues non codées'),
+    'mkh': ISOCodeData('', '', 'Mon-Khmer languages', 'môn-khmer, langues'),
+    'mlg': ISOCodeData('', 'mg', 'Malagasy', 'malgache'),
+    'mlt': ISOCodeData('', 'mt', 'Maltese', 'maltais'),
+    'mnc': ISOCodeData('', '', 'Manchu', 'mandchou'),
+    'mni': ISOCodeData('', '', 'Manipuri', 'manipuri'),
+    'mno': ISOCodeData('', '', 'Manobo languages', 'manobo, langues'),
+    'moh': ISOCodeData('', '', 'Mohawk', 'mohawk'),
+    'mon': ISOCodeData('', 'mn', 'Mongolian', 'mongol'),
+    'mos': ISOCodeData('', '', 'Mossi', 'moré'),
+    'mul': ISOCodeData('', '', 'Multiple languages', 'multilingue'),
+    'mun': ISOCodeData('', '', 'Munda languages', 'mounda, langues'),
+    'mus': ISOCodeData('', '', 'Creek', 'muskogee'),
+    'mwl': ISOCodeData('', '', 'Mirandese', 'mirandais'),
+    'mwr': ISOCodeData('', '', 'Marwari', 'marvari'),
+    'myn': ISOCodeData('', '', 'Mayan languages', 'maya, langues'),
+    'myv': ISOCodeData('', '', 'Erzya', 'erza'),
+    'nah': ISOCodeData('', '', 'Nahuatl languages', 'nahuatl, langues'),
+    'nai': ISOCodeData(
+        '',
+        '',
+        'North American Indian languages',
+        'nord-amérindiennes, langues',
+    ),
+    'nap': ISOCodeData('', '', 'Neapolitan', 'napolitain'),
+    'nau': ISOCodeData('', 'na', 'Nauru', 'nauruan'),
+    'nav': ISOCodeData('', 'nv', 'Navajo; Navaho', 'navaho'),
+    'nbl': ISOCodeData(
+        '',
+        'nr',
+        'Ndebele, South; South Ndebele',
+        'ndébélé du Sud',
+    ),
+    'nde': ISOCodeData(
+        '',
+        'nd',
+        'Ndebele, North; North Ndebele',
+        'ndébélé du Nord',
+    ),
+    'ndo': ISOCodeData('', 'ng', 'Ndonga', 'ndonga'),
+    'nds': ISOCodeData(
+        '',
+        '',
+        'Low German; Low Saxon; German, Low; Saxon, Low',
+        'bas allemand; bas saxon; allemand, bas; saxon, bas',
+    ),
+    'nep': ISOCodeData('', 'ne', 'Nepali', 'népalais'),
+    'new': ISOCodeData('', '', 'Nepal Bhasa; Newari', 'nepal bhasa; newari'),
+    'nia': ISOCodeData('', '', 'Nias', 'nias'),
+    'nic': ISOCodeData(
+        '',
+        '',
+        'Niger-Kordofanian languages',
+        'nigéro-kordofaniennes, langues',
+    ),
+    'niu': ISOCodeData('', '', 'Niuean', 'niué'),
+    'nno': ISOCodeData(
+        '',
+        'nn',
+        'Norwegian Nynorsk; Nynorsk, Norwegian',
+        'norvégien nynorsk; nynorsk, norvégien',
+    ),
+    'nob': ISOCodeData(
+        '',
+        'nb',
+        'Bokmål, Norwegian; Norwegian Bokmål',
+        'norvégien bokmål',
+    ),
+    'nog': ISOCodeData('', '', 'Nogai', 'nogaï; nogay'),
+    'non': ISOCodeData('', '', 'Norse, Old', 'norrois, vieux'),
+    'nor': ISOCodeData('', 'no', 'Norwegian', 'norvégien'),
+    'nqo': ISOCodeData('', '', "N'Ko", "n'ko"),
+    'nso': ISOCodeData(
+        '',
+        '',
+        'Pedi; Sepedi; Northern Sotho',
+        'pedi; sepedi; sotho du Nord',
+    ),
+    'nub': ISOCodeData('', '', 'Nubian languages', 'nubiennes, langues'),
+    'nwc': ISOCodeData(
+        '',
+        '',
+        'Classical Newari; Old Newari; Classical Nepal Bhasa',
+        'newari classique',
+    ),
+    'nya': ISOCodeData(
+        '',
+        'ny',
+        'Chichewa; Chewa; Nyanja',
+        'chichewa; chewa; nyanja',
+    ),
+    'nym': ISOCodeData('', '', 'Nyamwezi', 'nyamwezi'),
+    'nyn': ISOCodeData('', '', 'Nyankole', 'nyankolé'),
+    'nyo': ISOCodeData('', '', 'Nyoro', 'nyoro'),
+    'nzi': ISOCodeData('', '', 'Nzima', 'nzema'),
+    'oci': ISOCodeData(
+        '',
+        'oc',
+        'Occitan (post 1500)',
+        'occitan (après 1500)',
+    ),
+    'oji': ISOCodeData('', 'oj', 'Ojibwa', 'ojibwa'),
+    'ori': ISOCodeData('', 'or', 'Oriya', 'oriya'),
+    'orm': ISOCodeData('', 'om', 'Oromo', 'galla'),
+    'osa': ISOCodeData('', '', 'Osage', 'osage'),
+    'oss': ISOCodeData('', 'os', 'Ossetian; Ossetic', 'ossète'),
+    'ota': ISOCodeData(
+        '',
+        '',
+        'Turkish, Ottoman (1500-1928)',
+        'turc ottoman (1500-1928)',
+    ),
+    'oto': ISOCodeData('', '', 'Otomian languages', 'otomi, langues'),
+    'paa': ISOCodeData('', '', 'Papuan languages', 'papoues, langues'),
+    'pag': ISOCodeData('', '', 'Pangasinan', 'pangasinan'),
+    'pal': ISOCodeData('', '', 'Pahlavi', 'pahlavi'),
+    'pam': ISOCodeData('', '', 'Pampanga; Kapampangan', 'pampangan'),
+    'pan': ISOCodeData('', 'pa', 'Panjabi; Punjabi', 'pendjabi'),
+    'pap': ISOCodeData('', '', 'Papiamento', 'papiamento'),
+    'pau': ISOCodeData('', '', 'Palauan', 'palau'),
+    'peo': ISOCodeData(
+        '',
+        '',
+        'Persian, Old (ca.600-400 B.C.)',
+        'perse, vieux (ca. 600-400 av. J.-C.)',
+    ),
+    'per': ISOCodeData('fas', 'fa', 'Persian', 'persan'),
+    'phi': ISOCodeData(
+        '',
+        '',
+        'Philippine languages',
+        'philippines, langues',
+    ),
+    'phn': ISOCodeData('', '', 'Phoenician', 'phénicien'),
+    'pli': ISOCodeData('', 'pi', 'Pali', 'pali'),
+    'pol': ISOCodeData('', 'pl', 'Polish', 'polonais'),
+    'pon': ISOCodeData('', '', 'Pohnpeian', 'pohnpei'),
+    'por': ISOCodeData('', 'pt', 'Portuguese', 'portugais'),
+    'pra': ISOCodeData('', '', 'Prakrit languages', 'prâkrit, langues'),
+    'pro': ISOCodeData(
+        '',
+        '',
+        'Provençal, Old (to 1500); Occitan, Old (to 1500)',
+        "provençal ancien (jusqu'à 1500); occitan ancien (jusqu'à 1500)",
+    ),
+    'pus': ISOCodeData('', 'ps', 'Pushto; Pashto', 'pachto'),
+    'qaa': ISOCodeData(
+        '',
+        '',
+        'Reserved for local use',
+        "réservée à l'usage local",
+    ),
+    'que': ISOCodeData('', 'qu', 'Quechua', 'quechua'),
+    'raj': ISOCodeData('', '', 'Rajasthani', 'rajasthani'),
+    'rap': ISOCodeData('', '', 'Rapanui', 'rapanui'),
+    'rar': ISOCodeData(
+        '',
+        '',
+        'Rarotongan; Cook Islands Maori',
+        'rarotonga; maori des îles Cook',
+    ),
+    'roa': ISOCodeData('', '', 'Romance languages', 'romanes, langues'),
+    'roh': ISOCodeData('', 'rm', 'Romansh', 'romanche'),
+    'rom': ISOCodeData('', '', 'Romany', 'tsigane'),
+    'rum': ISOCodeData(
+        'ron',
+        'ro',
+        'Romanian; Moldavian; Moldovan',
+        'roumain; moldave',
+    ),
+    'run': ISOCodeData('', 'rn', 'Rundi', 'rundi'),
+    'rup': ISOCodeData(
+        '',
+        '',
+        'Aromanian; Arumanian; Macedo-Romanian',
+        'aroumain; macédo-roumain',
+    ),
+    'rus': ISOCodeData('', 'ru', 'Russian', 'russe'),
+    'sad': ISOCodeData('', '', 'Sandawe', 'sandawe'),
+    'sag': ISOCodeData('', 'sg', 'Sango', 'sango'),
+    'sah': ISOCodeData('', '', 'Yakut', 'iakoute'),
+    'sai': ISOCodeData(
+        '',
+        '',
+        'South American Indian languages',
+        'sud-amérindiennes, langues',
+    ),
+    'sal': ISOCodeData('', '', 'Salishan languages', 'salishennes, langues'),
+    'sam': ISOCodeData('', '', 'Samaritan Aramaic', 'samaritain'),
+    'san': ISOCodeData('', 'sa', 'Sanskrit', 'sanskrit'),
+    'sas': ISOCodeData('', '', 'Sasak', 'sasak'),
+    'sat': ISOCodeData('', '', 'Santali', 'santal'),
+    'scn': ISOCodeData('', '', 'Sicilian', 'sicilien'),
+    'sco': ISOCodeData('', '', 'Scots', 'écossais'),
+    'sel': ISOCodeData('', '', 'Selkup', 'selkoupe'),
+    'sem': ISOCodeData('', '', 'Semitic languages', 'sémitiques, langues'),
+    'sga': ISOCodeData(
+        '',
+        '',
+        'Irish, Old (to 900)',
+        "irlandais ancien (jusqu'à 900)",
+    ),
+    'sgn': ISOCodeData('', '', 'Sign Languages', 'langues des signes'),
+    'shn': ISOCodeData('', '', 'Shan', 'chan'),
+    'sid': ISOCodeData('', '', 'Sidamo', 'sidamo'),
+    'sin': ISOCodeData('', 'si', 'Sinhala; Sinhalese', 'singhalais'),
+    'sio': ISOCodeData('', '', 'Siouan languages', 'sioux, langues'),
+    'sit': ISOCodeData(
+        '',
+        '',
+        'Sino-Tibetan languages',
+        'sino-tibétaines, langues',
+    ),
+    'sla': ISOCodeData('', '', 'Slavic languages', 'slaves, langues'),
+    'slo': ISOCodeData('slk', 'sk', 'Slovak', 'slovaque'),
+    'slv': ISOCodeData('', 'sl', 'Slovenian', 'slovène'),
+    'sma': ISOCodeData('', '', 'Southern Sami', 'sami du Sud'),
+    'sme': ISOCodeData('', 'se', 'Northern Sami', 'sami du Nord'),
+    'smi': ISOCodeData('', '', 'Sami languages', 'sames, langues'),
+    'smj': ISOCodeData('', '', 'Lule Sami', 'sami de Lule'),
+    'smn': ISOCodeData('', '', 'Inari Sami', "sami d'Inari"),
+    'smo': ISOCodeData('', 'sm', 'Samoan', 'samoan'),
+    'sms': ISOCodeData('', '', 'Skolt Sami', 'sami skolt'),
+    'sna': ISOCodeData('', 'sn', 'Shona', 'shona'),
+    'snd': ISOCodeData('', 'sd', 'Sindhi', 'sindhi'),
+    'snk': ISOCodeData('', '', 'Soninke', 'soninké'),
+    'sog': ISOCodeData('', '', 'Sogdian', 'sogdien'),
+    'som': ISOCodeData('', 'so', 'Somali', 'somali'),
+    'son': ISOCodeData('', '', 'Songhai languages', 'songhai, langues'),
+    'sot': ISOCodeData('', 'st', 'Sotho, Southern', 'sotho du Sud'),
+    'spa': ISOCodeData('', 'es', 'Spanish; Castilian', 'espagnol; castillan'),
+    'srd': ISOCodeData('', 'sc', 'Sardinian', 'sarde'),
+    'srn': ISOCodeData('', '', 'Sranan Tongo', 'sranan tongo'),
+    'srp': ISOCodeData('', 'sr', 'Serbian', 'serbe'),
+    'srr': ISOCodeData('', '', 'Serer', 'sérère'),
+    'ssa': ISOCodeData(
+        '',
+        '',
+        'Nilo-Saharan languages',
+        'nilo-sahariennes, langues',
+    ),
+    'ssw': ISOCodeData('', 'ss', 'Swati', 'swati'),
+    'suk': ISOCodeData('', '', 'Sukuma', 'sukuma'),
+    'sun': ISOCodeData('', 'su', 'Sundanese', 'soundanais'),
+    'sus': ISOCodeData('', '', 'Susu', 'soussou'),
+    'sux': ISOCodeData('', '', 'Sumerian', 'sumérien'),
+    'swa': ISOCodeData('', 'sw', 'Swahili', 'swahili'),
+    'swe': ISOCodeData('', 'sv', 'Swedish', 'suédois'),
+    'syc': ISOCodeData('', '', 'Classical Syriac', 'syriaque classique'),
+    'syr': ISOCodeData('', '', 'Syriac', 'syriaque'),
+    'tah': ISOCodeData('', 'ty', 'Tahitian', 'tahitien'),
+    'tai': ISOCodeData('', '', 'Tai languages', 'tai, langues'),
+    'tam': ISOCodeData('', 'ta', 'Tamil', 'tamoul'),
+    'tat': ISOCodeData('', 'tt', 'Tatar', 'tatar'),
+    'tel': ISOCodeData('', 'te', 'Telugu', 'télougou'),
+    'tem': ISOCodeData('', '', 'Timne', 'temne'),
+    'ter': ISOCodeData('', '', 'Tereno', 'tereno'),
+    'tet': ISOCodeData('', '', 'Tetum', 'tetum'),
+    'tgk': ISOCodeData('', 'tg', 'Tajik', 'tadjik'),
+    'tgl': ISOCodeData('', 'tl', 'Tagalog', 'tagalog'),
+    'tha': ISOCodeData('', 'th', 'Thai', 'thaï'),
+    'tib': ISOCodeData('bod', 'bo', 'Tibetan', 'tibétain'),
+    'tig': ISOCodeData('', '', 'Tigre', 'tigré'),
+    'tir': ISOCodeData('', 'ti', 'Tigrinya', 'tigrigna'),
+    'tiv': ISOCodeData('', '', 'Tiv', 'tiv'),
+    'tkl': ISOCodeData('', '', 'Tokelau', 'tokelau'),
+    'tlh': ISOCodeData('', '', 'Klingon; tlhIngan-Hol', 'klingon'),
+    'tli': ISOCodeData('', '', 'Tlingit', 'tlingit'),
+    'tmh': ISOCodeData('', '', 'Tamashek', 'tamacheq'),
+    'tog': ISOCodeData('', '', 'Tonga (Nyasa)', 'tonga (Nyasa)'),
+    'ton': ISOCodeData(
+        '',
+        'to',
+        'Tonga (Tonga Islands)',
+        'tongan (Îles Tonga)',
+    ),
+    'tpi': ISOCodeData('', '', 'Tok Pisin', 'tok pisin'),
+    'tsi': ISOCodeData('', '', 'Tsimshian', 'tsimshian'),
+    'tsn': ISOCodeData('', 'tn', 'Tswana', 'tswana'),
+    'tso': ISOCodeData('', 'ts', 'Tsonga', 'tsonga'),
+    'tuk': ISOCodeData('', 'tk', 'Turkmen', 'turkmène'),
+    'tum': ISOCodeData('', '', 'Tumbuka', 'tumbuka'),
+    'tup': ISOCodeData('', '', 'Tupi languages', 'tupi, langues'),
+    'tur': ISOCodeData('', 'tr', 'Turkish', 'turc'),
+    'tut': ISOCodeData('', '', 'Altaic languages', 'altaïques, langues'),
+    'tvl': ISOCodeData('', '', 'Tuvalu', 'tuvalu'),
+    'twi': ISOCodeData('', 'tw', 'Twi', 'twi'),
+    'tyv': ISOCodeData('', '', 'Tuvinian', 'touva'),
+    'udm': ISOCodeData('', '', 'Udmurt', 'oudmourte'),
+    'uga': ISOCodeData('', '', 'Ugaritic', 'ougaritique'),
+    'uig': ISOCodeData('', 'ug', 'Uighur; Uyghur', 'ouïgour'),
+    'ukr': ISOCodeData('', 'uk', 'Ukrainian', 'ukrainien'),
+    'umb': ISOCodeData('', '', 'Umbundu', 'umbundu'),
+    'und': ISOCodeData('', '', 'Undetermined', 'indéterminée'),
+    'urd': ISOCodeData('', 'ur', 'Urdu', 'ourdou'),
+    'uzb': ISOCodeData('', 'uz', 'Uzbek', 'ouszbek'),
+    'vai': ISOCodeData('', '', 'Vai', 'vaï'),
+    'ven': ISOCodeData('', 've', 'Venda', 'venda'),
+    'vie': ISOCodeData('', 'vi', 'Vietnamese', 'vietnamien'),
+    'vol': ISOCodeData('', 'vo', 'Volapük', 'volapük'),
+    'vot': ISOCodeData('', '', 'Votic', 'vote'),
+    'wak': ISOCodeData('', '', 'Wakashan languages', 'wakashanes, langues'),
+    'wal': ISOCodeData('', '', 'Wolaitta; Wolaytta', 'wolaitta; wolaytta'),
+    'war': ISOCodeData('', '', 'Waray', 'waray'),
+    'was': ISOCodeData('', '', 'Washo', 'washo'),
+    'wel': ISOCodeData('cym', 'cy', 'Welsh', 'gallois'),
+    'wen': ISOCodeData('', '', 'Sorbian languages', 'sorabes, langues'),
+    'wln': ISOCodeData('', 'wa', 'Walloon', 'wallon'),
+    'wol': ISOCodeData('', 'wo', 'Wolof', 'wolof'),
+    'xal': ISOCodeData('', '', 'Kalmyk; Oirat', 'kalmouk; oïrat'),
+    'xho': ISOCodeData('', 'xh', 'Xhosa', 'xhosa'),
+    'yao': ISOCodeData('', '', 'Yao', 'yao'),
+    'yap': ISOCodeData('', '', 'Yapese', 'yapois'),
+    'yid': ISOCodeData('', 'yi', 'Yiddish', 'yiddish'),
+    'yor': ISOCodeData('', 'yo', 'Yoruba', 'yoruba'),
+    'ypk': ISOCodeData('', '', 'Yupik languages', 'yupik, langues'),
+    'zap': ISOCodeData('', '', 'Zapotec', 'zapotèque'),
+    'zbl': ISOCodeData(
+        '',
+        '',
+        'Blissymbols; Blissymbolics; Bliss',
+        'symboles Bliss; Bliss',
+    ),
+    'zen': ISOCodeData('', '', 'Zenaga', 'zenaga'),
+    'zgh': ISOCodeData(
+        '',
+        '',
+        'Standard Moroccan Tamazight',
+        'amazighe standard marocain',
+    ),
+    'zha': ISOCodeData('', 'za', 'Zhuang; Chuang', 'zhuang; chuang'),
+    'znd': ISOCodeData('', '', 'Zande languages', 'zandé, langues'),
+    'zul': ISOCodeData('', 'zu', 'Zulu', 'zoulou'),
+    'zun': ISOCodeData('', '', 'Zuni', 'zuni'),
+    'zxx': ISOCodeData(
+        '',
+        '',
+        'No linguistic content; Not applicable',
+        'pas de contenu linguistique; non applicable',
+    ),
+    'zza': ISOCodeData(
+        '',
+        '',
+        'Zaza; Dimili; Dimli; Kirdki; Kirmanjki; Zazaki',
+        'zaza; dimili; dimli; kirdki; kirmanjki; zazaki',
+    ),
+}
+
+
+def iso_639_2_from_3(iso3: str) -> str:
+    """Convert ISO 639-3 code to ISO 639-2 code."""
+    if iso3 in ISO_639_3:
+        return ISO_639_3[iso3].alpha_2
+    else:
+        return ""

src/ocrmypdf/optimize.py

@@ -11,9 +11,11 @@ import sys
 import tempfile
 import threading
 from collections import defaultdict
+from collections.abc import Iterator, MutableSet, Sequence
 from os import fspath
 from pathlib import Path
-from typing import Callable, Iterator, MutableSet, NamedTuple, NewType, Sequence
+from typing import Any, Callable, NamedTuple, NewType
+from warnings import warn
 from zlib import compress
 
 import img2pdf
@@ -33,6 +35,7 @@ from PIL import Image
 from ocrmypdf._concurrent import Executor, SerialExecutor
 from ocrmypdf._exec import jbig2enc, pngquant
 from ocrmypdf._jobcontext import PdfContext
+from ocrmypdf._progressbar import ProgressBar
 from ocrmypdf.exceptions import OutputFileAccessError
 from ocrmypdf.helpers import IMG2PDF_KWARGS, safe_symlink
 
@@ -68,11 +71,14 @@ def jpg_name(root: Path, xref: Xref) -> Path:
 
 
 def extract_image_filter(
-    pdf: Pdf, root: Path, image: Stream, xref: Xref
+    image: Stream, xref: Xref, *args
 ) -> tuple[PdfImage, tuple[Name, Object]] | None:
     """Determine if an image is extractable."""
-    del pdf  # unused args
+    if isinstance(image, Pdf):
-    del root
+        # Support deprecated old function signature
+        # TODO Remove for v16 and drop *args from current function signature
+        image, xref = args[0], args[1]
+        warn("extract_image_filter: pdf, root parameters ignored", DeprecationWarning)
 
     if image.Subtype != Name.Image:
         return None
@@ -131,7 +137,7 @@ def extract_image_jbig2(
     """Extract an image, saving it as a JBIG2 file."""
     del options  # unused arg
 
-    result = extract_image_filter(pdf, root, image, xref)
+    result = extract_image_filter(image, xref)
     if result is None:
         return None
     pim, filtdp = result
@@ -171,7 +177,7 @@ def extract_image_generic(
     *, pdf: Pdf, root: Path, image: Stream, xref: Xref, options
 ) -> XrefExt | None:
     """Generic image extraction."""
-    result = extract_image_filter(pdf, root, image, xref)
+    result = extract_image_filter(image, xref)
     if result is None:
         return None
     pim, filtdp = result
@@ -282,7 +288,7 @@ def _find_image_xrefs(pdf: Pdf):
 
     for pageno, page in enumerate(pdf.pages):
         _find_image_xrefs_container(
-            pdf, page, pageno, include_xrefs, exclude_xrefs, pageno_for_xref
+            pdf, page.obj, pageno, include_xrefs, exclude_xrefs, pageno_for_xref
         )
 
     working_xrefs = include_xrefs - exclude_xrefs
@@ -385,15 +391,15 @@ def _produce_jbig2_images(
 
     if options.jbig2_page_group_size > 1:
         jbig2_args = jbig2_group_args
-        jbig2_convert = jbig2enc.convert_group_mp
+        jbig2_convert = jbig2enc.convert_group
     else:
         jbig2_args = jbig2_single_args
-        jbig2_convert = jbig2enc.convert_single_mp
+        jbig2_convert = jbig2enc.convert_single
 
     executor(
         use_threads=True,
         max_workers=options.jobs,
-        tqdm_kwargs=dict(
+        progress_kwargs=dict(
             total=len(jbig2_groups),
             desc="JBIG2",
             unit='item',
@@ -449,9 +455,9 @@ def convert_to_jbig2(
             )
 
 
-def _optimize_jpeg(args: tuple[Xref, Path, Path, int]) -> tuple[Xref, Path | None]:
+def _optimize_jpeg(
-    xref, in_jpg, opt_jpg, jpeg_quality = args
+    xref: Xref, in_jpg: Path, opt_jpg: Path, jpeg_quality: int
-
+) -> tuple[Xref, Path | None]:
     with Image.open(in_jpg) as im:
         im.save(opt_jpg, optimize=True, quality=jpeg_quality)
 
@@ -473,7 +479,7 @@ def transcode_jpegs(
             opt_jpg = in_jpg.with_suffix('.opt.jpg')
             yield xref, in_jpg, opt_jpg, options.jpeg_quality
 
-    def finish_jpeg(result: tuple[Xref, Path | None], pbar):
+    def finish_jpeg(result: tuple[Xref, Path | None], pbar: ProgressBar):
         xref, opt_jpg = result
         if opt_jpg:
             compdata = opt_jpg.read_bytes()  # JPEG can inserted into PDF as is
@@ -484,7 +490,7 @@ def transcode_jpegs(
     executor(
         use_threads=True,  # Processes are significantly slower at this task
         max_workers=options.jobs,
-        tqdm_kwargs=dict(
+        progress_kwargs=dict(
             desc="Recompressing JPEGs",
             total=len(jpegs),
             unit='image',
@@ -499,7 +505,7 @@ def transcode_jpegs(
 def _find_deflatable_jpeg(
     *, pdf: Pdf, root: Path, image: Stream, xref: Xref, options
 ) -> XrefExt | None:
-    result = extract_image_filter(pdf, root, image, xref)
+    result = extract_image_filter(image, xref)
     if result is None:
         return None
     _pim, filtdp = result
@@ -510,8 +516,9 @@ def _find_deflatable_jpeg(
     return None
 
 
-def _deflate_jpeg(args: tuple[Pdf, threading.Lock, Xref, int]) -> tuple[Xref, bytes]:
+def _deflate_jpeg(
-    pdf, lock, xref, complevel = args
+    pdf: Pdf, lock: threading.Lock, xref: Xref, complevel: int
+) -> tuple[Xref, bytes]:
     with lock:
         xobj = pdf.get_object(xref, 0)
         try:
@@ -546,7 +553,7 @@ def deflate_jpegs(pdf: Pdf, root: Path, options, executor: Executor) -> None:
         for xref in jpegs:
             yield pdf, lock, xref, complevel
 
-    def finish(result, pbar):
+    def finish(result: tuple[Xref, bytes], pbar: ProgressBar):
         xref, compdata = result
         if len(compdata) > 0:
             with lock:
@@ -557,7 +564,7 @@ def deflate_jpegs(pdf: Pdf, root: Path, options, executor: Executor) -> None:
     executor(
         use_threads=True,  # We're sharing the pdf directly, must use threads
         max_workers=options.jobs,
-        tqdm_kwargs=dict(
+        progress_kwargs=dict(
             desc="Deflating JPEGs",
             total=len(jpegs),
             unit='image',
@@ -616,7 +623,7 @@ def transcode_pngs(
     image_name_fn: Callable[[Path, Xref], Path],
     root: Path,
     options,
-    executor,
+    executor: Executor,
 ) -> None:
     """Apply lossy transcoding to PNGs."""
     modified: MutableSet[Xref] = set()
@@ -640,13 +647,13 @@ def transcode_pngs(
         executor(
             use_threads=True,
             max_workers=options.jobs,
-            tqdm_kwargs=dict(
+            progress_kwargs=dict(
                 desc="PNGs",
                 total=len(images),
                 unit='image',
                 disable=not options.progress_bar,
             ),
-            task=pngquant.quantize_mp,
+            task=pngquant.quantize,
             task_arguments=pngquant_args(),
         )
 
@@ -661,8 +668,8 @@ DEFAULT_EXECUTOR = SerialExecutor()
 def optimize(
     input_file: Path,
     output_file: Path,
-    context,
+    context: PdfContext,
-    save_settings,
+    save_settings: dict[str, Any],
     executor: Executor = DEFAULT_EXECUTOR,
 ) -> Path:
     """Optimize images in a PDF file."""

src/ocrmypdf/pdfa.py

@@ -6,9 +6,9 @@
 from __future__ import annotations
 
 import base64
+from collections.abc import Iterator
 from importlib.resources import files as package_files
 from pathlib import Path
-from typing import Iterator
 
 import pikepdf
 

src/ocrmypdf/pdfinfo/info.py

@@ -12,32 +12,36 @@ import re
 import statistics
 import sys
 from collections import defaultdict
-from contextlib import ExitStack
+from collections.abc import Container, Iterable, Iterator, Mapping, Sequence
+from contextlib import contextmanager
 from decimal import Decimal
 from enum import Enum, auto
 from functools import partial
 from math import hypot, inf, isclose
 from os import PathLike
 from pathlib import Path
-from typing import Container, Iterable, Iterator, Mapping, NamedTuple, Sequence, Tuple
+from typing import Callable, NamedTuple
 from warnings import warn
 
+from pdfminer.layout import LTPage, LTTextBox
 from pikepdf import (
+    Matrix,
     Name,
     Object,
     Page,
     Pdf,
     PdfImage,
     PdfInlineImage,
-    PdfMatrix,
+    Stream,
     UnsupportedImageTypeError,
     parse_content_stream,
 )
 
 from ocrmypdf._concurrent import Executor, SerialExecutor
+from ocrmypdf._progressbar import ProgressBar
 from ocrmypdf.exceptions import EncryptedPdfError, InputFileError
 from ocrmypdf.helpers import Resolution, available_cpu_count, pikepdf_enable_mmap
-from ocrmypdf.pdfinfo.layout import get_page_analysis, get_text_boxes
+from ocrmypdf.pdfinfo.layout import LTStateAwareChar, get_page_analysis, get_text_boxes
 
 logger = logging.getLogger()
 
@@ -73,7 +77,7 @@ class Encoding(Enum):
     runlength = auto()
 
 
-FloatRect = Tuple[float, float, float, float]
+FloatRect = tuple[float, float, float, float]
 
 FRIENDLY_COLORSPACE: dict[str, Colorspace] = {
     '/DeviceGray': Colorspace.gray,
@@ -205,7 +209,7 @@ def _interpret_contents(contentstream: Object, initial_shorthand=UNIT_SQUARE):
     CTM unchanged.
     """
     stack = []
-    ctm = PdfMatrix(initial_shorthand)
+    ctm = Matrix(initial_shorthand)
     xobject_settings: list[XobjectSettings] = []
     inline_images: list[InlineSettings] = []
     name_index = defaultdict(lambda: [])
@@ -236,7 +240,7 @@ def _interpret_contents(contentstream: Object, initial_shorthand=UNIT_SQUARE):
                 # to do. Just pretend nothing happened, keep calm and carry on.
                 warn("PDF graphics stack underflowed - PDF may be malformed")
         elif operator == 'cm':
-            ctm = PdfMatrix(operands) @ ctm
+            ctm = Matrix(operands) @ ctm
         elif operator == 'Do':
             image_name = operands[0]
             settings = XobjectSettings(
@@ -355,7 +359,7 @@ class ImageInfo:
         if inline is not None:
             self._origin = 'inline'
             pim = inline
-        elif pdfimage is not None:
+        elif pdfimage is not None and isinstance(pdfimage, Stream):
             self._origin = 'xobject'
             pim = PdfImage(pdfimage)
         else:
@@ -384,33 +388,43 @@ class ImageInfo:
         if self._enc == Encoding.jpeg2000:
             self._color = Colorspace.jpeg2000
 
-        if self._color == Colorspace.icc:
+        self._comp = None
-            # Check the ICC profile to determine actual colorspace
+        if self._color == Colorspace.icc and isinstance(pim, PdfImage):
-            try:
+            self._comp = self._init_icc(pim)
-                pim_icc = pim.icc
-                if pim_icc.profile.xcolor_space == 'GRAY':
-                    self._comp = 1
-                elif pim_icc.profile.xcolor_space == 'CMYK':
-                    self._comp = 4
-                else:
-                    self._comp = 3
-            except (AttributeError, UnsupportedImageTypeError) as ex:
-                self._comp = None
-                logger.warning(
-                    f"An image with a corrupt or unreadable ICC profile was found. "
-                    f"The output PDF may not match the input PDF visually: {ex}. {self}"
-                )
         else:
             if isinstance(self._color, Colorspace):
                 self._comp = FRIENDLY_COMP.get(self._color)
-            else:
-                self._comp = None
-
             # Bit of a hack... infer grayscale if component count is uncertain
             # but encoding only supports monochrome.
             if self._comp is None and self._enc in (Encoding.ccitt, Encoding.jbig2):
                 self._comp = FRIENDLY_COMP[Colorspace.gray]
 
+    def _init_icc(self, pim: PdfImage):
+        try:
+            icc = pim.icc
+        except UnsupportedImageTypeError as e:
+            logger.warning(
+                f"An image with a corrupt or unreadable ICC profile was found. "
+                f"Output PDF may not match the input PDF visually: {e}. {self}"
+            )
+            return None
+        # Check the ICC profile to determine actual colorspace
+        if icc is None or not hasattr(icc, 'profile'):
+            logger.warning(
+                f"An image with an ICC profile but no ICC profile data was found. "
+                f"The output PDF may not match the input PDF visually. {self}"
+            )
+            return None
+        try:
+            if icc.profile.xcolor_space == 'GRAY':
+                return 1
+            elif icc.profile.xcolor_space == 'CMYK':
+                return 4
+            else:
+                return 3
+        except AttributeError:
+            return None
+
     @property
     def name(self):
         """Name of the image as it appears in the PDF."""
@@ -600,12 +614,12 @@ def _process_content_streams(
     ):
         # Set the CTM to the state it was when the "Do" operator was
         # encountered that is drawing this instance of the Form XObject
-        ctm = PdfMatrix(shorthand) if shorthand else PdfMatrix.identity()
+        ctm = Matrix(shorthand) if shorthand else Matrix()
 
         # A Form XObject may provide its own matrix to map form space into
         # user space. Get this if one exists
-        form_shorthand = container.get(Name.Matrix, PdfMatrix.identity())
+        form_shorthand = container.get(Name.Matrix, Matrix())
-        form_matrix = PdfMatrix(form_shorthand)
+        form_matrix = Matrix(form_shorthand)
 
         # Concatenate form matrix with CTM to ensure CTM is correct for
         # drawing this instance of the XObject
@@ -654,7 +668,9 @@ def _page_has_text(text_blocks: Iterable[FloatRect], page_width, page_height) ->
     return has_text
 
 
-def simplify_textboxes(miner, textbox_getter) -> Iterator[TextboxInfo]:
+def simplify_textboxes(
+    miner: LTPage, textbox_getter: Callable[[LTPage], Iterator[LTTextBox]]
+) -> Iterator[TextboxInfo]:
     """Extract only limited content from text boxes.
 
     We do this to save memory and ensure that our objects are pickleable.
@@ -662,7 +678,8 @@ def simplify_textboxes(miner, textbox_getter) -> Iterator[TextboxInfo]:
     for box in textbox_getter(miner):
         first_line = box._objs[0]  # pylint: disable=protected-access
         first_char = first_line._objs[0]  # pylint: disable=protected-access
-
+        if not isinstance(first_char, LTStateAwareChar):
+            continue
         visible = first_char.rendermode != 3
         corrupt = first_char.get_text() == '\ufffd'
         yield TextboxInfo(box.bbox, visible, corrupt)
@@ -688,29 +705,41 @@ def _pdf_pageinfo_sync_init(pdf: Pdf, infile: Path, pdfminer_loglevel):
         atexit.register(on_process_close)
 
 
-def _pdf_pageinfo_sync(args):
+@contextmanager
-    pageno, thread_pdf, infile, check_pages, detailed_analysis = args
+def _pdf_pageinfo_sync_pdf(thread_pdf: Pdf | None, infile: Path):
-    pdf = thread_pdf if thread_pdf is not None else worker_pdf
+    if thread_pdf is not None:
-    with ExitStack() as stack:
+        yield thread_pdf
-        if not pdf:  # When called with SerialExecutor
+    elif worker_pdf is not None:
-            pdf = stack.enter_context(Pdf.open(infile))
+        yield worker_pdf
-        page = PageInfo(pdf, pageno, infile, check_pages, detailed_analysis)
+    else:
-        return page
+        with Pdf.open(infile) as pdf:
+            yield pdf
+
+
+def _pdf_pageinfo_sync(
+    pageno: int,
+    thread_pdf: Pdf | None,
+    infile: Path,
+    check_pages: Container[int],
+    detailed_analysis: bool,
+) -> PageInfo:
+    with _pdf_pageinfo_sync_pdf(thread_pdf, infile) as pdf:
+        return PageInfo(pdf, pageno, infile, check_pages, detailed_analysis)
 
 
 def _pdf_pageinfo_concurrent(
     pdf,
     executor: Executor,
+    max_workers: int,
+    use_threads: bool,
     infile,
     progbar,
-    max_workers,
     check_pages,
-    detailed_analysis=False,
+    detailed_analysis: bool = False,
 ) -> Sequence[PageInfo | None]:
-    pages: Sequence[PageInfo | None] = [None] * len(pdf.pages)
+    pages: list[PageInfo | None] = [None] * len(pdf.pages)
 
-    def update_pageinfo(result, pbar):
+    def update_pageinfo(page: PageInfo, pbar: ProgressBar):
-        page = result
         if not page:
             raise InputFileError("Could read a page in the PDF")
         pages[page.pageno] = page
@@ -721,13 +750,17 @@ def _pdf_pageinfo_concurrent(
 
     total = len(pdf.pages)
 
-    use_threads = False  # No performance gain if threaded due to GIL
     n_workers = min(1 + len(pages) // 4, max_workers)
     if n_workers == 1:
-        # But if we decided on only one worker, there is no point in using
+        # If we decided on only one worker, there is no point in using
         # a separate process.
         use_threads = True
 
+    if use_threads and n_workers > 1:
+        # If we are using threads, there is no point in using more than one
+        # worker thread - they will just fight over the GIL.
+        n_workers = 1
+
     # If we use a thread, we can pass the already-open Pdf for them to use
     # If we use processes, we pass a None which tells the init function to open its
     # own
@@ -737,10 +770,15 @@ def _pdf_pageinfo_concurrent(
         (n, initial_pdf, infile, check_pages, detailed_analysis) for n in range(total)
     )
     assert n_workers == 1 if use_threads else n_workers >= 1, "Not multithreadable"
+    logger.debug(
+        f"Gathering info with {n_workers} "
+        + ('thread' if use_threads else 'process')
+        + " workers"
+    )
     executor(
         use_threads=use_threads,
         max_workers=n_workers,
-        tqdm_kwargs=dict(
+        progress_kwargs=dict(
             total=total, desc="Scanning contents", unit='page', disable=not progbar
         ),
         worker_initializer=partial(
@@ -812,7 +850,7 @@ class PageInfo:
         detailed_analysis: bool,
     ):
         page: Page = pdf.pages[pageno]
-        mediabox = [Decimal(d) for d in page.MediaBox.as_list()]
+        mediabox = [Decimal(d) for d in page.mediabox.as_list()]
         width_pt = mediabox[2] - mediabox[0]
         height_pt = mediabox[3] - mediabox[1]
 
@@ -821,7 +859,10 @@ class PageInfo:
         if check_this_page and detailed_analysis:
             pscript5_mode = str(pdf.docinfo.get(Name.Creator)).startswith('PScript5')
             miner = get_page_analysis(infile, pageno, pscript5_mode)
-            self._textboxes = list(simplify_textboxes(miner, get_text_boxes))
+            if miner is not None:
+                self._textboxes = list(simplify_textboxes(miner, get_text_boxes))
+            else:
+                self._textboxes = []
             bboxes = (box.bbox for box in self._textboxes)
 
             self._has_text = _page_has_text(bboxes, width_pt, height_pt)
@@ -937,7 +978,9 @@ class PageInfo:
     def get_textareas(self, visible: bool | None = None, corrupt: bool | None = None):
         """Return textareas bounding boxes in PDF coordinates on the page."""
 
-        def predicate(obj, want_visible, want_corrupt):
+        def predicate(
+            obj: TextboxInfo, want_visible: bool | None, want_corrupt: bool | None
+        ) -> bool:
             result = True
             if want_visible is not None:
                 if obj.is_visible != want_visible:
@@ -1028,7 +1071,11 @@ DEFAULT_EXECUTOR = SerialExecutor()
 
 
 class PdfInfo:
-    """Get summary information about a PDF."""
+    """Extract summary information about a PDF without retaining the PDF itself.
+
+    Crucially this lets us get the information in a pure Python format so that
+    it can be pickled and passed to a worker process.
+    """
 
     _has_acroform: bool = False
     _has_signature: bool = False
@@ -1036,11 +1083,12 @@ class PdfInfo:
 
     def __init__(
         self,
-        infile,
+        infile: Path,
         *,
         detailed_analysis: bool = False,
         progbar: bool = False,
         max_workers: int | None = None,
+        use_threads: bool = True,
         check_pages=None,
         executor: Executor = DEFAULT_EXECUTOR,
     ):
@@ -1055,9 +1103,10 @@ class PdfInfo:
             self._pages = _pdf_pageinfo_concurrent(
                 pdf,
                 executor,
+                max_workers,
+                use_threads,
                 infile,
                 progbar,
-                max_workers,
                 check_pages=check_pages,
                 detailed_analysis=detailed_analysis,
             )
@@ -1068,6 +1117,9 @@ class PdfInfo:
                 elif Name.XFA in pdf.Root.AcroForm:
                     self._has_acroform = True
                 self._has_signature = bool(pdf.Root.AcroForm.get(Name.SigFlags, 0) & 1)
+            self._is_tagged = bool(
+                pdf.Root.get(Name.MarkInfo, {}).get(Name.Marked, False)
+            )
 
     @property
     def pages(self) -> Sequence[PageInfo | None]:
@@ -1095,6 +1147,11 @@ class PdfInfo:
         """Return True if the document annotations has a digital signature."""
         return self._has_signature
 
+    @property
+    def is_tagged(self) -> bool:
+        """Return True if the document catalog indicates this is a Tagged PDF."""
+        return self._is_tagged
+
     @property
     def filename(self) -> str | Path:
         """Return filename of PDF."""
@@ -1124,7 +1181,7 @@ class PdfInfo:
         return f"<PdfInfo('...'), page count={len(self)}>"
 
 
-def main():
+def main():  # pragma: no cover
     """Run as a script."""
     import argparse  # pylint: disable=import-outside-toplevel
     from pprint import pprint  # pylint: disable=import-outside-toplevel

src/ocrmypdf/pdfinfo/layout.py

@@ -5,8 +5,12 @@
 from __future__ import annotations
 
 import re
+from collections.abc import Mapping
+from contextlib import contextmanager
 from math import copysign
+from os import PathLike
 from pathlib import Path
+from typing import Any, Iterator
 from unittest.mock import patch
 
 import pdfminer
@@ -15,10 +19,13 @@ import pdfminer.pdfdevice
 import pdfminer.pdfinterp
 from pdfminer.converter import PDFLayoutAnalyzer
 from pdfminer.layout import LAParams, LTChar, LTPage, LTTextBox
+from pdfminer.pdfcolor import PDFColorSpace
+from pdfminer.pdfdevice import PDFTextSeq
 from pdfminer.pdfdocument import PDFTextExtractionNotAllowed
-from pdfminer.pdffont import PDFSimpleFont, PDFUnicodeNotDefined
+from pdfminer.pdffont import FontWidthDict, PDFFont, PDFSimpleFont, PDFUnicodeNotDefined
+from pdfminer.pdfinterp import PDFGraphicState, PDFResourceManager, PDFTextState
 from pdfminer.pdfpage import PDFPage
-from pdfminer.utils import bbox2str, matrix2str
+from pdfminer.utils import Matrix, bbox2str, matrix2str
 
 from ocrmypdf.exceptions import EncryptedPdfError, InputFileError
 
@@ -28,7 +35,12 @@ STRIP_NAME = re.compile(r'[0-9]+')
 original_pdfsimplefont_init = PDFSimpleFont.__init__
 
 
-def pdfsimplefont__init__(self, descriptor, widths, spec):
+def pdfsimplefont__init__(
+    self,
+    descriptor: Mapping[str, Any],
+    widths: FontWidthDict,
+    spec: Mapping[str, Any],
+) -> None:
     """Monkeypatch pdfminer.six PDFSimpleFont.__init__.
 
     If there is no ToUnicode and no Encoding, pdfminer.six assumes that Unicode
@@ -44,7 +56,7 @@ def pdfsimplefont__init__(self, descriptor, widths, spec):
     return
 
 
-PDFSimpleFont.__init__ = pdfsimplefont__init__
+setattr(PDFSimpleFont, '__init__', pdfsimplefont__init__)
 
 #
 # pdfminer patches when creator is PScript5.dll
@@ -85,6 +97,11 @@ def pdftype3font__pscript5_get_ascent(self):
     return self.ascent * copysign(1.0, self.vscale)
 
 
+def _is_undefined_char(s: str) -> bool:
+    """Check if a string is an undefined character."""
+    return s.startswith('(cid:') and s.endswith(')')
+
+
 class LTStateAwareChar(LTChar):
     """A subclass of LTChar that tracks text render mode at time of drawing."""
 
@@ -107,18 +124,18 @@ class LTStateAwareChar(LTChar):
 
     def __init__(
         self,
-        matrix,
+        matrix: Matrix,
-        font,
+        font: PDFFont,
-        fontsize,
+        fontsize: float,
-        scaling,
+        scaling: float,
-        rise,
+        rise: float,
-        text,
+        text: str,
-        textwidth,
+        textwidth: float,
-        textdisp,
+        textdisp: float | tuple[float | None, float],
-        ncs,
+        ncs: PDFColorSpace,
-        graphicstate,
+        graphicstate: PDFGraphicState,
-        textstate,
+        textstate: PDFTextState,
-    ):
+    ) -> None:
         """Initialize."""
         super().__init__(
             matrix,
@@ -134,7 +151,7 @@ class LTStateAwareChar(LTChar):
         )
         self.rendermode = textstate.render
 
-    def is_compatible(self, obj):
+    def is_compatible(self, obj: object) -> bool:
         """Check if characters can be combined into a textline.
 
         We consider characters compatible if:
@@ -142,23 +159,22 @@ class LTStateAwareChar(LTChar):
             - the Unicode mapping is unknown but both are part of the same font
         """
         # pylint: disable=protected-access
-        both_unicode_mapped = isinstance(self._text, str) and isinstance(obj._text, str)
+        if not isinstance(obj, LTStateAwareChar):
-        try:
-            if both_unicode_mapped:
-                return self.rendermode == obj.rendermode
-            font0, _ = self._text
-            font1, _ = obj._text
-            return font0 == font1 and self.rendermode == obj.rendermode
-        except (ValueError, AttributeError):
             return False
+        both_unicode_mapped = not _is_undefined_char(
+            self._text
+        ) and not _is_undefined_char(obj._text)
+        if both_unicode_mapped:
+            return self.rendermode == obj.rendermode
+        return self.fontname == obj.fontname and self.rendermode == obj.rendermode
 
-    def get_text(self):
+    def get_text(self) -> str:
         """Get text from this character."""
-        if isinstance(self._text, tuple):
+        if _is_undefined_char(self._text):
             return '\ufffd'  # standard 'Unknown symbol'
         return self._text
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         """Return a string representation of this object."""
         return (
             f"<{self.__class__.__name__} "
@@ -174,19 +190,24 @@ class LTStateAwareChar(LTChar):
 class TextPositionTracker(PDFLayoutAnalyzer):
     """A page layout analyzer that pays attention to text visibility."""
 
-    def __init__(self, rsrcmgr, pageno=1, laparams=None):
+    textstate: PDFTextState
+
+    def __init__(
+        self,
+        rsrcmgr: PDFResourceManager,
+        pageno: int = 1,
+        laparams: LAParams | None = None,
+    ):
         """Initialize the layout analyzer."""
         super().__init__(rsrcmgr, pageno, laparams)
-        self.textstate = None
+        self.result: LTPage | None = None
-        self.result = None
-        self.cur_item = None  # not defined in pdfminer code as it should be
 
-    def begin_page(self, page, ctm):
+    def begin_page(self, page: PDFPage, ctm: Matrix) -> None:
         """Begin processing of a page."""
         super().begin_page(page, ctm)
         self.cur_item = LTPage(self.pageno, page.mediabox)
 
-    def end_page(self, page):
+    def end_page(self, page: PDFPage) -> None:
         """End processing of a page."""
         assert not self._stack, str(len(self._stack))
         assert isinstance(self.cur_item, LTPage), str(type(self.cur_item))
@@ -195,14 +216,28 @@ class TextPositionTracker(PDFLayoutAnalyzer):
         self.pageno += 1
         self.receive_layout(self.cur_item)
 
-    def render_string(self, textstate, seq, ncs, graphicstate):
+    def render_string(
+        self,
+        textstate: PDFTextState,
+        seq: PDFTextSeq,
+        ncs: PDFColorSpace,
+        graphicstate: PDFGraphicState,
+    ) -> None:
         """Respond to render string event by updating text state."""
         self.textstate = textstate.copy()
         super().render_string(self.textstate, seq, ncs, graphicstate)
 
     def render_char(
-        self, matrix, font, fontsize, scaling, rise, cid, ncs, graphicstate
+        self,
-    ):
+        matrix: Matrix,
+        font: PDFFont,
+        fontsize: float,
+        scaling: float,
+        rise: float,
+        cid: int,
+        ncs: PDFColorSpace,
+        graphicstate: PDFGraphicState,
+    ) -> float:
         """Respond to render char event by updating text state."""
         try:
             text = font.to_unichr(cid)
@@ -227,21 +262,34 @@ class TextPositionTracker(PDFLayoutAnalyzer):
         self.cur_item.add(item)
         return item.adv
 
-    def handle_undefined_char(self, font, cid):
+    def receive_layout(self, ltpage: LTPage) -> None:
-        """Handle undefined character."""
-        # log.info('undefined: %r, %r', font, cid)
-        return (font.fontname, cid)
-
-    def receive_layout(self, ltpage):
         """Receive layout handler."""
         self.result = ltpage
 
-    def get_result(self):
+    def get_result(self) -> LTPage | None:
         """Get the result of the analysis."""
         return self.result
 
 
-def get_page_analysis(infile, pageno, pscript5_mode):
+@contextmanager
+def patch_pdfminer(pscript5_mode: bool):
+    """Patch pdfminer.six to work around bugs in PDFs created by PScript5."""
+    if pscript5_mode:
+        with patch.multiple(
+            'pdfminer.pdffont.PDFType3Font',
+            spec=True,
+            get_ascent=pdftype3font__pscript5_get_ascent,
+            get_descent=pdftype3font__pscript5_get_descent,
+            get_height=pdftype3font__pscript5_get_height,
+        ):
+            yield
+    else:
+        yield
+
+
+def get_page_analysis(
+    infile: PathLike, pageno: int, pscript5_mode: bool
+) -> LTPage | None:
     """Get the page analysis for a given page."""
     rman = pdfminer.pdfinterp.PDFResourceManager(caching=True)
     disable_boxes_flow = None
@@ -253,36 +301,23 @@ def get_page_analysis(infile, pageno, pscript5_mode):
     )
     interp = pdfminer.pdfinterp.PDFPageInterpreter(rman, dev)
 
-    patcher = None
+    with patch_pdfminer(pscript5_mode):
-    if pscript5_mode:
+        try:
-        patcher = patch.multiple(
+            with Path(infile).open('rb') as f:
-            'pdfminer.pdffont.PDFType3Font',
+                page_iter = PDFPage.get_pages(f, pagenos=[pageno], maxpages=0)
-            spec=True,
+                page = next(page_iter, None)
-            get_ascent=pdftype3font__pscript5_get_ascent,
+                if page is None:
-            get_descent=pdftype3font__pscript5_get_descent,
+                    raise InputFileError(
-            get_height=pdftype3font__pscript5_get_height,
+                        f"pdfminer could not process page {pageno} (counting from 0)."
-        )
+                    )
-        patcher.start()
+                interp.process_page(page)
-
+        except PDFTextExtractionNotAllowed as e:
-    try:
+            raise EncryptedPdfError() from e
-        with Path(infile).open('rb') as f:
-            page_iter = PDFPage.get_pages(f, pagenos=[pageno], maxpages=0)
-            page = next(page_iter, None)
-            if page is None:
-                raise InputFileError(
-                    f"pdfminer could not process page {pageno} (counting from 0)."
-                )
-            interp.process_page(page)
-    except PDFTextExtractionNotAllowed as e:
-        raise EncryptedPdfError() from e
-    finally:
-        if patcher is not None:
-            patcher.stop()
 
     return dev.get_result()
 
 
-def get_text_boxes(obj):
+def get_text_boxes(obj) -> Iterator[LTTextBox]:
     """Get the text boxes attached to the current node."""
     for child in obj:
         if isinstance(child, (LTTextBox)):

src/ocrmypdf/pluginspec.py

@@ -7,13 +7,15 @@ from __future__ import annotations
 
 from abc import ABC, abstractmethod
 from argparse import ArgumentParser, Namespace
+from collections.abc import Sequence, Set
 from logging import Handler
 from pathlib import Path
-from typing import TYPE_CHECKING, AbstractSet, NamedTuple, Sequence
+from typing import TYPE_CHECKING, NamedTuple
 
 import pluggy
 
 from ocrmypdf import Executor, PdfContext
+from ocrmypdf._progressbar import ProgressBar
 from ocrmypdf.helpers import Resolution
 
 if TYPE_CHECKING:
@@ -101,7 +103,6 @@ def check_options(options: Namespace) -> None:
             and the application should terminate gracefully with an informative
             message and error code.
 
-
     Note:
         This hook will be called from the main process, and may modify global state
         before child worker processes are forked.
@@ -109,7 +110,7 @@ def check_options(options: Namespace) -> None:
 
 
 @hookspec(firstresult=True)
-def get_executor(progressbar_class) -> Executor:
+def get_executor(progressbar_class: type[ProgressBar]) -> Executor:
     """Called to obtain an object that manages parallel execution.
 
     This may be used to replace OCRmyPDF's default parallel execution system
@@ -131,41 +132,24 @@ def get_executor(progressbar_class) -> Executor:
         This hook will be called from the main process, and may modify global state
         before child worker processes are forked.
 
-
     Note:
         This is a :ref:`firstresult hook<firstresult>`.
     """
 
 
 @hookspec(firstresult=True)
-def get_progressbar_class():
+def get_progressbar_class() -> type[ProgressBar]:
     """Called to obtain a class that can be used to monitor progress.
 
-    A progress bar is assumed, but this could be used for any type of monitoring.
+    OCRmyPDF will call this function when it wants to display a progress bar.
-
+    The class returned by this function must be compatible with the
-    The class should follow a tqdm-like protocol. Calling the class should return
+    :class:`ProgressBar` protocol.
-    a new progress bar object, which is activated with ``__enter__`` and terminated
-    ``__exit__``. An update method is called whenever the progress bar is updated.
-    Progress bar objects will not be reused; a new one will be created for each
-    group of tasks.
-
-    The progress bar is held in the main process/thread and not updated by child
-    process/threads. When a child notifies the parent of completed work, the
-    parent updates the progress bar.
-
-    The arguments are the same as `tqdm <https://github.com/tqdm/tqdm>`_ accepts.
-
-    Progress bars should never write to ``sys.stdout``, or they will corrupt the
-    output if OCRmyPDF writes a PDF to standard output.
-
-    The type of events that OCRmyPDF reports to a progress bar may change in
-    minor releases.
 
     Here is how OCRmyPDF will use the progress bar:
 
     Example:
         pbar_class = pm.hook.get_progressbar_class()
-        with pbar_class(**tqdm_kwargs) as pbar:
+        with pbar_class(**progress_kwargs) as pbar:
             ...
             pbar.update(1)
     """
@@ -186,7 +170,6 @@ def validate(pdfinfo: PdfInfo, options: Namespace) -> None:
             and the application should terminate gracefully with an informative
             message and error code.
 
-
     Note:
         This hook will be called from the main process, and may modify global state
         before child worker processes are forked.
@@ -230,6 +213,7 @@ def rasterize_pdf_page(
 
     Returns:
         Path: output_file if successful
+
     Note:
         This hook will be called from child processes. Modifying global state
         will not affect the main process or other child processes.
@@ -269,7 +253,6 @@ def filter_ocr_image(page: PageContext, image: Image.Image) -> Image.Image:
         This hook will be called from child processes. Modifying global state
         will not affect the main process or other child processes.
 
-
     Note:
         This is a :ref:`firstresult hook<firstresult>`.
     """
@@ -307,7 +290,6 @@ def filter_page_image(page: PageContext, image_filename: Path) -> Path:
         This hook will be called from child processes. Modifying global state
         will not affect the main process or other child processes.
 
-
     Note:
         This is a :ref:`firstresult hook<firstresult>`.
     """
@@ -405,7 +387,7 @@ class OcrEngine(ABC):
 
     @staticmethod
     @abstractmethod
-    def languages(options: Namespace) -> AbstractSet[str]:
+    def languages(options: Namespace) -> Set[str]:
         """Returns the set of all languages that are supported by the engine.
 
         Languages are typically given in 3-letter ISO 3166-1 codes, but actually
@@ -427,21 +409,42 @@ class OcrEngine(ABC):
     def generate_hocr(
         input_file: Path, output_hocr: Path, output_text: Path, options: Namespace
     ) -> None:
-        """Called to produce a hOCR file and sidecar text file."""
+        """Called to produce a hOCR file from a page image and sidecar text file.
+
+        A hOCR file is an HTML-like file that describes the position of text on a
+        page. OCRmyPDF can create a text only PDF from the hOCR file and graft it
+        onto the output PDF.
+
+        This function executes in a worker thread or worker process. OCRmyPDF
+        automatically parallelizes OCR over pages. The OCR engine should not
+        introduce more parallelism.
+
+        Args:
+            input_file: A page image on which to perform OCR.
+            output_hocr: The expected name of the output hOCR file.
+            output_text: The expected name of a text file containing the
+                recognized text.
+            options: The command line options.
+        """
 
     @staticmethod
     @abstractmethod
     def generate_pdf(
         input_file: Path, output_pdf: Path, output_text: Path, options: Namespace
     ) -> None:
-        """Called to produce a text only PDF.
+        """Called to produce a text only PDF from a page image.
+
+        A text only PDF should contain no visible material of any kind, as it
+        will be grafted onto the input PDF page. It must be sized to the
+        exact dimensions of the input image.
+
+        This function executes in a worker thread or worker process. OCRmyPDF
+        automatically parallelizes OCR over pages. The OCR engine should not
+        introduce more parallelism.
 
         Args:
             input_file: A page image on which to perform OCR.
-            output_pdf: The expected name of the output PDF, which must be
+            output_pdf: The expected name of the output PDF.
-                a single page PDF with no visible content of any kind, sized
-                to the dimensions implied by the input_file's width, height
-                and DPI. The image will be grafted onto the input PDF page.
             output_text: The expected name of a text file containing the
                 recognized text.
             options: The command line options.
@@ -468,7 +471,7 @@ def generate_pdfa(
     context: PdfContext,
     pdf_version: str,
     pdfa_part: str,
-    progressbar_class,
+    progressbar_class: type[ProgressBar] | None,
     stop_on_soft_error: bool,
 ) -> Path:
     """Generate a PDF/A.
@@ -488,14 +491,8 @@ def generate_pdfa(
             At its own discretion, the PDF/A generator may raise the version,
             but should not lower it.
         pdfa_part: The desired PDF/A compliance level, such as ``'2B'``.
-        progressbar_class: The class of a progress bar with a tqdm-like API. An
+        progressbar_class: The class of a progress bar, which must implement
-            instance of this class will be initialized when PDF/A conversion
+            the ProgressBar protocol. If None, no progress is reported.
-            begins, using
-            ``instance = progressbar_class(total: int, desc: str, unit:str)``,
-            defining the number of work units, a user-visible description,
-            and the name of the work units ("page"). Then ``instance.update()``
-            will be called when a work unit is completed. If ``None``, no
-            progress information is reported.
         stop_on_soft_error: If there is an "soft error" such that PDF/A generation
             can proceed and produce a valid PDF/A, but output may be invalid or
             may not visually resemble the original, the implementer of this hook
@@ -513,9 +510,6 @@ def generate_pdfa(
         Before version 15.0.0, the ``context`` was not provided and ``compression``
         was provided instead. Plugins should now read the context object to determine
         if compression is requested.
-
-    See Also:
-        https://github.com/tqdm/tqdm
     """
 
 

src/ocrmypdf/quality.py

@@ -7,7 +7,7 @@
 from __future__ import annotations
 
 import re
-from typing import Iterable
+from collections.abc import Iterable
 
 
 class OcrQualityDictionary:

src/ocrmypdf/subprocess/__init__.py

@@ -8,11 +8,12 @@ import logging
 import os
 import re
 import sys
+from collections.abc import Mapping, Sequence
 from contextlib import suppress
 from pathlib import Path
 from subprocess import PIPE, STDOUT, CalledProcessError, CompletedProcess, Popen
 from subprocess import run as subprocess_run
-from typing import Callable, Mapping, Sequence, Union
+from typing import Callable, Union
 
 from packaging.version import Version
 

src/ocrmypdf/subprocess/_windows.py

@@ -9,9 +9,10 @@ import os
 import re
 import shutil
 import sys
+from collections.abc import Iterable, Iterator
 from itertools import chain
 from pathlib import Path
-from typing import Any, Callable, Iterable, Iterator, TypeVar
+from typing import Any, Callable, TypeVar
 
 from packaging.version import InvalidVersion, Version
 

tests/cache/multipage/__-l__eng__000002_ocr.png__000002_ocr_hocr__hocr__txt/hocr.bin

@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+  <title></title>
+  <meta http-equiv="Content-Type" content="text/html;charset=utf-8"/>
+  <meta name='ocr-system' content='tesseract 5.3.2' />
+  <meta name='ocr-capabilities' content='ocr_page ocr_carea ocr_par ocr_line ocrx_word ocrp_wconf'/>
+ </head>
+ <body>
+  <div class='ocr_page' id='page_1' title='image "/tmp/pytest-of-jb/pytest-25/popen-gw2/test_hocr_api0/000002_ocr.png"; bbox 0 0 9000 9000; ppageno 0; scan_res 300 300'>
+   <div class='ocr_carea' id='block_1_1' title="bbox 791 740 4415 1463">
+    <p class='ocr_par' id='par_1_1' lang='eng' title="bbox 791 740 4415 1463">
+     <span class='ocr_line' id='line_1_1' title="bbox 793 740 4415 1009; baseline -0.001 -51; x_size 274.5; x_descenders 57.5; x_ascenders 55.5">
+      <span class='ocrx_word' id='word_1_1' title='bbox 793 744 2238 958; x_wconf 34'>9OO0Ox9000</span>
+      <span class='ocrx_word' id='word_1_2' title='bbox 2350 740 3016 1009; x_wconf 92'>pixels</span>
+      <span class='ocrx_word' id='word_1_3' title='bbox 3120 752 3334 958; x_wconf 95'>at</span>
+      <span class='ocrx_word' id='word_1_4' title='bbox 3432 744 3914 958; x_wconf 75'>GOO</span>
+      <span class='ocrx_word' id='word_1_5' title='bbox 4025 740 4415 954; x_wconf 64'>DPI</span>
+     </span>
+     <span class='ocr_line' id='line_1_2' title="bbox 791 1190 2588 1463; baseline 0 -55; x_size 273; x_descenders 57; x_ascenders 55">
+      <span class='ocrx_word' id='word_1_6' title='bbox 791 1194 1045 1408; x_wconf 57'>S|]</span>
+      <span class='ocrx_word' id='word_1_7' title='bbox 1213 1190 2588 1463; x_wconf 59'>megapixels</span>
+     </span>
+    </p>
+   </div>
+  </div>
+ </body>
+</html>

tests/cache/multipage/__-l__eng__000002_ocr.png__000002_ocr_hocr__hocr__txt/txt.bin

@@ -0,0 +1,2 @@
+9OO0Ox9000 pixels at GOO DPI
+S|] megapixels

tests/conftest.py

@@ -25,9 +25,8 @@ def is_macos():
 
 
 def running_in_docker():
-    # Docker creates a file named /.dockerenv (newer versions) or
+    # Docker creates a file named /.dockerenv in all supported versions
-    # /.dockerinit (older) -- this is undocumented, not an offical test
+    return Path('/.dockerenv').exists()
-    return Path('/.dockerenv').exists() or Path('/.dockerinit').exists()
 
 
 def have_unpaper():
@@ -92,7 +91,7 @@ def check_ocrmypdf(input_file: Path, output_file: Path, *args) -> Path:
 
     _parser, options, plugin_manager = get_parser_options_plugins(args=api_args)
     api.check_options(options, plugin_manager)
-    result = api.run_pipeline(options, plugin_manager=plugin_manager, api=True)
+    result = api.run_pipeline(options, plugin_manager=plugin_manager)
 
     assert result == 0
     assert output_file.exists(), "Output file not created"
@@ -102,14 +101,14 @@ def check_ocrmypdf(input_file: Path, output_file: Path, *args) -> Path:
 
 
 def run_ocrmypdf_api(input_file: Path, output_file: Path, *args) -> ExitCode:
-    """Run ocrmypdf via its API in-process, and let test deal with results.
+    """Run ocrmypdf via its API in-process, but return CLI-style ExitCode.
 
-    This simulates calling the command line interface in a subprocess, but
+    This simulates calling the command line interface in a subprocess and allows us
-    is easier for debuggers and code coverage to follow.
+    to check that the command line interface is working correctly, but since it is
+    in-process it is easier to trace with a debugger or coverage tool.
 
     Any exception raised will be trapped and converted to an exit code.
-    The return code must always be checked or the test may declare a failure
+    The return code must be checked by the caller to determine if the test passed.
-    to be pass.
     """
     api_args = [str(input_file), str(output_file)] + [
         str(arg) for arg in args if arg is not None
@@ -117,7 +116,7 @@ def run_ocrmypdf_api(input_file: Path, output_file: Path, *args) -> ExitCode:
     _parser, options, plugin_manager = get_parser_options_plugins(args=api_args)
 
     api.check_options(options, plugin_manager)
-    return api.run_pipeline(options, plugin_manager=None, api=False)
+    return api.run_pipeline_cli(options, plugin_manager=plugin_manager)
 
 
 def run_ocrmypdf(

tests/plugins/gs_raster_failure.py

@@ -13,7 +13,7 @@ from ocrmypdf.builtin_plugins import ghostscript
 
 def raise_gs_fail(*args, **kwargs):
     raise CalledProcessError(
-        1, 'gs', output=b"", stderr=b"ERROR: Ghost story archive not found"
+        1, 'gs', output=b"", stderr=b"TEST ERROR: gs_raster_failure.py"
     )
 
 

tests/plugins/gs_render_failure.py

@@ -12,7 +12,7 @@ from ocrmypdf.builtin_plugins import ghostscript
 
 def raise_gs_fail(*args, **kwargs):
     raise CalledProcessError(
-        1, 'gs', output=b"", stderr=b"ERROR: Casper is not a friendly ghost"
+        1, 'gs', output=b"", stderr=b"TEST ERROR: gs_render_failure.py"
     )
 
 

tests/resources/README.rst

@@ -19,9 +19,6 @@ the copyright holder(s) and license(s) applicable to these resources.
     *   - c02-22.pdf
         - `Project Gutenberg`_, Adventures of Huckleberry Finn, page 22
         - difficult OCR image (obscure fonts and illustrations)
-    *   - congress.jpg
-        - `US Congressional Records`_
-        - difficult OCR image (color background)
     *   - graph.pdf
         - `Wikimedia:Simple_line_graph_of_ACE_2012_results_by_candidate_sj01.png`_
         - image with slanted text
@@ -125,8 +122,6 @@ These test resources are assemblies or derivatives from other previously mention
 
 .. _`Project Gutenberg`: https://www.gutenberg.org/files/76/76-h/76-h.htm#c2
 
-.. _`US Congressional Records`: http://www.baxleystamps.com/litho/meiji/courts_1871.jpg
-
 .. _`Wikimedia: Simple_line_graph_of_ACE_2012_results_by_candidate_sj01.png`: https://en.wikipedia.org/wiki/File:Simple_line_graph_of_ACE_2012_results_by_candidate_sj01.png
 
 .. _`Wikimedia: JPEG2000 Lichtenstein`: https://en.wikipedia.org/wiki/JPEG_2000#/media/File:Jpeg2000_2-level_wavelet_transform-lichtenstein.png

tests/test_api.py

@@ -3,12 +3,14 @@
 
 from __future__ import annotations
 
-import logging
+from io import BytesIO
-from io import BytesIO, StringIO
+from pathlib import Path
 
 import pytest
+from pdfminer.high_level import extract_text
 
 import ocrmypdf
+import ocrmypdf.api
 
 
 def test_language_list():
@@ -18,10 +20,45 @@ def test_language_list():
         ocrmypdf.ocr('doesnotexist.pdf', '_.pdf', language=['eng', 'deu'])
 
 
-def test_stream_api(resources):
+def test_stream_api(resources: Path):
     in_ = (resources / 'graph.pdf').open('rb')
     out = BytesIO()
 
     ocrmypdf.ocr(in_, out, tesseract_timeout=0.0)
     out.seek(0)
     assert b'%PDF' in out.read(1024)
+
+
+def test_hocr_api_multipage(resources: Path, outdir: Path, outpdf: Path):
+    ocrmypdf.api._pdf_to_hocr(
+        resources / 'multipage.pdf',
+        outdir,
+        language='eng',
+        skip_text=True,
+        plugins=['tests/plugins/tesseract_cache.py'],
+    )
+    assert (outdir / '000001_ocr_hocr.hocr').exists()
+    assert (outdir / '000006_ocr_hocr.hocr').exists()
+    assert not (outdir / '000004_ocr_hocr.hocr').exists()
+
+    ocrmypdf.api._hocr_to_ocr_pdf(outdir, outpdf)
+    assert outpdf.exists()
+
+
+def test_hocr_to_pdf_api(resources: Path, outdir: Path, outpdf: Path):
+    ocrmypdf.api._pdf_to_hocr(
+        resources / 'ccitt.pdf',
+        outdir,
+        language='eng',
+        skip_text=True,
+        plugins=['tests/plugins/tesseract_cache.py'],
+    )
+    assert (outdir / '000001_ocr_hocr.hocr').exists()
+    hocr = (outdir / '000001_ocr_hocr.hocr').read_text(encoding='utf-8')
+    mangled = hocr.replace('the', 'hocr')
+    (outdir / '000001_ocr_hocr.hocr').write_text(mangled, encoding='utf-8')
+
+    ocrmypdf.api._hocr_to_ocr_pdf(outdir, outpdf, optimize=0)
+
+    text = extract_text(outpdf)
+    assert 'hocr' in text and 'the' not in text

tests/test_concurrency.py

@@ -12,13 +12,13 @@ from ocrmypdf import ExitCode
 from .conftest import run_ocrmypdf_api
 
 
-@pytest.mark.skipif(True, reason="--use-threads is currently default")
 @pytest.mark.skipif(os.name == 'nt', reason="Windows doesn't have SIGKILL")
 def test_simulate_oom_killer(multipage, no_outpdf):
     exitcode = run_ocrmypdf_api(
         multipage,
         no_outpdf,
         '--force-ocr',
+        '--no-use-threads',
         '--plugin',
         'tests/plugins/tesseract_simulate_oom_killer.py',
     )

tests/test_ghostscript.py

@@ -4,6 +4,7 @@
 from __future__ import annotations
 
 import logging
+import secrets
 import subprocess
 from decimal import Decimal
 from unittest.mock import patch
@@ -13,10 +14,10 @@ import pytest
 from PIL import Image, UnidentifiedImageError
 
 from ocrmypdf._exec.ghostscript import DuplicateFilter, rasterize_pdf
-from ocrmypdf.exceptions import ExitCode
+from ocrmypdf.exceptions import ColorConversionNeededError, ExitCode
 from ocrmypdf.helpers import Resolution
 
-from .conftest import check_ocrmypdf, run_ocrmypdf
+from .conftest import check_ocrmypdf, run_ocrmypdf_api
 
 # pylint: disable=redefined-outer-name
 
@@ -29,8 +30,8 @@ def francais(resources):
 
 def test_rasterize_size(francais, outdir):
     path, pdf = francais
-    page_size_pts = (pdf.pages[0].MediaBox[2], pdf.pages[0].MediaBox[3])
+    page_size_pts = (pdf.pages[0].mediabox[2], pdf.pages[0].mediabox[3])
-    assert pdf.pages[0].MediaBox[0] == pdf.pages[0].MediaBox[1] == 0
+    assert pdf.pages[0].mediabox[0] == pdf.pages[0].mediabox[1] == 0
     page_size = (page_size_pts[0] / Decimal(72), page_size_pts[1] / Decimal(72))
     target_size = Decimal('50.0'), Decimal('30.0')
     forced_dpi = Resolution(42.0, 4242.0)
@@ -52,8 +53,8 @@ def test_rasterize_size(francais, outdir):
 
 def test_rasterize_rotated(francais, outdir, caplog):
     path, pdf = francais
-    page_size_pts = (pdf.pages[0].MediaBox[2], pdf.pages[0].MediaBox[3])
+    page_size_pts = (pdf.pages[0].mediabox[2], pdf.pages[0].mediabox[3])
-    assert pdf.pages[0].MediaBox[0] == pdf.pages[0].MediaBox[1] == 0
+    assert pdf.pages[0].mediabox[0] == pdf.pages[0].mediabox[1] == 0
     page_size = (page_size_pts[0] / Decimal(72), page_size_pts[1] / Decimal(72))
     target_size = Decimal('50.0'), Decimal('30.0')
     forced_dpi = Resolution(42.0, 4242.0)
@@ -75,8 +76,8 @@ def test_rasterize_rotated(francais, outdir, caplog):
         assert im.info['dpi'] == forced_dpi.flip_axis()
 
 
-def test_gs_render_failure(resources, outpdf):
+def test_gs_render_failure(resources, outpdf, caplog):
-    p = run_ocrmypdf(
+    exitcode = run_ocrmypdf_api(
         resources / 'blank.pdf',
         outpdf,
         '--plugin',
@@ -84,12 +85,12 @@ def test_gs_render_failure(resources, outpdf):
         '--plugin',
         'tests/plugins/gs_render_failure.py',
     )
-    assert 'Casper is not a friendly ghost' in p.stderr
+    assert 'TEST ERROR: gs_render_failure.py' in caplog.text
-    assert p.returncode == ExitCode.child_process_error
+    assert exitcode == ExitCode.child_process_error
 
 
-def test_gs_raster_failure(resources, outpdf):
+def test_gs_raster_failure(resources, outpdf, caplog):
-    p = run_ocrmypdf(
+    exitcode = run_ocrmypdf_api(
         resources / 'francais.pdf',
         outpdf,
         '--plugin',
@@ -97,12 +98,12 @@ def test_gs_raster_failure(resources, outpdf):
         '--plugin',
         'tests/plugins/gs_raster_failure.py',
     )
-    assert 'Ghost story archive not found' in p.stderr
+    assert 'TEST ERROR: gs_raster_failure.py' in caplog.text
-    assert p.returncode == ExitCode.child_process_error
+    assert exitcode == ExitCode.child_process_error
 
 
-def test_ghostscript_pdfa_failure(resources, outpdf):
+def test_ghostscript_pdfa_failure(resources, outpdf, caplog):
-    p = run_ocrmypdf(
+    exitcode = run_ocrmypdf_api(
         resources / 'francais.pdf',
         outpdf,
         '--plugin',
@@ -111,7 +112,7 @@ def test_ghostscript_pdfa_failure(resources, outpdf):
         'tests/plugins/gs_pdfa_failure.py',
     )
     assert (
-        p.returncode == ExitCode.pdfa_conversion_failed
+        exitcode == ExitCode.pdfa_conversion_failed
     ), "Unexpected return when PDF/A fails"
 
 
@@ -126,6 +127,16 @@ def test_ghostscript_feature_elision(resources, outpdf):
     )
 
 
+def test_ghostscript_mandatory_color_conversion(resources, outpdf):
+    with pytest.raises(ColorConversionNeededError):
+        check_ocrmypdf(
+            resources / 'jbig2_baddevicen.pdf',
+            outpdf,
+            '--plugin',
+            'tests/plugins/tesseract_noop.py',
+        )
+
+
 def test_rasterize_pdf_errors(resources, no_outpdf, caplog):
     with patch('ocrmypdf._exec.ghostscript.run') as mock:
         # ghostscript can produce
@@ -144,9 +155,10 @@ def test_rasterize_pdf_errors(resources, no_outpdf, caplog):
 
 
 class TestDuplicateFilter:
-    @pytest.fixture(scope='class', autouse=True)
+    @pytest.fixture(scope='function')
     def duplicate_filter_logger(self):
-        logger = logging.getLogger(__name__)
+        # token_urlsafe: ensure the logger has a unique name so tests are isolated
+        logger = logging.getLogger(__name__ + secrets.token_urlsafe(8))
         logger.setLevel(logging.DEBUG)
         logger.addFilter(DuplicateFilter(logger))
         return logger
@@ -162,9 +174,9 @@ class TestDuplicateFilter:
 
         assert len(caplog.records) == 5
         assert caplog.records[0].msg == "test error message"
-        assert caplog.records[1].msg == "(previous message repeated 2 times)"
+        assert caplog.records[1].msg == "(suppressed 2 repeated lines)"
         assert caplog.records[2].msg == "another error message"
-        assert caplog.records[3].msg == "(previous message repeated 1 times)"
+        assert caplog.records[3].msg == "(suppressed 1 repeated lines)"
         assert caplog.records[4].msg == "yet another error message"
 
     def test_filter_does_not_affect_unique_messages(
@@ -179,3 +191,20 @@ class TestDuplicateFilter:
         assert caplog.records[0].msg == "test error message"
         assert caplog.records[1].msg == "another error message"
         assert caplog.records[2].msg == "yet another error message"
+
+    def test_filter_alt_messages(self, duplicate_filter_logger, caplog):
+        log = duplicate_filter_logger
+        log.error("test error message")
+        log.error("another error message")
+        log.error("test error message")
+        log.error("another error message")
+        log.error("test error message")
+        log.error("test error message")
+        log.error("another error message")
+        log.error("yet another error message")
+
+        assert len(caplog.records) == 4
+        assert caplog.records[0].msg == "test error message"
+        assert caplog.records[1].msg == "another error message"
+        assert caplog.records[2].msg == "(suppressed 5 repeated lines)"
+        assert caplog.records[3].msg == "yet another error message"

tests/test_hocrtransform.py

@@ -16,7 +16,7 @@ from pdfminer.pdfparser import PDFParser
 from PIL import Image
 
 from ocrmypdf import hocrtransform
-from ocrmypdf._exec.tesseract import HOCR_TEMPLATE
+from ocrmypdf._exec.tesseract import generate_hocr
 from ocrmypdf.helpers import check_pdf
 
 from .conftest import check_ocrmypdf
@@ -40,9 +40,22 @@ def text_from_pdf(filename):
 
 @pytest.fixture
 def blank_hocr(tmp_path):
-    filename = tmp_path / "blank.hocr"
+    im = Image.new('1', (8, 8), 0)
-    filename.write_text(HOCR_TEMPLATE)
+    im.save(tmp_path / 'blank.tif', format='TIFF')
-    return filename
+    generate_hocr(
+        input_file=tmp_path / 'blank.tif',
+        output_hocr=tmp_path / 'blank.hocr',
+        output_text=tmp_path / 'blank.txt',
+        languages=['eng'],
+        engine_mode=1,
+        tessconfig=[],
+        pagesegmode=3,
+        thresholding=0,
+        user_words=None,
+        user_patterns=None,
+        timeout=None,
+    )
+    return tmp_path / 'blank.hocr'
 
 
 def test_mono_image(blank_hocr, outdir):

tests/test_image_input.py

@@ -75,7 +75,7 @@ def test_img2pdf_fails(resources, no_outpdf):
 @pytest.mark.xfail(reason="remove background disabled")
 def test_jpeg_in_jpeg_out(resources, outpdf):
     check_ocrmypdf(
-        resources / 'congress.jpg',
+        resources / 'baiona_color.jpg',
         outpdf,
         '--image-dpi',
         '100',

tests/test_logging.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 
 import logging
 
-from ocrmypdf._sync import configure_debug_logging
+from ocrmypdf._pipelines._common import configure_debug_logging
 
 
 def test_debug_logging(tmp_path):
@@ -13,6 +13,6 @@ def test_debug_logging(tmp_path):
     # See https://github.com/pytest-dev/pytest/issues/5502 for pytest logging quirks
     prefix = 'test_debug_logging'
     log = logging.getLogger(prefix)
-    handler = configure_debug_logging(tmp_path / 'test.log', prefix)
+    _handler, remover = configure_debug_logging(tmp_path / 'test.log', prefix)
     log.info("test message")
-    log.removeHandler(handler)
+    remover()

tests/test_main.py

@@ -16,7 +16,7 @@ from PIL import Image
 
 import ocrmypdf
 from ocrmypdf._exec import tesseract
-from ocrmypdf.exceptions import ExitCode, MissingDependencyError
+from ocrmypdf.exceptions import ExitCode, MissingDependencyError, OutputFileAccessError
 from ocrmypdf.pdfa import file_claims_pdfa
 from ocrmypdf.pdfinfo import Colorspace, Encoding, PdfInfo
 from ocrmypdf.subprocess import get_version
@@ -201,14 +201,14 @@ def test_force_ocr_on_pdf_with_no_images(resources, no_outpdf):
     # As a correctness test, make sure that --force-ocr on a PDF with no
     # content still triggers tesseract. If tesseract crashes, then it was
     # called.
-    p = run_ocrmypdf(
+    exitcode = run_ocrmypdf_api(
         resources / 'blank.pdf',
         no_outpdf,
         '--force-ocr',
         '--plugin',
         'tests/plugins/tesseract_crash.py',
     )
-    assert p.returncode == ExitCode.child_process_error
+    assert exitcode == ExitCode.child_process_error
     assert not no_outpdf.exists()
 
 
@@ -239,8 +239,8 @@ def test_german(resources, outdir):
 
 
 def test_klingon(resources, outpdf):
-    p = run_ocrmypdf(resources / 'francais.pdf', outpdf, '-l', 'klz')
+    with pytest.raises(MissingDependencyError):
-    assert p.returncode == ExitCode.missing_dependency
+        run_ocrmypdf_api(resources / 'francais.pdf', outpdf, '-l', 'klz')
 
 
 def test_missing_docinfo(resources, outpdf):
@@ -345,8 +345,8 @@ def test_tesseract_thresholding_invalid(value, resources, no_outpdf):
 
 
 @pytest.mark.parametrize('renderer', RENDERERS)
-def test_tesseract_crash(renderer, resources, no_outpdf):
+def test_tesseract_crash(renderer, resources, no_outpdf, caplog):
-    p = run_ocrmypdf(
+    exitcode = run_ocrmypdf_api(
         resources / 'ccitt.pdf',
         no_outpdf,
         '-v',
@@ -356,24 +356,22 @@ def test_tesseract_crash(renderer, resources, no_outpdf):
         '--plugin',
         'tests/plugins/tesseract_crash.py',
     )
-    assert p.returncode == ExitCode.child_process_error
+    assert exitcode == ExitCode.child_process_error
     assert not no_outpdf.exists()
-    assert "SubprocessOutputError" in p.stderr
+    assert "SubprocessOutputError" in caplog.text
 
 
-def test_tesseract_crash_autorotate(resources, no_outpdf):
+def test_tesseract_crash_autorotate(resources, no_outpdf, caplog):
-    p = run_ocrmypdf(
+    exitcode = run_ocrmypdf_api(
         resources / 'ccitt.pdf',
         no_outpdf,
         '-r',
         '--plugin',
         'tests/plugins/tesseract_crash.py',
     )
-    assert p.returncode == ExitCode.child_process_error
+    assert exitcode == ExitCode.child_process_error
     assert not no_outpdf.exists()
-    assert "uncaught exception" in p.stderr
+    assert "uncaught exception" in caplog.text
-    print(p.stdout)
-    print(p.stderr)
 
 
 @pytest.mark.parametrize('renderer', RENDERERS)
@@ -482,13 +480,13 @@ def protected_file(outdir):
     os.name == 'nt' or os.geteuid() == 0, reason="root can write to anything"
 )
 def test_destination_not_writable(resources, protected_file):
-    p = run_ocrmypdf(
+    exitcode = run_ocrmypdf_api(
         resources / 'jbig2.pdf',
         protected_file,
         '--plugin',
         'tests/plugins/tesseract_noop.py',
     )
-    assert p.returncode == ExitCode.file_access_error, "Expected error"
+    assert exitcode == ExitCode.file_access_error
 
 
 @pytest.fixture
@@ -625,7 +623,7 @@ def test_no_contents(resources, outpdf):
 
 
 @pytest.mark.parametrize(
-    'image', ['baiona.png', 'baiona_gray.png', 'baiona_alpha.png', 'congress.jpg']
+    'image', ['baiona.png', 'baiona_gray.png', 'baiona_alpha.png', 'baiona_color.jpg']
 )
 def test_compression_preserved(ocrmypdf_exec, resources, image, outpdf):
     input_file = str(resources / image)
@@ -681,7 +679,7 @@ def test_compression_preserved(ocrmypdf_exec, resources, image, outpdf):
     [
         ('baiona.png', 'jpeg'),
         ('baiona_gray.png', 'lossless'),
-        ('congress.jpg', 'lossless'),
+        ('baiona_color.jpg', 'lossless'),
     ],
 )
 def test_compression_changed(ocrmypdf_exec, resources, image, compression, outpdf):
@@ -791,15 +789,18 @@ def test_pdfa_n(pdfa_level, resources, outpdf):
     assert pdfa_info['conformance'] == f'PDF/A-{pdfa_level}B'
 
 
-def test_decompression_bomb_error(resources, outpdf):
+def test_decompression_bomb_error(resources, outpdf, caplog):
-    p = run_ocrmypdf(resources / 'hugemono.pdf', outpdf)
+    run_ocrmypdf_api(resources / 'hugemono.pdf', outpdf)
-    assert 'decompression bomb' in p.stderr and '--max-image-mpixels' in p.stderr
+    assert 'decompression bomb' in caplog.text
+    assert 'max-image-mpixels' in caplog.text
 
 
 @pytest.mark.slow
 def test_decompression_bomb_succeeds(resources, outpdf):
-    p = run_ocrmypdf(resources / 'hugemono.pdf', outpdf, '--max-image-mpixels', '2000')
+    exitcode = run_ocrmypdf_api(
-    assert p.returncode == 0
+        resources / 'hugemono.pdf', outpdf, '--max-image-mpixels', '2000'
+    )
+    assert exitcode == 0
 
 
 def test_text_curves(resources, outpdf):
@@ -829,37 +830,37 @@ def test_text_curves_force(resources, outpdf):
         assert len(info.pages[0].images) != 0, "force did not rasterize"
 
 
-def test_output_is_dir(resources, outdir):
+def test_output_is_dir(resources, outdir, caplog):
-    p = run_ocrmypdf(
+    exitcode = run_ocrmypdf_api(
         resources / 'trivial.pdf',
         outdir,
         '--force-ocr',
         '--plugin',
         'tests/plugins/tesseract_noop.py',
     )
-    assert p.returncode == ExitCode.file_access_error
+    assert exitcode == ExitCode.file_access_error
-    assert 'is not a writable file' in p.stderr
+    assert 'is not a writable file' in caplog.text
 
 
 @pytest.mark.skipif(os.name == 'nt', reason="symlink needs admin permissions")
 def test_output_is_symlink(resources, outdir):
     sym = Path(outdir / 'this_is_a_symlink')
     sym.symlink_to(outdir / 'out.pdf')
-    p = run_ocrmypdf(
+    exitcode = run_ocrmypdf_api(
         resources / 'trivial.pdf',
         sym,
         '--force-ocr',
         '--plugin',
         'tests/plugins/tesseract_noop.py',
     )
-    assert p.returncode == ExitCode.ok, p.stderr
+    assert exitcode == ExitCode.ok
     assert (outdir / 'out.pdf').stat().st_size > 0, 'target file not created'
 
 
-def test_livecycle(resources, no_outpdf):
+def test_livecycle(resources, no_outpdf, caplog):
-    p = run_ocrmypdf(resources / 'livecycle.pdf', no_outpdf)
+    exitcode = run_ocrmypdf_api(resources / 'livecycle.pdf', no_outpdf)
 
-    assert p.returncode == ExitCode.input_file, p.stderr
+    assert exitcode == ExitCode.input_file, caplog.text
 
 
 def test_version_check():
@@ -928,7 +929,7 @@ def test_outputtype_none_bad_setup(resources, outpdf):
 
 
 def test_outputtype_none(resources, outtxt):
-    p = run_ocrmypdf(
+    exitcode = run_ocrmypdf_api(
         resources / 'trivial.pdf',
         '-',
         '--output-type=none',
@@ -937,7 +938,8 @@ def test_outputtype_none(resources, outtxt):
         '--plugin',
         'tests/plugins/tesseract_noop.py',
     )
-    assert p.returncode == ExitCode.ok
+    assert exitcode == ExitCode.ok
+    assert outtxt.exists()
 
 
 @pytest.fixture

tests/test_metadata.py

@@ -15,13 +15,14 @@ from pikepdf.models.metadata import decode_pdf_date
 
 from ocrmypdf._exec import ghostscript
 from ocrmypdf._jobcontext import PdfContext
-from ocrmypdf._pipeline import convert_to_pdfa, metadata_fixup
+from ocrmypdf._metadata import metadata_fixup
+from ocrmypdf._pipeline import convert_to_pdfa
 from ocrmypdf._plugin_manager import get_parser_options_plugins, get_plugin_manager
 from ocrmypdf.exceptions import ExitCode
 from ocrmypdf.pdfa import file_claims_pdfa, generate_pdfa_ps
 from ocrmypdf.pdfinfo import PdfInfo
 
-from .conftest import check_ocrmypdf, run_ocrmypdf
+from .conftest import check_ocrmypdf, run_ocrmypdf, run_ocrmypdf_api
 
 
 @pytest.mark.parametrize("output_type", ['pdfa', 'pdf'])
@@ -44,12 +45,12 @@ def test_preserve_docinfo(output_type, resources, outpdf):
 
 
 @pytest.mark.parametrize("output_type", ['pdfa', 'pdf'])
-def test_override_metadata(output_type, resources, outpdf):
+def test_override_metadata(output_type, resources, outpdf, caplog):
     input_file = resources / 'c02-22.pdf'
     german = 'Du siehst den Wald vor lauter Bäumen nicht.'
     chinese = '孔子'
 
-    p = run_ocrmypdf(
+    exitcode = run_ocrmypdf_api(
         input_file,
         outpdf,
         '--title',
@@ -62,7 +63,7 @@ def test_override_metadata(output_type, resources, outpdf):
         'tests/plugins/tesseract_noop.py',
     )
 
-    assert p.returncode == ExitCode.ok, p.stderr
+    assert exitcode == ExitCode.ok, caplog.text
 
     with pikepdf.open(input_file) as before, pikepdf.open(outpdf) as after:
         assert after.docinfo.Title == german, after.docinfo
@@ -79,7 +80,7 @@ def test_override_metadata(output_type, resources, outpdf):
 
 @pytest.mark.parametrize('output_type', ['pdfa', 'pdf', 'pdfa-1', 'pdfa-2', 'pdfa-3'])
 @pytest.mark.parametrize('field', ['title', 'author', 'subject', 'keywords'])
-def test_unset_metadata(output_type, field, resources, outpdf):
+def test_unset_metadata(output_type, field, resources, outpdf, caplog):
     input_file = resources / 'meta.pdf'
 
     # magic strings contained in the input pdf metadata
@@ -90,7 +91,7 @@ def test_unset_metadata(output_type, field, resources, outpdf):
         'keywords': b's9EeALwUg7urA7fnnhm5EtUyC54sW2WPUzqh',
     }
 
-    p = run_ocrmypdf(
+    exitcode = run_ocrmypdf_api(
         input_file,
         outpdf,
         f'--{field}',
@@ -101,7 +102,7 @@ def test_unset_metadata(output_type, field, resources, outpdf):
         'tests/plugins/tesseract_noop.py',
     )
 
-    assert p.returncode == ExitCode.ok, p.stderr
+    assert exitcode == ExitCode.ok, caplog.text
 
     # We mainly want to ensure that when '' is passed, the corresponding
     # metadata is unset in the output pdf. Since metedata is not compressed,
@@ -332,7 +333,9 @@ def test_metadata_fixup_warning(resources, outdir, caplog):
     context = PdfContext(
         options, outdir, outdir / 'graph.pdf', None, get_plugin_manager([])
     )
-    metadata_fixup(working_file=outdir / 'graph.pdf', context=context)
+    metadata_fixup(
+        working_file=outdir / 'graph.pdf', context=context, pdf_save_settings={}
+    )
     for record in caplog.records:
         assert record.levelname != 'WARNING', "Unexpected warning"
 
@@ -345,7 +348,9 @@ def test_metadata_fixup_warning(resources, outdir, caplog):
     context = PdfContext(
         options, outdir, outdir / 'graph_mod.pdf', None, get_plugin_manager([])
     )
-    metadata_fixup(working_file=outdir / 'graph.pdf', context=context)
+    metadata_fixup(
+        working_file=outdir / 'graph.pdf', context=context, pdf_save_settings={}
+    )
     assert any(record.levelname == 'WARNING' for record in caplog.records)
 
 
@@ -394,9 +399,12 @@ def test_prevent_gs_invalid_xml(resources, outdir):
     assert contents.find(b'\x00', xmp_start, xmp_end) == -1
 
 
-@pytest.mark.skipif(
+@pytest.mark.xfail(
-    ghostscript.version() >= Version('10.2.0'),
+    ghostscript.version() >= Version('10.01.2'),
-    reason="Ghostscript 10.2.0+ exit with an error on invalid DocumentInfo",
+    reason=(
+        "Ghostscript now exits with an error on invalid DocumentInfo, defeating "
+        "this test.",
+    ),
 )
 def test_malformed_docinfo(caplog, resources, outdir):
     generate_pdfa_ps(outdir / 'pdfa.ps')

tests/test_optimize.py

@@ -3,21 +3,23 @@
 
 from __future__ import annotations
 
+from io import BytesIO
 from os import fspath
 from pathlib import Path
-from unittest.mock import patch
+from unittest.mock import MagicMock, patch
 
 import img2pdf
 import pikepdf
 import pytest
+from pikepdf import Array, Dictionary, Name
 from PIL import Image, ImageDraw
 
 from ocrmypdf import optimize as opt
 from ocrmypdf._exec import jbig2enc, pngquant
 from ocrmypdf._exec.ghostscript import rasterize_pdf
 from ocrmypdf.helpers import IMG2PDF_KWARGS, Resolution
-
+from ocrmypdf.optimize import PdfImage, extract_image_filter
-from .conftest import check_ocrmypdf
+from tests.conftest import check_ocrmypdf
 
 needs_pngquant = pytest.mark.skipif(
     not pngquant.available(), reason="pngquant not installed"
@@ -195,22 +197,124 @@ def test_optimize_off(resources, outpdf):
     )
 
 
-def test_group3(resources, outdir):
+def test_group3(resources):
     with pikepdf.open(resources / 'ccitt.pdf') as pdf:
         im = pdf.pages[0].Resources.XObject['/Im1']
         assert (
-            opt.extract_image_filter(pdf, outdir, im, im.objgen[0]) is not None
+            opt.extract_image_filter(im, im.objgen[0]) is not None
         ), "Group 4 should be allowed"
 
         im.DecodeParms['/K'] = 0
         assert (
-            opt.extract_image_filter(pdf, outdir, im, im.objgen[0]) is None
+            opt.extract_image_filter(im, im.objgen[0]) is None
         ), "Group 3 should be disallowed"
 
 
-def test_find_formx(resources, outdir):
+def test_find_formx(resources):
     with pikepdf.open(resources / 'formxobject.pdf') as pdf:
         working, pagenos = opt._find_image_xrefs(pdf)
         assert len(working) == 1
         xref = next(iter(working))
         assert pagenos[xref] == 0
+
+
+def test_extract_image_filter_with_pdf_image():
+    image = MagicMock()
+    image.Subtype = Name.Image
+    image.Length = 200
+    image.Width = 10
+    image.Height = 10
+    image.Filter = [Name.FlateDecode, Name.DCTDecode]
+    pdf_image = PdfImage(image)
+    image.BitsPerComponent = 8
+    assert extract_image_filter(image, None) == (
+        pdf_image,
+        pdf_image.filter_decodeparms[1],
+    )
+
+
+def test_extract_image_filter_with_non_image():
+    image = MagicMock()
+    image.Subtype = Name.Form
+    assert extract_image_filter(image, None) is None
+
+
+def test_extract_image_filter_with_small_stream_size():
+    image = MagicMock()
+    image.Subtype = Name.Image
+    image.Length = 50
+    assert extract_image_filter(image, None) is None
+
+
+def test_extract_image_filter_with_small_dimensions():
+    image = MagicMock()
+    image.Subtype = Name.Image
+    image.Length = 200
+    image.Width = 5
+    image.Height = 5
+    assert extract_image_filter(image, None) is None
+
+
+def test_extract_image_filter_with_multiple_compression_filters():
+    image = MagicMock()
+    image.Subtype = Name.Image
+    image.Length = 200
+    image.Width = 10
+    image.Height = 10
+    image.BitsPerComponent = 8
+    image.Filter = [Name.ASCII85Decode, Name.FlateDecode, Name.DCTDecode]
+    assert extract_image_filter(image, None) is None
+
+
+def test_extract_image_filter_with_wide_gamut_image():
+    image = MagicMock()
+    image.Subtype = Name.Image
+    image.Length = 200
+    image.Width = 10
+    image.Height = 10
+    image.BitsPerComponent = 16
+    image.Filter = Name.FlateDecode
+    assert extract_image_filter(image, None) is None
+
+
+def test_extract_image_filter_with_jpeg2000_image():
+    im = Image.new('RGB', (10, 10))
+    bio = BytesIO()
+    im.save(bio, format='JPEG2000')
+    pdf = pikepdf.new()
+    stream = pdf.make_stream(
+        data=bio.getvalue(),
+        Subtype=Name.Image,
+        Length=200,
+        Width=10,
+        Height=10,
+        BitsPerComponent=8,
+        Filter=Name.JPXDecode,
+    )
+    assert extract_image_filter(stream, None) is None
+
+
+def test_extract_image_filter_with_ccitt_group_3_image():
+    image = MagicMock()
+    image.Subtype = Name.Image
+    image.Length = 200
+    image.Width = 10
+    image.Height = 10
+    image.BitsPerComponent = 1
+    image.Filter = Name.CCITTFaxDecode
+    image.DecodeParms = Array([Dictionary(K=1)])
+    assert extract_image_filter(image, None) is None
+
+
+# Triggers pikepdf bug
+# def test_extract_image_filter_with_decode_table():
+#     image = MagicMock()
+#     image.Subtype = Name.Image
+#     image.Length = 200
+#     image.Width = 10
+#     image.Height = 10
+#     image.Filter = Name.FlateDecode
+#     image.BitsPerComponent = 8
+#     image.ColorSpace = Name.DeviceGray
+#     image.Decode = [42, 0]
+#     assert extract_image_filter(image, None) is None

tests/test_preprocessing.py

@@ -46,11 +46,11 @@ def test_deskew_blank_page(resources, outpdf):
 @pytest.mark.xfail(reason="remove background disabled")
 def test_remove_background(resources, outdir):
     # Ensure the input image does not contain pure white/black
-    with Image.open(resources / 'congress.jpg') as im:
+    with Image.open(resources / 'baiona_color.jpg') as im:
         assert im.getextrema() != ((0, 255), (0, 255), (0, 255))
 
     output_pdf = check_ocrmypdf(
-        resources / 'congress.jpg',
+        resources / 'baiona_color.jpg',
         outdir / 'test_remove_bg.pdf',
         '--remove-background',
         '--image-dpi',

tests/test_rotation.py

@@ -19,17 +19,10 @@ from ocrmypdf._plugin_manager import get_plugin_manager
 from ocrmypdf.helpers import IMG2PDF_KWARGS, Resolution
 from ocrmypdf.pdfinfo import PdfInfo
 
-from .conftest import check_ocrmypdf, run_ocrmypdf
+from .conftest import check_ocrmypdf, run_ocrmypdf_api
 
 # pylintx: disable=unused-variable
 
-# Remove this workaround when we require Pillow >= 10
-try:
-    Transpose = Image.Transpose  # type: ignore
-except AttributeError:
-    # Pillow 9 shim
-    Transpose = Image  # type: ignore
-
 RENDERERS = ['hocr', 'sandwich']
 
 
@@ -220,13 +213,13 @@ def test_rotate_deskew_ocr_timeout(resources, outdir):
 @pytest.mark.slow
 @pytest.mark.parametrize('page_angle', (0, 90, 180, 270))
 @pytest.mark.parametrize('image_angle', (0, 90, 180, 270))
-def test_rotate_page_level(image_angle, page_angle, resources, outdir):
+def test_rotate_page_level(image_angle, page_angle, resources, outdir, caplog):
     def make_rotate_test(prefix, image_angle, page_angle):
         memimg = BytesIO()
         with Image.open(fspath(resources / 'typewriter.png')) as im:
             if image_angle != 0:
                 ccw_angle = -image_angle % 360
-                im = im.transpose(getattr(Transpose, f'ROTATE_{ccw_angle}'))
+                im = im.transpose(getattr(Image.Transpose, f'ROTATE_{ccw_angle}'))
             im.save(memimg, format='PNG')
         memimg.seek(0)
         mempdf = BytesIO()
@@ -247,17 +240,15 @@ def test_rotate_page_level(image_angle, page_angle, resources, outdir):
     test = make_rotate_test('test', image_angle, page_angle)
     out = test.with_suffix('.out.pdf')
 
-    p = run_ocrmypdf(
+    exitcode = run_ocrmypdf_api(
         test,
         out,
         '-O0',
         '--rotate-pages',
         '--rotate-pages-threshold',
         '0.001',
-        text=False,
     )
-    err = p.stderr.decode('utf-8', errors='replace')
+    assert exitcode == 0, caplog.text
-    assert p.returncode == 0, err
 
     assert compare_images_monochrome(outdir, reference, 1, out, 1) > 0.2
 
@@ -328,15 +319,15 @@ def test_simulated_scan(outdir):
 
     with pikepdf.open(outdir / 'out.pdf') as pdf:
         assert (
-            pdf.pages[1].MediaBox[2] > pdf.pages[1].MediaBox[3]
+            pdf.pages[1].mediabox[2] > pdf.pages[1].mediabox[3]
         ), "Wrong orientation: not landscape"
         assert (
-            pdf.pages[3].MediaBox[2] > pdf.pages[3].MediaBox[3]
+            pdf.pages[3].mediabox[2] > pdf.pages[3].mediabox[3]
         ), "Wrong orientation: Not landscape"
 
         assert (
-            pdf.pages[0].MediaBox[2] < pdf.pages[0].MediaBox[3]
+            pdf.pages[0].mediabox[2] < pdf.pages[0].mediabox[3]
         ), "Wrong orientation: Not portrait"
         assert (
-            pdf.pages[2].MediaBox[2] < pdf.pages[2].MediaBox[3]
+            pdf.pages[2].mediabox[2] < pdf.pages[2].mediabox[3]
         ), "Wrong orientation: Not portrait"

tests/test_semfree.py

@@ -0,0 +1,26 @@
+# SPDX-FileCopyrightText: 2023 James R. Barlow
+# SPDX-License-Identifier: MPL-2.0
+
+from __future__ import annotations
+
+import pytest
+
+from ocrmypdf.exceptions import ExitCode
+
+from .conftest import is_linux, run_ocrmypdf_api
+
+
+@pytest.mark.skipif(not is_linux(), reason='semfree plugin only works on Linux')
+def test_semfree(resources, outpdf):
+    exitcode = run_ocrmypdf_api(
+        resources / 'multipage.pdf',
+        outpdf,
+        '--skip-text',
+        '--skip-big',
+        '2',
+        '--plugin',
+        'ocrmypdf.extra_plugins.semfree',
+        '--plugin',
+        'tests/plugins/tesseract_noop.py',
+    )
+    assert exitcode == ExitCode.ok

tests/test_soft_error.py

@@ -9,11 +9,11 @@ import pytest
 
 from ocrmypdf.exceptions import ExitCode
 
-from .conftest import run_ocrmypdf
+from .conftest import run_ocrmypdf_api
 
 
 def test_raster_continue_on_soft_error(resources, outpdf):
-    p = run_ocrmypdf(
+    exitcode = run_ocrmypdf_api(
         resources / 'francais.pdf',
         outpdf,
         '--continue-on-soft-render-error',
@@ -22,11 +22,11 @@ def test_raster_continue_on_soft_error(resources, outpdf):
         '--plugin',
         'tests/plugins/gs_raster_soft_error.py',
     )
-    assert p.returncode == ExitCode.ok
+    assert exitcode == ExitCode.ok
 
 
 def test_raster_stop_on_soft_error(resources, outpdf):
-    p = run_ocrmypdf(
+    exitcode = run_ocrmypdf_api(
         resources / 'francais.pdf',
         outpdf,
         '--plugin',
@@ -34,11 +34,11 @@ def test_raster_stop_on_soft_error(resources, outpdf):
         '--plugin',
         'tests/plugins/gs_raster_soft_error.py',
     )
-    assert p.returncode == ExitCode.child_process_error
+    assert exitcode == ExitCode.child_process_error
 
 
 def test_render_continue_on_soft_error(resources, outpdf):
-    p = run_ocrmypdf(
+    exitcode = run_ocrmypdf_api(
         resources / 'francais.pdf',
         outpdf,
         '--continue-on-soft-render-error',
@@ -47,12 +47,12 @@ def test_render_continue_on_soft_error(resources, outpdf):
         '--plugin',
         'tests/plugins/gs_render_soft_error.py',
     )
-    assert p.returncode == ExitCode.ok
+    assert exitcode == ExitCode.ok
 
 
 @pytest.mark.skipif(os.name == 'nt', reason='Ghostscript on Windows errors out')
 def test_render_stop_on_soft_error(resources, outpdf):
-    p = run_ocrmypdf(
+    exitcode = run_ocrmypdf_api(
         resources / 'francais.pdf',
         outpdf,
         '--plugin',
@@ -60,4 +60,4 @@ def test_render_stop_on_soft_error(resources, outpdf):
         '--plugin',
         'tests/plugins/gs_render_soft_error.py',
     )
-    assert p.returncode == ExitCode.child_process_error
+    assert exitcode == ExitCode.child_process_error

tests/test_tagged.py

@@ -0,0 +1,24 @@
+# SPDX-FileCopyrightText: 2023 James R. Barlow
+# SPDX-License-Identifier: MPL-2.0
+
+from __future__ import annotations
+
+import pytest
+
+import ocrmypdf
+
+
+def test_block_tagged(resources):
+    with pytest.raises(ocrmypdf.exceptions.TaggedPDFError):
+        ocrmypdf.ocr(resources / 'tagged.pdf', '_.pdf')
+
+
+def test_force_tagged_warns(resources, outpdf, caplog):
+    caplog.set_level('WARNING')
+    ocrmypdf.ocr(
+        resources / 'tagged.pdf',
+        outpdf,
+        force_ocr=True,
+        plugins=['tests/plugins/tesseract_noop.py'],
+    )
+    assert 'marked as a Tagged PDF' in caplog.text

tests/test_tesseract.py

@@ -59,7 +59,7 @@ def test_content_preservation(resources, outpdf):
 
 
 @pytest.mark.skipif(
-    tesseract.version() > tesseract.TesseractVersion('5'), reason="doesn't fool Tess 5"
+    tesseract.version() >= tesseract.TesseractVersion('5'), reason="doesn't fool Tess 5"
 )
 def test_no_languages(tmp_path, monkeypatch):
     (tmp_path / 'tessdata').mkdir()
@@ -86,7 +86,7 @@ def test_image_too_large_hocr(monkeypatch, resources, outdir):
         user_words=None,
         user_patterns=None,
     )
-    assert "name='ocr-capabilities'" in Path(outdir / 'out.hocr').read_text()
+    assert Path(outdir / 'out.hocr').read_text() == ''
 
 
 def test_image_too_large_pdf(monkeypatch, resources, outdir):

tests/test_unpaper.py

@@ -13,9 +13,9 @@ from packaging.version import Version
 from ocrmypdf._exec import unpaper
 from ocrmypdf._plugin_manager import get_parser_options_plugins
 from ocrmypdf._validation import check_options
-from ocrmypdf.exceptions import ExitCode, MissingDependencyError
+from ocrmypdf.exceptions import BadArgsError, ExitCode, MissingDependencyError
 
-from .conftest import check_ocrmypdf, have_unpaper, run_ocrmypdf
+from .conftest import check_ocrmypdf, have_unpaper, run_ocrmypdf_api
 
 # pylint: disable=redefined-outer-name
 
@@ -73,23 +73,22 @@ def test_unpaper_args_valid(resources, outpdf):
 
 
 @needs_unpaper
-def test_unpaper_args_invalid_filename(resources, outpdf):
+def test_unpaper_args_invalid_filename(resources, outpdf, caplog):
-    p = run_ocrmypdf(
+    with pytest.raises(BadArgsError):
-        resources / "skew.pdf",
+        run_ocrmypdf_api(
-        outpdf,
+            resources / "skew.pdf",
-        "-c",
+            outpdf,
-        "--unpaper-args",
+            "-c",
-        "/etc/passwd",
+            "--unpaper-args",
-        '--plugin',
+            "/etc/passwd",
-        'tests/plugins/tesseract_noop.py',
+            '--plugin',
-    )
+            'tests/plugins/tesseract_noop.py',
-    assert "No filenames allowed" in p.stderr
+        )
-    assert p.returncode == ExitCode.bad_args
 
 
 @needs_unpaper
 def test_unpaper_args_invalid(resources, outpdf):
-    p = run_ocrmypdf(
+    exitcode = run_ocrmypdf_api(
         resources / "skew.pdf",
         outpdf,
         "-c",
@@ -100,7 +99,7 @@ def test_unpaper_args_invalid(resources, outpdf):
     )
     # Can't tell difference between unpaper choking on bad arguments or some
     # other unpaper failure
-    assert p.returncode == ExitCode.child_process_error
+    assert exitcode == ExitCode.child_process_error
 
 
 @needs_unpaper
@@ -114,3 +113,14 @@ def test_unpaper_image_too_big(resources, outdir, caplog):
             for rec in caplog.get_records('call')
             if rec.levelno == logging.WARNING
         )
+
+
+@needs_unpaper
+def test_palette_image(resources, outpdf):
+    check_ocrmypdf(
+        resources / "palette.pdf",
+        outpdf,
+        "-c",
+        '--plugin',
+        'tests/plugins/tesseract_noop.py',
+    )

tests/test_validation.py

@@ -71,7 +71,9 @@ def test_tesseract_not_installed(caplog):
 
 def test_lossless_redo():
     with pytest.raises(BadArgsError):
-        vd.check_options_output(make_opts(redo_ocr=True, deskew=True))
+        options = make_opts(redo_ocr=True, deskew=True)
+        vd.check_options_output(options)
+        vd.set_lossless_reconstruction(options)
 
 
 def test_mutex_options():