Try to retain/copy xattrs

pip attempts to download musllinux_1_1 wheels and can't find pikepdf's which is only 1_2. Not clear how to fix.

.docker/Dockerfile.alpine

@@ -0,0 +1,83 @@
+# SPDX-FileCopyrightText: 2023 James R. Barlow
+# SPDX-License-Identifier: MPL-2.0
+
+FROM alpine:3.18 as base
+
+ENV LANG=C.UTF-8
+ENV TZ=UTC
+
+RUN apk add --no-cache \
+    python3 \
+    zlib
+
+FROM base as builder
+
+RUN apk add --no-cache \
+    ca-certificates \
+    git \
+    python3-dev \
+    py3-pip
+
+# On arm64, we need to build cffi from source.
+ARG TARGETPLATFORM
+
+RUN if [ "${TARGETPLATFORM}" == "linux/arm64" ]; then \
+    apk add --no-cache \
+    build-base \
+    autoconf \
+    automake \
+    libtool \
+    zlib-dev \
+    libffi-dev \
+    cairo-dev \
+    pkgconfig \
+    ;  \
+    fi
+
+COPY . /app
+
+WORKDIR /app
+
+RUN python3 -m venv .venv
+
+RUN source .venv/bin/activate \
+    && python3 -m pip install --no-cache-dir --upgrade pip \
+    && python3 -m pip install --no-cache-dir wheel \
+    && python3 -m pip install --no-cache-dir .[test,webservice,watcher]
+
+FROM base
+
+RUN apk add --no-cache \
+    ghostscript \
+    jbig2dec \
+    jbig2enc \
+    pngquant \
+    tesseract-ocr \
+    tesseract-ocr-data-chi_sim \
+    tesseract-ocr-data-deu \
+    tesseract-ocr-data-eng \
+    tesseract-ocr-data-fra \
+    tesseract-ocr-data-osd \
+    tesseract-ocr-data-por \
+    tesseract-ocr-data-spa \
+    ttf-droid \
+    unpaper \
+    && rm -rf /var/cache/apk/*
+
+WORKDIR /app
+
+COPY --from=builder /usr/local/lib/ /usr/local/lib/
+COPY --from=builder /usr/local/bin/ /usr/local/bin/
+
+COPY --from=builder /app/.venv/ /app/.venv/
+
+COPY --from=builder /app/misc/webservice.py /app/
+COPY --from=builder /app/misc/watcher.py /app/
+
+# Copy minimal project files to get the test suite.
+COPY --from=builder /app/pyproject.toml /app/README.md /app/
+COPY --from=builder /app/tests /app/tests
+
+ENV PATH="/app/.venv/bin:${PATH}"
+
+ENTRYPOINT ["/app/.venv/bin/ocrmypdf"]

.github/workflows/build.yml

@@ -28,11 +28,14 @@ jobs:
             python: "3.10"
           - os: ubuntu-22.04
             python: "3.11"
-          #- os: ubuntu-latest
-          #  python: "pypy3.9"
           - os: ubuntu-22.04
             python: "3.9"
             tesseract5: true
+          - os: ubuntu-latest
+            python: "3.12"
+            tesseract5: true
+            #- os: ubuntu-latest
+            #  python: "pypy3.9"
 
     env:
       OS: ${{ matrix.os }}
@@ -44,9 +47,10 @@ jobs:
           fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags
 
       - uses: actions/setup-python@v4
-        name: Install Python
+        name: Setup Python
         with:
           python-version: ${{ matrix.python }}
+          cache: "pip"
 
       - name: Install Tesseract 5
         if: matrix.tesseract5
@@ -109,7 +113,7 @@ jobs:
     strategy:
       matrix:
         os: [macos-latest]
-        python: ["3.10", "3.11"]
+        python: ["3.10", "3.11", "3.12"]
 
     env:
       OS: ${{ matrix.os }}
@@ -129,13 +133,15 @@ jobs:
             ghostscript \
             jbig2enc \
             openjpeg \
+            openssl \
             pngquant \
             tesseract
 
       - uses: actions/setup-python@v4
-        name: Install Python
+        name: Setup Python
         with:
           python-version: ${{ matrix.python }}
+          cache: "pip"
 
       - name: Install Python packages
         run: |
@@ -165,7 +171,7 @@ jobs:
     strategy:
       matrix:
         os: [windows-latest]
-        python: ["3.10", "3.11"]
+        python: ["3.10", "3.11", "3.12"]
 
     env:
       OS: ${{ matrix.os }}
@@ -177,9 +183,10 @@ jobs:
           fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags
 
       - uses: actions/setup-python@v4
-        name: Install Python
+        name: Setup Python
         with:
           python-version: ${{ matrix.python }}
+          cache: "pip"
 
       - name: Install system packages
         run: |
@@ -210,9 +217,10 @@ jobs:
           fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags
 
       - uses: actions/setup-python@v4
-        name: Install Python
+        name: Setup Python
         with:
           python-version: "3.9"
+          cache: "pip"
 
       - name: Make wheels and sdist
         run: |
@@ -268,8 +276,8 @@ jobs:
             ./dist/*.whl
             ./dist/*.tar.gz
 
-  docker:
+  docker_ubuntu:
-    name: Build Docker images
+    name: Build Ubuntu-based Docker image
     needs: [wheel_sdist_linux, test_linux, test_macos, test_windows]
     runs-on: ubuntu-latest
     if: github.event_name != 'pull_request'
@@ -313,4 +321,52 @@ jobs:
             --push \
             --platform linux/arm64/v8,linux/amd64  \
             --tag "${DOCKER_REPOSITORY}/${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}" \
+            --tag "${DOCKER_REPOSITORY}/${DOCKER_IMAGE_NAME}-ubuntu:${DOCKER_IMAGE_TAG}" \
             --file .docker/Dockerfile .
+
+  docker_alpine:
+    name: Build Alpine-based Docker images
+    needs: [wheel_sdist_linux, test_linux, test_macos, test_windows]
+    runs-on: ubuntu-latest
+    if: github.event_name != 'pull_request'
+    steps:
+      - name: Set image tag to release or branch
+        run: echo "DOCKER_IMAGE_TAG=${GITHUB_REF##*/}" >> $GITHUB_ENV
+
+      - name: If main, set to latest
+        run: echo 'DOCKER_IMAGE_TAG=latest' >> $GITHUB_ENV
+        if: env.DOCKER_IMAGE_TAG == 'main'
+
+      - name: Set Docker Hub repository to username
+        run: echo "DOCKER_REPOSITORY=jbarlow83" >> $GITHUB_ENV
+
+      - name: Set image name
+        run: echo "DOCKER_IMAGE_NAME=ocrmypdf-alpine" >> $GITHUB_ENV
+
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: jbarlow83
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Print image tag
+        run: echo "Building image ${DOCKER_REPOSITORY}/${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}"
+
+      - name: Build
+        run: |
+          docker buildx build \
+            --push \
+            --platform linux/amd64  \
+            --tag "${DOCKER_REPOSITORY}/${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}" \
+            --file .docker/Dockerfile.alpine .

README.md

@@ -39,8 +39,8 @@ ocrmypdf                      # it's a scriptable command line program
 - Distributes work across all available CPU cores
 - Uses [Tesseract OCR](https://github.com/tesseract-ocr/tesseract) engine to recognize more than [100 languages](https://github.com/tesseract-ocr/tessdata)
 - Keeps your private data private.
-- Scales properly to handle files with thousands of pages
+- Scales properly to handle files with thousands of pages.
-- Battle-tested on millions of PDFs
+- Battle-tested on millions of PDFs.
 
 <img src="misc/screencast/demo.svg" alt="Demo of OCRmyPDF in a terminal session">
 

docs/advanced.rst

@@ -117,14 +117,14 @@ exceed a certain number of megapixels with ``--skip-big``. (A 300 DPI,
 OCR for huge images
 -------------------
 
-Separate from these settings, Tesseract has internal limits on the size
+Tesseract has internal limits on the size
 of images it will process. If you issue
 ``--tesseract-downsample-large-images``, OCRmyPDF will downsample images
 to fit Tesseract limits. (The limits are usually entered only for scanned
 images of oversized media, such as large maps or blueprints exceeding
 110 cm or 43 inches in either dimension, and at high DPI.)
 
-``--tesseract-downsample-above`` adjusts the threshold at which images
+``--tesseract-downsample-above Npixels`` adjusts the threshold at which images
 will be downsampled. By default, only images that exceed any of Tesseract's
 internal limits are downsampled.
 
@@ -195,10 +195,10 @@ In each case OCRmyPDF will search the ``PATH`` environment variable to
 locate the binaries. By modifying the ``PATH`` environment variable, you
 can override the binaries that OCRmyPDF uses.
 
-Changing tesseract configuration variables
+Changing Tesseract configuration variables
 ------------------------------------------
 
-You can override tesseract's default `control
+You can override Tesseract's default `control
 parameters <https://tesseract-ocr.github.io/tessdoc/tess3/ControlParams.html>`__
 with a configuration file.
 
@@ -273,7 +273,7 @@ Unlike ``sandwich`` this renderer is implemented within OCRmyPDF; anyone
 looking to customize how OCR is presented should look here. A major
 disadvantage of this renderer is it not capable of correctly handling
 text outside the Latin alphabet (specifically, it supports the ISO 8859-1
-character). Pull requests to improve the situation are welcome.
+character set). Pull requests to improve the situation are welcome.
 
 Currently, this renderer has the best compatibility with Mozilla's
 PDF.js viewer.
@@ -286,11 +286,31 @@ Rendering and rasterizing options
 .. versionadded:: 14.3.0
 
 The ``--continue-on-soft-render-error`` option allows OCRmyPDF to
-proceed if a page cannot be rasterized rendered. This is useful if you are
+proceed if a page cannot be rasterized/rendered. This is useful if you are
 trying to get the best possible OCR from a PDF that is not well-formed,
 and you are willing to accept some pages that may not visually match the
 input, and that may not OCR well.
 
+Color conversion strategy
+=========================
+
+.. versionadded:: 15.0.0
+
+OCRmyPDF uses Ghostscript to convert PDF to PDF/A. In some cases, this
+conversion requires color conversion. The default strategy is to convert
+using the ``LeaveColorUnchanged`` strategy, which preserves the original
+color space wherever possible (some rare color spaces might still be
+converted).
+
+Usually document scanners produce PDFs in the sRGB color space, and do
+not need to be converted, so the default strategy is appropriate.
+
+Suppose that you have a document that was prepared for professional
+printing in a Separation or CMYK color space, and text was converted to
+curves. In this case, you may want to use a different color conversion
+strategy. The ``--color-conversion-strategy`` option allows you to select a
+different strategy, such as ``RGB``.
+
 Return code policy
 ==================
 

docs/api.rst

@@ -39,9 +39,7 @@ Parent process requirements
 The :func:`ocrmypdf.ocr` function runs OCRmyPDF similar to command line
 execution. To do this, it will:
 
-- create a monitoring thread
+- create worker processes or threads
-- create worker processes (on Linux, forking itself; on Windows and macOS, by
-  spawning)
 - manage the signal flags of its worker processes
 - execute other subprocesses (forking and executing other programs)
 
@@ -54,7 +52,19 @@ processes.
 
 Creating a child process to call :func:`ocrmypdf.ocr()` is suggested. That
 way your application will survive and remain interactive even if
-OCRmyPDF fails for any reason.
+OCRmyPDF fails for any reason. For example:
+
+.. code-block:: python
+
+    from multiprocessing import Process
+
+    def ocrmypdf_process():
+        ocrmypdf.ocr('input.pdf', 'output.pdf')
+
+    def call_ocrmypdf_from_my_app():
+        p = Process(target=ocrmypdf_process)
+        p.start()
+        p.join()
 
 Programs that call :func:`ocrmypdf.ocr()` should also install a SIGBUS signal
 handler (except on Windows), to raise an exception if access to a memory
@@ -89,12 +99,21 @@ your use case.
 Progress monitoring
 -------------------
 
-OCRmyPDF uses the ``tqdm`` package to implement its progress bars.
+OCRmyPDF uses the ``rich`` package to implement its progress bars.
 :func:`ocrmypdf.configure_logging` will set up logging output to
 ``sys.stderr`` in a way that is compatible with the display of the
 progress bar. Use ``ocrmypdf.ocr(...progress_bar=False)`` to disable
 the progress bar.
 
+Standard output
+---------------
+
+OCRmyPDF is strict about not writing to standard output so that
+users can safely use it in a pipeline and produce a valid output
+file. A caller application will have to ensure it does not write to
+standard output either, if it wants to be compatible with this
+behavior and support piping to a file.
+
 Exceptions
 ----------
 
@@ -104,9 +123,6 @@ exceptions, some exceptions related to multiprocessing, and
 handler. OCRmyPDF will clean up its temporary files and worker processes
 automatically when an exception occurs.
 
-Programs that call OCRmyPDF should consider trapping KeyboardInterrupt
-so that they allow OCR to terminate with the whole program terminating.
-
 When OCRmyPDF succeeds conditionally, it returns an integer exit code.
 
 Reference

docs/batch.rst

@@ -152,7 +152,7 @@ The watcher service is included in the OCRmyPDF Docker image. To run it:
     docker run \
         --volume <path to files to convert>:/input \
         --volume <path to store results>:/output \
-        --volume <path to store processed originals>:/archive \
+        --volume <path to store processed originals>:/processed \
         --env OCR_OUTPUT_DIRECTORY_YEAR_MONTH=1 \
         --env OCR_ON_SUCCESS_ARCHIVE=1 \
         --env OCR_DESKEW=1 \
@@ -163,7 +163,7 @@ The watcher service is included in the OCRmyPDF Docker image. To run it:
 
 This service will watch for a file that matches ``/input/\*.pdf``,
 convert it to a OCRed PDF in ``/output/``, and move the processed
-original to ``/archive``. The parameters to this image are:
+original to ``/processed``. The parameters to this image are:
 
 .. csv-table:: watcher.py parameters for Docker
     :header: "Parameter", "Description"
@@ -171,7 +171,7 @@ original to ``/archive``. The parameters to this image are:
 
     "``--volume <path to files to convert>:/input``", "Files placed in this location will be OCRed"
     "``--volume <path to store results>:/output``", "This is where OCRed files will be stored"
-    "``--volume <path to store processed originals>:/archive``", "Archive processed originals here"
+    "``--volume <path to store processed originals>:/processed``", "Archive processed originals here"
     "``--env OCR_OUTPUT_DIRECTORY_YEAR_MONTH=1``", "Define environment variable ``OCR_OUTPUT_DIRECTORY_YEAR_MONTH=1`` to place files in the output in ``{output}/{year}/{month}/{filename}``"
     "``--env OCR_ON_SUCCESS_ARCHIVE=1``", "Define environment variable ``OCR_ON_SUCCESS_ARCHIVE`` to move processed originals"
     "``--env OCR_DESKEW=1``", "Define environment variable ``OCR_DESKEW``  to apply deskew to crooked input PDFs"

docs/docker.rst

@@ -8,17 +8,16 @@
 OCRmyPDF Docker image
 =====================
 
-OCRmyPDF is also available in a Docker image that packages recent
+OCRmyPDF is also available in Docker images that packages recent
 versions of all dependencies.
 
 For users who already have Docker installed this may be an easy and
-convenient option. However, it is less performant than a system
+convenient option.
-installation and may require Docker engine configuration.
 
-OCRmyPDF needs a generous amount of RAM, CPU cores, temporary storage
+On platforms other than Linux, Docker runs in a virtual machine, and so may
-space, whether running in a Docker container or on its own. It may be
+be less performant. You may also want to adjust the Docker virtual machine's
-necessary to ensure the container is provisioned with additional
+memory and CPU allocation. On Linux, the Docker image runs natively and
-resources.
+performance is comparable to a system installation.
 
 .. _docker-install:
 
@@ -35,28 +34,37 @@ execute the image:
 
    docker run hello-world
 
-The recommended OCRmyPDF Docker image is currently named ``ocrmypdf``:
+.. list-table:: Docker images
+   :width: 30 20 50
+   :header-rows: 1
+
+   * - Image
+     - Architecture
+     - Description
+   * - ``jbarlow83/ocrmypdf-alpine``
+     - x86_64 only
+     - Recommended image, based on Alpine Linux.
+   * - ``jbarlow83/ocrmypdf-ubuntu``
+     - x86_64 and arm64
+     - Alternate image, based on Ubuntu. When the Alpine image is considered
+       stable and available for arm64, this image will be deprecated.
+   * - ``jbarlow83/ocrmypdf``
+     - x86_64 and arm64
+     - Currently an alias for ocrmypdf-ubuntu. When the Alpine image is
+       considered stable and available for arm64, this name point to the
+       Alpine image. If you don't about the difference between Alpine and
+       Ubuntu, use this image.
+
+To install:
 
 .. code-block:: bash
 
-   docker pull jbarlow83/ocrmypdf
+   docker pull jbarlow83/ocrmypdf-alpine
 
+The ``ocrmypdf`` image is also available, but is deprecated and will be removed
+in the future.
 
-OCRmyPDF will use all available CPU cores. By default, the VirtualBox
+OCRmyPDF will use all available CPU cores. See the Docker documentation for
-machine instance on Windows and macOS has only a single CPU core
-enabled. Use the VirtualBox Manager to determine the name of your Docker
-engine host, and then follow these optional steps to enable multiple
-CPUs:
-
-.. code-block:: bash
-
-   # Optional step for Mac OS X users
-   docker-machine stop "yourVM"
-   VBoxManage modifyvm "yourVM" --cpus 2  # or whatever number of core is desired
-   docker-machine start "yourVM"
-   eval $(docker-machine env "yourVM")
-
-See the Docker documentation for
 `adjusting memory and CPU on other platforms <https://docs.docker.com/config/containers/resource_constraints/>`__.
 
 Using the Docker image on the command line
@@ -66,6 +74,8 @@ Using the Docker image on the command line
 container is ephemeral – it runs for one OCR job and terminates, just like a
 command line program. We are using Docker to deliver an application (as opposed
 to the more conventional case, where a Docker container runs as a server).
+For that reason we usually use the ``--rm`` argument to delete the container
+when it exits.
 
 To start a Docker container (instance of the image):
 
@@ -116,7 +126,7 @@ Dockerfile based on the public one.
    # Example: add Italian
    RUN apt install tesseract-ocr-ita
 
-To install language packs (training data) such as the 
+To install language packs (training data) such as the
 `tessdata_best <https://github.com/tesseract-ocr/tessdata_best>`_ suite or
 custom data, you first need to determine the version of Tesseract data files, which
 may differ from the Tesseract program version. Use this command to determine the data
@@ -132,17 +142,35 @@ You can then add new data with either a Dockerfile:
 
 .. code-block:: dockerfile
 
-   FROM jbarlow83/ocrmypdf
+   FROM jbarlow83/ocrmypdf:{TAG}
 
    # Example: add a tessdata_best file
    COPY chi_tra_vert.traineddata /usr/share/tesseract-ocr/<data version>/tessdata/
 
+When creating your own image, you should always pin a specific version of the
+OCRmyPDF Docker image. This ensures that your image will not break when a new
+version of OCRmyPDF is released.
+
 Alternately, you can copy training data into a Docker container as follows:
 
 .. code-block:: bash
 
    docker cp mycustomtraining.traineddata name_of_container:/usr/share/tesseract-ocr/<tesseract version>/tessdata/
 
+Extending the Docker image
+==========================
+
+You can extend the Docker image with your own customizations, similar to the way
+it is extended to add language packs.
+
+Note that the Docker image is subject to change at any time. For example, the base
+image may be updated to a newer version of Ubuntu or Debian. Such changes will be
+noted in the release notes but might occur at minor versions releases, unless the
+way a "casual" user of the Docker image is affected.
+
+If you extend the Docker image, you should pin a specific version of the OCRmyPDF
+Docker image.
+
 Executing the test suite
 ========================
 
@@ -150,16 +178,16 @@ The OCRmyPDF test suite is installed with image. To run it:
 
 .. code-block:: bash
 
-   docker run --entrypoint python3  jbarlow83/ocrmypdf -m pytest
+   docker run --rm --entrypoint python  jbarlow83/ocrmypdf -m pytest
 
 Accessing the shell
 ===================
 
-To use the bash shell in the Docker image:
+To use the shell in the Docker image:
 
 .. code-block:: bash
 
-   docker run -it --entrypoint bash  jbarlow83/ocrmypdf
+   docker run -it --entrypoint sh  jbarlow83/ocrmypdf
 
 Using the OCRmyPDF web service wrapper
 ======================================
@@ -169,7 +197,10 @@ service. The webservice may be launched as follows:
 
 .. code-block:: bash
 
-   docker run --entrypoint python3 -p 5000:5000  jbarlow83/ocrmypdf webservice.py
+   docker run --entrypoint python -p 5000:5000  jbarlow83/ocrmypdf webservice.py
+
+We omit the ``--rm`` parameter so that the container will not be
+automatically deleted when it exits.
 
 This will configure the machine to listen on port 5000. On Linux machines
 this is port 5000 of localhost. On macOS or Windows machines running

docs/index.rst

@@ -12,7 +12,7 @@ files, allowing them to be searched.
 
 PDF is the best format for storing and exchanging scanned documents.
 Unfortunately, PDFs can be difficult to modify. OCRmyPDF makes it easy to apply
-image processing and OCR to existing PDFs.
+image processing and OCR (recognized, searchable text) to existing PDFs.
 
 .. toctree::
    :maxdepth: 1

docs/installation.rst

@@ -72,7 +72,7 @@ Debian and Ubuntu 20.04 or newer
 | |ubu-2004| |ubu-2204|                         |
 +-----------------------------------------------+
 
-Users of Debian 11, or Ubuntu 20.04 LTS, or newer may simply
+Users of Debian or Ubuntu may simply
 
 .. code-block:: bash
 
@@ -99,11 +99,11 @@ For full details on version availability for your platform, check the
 Fedora
 ------
 
-.. |fedora-35| image:: https://repology.org/badge/version-for-repo/fedora_35/ocrmypdf.svg
+.. |fedora-37| image:: https://repology.org/badge/version-for-repo/fedora_37/ocrmypdf.svg
-    :alt: Fedora 35
+    :alt: Fedora 37
 
-.. |fedora-36| image:: https://repology.org/badge/version-for-repo/fedora_36/ocrmypdf.svg
+.. |fedora-38| image:: https://repology.org/badge/version-for-repo/fedora_38/ocrmypdf.svg
-    :alt: Fedora 36
+    :alt: Fedora 38
 
 .. |fedora-rawhide| image:: https://repology.org/badge/version-for-repo/fedora_rawhide/ocrmypdf.svg
     :alt: Fedore Rawhide
@@ -113,10 +113,10 @@ Fedora
 +-----------------------------------------------+
 | |latest|                                      |
 +-----------------------------------------------+
-| |fedora-35| |fedora-36| |fedora-rawhide|      |
+| |fedora-37| |fedora-38| |fedora-rawhide|      |
 +-----------------------------------------------+
 
-Users of Fedora 29 or later may simply
+Users of Fedora may simply
 
 .. code-block:: bash
 
@@ -155,7 +155,7 @@ user, follow these steps:
 
 If you get the message ``WARNING: The script ocrmypdf is installed in
 '/home/$USER/.local/bin' which is not on PATH.``, you may need to re-login
-or open a new shell, or manually add this to your user's PATH.
+or open a new shell, or manually adjust your PATH.
 
 To add JBIG2 encoding, see :ref:`jbig2`.
 
@@ -382,29 +382,33 @@ Native Windows
 
 You must install the following for Windows:
 
-* Python 3.9 (64-bit) or later
+* Python 64-bit
-* Tesseract 4.1.1 (64-bit) or later
+* Tesseract 64-bit
-* Ghostscript 9.50 (64-bit) or later
+* Ghostscript 64-bit
 
-Using the `Chocolatey <https://chocolatey.org/>`_ package manager, install the
+Using the `winget <https://docs.microsoft.com/en-us/windows/package-manager/winget/>`_
-following when running in an Administrator command prompt:
+package manager:
+
+* ``winget install -e --id Python.Python.3.11``
+* ``winget install -e --id UB-Mannheim.TesseractOCR``
+* ``winget install -e --id ArtifexSoftware.GhostScript``
+
+
+(Or alternately, using the `Chocolatey <https://chocolatey.org/>`_ package manager, install
+the following when running in an Administrator command prompt):
 
 * ``choco install python3``
 * ``choco install --pre tesseract``
 * ``choco install ghostscript``
 * ``choco install pngquant`` (optional)
 
-The commands above will install Python 3.x (latest version), Tesseract, Ghostscript
+Either set of commands will install the required software. At the mmoment there is no
-and pngquant. Chocolatey may also need to install the Windows Visual C++ Runtime
+single command to install Windows.
-DLLs or other Windows patches, and may require a reboot.
 
 You may then use ``pip`` to install ocrmypdf. (This can performed by a user or
 Administrator.):
 
-* ``pip install ocrmypdf``
+* ``python3 -m pip install ocrmypdf``
-
-Chocolatey automatically selects appropriate versions of these applications. Please make sure
-you are installing the 64-bit versions.
 
 OCRmyPDF will check the Windows Registry and standard locations in your Program Files
 for third party software it needs (specifically, Tesseract and Ghostscript). To
@@ -416,12 +420,12 @@ to change the PATH.
 
     As of early 2021, users have reported problems with the Microsoft Store version of
     Python and OCRmyPDF. These issues affect many other third party Python packages.
-    Please download Python from Python.org or Chocolatey instead, and do not use the
+    Please download Python from Python.org or a package manager instead of the
     Microsoft Store version.
 
 .. warning::
 
-    32-bit Windows might work, but is not supported.
+    32-bit Windows is not supported.
 
 Windows Subsystem for Linux
 ---------------------------
@@ -448,7 +452,7 @@ Cygwin64
 
 First install the the following prerequisite Cygwin packages using ``setup-x86_64.exe``::
 
-    python38 (or later)
+    python39 (or later)
     python3?-devel
     python3?-pip
     python3?-lxml
@@ -457,7 +461,7 @@ First install the the following prerequisite Cygwin packages using ``setup-x86_6
        (where 3? means match the version of python3 you installed)
 
     gcc-g++
-    ghostscript (<=9.50 or >=9.52-2 see note below)
+    ghostscript
     libexempi3
     libexempi-devel
     libffi6
@@ -468,13 +472,6 @@ First install the the following prerequisite Cygwin packages using ``setup-x86_6
     tesseract-ocr
     tesseract-ocr-devel
 
-.. note::
-
-    The Cygwin package for Ghostscript in versions 9.52 and
-    9.52-1 contained a bug that caused an exception to occur when
-    ocrmypdf invoked gs.  Make sure you have either 9.50 (or earlier)
-    or 9.52-2 (or later).
-
 Then open a Cygwin terminal (i.e. ``mintty``), run the following commands. Note
 that if you are using the version of ``pip`` that was installed with the Cygwin
 Python package, the command name will be ``pip3``.  If you have since updated
@@ -554,12 +551,15 @@ manager. ``pip`` cannot provide them.
 The following versions are required:
 
 -  Python 3.9 or newer
--  Ghostscript 9.50 or newer
+-  Ghostscript 9.55 or newer
 -  Tesseract 4.1.1 or newer
 -  jbig2enc 0.29 or newer
 -  pngquant 2.5 or newer
 -  unpaper 6.1
 
+We recommend 64-bit versions of all software. (32-bit versions are not
+supported, although on Linux, they may still work.)
+
 jbig2enc, pngquant, and unpaper are optional. If missing certain
 features are disabled. OCRmyPDF will discover them as soon as they are
 available.
@@ -665,3 +665,18 @@ To manually install the ``bash`` completion, copy
 To manually install the ``fish`` completion, copy
 ``misc/completion/ocrmypdf.fish`` to
 ``~/.config/fish/completions/ocrmypdf.fish``.
+
+Note on 32-bit support
+======================
+
+Many Python libraries no longer 32-bit binary wheels for Linux. This
+includes many of the libraries that OCRmyPDF depends on, such as
+Pillow. The easiest way to express this to end users is to say we don't
+support 32-bit Linux.
+
+However, if your Linux distribution still supports 32-bit binaries, you
+can still install and use OCRmyPDF. A warning message will appear.
+In practice, OCRmyPDF may need more than 32-bit memory space to run when
+large documents are processed, so there are practical limitations to what
+users can accomplish with it. Still, for the common use case of an 32-bit
+ARM NAS or Raspberry Pi processing small documents, it should work.

docs/jbig2.rst

@@ -59,5 +59,9 @@ To turn on JBIG2 lossy mode, add the argument ``--jbig2-lossy``.
 also required. Also, a JBIG2 encoder must be installed as described in
 the previous section.
 
+You can adjust the threshold for JBIG2 compression with the
+``--jbig2-threshold``. The default is 0.85, meaning that if two symbols
+are 85% similar, they will be compressed together.
+
 *Due to an oversight, ocrmypdf v7.0 and v7.1 used lossy mode by
 default.*

docs/maintainers.rst

@@ -56,4 +56,12 @@ improve OCRmyPDF's compression.
 Command line completions
 ------------------------
 
-Please ensure that command line completions are installed.
+Please ensure that command line completions are installed, as described in the
+installation documentation.
+
+32-bit Linux support
+--------------------
+
+If you maintain a Linux distribution that supports 32-bit x86 or ARM, OCRmyPDF
+should continue to work as long as all of its dependencies continue to be
+available in 32-bit form. Please note we do not test on 32-bit platforms.

docs/optimizer.rst

@@ -45,8 +45,8 @@ Optimizations that always occurs
 ================================
 
 OCRmyPDF will automatically replace obsolete or inferior compression schemes
-such as RLE or LZW with superior schemes such as Deflate and converting
+such as RLE or LZW with superior schemes such as Deflate, and convert
-monochrome images to CCITT G4. Since this is harmless it always occurs and there
+monochrome images to CCITT G4. Since this is lossless, it always occurs and there
 is no way to disable it. Other non-image compressed objects are compressed as
 well.
 

docs/release_notes.rst

@@ -28,31 +28,87 @@ tagged yet.
 
 .. |OCRmyPDF PyPI| image:: https://img.shields.io/pypi/v/ocrmypdf.svg
 
+v15.3.1
+=======
+
+-  Fixed an issue with logging settings for misc/watcher.py introduced in the
+   previous release. :issue:`1180`
+-  Updated documentation on Docker performance concerns.
+
+v15.3.0
+=======
+
+-  Update misc/watcher.py to improve command line interface using Typer, and
+   support ``.env`` specification of environment variables. Improved error
+   messages. Thanks to @mflagg2814 for the PR that prompted this improvement.
+-  Improved error message when a file cannot be read because we are running in
+   a snap container.
+
+v15.2.0
+=======
+
+-  Added a Docker image based on Alpine Linux. This image is smaller than the
+   Ubuntu-based image and may be useful in some situations. Currently hosted at
+   jbarlow83/ocrmypdf-alpine. Currently not available in ARM flavor.
+-  The Ubuntu Docker is now aliased to jbarlow83/ocrmypdf-ubuntu.
+-  Updated Docker documentation.
+
+v15.1.0
+=======
+
+-  We now require Pillow 10.0.1, due a serious security vulnerability in all earlier
+   versions of that dependency. The vulnerability concerns WebP images and could
+   be triggered in OCRmyPDF when creating a PDF from a malicious WebP image.
+-  Added some keyword arguments to ``ocrmypdf.ocr`` that were previously accepted
+   but undocumented.
+-  Documentation updates and typing improvements.
+
+v15.0.2
+=======
+
+-  Added Python 3.12 to test matrix.
+-  Updated documentation for notes on Python 3.12, 32-bit support and some new
+   features in v15.
+
+v15.0.1
+=======
+
+-  Wheels Python tag changed to py39.
+-  Marked as a expected fail a test that fails on recent Ghostscript versions.
+-  Clarified documentation and release notes around the extent of 32-bit support.
+-  Updated installation documentation to changes in v15.
+
 v15.0.0
 =======
 
 -  Dropped support for Python 3.8.
--  Dropped support many older dependencies - see ``pyproject.toml`` for details.
+-  Dropped support some older dependencies, specifically ``coloredlogs`` and
-   Generally speaking, Ubuntu 22.04 is our baseline system.
+   ``tqdm`` in favor of rich - see ``pyproject.toml`` for details.
--  Dropped support 32-bit Windows and Linux. You must use a 64-bit operating system,
+   Generally speaking, Ubuntu 22.04 is our new baseline system.
-   and 64-bit versions of Python, Tesseract and Ghostscript to use OCRmyPDF. Many of
+-  Tightened version requirements for some dependencies.
-   our dependencies are dropping 32-bit support (e.g. Pillow), and we are following
+-  Dropped support for 32-bit Linux wheels. We strongly recommend a 64-bit operating
-   suit.
+   system, and 64-bit versions of Python, Tesseract and Ghostscript to use OCRmyPDF.
+   Many of our dependencies are dropping 32-bit builds (e.g. Pillow), and we are
+   following suit. (Maintainers may still build 32-bit versions from source.)
 -  Changed to trusted release for PyPI publishing.
--  pikepdf memory mapping is enabled again for improved performance, now an issue
+-  pikepdf memory mapping is enabled again for improved performance, now that an
-   with pikepdf has been fixed.
+   issue with feature in pikepdf is fixed.
 -  ``ocrmypdf.helpers.calculate_downsample`` previously had two variants, one
    that took a ``PIL.Image`` and one that took a ``tuple[int, int]``. The latter
    was removed.
 -  The snap version of ocrmypdf is now based on Ubuntu core22.
--  We now account situations where a small portion of an image on a page reports a
+-  We now account for situations where a small portion of an image on a page is drawn
-   high DPI (resolution). Previously, the entire page would be rasterized at the
+   at high DPI (resolution). Previously, the entire page would be rasterized at the
-   highest resolution, which caused performance problems. Now, the page is rasterized
+   highest resolution of any feature, which caused performance problems. Now,
+   the page is rasterized
    at a resolution based on the average DPI of the page, weighted by the area that
    each feature occupies. Typically, small areas of high resolution in PDFs are
    errors or quirks from the repeated use of assets and high resolution is not
    beneficial. :issue:`1010,1104,1004,1079,1010`
--  Ghostscript color conversion strategy is now configurable. :issue:`1143`
+-  Ghostscript color conversion strategy is now configurable using
+   ``--color-conversion-strategy``. :issue:`1143`
+-  JBIG2 threshold for optimization is now configurable using
+   ``--jbig2-threshold``. :issue:`1133`
 
 v14.4.0
 =======

misc/watcher.py

@@ -5,107 +5,122 @@
 
 """Watch a directory for new PDFs and OCR them."""
 
-from __future__ import annotations
+# Do not enable annotations!
+# https://github.com/tiangolo/typer/discussions/598
+# from __future__ import annotations
 
 import json
 import logging
-import os
 import shutil
 import sys
 import time
 from datetime import datetime
+from enum import Enum
 from pathlib import Path
+from typing import Annotated, Any
 
 import pikepdf
+import typer
+from dotenv import load_dotenv
 from watchdog.events import PatternMatchingEventHandler
 from watchdog.observers import Observer
 from watchdog.observers.polling import PollingObserver
 
 import ocrmypdf
 
+load_dotenv()
+
+
 # pylint: disable=logging-format-interpolation
-
+app = typer.Typer(name="ocrmypdf-watcher")
-
-def getenv_bool(name: str, default: str = 'False'):
-    return os.getenv(name, default).lower() in ('true', 'yes', 'y', '1')
-
-
-INPUT_DIRECTORY = os.getenv('OCR_INPUT_DIRECTORY', '/input')
-OUTPUT_DIRECTORY = os.getenv('OCR_OUTPUT_DIRECTORY', '/output')
-ARCHIVE_DIRECTORY = os.getenv('OCR_ARCHIVE_DIRECTORY', '/processed')
-OUTPUT_DIRECTORY_YEAR_MONTH = getenv_bool('OCR_OUTPUT_DIRECTORY_YEAR_MONTH')
-ON_SUCCESS_DELETE = getenv_bool('OCR_ON_SUCCESS_DELETE')
-ON_SUCCESS_ARCHIVE = getenv_bool('OCR_ON_SUCCESS_ARCHIVE')
-DESKEW = getenv_bool('OCR_DESKEW')
-OCR_JSON_SETTINGS = json.loads(os.getenv('OCR_JSON_SETTINGS', '{}'))
-POLL_NEW_FILE_SECONDS = int(os.getenv('OCR_POLL_NEW_FILE_SECONDS', '1'))
-USE_POLLING = getenv_bool('OCR_USE_POLLING')
-RETRIES_LOADING_FILE = int(os.getenv('OCR_RETRIES_LOADING_FILE', '5'))
-LOGLEVEL = os.getenv('OCR_LOGLEVEL', 'INFO')
-PATTERNS = ['*.pdf', '*.PDF']
 
 log = logging.getLogger('ocrmypdf-watcher')
 
 
-def get_output_dir(root, basename):
+class LoggingLevelEnum(str, Enum):
-    if OUTPUT_DIRECTORY_YEAR_MONTH:
+    """Enum for logging levels."""
+
+    DEBUG = "DEBUG"
+    INFO = "INFO"
+    WARNING = "WARNING"
+    ERROR = "ERROR"
+    CRITICAL = "CRITICAL"
+
+
+def get_output_dir(root: Path, basename: str, output_dir_year_month: bool) -> Path:
+    if output_dir_year_month:
         today = datetime.today()
-        output_directory_year_month = (
+        output_directory_year_month = root / str(today.year) / f'{today.month:02d}'
-            Path(root) / str(today.year) / f'{today.month:02d}'
-        )
         if not output_directory_year_month.exists():
             output_directory_year_month.mkdir(parents=True, exist_ok=True)
         output_path = Path(output_directory_year_month) / basename
     else:
-        output_path = Path(OUTPUT_DIRECTORY) / basename
+        output_path = root / basename
     return output_path
 
 
-def wait_for_file_ready(file_path):
+def wait_for_file_ready(
+    file_path: Path, poll_new_file_seconds: int, retries_loading_file: int
+):
     # This loop waits to make sure that the file is completely loaded on
     # disk before attempting to read. Docker sometimes will publish the
     # watchdog event before the file is actually fully on disk, causing
     # pikepdf to fail.
 
-    retries = RETRIES_LOADING_FILE
+    tries = retries_loading_file + 1
-    while retries:
+    while tries:
         try:
-            pdf = pikepdf.open(file_path)
+            with pikepdf.Pdf.open(file_path) as pdf:
-        except (FileNotFoundError, pikepdf.PdfError) as e:
+                log.debug(f"{file_path} ready with {pdf.pages} pages")
+                return True
+        except (FileNotFoundError, OSError) as e:
             log.info(f"File {file_path} is not ready yet")
             log.debug("Exception was", exc_info=e)
-            time.sleep(POLL_NEW_FILE_SECONDS)
+            time.sleep(poll_new_file_seconds)
-            retries -= 1
+            tries -= 1
-        else:
+        except pikepdf.PdfError as e:
-            pdf.close()
+            log.info(f"File {file_path} is not full written yet")
-            return True
+            log.debug("Exception was", exc_info=e)
+            time.sleep(poll_new_file_seconds)
+            tries -= 1
 
     return False
 
 
-def execute_ocrmypdf(file_path):
+def execute_ocrmypdf(
-    file_path = Path(file_path)
+    *,
-    output_path = get_output_dir(OUTPUT_DIRECTORY, file_path.name)
+    file_path: Path,
+    archive_dir: Path,
+    output_dir: Path,
+    deskew: bool,
+    ocrmypdf_kwargs: dict[str, Any],
+    on_success_delete: bool,
+    on_success_archive: bool,
+    poll_new_file_seconds: int,
+    retries_loading_file: int,
+    output_dir_year_month: bool,
+):
+    output_path = get_output_dir(output_dir, file_path.name, output_dir_year_month)
 
     log.info("-" * 20)
-    log.info(f'New file: {file_path}. Waiting until fully loaded...')
+    log.info(f'New file: {file_path}. Waiting until fully written...')
-    if not wait_for_file_ready(file_path):
+    if not wait_for_file_ready(file_path, poll_new_file_seconds, retries_loading_file):
         log.info(f"Gave up waiting for {file_path} to become ready")
         return
     log.info(f'Attempting to OCRmyPDF to: {output_path}')
     exit_code = ocrmypdf.ocr(
         input_file=file_path,
         output_file=output_path,
-        deskew=DESKEW,
+        deskew=deskew,
-        **OCR_JSON_SETTINGS,
+        **ocrmypdf_kwargs,
     )
     if exit_code == 0:
-        if ON_SUCCESS_DELETE:
+        if on_success_delete:
             log.info(f'OCR is done. Deleting: {file_path}')
             file_path.unlink()
-        elif ON_SUCCESS_ARCHIVE:
+        elif on_success_archive:
-            log.info(f'OCR is done. Archiving {file_path.name} to {ARCHIVE_DIRECTORY}')
+            log.info(f'OCR is done. Archiving {file_path.name} to {archive_dir}')
-            shutil.move(file_path, f'{ARCHIVE_DIRECTORY}/{file_path.name}')
+            shutil.move(file_path, f'{archive_dir}/{file_path.name}')
         else:
             log.info('OCR is done')
     else:
@@ -113,61 +128,202 @@ def execute_ocrmypdf(file_path):
 
 
 class HandleObserverEvent(PatternMatchingEventHandler):
+    def __init__(
+        self,
+        patterns=None,
+        ignore_patterns=None,
+        ignore_directories=False,
+        case_sensitive=False,
+        settings={},
+    ):
+        super().__init__(
+            patterns=patterns,
+            ignore_patterns=ignore_patterns,
+            ignore_directories=ignore_directories,
+            case_sensitive=case_sensitive,
+        )
+        self._settings = settings
+
     def on_any_event(self, event):
         if event.event_type in ['created']:
-            execute_ocrmypdf(event.src_path)
+            execute_ocrmypdf(event.src_path, **self._settings)
 
 
-def main():
+@app.command()
+def main(
+    input_dir: Annotated[
+        Path,
+        typer.Argument(
+            envvar='OCR_INPUT_DIRECTORY',
+            exists=True,
+            file_okay=False,
+            dir_okay=True,
+            readable=True,
+            resolve_path=True,
+        ),
+    ] = '/input',
+    output_dir: Annotated[
+        Path,
+        typer.Argument(
+            envvar='OCR_OUTPUT_DIRECTORY',
+            exists=True,
+            file_okay=False,
+            dir_okay=True,
+            writable=True,
+            resolve_path=True,
+        ),
+    ] = '/output',
+    archive_dir: Annotated[
+        Path,
+        typer.Argument(
+            envvar='OCR_ARCHIVE_DIRECTORY',
+            exists=True,
+            file_okay=False,
+            dir_okay=True,
+            writable=True,
+            resolve_path=True,
+        ),
+    ] = '/processed',
+    output_dir_year_month: Annotated[
+        bool,
+        typer.Option(
+            envvar='OCR_OUTPUT_DIRECTORY_YEAR_MONTH',
+            help='Create a subdirectory in the output directory for each year and month',
+        ),
+    ] = False,
+    on_success_delete: Annotated[
+        bool,
+        typer.Option(
+            envvar='OCR_ON_SUCCESS_DELETE',
+            help='Delete the input file after successful OCR',
+        ),
+    ] = False,
+    on_success_archive: Annotated[
+        bool,
+        typer.Option(
+            envvar='OCR_ON_SUCCESS_ARCHIVE',
+            help='Archive the input file after successful OCR',
+        ),
+    ] = False,
+    deskew: Annotated[
+        bool,
+        typer.Option(
+            envvar='OCR_DESKEW',
+            help='Deskew the input file before OCR',
+        ),
+    ] = False,
+    ocr_json_settings: Annotated[
+        typer.FileText,
+        typer.Option(
+            envvar='OCR_JSON_SETTINGS',
+            help='JSON settings to pass to OCRmyPDF',
+        ),
+    ] = None,
+    poll_new_file_seconds: Annotated[
+        int,
+        typer.Option(
+            envvar='OCR_POLL_NEW_FILE_SECONDS',
+            help='Seconds to wait before polling a new file',
+            min=0,
+        ),
+    ] = 1,
+    use_polling: Annotated[
+        bool,
+        typer.Option(
+            envvar='OCR_USE_POLLING',
+            help='Use polling instead of filesystem events',
+        ),
+    ] = False,
+    retries_loading_file: Annotated[
+        int,
+        typer.Option(
+            envvar='OCR_RETRIES_LOADING_FILE',
+            help='Number of times to retry loading a file before giving up',
+            min=0,
+        ),
+    ] = 5,
+    loglevel: Annotated[
+        LoggingLevelEnum,
+        typer.Option(
+            envvar='OCR_LOGLEVEL',
+            help='Logging level',
+        ),
+    ] = LoggingLevelEnum.INFO,
+    patterns: Annotated[
+        str,
+        typer.Option(
+            envvar='OCR_PATTERNS',
+            help='File patterns to watch',
+        ),
+    ] = '*.pdf,*.PDF',
+):
     ocrmypdf.configure_logging(
         verbosity=(
             ocrmypdf.Verbosity.default
-            if LOGLEVEL != 'DEBUG'
+            if loglevel != LoggingLevelEnum.DEBUG
             else ocrmypdf.Verbosity.debug
         ),
         manage_root_logger=True,
     )
-    log.setLevel(LOGLEVEL)
+    log.setLevel(loglevel.value)
     log.info(
         f"Starting OCRmyPDF watcher with config:\n"
-        f"Input Directory: {INPUT_DIRECTORY}\n"
+        f"Input Directory: {input_dir}\n"
-        f"Output Directory: {OUTPUT_DIRECTORY}\n"
+        f"Output Directory: {output_dir}\n"
-        f"Output Directory Year & Month: {OUTPUT_DIRECTORY_YEAR_MONTH}\n"
+        f"Output Directory Year & Month: {output_dir_year_month}\n"
-        f"Archive Directory: {ARCHIVE_DIRECTORY}"
+        f"Archive Directory: {archive_dir}"
     )
     log.debug(
-        f"INPUT_DIRECTORY: {INPUT_DIRECTORY}\n"
+        f"INPUT_DIRECTORY: {input_dir}\n"
-        f"OUTPUT_DIRECTORY: {OUTPUT_DIRECTORY}\n"
+        f"OUTPUT_DIRECTORY: {output_dir}\n"
-        f"OUTPUT_DIRECTORY_YEAR_MONTH: {OUTPUT_DIRECTORY_YEAR_MONTH}\n"
+        f"OUTPUT_DIRECTORY_YEAR_MONTH: {output_dir_year_month}\n"
-        f"ARCHIVE_DIRECTORY: {ARCHIVE_DIRECTORY}\n"
+        f"ARCHIVE_DIRECTORY: {archive_dir}\n"
-        f"ON_SUCCESS_DELETE: {ON_SUCCESS_DELETE}\n"
+        f"ON_SUCCESS_DELETE: {on_success_delete}\n"
-        f"ON_SUCCESS_ARCHIVE: {ON_SUCCESS_ARCHIVE}\n"
+        f"ON_SUCCESS_ARCHIVE: {on_success_archive}\n"
-        f"DESKEW: {DESKEW}\n"
+        f"DESKEW: {deskew}\n"
-        f"ARGS: {OCR_JSON_SETTINGS}\n"
+        f"ARGS: {ocr_json_settings}\n"
-        f"POLL_NEW_FILE_SECONDS: {POLL_NEW_FILE_SECONDS}\n"
+        f"POLL_NEW_FILE_SECONDS: {poll_new_file_seconds}\n"
-        f"RETRIES_LOADING_FILE: {RETRIES_LOADING_FILE}\n"
+        f"RETRIES_LOADING_FILE: {retries_loading_file}\n"
-        f"USE_POLLING: {USE_POLLING}\n"
+        f"USE_POLLING: {use_polling}\n"
-        f"LOGLEVEL: {LOGLEVEL}"
+        f"LOGLEVEL: {loglevel.value}"
     )
 
-    if 'input_file' in OCR_JSON_SETTINGS or 'output_file' in OCR_JSON_SETTINGS:
+    json_settings = json.loads(ocr_json_settings.read() if ocr_json_settings else '{}')
-        log.error('OCR_JSON_SETTINGS should not specify input file or output file')
+
+    if 'input_file' in json_settings or 'output_file' in json_settings:
+        log.error(
+            'OCR_JSON_SETTINGS (--ocr-json-settings) may not specify input/output file'
+        )
         sys.exit(1)
 
-    handler = HandleObserverEvent(patterns=PATTERNS)
+    handler = HandleObserverEvent(
-    if USE_POLLING:
+        patterns=patterns.split(','),
+        settings={
+            'archive_dir': archive_dir,
+            'output_dir': output_dir,
+            'deskew': deskew,
+            'ocrmypdf_kwargs': json_settings,
+            'on_success_delete': on_success_delete,
+            'on_success_archive': on_success_archive,
+            'poll_new_file_seconds': poll_new_file_seconds,
+            'retries_loading_file': retries_loading_file,
+            'output_dir_year_month': output_dir_year_month,
+        },
+    )
+    if use_polling:
         observer = PollingObserver()
     else:
         observer = Observer()
-    observer.schedule(handler, INPUT_DIRECTORY, recursive=True)
+    observer.schedule(handler, input_dir, recursive=True)
     observer.start()
+    typer.echo(f"Watching {input_dir} for new PDFs. Press Ctrl+C to exit.")
     try:
         while True:
-            time.sleep(1)
+            time.sleep(30)
     except KeyboardInterrupt:
         observer.stop()
     observer.join()
 
 
 if __name__ == "__main__":
-    main()
+    app()

pyproject.toml

@@ -12,7 +12,7 @@ readme = "README.md"
 license = { text = "MPL-2.0" }
 requires-python = ">=3.9"
 dependencies = [
-  "Pillow>=9.0.1",
+  "Pillow>=10.0.1",
   "deprecation>=2.1.0",
   "img2pdf>=0.4.4",
   "packaging>=20",
@@ -61,7 +61,7 @@ test = [
   "types-Pillow",
   "types-humanfriendly",
 ]
-watcher = ["watchdog>=1.0.2"]
+watcher = ["watchdog>=1.0.2", "typer[all]", "python-dotenv"]
 webservice = ["Flask>=2.0.1"]
 
 [project.scripts]
@@ -77,11 +77,11 @@ namespaces = false
 [tool.setuptools_scm]
 
 [tool.distutils.bdist_wheel]
-python-tag = "py38"
+python-tag = "py39"
 
 [tool.black]
 line-length = 88
-target-version = ["py38", "py39", "py310", "py311"]
+target-version = ["py39", "py310", "py311"]
 skip-string-normalization = true
 include = '\.pyi?$'
 exclude = '''
@@ -157,7 +157,7 @@ select = [
   "I001", # isort
   "UP",   # pyupgrade
 ]
-target-version = "py38"
+target-version = "py39"
 
 [tool.ruff.isort]
 known-first-party = ["ocrmypdf"]

src/ocrmypdf/_concurrent.py

@@ -7,7 +7,8 @@ from __future__ import annotations
 
 import threading
 from abc import ABC, abstractmethod
-from typing import Callable, Iterable
+from collections.abc import Iterable
+from typing import Callable
 
 
 def _task_noop(*_args, **_kwargs):

src/ocrmypdf/_exec/ghostscript.py

@@ -8,7 +8,6 @@ from __future__ import annotations
 import logging
 import os
 import re
-import sys
 from io import BytesIO
 from os import fspath
 from pathlib import Path
@@ -21,14 +20,6 @@ from ocrmypdf.exceptions import SubprocessOutputError
 from ocrmypdf.helpers import Resolution
 from ocrmypdf.subprocess import get_version, run, run_polling_stderr
 
-# Remove this workaround when we require Pillow >= 10
-try:
-    Transpose = Image.Transpose  # type: ignore
-except AttributeError:
-    # Pillow 9 shim
-    Transpose = Image  # type: ignore
-
-
 COLOR_CONVERSION_STRATEGIES = frozenset(
     [
         'CMYK',
@@ -138,11 +129,11 @@ def rasterize_pdf(
                 # rotation is a clockwise angle and Image.ROTATE_* is
                 # counterclockwise so this cancels out the rotation
                 if rotation == 90:
-                    im = im.transpose(Transpose.ROTATE_90)
+                    im = im.transpose(Image.Transpose.ROTATE_90)
                 elif rotation == 180:
-                    im = im.transpose(Transpose.ROTATE_180)
+                    im = im.transpose(Image.Transpose.ROTATE_180)
                 elif rotation == 270:
-                    im = im.transpose(Transpose.ROTATE_270)
+                    im = im.transpose(Image.Transpose.ROTATE_270)
                 if rotation % 180 == 90:
                     page_dpi = page_dpi.flip_axis()
             im.save(fspath(output_file), dpi=page_dpi)

src/ocrmypdf/_exec/tesseract.py

@@ -7,6 +7,7 @@ from __future__ import annotations
 
 import logging
 import re
+from contextlib import suppress
 from math import pi
 from os import fspath
 from pathlib import Path
@@ -350,7 +351,7 @@ def generate_hocr(
         tesseract_log_output(stdout)
         # The sidecar text file will get the suffix .txt; rename it to
         # whatever caller wants it named
-        if prefix.with_suffix('.txt').exists():
+        with suppress(FileNotFoundError):
             prefix.with_suffix('.txt').replace(output_text)
 
 
@@ -406,7 +407,7 @@ def generate_pdf(
     try:
         p = run(args_tesseract, stdout=PIPE, stderr=STDOUT, timeout=timeout, check=True)
         stdout = p.stdout
-        if prefix.with_suffix('.txt').exists():
+        with suppress(FileNotFoundError):
             prefix.with_suffix('.txt').replace(output_text)
     except TimeoutExpired:
         page_timedout(timeout)

src/ocrmypdf/_exec/unpaper.py

@@ -9,11 +9,12 @@ import logging
 import os
 import shlex
 import sys
+from collections.abc import Iterator
 from contextlib import contextmanager
 from decimal import Decimal
 from pathlib import Path
 from subprocess import PIPE, STDOUT
-from typing import Iterator, Union
+from typing import Union
 
 from packaging.version import Version
 from PIL import Image

src/ocrmypdf/_jobcontext.py

@@ -9,9 +9,9 @@ import os
 import shutil
 import sys
 from argparse import Namespace
+from collections.abc import Iterator
 from copy import copy
 from pathlib import Path
-from typing import Iterator
 
 from pluggy import PluginManager
 

src/ocrmypdf/_pipeline.py

@@ -10,11 +10,12 @@ import logging
 import os
 import re
 import sys
+from collections.abc import Iterable, Iterator, Sequence
 from contextlib import suppress
 from datetime import datetime, timezone
 from pathlib import Path
-from shutil import copyfileobj
+from shutil import copyfileobj, copystat
-from typing import Any, BinaryIO, Iterable, Iterator, Sequence, cast
+from typing import Any, BinaryIO, TypeVar, cast
 
 import img2pdf
 import pikepdf
@@ -32,6 +33,7 @@ from ocrmypdf.exceptions import (
     EncryptedPdfError,
     InputFileError,
     PriorOcrFoundError,
+    TaggedPDFError,
     UnsupportedImageFormatError,
 )
 from ocrmypdf.helpers import IMG2PDF_KWARGS, Resolution, safe_symlink
@@ -40,13 +42,7 @@ from ocrmypdf.pdfa import generate_pdfa_ps
 from ocrmypdf.pdfinfo import Colorspace, Encoding, PageInfo, PdfInfo
 from ocrmypdf.pluginspec import OrientationConfidence
 
-# Remove this workaround when we require Pillow >= 10
+T = TypeVar("T")
-try:
-    BICUBIC = Image.Resampling.BICUBIC  # type: ignore
-except AttributeError:  # pragma: no cover
-    # Pillow 9 shim
-    BICUBIC = Image.BICUBIC  # type: ignore
-
 log = logging.getLogger(__name__)
 
 VECTOR_PAGE_DPI = 400
@@ -145,6 +141,7 @@ def _pdf_guess_version(input_file: Path, search_window=1024) -> str:
 def triage(
     original_filename: str, input_file: Path, output_file: Path, options
 ) -> Path:
+    """Triage the input file. We can handle PDFs and images."""
     try:
         if _pdf_guess_version(input_file):
             if options.image_dpi:
@@ -173,6 +170,7 @@ def get_pdfinfo(
     max_workers: int | None = None,
     check_pages=None,
 ) -> PdfInfo:
+    """Get the PDF info."""
     try:
         return PdfInfo(
             input_file,
@@ -189,6 +187,7 @@ def get_pdfinfo(
 
 
 def validate_pdfinfo_options(context: PdfContext) -> None:
+    """Validate the PDF info options."""
     pdfinfo = context.pdfinfo
     options = context.options
 
@@ -220,6 +219,16 @@ def validate_pdfinfo_options(context: PdfContext) -> None:
                     "form and all filled form fields. The output PDF will be "
                     "'flattened' and will no longer be fillable."
                 )
+    if pdfinfo.is_tagged:
+        if options.force_ocr or options.skip_text or options.redo_ocr:
+            log.warning(
+                "This PDF is marked as a Tagged PDF. This often indicates "
+                "that the PDF was generated from an office document and does "
+                "not need OCR. PDF pages processed by OCRmyPDF may not be "
+                "tagged correctly."
+            )
+        else:
+            raise TaggedPDFError()
     context.plugin_manager.hook.validate(pdfinfo=pdfinfo, options=options)
 
 
@@ -434,6 +443,7 @@ def get_orientation_correction(preview: Path, page_context: PageContext) -> int:
 
 
 def calculate_image_dpi(page_context: PageContext) -> Resolution:
+    """Calculate the DPI for the page image."""
     pageinfo = page_context.pageinfo
     dpi_profile = pageinfo.page_dpi_profile()
     if dpi_profile and dpi_profile.average_to_max_dpi_ratio < 0.8:
@@ -532,6 +542,7 @@ def rasterize(
 
 
 def preprocess_remove_background(input_file: Path, page_context: PageContext) -> Path:
+    """Remove the background from the input image (temporarily disabled)."""
     if any(image.bpc > 1 for image in page_context.pageinfo.images):
         raise NotImplementedError("--remove-background is temporarily not implemented")
         # output_file = page_context.get_path('pp_rm_bg.png')
@@ -562,7 +573,7 @@ def preprocess_deskew(input_file: Path, page_context: PageContext) -> Path:
         # resampling if image is mode '1' or 'P'
         deskewed = im.rotate(
             deskew_angle_degrees,
-            resample=BICUBIC,
+            resample=Image.Resampling.BICUBIC,
             fillcolor=ImageColor.getcolor('white', mode=im.mode),  # type: ignore
         )
         deskewed.save(output_file, dpi=dpi)
@@ -571,6 +582,7 @@ def preprocess_deskew(input_file: Path, page_context: PageContext) -> Path:
 
 
 def preprocess_clean(input_file: Path, page_context: PageContext) -> Path:
+    """Clean the input image using unpaper."""
     output_file = page_context.get_path('pp_clean.png')
     dpi = get_page_square_dpi(page_context, calculate_image_dpi(page_context))
     return unpaper.clean(
@@ -631,6 +643,7 @@ def create_ocr_image(image: Path, page_context: PageContext) -> Path:
 
 
 def ocr_engine_hocr(input_file: Path, page_context: PageContext) -> tuple[Path, Path]:
+    """Run the OCR engine and generate hOCR output."""
     hocr_out = page_context.get_path('ocr_hocr.hocr')
     hocr_text_out = page_context.get_path('ocr_hocr.txt')
     options = page_context.options
@@ -662,6 +675,10 @@ def should_visible_page_image_use_jpg(pageinfo: PageInfo) -> bool:
 
 
 def create_visible_page_jpg(image: Path, page_context: PageContext) -> Path:
+    """Create a visible page image in JPEG format.
+
+    This is intended to be used when all images on the page were originally JPEGs.
+    """
     output_file = page_context.get_path('visible.jpg')
     with Image.open(image) as im:
         # At this point the image should be a .png, but deskew, unpaper
@@ -683,6 +700,7 @@ def create_visible_page_jpg(image: Path, page_context: PageContext) -> Path:
 def create_pdf_page_from_image(
     image: Path, page_context: PageContext, orientation_correction: int
 ) -> Path:
+    """Create a PDF page from a page image."""
     # We rasterize a square DPI version of each page because most image
     # processing tools don't support rectangular DPI. Use the square DPI as it
     # accurately describes the image. It would be possible to resample the image
@@ -714,6 +732,7 @@ def create_pdf_page_from_image(
 
 
 def render_hocr_page(hocr: Path, page_context: PageContext) -> Path:
+    """Render the hOCR page to a PDF."""
     options = page_context.options
     output_file = page_context.get_path('ocr_hocr.pdf')
     dpi = get_page_square_dpi(page_context, calculate_image_dpi(page_context))
@@ -733,6 +752,7 @@ def render_hocr_page(hocr: Path, page_context: PageContext) -> Path:
 def ocr_engine_textonly_pdf(
     input_image: Path, page_context: PageContext
 ) -> tuple[Path, Path]:
+    """Run the OCR engine and generate a text-only PDF (will look blank)."""
     output_pdf = page_context.get_path('ocr_tess.pdf')
     output_text = page_context.get_path('ocr_tess.txt')
     options = page_context.options
@@ -748,6 +768,7 @@ def ocr_engine_textonly_pdf(
 
 
 def get_docinfo(base_pdf: pikepdf.Pdf, context: PdfContext) -> dict[str, str]:
+    """Read the document info and store it in a dictionary."""
     options = context.options
 
     def from_document_info(key):
@@ -793,6 +814,14 @@ def generate_postscript_stub(context: PdfContext) -> Path:
 
 
 def convert_to_pdfa(input_pdf: Path, input_ps_stub: Path, context: PdfContext) -> Path:
+    """Converts the given PDF to PDF/A.
+
+    Args:
+        input_pdf: The input PDF file path (presumably not PDF/A).
+        input_ps_stub: The input PostScript file path, containing instructions
+            for the PDF/A generator to use.
+        context: The PDF context.
+    """
     options = context.options
     input_pdfinfo = context.pdfinfo
     fix_docinfo_file = context.get_path('fix_docinfo.pdf')
@@ -847,6 +876,10 @@ def _repair_docinfo_nuls(pdf):
 
 
 def should_linearize(working_file: Path, context: PdfContext) -> bool:
+    """Determine whether the PDF should be linearized.
+
+    For smaller files, linearization is not worth the effort.
+    """
     filesize = os.stat(working_file).st_size
     if filesize > (context.options.fast_web_view * 1_000_000):
         return True
@@ -854,6 +887,11 @@ def should_linearize(working_file: Path, context: PdfContext) -> bool:
 
 
 def get_pdf_save_settings(output_type: str) -> dict[str, Any]:
+    """Get pikepdf.Pdf.save settings for the given output type.
+
+    Essentially, don't use features that are incompatible with a given
+    PDF/A specification.
+    """
     if output_type == 'pdfa-1':
         # Trigger recompression to ensure object streams are removed, because
         # Acrobat complains about them in PDF/A-1b validation.
@@ -872,6 +910,11 @@ def get_pdf_save_settings(output_type: str) -> dict[str, Any]:
 
 
 def metadata_fixup(working_file: Path, context: PdfContext) -> Path:
+    """Fix certain metadata fields after Ghostscript PDF/A conversion.
+
+    Also report on metadata in the input file that was not retained during
+    PDF/A conversion.
+    """
     output_file = context.get_path('metafix.pdf')
     options = context.options
 
@@ -894,7 +937,9 @@ def metadata_fixup(working_file: Path, context: PdfContext) -> Path:
 
     with pikepdf.open(context.origin) as original, pikepdf.open(working_file) as pdf:
         docinfo = get_docinfo(original, context)
-        with pdf.open_metadata() as meta_pdf:
+        with original.open_metadata(
+            set_pikepdf_as_editor=False, update_docinfo=False, strict=False
+        ) as meta_original, pdf.open_metadata() as meta_pdf:
             meta_pdf.load_from_docinfo(
                 docinfo, delete_missing=False, raise_failure=False
             )
@@ -902,37 +947,33 @@ def metadata_fixup(working_file: Path, context: PdfContext) -> Path:
             # ensure consistency with Ghostscript.
             if 'xmp:CreateDate' not in meta_pdf:
                 meta_pdf['xmp:CreateDate'] = meta_pdf.get('xmp:ModifyDate', '')
-
+            if meta_pdf.get('dc:title') == 'Untitled':
-            with original.open_metadata(
+                # Ghostscript likes to set title to Untitled if omitted from input.
-                set_pikepdf_as_editor=False, update_docinfo=False, strict=False
+                # Reverse this, because PDF/A TechNote 0003:Metadata in PDF/A-1
-            ) as meta_original:
+                # and the XMP Spec do not make this recommendation.
-                if meta_pdf.get('dc:title') == 'Untitled':
+                if 'dc:title' not in meta_original:
-                    # Ghostscript likes to set title to Untitled if omitted from input.
+                    del meta_pdf['dc:title']
-                    # Reverse this, because PDF/A TechNote 0003:Metadata in PDF/A-1
+            # If the user explicitly specified an empty string for any of the
-                    # and the XMP Spec do not make this recommendation.
+            # following, they should be unset and not reported as missing in
-                    if 'dc:title' not in meta_original:
+            # the output pdf. Note that some metadata fields use differing names
-                        del meta_pdf['dc:title']
+            # between PDF-A and PDF.
-                # If the user explicitly specified an empty string for any of the
+            for meta in [meta_pdf, meta_original]:
-                # following, they should be unset and not reported as missing in
+                if options.title == '' and 'dc:title' in meta:
-                # the output pdf. Note that some metadata fields use differing names
+                    del meta['dc:title']  # PDF-A and PDF
-                # between PDF-A and PDF.
+                if options.author == '':
-                for meta in [meta_pdf, meta_original]:
+                    if 'dc:creator' in meta:
-                    if options.title == '' and 'dc:title' in meta:
+                        del meta['dc:creator']  # PDF-A (Not xmp:CreatorTool)
-                        del meta['dc:title']  # PDF-A and PDF
+                    if 'pdf:Author' in meta:
-                    if options.author == '':
+                        del meta['pdf:Author']  # PDF
-                        if 'dc:creator' in meta:
+                if options.subject == '':
-                            del meta['dc:creator']  # PDF-A (Not xmp:CreatorTool)
+                    if 'dc:description' in meta:
-                        if 'pdf:Author' in meta:
+                        del meta['dc:description']  # PDF-A
-                            del meta['pdf:Author']  # PDF
+                    if 'dc:subject' in meta:
-                    if options.subject == '':
+                        del meta['dc:subject']  # PDF
-                        if 'dc:description' in meta:
+                if options.keywords == '' and 'pdf:Keywords' in meta:
-                            del meta['dc:description']  # PDF-A
+                    del meta['pdf:Keywords']  # PDF-A and PDF
-                        if 'dc:subject' in meta:
+            meta_missing = set(meta_original.keys()) - set(meta_pdf.keys())
-                            del meta['dc:subject']  # PDF
+            report_on_metadata(meta_missing)
-                    if options.keywords == '' and 'pdf:Keywords' in meta:
-                        del meta['pdf:Keywords']  # PDF-A and PDF
-                meta_missing = set(meta_original.keys()) - set(meta_pdf.keys())
-                report_on_metadata(meta_missing)
 
         optimizing = context.plugin_manager.hook.is_optimization_enabled(
             context=context
@@ -974,6 +1015,7 @@ def _file_size_ratio(
 def optimize_pdf(
     input_file: Path, context: PdfContext, executor: Executor
 ) -> tuple[Path, Sequence[str]]:
+    """Optimize the given PDF file."""
     output_file = context.get_path('optimize.pdf')
     output_pdf, messages = context.plugin_manager.hook.optimize_pdf(
         input_pdf=input_file,
@@ -993,8 +1035,8 @@ def optimize_pdf(
 
 
 def enumerate_compress_ranges(
-    iterable: Iterable,
+    iterable: Iterable[T],
-) -> Iterator[tuple[tuple[int, int], Any]]:
+) -> Iterator[tuple[tuple[int, int], T]]:
     """Enumerate the ranges of non-empty elements in an iterable.
 
     Compresses consecutive ranges of length 1 into single elements.
@@ -1022,21 +1064,22 @@ def enumerate_compress_ranges(
 
 
 def merge_sidecars(txt_files: Iterable[Path | None], context: PdfContext) -> Path:
+    """Merge the page sidecar files into a single file.
+
+    Sidecar files are created by the OCR engine and contain the text for each
+    page in the PDF. This function merges the sidecar files into a single file
+    and returns the path to the merged file.
+    """
     output_file = context.get_path('sidecar.txt')
     with open(output_file, 'w', encoding="utf-8") as stream:
         for (from_, to_), txt_file in enumerate_compress_ranges(txt_files):
             if from_ != 1:
-                stream.write('\f')  # Form feed between pages
+                stream.write('\f')  # Form feed between pages for all pages after first
             if txt_file:
-                with open(txt_file, encoding="utf-8") as in_:
+                txt = txt_file.read_text(encoding="utf-8")
-                    txt = in_.read()
+                # Some versions of Tesseract add a form feed at the end and
-                    # Some OCR engines (e.g. Tesseract v4 alpha) add form feeds
+                # others don't. Remove it if it exists, since we add one manually.
-                    # between pages, and some do not. For consistency, we ignore
+                stream.write(txt.removesuffix('\f'))
-                    # any added by the OCR engine and them on our own.
-                    if txt.endswith('\f'):
-                        stream.write(txt[:-1])
-                    else:
-                        stream.write(txt)
             else:
                 if from_ != to_:
                     pages = f'{from_}-{to_}'
@@ -1047,14 +1090,14 @@ def merge_sidecars(txt_files: Iterable[Path | None], context: PdfContext) -> Pat
 
 
 def copy_final(
-    input_file: Path, output_file: str | Path | BinaryIO, _context: PdfContext
+    input_file: Path, output_file: str | Path | BinaryIO, context: PdfContext
 ) -> None:
     """Copy the final temporary file to the output destination.
 
     Args:
-        input_file (Path): The input file to copy.
+        input_file (Path): The intermediate input file to copy.
         output_file (str | Path | BinaryIO): The output file to copy to.
-        _context (PdfContext): The PDF context.
+        context (PdfContext): The PDF context.
 
     Returns:
         None
@@ -1073,5 +1116,11 @@ def copy_final(
             # At this point we overwrite the output_file specified by the user
             # use copyfileobj because then we use open() to create the file and
             # get the appropriate umask, ownership, etc.
-            with open(output_file, 'wb') as output_stream:
+            with open(output_file, 'w+b') as output_stream:
                 copyfileobj(input_stream, output_stream)
+            # Attempt to copy file attributes from input to output
+            with suppress(OSError):
+                # Copy original file's permissions, ownership, etc. if possible
+                copystat(context.options.input_file, output_file)
+                # Set output file's modification time to now
+                Path(output_file).touch(exist_ok=True)

src/ocrmypdf/_plugin_manager.py

@@ -10,8 +10,8 @@ import importlib
 import importlib.util
 import pkgutil
 import sys
+from collections.abc import Sequence
 from pathlib import Path
-from typing import Sequence
 
 import pluggy
 

src/ocrmypdf/_sync.py

@@ -13,12 +13,13 @@ import logging.handlers
 import os
 import sys
 import threading
+from collections.abc import Sequence
 from concurrent.futures.process import BrokenProcessPool
 from concurrent.futures.thread import BrokenThreadPool
 from functools import partial
 from pathlib import Path
 from tempfile import mkdtemp
-from typing import NamedTuple, Sequence, cast
+from typing import NamedTuple, cast
 
 import PIL
 
@@ -104,6 +105,7 @@ def preprocess(
     deskew: bool,
     clean: bool,
 ) -> Path:
+    """Preprocess an image."""
     if remove_background:
         image = preprocess_remove_background(image, page_context)
     if deskew:
@@ -116,6 +118,7 @@ def preprocess(
 def make_intermediate_images(
     page_context: PageContext, orientation_correction: int
 ) -> tuple[Path, Path | None]:
+    """Create intermediate and preprocessed images for OCR."""
     options = page_context.options
 
     ocr_image = preprocess_out = None
@@ -173,6 +176,7 @@ def make_intermediate_images(
 
 
 def exec_page_sync(page_context: PageContext) -> PageResult:
+    """Execute a pipeline for a single page synchronously."""
     options = page_context.options
     tls.pageno = page_context.pageno + 1
 
@@ -233,6 +237,7 @@ def exec_page_sync(page_context: PageContext) -> PageResult:
 def post_process(
     pdf_file: Path, context: PdfContext, executor: Executor
 ) -> tuple[Path, Sequence[str]]:
+    """Postprocess the PDF file."""
     pdf_out = pdf_file
     if context.options.output_type.startswith('pdfa'):
         ps_stub_out = generate_postscript_stub(context)
@@ -243,6 +248,7 @@ def post_process(
 
 
 def worker_init(max_pixels: int) -> None:
+    """Initialize a worker thread or process."""
     # In Windows, child process will not inherit our change to this value in
     # the parent process, so ensure workers get it set. Not needed when running
     # threaded, but harmless to set again.
@@ -251,8 +257,8 @@ def worker_init(max_pixels: int) -> None:
 
 
 def exec_concurrent(context: PdfContext, executor: Executor) -> Sequence[str]:
-    """Execute the pipeline concurrently."""
+    """Execute the OCR pipeline concurrently."""
-    # Run exec_page_sync on every page context
+    # Run exec_page_sync on every page
     options = context.options
     max_workers = min(len(context.pdfinfo), options.jobs)
     if max_workers > 1:
@@ -262,6 +268,7 @@ def exec_concurrent(context: PdfContext, executor: Executor) -> Sequence[str]:
     ocrgraft = OcrGrafter(context)
 
     def update_page(result: PageResult, pbar):
+        """After OCR is complete for a page, update the PDF."""
         try:
             tls.pageno = result.pageno + 1
             sidecars[result.pageno] = result.text
@@ -317,7 +324,7 @@ def configure_debug_logging(
 ) -> logging.FileHandler:
     """Create a debug log file at a specified location.
 
-    Arguments:
+    Args:
         log_filename: Where to the put the log file.
         prefix: The logging domain prefix that should be sent to the log.
     """
@@ -338,6 +345,17 @@ def run_pipeline(
     plugin_manager: OcrmypdfPluginManager | None,
     api: bool = False,
 ) -> ExitCode:
+    """Run the OCR pipeline.
+
+    Args:
+        options: The parsed command line options.
+        plugin_manager: The plugin manager to use. If not provided, one will be
+            created.
+        api: If ``True``, the pipeline is being run from the API. This is used
+            to manage exceptions in a way appropriate for API or CLI usage.
+            For CLI (``api=False``), exceptions are printed and described;
+            for API use, they are propagated to the caller.
+    """
     # Any changes to options will not take effect for options that are already
     # bound to function parameters in the pipeline. (For example
     # options.input_file, options.pdf_renderer are already bound.)

src/ocrmypdf/_validation.py

@@ -12,9 +12,9 @@ import os
 import sys
 import unicodedata
 from argparse import Namespace
+from collections.abc import Sequence
 from pathlib import Path
 from shutil import copyfileobj
-from typing import Sequence
 
 import pikepdf
 import PIL
@@ -44,7 +44,7 @@ log = logging.getLogger(__name__)
 
 def check_platform() -> None:
     if sys.maxsize <= 2**32:  # pragma: no cover
-        log.error(
+        log.warning(
             "You are running OCRmyPDF in a 32-bit (x86) Python interpreter. "
             "This is not supported. 32-bit does not have enough address space "
             "to process large files. "
@@ -245,6 +245,18 @@ def check_options(options: Namespace, plugin_manager: PluginManager) -> None:
     _check_plugin_options(options, plugin_manager)
 
 
+def _in_docker():
+    return Path('/.dockerenv').exists()
+
+
+def _in_snap():
+    try:
+        cgroup_text = Path('/proc/self/cgroup').read_text()
+        return 'snap.ocrmypdf' in cgroup_text
+    except FileNotFoundError:
+        return False
+
+
 def create_input_file(options: Namespace, work_folder: Path) -> tuple[Path, str]:
     if options.input_file == '-':
         # stdin
@@ -268,7 +280,7 @@ def create_input_file(options: Namespace, work_folder: Path) -> tuple[Path, str]
             return target, os.fspath(options.input_file)
         except FileNotFoundError as e:
             msg = f"File not found - {options.input_file}"
-            if Path('/.dockerenv').exists():  # pragma: no cover
+            if _in_docker():  # pragma: no cover
                 msg += (
                     "\nDocker cannot your working directory unless you "
                     "explicitly share it with the Docker container and set up"
@@ -278,6 +290,15 @@ def create_input_file(options: Namespace, work_folder: Path) -> tuple[Path, str]
                     "\tdocker run -i --rm jbarlow83/ocrmypdf - - <input.pdf >output.pdf"
                     "\n"
                 )
+            elif _in_snap():  # pragma: no cover
+                msg += (
+                    "\nSnap applications cannot access files outside of "
+                    "your home directory unless you explicitly allow it. "
+                    "You may find it easier to use stdin/stdout:"
+                    "\n"
+                    "\tsnap run ocrmypdf - - <input.pdf >output.pdf"
+                    "\n"
+                )
             raise InputFileError(msg) from e
 
 

src/ocrmypdf/api.py

@@ -10,10 +10,11 @@ import os
 import sys
 import threading
 from argparse import Namespace
+from collections.abc import Iterable
 from enum import IntEnum
 from io import IOBase
 from pathlib import Path
-from typing import AnyStr, BinaryIO, Iterable, Union
+from typing import AnyStr, BinaryIO, Union
 from warnings import warn
 
 import pluggy
@@ -209,7 +210,7 @@ def create_options(
     return options
 
 
-def ocr(  # noqa: ruff: disable=D417
+def ocr(  # noqa: D417
     input_file: PathOrIO,
     output_file: PathOrIO,
     *,
@@ -240,6 +241,7 @@ def ocr(  # noqa: ruff: disable=D417
     png_quality: int | None = None,
     jbig2_lossy: bool | None = None,
     jbig2_page_group_size: int | None = None,
+    jbig2_threshold: float | None = None,
     pages: str | None = None,
     max_image_mpixels: float | None = None,
     tesseract_config: Iterable[str] | None = None,
@@ -249,12 +251,16 @@ def ocr(  # noqa: ruff: disable=D417
     pdf_renderer: str | None = None,
     tesseract_timeout: float | None = None,
     tesseract_non_ocr_timeout: float | None = None,
+    tesseract_downsample_above: int | None = None,
+    tesseract_downsample_large_images: bool | None = None,
     rotate_pages_threshold: float | None = None,
     pdfa_image_compression: str | None = None,
+    color_conversion_strategy: str | None = None,
     user_words: os.PathLike | None = None,
     user_patterns: os.PathLike | None = None,
     fast_web_view: float | None = None,
     continue_on_soft_render_error: bool | None = None,
+    invalidate_digital_signatures: bool | None = None,
     plugins: Iterable[StrPath] | None = None,
     plugin_manager=None,
     keep_temporary_files: bool | None = None,

src/ocrmypdf/builtin_plugins/concurrency.py

@@ -12,9 +12,10 @@ import queue
 import signal
 import sys
 import threading
+from collections.abc import Iterable
 from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed
 from contextlib import suppress
-from typing import Callable, Iterable, Type, Union
+from typing import Callable, Union
 
 from rich.console import Console as RichConsole
 
@@ -23,7 +24,7 @@ from ocrmypdf._logging import RichLoggingHandler, RichTqdmProgressAdapter
 from ocrmypdf.exceptions import InputFileError
 from ocrmypdf.helpers import remove_all_log_handlers
 
-FuturesExecutorClass = Union[Type[ThreadPoolExecutor], Type[ProcessPoolExecutor]]
+FuturesExecutorClass = Union[type[ThreadPoolExecutor], type[ProcessPoolExecutor]]
 Queue = Union[multiprocessing.Queue, queue.Queue]
 UserInit = Callable[[], None]
 WorkerInit = Callable[[Queue, UserInit, int], None]
@@ -118,6 +119,11 @@ class StandardExecutor(Executor):
         # Regardless of whether we use_threads for worker processes, the log_listener
         # must be a thread. Make sure we create the listener after the worker pool,
         # so that it does not get forked into the workers.
+        # If use_threads is False, we are currently guilty of creating a thread before
+        # forking on Linux, which is not recommended. However, we take a big
+        # performance hit in pdfinfo if we can't fork. Long term solution is to
+        # replace most of this with an asyncio implementation, and probably to
+        # migrate some of pdfinfo into C++ or Rust.
         listener = threading.Thread(target=log_listener, args=(log_queue,))
         listener.start()
 

src/ocrmypdf/builtin_plugins/optimize.py

@@ -6,8 +6,8 @@ from __future__ import annotations
 
 import argparse
 import logging
+from collections.abc import Sequence
 from pathlib import Path
-from typing import Sequence
 
 from ocrmypdf import Executor, PdfContext, hookimpl
 from ocrmypdf._exec import jbig2enc, pngquant

src/ocrmypdf/cli.py

@@ -6,7 +6,8 @@
 from __future__ import annotations
 
 import argparse
-from typing import Any, Callable, Mapping, TypeVar
+from collections.abc import Mapping
+from typing import Any, Callable, TypeVar
 
 from ocrmypdf._version import PROGRAM_NAME as _PROGRAM_NAME
 from ocrmypdf._version import __version__ as _VERSION

src/ocrmypdf/exceptions.py

@@ -108,10 +108,16 @@ class EncryptedPdfError(ExitCodeException):
     )
 
 
-class DigitalSignatureError(ExitCodeException):
+class TesseractConfigError(ExitCodeException):
+    """Tesseract config can't be parsed."""
+
+    exit_code = ExitCode.invalid_config
+    message = "Error occurred while parsing a Tesseract configuration file"
+
+
+class DigitalSignatureError(InputFileError):
     """PDF has a digital signature."""
 
-    exit_code = ExitCode.input_file
     message = dedent(
         """\
         Input PDF has a digital signature. OCR would alter the document,
@@ -120,8 +126,14 @@ class DigitalSignatureError(ExitCodeException):
     )
 
 
-class TesseractConfigError(ExitCodeException):
+class TaggedPDFError(InputFileError):
-    """Tesseract config can't be parsed."""
+    """PDF is tagged."""
 
-    exit_code = ExitCode.invalid_config
+    message = dedent(
-    message = "Error occurred while parsing a Tesseract configuration file"
+        """\
+        This PDF is marked as a Tagged PDF. This often indicates
+        that the PDF was generated from an office document and does
+        not need OCR. Use --force-ocr, --skip-text or --redo-ocr to
+        override this error.
+        """
+    )

src/ocrmypdf/extra_plugins/semfree.py

@@ -20,12 +20,13 @@ from __future__ import annotations
 import logging
 import logging.handlers
 import signal
+from collections.abc import Iterable, Iterator
 from contextlib import suppress
 from enum import Enum, auto
 from itertools import islice, repeat, takewhile, zip_longest
 from multiprocessing import Pipe, Process
 from multiprocessing.connection import Connection, wait
-from typing import Callable, Iterable, Iterator
+from typing import Callable
 
 from ocrmypdf import Executor, hookimpl
 from ocrmypdf._concurrent import NullProgressBar

src/ocrmypdf/helpers.py

@@ -10,7 +10,7 @@ import multiprocessing
 import os
 import shutil
 import warnings
-from collections.abc import Iterable
+from collections.abc import Iterable, Sequence
 from contextlib import suppress
 from decimal import Decimal
 from io import StringIO
@@ -21,9 +21,6 @@ from typing import (
     Any,
     Callable,
     Generic,
-    Sequence,
-    SupportsFloat,
-    SupportsRound,
     TypeVar,
 )
 
@@ -324,8 +321,10 @@ def remove_all_log_handlers(logger: logging.Logger) -> None:
 def pikepdf_enable_mmap() -> None:
     """Enable pikepdf mmap."""
     try:
-        if pikepdf._core.set_access_default_mmap(True):
+        pikepdf._core.set_access_default_mmap(True)
-            log.debug("pikepdf mmap enabled")
+        log.debug(
+            "pikepdf mmap "
+            + ('enabled' if pikepdf._core.get_access_default_mmap() else 'disabled')
+        )
     except AttributeError:
         log.debug("pikepdf mmap not available")
-    log.debug("pikepdf mmap disabled")

src/ocrmypdf/imageops.py

@@ -7,18 +7,10 @@ from __future__ import annotations
 
 import logging
 from math import floor, sqrt
-from typing import Optional, Tuple
+from typing import Optional
 
 from PIL import Image
 
-# Remove this workaround when we require Pillow >= 9.1.0
-try:
-    Resampling = Image.Resampling  # type: ignore
-except AttributeError:
-    # Pillow 9 shim
-    Resampling = Image  # type: ignore
-
-
 # While from __future__ import annotations, we use singledispatch here, which
 # does not support annotations. Disable check about using old-style typing
 # until Python 3.10, OR when drop singledispatch in ocrmypdf 15.
@@ -43,13 +35,13 @@ def bytes_per_pixel(mode: str) -> int:
 
 
 def _calculate_downsample(
-    image_size: Tuple[int, int],
+    image_size: tuple[int, int],
     bytes_per_pixel: int,
     *,
-    max_size: Optional[Tuple[int, int]] = None,
+    max_size: Optional[tuple[int, int]] = None,
     max_pixels: Optional[int] = None,
     max_bytes: Optional[int] = None,
-) -> Tuple[int, int]:
+) -> tuple[int, int]:
     """Calculate image size required to downsample an image to fit limits.
 
     If no limit is exceeded, the input image's size is returned.
@@ -106,10 +98,10 @@ def _calculate_downsample(
 def calculate_downsample(
     image: Image.Image,
     *,
-    max_size: Optional[Tuple[int, int]] = None,
+    max_size: Optional[tuple[int, int]] = None,
     max_pixels: Optional[int] = None,
     max_bytes: Optional[int] = None,
-) -> Tuple[int, int]:
+) -> tuple[int, int]:
     """Calculate image size required to downsample an image to fit limits.
 
     If no limit is exceeded, the input image's size is returned.
@@ -135,7 +127,7 @@ def downsample_image(
     image: Image.Image,
     new_size: tuple[int, int],
     *,
-    resample_mode: Image.Resampling = Resampling.BICUBIC,
+    resample_mode: Image.Resampling = Image.Resampling.BICUBIC,
     reducing_gap: int = 3,
 ) -> Image.Image:
     """Downsample an image to fit within the given limits.

src/ocrmypdf/optimize.py

@@ -11,9 +11,10 @@ import sys
 import tempfile
 import threading
 from collections import defaultdict
+from collections.abc import Iterator, MutableSet, Sequence
 from os import fspath
 from pathlib import Path
-from typing import Callable, Iterator, MutableSet, NamedTuple, NewType, Sequence
+from typing import Callable, NamedTuple, NewType
 from zlib import compress
 
 import img2pdf

src/ocrmypdf/pdfa.py

@@ -6,9 +6,9 @@
 from __future__ import annotations
 
 import base64
+from collections.abc import Iterator
 from importlib.resources import files as package_files
 from pathlib import Path
-from typing import Iterator
 
 import pikepdf
 

src/ocrmypdf/pdfinfo/info.py

@@ -12,6 +12,7 @@ import re
 import statistics
 import sys
 from collections import defaultdict
+from collections.abc import Container, Iterable, Iterator, Mapping, Sequence
 from contextlib import ExitStack
 from decimal import Decimal
 from enum import Enum, auto
@@ -19,9 +20,10 @@ from functools import partial
 from math import hypot, inf, isclose
 from os import PathLike
 from pathlib import Path
-from typing import Container, Iterable, Iterator, Mapping, NamedTuple, Sequence, Tuple
+from typing import Callable, NamedTuple
 from warnings import warn
 
+from pdfminer.layout import LTPage, LTTextBox
 from pikepdf import (
     Name,
     Object,
@@ -37,7 +39,7 @@ from pikepdf import (
 from ocrmypdf._concurrent import Executor, SerialExecutor
 from ocrmypdf.exceptions import EncryptedPdfError, InputFileError
 from ocrmypdf.helpers import Resolution, available_cpu_count, pikepdf_enable_mmap
-from ocrmypdf.pdfinfo.layout import get_page_analysis, get_text_boxes
+from ocrmypdf.pdfinfo.layout import LTStateAwareChar, get_page_analysis, get_text_boxes
 
 logger = logging.getLogger()
 
@@ -73,7 +75,7 @@ class Encoding(Enum):
     runlength = auto()
 
 
-FloatRect = Tuple[float, float, float, float]
+FloatRect = tuple[float, float, float, float]
 
 FRIENDLY_COLORSPACE: dict[str, Colorspace] = {
     '/DeviceGray': Colorspace.gray,
@@ -654,7 +656,9 @@ def _page_has_text(text_blocks: Iterable[FloatRect], page_width, page_height) ->
     return has_text
 
 
-def simplify_textboxes(miner, textbox_getter) -> Iterator[TextboxInfo]:
+def simplify_textboxes(
+    miner: LTPage, textbox_getter: Callable[[LTPage], Iterator[LTTextBox]]
+) -> Iterator[TextboxInfo]:
     """Extract only limited content from text boxes.
 
     We do this to save memory and ensure that our objects are pickleable.
@@ -662,7 +666,8 @@ def simplify_textboxes(miner, textbox_getter) -> Iterator[TextboxInfo]:
     for box in textbox_getter(miner):
         first_line = box._objs[0]  # pylint: disable=protected-access
         first_char = first_line._objs[0]  # pylint: disable=protected-access
-
+        if not isinstance(first_char, LTStateAwareChar):
+            continue
         visible = first_char.rendermode != 3
         corrupt = first_char.get_text() == '\ufffd'
         yield TextboxInfo(box.bbox, visible, corrupt)
@@ -821,7 +826,10 @@ class PageInfo:
         if check_this_page and detailed_analysis:
             pscript5_mode = str(pdf.docinfo.get(Name.Creator)).startswith('PScript5')
             miner = get_page_analysis(infile, pageno, pscript5_mode)
-            self._textboxes = list(simplify_textboxes(miner, get_text_boxes))
+            if miner is not None:
+                self._textboxes = list(simplify_textboxes(miner, get_text_boxes))
+            else:
+                self._textboxes = []
             bboxes = (box.bbox for box in self._textboxes)
 
             self._has_text = _page_has_text(bboxes, width_pt, height_pt)
@@ -937,7 +945,9 @@ class PageInfo:
     def get_textareas(self, visible: bool | None = None, corrupt: bool | None = None):
         """Return textareas bounding boxes in PDF coordinates on the page."""
 
-        def predicate(obj, want_visible, want_corrupt):
+        def predicate(
+            obj: TextboxInfo, want_visible: bool | None, want_corrupt: bool | None
+        ) -> bool:
             result = True
             if want_visible is not None:
                 if obj.is_visible != want_visible:
@@ -1028,7 +1038,11 @@ DEFAULT_EXECUTOR = SerialExecutor()
 
 
 class PdfInfo:
-    """Get summary information about a PDF."""
+    """Extract summary information about a PDF without retaining the PDF itself.
+
+    Crucially this lets us get the information in a pure Python format so that
+    it can be pickled and passed to a worker process.
+    """
 
     _has_acroform: bool = False
     _has_signature: bool = False
@@ -1068,6 +1082,9 @@ class PdfInfo:
                 elif Name.XFA in pdf.Root.AcroForm:
                     self._has_acroform = True
                 self._has_signature = bool(pdf.Root.AcroForm.get(Name.SigFlags, 0) & 1)
+            self._is_tagged = bool(
+                pdf.Root.get(Name.MarkInfo, {}).get(Name.Marked, False)
+            )
 
     @property
     def pages(self) -> Sequence[PageInfo | None]:
@@ -1095,6 +1112,11 @@ class PdfInfo:
         """Return True if the document annotations has a digital signature."""
         return self._has_signature
 
+    @property
+    def is_tagged(self) -> bool:
+        """Return True if the document catalog indicates this is a Tagged PDF."""
+        return self._is_tagged
+
     @property
     def filename(self) -> str | Path:
         """Return filename of PDF."""

src/ocrmypdf/pdfinfo/layout.py

@@ -5,8 +5,12 @@
 from __future__ import annotations
 
 import re
+from collections.abc import Mapping
+from contextlib import contextmanager
 from math import copysign
+from os import PathLike
 from pathlib import Path
+from typing import Any, Iterator
 from unittest.mock import patch
 
 import pdfminer
@@ -15,10 +19,13 @@ import pdfminer.pdfdevice
 import pdfminer.pdfinterp
 from pdfminer.converter import PDFLayoutAnalyzer
 from pdfminer.layout import LAParams, LTChar, LTPage, LTTextBox
+from pdfminer.pdfcolor import PDFColorSpace
+from pdfminer.pdfdevice import PDFTextSeq
 from pdfminer.pdfdocument import PDFTextExtractionNotAllowed
-from pdfminer.pdffont import PDFSimpleFont, PDFUnicodeNotDefined
+from pdfminer.pdffont import FontWidthDict, PDFFont, PDFSimpleFont, PDFUnicodeNotDefined
+from pdfminer.pdfinterp import PDFGraphicState, PDFResourceManager, PDFTextState
 from pdfminer.pdfpage import PDFPage
-from pdfminer.utils import bbox2str, matrix2str
+from pdfminer.utils import Matrix, bbox2str, matrix2str
 
 from ocrmypdf.exceptions import EncryptedPdfError, InputFileError
 
@@ -28,7 +35,12 @@ STRIP_NAME = re.compile(r'[0-9]+')
 original_pdfsimplefont_init = PDFSimpleFont.__init__
 
 
-def pdfsimplefont__init__(self, descriptor, widths, spec):
+def pdfsimplefont__init__(
+    self,
+    descriptor: Mapping[str, Any],
+    widths: FontWidthDict,
+    spec: Mapping[str, Any],
+) -> None:
     """Monkeypatch pdfminer.six PDFSimpleFont.__init__.
 
     If there is no ToUnicode and no Encoding, pdfminer.six assumes that Unicode
@@ -44,7 +56,7 @@ def pdfsimplefont__init__(self, descriptor, widths, spec):
     return
 
 
-PDFSimpleFont.__init__ = pdfsimplefont__init__
+setattr(PDFSimpleFont, '__init__', pdfsimplefont__init__)
 
 #
 # pdfminer patches when creator is PScript5.dll
@@ -85,6 +97,11 @@ def pdftype3font__pscript5_get_ascent(self):
     return self.ascent * copysign(1.0, self.vscale)
 
 
+def _is_undefined_char(s: str) -> bool:
+    """Check if a string is an undefined character."""
+    return s.startswith('(cid:') and s.endswith(')')
+
+
 class LTStateAwareChar(LTChar):
     """A subclass of LTChar that tracks text render mode at time of drawing."""
 
@@ -107,18 +124,18 @@ class LTStateAwareChar(LTChar):
 
     def __init__(
         self,
-        matrix,
+        matrix: Matrix,
-        font,
+        font: PDFFont,
-        fontsize,
+        fontsize: float,
-        scaling,
+        scaling: float,
-        rise,
+        rise: float,
-        text,
+        text: str,
-        textwidth,
+        textwidth: float,
-        textdisp,
+        textdisp: float | tuple[float | None, float],
-        ncs,
+        ncs: PDFColorSpace,
-        graphicstate,
+        graphicstate: PDFGraphicState,
-        textstate,
+        textstate: PDFTextState,
-    ):
+    ) -> None:
         """Initialize."""
         super().__init__(
             matrix,
@@ -134,7 +151,7 @@ class LTStateAwareChar(LTChar):
         )
         self.rendermode = textstate.render
 
-    def is_compatible(self, obj):
+    def is_compatible(self, obj: object) -> bool:
         """Check if characters can be combined into a textline.
 
         We consider characters compatible if:
@@ -142,23 +159,22 @@ class LTStateAwareChar(LTChar):
             - the Unicode mapping is unknown but both are part of the same font
         """
         # pylint: disable=protected-access
-        both_unicode_mapped = isinstance(self._text, str) and isinstance(obj._text, str)
+        if not isinstance(obj, LTStateAwareChar):
-        try:
-            if both_unicode_mapped:
-                return self.rendermode == obj.rendermode
-            font0, _ = self._text
-            font1, _ = obj._text
-            return font0 == font1 and self.rendermode == obj.rendermode
-        except (ValueError, AttributeError):
             return False
+        both_unicode_mapped = not _is_undefined_char(
+            self._text
+        ) and not _is_undefined_char(obj._text)
+        if both_unicode_mapped:
+            return self.rendermode == obj.rendermode
+        return self.fontname == obj.fontname and self.rendermode == obj.rendermode
 
-    def get_text(self):
+    def get_text(self) -> str:
         """Get text from this character."""
-        if isinstance(self._text, tuple):
+        if _is_undefined_char(self._text):
             return '\ufffd'  # standard 'Unknown symbol'
         return self._text
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         """Return a string representation of this object."""
         return (
             f"<{self.__class__.__name__} "
@@ -174,19 +190,24 @@ class LTStateAwareChar(LTChar):
 class TextPositionTracker(PDFLayoutAnalyzer):
     """A page layout analyzer that pays attention to text visibility."""
 
-    def __init__(self, rsrcmgr, pageno=1, laparams=None):
+    textstate: PDFTextState
+
+    def __init__(
+        self,
+        rsrcmgr: PDFResourceManager,
+        pageno: int = 1,
+        laparams: LAParams | None = None,
+    ):
         """Initialize the layout analyzer."""
         super().__init__(rsrcmgr, pageno, laparams)
-        self.textstate = None
+        self.result: LTPage | None = None
-        self.result = None
-        self.cur_item = None  # not defined in pdfminer code as it should be
 
-    def begin_page(self, page, ctm):
+    def begin_page(self, page: PDFPage, ctm: Matrix) -> None:
         """Begin processing of a page."""
         super().begin_page(page, ctm)
         self.cur_item = LTPage(self.pageno, page.mediabox)
 
-    def end_page(self, page):
+    def end_page(self, page: PDFPage) -> None:
         """End processing of a page."""
         assert not self._stack, str(len(self._stack))
         assert isinstance(self.cur_item, LTPage), str(type(self.cur_item))
@@ -195,14 +216,28 @@ class TextPositionTracker(PDFLayoutAnalyzer):
         self.pageno += 1
         self.receive_layout(self.cur_item)
 
-    def render_string(self, textstate, seq, ncs, graphicstate):
+    def render_string(
+        self,
+        textstate: PDFTextState,
+        seq: PDFTextSeq,
+        ncs: PDFColorSpace,
+        graphicstate: PDFGraphicState,
+    ) -> None:
         """Respond to render string event by updating text state."""
         self.textstate = textstate.copy()
         super().render_string(self.textstate, seq, ncs, graphicstate)
 
     def render_char(
-        self, matrix, font, fontsize, scaling, rise, cid, ncs, graphicstate
+        self,
-    ):
+        matrix: Matrix,
+        font: PDFFont,
+        fontsize: float,
+        scaling: float,
+        rise: float,
+        cid: int,
+        ncs: PDFColorSpace,
+        graphicstate: PDFGraphicState,
+    ) -> float:
         """Respond to render char event by updating text state."""
         try:
             text = font.to_unichr(cid)
@@ -227,21 +262,34 @@ class TextPositionTracker(PDFLayoutAnalyzer):
         self.cur_item.add(item)
         return item.adv
 
-    def handle_undefined_char(self, font, cid):
+    def receive_layout(self, ltpage: LTPage) -> None:
-        """Handle undefined character."""
-        # log.info('undefined: %r, %r', font, cid)
-        return (font.fontname, cid)
-
-    def receive_layout(self, ltpage):
         """Receive layout handler."""
         self.result = ltpage
 
-    def get_result(self):
+    def get_result(self) -> LTPage | None:
         """Get the result of the analysis."""
         return self.result
 
 
-def get_page_analysis(infile, pageno, pscript5_mode):
+@contextmanager
+def patch_pdfminer(pscript5_mode: bool):
+    """Patch pdfminer.six to work around bugs in PDFs created by PScript5."""
+    if pscript5_mode:
+        with patch.multiple(
+            'pdfminer.pdffont.PDFType3Font',
+            spec=True,
+            get_ascent=pdftype3font__pscript5_get_ascent,
+            get_descent=pdftype3font__pscript5_get_descent,
+            get_height=pdftype3font__pscript5_get_height,
+        ):
+            yield
+    else:
+        yield
+
+
+def get_page_analysis(
+    infile: PathLike, pageno: int, pscript5_mode: bool
+) -> LTPage | None:
     """Get the page analysis for a given page."""
     rman = pdfminer.pdfinterp.PDFResourceManager(caching=True)
     disable_boxes_flow = None
@@ -253,36 +301,23 @@ def get_page_analysis(infile, pageno, pscript5_mode):
     )
     interp = pdfminer.pdfinterp.PDFPageInterpreter(rman, dev)
 
-    patcher = None
+    with patch_pdfminer(pscript5_mode):
-    if pscript5_mode:
+        try:
-        patcher = patch.multiple(
+            with Path(infile).open('rb') as f:
-            'pdfminer.pdffont.PDFType3Font',
+                page_iter = PDFPage.get_pages(f, pagenos=[pageno], maxpages=0)
-            spec=True,
+                page = next(page_iter, None)
-            get_ascent=pdftype3font__pscript5_get_ascent,
+                if page is None:
-            get_descent=pdftype3font__pscript5_get_descent,
+                    raise InputFileError(
-            get_height=pdftype3font__pscript5_get_height,
+                        f"pdfminer could not process page {pageno} (counting from 0)."
-        )
+                    )
-        patcher.start()
+                interp.process_page(page)
-
+        except PDFTextExtractionNotAllowed as e:
-    try:
+            raise EncryptedPdfError() from e
-        with Path(infile).open('rb') as f:
-            page_iter = PDFPage.get_pages(f, pagenos=[pageno], maxpages=0)
-            page = next(page_iter, None)
-            if page is None:
-                raise InputFileError(
-                    f"pdfminer could not process page {pageno} (counting from 0)."
-                )
-            interp.process_page(page)
-    except PDFTextExtractionNotAllowed as e:
-        raise EncryptedPdfError() from e
-    finally:
-        if patcher is not None:
-            patcher.stop()
 
     return dev.get_result()
 
 
-def get_text_boxes(obj):
+def get_text_boxes(obj) -> Iterator[LTTextBox]:
     """Get the text boxes attached to the current node."""
     for child in obj:
         if isinstance(child, (LTTextBox)):

src/ocrmypdf/pluginspec.py

@@ -7,9 +7,10 @@ from __future__ import annotations
 
 from abc import ABC, abstractmethod
 from argparse import ArgumentParser, Namespace
+from collections.abc import Sequence, Set
 from logging import Handler
 from pathlib import Path
-from typing import TYPE_CHECKING, AbstractSet, NamedTuple, Sequence
+from typing import TYPE_CHECKING, NamedTuple
 
 import pluggy
 
@@ -405,7 +406,7 @@ class OcrEngine(ABC):
 
     @staticmethod
     @abstractmethod
-    def languages(options: Namespace) -> AbstractSet[str]:
+    def languages(options: Namespace) -> Set[str]:
         """Returns the set of all languages that are supported by the engine.
 
         Languages are typically given in 3-letter ISO 3166-1 codes, but actually

src/ocrmypdf/quality.py

@@ -7,7 +7,7 @@
 from __future__ import annotations
 
 import re
-from typing import Iterable
+from collections.abc import Iterable
 
 
 class OcrQualityDictionary:

src/ocrmypdf/subprocess/__init__.py

@@ -8,11 +8,12 @@ import logging
 import os
 import re
 import sys
+from collections.abc import Mapping, Sequence
 from contextlib import suppress
 from pathlib import Path
 from subprocess import PIPE, STDOUT, CalledProcessError, CompletedProcess, Popen
 from subprocess import run as subprocess_run
-from typing import Callable, Mapping, Sequence, Union
+from typing import Callable, Union
 
 from packaging.version import Version
 

src/ocrmypdf/subprocess/_windows.py

@@ -9,9 +9,10 @@ import os
 import re
 import shutil
 import sys
+from collections.abc import Iterable, Iterator
 from itertools import chain
 from pathlib import Path
-from typing import Any, Callable, Iterable, Iterator, TypeVar
+from typing import Any, Callable, TypeVar
 
 from packaging.version import InvalidVersion, Version
 

tests/conftest.py

@@ -25,9 +25,8 @@ def is_macos():
 
 
 def running_in_docker():
-    # Docker creates a file named /.dockerenv (newer versions) or
+    # Docker creates a file named /.dockerenv in all supported versions
-    # /.dockerinit (older) -- this is undocumented, not an offical test
+    return Path('/.dockerenv').exists()
-    return Path('/.dockerenv').exists() or Path('/.dockerinit').exists()
 
 
 def have_unpaper():

tests/test_metadata.py

@@ -394,9 +394,12 @@ def test_prevent_gs_invalid_xml(resources, outdir):
     assert contents.find(b'\x00', xmp_start, xmp_end) == -1
 
 
-@pytest.mark.skipif(
+@pytest.mark.xfail(
-    ghostscript.version() >= Version('10.2.0'),
+    ghostscript.version() >= Version('10.01.2'),
-    reason="Ghostscript 10.2.0+ exit with an error on invalid DocumentInfo",
+    reason=(
+        "Ghostscript now exits with an error on invalid DocumentInfo, defeating "
+        "this test.",
+    ),
 )
 def test_malformed_docinfo(caplog, resources, outdir):
     generate_pdfa_ps(outdir / 'pdfa.ps')

tests/test_rotation.py

@@ -23,13 +23,6 @@ from .conftest import check_ocrmypdf, run_ocrmypdf
 
 # pylintx: disable=unused-variable
 
-# Remove this workaround when we require Pillow >= 10
-try:
-    Transpose = Image.Transpose  # type: ignore
-except AttributeError:
-    # Pillow 9 shim
-    Transpose = Image  # type: ignore
-
 RENDERERS = ['hocr', 'sandwich']
 
 
@@ -226,7 +219,7 @@ def test_rotate_page_level(image_angle, page_angle, resources, outdir):
         with Image.open(fspath(resources / 'typewriter.png')) as im:
             if image_angle != 0:
                 ccw_angle = -image_angle % 360
-                im = im.transpose(getattr(Transpose, f'ROTATE_{ccw_angle}'))
+                im = im.transpose(getattr(Image.Transpose, f'ROTATE_{ccw_angle}'))
             im.save(memimg, format='PNG')
         memimg.seek(0)
         mempdf = BytesIO()

tests/test_tagged.py

@@ -0,0 +1,26 @@
+# SPDX-FileCopyrightText: 2023 James R. Barlow
+# SPDX-License-Identifier: MPL-2.0
+
+from __future__ import annotations
+
+import argparse
+
+import pytest
+
+import ocrmypdf
+
+
+def test_block_tagged(resources):
+    with pytest.raises(ocrmypdf.exceptions.TaggedPDFError):
+        ocrmypdf.ocr(resources / 'tagged.pdf', '_.pdf')
+
+
+def test_force_tagged_warns(resources, outpdf, caplog):
+    caplog.set_level('WARNING')
+    ocrmypdf.ocr(
+        resources / 'tagged.pdf',
+        outpdf,
+        force_ocr=True,
+        plugins=['tests/plugins/tesseract_noop.py'],
+    )
+    assert 'marked as a Tagged PDF' in caplog.text