v16.10.0 release notes

I prefer to write the name in full aka `jbarlow83/ocrmypdf-alpine` and I'd also suggest to document this because:
* if you use `docker tag` this AFAIK only tags the currently downloaded (=pulled) version of that image
* in case a new update comes out, the new one will not be pulled automatically and one would have to pull and tag the image locally, again
* This `docker tag`  command is easily overlooked, if users just run `docker run ocrmypdf` this may or may not work, depending on how it is resolved.
   Also, AFAIK if one could get Docker to register https://hub.docker.com/ocrmypdf then this would suddenly be used instead of your image (currently `podman pull docker.io/ocrmypdf` returns a 404 for me, though)
* It is more common to write at least the user namespace there and the project, to prevent such errors.

Also, default [Docker has many shortcuts for this and e.g. assumes Docker-Hub is always being used](https://stackoverflow.com/questions/37861791/how-are-docker-image-names-parsed). Podman usually does not, that's why I personally prefer to use the very full and clear `docker.io/jbarlow83/ocrmypdf-alpine:latest` e.g. for alpine. This makes it not only clear which version is used, but also where it is pulled from (should one have configured different Docker registries).

.docker/Dockerfile

@@ -1,7 +1,7 @@
-# SPDX-FileCopyrightText: 2022 James R. Barlow
+# SPDX-FileCopyrightText: 2024 James R. Barlow
 # SPDX-License-Identifier: MPL-2.0
 
-FROM ubuntu:22.04 as base
+FROM ubuntu:24.04 AS base
 
 ENV LANG=C.UTF-8
 ENV TZ=UTC
@@ -9,19 +9,15 @@ RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selectio
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
   python3 \
-  libqpdf-dev \
-  zlib1g \
-  liblept5
+  python-is-python3
 
-FROM base as builder
+FROM base AS builder
 
 # Note we need leptonica here to build jbig2
 RUN apt-get update && apt-get install -y --no-install-recommends \
   build-essential autoconf automake libtool \
   libleptonica-dev \
   zlib1g-dev \
-  python3-dev \
-  python3-distutils \
   libffi-dev \
   ca-certificates \
   curl \
@@ -29,42 +25,50 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   libcairo2-dev \
   pkg-config
 
-# Get the latest pip (Ubuntu version doesn't support manylinux2010)
-RUN \
-  curl https://bootstrap.pypa.io/get-pip.py | python3
-
 # Compile and install jbig2
 # Needs libleptonica-dev, zlib1g-dev
 RUN \
   mkdir jbig2 \
-  && curl -L https://github.com/agl/jbig2enc/archive/ea6a40a.tar.gz | \
+  && curl -L https://github.com/agl/jbig2enc/archive/c0141bf.tar.gz | \
   tar xz -C jbig2 --strip-components=1 \
   && cd jbig2 \
   && ./autogen.sh && ./configure && make && make install \
   && cd .. \
   && rm -rf jbig2
 
-COPY . /app
 
 WORKDIR /app
 
-RUN pip3 install --no-cache-dir .[test,webservice,watcher]
+# Copy uv from ghcr
+COPY --from=ghcr.io/astral-sh/uv:0.5.5 /uv /uvx /bin/
+
+ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy
+
+# Install the project's dependencies using the lockfile and settings
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    uv sync --frozen --no-install-project --no-dev
+
+# Then, add the rest of the project source code and install it
+# Installing separately from its dependencies allows optimal layer caching
+COPY . /app
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --frozen \
+        --extra test --extra webservice --extra watcher --no-dev \
+        --no-install-package pyarrow
 
 FROM base
 
-# For Tesseract 5
-RUN apt-get update && apt-get install -y --no-install-recommends \
-  software-properties-common gpg-agent
-RUN add-apt-repository -y ppa:alex-p/tesseract-ocr-devel
+RUN apt-get update && apt-get install -y software-properties-common
+
+RUN add-apt-repository -y ppa:alex-p/tesseract-ocr5
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
   ghostscript \
   fonts-droid-fallback \
   jbig2dec \
-  img2pdf \
-  libsm6 libxext6 libxrender-dev \
   pngquant \
-  python-is-python3 \
   tesseract-ocr \
   tesseract-ocr-chi-sim \
   tesseract-ocr-deu \
@@ -80,11 +84,13 @@ WORKDIR /app
 COPY --from=builder /usr/local/lib/ /usr/local/lib/
 COPY --from=builder /usr/local/bin/ /usr/local/bin/
 
-COPY --from=builder /app/misc/webservice.py /app/
-COPY --from=builder /app/misc/watcher.py /app/
+COPY --from=builder --chown=app:app /app /app
 
-# Copy minimal project files to get the test suite.
-COPY --from=builder /app/pyproject.toml /app/README.md /app/
-COPY --from=builder /app/tests /app/tests
+RUN rm -rf /app/.git && \
+ln -s /app/misc/webservice.py /app/webservice.py && \
+ln -s /app/misc/watcher.py /app/watcher.py
+
+ENV PATH="/app/.venv/bin:${PATH}"
+
+ENTRYPOINT ["/app/.venv/bin/ocrmypdf"]
 
-ENTRYPOINT ["/usr/local/bin/ocrmypdf"]

.docker/Dockerfile.alpine

@@ -1,7 +1,13 @@
 # SPDX-FileCopyrightText: 2023 James R. Barlow
 # SPDX-License-Identifier: MPL-2.0
 
-FROM alpine:3.19.1 as base
+# Note: Alpine 3.20 builds tesseract with --enable-opencl, which is not
+# supported by anyone. OCRmyPDF is not compatible with Alpine 3.20.0
+# through 3.20.3. The issue is fixed in 3.21.
+# Details
+#  https://gitlab.alpinelinux.org/alpine/aports/-/issues/16143
+#  https://github.com/ocrmypdf/OCRmyPDF/issues/1395
+FROM alpine:3.21 AS base
 
 ENV LANG=C.UTF-8
 ENV TZ=UTC
@@ -10,40 +16,37 @@ RUN apk add --no-cache \
     python3 \
     zlib
 
-FROM base as builder
+FROM base AS builder
 
+# Yes it really is python3-dev, and py3-package
 RUN apk add --no-cache \
     ca-certificates \
     git \
     python3-dev \
-    py3-pip
-
-# On arm64, we need to build cffi from source.
-ARG TARGETPLATFORM
-
-RUN if [ "${TARGETPLATFORM}" == "linux/arm64" ]; then \
-    apk add --no-cache \
-    build-base \
-    autoconf \
-    automake \
-    libtool \
-    zlib-dev \
-    libffi-dev \
-    cairo-dev \
-    pkgconfig \
-    ;  \
-    fi
-
-COPY . /app
+    py3-pyarrow \
+    curl
 
 WORKDIR /app
 
-RUN python3 -m venv .venv
+COPY --from=ghcr.io/astral-sh/uv:0.5.5 /uv /uvx /bin/
 
-RUN source .venv/bin/activate \
-    && python3 -m pip install --no-cache-dir --upgrade pip \
-    && python3 -m pip install --no-cache-dir wheel \
-    && python3 -m pip install --no-cache-dir .[test,webservice,watcher]
+ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy
+
+RUN uv venv --system-site-packages .venv
+
+# Install the project's dependencies using the lockfile and settings
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    uv sync --frozen --no-install-project --no-dev
+
+# Then, add the rest of the project source code and install it
+# Installing separately from its dependencies allows optimal layer caching
+COPY . /app
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --frozen \
+        --extra test --extra webservice --extra watcher --no-dev \
+        --no-install-package pyarrow
 
 FROM base
 
@@ -66,17 +69,11 @@ RUN apk add --no-cache \
 
 WORKDIR /app
 
-COPY --from=builder /usr/local/lib/ /usr/local/lib/
-COPY --from=builder /usr/local/bin/ /usr/local/bin/
+COPY --from=builder --chown=app:app /app /app
 
-COPY --from=builder /app/.venv/ /app/.venv/
-
-COPY --from=builder /app/misc/webservice.py /app/
-COPY --from=builder /app/misc/watcher.py /app/
-
-# Copy minimal project files to get the test suite.
-COPY --from=builder /app/pyproject.toml /app/README.md /app/
-COPY --from=builder /app/tests /app/tests
+RUN rm -rf /app/.git && \
+    ln -s /app/misc/webservice.py /app/webservice.py && \
+    ln -s /app/misc/watcher.py /app/watcher.py
 
 ENV PATH="/app/.venv/bin:${PATH}"
 

.github/workflows/build.yml

@@ -21,18 +21,13 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
+        os: [ubuntu-22.04, ubuntu-24.04]
+        python: ["3.10", "3.11", "3.12", "3.13"]
         include:
           - os: ubuntu-22.04
+            tesseract_ppa: "ppa"
             python: "3.10"
-          - os: ubuntu-22.04
-            python: "3.11"
-          - os: ubuntu-22.04
-            python: "3.10"
-            tesseract5: true
-          - os: ubuntu-latest
-            python: "3.12"
-            tesseract5: true
-          - os: ubuntu-latest
+          - os: ubuntu-24.04
             python: "pypy3.10"
 
     env:
@@ -44,16 +39,20 @@ jobs:
         with:
           fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags
 
-      - uses: actions/setup-python@v5
-        name: Setup Python
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "0.5.x"
+
+      - name: "Set up Python"
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
-          cache: "pip"
 
-      - name: Install Tesseract 5
-        if: matrix.tesseract5
+      - name: Install Tesseract from PPA
+        if: matrix.tesseract_ppa == 'ppa'
         run: |
-          sudo add-apt-repository -y ppa:alex-p/tesseract-ocr-devel
+          sudo add-apt-repository -y ppa:alex-p/tesseract-ocr5.3
 
       - name: Install common packages
         run: |
@@ -85,8 +84,7 @@ jobs:
 
       - name: Install Python packages
         run: |
-          python -m pip install --upgrade pip wheel
-          python -m pip install --prefer-binary .[test]
+          uv sync --extra test --no-dev
 
       - name: Report versions
         run: |
@@ -94,28 +92,27 @@ jobs:
           gs --version
           pngquant --version
           unpaper --version
-          img2pdf --version
+          uv run --no-dev img2pdf --version
 
       - name: Test
         run: |
-          python -m pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
+          uv run --no-dev pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
 
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:
           files: ./coverage.xml
           env_vars: OS,PYTHON
 
-
   test_macos:
     name: Test macOS
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         os: [macos-latest, macos-13]  # macos-latest is arm64, macos-13 is x86_64
-        python: ["3.10", "3.11", "3.12"]
+        python: ["3.10", "3.11", "3.12", "3.13"]
 
     env:
       OS: ${{ matrix.os }}
@@ -138,30 +135,33 @@ jobs:
             pngquant \
             tesseract
 
-      - uses: actions/setup-python@v5
-        name: Setup Python
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "0.5.x"
+
+      - name: "Set up Python"
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
-          cache: "pip"
 
       - name: Install Python packages
         run: |
-          python -m pip install --upgrade pip wheel
-          python -m pip install --prefer-binary .[test]
+          uv sync --extra test --no-dev
 
       - name: Report versions
         run: |
           tesseract --version
           gs --version
           pngquant --version
-          img2pdf --version
+          uv run --no-dev img2pdf --version
 
       - name: Test
         run: |
-          python -m pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
+          uv run --no-dev pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
 
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:
@@ -174,7 +174,7 @@ jobs:
     strategy:
       matrix:
         os: [windows-latest]
-        python: ["3.10", "3.11", "3.12"]
+        python: ["3.10", "3.11", "3.12", "3.13"]
 
     env:
       OS: ${{ matrix.os }}
@@ -185,11 +185,15 @@ jobs:
         with:
           fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags
 
-      - uses: actions/setup-python@v5
-        name: Setup Python
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "0.5.x"
+
+      - name: "Set up Python"
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
-          cache: "pip"
 
       - name: Install system packages
         run: |
@@ -198,15 +202,14 @@ jobs:
 
       - name: Install Python packages
         run: |
-          python -m pip install --upgrade pip wheel
-          python -m pip install --prefer-binary .[test]
+          uv sync --extra test --no-dev
 
       - name: Test
         run: |
-          python -m pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
+          uv run --no-dev pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
 
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:
@@ -221,16 +224,14 @@ jobs:
         with:
           fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags
 
-      - uses: actions/setup-python@v5
-        name: Setup Python
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
         with:
-          python-version: "3.10"
-          cache: "pip"
+          version: "0.5.x"
 
       - name: Make wheels and sdist
         run: |
-          python -m pip install --upgrade pip wheel build
-          python -m build --sdist --wheel
+          uv build --sdist --wheel
 
       - uses: actions/upload-artifact@v4
         with:
@@ -258,29 +259,45 @@ jobs:
 
   create_release:
     name: Create GitHub release
-    needs: [wheel_sdist_linux, test_linux, test_macos, test_windows]
+    needs: [upload_pypi]
     runs-on: ubuntu-latest
     if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v')
     permissions:
       # Required to create a release
       contents: write
+      id-token: write
     steps:
       - uses: actions/download-artifact@v4
         with:
           name: artifact
           path: dist
 
-      - name: Create Release
-        id: create-release
-        uses: shogo82148/actions-create-release@v1
-
-      - name: Upload Assets
-        uses: shogo82148/actions-upload-release-asset@v1
+      - name: Sign the dists with Sigstore
+        uses: sigstore/gh-action-sigstore-python@v3.0.0
         with:
-          upload_url: ${{ steps.create-release.outputs.upload_url }}
-          asset_path: |
-            ./dist/*.whl
+          inputs: |
             ./dist/*.tar.gz
+            ./dist/*.whl
+
+      - name: Create GitHub Release
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: >-
+          gh release create
+          "$GITHUB_REF_NAME"
+          --repo "$GITHUB_REPOSITORY"
+          --notes ""
+
+      - name: Upload artifact signatures to GitHub Release
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        # Upload to GitHub Release using the `gh` CLI.
+        # `dist/` contains the built packages, and the
+        # sigstore-produced signatures and certificates.
+        run: >-
+          gh release upload
+          "$GITHUB_REF_NAME" dist/**
+          --repo "$GITHUB_REPOSITORY"
 
   docker_ubuntu:
     name: Build Ubuntu-based Docker image

.github/workflows/triage.yml

@@ -0,0 +1,32 @@
+# SPDX-FileCopyrightText: 2024 James R. Barlow
+# SPDX-License-Identifier: MPL-2.0
+
+name: Remove Triage Label on Reply
+
+on:
+  issue_comment:
+    types:
+      - created
+
+jobs:
+  remove-triage-label:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check if comment is by the repository owner
+        id: check_comment
+        run: |
+          echo "::set-output name=is_owner::$(
+            if [[ '${{ github.event.comment.user.login }}' == 'jbarlow83' ]]; then
+              echo 'true';
+            else
+              echo 'false';
+            fi
+          )"
+
+      - name: Remove 'triage' label
+        if: ${{ steps.check_comment.outputs.is_owner == 'true' }}
+        uses: actions-ecosystem/action-remove-labels@v1
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          labels: triage

.gitignore

@@ -44,3 +44,4 @@ docs/_build/
 docs/_static/
 docs/_templates/
 docs/Makefile
+src/ocrmypdf/_version.py

.readthedocs.yaml

@@ -11,10 +11,6 @@ version: 2
 sphinx:
   configuration: docs/conf.py
 
-# Optionally build your docs in additional formats such as PDF
-formats:
-  - pdf
-
 # Optionally set the version of Python and requirements required to build your docs
 build:
   os: ubuntu-22.04

README.md

@@ -74,7 +74,6 @@ Linux, Windows, macOS and FreeBSD are supported. Docker images are also availabl
 | macOS (nix)                   | ``nix-env -i ocrmypdf``       |
 | LinuxBrew                     | ``brew install ocrmypdf``     |
 | FreeBSD                       | ``pkg install py-ocrmypdf``   |
-| Conda                         | ``conda install ocrmypdf``    |
 | Ubuntu Snap                   | ``snap install ocrmypdf``     |
 
 For everyone else, [see our documentation](https://ocrmypdf.readthedocs.io/en/latest/installation.html) for installation steps.
@@ -113,9 +112,33 @@ Our [documentation is served on Read the Docs](https://ocrmypdf.readthedocs.io/e
 
 Please report issues on our [GitHub issues](https://github.com/ocrmypdf/OCRmyPDF/issues) page, and follow the issue template for quick response.
 
+## Feature demo
+
+```bash
+# Add an OCR layer and convert to PDF/A
+ocrmypdf input.pdf output.pdf
+
+# Convert an image to single page PDF
+ocrmypdf input.jpg output.pdf
+
+# Add OCR to a file in place (only modifies file on success)
+ocrmypdf myfile.pdf myfile.pdf
+
+# OCR with non-English languages (look up your language's ISO 639-3 code)
+ocrmypdf -l fra LeParisien.pdf LeParisien.pdf
+
+# OCR multilingual documents
+ocrmypdf -l eng+fra Bilingual-English-French.pdf Bilingual-English-French.pdf
+
+# Deskew (straighten crooked pages)
+ocrmypdf --deskew input.pdf output.pdf
+```
+
+For more features, see the [documentation](https://ocrmypdf.readthedocs.io/en/latest/index.html).
+
 ## Requirements
 
-In addition to the required Python version (3.8+), OCRmyPDF requires external program installations of Ghostscript and Tesseract OCR. OCRmyPDF is pure Python, and runs on pretty much everything: Linux, macOS, Windows and FreeBSD.
+In addition to the required Python version, OCRmyPDF requires external program installations of Ghostscript and Tesseract OCR. OCRmyPDF is pure Python, and runs on pretty much everything: Linux, macOS, Windows and FreeBSD.
 
 ## Press & Media
 

docs/advanced.rst

@@ -235,7 +235,7 @@ The directive ``--tesseract-pagesegmode Nmode`` forwards the desired page segmen
 mode to Tesseract OCR. The default is 3.
 
 Page segmentation can improve OCR results when you know that a PDF ought to be
-analyzed a particular way, such as PDFs whose pages contain only a single line of 
+analyzed a particular way, such as PDFs whose pages contain only a single line of
 text. For the vast majority of users, changing the page segmentation mode will only
 make things worse.
 
@@ -244,37 +244,37 @@ As of June 2024, the Tesseract page segmentation modes are:
 +-----+----------------------------------------------------------------------------------+
 | ID  | Description                                                                      |
 +=====+==================================================================================+
-|  0  | Orientation and script detection (OSD) only.                                      |
+|  0  | Orientation and script detection (OSD) only.                                     |
 +-----+----------------------------------------------------------------------------------+
-|  1  | Automatic page segmentation with OSD.                                             |
+|  1  | Automatic page segmentation with OSD.                                            |
 +-----+----------------------------------------------------------------------------------+
 |  2  | Automatic page segmentation, but no OSD, or OCR. (not implemented)               |
 +-----+----------------------------------------------------------------------------------+
 |  3  | Fully automatic page segmentation, but no OSD. (Default)                         |
 +-----+----------------------------------------------------------------------------------+
-|  4  | Assume a single column of text of variable sizes.                                 |
+|  4  | Assume a single column of text of variable sizes.                                |
 +-----+----------------------------------------------------------------------------------+
-|  5  | Assume a single uniform block of vertically aligned text.                         |
+|  5  | Assume a single uniform block of vertically aligned text.                        |
 +-----+----------------------------------------------------------------------------------+
-|  6  | Assume a single uniform block of text.                                            |
+|  6  | Assume a single uniform block of text.                                           |
 +-----+----------------------------------------------------------------------------------+
-|  7  | Treat the image as a single text line.                                            |
+|  7  | Treat the image as a single text line.                                           |
 +-----+----------------------------------------------------------------------------------+
-|  8  | Treat the image as a single word.                                                 |
+|  8  | Treat the image as a single word.                                                |
 +-----+----------------------------------------------------------------------------------+
-|  9  | Treat the image as a single word in a circle.                                     |
+|  9  | Treat the image as a single word in a circle.                                    |
 +-----+----------------------------------------------------------------------------------+
-| 10  | Treat the image as a single character.                                            |
+| 10  | Treat the image as a single character.                                           |
 +-----+----------------------------------------------------------------------------------+
-| 11  | Sparse text. Find as much text as possible in no particular order.                |
+| 11  | Sparse text. Find as much text as possible in no particular order.               |
 +-----+----------------------------------------------------------------------------------+
 | 12  | Sparse text with OSD.                                                            |
 +-----+----------------------------------------------------------------------------------+
 | 13  | Raw line. Treat the image as a single text line, bypassing hacks that are        |
-|     | Tesseract-specific.                                                               |
+|     | Tesseract-specific.                                                              |
 +-----+----------------------------------------------------------------------------------+
 
-Modes 0, 1, 2, and 12 (all of those that enable orientation and script detection) 
+Modes 0, 1, 2, and 12 (all of those that enable orientation and script detection)
 are not compatible with OCRmyPDF, which performs OSD in a separate step from OCR.
 Their use may interfere with ``--rotate-pages`` and other features.
 
@@ -445,7 +445,7 @@ Debugging the intermediate files
 OCRmyPDF normally saves its intermediate results to a temporary folder
 and deletes this folder when it exits, whether it succeeded or failed.
 
-If the ``--keep-temporary-files`` (``-k```) argument is issued on the
+If the ``--keep-temporary-files`` (``-k``) argument is issued on the
 command line, OCRmyPDF will keep the temporary folder and print the location,
 whether it succeeded or failed. An example message is:
 
@@ -454,6 +454,10 @@ whether it succeeded or failed. An example message is:
     Temporary working files retained at:
     /tmp/ocrmypdf.io.u20wpz07
 
+When OCRmyPDF is launched as a snap, this corresponds to the snap filesystem, for instance:
+
+    /tmp/snap-private-tmp/snap.ocrmypdf/tmp/ocrmypdf.io.u20wpz07
+
 The organization of this folder is an implementation detail and subject
 to change between releases. However the general organization is that
 working files on a per page basis have the page number as a prefix
@@ -465,9 +469,9 @@ suffix indicates the file type. Some important files include:
    on arguments this may differ from the presentation image
 -  ``_pp_deskew.png`` - the image, after deskewing
 -  ``_pp_clean.png`` - the image, after cleaning with unpaper
--  ``_ocr_tess.pdf`` - the OCR file; appears as a blank page with invisible
+-  ``_ocr_hocr.pdf`` - the OCR file; appears as a blank page with invisible
    text embedded
--  ``_ocr_tess.txt`` - the OCR text (not necessarily all text on the page,
+-  ``_ocr_hocr.txt`` - the OCR text (not necessarily all text on the page,
    if the page is mixed format)
 -  ``fix_docinfo.pdf`` - a temporary file created to fix the PDF DocumentInfo
    data structure

docs/apiref.rst

@@ -9,27 +9,12 @@ API reference
 This page summarizes the rest of the public API. Generally speaking this
 should be mainly of interest to plugin developers.
 
-ocrmypdf
-========
+ocrmypdf.api
+============
 
-.. autoclass:: ocrmypdf.PageContext
+.. automodule:: ocrmypdf.api
     :members:
 
-.. autoclass:: ocrmypdf.PdfContext
-    :members:
-
-.. autoclass:: ocrmypdf.Verbosity
-    :members:
-    :undoc-members:
-
-.. autofunction:: ocrmypdf.configure_logging
-
-.. autofunction:: ocrmypdf.ocr
-
-.. autofunction:: ocrmypdf.pdf_to_hocr
-
-.. autofunction:: ocrmypdf.hocr_to_ocr_pdf
-
 ocrmypdf.exceptions
 ===================
 

docs/batch.rst

@@ -44,7 +44,7 @@ place, and printing each filename in between runs:
 
 .. code-block:: bash
 
-   find . -printf '%p\n' -name '*.pdf' -exec ocrmypdf '{}' '{}' \;
+   find . -name '*.pdf' -printf '%p\n' -exec ocrmypdf '{}' '{}' \;
 
 This only runs one ``ocrmypdf`` process at a time. This variation uses
 ``find`` to create a directory list and ``parallel`` to parallelize runs
@@ -135,7 +135,7 @@ Users may need to customize the script to meet their requirements.
     "OCR_ON_SUCCESS_ARCHIVE", "This will move the processed original file to ``OCR_ARCHIVE_DIRECTORY`` if the exit code is 0 (OK). Note that ``OCR_ON_SUCCESS_DELETE`` takes precedence over this option, i.e. if both options are set, the input file will be deleted."
     "OCR_OUTPUT_DIRECTORY_YEAR_MONTH", "This will place files in the output in ``{output}/{year}/{month}/{filename}``"
     "OCR_DESKEW", "Apply deskew to crooked input PDFs"
-    "OCR_JSON_SETTINGS", "A JSON string specifying any other arguments for ``ocrmypdf.ocr``, e.g. ``'OCR_JSON_SETTINGS={""rotate_pages"": true}'``."
+    "OCR_JSON_SETTINGS", "A JSON string specifying any other arguments for ``ocrmypdf.ocr``, e.g. ``'OCR_JSON_SETTINGS={""rotate_pages"": true, ""optimize"": ""3""}'``."
     "OCR_POLL_NEW_FILE_SECONDS", "Polling interval"
     "OCR_LOGLEVEL", "Level of log messages to report"
 

docs/conf.py

@@ -30,6 +30,8 @@
 #
 # needs_sphinx = '1.0'
 
+import datetime
+
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
@@ -38,11 +40,12 @@ extensions = [
     'sphinx.ext.intersphinx',
     'sphinx.ext.autosummary',
     'sphinx.ext.napoleon',
+    'sphinx.ext.imgconverter',  # PDF docs needs this for SVG to PNG conversion
     'sphinx_issues',
 ]
 
 # Extension settings
-intersphinx_mapping = {'https://docs.python.org/': None}
+intersphinx_mapping = {'python': ('https://docs.python.org/3', None)}
 napoleon_use_rtype = False
 issues_github_path = "ocrmypdf/OCRmyPDF"
 
@@ -50,10 +53,7 @@ issues_github_path = "ocrmypdf/OCRmyPDF"
 templates_path = ['_templates']
 
 # The suffix(es) of source filenames.
-# You can specify multiple suffix as a list of string:
-#
-# source_suffix = ['.rst', '.md']
-source_suffix = '.rst'
+source_suffix = {'.rst': 'restructuredtext', '.md': 'markdown'}
 
 # The encoding of source files.
 #
@@ -64,8 +64,11 @@ master_doc = 'index'
 
 # General information about the project.
 project = 'ocrmypdf'
+
+year = str(datetime.date.today().year)
 copyright = (
-    '2023, James R. Barlow. Licensed under Creative Commons Attribution-ShareAlike 4.0.'
+    f'{year}, James R. Barlow. ',
+    'Licensed under Creative Commons Attribution-ShareAlike 4.0.',
 )
 author = 'James R. Barlow'
 
@@ -92,6 +95,7 @@ if on_rtd:
 
     MOCK_MODULES = [
         'pikepdf',
+        'pikepdf.canvas',
         'pikepdf.models',
         'pikepdf.models.metadata',
     ]
@@ -108,7 +112,7 @@ version = '.'.join(release.split('.')[:2])
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = 'en'
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
@@ -158,19 +162,18 @@ todo_include_todos = False
 
 # -- Options for HTML output ----------------------------------------------
 
-import sphinx_rtd_theme
+import sphinx_rtd_theme  # noqa: F401
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
 html_theme = 'sphinx_rtd_theme'
-html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
 #
-html_theme_options = {'display_version': False}
+html_theme_options = {}
 
 # Add any paths that contain custom themes here, relative to this directory.
 # html_theme_path = []
@@ -198,7 +201,7 @@ html_theme_options = {'display_version': False}
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+# html_static_path = ['_static']
 
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied

docs/cookbook.rst

@@ -399,7 +399,7 @@ Some users may consider enabling lossy JBIG2. See: :ref:`jbig2-lossy`.
 Digitally signed PDFs
 =====================
 
-OCRmyPDF cannot preserve digital signatures in PDFs and also add to OCR to them.
+OCRmyPDF cannot preserve digital signatures in PDFs and also add OCR to them.
 By default, it will refuse to modify a signed PDF regardless of other settings. You can
 override this behavior with ``--invalidate-digital-signatures``; as the name suggests,
 any digital signatures will be invalidated.

docs/docker.rst

@@ -35,7 +35,7 @@ execute the image:
    docker run hello-world
 
 .. list-table:: Docker images
-   :width: 30 20 50
+   :widths: 30 20 50
    :header-rows: 1
 
    * - Image
@@ -65,7 +65,13 @@ The ``ocrmypdf`` image is also available, but is deprecated and will be removed
 in the future.
 
 OCRmyPDF will use all available CPU cores. See the Docker documentation for
-`adjusting memory and CPU on other platforms <https://docs.docker.com/config/containers/resource_constraints/>`__.
+`adjusting memory and CPU on other platforms <https://docs.docker.com/config/containers/resource_constraints/>`__
+if you are using Docker on macOS or Windows, where you may need to manually assign
+more resources. On Linux, all resources will be available automatically.
+
+The underlying operating system and other details in Docker images are considered
+implementation details and **subject to change at minor releases**. If you are
+modifying the image, you should pin the version you intend to use.
 
 Using the Docker image on the command line
 ==========================================
@@ -81,8 +87,7 @@ To start a Docker container (instance of the image):
 
 .. code-block:: bash
 
-   docker tag jbarlow83/ocrmypdf-alpine ocrmypdf
-   docker run --rm -i ocrmypdf (... all other arguments here...) - -
+   docker run --rm -i jbarlow83/ocrmypdf-alpine (... all other arguments here...) - -
 
 For convenience, create a shell alias to hide the Docker command. It is
 easier to send the input file as stdin and read the output from
@@ -90,7 +95,7 @@ stdout – **this avoids the messy permission issues with Docker entirely**.
 
 .. code-block:: bash
 
-   alias docker_ocrmypdf='docker run --rm -i ocrmypdf'
+   alias docker_ocrmypdf='docker run --rm -i jbarlow83/ocrmypdf-alpine'
    docker_ocrmypdf --version  # runs docker version
    docker_ocrmypdf - - <input.pdf >output.pdf
 
@@ -98,7 +103,7 @@ Or in the wonderful `fish shell <https://fishshell.com/>`__:
 
 .. code-block:: fish
 
-   alias docker_ocrmypdf 'docker run --rm ocrmypdf'
+   alias docker_ocrmypdf 'docker run --rm jbarlow83/ocrmypdf-alpine'
    funcsave docker_ocrmypdf
 
 Alternately, you could mount the local current working directory as a
@@ -106,9 +111,27 @@ Docker volume:
 
 .. code-block:: bash
 
-   alias docker_ocrmypdf='docker run --rm  -i --user "$(id -u):$(id -g)" --workdir /data -v "$PWD:/data" ocrmypdf'
+   alias docker_ocrmypdf='docker run --rm  -i --user "$(id -u):$(id -g)" --workdir /data -v "$PWD:/data" jbarlow83/ocrmypdf-alpine'
    docker_ocrmypdf /data/input.pdf /data/output.pdf
 
+Podman
+======
+
+Especially if you use `Podman <https://podman.io/>`__ (or have SELinux enabled on your
+system), you may need to add ``--userns keep-id`` there, otherwise you may get access
+errors, because the user is otherwise not mapped to the same UID as on the host:
+
+.. code-block:: bash
+
+   alias podman_ocrmypdf='podman run --rm  -i --user "$(id -u):$(id -g)" --userns keep-id --workdir /data -v "$PWD:/data" ocrmypdf'
+   podman_ocrmypdf /data/input.pdf /data/output.pdf
+
+If you use SELinux you may additionally need to add the ``:Z`` `suffix to the volume
+<https://docs.podman.io/en/stable/markdown/podman-run.1.html#volume-v-source-volume-host-dir-container-dir-options>`__
+or disable SELinux for the container using ``--security-opt label=disable``, which is
+suggested for system files as they should not be re-labelled. Please refer to the „Note”
+section at the end of the linked podman documentation for details.
+
 .. _docker-lang-packs:
 
 Adding languages to the Docker image

docs/installation.rst

@@ -31,8 +31,6 @@ These platforms have one-liner installs:
 +-------------------------------+-----------------------------------------+
 | FreeBSD                       | ``pkg install textproc/py-ocrmypdf``    |
 +-------------------------------+-----------------------------------------+
-| Conda (WSL, macOS, Linux)     | ``conda install ocrmypdf``              |
-+-------------------------------+-----------------------------------------+
 | Snap (snapcraft packaging)    | ``snap install ocrmypdf``               |
 +-------------------------------+-----------------------------------------+
 
@@ -341,7 +339,7 @@ OCRmyPDF is includes in MacPorts:
     sudo port install ocrmypdf
 
 Note that while this will install tesseract you will need to install
-the appropriate tesseract `language ports <https://ports.macports.org/search/?selected_facets=categories_exact%3Atextproc&installed_file=&q=tesseract&name=on>`__. 
+the appropriate tesseract `language ports <https://ports.macports.org/search/?selected_facets=categories_exact%3Atextproc&installed_file=&q=tesseract&name=on>`__.
 
 Manual installation on macOS
 ----------------------------
@@ -396,6 +394,10 @@ Installing on Windows
 Native Windows
 --------------
 
+..
+  If you have a Windows that is not the Home edition, you can use Windows Sandbox to test on a blank Windows instance.
+  https://learn.microsoft.com/en-us/windows/security/application-security/application-isolation/windows-sandbox/
+
 .. note::
 
     Administrator privileges will be required for some of these steps.
@@ -432,6 +434,25 @@ Administrator.):
 
 * ``python3 -m pip install ocrmypdf``
 
+..
+  The Windows Python versions do not place any python or python3 executable in the path.
+  They add the py launcher to the path:
+  https://docs.python.org/3/using/windows.html#python-launcher-for-windows
+
+If you installed Python using WinGet, then use the following command instead:
+
+* ``py -m pip install ocrmypdf``
+
+and use:
+
+* ``py -m ocrmypdf``
+
+To start OCRmyPDF.
+
+If you intend to use more Python software on your Windows machine, consider the use of
+`pipx <https://pipx.pypa.io/stable/>`_ or a similar tool to create isolated Python
+environments for each Python software that you want to use.
+
 OCRmyPDF will check the Windows Registry and standard locations in your Program Files
 for third party software it needs (specifically, Tesseract and Ghostscript). To
 override the versions OCRmyPDF selects, you can modify the ``PATH`` environment
@@ -640,8 +661,7 @@ environment:
 
     pip install git+https://github.com/ocrmypdf/OCRmyPDF.git
 
-Or, to install in `development
-mode <https://packaging.python.org/en/latest/guides/distributing-packages-using-setuptools/#working-in-development-mode>`__,
+Or, to install in editable mode
 allowing customization of OCRmyPDF, use the ``-e`` flag:
 
 .. code-block:: bash

docs/jbig2.rst

@@ -37,8 +37,6 @@ For all other platforms, you would need to build the JBIG2 encoder from source:
    ./configure && make
    [sudo] make install
 
-.. _jbig2-lossy:
-
 Dependencies include libtoolize and libleptonica, which on Ubuntu systems
 are packaged as libtool and libleptonica-dev. On Fedora (35) they are packaged
 as libtool and leptonica-devel. For this to work, please make sure to install
@@ -48,8 +46,8 @@ installed.
 .. code-block:: bash
 
     [sudo] apt install autotools-dev automake libtool libleptonica-dev
-..
 
+.. _jbig2-lossy:
 
 Lossy mode JBIG2
 ================

docs/languages.rst

@@ -68,8 +68,8 @@ to what languages it should search for. Multiple languages can be
 requested using either ``-l eng+fra`` (English and French) or
 ``-l eng -l fra``.
 
-Archlinux
-------
+Arch Linux
+----------
 
 .. code-block:: bash
 

docs/maintainers.rst

@@ -38,12 +38,13 @@ on ARM and x86_64. Performance may be poor on other processor architectures.
 Versioning scheme
 -----------------
 
-OCRmyPDF uses setuptools-scm for versioning, which derives the version from
+OCRmyPDF uses hatch-vcs for versioning, which derives the version from
 Git as a single source of truth. This may be unsuitable for some distributions, e.g.
 to indicate that your distribution modifies OCRmyPDF in some way.
 
 You can patch the ``__version__`` variable in ``src/ocrmypdf/_version.py`` if
-necessary.
+necessary, or set the environment variable ``SETUPTOOLS_SCM_PRETEND_VERSION``
+to the required version, if you need to override versioning for some reason.
 
 jbig2enc
 --------

docs/optimizer.rst

@@ -14,8 +14,8 @@ perform other possible optimizations such as deduplicating resources,
 consolidating fonts, simplifying vector drawings, or anything of that nature.
 
 .. list-table:: Title
-   :widths: 33 6 60
-   :header-rows: 1
+    :widths: 33 6 60
+    :header-rows: 1
 
     * - Optimization level
       - Shorthand

docs/pdfsecurity.rst

@@ -2,6 +2,8 @@
 ..
 .. SPDX-License-Identifier: CC-BY-SA-4.0
 
+.. _security:
+
 ===================
 PDF security issues
 ===================

docs/plugins.rst

@@ -29,6 +29,9 @@ conventions. Note that: plugins installed with as setuptools entrypoints are
 not checked currently, because OCRmyPDF assumes you may not want to enable
 plugins for all files.
 
+See [OCRmyPDF-EasyOCR](https://github.com/ocrmypdf/OCRmyPDF-EasyOCR) for an
+example of a straightforward, fully working plugin.
+
 Script plugins
 ==============
 
@@ -70,14 +73,15 @@ similar to ``pytest`` packages such as ``pytest-cov`` (the package) and
     module), just like pytest plugins. At the same time, please make it clear
     that your package is not official.
 
-Setuptools plugins
-==================
+Plugins
+=======
 
 You can also create a plugin that OCRmyPDF will always automatically load if both are
-installed in the same virtual environment, using a setuptools entrypoint.
+installed in the same virtual environment, using a project entrypoint.
+OCRmyPDF uses the entrypoint namespace "ocrmypdf".
 
-Your package's ``pyproject.toml`` would need to contain the following, for a plugin
-named ``ocrmypdf-exampleplugin``:
+For example, ``pyproject.toml`` would need to contain the following, for a plugin named
+``ocrmypdf-exampleplugin``:
 
 .. code-block:: toml
 
@@ -87,13 +91,6 @@ named ``ocrmypdf-exampleplugin``:
     [project.entry-points."ocrmypdf"]
     exampleplugin = "exampleplugin.pluginmodule"
 
-.. code-block:: ini
-
-    # equivalent setup.cfg
-    [options.entry_points]
-    ocrmypdf =
-        exampleplugin = exampleplugin.pluginmodule
-
 Plugin requirements
 ===================
 

docs/release_notes.rst

@@ -30,6 +30,106 @@ OCRmyPDF typically supports the three most recent Python versions.
 
 .. |OCRmyPDF PyPI| image:: https://img.shields.io/pypi/v/ocrmypdf.svg
 
+v16.10.0
+========
+
+-  Added hocr textangle processing, improving handling of text at angles.
+   Thanks @0dinD :issue:`1467`
+-  Docker documentation updates related to podman. Thanks @rugk. :issue:`1489,1488`
+-  Dropped webservice.py's fragile use of ttyd. Instead, messages from ocrmypdf are
+   printed to the console.
+-  Fixed broken test test_hocrtransform_matches_sandwich, which had become
+   an invalid test. Thanks @QuLogic for reporting.
+-  Improved install instructions for Windows. Thanks @alex.
+
+v16.9.0
+=======
+
+-  Added hocr caption processing. Thanks @0dinD :issue:`1466`
+-  ocrmypdf-alpine Docker image is now built with Alpine 3.21.
+-  Fixed error handling of PDFs that contain invalid images with both ImageMask
+   and ColorSpace defined. :issue:`1453`
+-  Fixed test suite regression when only older Ghostscripts are installed.
+-  Improved documetnation of _progressbar.py. Thanks @QuentinFuxa. :issue:`1456`
+-  Disabling building of documentation as PDF on ReadTheDocs, as this caused
+   complex build issues deemed not worth solving.
+
+v16.8.0
+=======
+
+-  Upgraded webservice.py demonstration using streamlit. It's now possible to
+   exercise most of OCRmyPDF's functionality in a simple web UI.
+-  Added cache to Dockerfiles to improve build speed.
+-  Fixed numerous formatting errors in the documentation that prevented some
+   parts of documentation from generating correctly.
+-  Improved OCR text rendering by suppressing negative-width spaces. Thanks
+   @pajowu. :issue:`1446`
+-  Improved detecting of invisible text when using `--redo-ocr`. Thanks
+   @pajowu. :issue:`1448``
+
+v16.7.0
+=======
+
+-  Fixed further issues with Docker build and updated some versions.
+-  Main Docker image returned to Ubuntu 24.04 since the fix in v16.6.2 resolved
+   that concern.
+-  Code that previously sent Ghostscript output to stdout has been changed to
+   output to temporary files, since Ghostscript was doing that anyway internally.
+   This is a modest efficiency improvement.
+-  Fixed an issue with debug log output being parsed as rich markup. :issue:`1444`
+
+v16.6.2
+=======
+
+-  Remove invalid hyperlink annotations to satisfy Ghostscript 10.x during PDF/A
+   conversion. :issue:`1425`
+
+v16.6.1
+=======
+
+-  Fixed some issues with Docker build, such as removing unnecessary content and using
+   a stable Tesseract version.
+-  Reverted Docker image to Ubuntu 22.04 to access older/more stable Ghostscript
+   for now.
+-  Clarified batch commands in documentation.
+-  Fixed an issue with JSON serialization and pickling of HOCRResult. :issue:`1427`
+
+v16.6.0
+=======
+
+-  Fixed an issue where damaged PDFs would fail with ``--redo-ocr``. :issue:`1403`
+-  Fixed an error that prevented JBIG2 optimization on Windows if the image
+   was optimized in an earlier step. :issue:`1396`
+-  Fixed an error detecting the version of unpaper 7.0.0. :issue:`1409`
+-  Fixed a performance regression when scanning pages. :issue:`1378`. Thanks @aliemjay.
+-  Fixed Alpine Docker image by enforcing Alpine 3.19. Alpine 3.20 includes a
+   defective version of Tesseract OCR and so is not usable.
+-  Upgraded Ubuntu Docker image to use Ubuntu 24.04.
+-  Build and test scripts/actions switched to uv.
+-  When running in a container, we now remind the user that temporary folders
+   are inside the container and may not be accessible.
+-  Fixed Linux test coverage matrix, which was missing some key versions.
+
+v16.5.0
+=======
+
+-  Fixed issue with interpreting PDFs that have images with array masks.
+   :issue:`1377`
+-  Enabled testing on Python 3.13.
+-  Fixed a test that did not work correctly but still passed. :issue:`1382`
+-  Improved "PDF/A conversion failed" warning message to better describe implications.
+-  Updated documentation to better explain OCR_JSON_SETTINGS in batch processing.
+-  Build backend changed from setuptools to hatchling.
+
+v16.4.3
+=======
+
+-  Work around pdfminer.six issue where a token on the buffer boundary is incorrectly
+   parsed as two tokens. :issue:`1361`
+-  New rules are applied to stencil masks and explicit masks when calculating the
+   optimal page DPI for rendering. :issue:`1362`
+-  Fixed attempts to use an incompatible jbig2.EXE provided by TeX Live. :issue:`1363`
+
 v16.4.2
 =======
 

misc/_webservice.py

@@ -0,0 +1,241 @@
+# SPDX-FileCopyrightText: 2025 James R. Barlow
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+"""This is a simple web service/HTTP wrapper for OCRmyPDF.
+
+This may be more convenient than the command line tool for some Docker users.
+Note that OCRmyPDF uses Ghostscript, which is licensed under AGPLv3+. While
+OCRmyPDF is under GPLv3, this file is distributed under the Affero GPLv3+ license,
+to emphasize that SaaS deployments should make sure they comply with
+Ghostscript's license as well as OCRmyPDF's.
+"""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+import time
+from functools import partial
+from operator import getitem
+from pathlib import Path
+from shutil import which
+from tempfile import NamedTemporaryFile
+
+import pikepdf
+import streamlit as st
+from streamlit.components.v1 import iframe
+
+from ocrmypdf._defaults import DEFAULT_ROTATE_PAGES_THRESHOLD
+
+
+def get_host_url_with_port(port: int) -> str:
+    """Get the host URL for the web service. Hacky."""
+    host_url = st.context.headers["host"]
+    try:
+        host, _streamlit_port = host_url.split(":", maxsplit=1)
+    except ValueError:
+        host = host_url
+    return f"//{host}:{port}"  # Use the same protocol
+
+
+st.title("OCRmyPDF Web Service")
+
+uploaded = st.file_uploader("Upload input PDF or image", type=["pdf"], key="file")
+
+mode = st.selectbox("Mode", options=["normal", "skip-text", "force-ocr", "redo-ocr"])
+
+with st.expander("Input options"):
+    invalidate_digital_signatures = st.checkbox(
+        "Invalidate digital signatures", value=False
+    )
+    language = st.selectbox("Language", options=["eng", "deu", "fra", "spa"])
+
+    image_dpi = st.slider(
+        "Image DPI", value=300, key="image_dpi", min_value=1, max_value=5000, step=50
+    )
+with st.expander("Preprocessing"):
+    skip_big = st.checkbox("Skip OCR on big pages", value=False, key="skip_big")
+    oversample = st.slider("Oversample", min_value=0, max_value=5000, value=0, step=50)
+    rotate_pages = st.checkbox("Rotate pages", value=False, key="rotate")
+    deskew = st.checkbox("Deskew pages", value=False, key="deskew")
+    clean = st.checkbox("Clean pages before OCR", value=False, key="clean")
+    clean_final = st.checkbox("Clean final", value=False, key="clean_final")
+    remove_vectors = st.checkbox("Remove vectors", value=False, key="remove_vectors")
+
+
+with st.expander("Output options"):
+    output_type = st.selectbox(
+        "Output type", options=["pdfa", "pdfa", "pdfa-1", "pdfa-2", "pdfa-3", "none"]
+    )
+
+    pdf_renderer = st.selectbox(
+        "PDF rendereer", options=["auto", "hocr", "hocrdebug", "sandwich"]
+    )
+
+    optimize = st.selectbox("Optimize", options=["0", "1", "2", "3"])
+
+    st.selectbox("PDF/A compression", options=["auto", "jpeg", "lossless"])
+
+with st.expander("Metadata"):
+    title = author = keywords = subject = None
+    if uploaded:
+        with pikepdf.open(uploaded) as pdf, pdf.open_metadata() as meta:
+            st.code(str(meta), language="xml")
+            title = st.text_input("Title", value=meta.get('dc:title', ''))
+            author = st.text_input("Author", value=meta.get('dc:creator', ''))
+            keywords = st.text_input("Keywords", value=meta.get('dc:subject', ''))
+            subject = st.text_input("Subject", value=meta.get('dc:description', ''))
+
+
+with st.expander("Optimization after OCR"):
+    jpeg_quality = st.slider(
+        "JPEG quality", min_value=0, max_value=100, value=75, key="jpeg_quality"
+    )
+    png_quality = st.slider(
+        "PNG quality", min_value=0, max_value=100, value=75, key="png_quality"
+    )
+    jbig2_lossy = st.checkbox("JBIG2 lossy (dangerous)", value=False, key="jbig2_lossy")
+    jbig2_threshold = st.number_input("JBIG2 threshold", value=0, key="jbig2_threshold")
+
+with st.expander("Advanced options"):
+    jobs = st.slider(
+        "Threads",
+        min_value=1,
+        max_value=os.cpu_count(),
+        value=os.cpu_count(),
+        key="threads",
+    )
+    pages = st.text_input(
+        "Pages", value="", help="Comma-separated list of pages to process"
+    )
+    max_image_mpixels = st.number_input(
+        "Max image size",
+        value=250.0,
+        min_value=0.0,
+        help="Maximum image size in megapixels",
+    )
+    rotate_pages_threshold = st.number_input(
+        "Rotate pages threshold",
+        value=DEFAULT_ROTATE_PAGES_THRESHOLD,
+        min_value=0.0,
+        max_value=1000.0,
+        help="Threshold for automatic page rotation",
+    )
+    fast_web_view = st.number_input(
+        "Fast web view",
+        value=1.0,
+        min_value=0.0,
+        help="Linearize files above this size in MB",
+    )
+    continue_on_soft_render_error = st.checkbox(
+        "Continue on soft render error", value=True
+    )
+    verbose_labels = ["quiet", "default", "debug", "debug_all"]
+    verbose = st.selectbox(
+        "Verbosity level",
+        options=[-1, 0, 1, 2],
+        index=1,
+        format_func=partial(getitem, verbose_labels),
+    )
+
+if uploaded:
+    args = []
+    if mode and mode != 'normal':
+        args.append(f"--{mode}")
+    if language:
+        args.append(f"--language={language}")
+    if not uploaded.name.lower().endswith(".pdf") and image_dpi:
+        args.append(f"--image-dpi={image_dpi}")
+    if skip_big:
+        args.append("--skip-big")
+    if oversample:
+        args.append(f"--oversample={oversample}")
+    if rotate_pages:
+        args.append("--rotate-pages")
+    if deskew:
+        args.append("--deskew")
+    if clean:
+        args.append("--clean")
+    if clean_final:
+        args.append("--clean-final")
+    if remove_vectors:
+        args.append("--remove-vectors")
+    if output_type:
+        args.append(f"--output-type={output_type}")
+    if pdf_renderer:
+        args.append(f"--pdf-renderer={pdf_renderer}")
+    if optimize:
+        args.append(f"--optimize={optimize}")
+    if title:
+        args.append(f"--title={title}")
+    if author:
+        args.append(f"--author={author}")
+    if keywords:
+        args.append(f"--keywords={keywords}")
+    if subject:
+        args.append(f"--subject={subject}")
+    if pages:
+        args.append(f"--pages={pages}")
+    if max_image_mpixels:
+        args.append(f"--max-image-mpixels={max_image_mpixels}")
+    if rotate_pages_threshold:
+        args.append(f"--rotate-pages-threshold={rotate_pages_threshold}")
+    if fast_web_view:
+        args.append(f"--fast-web-view={fast_web_view}")
+    if continue_on_soft_render_error:
+        args.append("--continue-on-soft-render-error")
+    if verbose:
+        args.append(f"--verbose={verbose}")
+    if optimize > '0' and jpeg_quality:
+        args.append(f"--jpeg-quality={jpeg_quality}")
+    if optimize > '0' and png_quality:
+        args.append(f"--png-quality={png_quality}")
+    if jbig2_lossy:
+        args.append("--jbig2-lossy")
+    if jbig2_threshold:
+        args.append(f"--jbig2-threshold={jbig2_threshold}")
+    if jobs:
+        args.append(f"--jobs={jobs}")
+    input_file = NamedTemporaryFile(delete=True, suffix=f"_{uploaded.name}")
+    input_file.write(uploaded.getvalue())
+    input_file.flush()
+    input_file.seek(0)
+    args.append(str(input_file.name))
+    output_file = NamedTemporaryFile(delete=True, suffix=".pdf")
+    args.append(str(output_file.name))
+
+    st.session_state['running'] = (
+        'run_button' in st.session_state and st.session_state.run_button
+    )
+    if st.button(
+        "Run OCRmyPDF",
+        disabled=st.session_state.get("running", False),
+        key='run_button',
+    ):
+        st.session_state['running'] = True
+        args = [sys.executable, '-u', '-m', "ocrmypdf"] + args
+
+        proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        with st.container(border=True):
+            while proc.poll() is None:
+                line = proc.stderr.readline()
+                if line:
+                    st.html("<code>" + line.decode().strip() + "</code>")
+
+        if proc.returncode != 0:
+            st.error(f"ocrmypdf failed with exit code {proc.returncode}")
+            st.session_state['running'] = False
+            st.stop()
+
+        if Path(output_file.name).stat().st_size == 0:
+            st.error("No output PDF file was generated")
+            st.stop()
+
+        st.download_button(
+            label="Download output PDF",
+            data=input_file.read(),
+            file_name=uploaded.name,
+            mime="application/pdf",
+        )
+        st.session_state['running'] = False

misc/bisect_pdf.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: 2024 James R. Barlow
+# SPDX-License-Identifier: MIT
+
+"""Helper script for bisecting PDFs to find a page with an issue."""
+
+import sys
+
+import pikepdf
+
+if len(sys.argv) != 2:
+    print(f"Usage: {sys.argv[0]} <input.pdf>")
+    sys.exit(1)
+
+with pikepdf.open(sys.argv[1]) as pdf:
+    num_pages = len(pdf.pages)
+    low = 0
+    high = num_pages - 1
+    while low <= high:
+        mid = (low + high) // 2
+        with pikepdf.new() as new_pdf:
+            new_pdf.pages.extend(pdf.pages[low : mid + 1])
+            new_pdf.save(f"bisect-issue-{low + 1}-{mid + 1}.pdf")
+        print(f"Is bisect-issue-{low + 1}-{mid + 1}.pdf good or bad?", end=" ")
+        while True:
+            response = input().lower()
+            if response == "good":
+                low = mid + 1
+                break
+            elif response == "bad":
+                high = mid - 1
+                break
+            else:
+                print("Please respond with 'good' or 'bad'.")
+    print(f"The issue is on page {low + 1} of the original PDF.")
+    with pikepdf.new() as new_pdf:
+        new_pdf.pages.extend(pdf.pages[low])
+        new_pdf.save(f"bisect-issue-bad-{low + 1}.pdf")
+    with pikepdf.new() as new_pdf:
+        new_pdf.pages.extend(pdf.pages[:low])
+        new_pdf.pages.extend(pdf.pages[low + 1 :])
+        new_pdf.save(f"bisect-issue-good-{low + 1}.pdf")

misc/ocrmypdf_compare.py

@@ -0,0 +1,123 @@
+# SPDX-FileCopyrightText: 2025 James R. Barlow
+# SPDX-License-Identifier: MIT
+
+"""Run OCRmyPDF on the same PDF with different options."""
+
+from __future__ import annotations
+
+import os
+import shlex
+from io import BytesIO
+from pathlib import Path
+from subprocess import check_output, run
+from tempfile import TemporaryDirectory
+
+import pikepdf
+import pymupdf
+import streamlit as st
+from lxml import etree
+from streamlit_pdf_viewer import pdf_viewer
+
+
+def main():
+    st.set_page_config(layout="wide")
+
+    st.title("OCRmyPDF Compare")
+    st.write("Run OCRmyPDF on the same PDF with different options.")
+
+    uploaded_pdf = st.file_uploader("Upload a PDF", type=["pdf"])
+    if uploaded_pdf is None:
+        return
+
+    pdf_bytes = uploaded_pdf.read()
+
+    with pikepdf.open(BytesIO(pdf_bytes)) as p, TemporaryDirectory() as d:
+        with st.expander("PDF Metadata"):
+            with p.open_metadata() as meta:
+                xml_txt = str(meta)
+                parser = etree.XMLParser(remove_blank_text=True)
+                tree = etree.fromstring(xml_txt, parser=parser)
+                st.code(
+                    etree.tostring(tree, pretty_print=True).decode("utf-8"),
+                    language="xml",
+                )
+            st.write(p.docinfo)
+            st.write("Number of pages:", len(p.pages))
+
+        col1, col2 = st.columns(2)
+        with col1:
+            cli1 = st.text_area(
+                "Command line arguments for A",
+                key="args1",
+                value="ocrmypdf {in_} {out}",
+            )
+            env1 = st.text_area("Environment variables for A", key="env1")
+            args1 = shlex.split(
+                cli1.format(
+                    in_=os.path.join(d, "input.pdf"),
+                    out=os.path.join(d, "output1.pdf"),
+                )
+            )
+            st.code(shlex.join(args1))
+        with col2:
+            cli2 = st.text_area(
+                "Command line arguments for B",
+                key="args2",
+                value="ocrmypdf {in_} {out}",
+            )
+            env2 = st.text_area("Environment variables for B", key="env2")
+            args2 = shlex.split(
+                cli2.format(
+                    in_=os.path.join(d, "input.pdf"),
+                    out=os.path.join(d, "output2.pdf"),
+                )
+            )
+            st.code(shlex.join(args2))
+
+        if not st.button("Execute and Compare"):
+            return
+        with st.spinner("Executing..."):
+            Path(d, "input.pdf").write_bytes(pdf_bytes)
+            run(args1, env=dict(os.environ, **eval(env1 or "{}")))
+            run(args2, env=dict(os.environ, **eval(env2 or "{}")))
+
+            col1, col2 = st.columns(2)
+            with col1:
+                st.text(
+                    "Ghostscript version A: "
+                    + check_output(
+                        ["gs", "--version"],
+                        env=dict(os.environ, **eval(env1 or "{}")),
+                        text=True,
+                    )
+                )
+            with col2:
+                st.text(
+                    "Ghostscript version B: "
+                    + check_output(
+                        ["gs", "--version"],
+                        env=dict(os.environ, **eval(env2 or "{}")),
+                        text=True,
+                    )
+                )
+
+            doc1 = pymupdf.open(os.path.join(d, "output1.pdf"))
+            doc2 = pymupdf.open(os.path.join(d, "output2.pdf"))
+            for i, page1_2 in enumerate(zip(doc1, doc2)):
+                st.write(f"Page {i+1}")
+                page1, page2 = page1_2
+                col1, col2 = st.columns(2)
+                with col1, st.container(border=True):
+                    st.write(page1.get_text())
+                with col2, st.container(border=True):
+                    st.write(page2.get_text())
+
+            col1, col2 = st.columns(2)
+            with col1, st.expander("PDF Viewer"):
+                pdf_viewer(Path(d, "output1.pdf"))
+            with col2, st.expander("PDF Viewer"):
+                pdf_viewer(Path(d, "output2.pdf"))
+
+
+if __name__ == "__main__":
+    main()

misc/pdf_compare.py

@@ -0,0 +1,83 @@
+# SPDX-FileCopyrightText: 2025 James R. Barlow
+# SPDX-License-Identifier: MIT
+
+"""Compare two PDFs."""
+
+from __future__ import annotations
+
+import os
+from io import BytesIO
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+import pikepdf
+import pymupdf
+import streamlit as st
+from lxml import etree
+from streamlit_pdf_viewer import pdf_viewer
+
+
+def do_metadata(pdf):
+    with pikepdf.open(pdf) as pdf:
+        with pdf.open_metadata() as meta:
+            xml_txt = str(meta)
+            parser = etree.XMLParser(remove_blank_text=True)
+            tree = etree.fromstring(xml_txt, parser=parser)
+            st.code(
+                etree.tostring(tree, pretty_print=True).decode("utf-8"),
+                language="xml",
+            )
+        st.write(pdf.docinfo)
+        st.write("Number of pages:", len(pdf.pages))
+
+
+def main():
+    st.set_page_config(layout="wide")
+
+    st.title("PDF Compare")
+    st.write("Compare two PDFs.")
+
+    col1, col2 = st.columns(2)
+    with col1:
+        uploaded_pdf1 = st.file_uploader("Upload a PDF", type=["pdf"], key='pdf1')
+    with col2:
+        uploaded_pdf2 = st.file_uploader("Upload a PDF", type=["pdf"], key='pdf2')
+    if uploaded_pdf1 is None or uploaded_pdf2 is None:
+        return
+
+    pdf_bytes1 = uploaded_pdf1.getvalue()
+    pdf_bytes2 = uploaded_pdf2.getvalue()
+
+    with st.expander("PDF Metadata"):
+        col1, col2 = st.columns(2)
+        with col1:
+            do_metadata(BytesIO(pdf_bytes1))
+        with col2:
+            do_metadata(BytesIO(pdf_bytes2))
+
+    with TemporaryDirectory() as d:
+        Path(d, "1.pdf").write_bytes(pdf_bytes1)
+        Path(d, "2.pdf").write_bytes(pdf_bytes2)
+
+        with st.expander("Text"):
+            doc1 = pymupdf.open(os.path.join(d, "1.pdf"))
+            doc2 = pymupdf.open(os.path.join(d, "2.pdf"))
+            for i, page1_2 in enumerate(zip(doc1, doc2)):
+                st.write(f"Page {i+1}")
+                page1, page2 = page1_2
+                col1, col2 = st.columns(2)
+                with col1, st.container(border=True):
+                    st.write(page1.get_text())
+                with col2, st.container(border=True):
+                    st.write(page2.get_text())
+
+        with st.expander("PDF Viewer"):
+            col1, col2 = st.columns(2)
+            with col1:
+                pdf_viewer(Path(d, "1.pdf"), key='pdf_viewer1', render_text=True)
+            with col2:
+                pdf_viewer(Path(d, "2.pdf"), key='pdf_viewer2', render_text=True)
+
+
+if __name__ == "__main__":
+    main()

misc/pdf_text_diff.py

@@ -0,0 +1,46 @@
+# SPDX-FileCopyrightText: 2025 James R. Barlow
+# SPDX-License-Identifier: MPL-2.0
+
+"""Compare text in PDFs."""
+
+from __future__ import annotations
+
+from subprocess import run
+from tempfile import NamedTemporaryFile
+from typing import Annotated
+
+import typer
+
+
+def main(
+    pdf1: Annotated[typer.FileBinaryRead, typer.Argument()],
+    pdf2: Annotated[typer.FileBinaryRead, typer.Argument()],
+    engine: Annotated[str, typer.Option()] = 'pdftotext',
+):
+    """Compare text in PDFs."""
+
+    text1 = run(
+        ['pdftotext', '-layout', '-', '-'], stdin=pdf1, capture_output=True, check=True
+    )
+    text2 = run(
+        ['pdftotext', '-layout', '-', '-'], stdin=pdf2, capture_output=True, check=True
+    )
+
+    with NamedTemporaryFile() as f1, NamedTemporaryFile() as f2:
+        f1.write(text1.stdout)
+        f1.flush()
+        f2.write(text2.stdout)
+        f2.flush()
+        diff = run(
+            ['diff', '--color=always', '--side-by-side', f1.name, f2.name],
+            capture_output=True,
+        )
+        run(['less', '-R'], input=diff.stdout, check=True)
+        if text1.stdout.strip() != text2.stdout.strip():
+            return 1
+
+    return 0
+
+
+if __name__ == '__main__':
+    typer.run(main)

misc/webservice.py

@@ -1,107 +1,29 @@
-#!/usr/bin/env python3
-# SPDX-FileCopyrightText: 2019 James R. Barlow
+#!/usr/bin/env python
+# SPDX-FileCopyrightText: 2025 James R. Barlow
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
-"""This is a simple web service/HTTP wrapper for OCRmyPDF.
-
-This may be more convenient than the command line tool for some Docker users.
-Note that OCRmyPDF uses Ghostscript, which is licensed under AGPLv3+. While
-OCRmyPDF is under GPLv3, this file is distributed under the Affero GPLv3+ license,
-to emphasize that SaaS deployments should make sure they comply with
-Ghostscript's license as well as OCRmyPDF's.
-"""
-
-from __future__ import annotations
+"""Run the OCRmyPDF web service."""
 
 import os
-import shlex
-from subprocess import run
-from tempfile import TemporaryDirectory
+import sys
 
-from flask import Flask, Response, request, send_from_directory
-from werkzeug.utils import secure_filename
+try:
+    import streamlit  # noqa: F401
+except ImportError:
+    raise ImportError(
+        'You need to install streamlit in the Python environment '
+        'to run the web service.\n'
+    )
 
-app = Flask(__name__)
-app.secret_key = "secret"
-app.config['MAX_CONTENT_LENGTH'] = 50_000_000
-app.config.from_envvar("OCRMYPDF_WEBSERVICE_SETTINGS", silent=True)
-
-ALLOWED_EXTENSIONS = {"pdf"}
-
-
-def allowed_file(filename):
-    return "." in filename and filename.rsplit(".", 1)[1].lower() in ALLOWED_EXTENSIONS
-
-
-def do_ocrmypdf(file):
-    uploaddir = TemporaryDirectory(prefix="ocrmypdf-upload")
-    downloaddir = TemporaryDirectory(prefix="ocrmypdf-download")
-
-    filename = secure_filename(file.filename)
-    up_file = os.path.join(uploaddir.name, filename)
-    file.save(up_file)
-
-    down_file = os.path.join(downloaddir.name, filename)
-
-    cmd_args = [arg for arg in shlex.split(request.form["params"])]
-    if "--sidecar" in cmd_args:
-        return Response("--sidecar not supported", 501, mimetype='text/plain')
-
-    ocrmypdf_args = ["ocrmypdf", *cmd_args, up_file, down_file]
-    proc = run(ocrmypdf_args, capture_output=True, encoding="utf-8", check=False)
-    if proc.returncode != 0:
-        stderr = proc.stderr
-        return Response(stderr, 400, mimetype='text/plain')
-
-    return send_from_directory(downloaddir.name, filename)
-
-
-@app.route("/", methods=["GET", "POST"])
-def upload_file():
-    if request.method == "POST":
-        if "file" not in request.files:
-            return Response("No file in POST", 400, mimetype='text/plain')
-        file = request.files["file"]
-        if file.filename == "":
-            return Response("Empty filename", 400, mimetype='text/plain')
-        if not allowed_file(file.filename):
-            return Response("Invalid filename", 400, mimetype='text/plain')
-        if file and allowed_file(file.filename):
-            return do_ocrmypdf(file)
-        return Response("Some other problem", 400, mimetype='text/plain')
-
-    return """
-    <!doctype html>
-    <title>OCRmyPDF webservice</title>
-    <h1>Upload a PDF (debug UI)</h1>
-    <form method=post enctype=multipart/form-data>
-      <label for="args">Command line parameters</label>
-      <input type=textbox name=params>
-      <label for="file">File to upload</label>
-      <input type=file name=file>
-      <input type=submit value=Upload>
-    </form>
-    <h4>Notice</h2>
-    <div style="font-size: 70%; max-width: 34em;">
-    <p>This is a webservice wrapper for OCRmyPDF.</p>
-    <p>Copyright 2019 James R. Barlow</p>
-    <p>This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Affero General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-    </p>
-    <p>This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-    </p>
-    <p>
-    You should have received a copy of the GNU Affero General Public License
-    along with this program.  If not, see &lt;http://www.gnu.org/licenses/&gt;.
-    </p>
-    </div>
-    """
-
-
-if __name__ == "__main__":
-    app.run(host='0.0.0.0', port=5000)
+if __name__ == '__main__':
+    os.execvp(
+        sys.executable,
+        [
+            sys.executable,
+            '-m',
+            'streamlit',
+            'run',
+            'misc/_webservice.py',
+            *sys.argv[1:],
+        ],
+    )

pyproject.toml

@@ -1,8 +1,8 @@
 # SPDX-FileCopyrightText: 2022 James R. Barlow
 # SPDX-License-Identifier: MPL-2.0
 [build-system]
-requires = ["setuptools >= 61", "setuptools_scm[toml] >= 7.0.5", "wheel"]
-build-backend = "setuptools.build_meta"
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
 
 [project]
 name = "ocrmypdf"
@@ -46,6 +46,7 @@ keywords = ["PDF", "OCR", "optical character recognition", "PDF/A", "scanning"]
 Documentation = "https://ocrmypdf.readthedocs.io/"
 Source = "https://github.com/ocrmypdf/OCRmyPDF"
 Tracker = "https://github.com/ocrmypdf/OCRmyPDF/issues"
+Changelog = "https://github.com/ocrmypdf/OCRmyPDF/docs/release_notes.rst"
 
 [project.optional-dependencies]
 docs = ["sphinx", "sphinx-issues", "sphinx-rtd-theme"]
@@ -62,19 +63,16 @@ test = [
   "types-humanfriendly",
 ]
 watcher = ["watchdog>=1.0.2", "typer-slim[standard]", "python-dotenv"]
-webservice = ["Flask>=2.0.1"]
+webservice = ["streamlit>=1.41.0"]
 
 [project.scripts]
 ocrmypdf = "ocrmypdf.__main__:run"
 
-[tool.setuptools.package-data]
-ocrmypdf = ["data/sRGB.icc", "py.typed"]
+[tool.hatch.version]
+source = "vcs"
 
-[tool.setuptools.packages.find]
-where = ["src"]
-namespaces = false
-
-[tool.setuptools_scm]
+[tool.hatch.build.hooks.vcs]
+version-file = "src/ocrmypdf/_version.py"
 
 [tool.distutils.bdist_wheel]
 python-tag = "py310"
@@ -120,8 +118,6 @@ filterwarnings = [
 [[tool.mypy.overrides]]
 module = [
   'pluggy',
-  'tqdm',
-  'coloredlogs',
   'img2pdf',
   'pdfminer.*',
   'reportlab.*',
@@ -157,3 +153,12 @@ convention = "google"
 
 [tool.ruff.format]
 quote-style = "preserve"
+
+[dependency-groups]
+dev = [
+  "mypy>=1.13.0",
+  "pymupdf>=1.24.14",
+  "streamlit-pdf-viewer>=0.0.19",
+  "streamlit>=1.40.2",
+  "ipykernel>=6.29.5",
+]

src/ocrmypdf/__init__.py

@@ -9,11 +9,12 @@ from pluggy import HookimplMarker as _HookimplMarker
 
 from ocrmypdf import helpers, hocrtransform, pdfa, pdfinfo
 from ocrmypdf._concurrent import Executor
+from ocrmypdf._defaults import PROGRAM_NAME
 from ocrmypdf._jobcontext import PageContext, PdfContext
 from ocrmypdf._pipelines._common import (
     configure_debug_logging,
 )
-from ocrmypdf._version import PROGRAM_NAME, __version__
+from ocrmypdf._version import __version__
 from ocrmypdf.api import (
     Verbosity,
     configure_logging,
@@ -37,7 +38,6 @@ from ocrmypdf.pluginspec import OcrEngine, OrientationConfidence
 
 hookimpl = _HookimplMarker('ocrmypdf')
 
-
 __all__ = [
     '__version__',
     'BadArgsError',

src/ocrmypdf/_annots.py

@@ -0,0 +1,66 @@
+# SPDX-FileCopyrightText: 2024 James R. Barlow
+# SPDX-License-Identifier: MPL-2.0
+
+"""OCRmyPDF PDF annotation cleanup."""
+
+from __future__ import annotations
+
+import logging
+
+from pikepdf import Dictionary, Name, NameTree, Pdf
+
+log = logging.getLogger(__name__)
+
+
+def remove_broken_goto_annotations(pdf: Pdf) -> bool:
+    """Remove broken goto annotations from a PDF.
+
+    If a PDF contains a GoTo Action that points to a named destination that does not
+    exist, Ghostscript PDF/A conversion will fail. In any event, a named destination
+    that is not defined is not useful.
+
+    Args:
+        pdf: Opened PDF file.
+
+    Returns:
+        bool: True if the file was modified, False if not.
+    """
+    modified = False
+
+    # Check if there are any named destinations
+    if Name.Names not in pdf.Root:
+        return modified
+    if Name.Dests not in pdf.Root[Name.Names]:
+        return modified
+
+    dests = pdf.Root[Name.Names][Name.Dests]
+    if not isinstance(dests, Dictionary):
+        return modified
+    nametree = NameTree(dests)
+
+    # Create a set of all named destinations
+    names = set(k for k in nametree.keys())
+
+    for n, page in enumerate(pdf.pages):
+        if Name.Annots not in page:
+            continue
+        for annot in page[Name.Annots]:
+            if not isinstance(annot, Dictionary):
+                continue
+            if Name.A not in annot or Name.D not in annot[Name.A]:
+                continue
+            # We found an annotation that points to a named destination
+            named_destination = str(annot[Name.A][Name.D])
+            if named_destination not in names:
+                # If there is no corresponding named destination, remove the
+                # annotation. Having no destination set is still valid and just
+                # makes the link non-functional.
+                log.warning(
+                    f"Disabling a hyperlink annotation on page {n + 1} to a "
+                    "non-existent named destination "
+                    f"{named_destination}."
+                )
+                del annot[Name.A][Name.D]
+                modified = True
+
+    return modified

src/ocrmypdf/_defaults.py

@@ -6,3 +6,5 @@ DEFAULT_LANGUAGE = 'eng'
 
 # Default rotation threshold
 DEFAULT_ROTATE_PAGES_THRESHOLD = 14.0
+
+PROGRAM_NAME = 'OCRmyPDF'

src/ocrmypdf/_exec/ghostscript.py

@@ -17,7 +17,11 @@ from subprocess import PIPE, CalledProcessError
 from packaging.version import Version
 from PIL import Image, UnidentifiedImageError
 
-from ocrmypdf.exceptions import ColorConversionNeededError, SubprocessOutputError
+from ocrmypdf.exceptions import (
+    ColorConversionNeededError,
+    InputFileError,
+    SubprocessOutputError,
+)
 from ocrmypdf.helpers import Resolution
 from ocrmypdf.subprocess import get_version, run, run_polling_stderr
 
@@ -111,7 +115,6 @@ def rasterize_pdf(
     args_gs = (
         [
             GS,
-            '-dQUIET',
             '-dSAFER',
             '-dBATCH',
             '-dNOPAUSE',
@@ -125,7 +128,7 @@ def rasterize_pdf(
         + (['-dPDFSTOPONERROR'] if stop_on_error else [])
         + [
             '-o',
-            '-',
+            fspath(output_file),
             '-sstdout=%stderr',  # Literal %s, not string interpolation
             '-dAutoRotatePages=/None',  # Probably has no effect on raster
             '-f',
@@ -137,14 +140,23 @@ def rasterize_pdf(
         p = run(args_gs, stdout=PIPE, stderr=PIPE, check=True)
     except CalledProcessError as e:
         log.error(e.stderr.decode(errors='replace'))
-        raise SubprocessOutputError('Ghostscript rasterizing failed') from e
-    else:
-        stderr = p.stderr.decode(errors='replace')
-        if _gs_error_reported(stderr):
-            log.error(stderr)
+        Path(output_file).unlink(missing_ok=True)
+        raise SubprocessOutputError("Ghostscript rasterizing failed") from e
+
+    stderr = p.stderr.decode(errors='replace')
+    if _gs_error_reported(stderr):
+        log.error(stderr)
+        if stop_on_error and "recoverable image error" in stderr:
+            Path(output_file).unlink(missing_ok=True)
+            raise InputFileError(
+                "Ghostscript rasterizing failed. The input file contains errors that "
+                "cause PDF viewers to interpret it differently and incorrectly. "
+                "Try using --continue-on-soft-render-error and manually inspect the "
+                "input and output files to check for visual differences or errors."
+            )
 
     try:
-        with Image.open(BytesIO(p.stdout)) as im:
+        with Image.open(output_file) as im:
             if rotation is not None:
                 log.debug("Rotating output by %i", rotation)
                 # rotation is a clockwise angle and Image.ROTATE_* is
@@ -157,13 +169,19 @@ def rasterize_pdf(
                     im = im.transpose(Image.Transpose.ROTATE_270)
                 if rotation % 180 == 90:
                     page_dpi = page_dpi.flip_axis()
-            im.save(fspath(output_file), dpi=page_dpi)
+            im.save(output_file, dpi=page_dpi)
     except UnidentifiedImageError:
         log.error(
             f"Ghostscript (using {raster_device} at {raster_dpi} dpi) produced "
             "an invalid page image file."
         )
         raise
+    except OSError as e:
+        log.error(
+            f"Ghostscript (using {raster_device} at {raster_dpi} dpi) produced "
+            "an invalid page image file."
+        )
+        raise UnidentifiedImageError() from e
 
 
 class GhostscriptFollower:
@@ -271,19 +289,15 @@ def generate_pdfa(
             f"-dPDFA={pdfa_part}",
             "-dPDFACompatibilityPolicy=1",
             "-o",
-            "-",
+            fspath(output_file),
             "-sstdout=%stderr",  # Literal %s, not string interpolation
         ]
     )
     args_gs.extend(fspath(s) for s in pdf_pages)  # Stringify Path objs
     try:
-        with (
-            Path(output_file).open('wb') as output,
-            GhostscriptFollower(progressbar_class) as pbar,
-        ):
+        with GhostscriptFollower(progressbar_class) as pbar:
             p = run_polling_stderr(
                 args_gs,
-                stdout=output,
                 stderr=PIPE,
                 check=True,
                 text=True,

src/ocrmypdf/_exec/jbig2enc.py

@@ -5,7 +5,7 @@
 
 from __future__ import annotations
 
-from subprocess import PIPE
+from subprocess import PIPE, CalledProcessError
 
 from packaging.version import Version
 
@@ -14,7 +14,13 @@ from ocrmypdf.subprocess import get_version, run
 
 
 def version() -> Version:
-    return Version(get_version('jbig2', regex=r'jbig2enc (\d+(\.\d+)*).*'))
+    try:
+        version = get_version('jbig2', regex=r'jbig2enc (\d+(\.\d+)*).*')
+    except CalledProcessError as e:
+        # TeX Live for Windows provides an incompatible jbig2.EXE which may
+        # be on the PATH.
+        raise MissingDependencyError('jbig2enc') from e
+    return Version(version)
 
 
 def available():

src/ocrmypdf/_exec/unpaper.py

@@ -48,7 +48,7 @@ class UnpaperImageTooLargeError(Exception):
 
 
 def version() -> Version:
-    return Version(get_version('unpaper'))
+    return Version(get_version('unpaper', regex=r'(?m).*?(\d+(\.\d+)(\.\d+)?)'))
 
 
 @contextmanager

src/ocrmypdf/_graft.py

@@ -61,19 +61,31 @@ def strip_invisible_text(pdf: Pdf, page: Page):
     stream = []
     in_text_obj = False
     render_mode = 0
+    render_mode_stack = []
     text_objects = []
 
     for operands, operator in parse_content_stream(page, ''):
+        if operator == Operator('Tr'):
+            render_mode = operands[0]
+
+        if operator == Operator('q'):
+            render_mode_stack.append(render_mode)
+
+        if operator == Operator('Q'):
+            try:
+                render_mode = render_mode_stack.pop()
+            except IndexError:
+                # Stack underflow: content stream is malformed
+                # but try to carry on
+                pass
+
         if not in_text_obj:
             if operator == Operator('BT'):
                 in_text_obj = True
-                render_mode = 0
                 text_objects.append((operands, operator))
             else:
                 stream.append((operands, operator))
         else:
-            if operator == Operator('Tr'):
-                render_mode = operands[0]
             text_objects.append((operands, operator))
             if operator == Operator('ET'):
                 in_text_obj = False

src/ocrmypdf/_logging.py

@@ -26,5 +26,5 @@ class PageNumberFilter(logging.Filter):
 class RichLoggingHandler(RichHandler):
     def __init__(self, console: Console, **kwargs):
         super().__init__(
-            console=console, show_level=False, show_time=False, markup=True, **kwargs
+            console=console, show_level=False, show_time=False, markup=False, **kwargs
         )

src/ocrmypdf/_metadata.py

@@ -15,8 +15,9 @@ from pikepdf import Dictionary, Name, Pdf
 from pikepdf import __version__ as PIKEPDF_VERSION
 from pikepdf.models.metadata import PdfMetadata, encode_pdf_date
 
+from ocrmypdf._annots import remove_broken_goto_annotations
+from ocrmypdf._defaults import PROGRAM_NAME
 from ocrmypdf._jobcontext import PdfContext
-from ocrmypdf._version import PROGRAM_NAME
 from ocrmypdf._version import __version__ as OCRMYPF_VERSION
 from ocrmypdf.languages import iso_639_2_from_3
 

src/ocrmypdf/_pipeline.py

@@ -79,6 +79,14 @@ def triage_image_file(input_file: Path, output_file: Path, options) -> None:
     except OSError as e:
         # Recover the original filename
         log.error(str(e).replace(str(input_file), str(options.input_file)))
+        if not input_file.exists():
+            log.error("Input file does not exist: %s", input_file)
+        if input_file.is_dir():
+            log.error("Input file is a directory: %s", input_file)
+        if input_file.is_file():
+            log.error("Input file is a file: %s", input_file)
+        if input_file.stat().st_size == 0:
+            log.error("Input file is empty: %s", input_file)
         raise UnsupportedImageFormatError() from e
 
     with im:
@@ -159,8 +167,13 @@ def triage(
                     "Argument --image-dpi is being ignored because the "
                     "input file is a PDF, not an image."
                 )
-            # Origin file is a pdf create a symlink with pdf extension
-            safe_symlink(input_file, output_file)
+            try:
+                with pikepdf.open(input_file) as pdf:
+                    pdf.save(output_file)
+            except pikepdf.PdfError as e:
+                raise InputFileError() from e
+            except pikepdf.PasswordError as e:
+                raise EncryptedPdfError() from e
             return output_file
     except OSError as e:
         log.debug(f"Temporary file was at: {input_file}")
@@ -854,7 +867,7 @@ def fix_pagepdf_boxes(
             page.CropBox = cropbox
             page.TrimBox = trimbox
         pdf.save(out_file)
-    return pdf
+    return out_file
 
 
 def generate_postscript_stub(context: PdfContext) -> Path:

src/ocrmypdf/_pipelines/_common.py

@@ -20,7 +20,9 @@ from pathlib import Path
 from typing import NamedTuple, cast
 
 import PIL
+from pikepdf import Pdf
 
+from ocrmypdf._annots import remove_broken_goto_annotations
 from ocrmypdf._concurrent import Executor, setup_executor
 from ocrmypdf._jobcontext import PageContext, PdfContext
 from ocrmypdf._logging import PageNumberFilter
@@ -33,6 +35,7 @@ from ocrmypdf._pipeline import (
     generate_postscript_stub,
     get_orientation_correction,
     get_pdf_save_settings,
+    get_pdfinfo,
     optimize_pdf,
     preprocess_clean,
     preprocess_deskew,
@@ -51,9 +54,12 @@ from ocrmypdf.helpers import (
     available_cpu_count,
     check_pdf,
     pikepdf_enable_mmap,
+    running_in_docker,
+    running_in_snap,
     samefile,
 )
 from ocrmypdf.pdfa import file_claims_pdfa
+from ocrmypdf.pdfinfo import PdfInfo
 
 log = logging.getLogger(__name__)
 tls = threading.local()
@@ -100,6 +106,23 @@ class PageResult(NamedTuple):
     """Orientation correction in degrees."""
 
 
+class HOCRResultEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, Path):
+            return {'Path': str(obj)}
+        return super().default(obj)
+
+
+class HOCRResultDecoder(json.JSONDecoder):
+    def __init__(self, *args, **kwargs):
+        super().__init__(object_hook=self.dict_to_object, *args, **kwargs)
+
+    def dict_to_object(self, d):
+        if 'Path' in d:
+            return Path(d['Path'])
+        return d
+
+
 @dataclass
 class HOCRResult:
     """Result when hOCR is finished processing."""
@@ -119,38 +142,14 @@ class HOCRResult:
     orientation_correction: int = 0
     """Orientation correction in degrees."""
 
-    def __getstate__(self):
-        """Return state values to be pickled."""
-        return {
-            k: (
-                ('Path://' + str(v))
-                if k in ('pdf_page_from_image', 'hocr', 'textpdf') and v is not None
-                else v
-            )
-            for k, v in self.__dict__.items()
-        }
-
-    def __setstate__(self, state):
-        """Restore state from the unpickled state values."""
-        self.__dict__.update(
-            {
-                k: (
-                    Path(v.removeprefix('Path://'))
-                    if k in ('pdf_page_from_image', 'hocr', 'textpdf') and v is not None
-                    else v
-                )
-                for k, v in state.items()
-            }
-        )
-
     @classmethod
     def from_json(cls, json_str: str) -> HOCRResult:
         """Create an instance from a dict."""
-        return cls(**json.loads(json_str))
+        return cls(**json.loads(json_str, cls=HOCRResultDecoder))
 
     def to_json(self) -> str:
         """Serialize to a JSON string."""
-        return json.dumps(self.__getstate__())
+        return json.dumps(self.__dict__, cls=HOCRResultEncoder)
 
 
 def configure_debug_logging(
@@ -183,7 +182,7 @@ def configure_debug_logging(
     return log_file_handler, remover
 
 
-def worker_init(max_pixels: int) -> None:
+def worker_init(max_pixels: int | None) -> None:
     """Initialize a worker thread or process."""
     # In Windows, child process will not inherit our change to this value in
     # the parent process, so ensure workers get it set. Not needed when running
@@ -215,6 +214,22 @@ def manage_debug_log_handler(
             remover()
 
 
+def _print_temp_folder_location(work_folder: Path):
+    """Print the location of the temporary work folder."""
+    msgs = [f"Temporary working files retained at:\n{work_folder}"]
+    if running_in_docker():  # pragma: no cover
+        msgs.append(
+            "OCRmyPDF is running in a Docker container, "
+            "so the files will be inside the container."
+        )
+    elif running_in_snap():  # pragma: no cover
+        msgs.append(
+            "OCRmyPDF is running in a Snap container, "
+            "so the files will be inside the container."
+        )
+    print('\n'.join(msgs), file=sys.stderr)
+
+
 @contextmanager
 def manage_work_folder(*, work_folder: Path, retain: bool, print_location: bool):
     try:
@@ -222,10 +237,7 @@ def manage_work_folder(*, work_folder: Path, retain: bool, print_location: bool)
     finally:
         if retain:
             if print_location:
-                print(
-                    f"Temporary working files retained at:\n{work_folder}",
-                    file=sys.stderr,
-                )
+                _print_temp_folder_location(work_folder)
         else:
             shutil.rmtree(work_folder, ignore_errors=True)
 
@@ -300,6 +312,20 @@ def setup_pipeline(
     return executor
 
 
+def do_get_pdfinfo(
+    pdf_path: Path, executor: Executor, options: argparse.Namespace
+) -> PdfInfo:
+    return get_pdfinfo(
+        pdf_path,
+        executor=executor,
+        detailed_analysis=options.redo_ocr,
+        progbar=options.progress_bar,
+        max_workers=options.jobs,
+        use_threads=options.use_threads,
+        check_pages=options.pages,
+    )
+
+
 def preprocess(
     page_context: PageContext,
     image: Path,
@@ -414,7 +440,14 @@ def postprocess(
     pdf_file: Path, context: PdfContext, executor: Executor
 ) -> tuple[Path, Sequence[str]]:
     """Postprocess the PDF file."""
-    pdf_out = pdf_file
+    # pdf_out = pdf_file
+    with Pdf.open(pdf_file) as pdf:
+        fix_annots = context.get_path('fix_annots.pdf')
+        if remove_broken_goto_annotations(pdf):
+            pdf.save(fix_annots)
+            pdf_out = fix_annots
+        else:
+            pdf_out = pdf_file
     if context.options.output_type.startswith('pdfa'):
         ps_stub_out = generate_postscript_stub(context)
         pdf_out = convert_to_pdfa(pdf_out, ps_stub_out, context)
@@ -441,7 +474,8 @@ def report_output_pdf(options, start_input_file, optimize_messages) -> ExitCode:
                 log.info("Output file is a %s (as expected)", pdfa_info['conformance'])
             else:
                 log.warning(
-                    "Output file is okay but is not PDF/A (seems to be %s)",
+                    "Output file is a valid PDF, but conversion to PDF/A did not "
+                    "succeed (issue: %s)",
                     pdfa_info['conformance'],
                 )
                 return ExitCode.pdfa_conversion_failed

src/ocrmypdf/_pipelines/hocr_to_ocr_pdf.py

@@ -19,11 +19,11 @@ from ocrmypdf._graft import OcrGrafter
 from ocrmypdf._jobcontext import PageContext, PdfContext
 from ocrmypdf._pipeline import (
     copy_final,
-    get_pdfinfo,
     render_hocr_page,
 )
 from ocrmypdf._pipelines._common import (
     HOCRResult,
+    do_get_pdfinfo,
     manage_work_folder,
     postprocess,
     report_output_pdf,
@@ -117,15 +117,7 @@ def run_hocr_to_ocr_pdf_pipeline(
         origin_pdf = work_folder / 'origin.pdf'
 
         # Gather pdfinfo and create context
-        pdfinfo = get_pdfinfo(
-            origin_pdf,
-            executor=executor,
-            detailed_analysis=options.redo_ocr,
-            progbar=options.progress_bar,
-            max_workers=options.jobs,
-            use_threads=options.use_threads,
-            check_pages=options.pages,
-        )
+        pdfinfo = do_get_pdfinfo(origin_pdf, executor, options)
         context = PdfContext(options, work_folder, origin_pdf, pdfinfo, plugin_manager)
         plugin_manager.hook.check_options(options=options)
         optimize_messages = exec_hocr_to_ocr_pdf(context, executor)

src/ocrmypdf/_pipelines/ocr.py

@@ -21,7 +21,6 @@ from ocrmypdf._graft import OcrGrafter
 from ocrmypdf._jobcontext import PageContext, PdfContext
 from ocrmypdf._pipeline import (
     copy_final,
-    get_pdfinfo,
     is_ocr_required,
     merge_sidecars,
     ocr_engine_hocr,
@@ -33,6 +32,7 @@ from ocrmypdf._pipeline import (
 from ocrmypdf._pipelines._common import (
     PageResult,
     cli_exception_handler,
+    do_get_pdfinfo,
     manage_debug_log_handler,
     manage_work_folder,
     postprocess,
@@ -171,16 +171,7 @@ def _run_pipeline(
         )
 
         # Gather pdfinfo and create context
-        pdfinfo = get_pdfinfo(
-            origin_pdf,
-            executor=executor,
-            detailed_analysis=options.redo_ocr,
-            progbar=options.progress_bar,
-            max_workers=options.jobs,
-            use_threads=options.use_threads,
-            check_pages=options.pages,
-        )
-
+        pdfinfo = do_get_pdfinfo(origin_pdf, executor, options)
         context = PdfContext(options, work_folder, origin_pdf, pdfinfo, plugin_manager)
 
         # Validate options are okay for this pdf

src/ocrmypdf/_pipelines/pdf_to_hocr.py

@@ -17,13 +17,13 @@ import PIL
 from ocrmypdf._concurrent import Executor
 from ocrmypdf._jobcontext import PageContext, PdfContext
 from ocrmypdf._pipeline import (
-    get_pdfinfo,
     is_ocr_required,
     ocr_engine_hocr,
     validate_pdfinfo_options,
 )
 from ocrmypdf._pipelines._common import (
     HOCRResult,
+    do_get_pdfinfo,
     manage_work_folder,
     process_page,
     set_thread_pageno,
@@ -94,18 +94,11 @@ def run_hocr_pipeline(
         work_folder=options.output_folder, retain=True, print_location=False
     ) as work_folder:
         executor = setup_pipeline(options, plugin_manager)
-        shutil.copy2(options.input_file, work_folder / 'origin.pdf')
+        origin_pdf = work_folder / 'origin.pdf'
+        shutil.copy2(options.input_file, origin_pdf)
 
         # Gather pdfinfo and create context
-        pdfinfo = get_pdfinfo(
-            options.input_file,
-            executor=executor,
-            detailed_analysis=options.redo_ocr,
-            progbar=options.progress_bar,
-            max_workers=options.jobs,
-            use_threads=options.use_threads,
-            check_pages=options.pages,
-        )
+        pdfinfo = do_get_pdfinfo(origin_pdf, executor, options)
         context = PdfContext(
             options, work_folder, options.input_file, pdfinfo, plugin_manager
         )

src/ocrmypdf/_progressbar.py

@@ -32,12 +32,76 @@ class ProgressBar(Protocol):
     The progress bar is held in the main process/thread and not updated by child
     process/threads. When a child notifies the parent of completed work, the
     parent updates the progress bar.
-
     Progress bars should never write to ``sys.stdout``, or they will corrupt the
     output if OCRmyPDF writes a PDF to standard output.
 
-    The type of events that OCRmyPDF reports to a progress bar may change in
+    Note:
+        The type of events that OCRmyPDF reports to a progress bar may change in
     minor releases.
+
+    Args:
+        total (int | float | None):
+            The total number of work units expected. If ``None``, the total is unknown.
+            For example, if you are processing pages, this might be the number of pages,
+            or if you are measuring overall progress in percent, this might be 100.
+        desc (str | None):
+            A brief description of the current step (e.g. "Scanning contents",
+            "OCR", "PDF/A conversion"). OCRmyPDF updates this before each major step.
+        unit (str | None):
+            A short label for the type of work being tracked (e.g. "page", "%", "image").
+        disable (bool):
+            If ``True``, progress updates are suppressed (no output). Defaults to ``False``.
+        **kwargs:
+            Future or extra parameters that OCRmyPDF might pass. Implementations
+            should accept and ignore unrecognized keywords gracefully.
+
+    Example:
+        A simple plugin implementation could look like this:
+
+        .. code-block:: python
+
+            from ocrmypdf.pluginspec import ProgressBar
+            from ocrmypdf import hookimpl
+
+            class ConsoleProgressBar(ProgressBar):
+                def __init__(self, *, total=None, desc=None, unit=None, disable=False, **kwargs):
+                    self.total = total
+                    self.desc = desc
+                    self.unit = unit
+                    self.disable = disable
+                    self.current = 0
+
+                def __enter__(self):
+                    if not self.disable:
+                        print(f"Starting {self.desc or 'an OCR task'} (total={self.total} {self.unit})")
+                    return self
+
+                def __exit__(self, exc_type, exc_value, traceback):
+                    if not self.disable:
+                        if exc_type is None:
+                            print("Completed successfully.")
+                        else:
+                            print(f"Task ended with error: {exc_value}")
+                    return False  # Let OCRmyPDF raise any exceptions
+
+                def update(self, n=1, *, completed=None):
+                    if completed is not None:
+                        # If 'completed' is given, you could set self.current = completed
+                        # but let's just read it to show usage
+                        print(f"Absolute completion reported: {completed}")
+                    # Otherwise, we increment by 'n'
+                    self.current += n
+                    if not self.disable:
+                        if self.total:
+                            percent = (self.current / self.total) * 100
+                            print(f"{self.desc}: {self.current}/{self.total} ({percent:.1f}%)")
+                        else:
+                            print(f"{self.desc}: {self.current} units done")
+
+            @hookimpl
+            def get_progressbar_class():
+                return MyProgressBar
+
     """
 
     def __init__(
@@ -51,13 +115,22 @@ class ProgressBar(Protocol):
     ):
         """Initialize a progress bar.
 
-        *total* indicates the total number of work units. If None, the total
-        number of work units is unknown. If *disable* is True, the progress bar
-        should be disabled. *unit* is a description of the work unit.
-        *desc* is a description of the overall task to be performed.
+        This is called once before any work is done. OCRmyPDF supplies the total
+        number of units (or None if unknown), a description of the work, and the
+        type of units. The ``disable`` parameter can be used to turn off progress
+        reporting. Unrecognized keyword arguments should be ignored.
 
-        Unrecognized keyword arguments must be ignored, as the list of keyword
-        arguments may grow with time.
+        Args:
+            total (int | float | None):
+                The total amount of work. If ``None``, the total is unknown.
+            desc (str | None):
+                A description of the current task. May change for different stages.
+            unit (str | None):
+                A short label for the unit of work.
+            disable (bool):
+                If ``True``, no output or logging should be displayed.
+            **kwargs:
+                Extra parameters that may be passed by OCRmyPDF in future versions.
         """
 
     def __enter__(self):
@@ -66,10 +139,32 @@ class ProgressBar(Protocol):
     def __exit__(self, *args):
         """Exit a progress bar context."""
 
-    def update(self, n=1, *, completed=None):
-        """Update the progress bar by an increment.
+    def update(self, n: float = 1, *, completed: float | None = None):
+        """Increment the progress bar by ``n`` units, or set an absolute completion.
 
-        For use within a progress bar context.
+        OCRmyPDF calls this method repeatedly while processing pages or other tasks.
+        If your total is known and you track it, you might do something like:
+
+        .. code-block:: python
+
+            self.current += n
+            percent = (self.current / total) * 100
+
+        The ``completed`` argument can indicate an absolute position, which is
+        particularly helpful if you're tracking a percentage of work (e.g., 0 to 100)
+        and want precise updates. In contrast, the incremental parameter ``n`` is
+        often more useful for page-based increments.
+
+        Args:
+            n (float, optional):
+                The amount to increment the progress by. Defaults to 1. May be
+                fractional if OCRmyPDF performs partial steps. If you are tracking
+                pages, this is typically how many pages have been processed in the
+                most recent step.
+            completed (float | None, optional):
+                The absolute amount of work completed so far. This can override or
+                supplement the simple increment logic. It's particularly useful
+                for percentage-based tracking (e.g., when ``total`` is 100).
         """
 
 

src/ocrmypdf/_validation.py

@@ -28,7 +28,13 @@ from ocrmypdf.exceptions import (
     MissingDependencyError,
     OutputFileAccessError,
 )
-from ocrmypdf.helpers import is_file_writable, monotonic, safe_symlink
+from ocrmypdf.helpers import (
+    is_file_writable,
+    monotonic,
+    running_in_docker,
+    running_in_snap,
+    safe_symlink,
+)
 from ocrmypdf.subprocess import check_external_program
 
 log = logging.getLogger(__name__)
@@ -237,18 +243,6 @@ def check_options(options: Namespace, plugin_manager: PluginManager) -> None:
     _check_plugin_options(options, plugin_manager)
 
 
-def _in_docker():
-    return Path('/.dockerenv').exists()
-
-
-def _in_snap():
-    try:
-        cgroup_text = Path('/proc/self/cgroup').read_text()
-        return 'snap.ocrmypdf' in cgroup_text
-    except FileNotFoundError:
-        return False
-
-
 def create_input_file(options: Namespace, work_folder: Path) -> tuple[Path, str]:
     if options.input_file == '-':
         # stdin
@@ -272,7 +266,7 @@ def create_input_file(options: Namespace, work_folder: Path) -> tuple[Path, str]
             return target, os.fspath(options.input_file)
         except FileNotFoundError as e:
             msg = f"File not found - {options.input_file}"
-            if _in_docker():  # pragma: no cover
+            if running_in_docker():  # pragma: no cover
                 msg += (
                     "\nDocker cannot access your working directory unless you "
                     "explicitly share it with the Docker container and set up"
@@ -282,7 +276,7 @@ def create_input_file(options: Namespace, work_folder: Path) -> tuple[Path, str]
                     "\tdocker run -i --rm jbarlow83/ocrmypdf - - <input.pdf >output.pdf"
                     "\n"
                 )
-            elif _in_snap():  # pragma: no cover
+            elif running_in_snap():  # pragma: no cover
                 msg += (
                     "\nSnap applications cannot access files outside of "
                     "your home directory unless you explicitly allow it. "

src/ocrmypdf/_version.py

@@ -1,16 +0,0 @@
-# SPDX-FileCopyrightText: 2022 James R. Barlow
-# SPDX-License-Identifier: MPL-2.0
-
-"""Get version by introspecting package information.
-
-OCRmyPDF uses setuptools_scm to derive version from git tags.
-"""
-
-from __future__ import annotations
-
-from importlib.metadata import version as _package_version
-
-PROGRAM_NAME = 'ocrmypdf'
-
-# Official PEP 396
-__version__ = _package_version('ocrmypdf')

src/ocrmypdf/api.py

@@ -14,7 +14,7 @@ from collections.abc import Iterable, Sequence
 from enum import IntEnum
 from io import IOBase
 from pathlib import Path
-from typing import AnyStr, BinaryIO
+from typing import BinaryIO
 from warnings import warn
 
 import pluggy
@@ -28,7 +28,7 @@ from ocrmypdf._validation import check_options
 from ocrmypdf.cli import ArgumentParser, get_parser
 from ocrmypdf.helpers import is_iterable_notstr
 
-StrPath = Path | AnyStr
+StrPath = Path | str | bytes
 PathOrIO = BinaryIO | StrPath
 
 # Installing plugins affects the global state of the Python interpreter,
@@ -140,9 +140,9 @@ def configure_logging(
 
 def _kwargs_to_cmdline(
     *, defer_kwargs: set[str], **kwargs
-) -> tuple[list[str], dict[str, AnyStr]]:
+) -> tuple[list[str | bytes], dict[str, str | bytes]]:
     """Convert kwargs to command line arguments."""
-    cmdline = []
+    cmdline: list[str | bytes] = []
     deferred = {}
     for arg, val in kwargs.items():
         if val is None:
@@ -279,7 +279,7 @@ def ocr(  # noqa: D417
     fast_web_view: float | None = None,
     continue_on_soft_render_error: bool | None = None,
     invalidate_digital_signatures: bool | None = None,
-    plugins: Iterable[StrPath] | None = None,
+    plugins: Iterable[Path | str] | None = None,
     plugin_manager=None,
     keep_temporary_files: bool | None = None,
     progress_bar: bool | None = None,
@@ -420,7 +420,7 @@ def _pdf_to_hocr(  # noqa: D417
     continue_on_soft_render_error: bool | None = None,
     invalidate_digital_signatures: bool | None = None,
     plugin_manager=None,
-    plugins: Sequence[StrPath] | None = None,
+    plugins: Sequence[Path | str] | None = None,
     keep_temporary_files: bool | None = None,
     **kwargs,
 ):
@@ -491,7 +491,7 @@ def _hocr_to_ocr_pdf(  # noqa: D417
     color_conversion_strategy: str | None = None,
     fast_web_view: float | None = None,
     plugin_manager=None,
-    plugins: Sequence[StrPath] | None = None,
+    plugins: Sequence[Path | str] | None = None,
     **kwargs,
 ):
     """Run OCRmyPDF on a work folder and produce an output PDF.

src/ocrmypdf/cli.py

@@ -10,7 +10,7 @@ from collections.abc import Callable, Mapping
 from typing import Any, TypeVar
 
 from ocrmypdf._defaults import DEFAULT_ROTATE_PAGES_THRESHOLD
-from ocrmypdf._version import PROGRAM_NAME as _PROGRAM_NAME
+from ocrmypdf._defaults import PROGRAM_NAME as _PROGRAM_NAME
 from ocrmypdf._version import __version__ as _VERSION
 
 T = TypeVar('T', int, float)

src/ocrmypdf/helpers.py

@@ -335,3 +335,17 @@ def pikepdf_enable_mmap() -> None:
         )
     except AttributeError:
         log.debug("pikepdf mmap not available")
+
+
+def running_in_docker() -> bool:
+    """Returns True if we seem to be running in a Docker container."""
+    return Path('/.dockerenv').exists()
+
+
+def running_in_snap() -> bool:
+    """Returns True if we seem to be running in a Snap container."""
+    try:
+        cgroup_text = Path('/proc/self/cgroup').read_text()
+        return 'snap.ocrmypdf' in cgroup_text
+    except FileNotFoundError:
+        return False

src/ocrmypdf/hocrtransform/_hocr.py

@@ -1,6 +1,7 @@
 # SPDX-FileCopyrightText: 2010 Jonathan Brinley
 # SPDX-FileCopyrightText: 2013-2014 Julien Pfefferkorn
 # SPDX-FileCopyrightText: 2023 James R. Barlow
+# SPDX-FileCopyrightText: 2025 Odin Dahlström
 # SPDX-License-Identifier: MIT
 
 """hOCR transform implementation."""
@@ -13,7 +14,7 @@ import re
 import unicodedata
 from dataclasses import dataclass
 from itertools import pairwise
-from math import atan, cos, pi
+from math import atan, pi
 from pathlib import Path
 from xml.etree import ElementTree
 
@@ -82,6 +83,13 @@ class HocrTransform:
         ''',
         re.VERBOSE,
     )
+    textangle_pattern = re.compile(
+        r'''
+        textangle \s+
+        ([\-\+]?\d*\.?\d*)  # +/- decimal float
+        ''',
+        re.VERBOSE,
+    )
 
     def __init__(
         self,
@@ -156,6 +164,14 @@ class HocrTransform:
             return (0.0, 0.0)
         return float(matches.group(1)), int(matches.group(2))
 
+    @classmethod
+    def textangle(cls, element: Element) -> float:
+        """Get text angle of an element."""
+        matches = cls.textangle_pattern.search(element.attrib.get('title', ''))
+        if not matches:
+            return 0.0
+        return float(matches.group(1))
+
     def _child_xpath(self, html_tag: str, html_class: str | None = None) -> str:
         xpath = f".//{self.xmlns}{html_tag}"
         if html_class:
@@ -210,7 +226,7 @@ class HocrTransform:
                     for element in par.iterfind(self._child_xpath('span'))
                     if 'class' in element.attrib
                     and element.attrib['class']
-                    in {'ocr_header', 'ocr_line', 'ocr_textfloat'}
+                    in {'ocr_header', 'ocr_line', 'ocr_textfloat', 'ocr_caption'}
                 ):
                     found_lines = True
                     direction = self._get_text_direction(par)
@@ -290,50 +306,69 @@ class HocrTransform:
         """
         if line is None:
             return
-        line_box = self.element_coordinates(line)
-        if not line_box:
+        # line_min_aabb (which is created from the "bbox" hOCR property) is so named
+        # because a Rectangle instance is always an AABB (it has no orientation).
+        # However, this means that for non-zero values of the "textangle" hOCR
+        # property, line_min_aabb is not the true bounding box of the hOCR line,
+        # but rather the minimum AABB that encloses the bounding box of the line.
+        # The true bounding box of the line must be seen as an OBB, due to the
+        # existance of the "textangle" hOCR property.
+        line_min_aabb = self.element_coordinates(line)
+        if not line_min_aabb:
             return
-        if line_box.ury <= line_box.lly:
+        if line_min_aabb.ury <= line_min_aabb.lly:
             log.error(
                 "line box is invalid so we cannot render it: box=%s text=%s",
-                line_box,
+                line_min_aabb,
                 self._get_element_text(line),
             )
             return
+        self._debug_draw_line_bbox(canvas, line_min_aabb)
 
-        self._debug_draw_line_bbox(canvas, line_box)
+        # Even though line_min_aabb is not the true bounding box of the line,
+        # it is still possible to derive an AABB (Rectangle) from it that is
+        # the same size as the true bounding box of the line,
+        # if we use a coordinate system that is axis-aligned with respect to
+        # the rotation of the OBB (textangle).
+        # line_size_aabb_matrix is a transform matrix for such a coordinate
+        # system, and line_size_aabb is thus an AABB with the same
+        # size as the true bounding box of the line.
+        top_left_corner = (line_min_aabb.llx, line_min_aabb.lly)
+        line_size_aabb_matrix = (
+            Matrix()
+            .translated(*top_left_corner)
+            # Note: negative sign (textangle is counter-clockwise, see hOCR spec)
+            .rotated(-self.textangle(line))
+        )
+        line_size_aabb = line_size_aabb_matrix.inverse().transform(line_min_aabb)
 
-        # Baseline is a polynomial (usually straight line) that describes the
-        # text baseline relative to the bottom left corner of the line bounding
-        # box.
-        bottom_left_corner = line_box.llx, line_box.ury
         slope, intercept = self.baseline(line)
         if abs(slope) < 0.005:
             slope = 0.0
-        angle = atan(slope)
+        slope_angle = atan(slope)
 
-        # Setup a new coordinate system on the line box's intercept and rotated by
-        # its slope.
-        line_matrix = (
-            Matrix()
-            .translated(*bottom_left_corner)
+        # Final PDF-perspective (bottom-left corner) transform matrix for the
+        # text baseline, which has an intercept and slope relative to the OBB.
+        # See "bbox", "textangle" and "baseline" in the hOCR spec for more details.
+        baseline_matrix = (
+            line_size_aabb_matrix
+            # Translate from hOCR perspective (top-left corner) to PDF perspective
+            # (bottom-left corner).
+            # Note: it would be incorrect to use line_min_aabb.height here because
+            # it is not the true height of the OBB of the line, if textangle != 0.
+            .translated(0, line_size_aabb.height)
             .translated(0, intercept)
-            .rotated(angle / pi * 180)
+            .rotated(slope_angle / pi * 180)
         )
-        log.debug(line_matrix)
-        with canvas.do.save_state(cm=line_matrix):
-            text = Text(direction=text_direction)
 
-            # Don't allow the font to break out of the bounding box. Division by
-            # cos_a accounts for extra clearance between the glyph's vertical axis
-            # on a sloped baseline and the edge of the bounding box.
-            line_box_height = abs(line_box.height) / cos(angle)
-            fontsize = line_box_height + intercept
+        with canvas.do.save_state(cm=baseline_matrix):
+            text = Text(direction=text_direction)
+            fontsize = line_size_aabb.height + intercept
             text.font(self._fontname, fontsize)
             text.render_mode(3 if invisible_text else 0)
 
             self._debug_draw_baseline(
-                canvas, line_matrix.inverse().transform(line_box), 0
+                canvas, baseline_matrix.inverse().transform(line_min_aabb), 0
             )
 
             canvas.do.fill_color(BLACK)  # text in black
@@ -341,7 +376,7 @@ class HocrTransform:
             for elem, next_elem in pairwise(elements + [None]):
                 self._do_line_word(
                     canvas,
-                    line_matrix,
+                    baseline_matrix,
                     text,
                     fontsize,
                     elem,
@@ -357,7 +392,7 @@ class HocrTransform:
         line_matrix: Matrix,
         text: Text,
         fontsize: float,
-        elem: Element,
+        elem: Element | None,
         next_elem: Element | None,
         text_direction: TextDirection,
         inject_word_breaks: bool,
@@ -396,7 +431,7 @@ class HocrTransform:
         )
         if hocr_next_box is None:
             return
-        # Render a space this word and the next word. The explicit space helps
+        # Render a space between this word and the next word. The explicit space helps
         # PDF viewers identify the word break, and horizontally scaling it to
         # occupy the space the between the words helps the PDF viewer
         # avoid combiningthewordstogether.
@@ -409,7 +444,7 @@ class HocrTransform:
             space_box = Rectangle(next_box.urx, box.lly, box.llx, next_box.ury)
         self._debug_draw_space_bbox(canvas, space_box)
         space_width = self._font.text_width(' ', fontsize)
-        if space_width > 0:
+        if space_width > 0 and space_box.width > 0:
             if text_direction == TextDirection.LTR:
                 text.text_transform(Matrix(1, 0, 0, -1, space_box.llx, 0))
             elif text_direction == TextDirection.RTL:
@@ -434,7 +469,7 @@ class HocrTransform:
                 if ocr_par is None:
                     continue
                 canvas.do.rect(
-                    ocr_par.llx, ocr_par.lly, ocr_par.width, ocr_par.height, fill=0
+                    ocr_par.llx, ocr_par.lly, ocr_par.width, ocr_par.height, fill=False
                 )
 
     def _debug_draw_line_bbox(self, canvas: Canvas, line_box: Rectangle, color=BLUE):
@@ -443,7 +478,7 @@ class HocrTransform:
             return
         with canvas.do.save_state():
             canvas.do.stroke_color(color).line_width(0.15).rect(
-                line_box.llx, line_box.lly, line_box.width, line_box.height, fill=0
+                line_box.llx, line_box.lly, line_box.width, line_box.height, fill=False
             )
 
     def _debug_draw_word_triangle(
@@ -467,7 +502,7 @@ class HocrTransform:
             return
         with canvas.do.save_state():
             canvas.do.stroke_color(color).line_width(line_width).rect(
-                box.llx, box.lly, box.width, box.height, fill=0
+                box.llx, box.lly, box.width, box.height, fill=False
             )
 
     def _debug_draw_space_bbox(
@@ -478,7 +513,7 @@ class HocrTransform:
             return
         with canvas.do.save_state():
             canvas.do.fill_color(color).line_width(line_width).rect(
-                box.llx, box.lly, box.width, box.height, fill=1
+                box.llx, box.lly, box.width, box.height, fill=True
             )
 
     def _debug_draw_baseline(

src/ocrmypdf/optimize.py

@@ -75,10 +75,15 @@ def extract_image_filter(
     """Determine if an image is extractable."""
     if image.Subtype != Name.Image:
         return None
-    if image.Length < 100:
+    if not isinstance(image.Length, int) or image.Length < 100:
         log.debug(f"xref {xref}: skipping image with small stream size")
         return None
-    if image.Width < 8 or image.Height < 8:  # Issue 732
+    if (
+        not isinstance(image.Width, int)
+        or not isinstance(image.Height, int)
+        or image.Width < 8
+        or image.Height < 8
+    ):  # Issue 732
         log.debug(f"xref {xref}: skipping image with unusually small dimensions")
         return None
 
@@ -154,7 +159,10 @@ def extract_image_jbig2(
                 imgname = root / f'{xref:08d}'
                 with imgname.open('wb') as f:
                     ext = pim.extract_to(stream=f)
-                imgname.rename(imgname.with_suffix(ext))
+                # Rename the file so it has .prejbig2.ext extension
+                # Making it unique avoids problems with Windows if the
+                # same image is extracted multiple times
+                imgname.rename(imgname.with_suffix(".prejbig2" + ext))
             except NotImplementedError as e:
                 if '/Decode' in str(e):
                     log.debug(
@@ -170,7 +178,7 @@ def extract_image_jbig2(
                     pim.obj.ColorSpace = colorspace
                 else:
                     del pim.obj.ColorSpace
-            return XrefExt(xref, ext)
+            return XrefExt(xref, ".prejbig2" + ext)
     return None
 
 

src/ocrmypdf/pdfinfo/info.py

@@ -12,7 +12,7 @@ import re
 import statistics
 from collections import defaultdict
 from collections.abc import Callable, Container, Iterable, Iterator, Mapping, Sequence
-from contextlib import contextmanager
+from contextlib import contextmanager, nullcontext
 from decimal import Decimal
 from enum import Enum, auto
 from functools import partial
@@ -24,6 +24,7 @@ from warnings import warn
 
 from pdfminer.layout import LTPage, LTTextBox
 from pikepdf import (
+    Dictionary,
     Matrix,
     Name,
     Object,
@@ -40,7 +41,12 @@ from ocrmypdf._concurrent import Executor, SerialExecutor
 from ocrmypdf._progressbar import ProgressBar
 from ocrmypdf.exceptions import EncryptedPdfError, InputFileError
 from ocrmypdf.helpers import Resolution, available_cpu_count, pikepdf_enable_mmap
-from ocrmypdf.pdfinfo.layout import LTStateAwareChar, get_page_analysis, get_text_boxes
+from ocrmypdf.pdfinfo.layout import (
+    LTStateAwareChar,
+    PdfMinerState,
+    get_page_analysis,
+    get_text_boxes,
+)
 
 logger = logging.getLogger()
 
@@ -369,18 +375,26 @@ class ImageInfo:
             pim = PdfImage(pdfimage)
         else:
             raise ValueError("Either pdfimage or inline must be set")
-        if pim.obj.get(Name.SMask, None) is not None:
+
+        self._width = pim.width
+        self._height = pim.height
+        if (smask := pim.obj.get(Name.SMask, None)) is not None:
             # SMask is pretty much an alpha channel, but in PDF it's possible
             # for channel to have different dimensions than the image
             # itself. Some PDF writers use this to create a grayscale stencil
             # mask. For our purposes, the effective size is the size of the
             # larger component (image or smask).
-            smask = pim.obj[Name.SMask]
-            self._width = max(smask.get(Name.Width, 0), pim.width)
-            self._height = max(smask.get(Name.Height, 0), pim.height)
-        else:
-            self._width = pim.width
-            self._height = pim.height
+            if isinstance(smask, Stream | Dictionary):
+                self._width = max(smask.get(Name.Width, 0), self._width)
+                self._height = max(smask.get(Name.Height, 0), self._height)
+        if (mask := pim.obj.get(Name.Mask, None)) is not None:
+            # If the image has a /Mask entry, it has an explicit mask.
+            # /Mask can be a Stream or an Array. If it's a Stream,
+            # use its /Width and /Height if they are larger than the main
+            # image's.
+            if isinstance(mask, Stream | Dictionary):
+                self._width = max(mask.get(Name.Width, 0), self._width)
+                self._height = max(mask.get(Name.Height, 0), self._height)
 
         # If /ImageMask is true, then this image is a stencil mask
         # (Images that draw with this stencil mask will have a reference to
@@ -484,9 +498,18 @@ class ImageInfo:
     def renderable(self) -> bool:
         """Whether the image is renderable.
 
-        Some PDFs in the wild have invalid images that are not renderable.
+        Some PDFs in the wild have invalid images that are not renderable,
+        due to unusual dimensions.
+
+        Stencil masks are not also not renderable, since they are not
+        drawn, but rather they control how rendering happens.
         """
-        return self.dpi.is_finite and self.width >= 0 and self.height >= 0
+        return (
+            self.dpi.is_finite
+            and self.width >= 0
+            and self.height >= 0
+            and self.type_ != 'stencil'
+        )
 
     @property
     def dpi(self) -> Resolution:
@@ -684,13 +707,13 @@ def _page_has_text(text_blocks: Iterable[FloatRect], page_width, page_height) ->
 
 
 def simplify_textboxes(
-    miner: LTPage, textbox_getter: Callable[[LTPage], Iterator[LTTextBox]]
+    miner_page: LTPage, textbox_getter: Callable[[LTPage], Iterator[LTTextBox]]
 ) -> Iterator[TextboxInfo]:
     """Extract only limited content from text boxes.
 
     We do this to save memory and ensure that our objects are pickleable.
     """
-    for box in textbox_getter(miner):
+    for box in textbox_getter(miner_page):
         first_line = box._objs[0]  # pylint: disable=protected-access
         first_char = first_line._objs[0]  # pylint: disable=protected-access
         if not isinstance(first_char, LTStateAwareChar):
@@ -737,9 +760,12 @@ def _pdf_pageinfo_sync(
     infile: Path,
     check_pages: Container[int],
     detailed_analysis: bool,
+    miner_state: PdfMinerState | None,
 ) -> PageInfo:
     with _pdf_pageinfo_sync_pdf(thread_pdf, infile) as pdf:
-        return PageInfo(pdf, pageno, infile, check_pages, detailed_analysis)
+        return PageInfo(
+            pdf, pageno, infile, check_pages, detailed_analysis, miner_state
+        )
 
 
 def _pdf_pageinfo_concurrent(
@@ -751,6 +777,7 @@ def _pdf_pageinfo_concurrent(
     progbar,
     check_pages,
     detailed_analysis: bool = False,
+    miner_state: PdfMinerState | None = None,
 ) -> Sequence[PageInfo | None]:
     pages: list[PageInfo | None] = [None] * len(pdf.pages)
 
@@ -782,7 +809,8 @@ def _pdf_pageinfo_concurrent(
     initial_pdf = pdf if use_threads else None
 
     contexts = (
-        (n, initial_pdf, infile, check_pages, detailed_analysis) for n in range(total)
+        (n, initial_pdf, infile, check_pages, detailed_analysis, miner_state)
+        for n in range(total)
     )
     assert n_workers == 1 if use_threads else n_workers >= 1, "Not multithreadable"
     logger.debug(
@@ -849,12 +877,15 @@ class PageInfo:
         infile: PathLike,
         check_pages: Container[int],
         detailed_analysis: bool = False,
+        miner_state: PdfMinerState | None = None,
     ):
         """Initialize a PageInfo object."""
         self._pageno = pageno
         self._infile = infile
         self._detailed_analysis = detailed_analysis
-        self._gather_pageinfo(pdf, pageno, infile, check_pages, detailed_analysis)
+        self._gather_pageinfo(
+            pdf, pageno, infile, check_pages, detailed_analysis, miner_state
+        )
 
     def _gather_pageinfo(
         self,
@@ -863,6 +894,7 @@ class PageInfo:
         infile: PathLike,
         check_pages: Container[int],
         detailed_analysis: bool,
+        miner_state: PdfMinerState | None,
     ):
         page: Page = pdf.pages[pageno]
         mediabox = [Decimal(d) for d in page.mediabox.as_list()]
@@ -878,10 +910,11 @@ class PageInfo:
         check_this_page = pageno in check_pages
 
         if check_this_page and detailed_analysis:
-            pscript5_mode = str(pdf.docinfo.get(Name.Creator)).startswith('PScript5')
-            miner = get_page_analysis(infile, pageno, pscript5_mode)
-            if miner is not None:
-                self._textboxes = list(simplify_textboxes(miner, get_text_boxes))
+            page_analysis = miner_state.get_page_analysis(pageno)
+            if page_analysis is not None:
+                self._textboxes = list(
+                    simplify_textboxes(page_analysis, get_text_boxes)
+                )
             else:
                 self._textboxes = []
             bboxes = (box.bbox for box in self._textboxes)
@@ -1065,10 +1098,14 @@ class PageInfo:
 
         Returns None if there is no meaningful DPI for the page.
         """
-        image_dpis = [
-            image.dpi.to_scalar() for image in self._images if image.renderable
-        ]
-        image_areas = [image.printed_area for image in self._images if image.renderable]
+        image_dpis = []
+        image_areas = []
+        for image in self._images:
+            if not image.renderable:
+                continue
+            image_dpis.append(image.dpi.to_scalar())
+            image_areas.append(image.printed_area)
+
         total_drawn_area = sum(image_areas)
         if total_drawn_area == 0:
             return None
@@ -1130,16 +1167,26 @@ class PdfInfo:
         with Pdf.open(infile) as pdf:
             if pdf.is_encrypted:
                 raise EncryptedPdfError()  # Triggered by encryption with empty passwd
-            self._pages = _pdf_pageinfo_concurrent(
-                pdf,
-                executor,
-                max_workers,
-                use_threads,
-                infile,
-                progbar,
-                check_pages=check_pages,
-                detailed_analysis=detailed_analysis,
+            pscript5_mode = str(pdf.docinfo.get(Name.Creator, "")).startswith(
+                'PScript5'
             )
+            self._miner_state = (
+                PdfMinerState(infile, pscript5_mode)
+                if detailed_analysis
+                else nullcontext()
+            )
+            with self._miner_state as miner_state:
+                self._pages = _pdf_pageinfo_concurrent(
+                    pdf,
+                    executor,
+                    max_workers,
+                    use_threads,
+                    infile,
+                    progbar,
+                    check_pages=check_pages,
+                    detailed_analysis=detailed_analysis,
+                    miner_state=miner_state,
+                )
             self._needs_rendering = pdf.Root.get(Name.NeedsRendering, False)
             if Name.AcroForm in pdf.Root:
                 if len(pdf.Root.AcroForm.get(Name.Fields, [])) > 0:

src/ocrmypdf/pdfinfo/layout.py

@@ -17,6 +17,8 @@ import pdfminer
 import pdfminer.encodingdb
 import pdfminer.pdfdevice
 import pdfminer.pdfinterp
+import pdfminer.psparser
+from deprecation import deprecated
 from pdfminer.converter import PDFLayoutAnalyzer
 from pdfminer.layout import LAParams, LTChar, LTPage, LTTextBox
 from pdfminer.pdfcolor import PDFColorSpace
@@ -58,9 +60,10 @@ def pdfsimplefont__init__(
 
 setattr(PDFSimpleFont, '__init__', pdfsimplefont__init__)
 
-#
-# pdfminer patches when creator is PScript5.dll
-#
+# Patch pdfminer.six buffer size
+# The parser doesn't properly handle keyword tokens are split across the end of the
+# buffer, so increase the buffer size something far larger than will ever be seen.
+pdfminer.psparser.PSBaseParser.BUFSIZ = 256 * 1024 * 1024
 
 
 def pdftype3font__pscript5_get_height(self):
@@ -287,6 +290,7 @@ def patch_pdfminer(pscript5_mode: bool):
         yield
 
 
+@deprecated(deprecated_in='16.6.0', details='Use PdfMinerState instead.')
 def get_page_analysis(
     infile: PathLike, pageno: int, pscript5_mode: bool
 ) -> LTPage | None:
@@ -317,6 +321,68 @@ def get_page_analysis(
     return dev.get_result()
 
 
+class PdfMinerState:
+    """Provide a context manager for using pdfminer.six.
+
+    This ensures that the file is closed. It also provides a cache of pages
+    from the PDF so that they can be reused if needed, to improve performance.
+    """
+
+    def __init__(self, infile: Path, pscript5_mode: bool) -> None:
+        """Initialize the context manager.
+
+        Args:
+            infile: The path to the PDF file to be analyzed.
+            pscript5_mode: Whether the PDF was generated by PScript5.dll.
+        """
+        self.infile = infile
+        self.rman = pdfminer.pdfinterp.PDFResourceManager(caching=True)
+        self.disable_boxes_flow = None
+        self.page_iter = None
+        self.page_cache: list[PDFPage] = []
+        self.pscript5_mode = pscript5_mode
+        self.file = None
+
+    def __enter__(self):
+        """Enter the context manager."""
+        self.file = Path(self.infile).open('rb')
+        self.page_iter = PDFPage.get_pages(self.file)
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        """Exit the context manager."""
+        if self.file:
+            self.file.close()
+        return True
+
+    def get_page_analysis(self, pageno: int):
+        """Get the page analysis for a given page."""
+        while len(self.page_cache) <= pageno:
+            try:
+                self.page_cache.append(next(self.page_iter))
+            except StopIteration:
+                raise InputFileError(
+                    f"pdfminer did not find page {pageno} in the input file."
+                )
+        page = self.page_cache[pageno]
+        if not page:
+            raise InputFileError(
+                f"pdfminer could not process page {pageno} (counting from 0)."
+            )
+        dev = TextPositionTracker(
+            self.rman,
+            laparams=LAParams(
+                all_texts=True, detect_vertical=True, boxes_flow=self.disable_boxes_flow
+            ),
+        )
+        interp = pdfminer.pdfinterp.PDFPageInterpreter(self.rman, dev)
+
+        with patch_pdfminer(self.pscript5_mode):
+            interp.process_page(page)
+
+        return dev.get_result()
+
+
 def get_text_boxes(obj) -> Iterator[LTTextBox]:
     """Get the text boxes attached to the current node."""
     for child in obj:

src/ocrmypdf/pluginspec.py

@@ -145,13 +145,15 @@ def get_progressbar_class() -> type[ProgressBar]:
     The class returned by this function must be compatible with the
     :class:`ProgressBar` protocol.
 
-    Here is how OCRmyPDF will use the progress bar:
-
     Example:
-        pbar_class = pm.hook.get_progressbar_class()
-        with pbar_class(**progress_kwargs) as pbar:
-            ...
-            pbar.update(1)
+        Here is how OCRmyPDF will use the progress bar:
+
+        .. code-block:: python
+
+            pbar_class = pm.hook.get_progressbar_class()
+            with pbar_class(**progress_kwargs) as pbar:
+                ... # do some work
+                pbar.update(1)
     """
 
 

src/ocrmypdf/subprocess/__init__.py

@@ -215,8 +215,10 @@ to have {found_version}.  Please update this program.
 
 OLD_VERSION_REQUIRED_FOR = '''
 OCRmyPDF requires '{program}' {need_version} or higher when run with the
-{required_for} arguments.  If you omit these arguments, OCRmyPDF may be able to
-proceed.  For best results, install the program.
+{required_for} arguments.  {program} {found_version} is installed.
+
+If you omit these arguments, OCRmyPDF may be able to
+proceed.  For best results, update the program.
 '''
 
 OSX_INSTALL_ADVICE = '''

tests/conftest.py

@@ -24,11 +24,6 @@ def is_macos():
     return platform.system() == 'Darwin'
 
 
-def running_in_docker():
-    # Docker creates a file named /.dockerenv in all supported versions
-    return Path('/.dockerenv').exists()
-
-
 def have_unpaper():
     try:
         unpaper.version()

tests/test_annots.py

@@ -0,0 +1,31 @@
+# SPDX-FileCopyrightText: 2024 James R. Barlow
+# SPDX-License-Identifier: MPL-2.0
+
+from __future__ import annotations
+
+import pytest
+from pikepdf import Array, Dictionary, Name, NameTree, Pdf
+
+from ocrmypdf._annots import remove_broken_goto_annotations
+
+
+def test_remove_broken_goto_annotations(resources):
+    with Pdf.open(resources / 'link.pdf') as pdf:
+        assert not remove_broken_goto_annotations(pdf), "File should not be modified"
+
+        # Construct Dests nametree
+        nt = NameTree.new(pdf)
+        names = pdf.Root[Name.Names] = pdf.make_indirect(Dictionary())
+        names[Name.Dests] = nt.obj
+        # Create a broken named destination
+        nt['Invalid'] = pdf.make_indirect(Dictionary())
+        # Create a valid named destination
+        nt['Valid'] = Array([pdf.pages[0].obj, Name.XYZ, 0, 0, 0])
+
+        pdf.pages[0].Annots[0].A.D = 'Missing'
+        pdf.pages[1].Annots[0].A.D = 'Valid'
+
+        assert remove_broken_goto_annotations(pdf), "File should be modified"
+
+        assert Name.D not in pdf.pages[0].Annots[0].A
+        assert Name.D in pdf.pages[1].Annots[0].A

tests/test_api.py

@@ -3,6 +3,7 @@
 
 from __future__ import annotations
 
+import pickle
 from io import BytesIO
 from pathlib import Path
 
@@ -10,6 +11,7 @@ import pytest
 from pdfminer.high_level import extract_text
 
 import ocrmypdf
+import ocrmypdf._pipelines
 import ocrmypdf.api
 
 
@@ -35,7 +37,7 @@ def test_sidecar_stringio(resources: Path, outdir: Path, outpdf: Path):
         resources / 'ccitt.pdf',
         outpdf,
         plugins=['tests/plugins/tesseract_cache.py'],
-        sidecar=s
+        sidecar=s,
     )
     s.seek(0)
     assert b'the' in s.getvalue()
@@ -75,3 +77,29 @@ def test_hocr_to_pdf_api(resources: Path, outdir: Path, outpdf: Path):
     text = extract_text(outpdf)
     assert 'hocr' in text and 'the' not in text
 
+
+def test_hocr_result_json():
+    result = ocrmypdf._pipelines._common.HOCRResult(
+        pageno=1,
+        pdf_page_from_image=Path('a'),
+        hocr=Path('b'),
+        textpdf=Path('c'),
+        orientation_correction=180,
+    )
+    assert (
+        result.to_json()
+        == '{"pageno": 1, "pdf_page_from_image": {"Path": "a"}, "hocr": {"Path": "b"}, '
+        '"textpdf": {"Path": "c"}, "orientation_correction": 180}'
+    )
+    assert ocrmypdf._pipelines._common.HOCRResult.from_json(result.to_json()) == result
+
+
+def test_hocr_result_pickle():
+    result = ocrmypdf._pipelines._common.HOCRResult(
+        pageno=1,
+        pdf_page_from_image=Path('a'),
+        hocr=Path('b'),
+        textpdf=Path('c'),
+        orientation_correction=180,
+    )
+    assert result == pickle.loads(pickle.dumps(result))

tests/test_completion.py

@@ -8,7 +8,7 @@ from subprocess import run
 
 import pytest
 
-from .conftest import running_in_docker
+from ocrmypdf.helpers import running_in_docker
 
 pytestmark = pytest.mark.skipif(
     running_in_docker(),

tests/test_ghostscript.py

@@ -11,10 +11,12 @@ from unittest.mock import patch
 
 import pikepdf
 import pytest
+from packaging.version import Version
 from PIL import Image, UnidentifiedImageError
 
+from ocrmypdf._exec import ghostscript
 from ocrmypdf._exec.ghostscript import DuplicateFilter, rasterize_pdf
-from ocrmypdf.exceptions import ColorConversionNeededError, ExitCode
+from ocrmypdf.exceptions import ColorConversionNeededError, ExitCode, InputFileError
 from ocrmypdf.helpers import Resolution
 
 from .conftest import check_ocrmypdf, run_ocrmypdf_api
@@ -139,7 +141,7 @@ def test_ghostscript_mandatory_color_conversion(resources, outpdf):
 
 def test_rasterize_pdf_errors(resources, no_outpdf, caplog):
     with patch('ocrmypdf._exec.ghostscript.run') as mock:
-        # ghostscript can produce
+        # ghostscript can produce empty files with return code 0
         mock.return_value = subprocess.CompletedProcess(
             ['fakegs'], returncode=0, stdout=b'', stderr=b'error this is an error'
         )
@@ -208,3 +210,71 @@ class TestDuplicateFilter:
         assert caplog.records[1].msg == "another error message"
         assert caplog.records[2].msg == "(suppressed 5 repeated lines)"
         assert caplog.records[3].msg == "yet another error message"
+
+
+@pytest.fixture
+def pdf_with_invalid_image(outdir):
+    # issue 1451
+    Name = pikepdf.Name
+    pdf = pikepdf.new()
+    pdf.add_blank_page()
+    pdf.pages[0].Contents = pdf.make_stream(b'612 0 0 612 0 0 cm /Image Do')
+    # Create an invalid image object that has both ColorSpace and ImageMask set
+    pdf.pages[0].Resources = pikepdf.Dictionary(
+        XObject=pdf.make_indirect(
+            pikepdf.Dictionary(
+                Image=pdf.make_stream(
+                    b"\xf0\x0f" * 8,
+                    ColorSpace=Name.DeviceGray,
+                    BitsPerComponent=1,
+                    Width=8,
+                    Height=8,
+                    ImageMask=True,
+                    Subtype=Name.Image,
+                    Type=Name.XObject,
+                )
+            )
+        )
+    )
+    pdf.save(outdir / 'invalid_image.pdf')
+    pdf.save('invalid_image.pdf')
+    return outdir / 'invalid_image.pdf'
+
+
+@pytest.mark.xfail(
+    ghostscript.version() < Version('10.04.0'),
+    reason="Older Ghostscript behavior is different",
+)
+def test_recoverable_image_error(pdf_with_invalid_image, outdir, caplog):
+    # When stop_on_error is False, we expect Ghostscript to print an error
+    # but continue
+    rasterize_pdf(
+        outdir / 'invalid_image.pdf',
+        outdir / 'out.png',
+        raster_device='pngmono',
+        raster_dpi=Resolution(10, 10),
+        stop_on_error=False,
+    )
+    assert 'Image has both ImageMask and ColorSpace' in caplog.text
+
+
+@pytest.mark.xfail(
+    ghostscript.version() < Version('10.04.0'),
+    reason="Older Ghostscript behavior is different",
+)
+def test_recoverable_image_error_with_stop(pdf_with_invalid_image, outdir, caplog):
+    # When stop_on_error is True, Ghostscript will print an error and exit
+    # but still produce a viable image. We intercept this case and raise
+    # InputFileError because it will contain an image of the whole page minus
+    # the image we are rendering.
+    with pytest.raises(
+        InputFileError, match="Try using --continue-on-soft-render-error"
+    ):
+        rasterize_pdf(
+            outdir / 'invalid_image.pdf',
+            outdir / 'out.png',
+            raster_device='pngmono',
+            raster_dpi=Resolution(100, 100),
+            stop_on_error=True,
+        )
+    # out2.png will not be created; if it were it would be blank.

tests/test_graft.py

@@ -40,3 +40,72 @@ def test_links(resources, outpdf):
         p2 = pdf.pages[1]
         assert p1.Annots[0].A.D[0].objgen == p2.objgen
         assert p2.Annots[0].A.D[0].objgen == p1.objgen
+
+
+def test_strip_invisble_text():
+    pdf = pikepdf.Pdf.new()
+    print(pikepdf.parse_content_stream(pikepdf.Stream(pdf, b'3 Tr')))
+    page = pdf.add_blank_page()
+    visible_text = [
+        pikepdf.ContentStreamInstruction((), pikepdf.Operator('BT')),
+        pikepdf.ContentStreamInstruction(
+            (pikepdf.Name('/F0'), 12), pikepdf.Operator('Tf')
+        ),
+        pikepdf.ContentStreamInstruction((288, 720), pikepdf.Operator('Td')),
+        pikepdf.ContentStreamInstruction(
+            (pikepdf.String('visible'),), pikepdf.Operator('Tj')
+        ),
+        pikepdf.ContentStreamInstruction((), pikepdf.Operator('ET')),
+    ]
+    invisible_text = [
+        pikepdf.ContentStreamInstruction((), pikepdf.Operator('BT')),
+        pikepdf.ContentStreamInstruction(
+            (pikepdf.Name('/F0'), 12), pikepdf.Operator('Tf')
+        ),
+        pikepdf.ContentStreamInstruction((288, 720), pikepdf.Operator('Td')),
+        pikepdf.ContentStreamInstruction(
+            (pikepdf.String('invisible'),), pikepdf.Operator('Tj')
+        ),
+        pikepdf.ContentStreamInstruction((), pikepdf.Operator('ET')),
+    ]
+    invisible_text_setting_tr = [
+        pikepdf.ContentStreamInstruction((), pikepdf.Operator('BT')),
+        pikepdf.ContentStreamInstruction([3], pikepdf.Operator('Tr')),
+        pikepdf.ContentStreamInstruction(
+            (pikepdf.Name('/F0'), 12), pikepdf.Operator('Tf')
+        ),
+        pikepdf.ContentStreamInstruction((288, 720), pikepdf.Operator('Td')),
+        pikepdf.ContentStreamInstruction(
+            (pikepdf.String('invisible'),), pikepdf.Operator('Tj')
+        ),
+        pikepdf.ContentStreamInstruction((), pikepdf.Operator('ET')),
+    ]
+    stream = [
+        pikepdf.ContentStreamInstruction([], pikepdf.Operator('q')),
+        pikepdf.ContentStreamInstruction([3], pikepdf.Operator('Tr')),
+        *invisible_text,
+        pikepdf.ContentStreamInstruction([], pikepdf.Operator('Q')),
+        *visible_text,
+        *invisible_text_setting_tr,
+        *invisible_text,
+    ]
+    content_stream = pikepdf.unparse_content_stream(stream)
+    page.Contents = pikepdf.Stream(pdf, content_stream)
+
+    def count(string, page):
+        return len(
+            [
+                True
+                for operands, operator in pikepdf.parse_content_stream(page)
+                if operator == pikepdf.Operator('Tj')
+                and operands[0] == pikepdf.String(string)
+            ]
+        )
+
+    nr_visible_pre = count('visible', page)
+    ocrmypdf._graft.strip_invisible_text(pdf, page)
+    nr_visible_post = count('visible', page)
+    assert (
+        nr_visible_pre == nr_visible_post
+    ), 'Number of visible text elements did not change'
+    assert count('invisible', page) == 0, 'No invisible elems left'

tests/test_helpers.py

@@ -13,8 +13,7 @@ import pytest
 from packaging.version import Version
 
 from ocrmypdf import helpers
-
-from .conftest import running_in_docker
+from ocrmypdf.helpers import running_in_docker
 
 needs_symlink = pytest.mark.skipif(os.name == 'nt', reason='needs posix symlink')
 windows_only = pytest.mark.skipif(os.name != 'nt', reason="Windows test")

tests/test_hocrtransform.py

@@ -84,13 +84,19 @@ def test_hocrtransform_matches_sandwich(resources, outdir):
     def clean(s):
         s = re.sub(r'\s+', ' ', s)
         words = s.split(' ')
-        return '\n'.join(sorted(words))
+        return set(words)
 
-    hocr_txt = clean(text_from_pdf(outdir / 'hocr.pdf'))
-    tess_txt = clean(text_from_pdf(outdir / 'tess.pdf'))
+    hocr_words = clean(text_from_pdf(outdir / 'hocr.pdf'))
+    tess_words = clean(text_from_pdf(outdir / 'tess.pdf'))
+
+    similarity = len(hocr_words & tess_words) / len(hocr_words | tess_words)
 
     # from pathlib import Path
-    # Path('hocr.txt').write_text(hocr_txt)
-    # Path('tess.txt').write_text(tess_txt)
 
-    assert hocr_txt == tess_txt
+    # Path('hocr.txt').write_text(sorted('\n'.join(hocr_words)))
+    # Path('tess.txt').write_text(sorted('\n'.join(tess_words)))
+    # Path('mismatch.txt').write_text(
+    #     '\n'.join(sorted(hocr_words ^ tess_words)), encoding='utf8'
+    # )
+
+    assert similarity > 0.99

tests/test_main.py

@@ -18,6 +18,7 @@ from PIL import Image
 import ocrmypdf
 from ocrmypdf._exec import tesseract
 from ocrmypdf.exceptions import ExitCode, MissingDependencyError
+from ocrmypdf.helpers import running_in_docker
 from ocrmypdf.pdfa import file_claims_pdfa
 from ocrmypdf.pdfinfo import Colorspace, Encoding, PdfInfo
 from ocrmypdf.subprocess import get_version
@@ -29,7 +30,6 @@ from .conftest import (
     is_macos,
     run_ocrmypdf,
     run_ocrmypdf_api,
-    running_in_docker,
 )
 
 # pylint: disable=redefined-outer-name

tests/test_pdfinfo.py

@@ -208,12 +208,13 @@ def test_pages_issue700(monkeypatch, resources):
     monkeypatch.setattr(PDFPage, 'get_pages', get_no_pages)
 
     with pytest.raises(InputFileError, match="pdfminer"):
-        pdfinfo.PdfInfo(
+        pi = pdfinfo.PdfInfo(
             resources / 'cardinal.pdf',
             detailed_analysis=True,
             progbar=False,
             max_workers=1,
         )
+        pi._miner_state.get_page_analysis(0)
 
 
 @pytest.fixture

tests/test_rotation.py

@@ -239,7 +239,9 @@ def make_rotate_test(imagefile, outdir, prefix, image_angle, page_angle):
 @pytest.mark.parametrize('image_angle', (0, 90, 180, 270))
 def test_rotate_page_level(image_angle, page_angle, resources, outdir, caplog):
     reference = make_rotate_test(resources / 'typewriter.png', outdir, 'ref', 0, 0)
-    test = make_rotate_test(resources, outdir, 'test', image_angle, page_angle)
+    test = make_rotate_test(
+        resources / 'typewriter.png', outdir, 'test', image_angle, page_angle
+    )
     out = test.with_suffix('.out.pdf')
 
     exitcode = run_ocrmypdf_api(

tests/test_unpaper.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 
 import logging
 from os import fspath
-from unittest.mock import patch
+from unittest.mock import Mock, patch
 
 import pytest
 from packaging.version import Version
@@ -48,6 +48,19 @@ def test_old_unpaper(resources, no_outpdf):
         mock.assert_called()
 
 
+def test_unpaper_version_chatter(resources, no_outpdf):
+    input_ = fspath(resources / "c02-22.pdf")
+    output = fspath(no_outpdf)
+
+    _parser, options, pm = get_parser_options_plugins(["--clean", input_, output])
+    with patch("ocrmypdf.subprocess.run") as mock:
+        mock.return_value = Mock(stdout='Warning: using insecure memory!\n7.0.0\n')
+
+        with pytest.raises(MissingDependencyError):
+            check_options(options, pm)
+        mock.assert_called()
+
+
 @needs_unpaper
 def test_clean(resources, outpdf):
     check_ocrmypdf(