v16.6.2 release notes

Closes #1425

.docker/Dockerfile

@@ -1,7 +1,7 @@
-# SPDX-FileCopyrightText: 2022 James R. Barlow
+# SPDX-FileCopyrightText: 2024 James R. Barlow
 # SPDX-License-Identifier: MPL-2.0
 
-FROM ubuntu:22.04 as base
+FROM ubuntu:22.04 AS base
 
 ENV LANG=C.UTF-8
 ENV TZ=UTC
@@ -9,19 +9,15 @@ RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selectio
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
   python3 \
-  libqpdf-dev \
+  python-is-python3
-  zlib1g \
-  liblept5
 
-FROM base as builder
+FROM base AS builder
 
 # Note we need leptonica here to build jbig2
 RUN apt-get update && apt-get install -y --no-install-recommends \
   build-essential autoconf automake libtool \
   libleptonica-dev \
   zlib1g-dev \
-  python3-dev \
-  python3-distutils \
   libffi-dev \
   ca-certificates \
   curl \
@@ -29,15 +25,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   libcairo2-dev \
   pkg-config
 
-# Get the latest pip (Ubuntu version doesn't support manylinux2010)
-RUN \
-  curl https://bootstrap.pypa.io/get-pip.py | python3
-
 # Compile and install jbig2
 # Needs libleptonica-dev, zlib1g-dev
 RUN \
   mkdir jbig2 \
-  && curl -L https://github.com/agl/jbig2enc/archive/ea6a40a.tar.gz | \
+  && curl -L https://github.com/agl/jbig2enc/archive/c0141bf.tar.gz | \
   tar xz -C jbig2 --strip-components=1 \
   && cd jbig2 \
   && ./autogen.sh && ./configure && make && make install \
@@ -48,23 +40,24 @@ COPY . /app
 
 WORKDIR /app
 
-RUN pip3 install --no-cache-dir .[test,webservice,watcher]
+RUN curl -LsSf https://astral.sh/uv/0.4.27/install.sh | sh
+
+ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy
+
+# Instead of restarting the shell, use uv directly from its installed location.
+RUN /root/.cargo/bin/uv sync --extra test --extra webservice --extra watcher
 
 FROM base
 
-# For Tesseract 5
+RUN apt-get update && apt-get install -y software-properties-common
-RUN apt-get update && apt-get install -y --no-install-recommends \
+
-  software-properties-common gpg-agent
+RUN add-apt-repository -y ppa:alex-p/tesseract-ocr5
-RUN add-apt-repository -y ppa:alex-p/tesseract-ocr-devel
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
   ghostscript \
   fonts-droid-fallback \
   jbig2dec \
-  img2pdf \
-  libsm6 libxext6 libxrender-dev \
   pngquant \
-  python-is-python3 \
   tesseract-ocr \
   tesseract-ocr-chi-sim \
   tesseract-ocr-deu \
@@ -80,11 +73,13 @@ WORKDIR /app
 COPY --from=builder /usr/local/lib/ /usr/local/lib/
 COPY --from=builder /usr/local/bin/ /usr/local/bin/
 
-COPY --from=builder /app/misc/webservice.py /app/
+COPY --from=builder --chown=app:app /app /app
-COPY --from=builder /app/misc/watcher.py /app/
 
-# Copy minimal project files to get the test suite.
+RUN rm -rf /app/.git && \
-COPY --from=builder /app/pyproject.toml /app/README.md /app/
+ln -s /app/misc/webservice.py /app/webservice.py && \
-COPY --from=builder /app/tests /app/tests
+ln -s /app/misc/watcher.py /app/watcher.py
+
+ENV PATH="/app/.venv/bin:${PATH}"
+
+ENTRYPOINT ["/app/.venv/bin/ocrmypdf"]
 
-ENTRYPOINT ["/usr/local/bin/ocrmypdf"]

.docker/Dockerfile.alpine

@@ -1,7 +1,14 @@
 # SPDX-FileCopyrightText: 2023 James R. Barlow
 # SPDX-License-Identifier: MPL-2.0
 
-FROM alpine:3.19.1 as base
+# Note: Alpine 3.20 builds tesseract with --enable-opencl, which is not
+# supported by anyone. OCRmyPDF is not compatible with Alpine 3.20.0
+# through 3.20.3. The Alpine issue should be fixed in 3.21.0. It is
+# not clear if 3.20.4+ will have the fix.
+# Details
+#  https://gitlab.alpinelinux.org/alpine/aports/-/issues/16143
+#  https://github.com/ocrmypdf/OCRmyPDF/issues/1395
+FROM alpine:3.19 AS base
 
 ENV LANG=C.UTF-8
 ENV TZ=UTC
@@ -10,40 +17,24 @@ RUN apk add --no-cache \
     python3 \
     zlib
 
-FROM base as builder
+FROM base AS builder
 
 RUN apk add --no-cache \
     ca-certificates \
     git \
     python3-dev \
-    py3-pip
+    curl
-
-# On arm64, we need to build cffi from source.
-ARG TARGETPLATFORM
-
-RUN if [ "${TARGETPLATFORM}" == "linux/arm64" ]; then \
-    apk add --no-cache \
-    build-base \
-    autoconf \
-    automake \
-    libtool \
-    zlib-dev \
-    libffi-dev \
-    cairo-dev \
-    pkgconfig \
-    ;  \
-    fi
 
 COPY . /app
 
 WORKDIR /app
 
-RUN python3 -m venv .venv
+RUN curl -LsSf https://astral.sh/uv/0.4.27/install.sh | sh
 
-RUN source .venv/bin/activate \
+ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy
-    && python3 -m pip install --no-cache-dir --upgrade pip \
+
-    && python3 -m pip install --no-cache-dir wheel \
+# Instead of restarting the shell, use uv directly from its installed location.
-    && python3 -m pip install --no-cache-dir .[test,webservice,watcher]
+RUN /root/.cargo/bin/uv sync --extra test --extra webservice --extra watcher
 
 FROM base
 
@@ -66,17 +57,11 @@ RUN apk add --no-cache \
 
 WORKDIR /app
 
-COPY --from=builder /usr/local/lib/ /usr/local/lib/
+COPY --from=builder --chown=app:app /app /app
-COPY --from=builder /usr/local/bin/ /usr/local/bin/
 
-COPY --from=builder /app/.venv/ /app/.venv/
+RUN rm -rf /app/.git && \
-
+    ln -s /app/misc/webservice.py /app/webservice.py && \
-COPY --from=builder /app/misc/webservice.py /app/
+    ln -s /app/misc/watcher.py /app/watcher.py
-COPY --from=builder /app/misc/watcher.py /app/
-
-# Copy minimal project files to get the test suite.
-COPY --from=builder /app/pyproject.toml /app/README.md /app/
-COPY --from=builder /app/tests /app/tests
 
 ENV PATH="/app/.venv/bin:${PATH}"
 

.github/workflows/build.yml

@@ -21,18 +21,13 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
+        os: [ubuntu-22.04, ubuntu-24.04]
+        python: ["3.10", "3.11", "3.12", "3.13"]
         include:
           - os: ubuntu-22.04
+            tesseract_ppa: "ppa"
             python: "3.10"
-          - os: ubuntu-22.04
+          - os: ubuntu-24.04
-            python: "3.11"
-          - os: ubuntu-22.04
-            python: "3.10"
-            tesseract5: true
-          - os: ubuntu-latest
-            python: "3.12"
-            tesseract5: true
-          - os: ubuntu-latest
             python: "pypy3.10"
 
     env:
@@ -44,16 +39,20 @@ jobs:
         with:
           fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags
 
-      - uses: actions/setup-python@v5
+      - name: Install uv
-        name: Setup Python
+        uses: astral-sh/setup-uv@v3
+        with:
+          version: "0.4.27"
+
+      - name: "Set up Python"
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
-          cache: "pip"
 
-      - name: Install Tesseract 5
+      - name: Install Tesseract from PPA
-        if: matrix.tesseract5
+        if: matrix.tesseract_ppa == 'ppa'
         run: |
-          sudo add-apt-repository -y ppa:alex-p/tesseract-ocr-devel
+          sudo add-apt-repository -y ppa:alex-p/tesseract-ocr5.3
 
       - name: Install common packages
         run: |
@@ -85,8 +84,7 @@ jobs:
 
       - name: Install Python packages
         run: |
-          python -m pip install --upgrade pip wheel
+          uv sync --extra test
-          python -m pip install --prefer-binary .[test]
 
       - name: Report versions
         run: |
@@ -94,11 +92,11 @@ jobs:
           gs --version
           pngquant --version
           unpaper --version
-          img2pdf --version
+          uv run img2pdf --version
 
       - name: Test
         run: |
-          python -m pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
+          uv run pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
@@ -108,14 +106,13 @@ jobs:
           files: ./coverage.xml
           env_vars: OS,PYTHON
 
-
   test_macos:
     name: Test macOS
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         os: [macos-latest, macos-13]  # macos-latest is arm64, macos-13 is x86_64
-        python: ["3.10", "3.11", "3.12"]
+        python: ["3.10", "3.11", "3.12", "3.13"]
 
     env:
       OS: ${{ matrix.os }}
@@ -138,27 +135,30 @@ jobs:
             pngquant \
             tesseract
 
-      - uses: actions/setup-python@v5
+      - name: Install uv
-        name: Setup Python
+        uses: astral-sh/setup-uv@v3
+        with:
+          version: "0.4.27"
+
+      - name: "Set up Python"
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
-          cache: "pip"
 
       - name: Install Python packages
         run: |
-          python -m pip install --upgrade pip wheel
+          uv sync --extra test
-          python -m pip install --prefer-binary .[test]
 
       - name: Report versions
         run: |
           tesseract --version
           gs --version
           pngquant --version
-          img2pdf --version
+          uv run img2pdf --version
 
       - name: Test
         run: |
-          python -m pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
+          uv run pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
@@ -174,7 +174,7 @@ jobs:
     strategy:
       matrix:
         os: [windows-latest]
-        python: ["3.10", "3.11", "3.12"]
+        python: ["3.10", "3.11", "3.12", "3.13"]
 
     env:
       OS: ${{ matrix.os }}
@@ -185,11 +185,15 @@ jobs:
         with:
           fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags
 
-      - uses: actions/setup-python@v5
+      - name: Install uv
-        name: Setup Python
+        uses: astral-sh/setup-uv@v3
+        with:
+          version: "0.4.27"
+
+      - name: "Set up Python"
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
-          cache: "pip"
 
       - name: Install system packages
         run: |
@@ -198,12 +202,11 @@ jobs:
 
       - name: Install Python packages
         run: |
-          python -m pip install --upgrade pip wheel
+          uv sync --extra test
-          python -m pip install --prefer-binary .[test]
 
       - name: Test
         run: |
-          python -m pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
+          uv run pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
@@ -221,16 +224,14 @@ jobs:
         with:
           fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags
 
-      - uses: actions/setup-python@v5
+      - name: Install uv
-        name: Setup Python
+        uses: astral-sh/setup-uv@v3
         with:
-          python-version: "3.10"
+          version: "0.4.27"
-          cache: "pip"
 
       - name: Make wheels and sdist
         run: |
-          python -m pip install --upgrade pip wheel build
+          uv build --sdist --wheel
-          python -m build --sdist --wheel
 
       - uses: actions/upload-artifact@v4
         with:
@@ -258,29 +259,45 @@ jobs:
 
   create_release:
     name: Create GitHub release
-    needs: [wheel_sdist_linux, test_linux, test_macos, test_windows]
+    needs: [upload_pypi]
     runs-on: ubuntu-latest
     if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v')
     permissions:
       # Required to create a release
       contents: write
+      id-token: write
     steps:
       - uses: actions/download-artifact@v4
         with:
           name: artifact
           path: dist
 
-      - name: Create Release
+      - name: Sign the dists with Sigstore
-        id: create-release
+        uses: sigstore/gh-action-sigstore-python@v3.0.0
-        uses: shogo82148/actions-create-release@v1
-
-      - name: Upload Assets
-        uses: shogo82148/actions-upload-release-asset@v1
         with:
-          upload_url: ${{ steps.create-release.outputs.upload_url }}
+          inputs: >-
-          asset_path: |
-            ./dist/*.whl
             ./dist/*.tar.gz
+            ./dist/*.whl
+
+      - name: Create GitHub Release
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: >-
+          gh release create
+          '${{ github.ref_name }}'
+          --repo '${{ github.repository }}'
+          --notes ""
+
+      - name: Upload artifact signatures to GitHub Release
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        # Upload to GitHub Release using the `gh` CLI.
+        # `dist/` contains the built packages, and the
+        # sigstore-produced signatures and certificates.
+        run: >-
+          gh release upload
+          '${{ github.ref_name }}' dist/**
+          --repo '${{ github.repository }}'
 
   docker_ubuntu:
     name: Build Ubuntu-based Docker image

.gitignore

@@ -44,3 +44,4 @@ docs/_build/
 docs/_static/
 docs/_templates/
 docs/Makefile
+src/ocrmypdf/_version.py

docs/advanced.rst

@@ -228,6 +228,59 @@ then run ocrmypdf as follows (along with any other desired arguments):
    Some combinations of control parameters will break Tesseract or break
    assumptions that OCRmyPDF makes about Tesseract's output.
 
+Changing page segmentation mode
+-------------------------------
+
+The directive ``--tesseract-pagesegmode Nmode`` forwards the desired page segmentation
+mode to Tesseract OCR. The default is 3.
+
+Page segmentation can improve OCR results when you know that a PDF ought to be
+analyzed a particular way, such as PDFs whose pages contain only a single line of 
+text. For the vast majority of users, changing the page segmentation mode will only
+make things worse.
+
+As of June 2024, the Tesseract page segmentation modes are:
+
++-----+----------------------------------------------------------------------------------+
+| ID  | Description                                                                      |
++=====+==================================================================================+
+|  0  | Orientation and script detection (OSD) only.                                      |
++-----+----------------------------------------------------------------------------------+
+|  1  | Automatic page segmentation with OSD.                                             |
++-----+----------------------------------------------------------------------------------+
+|  2  | Automatic page segmentation, but no OSD, or OCR. (not implemented)               |
++-----+----------------------------------------------------------------------------------+
+|  3  | Fully automatic page segmentation, but no OSD. (Default)                         |
++-----+----------------------------------------------------------------------------------+
+|  4  | Assume a single column of text of variable sizes.                                 |
++-----+----------------------------------------------------------------------------------+
+|  5  | Assume a single uniform block of vertically aligned text.                         |
++-----+----------------------------------------------------------------------------------+
+|  6  | Assume a single uniform block of text.                                            |
++-----+----------------------------------------------------------------------------------+
+|  7  | Treat the image as a single text line.                                            |
++-----+----------------------------------------------------------------------------------+
+|  8  | Treat the image as a single word.                                                 |
++-----+----------------------------------------------------------------------------------+
+|  9  | Treat the image as a single word in a circle.                                     |
++-----+----------------------------------------------------------------------------------+
+| 10  | Treat the image as a single character.                                            |
++-----+----------------------------------------------------------------------------------+
+| 11  | Sparse text. Find as much text as possible in no particular order.                |
++-----+----------------------------------------------------------------------------------+
+| 12  | Sparse text with OSD.                                                            |
++-----+----------------------------------------------------------------------------------+
+| 13  | Raw line. Treat the image as a single text line, bypassing hacks that are        |
+|     | Tesseract-specific.                                                               |
++-----+----------------------------------------------------------------------------------+
+
+Modes 0, 1, 2, and 12 (all of those that enable orientation and script detection) 
+are not compatible with OCRmyPDF, which performs OSD in a separate step from OCR.
+Their use may interfere with ``--rotate-pages`` and other features.
+
+It is currently not possible to use advanced Tesseract OCR features, such as creating
+OCR information, when using Tesseract through OCRmyPDF.
+
 Changing the PDF renderer
 =========================
 
@@ -401,6 +454,10 @@ whether it succeeded or failed. An example message is:
     Temporary working files retained at:
     /tmp/ocrmypdf.io.u20wpz07
 
+When OCRmyPDF is launched as a snap, this corresponds to the snap filesystem, for instance:
+
+    /tmp/snap-private-tmp/snap.ocrmypdf/tmp/ocrmypdf.io.u20wpz07
+
 The organization of this folder is an implementation detail and subject
 to change between releases. However the general organization is that
 working files on a per page basis have the page number as a prefix
@@ -412,9 +469,9 @@ suffix indicates the file type. Some important files include:
    on arguments this may differ from the presentation image
 -  ``_pp_deskew.png`` - the image, after deskewing
 -  ``_pp_clean.png`` - the image, after cleaning with unpaper
--  ``_ocr_tess.pdf`` - the OCR file; appears as a blank page with invisible
+-  ``_ocr_hocr.pdf`` - the OCR file; appears as a blank page with invisible
    text embedded
--  ``_ocr_tess.txt`` - the OCR text (not necessarily all text on the page,
+-  ``_ocr_hocr.txt`` - the OCR text (not necessarily all text on the page,
    if the page is mixed format)
 -  ``fix_docinfo.pdf`` - a temporary file created to fix the PDF DocumentInfo
    data structure

docs/batch.rst

@@ -44,7 +44,7 @@ place, and printing each filename in between runs:
 
 .. code-block:: bash
 
-   find . -printf '%p\n' -name '*.pdf' -exec ocrmypdf '{}' '{}' \;
+   find . -name '*.pdf' -printf '%p\n' -exec ocrmypdf '{}' '{}' \;
 
 This only runs one ``ocrmypdf`` process at a time. This variation uses
 ``find`` to create a directory list and ``parallel`` to parallelize runs
@@ -135,7 +135,7 @@ Users may need to customize the script to meet their requirements.
     "OCR_ON_SUCCESS_ARCHIVE", "This will move the processed original file to ``OCR_ARCHIVE_DIRECTORY`` if the exit code is 0 (OK). Note that ``OCR_ON_SUCCESS_DELETE`` takes precedence over this option, i.e. if both options are set, the input file will be deleted."
     "OCR_OUTPUT_DIRECTORY_YEAR_MONTH", "This will place files in the output in ``{output}/{year}/{month}/{filename}``"
     "OCR_DESKEW", "Apply deskew to crooked input PDFs"
-    "OCR_JSON_SETTINGS", "A JSON string specifying any other arguments for ``ocrmypdf.ocr``, e.g. ``'OCR_JSON_SETTINGS={""rotate_pages"": true}'``."
+    "OCR_JSON_SETTINGS", "A JSON string specifying any other arguments for ``ocrmypdf.ocr``, e.g. ``'OCR_JSON_SETTINGS={""rotate_pages"": true, ""optimize"": "3"}'``."
     "OCR_POLL_NEW_FILE_SECONDS", "Polling interval"
     "OCR_LOGLEVEL", "Level of log messages to report"
 

docs/docker.rst

@@ -65,7 +65,13 @@ The ``ocrmypdf`` image is also available, but is deprecated and will be removed
 in the future.
 
 OCRmyPDF will use all available CPU cores. See the Docker documentation for
-`adjusting memory and CPU on other platforms <https://docs.docker.com/config/containers/resource_constraints/>`__.
+`adjusting memory and CPU on other platforms <https://docs.docker.com/config/containers/resource_constraints/>`__
+if you are using Docker on macOS or Windows, where you may need to manually assign
+more resources. On Linux, all resources will be available automatically.
+
+The underlying operating system and other details in Docker images are subject
+to change at minor releases. If you are modifying the image, you should pin
+the version you intend to use.
 
 Using the Docker image on the command line
 ==========================================

docs/installation.rst

@@ -224,7 +224,7 @@ standard tooling needed to build packages, such as a compiler and binary tools.
 
 .. code-block:: bash
 
-   sudo pacman -S base-devel
+   sudo pacman -S --needed base-devel
 
 Now you are ready to install the OCRmyPDF package.
 
@@ -341,7 +341,7 @@ OCRmyPDF is includes in MacPorts:
     sudo port install ocrmypdf
 
 Note that while this will install tesseract you will need to install
-the appropriate tesseract `language ports <https://ports.macports.org/search/?selected_facets=categories_exact%3Atextproc&installed_file=&q=tesseract&name=on>`__. 
+the appropriate tesseract `language ports <https://ports.macports.org/search/?selected_facets=categories_exact%3Atextproc&installed_file=&q=tesseract&name=on>`__.
 
 Manual installation on macOS
 ----------------------------
@@ -640,8 +640,7 @@ environment:
 
     pip install git+https://github.com/ocrmypdf/OCRmyPDF.git
 
-Or, to install in `development
+Or, to install in editable mode
-mode <https://packaging.python.org/en/latest/guides/distributing-packages-using-setuptools/#working-in-development-mode>`__,
 allowing customization of OCRmyPDF, use the ``-e`` flag:
 
 .. code-block:: bash
@@ -681,7 +680,7 @@ To install all of the development and test requirements:
 .. code-block:: bash
 
     git clone -b main https://github.com/ocrmypdf/OCRmyPDF.git
-    python -m .venv
+    python -m venv .venv
     source .venv/bin/activate
     cd OCRmyPDF
     pip install -e .[test]

docs/maintainers.rst

@@ -38,12 +38,13 @@ on ARM and x86_64. Performance may be poor on other processor architectures.
 Versioning scheme
 -----------------
 
-OCRmyPDF uses setuptools-scm for versioning, which derives the version from
+OCRmyPDF uses hatch-vcs for versioning, which derives the version from
 Git as a single source of truth. This may be unsuitable for some distributions, e.g.
 to indicate that your distribution modifies OCRmyPDF in some way.
 
 You can patch the ``__version__`` variable in ``src/ocrmypdf/_version.py`` if
-necessary.
+necessary, or set the environment variable ``SETUPTOOLS_SCM_PRETEND_VERSION``
+to the required version, if you need to override versioning for some reason.
 
 jbig2enc
 --------

docs/plugins.rst

@@ -29,6 +29,9 @@ conventions. Note that: plugins installed with as setuptools entrypoints are
 not checked currently, because OCRmyPDF assumes you may not want to enable
 plugins for all files.
 
+See [OCRmyPDF-EasyOCR](https://github.com/ocrmypdf/OCRmyPDF-EasyOCR) for an
+example of a straightforward, fully working plugin.
+
 Script plugins
 ==============
 
@@ -70,14 +73,15 @@ similar to ``pytest`` packages such as ``pytest-cov`` (the package) and
     module), just like pytest plugins. At the same time, please make it clear
     that your package is not official.
 
-Setuptools plugins
+Plugins
-==================
+=======
 
 You can also create a plugin that OCRmyPDF will always automatically load if both are
-installed in the same virtual environment, using a setuptools entrypoint.
+installed in the same virtual environment, using a project entrypoint.
+OCRmyPDF uses the entrypoint namespace "ocrmypdf".
 
-Your package's ``pyproject.toml`` would need to contain the following, for a plugin
+For example, ``pyproject.toml`` would need to contain the following, for a plugin named
-named ``ocrmypdf-exampleplugin``:
+``ocrmypdf-exampleplugin``:
 
 .. code-block:: toml
 
@@ -87,13 +91,6 @@ named ``ocrmypdf-exampleplugin``:
     [project.entry-points."ocrmypdf"]
     exampleplugin = "exampleplugin.pluginmodule"
 
-.. code-block:: ini
-
-    # equivalent setup.cfg
-    [options.entry_points]
-    ocrmypdf =
-        exampleplugin = exampleplugin.pluginmodule
-
 Plugin requirements
 ===================
 

docs/release_notes.rst

@@ -30,6 +30,78 @@ OCRmyPDF typically supports the three most recent Python versions.
 
 .. |OCRmyPDF PyPI| image:: https://img.shields.io/pypi/v/ocrmypdf.svg
 
+v16.6.2
+=======
+
+-  Remove invalid hyperlink annotations to satisfy Ghostscript 10.x during PDF/A
+   conversion. :issue:`1425`
+
+v16.6.1
+=======
+
+-  Fixed some issues with Docker build, such as removing unnecessary content and using
+   a stable Tesseract version.
+-  Reverted Docker image to Ubuntu 22.04 to access older/more stable Ghostscript
+   for now.
+-  Clarified batch commands in documentation.
+-  Fixed an issue with JSON serialization and pickling of HOCRResult. :issue:`1427`
+
+v16.6.0
+=======
+
+-  Fixed an issue where damaged PDFs would fail with ``--redo-ocr``. :issue:`1403`
+-  Fixed an error that prevented JBIG2 optimization on Windows if the image
+   was optimized in an earlier step. :issue:`1396`
+-  Fixed an error detecting the version of unpaper 7.0.0. :issue:`1409`
+-  Fixed a performance regression when scanning pages. :issue:`1378`. Thanks @aliemjay.
+-  Fixed Alpine Docker image by enforcing Alpine 3.19. Alpine 3.20 includes a
+   defective version of Tesseract OCR and so is not usable.
+-  Upgraded Ubuntu Docker image to use Ubuntu 24.04.
+-  Build and test scripts/actions switched to uv.
+-  When running in a container, we now remind the user that temporary folders
+   are inside the container and may not be accessible.
+-  Fixed Linux test coverage matrix, which was missing some key versions.
+
+v16.5.0
+=======
+
+-  Fixed issue with interpreting PDFs that have images with array masks.
+   :issue:`1377`
+-  Enabled testing on Python 3.13.
+-  Fixed a test that did not work correctly but still passed. :issue:`1382`
+-  Improved "PDF/A conversion failed" warning message to better describe implications.
+-  Updated documentation to better explain OCR_JSON_SETTINGS in batch processing.
+-  Build backend changed from setuptools to hatchling.
+
+v16.4.3
+=======
+
+-  Work around pdfminer.six issue where a token on the buffer boundary is incorrectly
+   parsed as two tokens. :issue:`1361`
+-  New rules are applied to stencil masks and explicit masks when calculating the
+   optimal page DPI for rendering. :issue:`1362`
+-  Fixed attempts to use an incompatible jbig2.EXE provided by TeX Live. :issue:`1363`
+
+v16.4.2
+=======
+
+-  Fixed order of filenames passed to Ghostscript for PDF/A generation. :issue:`1359`
+-  Suppressed missing jbig2dec warning message. :issue:`1358`
+-  Fixed calculation of image size when soft mask dimensions don't match image
+   dimension. :issue:`1351`
+-  Several fixes to documentation. Thanks to users Iris and JoKalliauer
+   who contributed these changes.
+-  Fixed error on processing PDFs that are missing certain image metadata. :issue:`1315`
+
+v16.4.1
+=======
+
+-  Fixed calculation of image printed area (used in finding weighted DPI for OCR).
+   :issue:`1334`
+-  Fixed "NotImplementedError: not sure how to get colorspace" error
+   messages in logs which simply records a failure to optimize images with
+   print production colorspaces. :issue:`1315`
+
 v16.4.0
 =======
 

misc/bisect_pdf.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: 2024 James R. Barlow
+# SPDX-License-Identifier: MIT
+
+"""Helper script for bisecting PDFs to find a page with an issue."""
+
+import sys
+
+import pikepdf
+
+if len(sys.argv) != 2:
+    print(f"Usage: {sys.argv[0]} <input.pdf>")
+    sys.exit(1)
+
+with pikepdf.open(sys.argv[1]) as pdf:
+    num_pages = len(pdf.pages)
+    low = 0
+    high = num_pages - 1
+    while low <= high:
+        mid = (low + high) // 2
+        with pikepdf.new() as new_pdf:
+            new_pdf.pages.extend(pdf.pages[low : mid + 1])
+            new_pdf.save(f"bisect-issue-{low + 1}-{mid + 1}.pdf")
+        print(f"Is bisect-issue-{low + 1}-{mid + 1}.pdf good or bad?", end=" ")
+        while True:
+            response = input().lower()
+            if response == "good":
+                low = mid + 1
+                break
+            elif response == "bad":
+                high = mid - 1
+                break
+            else:
+                print("Please respond with 'good' or 'bad'.")
+    print(f"The issue is on page {low + 1} of the original PDF.")
+    with pikepdf.new() as new_pdf:
+        new_pdf.pages.extend(pdf.pages[low])
+        new_pdf.save(f"bisect-issue-bad-{low + 1}.pdf")
+    with pikepdf.new() as new_pdf:
+        new_pdf.pages.extend(pdf.pages[:low])
+        new_pdf.pages.extend(pdf.pages[low + 1 :])
+        new_pdf.save(f"bisect-issue-good-{low + 1}.pdf")

pyproject.toml

@@ -1,8 +1,8 @@
 # SPDX-FileCopyrightText: 2022 James R. Barlow
 # SPDX-License-Identifier: MPL-2.0
 [build-system]
-requires = ["setuptools >= 61", "setuptools_scm[toml] >= 7.0.5", "wheel"]
+requires = ["hatchling", "hatch-vcs"]
-build-backend = "setuptools.build_meta"
+build-backend = "hatchling.build"
 
 [project]
 name = "ocrmypdf"
@@ -46,6 +46,7 @@ keywords = ["PDF", "OCR", "optical character recognition", "PDF/A", "scanning"]
 Documentation = "https://ocrmypdf.readthedocs.io/"
 Source = "https://github.com/ocrmypdf/OCRmyPDF"
 Tracker = "https://github.com/ocrmypdf/OCRmyPDF/issues"
+Changelog = "https://github.com/ocrmypdf/OCRmyPDF/docs/release_notes.rst"
 
 [project.optional-dependencies]
 docs = ["sphinx", "sphinx-issues", "sphinx-rtd-theme"]
@@ -67,14 +68,11 @@ webservice = ["Flask>=2.0.1"]
 [project.scripts]
 ocrmypdf = "ocrmypdf.__main__:run"
 
-[tool.setuptools.package-data]
+[tool.hatch.version]
-ocrmypdf = ["data/sRGB.icc", "py.typed"]
+source = "vcs"
 
-[tool.setuptools.packages.find]
+[tool.hatch.build.hooks.vcs]
-where = ["src"]
+version-file = "src/ocrmypdf/_version.py"
-namespaces = false
-
-[tool.setuptools_scm]
 
 [tool.distutils.bdist_wheel]
 python-tag = "py310"
@@ -157,3 +155,6 @@ convention = "google"
 
 [tool.ruff.format]
 quote-style = "preserve"
+
+[dependency-groups]
+dev = ["mypy>=1.13.0"]

src/ocrmypdf/__init__.py

@@ -9,11 +9,12 @@ from pluggy import HookimplMarker as _HookimplMarker
 
 from ocrmypdf import helpers, hocrtransform, pdfa, pdfinfo
 from ocrmypdf._concurrent import Executor
+from ocrmypdf._defaults import PROGRAM_NAME
 from ocrmypdf._jobcontext import PageContext, PdfContext
 from ocrmypdf._pipelines._common import (
     configure_debug_logging,
 )
-from ocrmypdf._version import PROGRAM_NAME, __version__
+from ocrmypdf._version import __version__
 from ocrmypdf.api import (
     Verbosity,
     configure_logging,
@@ -37,7 +38,6 @@ from ocrmypdf.pluginspec import OcrEngine, OrientationConfidence
 
 hookimpl = _HookimplMarker('ocrmypdf')
 
-
 __all__ = [
     '__version__',
     'BadArgsError',

src/ocrmypdf/_annots.py

@@ -0,0 +1,66 @@
+# SPDX-FileCopyrightText: 2024 James R. Barlow
+# SPDX-License-Identifier: MPL-2.0
+
+"""OCRmyPDF PDF annotation cleanup."""
+
+from __future__ import annotations
+
+import logging
+
+from pikepdf import Dictionary, Name, NameTree, Pdf
+
+log = logging.getLogger(__name__)
+
+
+def remove_broken_goto_annotations(pdf: Pdf) -> bool:
+    """Remove broken goto annotations from a PDF.
+
+    If a PDF contains a GoTo Action that points to a named destination that does not
+    exist, Ghostscript PDF/A conversion will fail. In any event, a named destination
+    that is not defined is not useful.
+
+    Args:
+        pdf: Opened PDF file.
+
+    Returns:
+        bool: True if the file was modified, False if not.
+    """
+    modified = False
+
+    # Check if there are any named destinations
+    if Name.Names not in pdf.Root:
+        return modified
+    if Name.Dests not in pdf.Root[Name.Names]:
+        return modified
+
+    dests = pdf.Root[Name.Names][Name.Dests]
+    if not isinstance(dests, Dictionary):
+        return modified
+    nametree = NameTree(dests)
+
+    # Create a set of all named destinations
+    names = set(k for k in nametree.keys())
+
+    for n, page in enumerate(pdf.pages):
+        if Name.Annots not in page:
+            continue
+        for annot in page[Name.Annots]:
+            if not isinstance(annot, Dictionary):
+                continue
+            if Name.A not in annot or Name.D not in annot[Name.A]:
+                continue
+            # We found an annotation that points to a named destination
+            named_destination = str(annot[Name.A][Name.D])
+            if named_destination not in names:
+                # If there is no corresponding named destination, remove the
+                # annotation. Having no destination set is still valid and just
+                # makes the link non-functional.
+                log.warning(
+                    f"Disabling a hyperlink annotation on page {n + 1} to a "
+                    "non-existent named destination "
+                    f"{named_destination}."
+                )
+                del annot[Name.A][Name.D]
+                modified = True
+
+    return modified

src/ocrmypdf/_defaults.py

@@ -6,3 +6,5 @@ DEFAULT_LANGUAGE = 'eng'
 
 # Default rotation threshold
 DEFAULT_ROTATE_PAGES_THRESHOLD = 14.0
+
+PROGRAM_NAME = 'OCRmyPDF'

src/ocrmypdf/_exec/jbig2enc.py

@@ -5,7 +5,7 @@
 
 from __future__ import annotations
 
-from subprocess import PIPE
+from subprocess import PIPE, CalledProcessError
 
 from packaging.version import Version
 
@@ -14,7 +14,13 @@ from ocrmypdf.subprocess import get_version, run
 
 
 def version() -> Version:
-    return Version(get_version('jbig2', regex=r'jbig2enc (\d+(\.\d+)*).*'))
+    try:
+        version = get_version('jbig2', regex=r'jbig2enc (\d+(\.\d+)*).*')
+    except CalledProcessError as e:
+        # TeX Live for Windows provides an incompatible jbig2.EXE which may
+        # be on the PATH.
+        raise MissingDependencyError('jbig2enc') from e
+    return Version(version)
 
 
 def available():

src/ocrmypdf/_exec/unpaper.py

@@ -48,7 +48,7 @@ class UnpaperImageTooLargeError(Exception):
 
 
 def version() -> Version:
-    return Version(get_version('unpaper'))
+    return Version(get_version('unpaper', regex=r'(?m).*?(\d+(\.\d+)(\.\d+)?)'))
 
 
 @contextmanager

src/ocrmypdf/_metadata.py

@@ -15,8 +15,9 @@ from pikepdf import Dictionary, Name, Pdf
 from pikepdf import __version__ as PIKEPDF_VERSION
 from pikepdf.models.metadata import PdfMetadata, encode_pdf_date
 
+from ocrmypdf._annots import remove_broken_goto_annotations
+from ocrmypdf._defaults import PROGRAM_NAME
 from ocrmypdf._jobcontext import PdfContext
-from ocrmypdf._version import PROGRAM_NAME
 from ocrmypdf._version import __version__ as OCRMYPF_VERSION
 from ocrmypdf.languages import iso_639_2_from_3
 

src/ocrmypdf/_pipeline.py

@@ -159,8 +159,13 @@ def triage(
                     "Argument --image-dpi is being ignored because the "
                     "input file is a PDF, not an image."
                 )
-            # Origin file is a pdf create a symlink with pdf extension
+            try:
-            safe_symlink(input_file, output_file)
+                with pikepdf.open(input_file) as pdf:
+                    pdf.save(output_file)
+            except pikepdf.PdfError as e:
+                raise InputFileError() from e
+            except pikepdf.PasswordError as e:
+                raise EncryptedPdfError() from e
             return output_file
     except OSError as e:
         log.debug(f"Temporary file was at: {input_file}")
@@ -475,7 +480,7 @@ def calculate_raster_dpi(page_context: PageContext):
     page_dpi = get_page_square_dpi(page_context, image_dpi)
     if dpi_profile and dpi_profile.average_to_max_dpi_ratio < 0.8:
         log.warning(
-            "Weight average image DPI is %0.1f, max DPI is %0.1f. "
+            "Weighted average image DPI is %0.1f, max DPI is %0.1f. "
             "The discrepancy may indicate a high detail region on this page, "
             "but could also indicate a problem with the input PDF file. "
             "Page image will be rendered at %0.1f DPI.",
@@ -854,7 +859,7 @@ def fix_pagepdf_boxes(
             page.CropBox = cropbox
             page.TrimBox = trimbox
         pdf.save(out_file)
-    return pdf
+    return out_file
 
 
 def generate_postscript_stub(context: PdfContext) -> Path:

src/ocrmypdf/_pipelines/_common.py

@@ -20,7 +20,9 @@ from pathlib import Path
 from typing import NamedTuple, cast
 
 import PIL
+from pikepdf import Pdf
 
+from ocrmypdf._annots import remove_broken_goto_annotations
 from ocrmypdf._concurrent import Executor, setup_executor
 from ocrmypdf._jobcontext import PageContext, PdfContext
 from ocrmypdf._logging import PageNumberFilter
@@ -33,6 +35,7 @@ from ocrmypdf._pipeline import (
     generate_postscript_stub,
     get_orientation_correction,
     get_pdf_save_settings,
+    get_pdfinfo,
     optimize_pdf,
     preprocess_clean,
     preprocess_deskew,
@@ -51,9 +54,12 @@ from ocrmypdf.helpers import (
     available_cpu_count,
     check_pdf,
     pikepdf_enable_mmap,
+    running_in_docker,
+    running_in_snap,
     samefile,
 )
 from ocrmypdf.pdfa import file_claims_pdfa
+from ocrmypdf.pdfinfo import PdfInfo
 
 log = logging.getLogger(__name__)
 tls = threading.local()
@@ -100,6 +106,23 @@ class PageResult(NamedTuple):
     """Orientation correction in degrees."""
 
 
+class HOCRResultEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, Path):
+            return {'Path': str(obj)}
+        return super().default(obj)
+
+
+class HOCRResultDecoder(json.JSONDecoder):
+    def __init__(self, *args, **kwargs):
+        super().__init__(object_hook=self.dict_to_object, *args, **kwargs)
+
+    def dict_to_object(self, d):
+        if 'Path' in d:
+            return Path(d['Path'])
+        return d
+
+
 @dataclass
 class HOCRResult:
     """Result when hOCR is finished processing."""
@@ -119,38 +142,14 @@ class HOCRResult:
     orientation_correction: int = 0
     """Orientation correction in degrees."""
 
-    def __getstate__(self):
-        """Return state values to be pickled."""
-        return {
-            k: (
-                ('Path://' + str(v))
-                if k in ('pdf_page_from_image', 'hocr', 'textpdf') and v is not None
-                else v
-            )
-            for k, v in self.__dict__.items()
-        }
-
-    def __setstate__(self, state):
-        """Restore state from the unpickled state values."""
-        self.__dict__.update(
-            {
-                k: (
-                    Path(v.removeprefix('Path://'))
-                    if k in ('pdf_page_from_image', 'hocr', 'textpdf') and v is not None
-                    else v
-                )
-                for k, v in state.items()
-            }
-        )
-
     @classmethod
     def from_json(cls, json_str: str) -> HOCRResult:
         """Create an instance from a dict."""
-        return cls(**json.loads(json_str))
+        return cls(**json.loads(json_str, cls=HOCRResultDecoder))
 
     def to_json(self) -> str:
         """Serialize to a JSON string."""
-        return json.dumps(self.__getstate__())
+        return json.dumps(self.__dict__, cls=HOCRResultEncoder)
 
 
 def configure_debug_logging(
@@ -183,7 +182,7 @@ def configure_debug_logging(
     return log_file_handler, remover
 
 
-def worker_init(max_pixels: int) -> None:
+def worker_init(max_pixels: int | None) -> None:
     """Initialize a worker thread or process."""
     # In Windows, child process will not inherit our change to this value in
     # the parent process, so ensure workers get it set. Not needed when running
@@ -215,6 +214,22 @@ def manage_debug_log_handler(
             remover()
 
 
+def _print_temp_folder_location(work_folder: Path):
+    """Print the location of the temporary work folder."""
+    msgs = [f"Temporary working files retained at:\n{work_folder}"]
+    if running_in_docker():  # pragma: no cover
+        msgs.append(
+            "OCRmyPDF is running in a Docker container, "
+            "so the files will be inside the container."
+        )
+    elif running_in_snap():  # pragma: no cover
+        msgs.append(
+            "OCRmyPDF is running in a Snap container, "
+            "so the files will be inside the container."
+        )
+    print('\n'.join(msgs), file=sys.stderr)
+
+
 @contextmanager
 def manage_work_folder(*, work_folder: Path, retain: bool, print_location: bool):
     try:
@@ -222,10 +237,7 @@ def manage_work_folder(*, work_folder: Path, retain: bool, print_location: bool)
     finally:
         if retain:
             if print_location:
-                print(
+                _print_temp_folder_location(work_folder)
-                    f"Temporary working files retained at:\n{work_folder}",
-                    file=sys.stderr,
-                )
         else:
             shutil.rmtree(work_folder, ignore_errors=True)
 
@@ -300,6 +312,20 @@ def setup_pipeline(
     return executor
 
 
+def do_get_pdfinfo(
+    pdf_path: Path, executor: Executor, options: argparse.Namespace
+) -> PdfInfo:
+    return get_pdfinfo(
+        pdf_path,
+        executor=executor,
+        detailed_analysis=options.redo_ocr,
+        progbar=options.progress_bar,
+        max_workers=options.jobs,
+        use_threads=options.use_threads,
+        check_pages=options.pages,
+    )
+
+
 def preprocess(
     page_context: PageContext,
     image: Path,
@@ -414,7 +440,14 @@ def postprocess(
     pdf_file: Path, context: PdfContext, executor: Executor
 ) -> tuple[Path, Sequence[str]]:
     """Postprocess the PDF file."""
-    pdf_out = pdf_file
+    # pdf_out = pdf_file
+    with Pdf.open(pdf_file) as pdf:
+        fix_annots = context.get_path('fix_annots.pdf')
+        if remove_broken_goto_annotations(pdf):
+            pdf.save(fix_annots)
+            pdf_out = fix_annots
+        else:
+            pdf_out = pdf_file
     if context.options.output_type.startswith('pdfa'):
         ps_stub_out = generate_postscript_stub(context)
         pdf_out = convert_to_pdfa(pdf_out, ps_stub_out, context)
@@ -441,7 +474,8 @@ def report_output_pdf(options, start_input_file, optimize_messages) -> ExitCode:
                 log.info("Output file is a %s (as expected)", pdfa_info['conformance'])
             else:
                 log.warning(
-                    "Output file is okay but is not PDF/A (seems to be %s)",
+                    "Output file is a valid PDF, but conversion to PDF/A did not "
+                    "succeed (issue: %s)",
                     pdfa_info['conformance'],
                 )
                 return ExitCode.pdfa_conversion_failed

src/ocrmypdf/_pipelines/hocr_to_ocr_pdf.py

@@ -19,11 +19,11 @@ from ocrmypdf._graft import OcrGrafter
 from ocrmypdf._jobcontext import PageContext, PdfContext
 from ocrmypdf._pipeline import (
     copy_final,
-    get_pdfinfo,
     render_hocr_page,
 )
 from ocrmypdf._pipelines._common import (
     HOCRResult,
+    do_get_pdfinfo,
     manage_work_folder,
     postprocess,
     report_output_pdf,
@@ -117,15 +117,7 @@ def run_hocr_to_ocr_pdf_pipeline(
         origin_pdf = work_folder / 'origin.pdf'
 
         # Gather pdfinfo and create context
-        pdfinfo = get_pdfinfo(
+        pdfinfo = do_get_pdfinfo(origin_pdf, executor, options)
-            origin_pdf,
-            executor=executor,
-            detailed_analysis=options.redo_ocr,
-            progbar=options.progress_bar,
-            max_workers=options.jobs,
-            use_threads=options.use_threads,
-            check_pages=options.pages,
-        )
         context = PdfContext(options, work_folder, origin_pdf, pdfinfo, plugin_manager)
         plugin_manager.hook.check_options(options=options)
         optimize_messages = exec_hocr_to_ocr_pdf(context, executor)

src/ocrmypdf/_pipelines/ocr.py

@@ -21,7 +21,6 @@ from ocrmypdf._graft import OcrGrafter
 from ocrmypdf._jobcontext import PageContext, PdfContext
 from ocrmypdf._pipeline import (
     copy_final,
-    get_pdfinfo,
     is_ocr_required,
     merge_sidecars,
     ocr_engine_hocr,
@@ -33,6 +32,7 @@ from ocrmypdf._pipeline import (
 from ocrmypdf._pipelines._common import (
     PageResult,
     cli_exception_handler,
+    do_get_pdfinfo,
     manage_debug_log_handler,
     manage_work_folder,
     postprocess,
@@ -171,16 +171,7 @@ def _run_pipeline(
         )
 
         # Gather pdfinfo and create context
-        pdfinfo = get_pdfinfo(
+        pdfinfo = do_get_pdfinfo(origin_pdf, executor, options)
-            origin_pdf,
-            executor=executor,
-            detailed_analysis=options.redo_ocr,
-            progbar=options.progress_bar,
-            max_workers=options.jobs,
-            use_threads=options.use_threads,
-            check_pages=options.pages,
-        )
-
         context = PdfContext(options, work_folder, origin_pdf, pdfinfo, plugin_manager)
 
         # Validate options are okay for this pdf

src/ocrmypdf/_pipelines/pdf_to_hocr.py

@@ -17,13 +17,13 @@ import PIL
 from ocrmypdf._concurrent import Executor
 from ocrmypdf._jobcontext import PageContext, PdfContext
 from ocrmypdf._pipeline import (
-    get_pdfinfo,
     is_ocr_required,
     ocr_engine_hocr,
     validate_pdfinfo_options,
 )
 from ocrmypdf._pipelines._common import (
     HOCRResult,
+    do_get_pdfinfo,
     manage_work_folder,
     process_page,
     set_thread_pageno,
@@ -94,18 +94,11 @@ def run_hocr_pipeline(
         work_folder=options.output_folder, retain=True, print_location=False
     ) as work_folder:
         executor = setup_pipeline(options, plugin_manager)
-        shutil.copy2(options.input_file, work_folder / 'origin.pdf')
+        origin_pdf = work_folder / 'origin.pdf'
+        shutil.copy2(options.input_file, origin_pdf)
 
         # Gather pdfinfo and create context
-        pdfinfo = get_pdfinfo(
+        pdfinfo = do_get_pdfinfo(origin_pdf, executor, options)
-            options.input_file,
-            executor=executor,
-            detailed_analysis=options.redo_ocr,
-            progbar=options.progress_bar,
-            max_workers=options.jobs,
-            use_threads=options.use_threads,
-            check_pages=options.pages,
-        )
         context = PdfContext(
             options, work_folder, options.input_file, pdfinfo, plugin_manager
         )

src/ocrmypdf/_validation.py

@@ -28,7 +28,13 @@ from ocrmypdf.exceptions import (
     MissingDependencyError,
     OutputFileAccessError,
 )
-from ocrmypdf.helpers import is_file_writable, monotonic, safe_symlink
+from ocrmypdf.helpers import (
+    is_file_writable,
+    monotonic,
+    running_in_docker,
+    running_in_snap,
+    safe_symlink,
+)
 from ocrmypdf.subprocess import check_external_program
 
 log = logging.getLogger(__name__)
@@ -237,18 +243,6 @@ def check_options(options: Namespace, plugin_manager: PluginManager) -> None:
     _check_plugin_options(options, plugin_manager)
 
 
-def _in_docker():
-    return Path('/.dockerenv').exists()
-
-
-def _in_snap():
-    try:
-        cgroup_text = Path('/proc/self/cgroup').read_text()
-        return 'snap.ocrmypdf' in cgroup_text
-    except FileNotFoundError:
-        return False
-
-
 def create_input_file(options: Namespace, work_folder: Path) -> tuple[Path, str]:
     if options.input_file == '-':
         # stdin
@@ -272,7 +266,7 @@ def create_input_file(options: Namespace, work_folder: Path) -> tuple[Path, str]
             return target, os.fspath(options.input_file)
         except FileNotFoundError as e:
             msg = f"File not found - {options.input_file}"
-            if _in_docker():  # pragma: no cover
+            if running_in_docker():  # pragma: no cover
                 msg += (
                     "\nDocker cannot access your working directory unless you "
                     "explicitly share it with the Docker container and set up"
@@ -282,7 +276,7 @@ def create_input_file(options: Namespace, work_folder: Path) -> tuple[Path, str]
                     "\tdocker run -i --rm jbarlow83/ocrmypdf - - <input.pdf >output.pdf"
                     "\n"
                 )
-            elif _in_snap():  # pragma: no cover
+            elif running_in_snap():  # pragma: no cover
                 msg += (
                     "\nSnap applications cannot access files outside of "
                     "your home directory unless you explicitly allow it. "

src/ocrmypdf/_version.py

@@ -1,16 +0,0 @@
-# SPDX-FileCopyrightText: 2022 James R. Barlow
-# SPDX-License-Identifier: MPL-2.0
-
-"""Get version by introspecting package information.
-
-OCRmyPDF uses setuptools_scm to derive version from git tags.
-"""
-
-from __future__ import annotations
-
-from importlib.metadata import version as _package_version
-
-PROGRAM_NAME = 'ocrmypdf'
-
-# Official PEP 396
-__version__ = _package_version('ocrmypdf')

src/ocrmypdf/api.py

@@ -14,7 +14,7 @@ from collections.abc import Iterable, Sequence
 from enum import IntEnum
 from io import IOBase
 from pathlib import Path
-from typing import AnyStr, BinaryIO
+from typing import BinaryIO
 from warnings import warn
 
 import pluggy
@@ -28,7 +28,7 @@ from ocrmypdf._validation import check_options
 from ocrmypdf.cli import ArgumentParser, get_parser
 from ocrmypdf.helpers import is_iterable_notstr
 
-StrPath = Path | AnyStr
+StrPath = Path | str | bytes
 PathOrIO = BinaryIO | StrPath
 
 # Installing plugins affects the global state of the Python interpreter,
@@ -140,9 +140,9 @@ def configure_logging(
 
 def _kwargs_to_cmdline(
     *, defer_kwargs: set[str], **kwargs
-) -> tuple[list[str], dict[str, AnyStr]]:
+) -> tuple[list[str | bytes], dict[str, str | bytes]]:
     """Convert kwargs to command line arguments."""
-    cmdline = []
+    cmdline: list[str | bytes] = []
     deferred = {}
     for arg, val in kwargs.items():
         if val is None:
@@ -279,7 +279,7 @@ def ocr(  # noqa: D417
     fast_web_view: float | None = None,
     continue_on_soft_render_error: bool | None = None,
     invalidate_digital_signatures: bool | None = None,
-    plugins: Iterable[StrPath] | None = None,
+    plugins: Iterable[Path | str] | None = None,
     plugin_manager=None,
     keep_temporary_files: bool | None = None,
     progress_bar: bool | None = None,
@@ -420,7 +420,7 @@ def _pdf_to_hocr(  # noqa: D417
     continue_on_soft_render_error: bool | None = None,
     invalidate_digital_signatures: bool | None = None,
     plugin_manager=None,
-    plugins: Sequence[StrPath] | None = None,
+    plugins: Sequence[Path | str] | None = None,
     keep_temporary_files: bool | None = None,
     **kwargs,
 ):
@@ -491,7 +491,7 @@ def _hocr_to_ocr_pdf(  # noqa: D417
     color_conversion_strategy: str | None = None,
     fast_web_view: float | None = None,
     plugin_manager=None,
-    plugins: Sequence[StrPath] | None = None,
+    plugins: Sequence[Path | str] | None = None,
     **kwargs,
 ):
     """Run OCRmyPDF on a work folder and produce an output PDF.

src/ocrmypdf/builtin_plugins/ghostscript.py

@@ -129,7 +129,7 @@ def generate_pdfa(
 ):
     """Generate a PDF/A from the list of PDF pages and PDF/A metadata."""
     ghostscript.generate_pdfa(
-        pdf_pages=[*pdf_pages, pdfmark],
+        pdf_pages=[pdfmark, *pdf_pages],
         output_file=output_file,
         compression=context.options.pdfa_image_compression,
         color_conversion_strategy=context.options.color_conversion_strategy,

src/ocrmypdf/cli.py

@@ -10,7 +10,7 @@ from collections.abc import Callable, Mapping
 from typing import Any, TypeVar
 
 from ocrmypdf._defaults import DEFAULT_ROTATE_PAGES_THRESHOLD
-from ocrmypdf._version import PROGRAM_NAME as _PROGRAM_NAME
+from ocrmypdf._defaults import PROGRAM_NAME as _PROGRAM_NAME
 from ocrmypdf._version import __version__ as _VERSION
 
 T = TypeVar('T', int, float)

src/ocrmypdf/helpers.py

@@ -268,7 +268,9 @@ def check_pdf(input_file: Path) -> bool:
         return False
     else:
         with pdf:
-            messages = pdf.check()
+            with warnings.catch_warnings():
+                warnings.filterwarnings('ignore', message=r'pikepdf.*JBIG2.*')
+                messages = pdf.check()
             success = True
             for msg in messages:
                 if 'error' in msg.lower():
@@ -333,3 +335,17 @@ def pikepdf_enable_mmap() -> None:
         )
     except AttributeError:
         log.debug("pikepdf mmap not available")
+
+
+def running_in_docker() -> bool:
+    """Returns True if we seem to be running in a Docker container."""
+    return Path('/.dockerenv').exists()
+
+
+def running_in_snap() -> bool:
+    """Returns True if we seem to be running in a Snap container."""
+    try:
+        cgroup_text = Path('/proc/self/cgroup').read_text()
+        return 'snap.ocrmypdf' in cgroup_text
+    except FileNotFoundError:
+        return False

src/ocrmypdf/hocrtransform/_hocr.py

@@ -357,7 +357,7 @@ class HocrTransform:
         line_matrix: Matrix,
         text: Text,
         fontsize: float,
-        elem: Element,
+        elem: Element | None,
         next_elem: Element | None,
         text_direction: TextDirection,
         inject_word_breaks: bool,
@@ -434,7 +434,7 @@ class HocrTransform:
                 if ocr_par is None:
                     continue
                 canvas.do.rect(
-                    ocr_par.llx, ocr_par.lly, ocr_par.width, ocr_par.height, fill=0
+                    ocr_par.llx, ocr_par.lly, ocr_par.width, ocr_par.height, fill=False
                 )
 
     def _debug_draw_line_bbox(self, canvas: Canvas, line_box: Rectangle, color=BLUE):
@@ -443,7 +443,7 @@ class HocrTransform:
             return
         with canvas.do.save_state():
             canvas.do.stroke_color(color).line_width(0.15).rect(
-                line_box.llx, line_box.lly, line_box.width, line_box.height, fill=0
+                line_box.llx, line_box.lly, line_box.width, line_box.height, fill=False
             )
 
     def _debug_draw_word_triangle(
@@ -467,7 +467,7 @@ class HocrTransform:
             return
         with canvas.do.save_state():
             canvas.do.stroke_color(color).line_width(line_width).rect(
-                box.llx, box.lly, box.width, box.height, fill=0
+                box.llx, box.lly, box.width, box.height, fill=False
             )
 
     def _debug_draw_space_bbox(
@@ -478,7 +478,7 @@ class HocrTransform:
             return
         with canvas.do.save_state():
             canvas.do.fill_color(color).line_width(line_width).rect(
-                box.llx, box.lly, box.width, box.height, fill=1
+                box.llx, box.lly, box.width, box.height, fill=True
             )
 
     def _debug_draw_baseline(

src/ocrmypdf/optimize.py

@@ -28,6 +28,7 @@ from pikepdf import (
     Stream,
     UnsupportedImageTypeError,
 )
+from pikepdf.models.image import HifiPrintImageNotTranscodableError
 from PIL import Image
 
 from ocrmypdf._concurrent import Executor, SerialExecutor
@@ -74,10 +75,15 @@ def extract_image_filter(
     """Determine if an image is extractable."""
     if image.Subtype != Name.Image:
         return None
-    if image.Length < 100:
+    if not isinstance(image.Length, int) or image.Length < 100:
         log.debug(f"xref {xref}: skipping image with small stream size")
         return None
-    if image.Width < 8 or image.Height < 8:  # Issue 732
+    if (
+        not isinstance(image.Width, int)
+        or not isinstance(image.Height, int)
+        or image.Width < 8
+        or image.Height < 8
+    ):  # Issue 732
         log.debug(f"xref {xref}: skipping image with unusually small dimensions")
         return None
 
@@ -153,7 +159,10 @@ def extract_image_jbig2(
                 imgname = root / f'{xref:08d}'
                 with imgname.open('wb') as f:
                     ext = pim.extract_to(stream=f)
-                imgname.rename(imgname.with_suffix(ext))
+                # Rename the file so it has .prejbig2.ext extension
+                # Making it unique avoids problems with Windows if the
+                # same image is extracted multiple times
+                imgname.rename(imgname.with_suffix(".prejbig2" + ext))
             except NotImplementedError as e:
                 if '/Decode' in str(e):
                     log.debug(
@@ -169,7 +178,7 @@ def extract_image_jbig2(
                     pim.obj.ColorSpace = colorspace
                 else:
                     del pim.obj.ColorSpace
-            return XrefExt(xref, ext)
+            return XrefExt(xref, ".prejbig2" + ext)
     return None
 
 
@@ -200,7 +209,7 @@ def extract_image_generic(
             with imgname.open('wb') as f:
                 ext = pim.extract_to(stream=f)
             imgname.rename(imgname.with_suffix(ext))
-        except UnsupportedImageTypeError:
+        except (UnsupportedImageTypeError, HifiPrintImageNotTranscodableError):
             return None
         return XrefExt(xref, ext)
     elif (

src/ocrmypdf/pdfinfo/info.py

@@ -12,7 +12,7 @@ import re
 import statistics
 from collections import defaultdict
 from collections.abc import Callable, Container, Iterable, Iterator, Mapping, Sequence
-from contextlib import contextmanager
+from contextlib import contextmanager, nullcontext
 from decimal import Decimal
 from enum import Enum, auto
 from functools import partial
@@ -24,6 +24,7 @@ from warnings import warn
 
 from pdfminer.layout import LTPage, LTTextBox
 from pikepdf import (
+    Dictionary,
     Matrix,
     Name,
     Object,
@@ -40,7 +41,12 @@ from ocrmypdf._concurrent import Executor, SerialExecutor
 from ocrmypdf._progressbar import ProgressBar
 from ocrmypdf.exceptions import EncryptedPdfError, InputFileError
 from ocrmypdf.helpers import Resolution, available_cpu_count, pikepdf_enable_mmap
-from ocrmypdf.pdfinfo.layout import LTStateAwareChar, get_page_analysis, get_text_boxes
+from ocrmypdf.pdfinfo.layout import (
+    LTStateAwareChar,
+    PdfMinerState,
+    get_page_analysis,
+    get_text_boxes,
+)
 
 logger = logging.getLogger()
 
@@ -369,8 +375,26 @@ class ImageInfo:
             pim = PdfImage(pdfimage)
         else:
             raise ValueError("Either pdfimage or inline must be set")
+
         self._width = pim.width
         self._height = pim.height
+        if (smask := pim.obj.get(Name.SMask, None)) is not None:
+            # SMask is pretty much an alpha channel, but in PDF it's possible
+            # for channel to have different dimensions than the image
+            # itself. Some PDF writers use this to create a grayscale stencil
+            # mask. For our purposes, the effective size is the size of the
+            # larger component (image or smask).
+            if isinstance(smask, Stream | Dictionary):
+                self._width = max(smask.get(Name.Width, 0), self._width)
+                self._height = max(smask.get(Name.Height, 0), self._height)
+        if (mask := pim.obj.get(Name.Mask, None)) is not None:
+            # If the image has a /Mask entry, it has an explicit mask.
+            # /Mask can be a Stream or an Array. If it's a Stream,
+            # use its /Width and /Height if they are larger than the main
+            # image's.
+            if isinstance(mask, Stream | Dictionary):
+                self._width = max(mask.get(Name.Width, 0), self._width)
+                self._height = max(mask.get(Name.Height, 0), self._height)
 
         # If /ImageMask is true, then this image is a stencil mask
         # (Images that draw with this stencil mask will have a reference to
@@ -474,9 +498,18 @@ class ImageInfo:
     def renderable(self) -> bool:
         """Whether the image is renderable.
 
-        Some PDFs in the wild have invalid images that are not renderable.
+        Some PDFs in the wild have invalid images that are not renderable,
+        due to unusual dimensions.
+
+        Stencil masks are not also not renderable, since they are not
+        drawn, but rather they control how rendering happens.
         """
-        return self.dpi.is_finite and self.width >= 0 and self.height >= 0
+        return (
+            self.dpi.is_finite
+            and self.width >= 0
+            and self.height >= 0
+            and self.type_ != 'stencil'
+        )
 
     @property
     def dpi(self) -> Resolution:
@@ -491,7 +524,7 @@ class ImageInfo:
         """Physical area of the image in square inches."""
         if not self.renderable:
             return 0.0
-        return float(self.width * self.dpi.x * self.height * self.dpi.y)
+        return float((self.width / self.dpi.x) * (self.height / self.dpi.y))
 
     def __repr__(self):
         """Return a string representation of the image."""
@@ -573,7 +606,7 @@ def _find_form_xobject_images(pdf: Pdf, container: Object, contentsinfo: Content
     xobjs = resources[Name.XObject].as_dict()
     for xobj in xobjs:
         candidate = xobjs[xobj]
-        if candidate is None or candidate[Name.Subtype] != Name.Form:
+        if candidate is None or candidate.get(Name.Subtype) != Name.Form:
             continue
 
         form_xobject = candidate
@@ -674,13 +707,13 @@ def _page_has_text(text_blocks: Iterable[FloatRect], page_width, page_height) ->
 
 
 def simplify_textboxes(
-    miner: LTPage, textbox_getter: Callable[[LTPage], Iterator[LTTextBox]]
+    miner_page: LTPage, textbox_getter: Callable[[LTPage], Iterator[LTTextBox]]
 ) -> Iterator[TextboxInfo]:
     """Extract only limited content from text boxes.
 
     We do this to save memory and ensure that our objects are pickleable.
     """
-    for box in textbox_getter(miner):
+    for box in textbox_getter(miner_page):
         first_line = box._objs[0]  # pylint: disable=protected-access
         first_char = first_line._objs[0]  # pylint: disable=protected-access
         if not isinstance(first_char, LTStateAwareChar):
@@ -727,9 +760,12 @@ def _pdf_pageinfo_sync(
     infile: Path,
     check_pages: Container[int],
     detailed_analysis: bool,
+    miner_state: PdfMinerState | None,
 ) -> PageInfo:
     with _pdf_pageinfo_sync_pdf(thread_pdf, infile) as pdf:
-        return PageInfo(pdf, pageno, infile, check_pages, detailed_analysis)
+        return PageInfo(
+            pdf, pageno, infile, check_pages, detailed_analysis, miner_state
+        )
 
 
 def _pdf_pageinfo_concurrent(
@@ -741,6 +777,7 @@ def _pdf_pageinfo_concurrent(
     progbar,
     check_pages,
     detailed_analysis: bool = False,
+    miner_state: PdfMinerState | None = None,
 ) -> Sequence[PageInfo | None]:
     pages: list[PageInfo | None] = [None] * len(pdf.pages)
 
@@ -772,7 +809,8 @@ def _pdf_pageinfo_concurrent(
     initial_pdf = pdf if use_threads else None
 
     contexts = (
-        (n, initial_pdf, infile, check_pages, detailed_analysis) for n in range(total)
+        (n, initial_pdf, infile, check_pages, detailed_analysis, miner_state)
+        for n in range(total)
     )
     assert n_workers == 1 if use_threads else n_workers >= 1, "Not multithreadable"
     logger.debug(
@@ -839,12 +877,15 @@ class PageInfo:
         infile: PathLike,
         check_pages: Container[int],
         detailed_analysis: bool = False,
+        miner_state: PdfMinerState | None = None,
     ):
         """Initialize a PageInfo object."""
         self._pageno = pageno
         self._infile = infile
         self._detailed_analysis = detailed_analysis
-        self._gather_pageinfo(pdf, pageno, infile, check_pages, detailed_analysis)
+        self._gather_pageinfo(
+            pdf, pageno, infile, check_pages, detailed_analysis, miner_state
+        )
 
     def _gather_pageinfo(
         self,
@@ -853,6 +894,7 @@ class PageInfo:
         infile: PathLike,
         check_pages: Container[int],
         detailed_analysis: bool,
+        miner_state: PdfMinerState | None,
     ):
         page: Page = pdf.pages[pageno]
         mediabox = [Decimal(d) for d in page.mediabox.as_list()]
@@ -868,10 +910,11 @@ class PageInfo:
         check_this_page = pageno in check_pages
 
         if check_this_page and detailed_analysis:
-            pscript5_mode = str(pdf.docinfo.get(Name.Creator)).startswith('PScript5')
+            page_analysis = miner_state.get_page_analysis(pageno)
-            miner = get_page_analysis(infile, pageno, pscript5_mode)
+            if page_analysis is not None:
-            if miner is not None:
+                self._textboxes = list(
-                self._textboxes = list(simplify_textboxes(miner, get_text_boxes))
+                    simplify_textboxes(page_analysis, get_text_boxes)
+                )
             else:
                 self._textboxes = []
             bboxes = (box.bbox for box in self._textboxes)
@@ -1055,10 +1098,14 @@ class PageInfo:
 
         Returns None if there is no meaningful DPI for the page.
         """
-        image_dpis = [
+        image_dpis = []
-            image.dpi.to_scalar() for image in self._images if image.renderable
+        image_areas = []
-        ]
+        for image in self._images:
-        image_areas = [image.printed_area for image in self._images if image.renderable]
+            if not image.renderable:
+                continue
+            image_dpis.append(image.dpi.to_scalar())
+            image_areas.append(image.printed_area)
+
         total_drawn_area = sum(image_areas)
         if total_drawn_area == 0:
             return None
@@ -1071,7 +1118,6 @@ class PageInfo:
 
         arg_max_dpi = image_dpis.index(max_dpi)
         max_area_ratio = image_areas[arg_max_dpi] / total_drawn_area
-
         return PageResolutionProfile(
             weighted_dpi,
             max_dpi,
@@ -1121,16 +1167,26 @@ class PdfInfo:
         with Pdf.open(infile) as pdf:
             if pdf.is_encrypted:
                 raise EncryptedPdfError()  # Triggered by encryption with empty passwd
-            self._pages = _pdf_pageinfo_concurrent(
+            pscript5_mode = str(pdf.docinfo.get(Name.Creator, "")).startswith(
-                pdf,
+                'PScript5'
-                executor,
-                max_workers,
-                use_threads,
-                infile,
-                progbar,
-                check_pages=check_pages,
-                detailed_analysis=detailed_analysis,
             )
+            self._miner_state = (
+                PdfMinerState(infile, pscript5_mode)
+                if detailed_analysis
+                else nullcontext()
+            )
+            with self._miner_state as miner_state:
+                self._pages = _pdf_pageinfo_concurrent(
+                    pdf,
+                    executor,
+                    max_workers,
+                    use_threads,
+                    infile,
+                    progbar,
+                    check_pages=check_pages,
+                    detailed_analysis=detailed_analysis,
+                    miner_state=miner_state,
+                )
             self._needs_rendering = pdf.Root.get(Name.NeedsRendering, False)
             if Name.AcroForm in pdf.Root:
                 if len(pdf.Root.AcroForm.get(Name.Fields, [])) > 0:

src/ocrmypdf/pdfinfo/layout.py

@@ -17,6 +17,8 @@ import pdfminer
 import pdfminer.encodingdb
 import pdfminer.pdfdevice
 import pdfminer.pdfinterp
+import pdfminer.psparser
+from deprecation import deprecated
 from pdfminer.converter import PDFLayoutAnalyzer
 from pdfminer.layout import LAParams, LTChar, LTPage, LTTextBox
 from pdfminer.pdfcolor import PDFColorSpace
@@ -58,9 +60,10 @@ def pdfsimplefont__init__(
 
 setattr(PDFSimpleFont, '__init__', pdfsimplefont__init__)
 
-#
+# Patch pdfminer.six buffer size
-# pdfminer patches when creator is PScript5.dll
+# The parser doesn't properly handle keyword tokens are split across the end of the
-#
+# buffer, so increase the buffer size something far larger than will ever be seen.
+pdfminer.psparser.PSBaseParser.BUFSIZ = 256 * 1024 * 1024
 
 
 def pdftype3font__pscript5_get_height(self):
@@ -287,6 +290,7 @@ def patch_pdfminer(pscript5_mode: bool):
         yield
 
 
+@deprecated(deprecated_in='16.6.0', details='Use PdfMinerState instead.')
 def get_page_analysis(
     infile: PathLike, pageno: int, pscript5_mode: bool
 ) -> LTPage | None:
@@ -317,6 +321,73 @@ def get_page_analysis(
     return dev.get_result()
 
 
+class PdfMinerState:
+    """Provide a context manager for using pdfminer.six.
+
+    This ensures that the file is closed. It also provides a cache of pages
+    from the PDF so that they can be reused if needed, to improve performance.
+    """
+
+    def __init__(self, infile: Path, pscript5_mode: bool) -> None:
+        """Initialize the context manager.
+
+        Args:
+            infile: The path to the PDF file to be analyzed.
+            pscript5_mode: Whether the PDF was generated by PScript5.dll.
+        """
+        self.infile = infile
+        self.rman = pdfminer.pdfinterp.PDFResourceManager(caching=True)
+        self.disable_boxes_flow = None
+        self.page_cache: list[PDFPage] = []
+        self.pscript5_mode = pscript5_mode
+        self.file = None
+
+    def __enter__(self):
+        """Enter the context manager."""
+        self.file = Path(self.infile).open('rb')
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        """Exit the context manager."""
+        if self.file:
+            self.file.close()
+        return True
+
+    def _load_page_cache(self):
+        """Load the page cache."""
+        try:
+            self.page_cache = list(PDFPage.get_pages(self.file))
+            if not self.page_cache:
+                raise InputFileError(
+                    "pdfminer did not find any pages in the input file."
+                )
+            for n, page in enumerate(self.page_cache):
+                if page is None:
+                    raise InputFileError(
+                        f"pdfminer could not process page {n} (counting from 0)."
+                    )
+        except PDFTextExtractionNotAllowed as e:
+            raise EncryptedPdfError() from e
+
+    def get_page_analysis(self, pageno: int):
+        """Get the page analysis for a given page."""
+        if not self.page_cache:
+            self._load_page_cache()
+        page = self.page_cache[pageno]
+        dev = TextPositionTracker(
+            self.rman,
+            laparams=LAParams(
+                all_texts=True, detect_vertical=True, boxes_flow=self.disable_boxes_flow
+            ),
+        )
+        interp = pdfminer.pdfinterp.PDFPageInterpreter(self.rman, dev)
+
+        with patch_pdfminer(self.pscript5_mode):
+            interp.process_page(page)
+
+        return dev.get_result()
+
+
 def get_text_boxes(obj) -> Iterator[LTTextBox]:
     """Get the text boxes attached to the current node."""
     for child in obj:

src/ocrmypdf/subprocess/__init__.py

@@ -215,8 +215,10 @@ to have {found_version}.  Please update this program.
 
 OLD_VERSION_REQUIRED_FOR = '''
 OCRmyPDF requires '{program}' {need_version} or higher when run with the
-{required_for} arguments.  If you omit these arguments, OCRmyPDF may be able to
+{required_for} arguments.  {program} {found_version} is installed.
-proceed.  For best results, install the program.
+
+If you omit these arguments, OCRmyPDF may be able to
+proceed.  For best results, update the program.
 '''
 
 OSX_INSTALL_ADVICE = '''

tests/conftest.py

@@ -24,11 +24,6 @@ def is_macos():
     return platform.system() == 'Darwin'
 
 
-def running_in_docker():
-    # Docker creates a file named /.dockerenv in all supported versions
-    return Path('/.dockerenv').exists()
-
-
 def have_unpaper():
     try:
         unpaper.version()

tests/test_annots.py

@@ -0,0 +1,31 @@
+# SPDX-FileCopyrightText: 2024 James R. Barlow
+# SPDX-License-Identifier: MPL-2.0
+
+from __future__ import annotations
+
+import pytest
+from pikepdf import Array, Dictionary, Name, NameTree, Pdf
+
+from ocrmypdf._annots import remove_broken_goto_annotations
+
+
+def test_remove_broken_goto_annotations(resources):
+    with Pdf.open(resources / 'link.pdf') as pdf:
+        assert not remove_broken_goto_annotations(pdf), "File should not be modified"
+
+        # Construct Dests nametree
+        nt = NameTree.new(pdf)
+        names = pdf.Root[Name.Names] = pdf.make_indirect(Dictionary())
+        names[Name.Dests] = nt.obj
+        # Create a broken named destination
+        nt['Invalid'] = pdf.make_indirect(Dictionary())
+        # Create a valid named destination
+        nt['Valid'] = Array([pdf.pages[0].obj, Name.XYZ, 0, 0, 0])
+
+        pdf.pages[0].Annots[0].A.D = 'Missing'
+        pdf.pages[1].Annots[0].A.D = 'Valid'
+
+        assert remove_broken_goto_annotations(pdf), "File should be modified"
+
+        assert Name.D not in pdf.pages[0].Annots[0].A
+        assert Name.D in pdf.pages[1].Annots[0].A

tests/test_api.py

@@ -3,6 +3,7 @@
 
 from __future__ import annotations
 
+import pickle
 from io import BytesIO
 from pathlib import Path
 
@@ -10,6 +11,7 @@ import pytest
 from pdfminer.high_level import extract_text
 
 import ocrmypdf
+import ocrmypdf._pipelines
 import ocrmypdf.api
 
 
@@ -35,7 +37,7 @@ def test_sidecar_stringio(resources: Path, outdir: Path, outpdf: Path):
         resources / 'ccitt.pdf',
         outpdf,
         plugins=['tests/plugins/tesseract_cache.py'],
-        sidecar=s
+        sidecar=s,
     )
     s.seek(0)
     assert b'the' in s.getvalue()
@@ -75,3 +77,29 @@ def test_hocr_to_pdf_api(resources: Path, outdir: Path, outpdf: Path):
     text = extract_text(outpdf)
     assert 'hocr' in text and 'the' not in text
 
+
+def test_hocr_result_json():
+    result = ocrmypdf._pipelines._common.HOCRResult(
+        pageno=1,
+        pdf_page_from_image=Path('a'),
+        hocr=Path('b'),
+        textpdf=Path('c'),
+        orientation_correction=180,
+    )
+    assert (
+        result.to_json()
+        == '{"pageno": 1, "pdf_page_from_image": {"Path": "a"}, "hocr": {"Path": "b"}, '
+        '"textpdf": {"Path": "c"}, "orientation_correction": 180}'
+    )
+    assert ocrmypdf._pipelines._common.HOCRResult.from_json(result.to_json()) == result
+
+
+def test_hocr_result_pickle():
+    result = ocrmypdf._pipelines._common.HOCRResult(
+        pageno=1,
+        pdf_page_from_image=Path('a'),
+        hocr=Path('b'),
+        textpdf=Path('c'),
+        orientation_correction=180,
+    )
+    assert result == pickle.loads(pickle.dumps(result))

tests/test_completion.py

@@ -8,7 +8,7 @@ from subprocess import run
 
 import pytest
 
-from .conftest import running_in_docker
+from ocrmypdf.helpers import running_in_docker
 
 pytestmark = pytest.mark.skipif(
     running_in_docker(),

tests/test_helpers.py

@@ -13,8 +13,7 @@ import pytest
 from packaging.version import Version
 
 from ocrmypdf import helpers
-
+from ocrmypdf.helpers import running_in_docker
-from .conftest import running_in_docker
 
 needs_symlink = pytest.mark.skipif(os.name == 'nt', reason='needs posix symlink')
 windows_only = pytest.mark.skipif(os.name != 'nt', reason="Windows test")

tests/test_main.py

@@ -18,6 +18,7 @@ from PIL import Image
 import ocrmypdf
 from ocrmypdf._exec import tesseract
 from ocrmypdf.exceptions import ExitCode, MissingDependencyError
+from ocrmypdf.helpers import running_in_docker
 from ocrmypdf.pdfa import file_claims_pdfa
 from ocrmypdf.pdfinfo import Colorspace, Encoding, PdfInfo
 from ocrmypdf.subprocess import get_version
@@ -29,7 +30,6 @@ from .conftest import (
     is_macos,
     run_ocrmypdf,
     run_ocrmypdf_api,
-    running_in_docker,
 )
 
 # pylint: disable=redefined-outer-name

tests/test_page_reducing.py

@@ -0,0 +1,2 @@
+import hypothesis
+import pytest

tests/test_pdfinfo.py

@@ -208,12 +208,13 @@ def test_pages_issue700(monkeypatch, resources):
     monkeypatch.setattr(PDFPage, 'get_pages', get_no_pages)
 
     with pytest.raises(InputFileError, match="pdfminer"):
-        pdfinfo.PdfInfo(
+        pi = pdfinfo.PdfInfo(
             resources / 'cardinal.pdf',
             detailed_analysis=True,
             progbar=False,
             max_workers=1,
         )
+        pi._miner_state.get_page_analysis(0)
 
 
 @pytest.fixture

tests/test_rotation.py

@@ -239,7 +239,9 @@ def make_rotate_test(imagefile, outdir, prefix, image_angle, page_angle):
 @pytest.mark.parametrize('image_angle', (0, 90, 180, 270))
 def test_rotate_page_level(image_angle, page_angle, resources, outdir, caplog):
     reference = make_rotate_test(resources / 'typewriter.png', outdir, 'ref', 0, 0)
-    test = make_rotate_test(resources, outdir, 'test', image_angle, page_angle)
+    test = make_rotate_test(
+        resources / 'typewriter.png', outdir, 'test', image_angle, page_angle
+    )
     out = test.with_suffix('.out.pdf')
 
     exitcode = run_ocrmypdf_api(

tests/test_unpaper.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 
 import logging
 from os import fspath
-from unittest.mock import patch
+from unittest.mock import Mock, patch
 
 import pytest
 from packaging.version import Version
@@ -48,6 +48,19 @@ def test_old_unpaper(resources, no_outpdf):
         mock.assert_called()
 
 
+def test_unpaper_version_chatter(resources, no_outpdf):
+    input_ = fspath(resources / "c02-22.pdf")
+    output = fspath(no_outpdf)
+
+    _parser, options, pm = get_parser_options_plugins(["--clean", input_, output])
+    with patch("ocrmypdf.subprocess.run") as mock:
+        mock.return_value = Mock(stdout='Warning: using insecure memory!\n7.0.0\n')
+
+        with pytest.raises(MissingDependencyError):
+            check_options(options, pm)
+        mock.assert_called()
+
+
 @needs_unpaper
 def test_clean(resources, outpdf):
     check_ocrmypdf(