v16.12.0 release notes

Not an ideal fix, but it improves an issue affecting numerous users.

Fixes #1585.

.docker/Dockerfile

@@ -1,7 +1,7 @@
 # SPDX-FileCopyrightText: 2024 James R. Barlow
 # SPDX-License-Identifier: MPL-2.0
 
-FROM ubuntu:24.04 AS base
+FROM ubuntu:25.04 AS base
 
 ENV LANG=C.UTF-8
 ENV TZ=UTC
@@ -40,7 +40,7 @@ RUN \
 WORKDIR /app
 
 # Copy uv from ghcr
-COPY --from=ghcr.io/astral-sh/uv:0.6.14 /uv /uvx /bin/
+COPY --from=ghcr.io/astral-sh/uv:0.9.8 /uv /uvx /bin/
 
 ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy
 

.docker/Dockerfile.alpine

@@ -1,13 +1,7 @@
 # SPDX-FileCopyrightText: 2023 James R. Barlow
 # SPDX-License-Identifier: MPL-2.0
 
-# Note: Alpine 3.20 builds tesseract with --enable-opencl, which is not
-# supported by anyone. OCRmyPDF is not compatible with Alpine 3.20.0
-# through 3.20.3. The issue is fixed in 3.21.
-# Details
-#  https://gitlab.alpinelinux.org/alpine/aports/-/issues/16143
-#  https://github.com/ocrmypdf/OCRmyPDF/issues/1395
-FROM alpine:3.21 AS base
+FROM alpine:3.22 AS base
 
 ENV LANG=C.UTF-8
 ENV TZ=UTC
@@ -28,7 +22,7 @@ RUN apk add --no-cache \
 
 WORKDIR /app
 
-COPY --from=ghcr.io/astral-sh/uv:0.6.14 /uv /uvx /bin/
+COPY --from=ghcr.io/astral-sh/uv:0.9.8 /uv /uvx /bin/
 
 ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy
 

.github/workflows/build.yml

@@ -22,37 +22,35 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-22.04, ubuntu-24.04]
-        python: ["3.10", "3.11", "3.12", "3.13"]
+        python: ["3.10", "3.11", "3.12", "3.13", "3.14"]
         include:
           - os: ubuntu-22.04
             tesseract_ppa: "ppa"
             python: "3.10"
-          - os: ubuntu-24.04
-            python: "pypy3.10"
 
     env:
       OS: ${{ matrix.os }}
       PYTHON: ${{ matrix.python }}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v7
         with:
-          version: "0.5.x"
+          version: "0.9.x"
 
       - name: "Set up Python"
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python }}
 
       - name: Install Tesseract from PPA
         if: matrix.tesseract_ppa == 'ppa'
         run: |
-          sudo add-apt-repository -y ppa:alex-p/tesseract-ocr5.3
+          sudo add-apt-repository -y ppa:alex-p/tesseract-ocr5
 
       - name: Install common packages
         run: |
@@ -74,14 +72,6 @@ jobs:
             unpaper \
             zlib1g
 
-      - name: Install Ubuntu packages for PyPy
-        if: startsWith(matrix.python, 'pypy')
-        run: |
-          sudo apt-get install -y --no-install-recommends \
-            libxml2-dev \
-            libxslt1-dev \
-            pypy3-dev
-
       - name: Install Python packages
         run: |
           uv sync --extra test --no-dev
@@ -111,15 +101,15 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [macos-latest, macos-13]  # macos-latest is arm64, macos-13 is x86_64
-        python: ["3.10", "3.11", "3.12", "3.13"]
+        os: [macos-latest]
+        python: ["3.10", "3.11", "3.12", "3.13", "3.14"]
 
     env:
       OS: ${{ matrix.os }}
       PYTHON: ${{ matrix.python }}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags
 
@@ -136,12 +126,12 @@ jobs:
             tesseract
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v7
         with:
-          version: "0.5.x"
+          version: "0.9.x"
 
       - name: "Set up Python"
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python }}
 
@@ -174,24 +164,24 @@ jobs:
     strategy:
       matrix:
         os: [windows-latest]
-        python: ["3.10", "3.11", "3.12", "3.13"]
+        python: ["3.10", "3.11", "3.12", "3.13", "3.14"]
 
     env:
       OS: ${{ matrix.os }}
       PYTHON: ${{ matrix.python }}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v7
         with:
-          version: "0.5.x"
+          version: "0.9.x"
 
       - name: "Set up Python"
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python }}
 
@@ -220,20 +210,20 @@ jobs:
     name: Build sdist and wheels
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v7
         with:
-          version: "0.5.x"
+          version: "0.9.x"
 
       - name: Make wheels and sdist
         run: |
           uv build --sdist --wheel
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v5
         with:
           name: artifact
           path: |
@@ -246,10 +236,10 @@ jobs:
     runs-on: ubuntu-latest
     environment: release
     permissions:
-      id-token: write  # mandatory for PyPI publishing
+      id-token: write # mandatory for PyPI publishing
     if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v')
     steps:
-      - uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v6
         with:
           name: artifact
           path: dist
@@ -267,13 +257,13 @@ jobs:
       contents: write
       id-token: write
     steps:
-      - uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@v6
         with:
           name: artifact
           path: dist
 
       - name: Sign the dists with Sigstore
-        uses: sigstore/gh-action-sigstore-python@v3.0.0
+        uses: sigstore/gh-action-sigstore-python@v3.1.0
         with:
           inputs: |
             ./dist/*.tar.gz
@@ -318,7 +308,7 @@ jobs:
       - name: Set image name
         run: echo "DOCKER_IMAGE_NAME=ocrmypdf" >> $GITHUB_ENV
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags
 
@@ -366,7 +356,7 @@ jobs:
       - name: Set image name
         run: echo "DOCKER_IMAGE_NAME=ocrmypdf-alpine" >> $GITHUB_ENV
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags
 

.gitignore

@@ -46,4 +46,5 @@ docs/_templates/
 docs/Makefile
 src/ocrmypdf/_version.py
 
-.idea/
+.idea/
+.aider*

README.md

@@ -74,6 +74,7 @@ Linux, Windows, macOS and FreeBSD are supported. Docker images are also availabl
 | macOS (nix)                   | ``nix-env -i ocrmypdf``       |
 | LinuxBrew                     | ``brew install ocrmypdf``     |
 | FreeBSD                       | ``pkg install py-ocrmypdf``   |
+| OpenBSD                       | ``pkg_add ocrmypdf``          |
 | Ubuntu Snap                   | ``snap install ocrmypdf``     |
 
 For everyone else, [see our documentation](https://ocrmypdf.readthedocs.io/en/latest/installation.html) for installation steps.
@@ -92,6 +93,10 @@ apt-get install tesseract-ocr-chi-sim  # Example: Install Chinese Simplified lan
 # Arch Linux users
 pacman -S tesseract-data-eng tesseract-data-deu # Example: Install the English and German language packs
 
+# OpenBSD users
+pkg_info -aQ tesseract  # Display a list of all Tesseract language packs
+pkg_add tesseract-cym  # Example: Install the Welsh language pack
+
 # brew macOS users
 brew install tesseract-lang
 ```

docs/batch.md

@@ -196,6 +196,10 @@ it to a OCRed PDF in `/output/`, and move the processed original to
   - Define environment variable `OCR_DESKEW` to apply deskew to crooked input PDFs
 * - `--env PYTHONBUFFERED=1`
   - This will force `STDOUT` to be unbuffered and allow you to see messages in docker logs
+* - `--env OCR_LOGLEVEL='DEBUG'`
+  - Level of log messages
+* - `--env OCR_JSON_SETTINGS={"language":"deu+eng", "rotate_pages": true}`
+  - A JSON string specifying any other arguments for `ocrmypdf.ocr`
 :::
 
 This service relies on polling to check for changes to the filesystem.

docs/cookbook.md

@@ -221,7 +221,7 @@ works if all you want to is to apply image processing or PDF/A
 conversion.
 
 ```bash
-ocrmypdf --tesseract-timeout=0 --remove-background input.pdf output.pdf
+ocrmypdf --tesseract-timeout 0 --remove-background input.pdf output.pdf
 ```
 
 :::{versionchanged} v14.1.0
@@ -250,7 +250,7 @@ This command also removes OCR generated by third party tools.
 You can also optimize all images without performing any OCR:
 
 ```bash
-ocrmypdf --tesseract-timeout=0 --optimize 3 --skip-text input.pdf output.pdf
+ocrmypdf --tesseract-timeout 0 --optimize 3 --skip-text input.pdf output.pdf
 ```
 
 ### Process only certain pages

docs/plugins.md

@@ -118,7 +118,7 @@ A plugin may provide the following hooks. Hooks must be decorated with
 `ocrmypdf.hookimpl`, for example:
 
 ```python
-from ocrmpydf import hookimpl
+from ocrmypdf import hookimpl
 
 @hookimpl
 def add_options(parser):

docs/release_notes.md

@@ -25,6 +25,47 @@ about a forthcoming release that has not been tagged yet. A release is only
 official when it's tagged and posted to PyPI.
 :::
 
+## v16.12.0
+
+- Disable Ghostscript's subset fonts feature, which was found to corrupt text in
+  certain PDFs. Thanks @mnaegler for identifying this issue. {issue}`1592`
+- Users of Ghostscript 10.6.0+ reported that Ghostscript seems to generate corrupted
+  JPEGs. We force re-optimization of these JPEGs to mitigate the corruption until
+  Ghostscript fixes the issue. {issue}`1585`
+- OCRmyPDF now avoids applying flate compression to large JPEG images, unless maximum
+  optimization is requested, since flate+DCT compression reduces performances in PDF
+  viewers with large images.
+- Updated Dockerfiles to use more recent base operating systems.
+- Updated build and test matrix to include Python 3.14.
+- Minor documentation improvements.
+- pikepdf >= 10.0.0 is now required.
+
+## v16.11.1
+
+- Fixed issue with Tesseract changing an error message related to skew. {issue}`1576`
+- Dropped macOS 13 from build-test matrix since it is no longer supported by Apple.
+
+## v16.11.0
+
+- Deprecated "semfree" plugin in favor of falling back to threads if the platform
+  does not support semaphores. Fixes an issue with Python 3.14.
+- Fixed references to PDF/A compliances levels to be consistent with ISO nomenclature.
+  Thanks @5HT2. {issue}`1557`
+- Fixed an issue around using plugin_manager as an argument. {issue}`1555`
+- Added OpenBSD install steps to README. {issue}`1554`
+- Removed PyPy from test matrix due to declining support in third party libraries.
+- Documentation improvements.
+
+## v16.10.4
+
+- Corrected build errors in Python 3.13.3 and 3.13.4.
+
+## v16.10.3 (not released)
+
+- Blocked optimization of images with pre-blended soft masks. {issue}`1536`
+- Fixed warning from hypothesis on running tests.
+- Release incomplete due to new test failures in Python 3.13.3 and 3.13.4.
+
 ## v16.10.2
 
 - Blacklist pikepdf 9.8.0 due to an incompatible change.

misc/screencast/demo.cast

@@ -60,6 +60,6 @@
 [8.280789, "o", "\rRecompressing JPEGs: 0image [00:00, ?image/s]\rRecompressing JPEGs: 0image [00:00, ?image/s]\r\n\rDeflating JPEGs:   0%|                                                                                    | 0/4 [00:00<?, ?image/s]\rDeflating JPEGs: 100%|███████████████████████████████████████████████████████████████████████████| 4/4 [00:00<00:00, 238.28image/s]\r\n"]
 [8.28149, "o", "\rJBIG2: 0item [00:00, ?item/s]\rJBIG2: 0item [00:00, ?item/s]\r\n"]
 [8.289998, "o", "Image optimization ratio: 1.01 savings: 1.3%\r\nTotal file size ratio: 1.02 savings: 1.6%\r\n"]
-[8.291209, "o", "Output file is a PDF/A-2B (as expected)\r\n"]
+[8.291209, "o", "Output file is a PDF/A-2b (as expected)\r\n"]
 [8.361316, "o", "\u001b[2m⏎\u001b(B\u001b[m                                                                                                                                  \r⏎ \r\u001b[K\u001b[?2004h\u001b]0;fish /home/jb/src/ocrmypdf/tests/resources\u0007\u001b[30m\u001b(B\u001b[m> \u001b[K\r\u001b[C\u001b[C"]
 [8.862206, "o", "\r\n\u001b[30m\u001b(B\u001b[m\u001b[30m\u001b(B\u001b[m\u001b[?2004l"]

misc/watcher.py

@@ -5,8 +5,7 @@
 
 """Watch a directory for new PDFs and OCR them."""
 
-# Do not enable annotations!
-# https://github.com/tiangolo/typer/discussions/598
+from __future__ import annotations
 
 import json
 import logging
@@ -278,11 +277,11 @@ def main(
         f"Output Directory Year & Month: {output_dir_year_month}\n"
         f"Archive Directory: {archive_dir}"
     )
-    log.debug(
+    log.info(
         f"INPUT_DIRECTORY: {input_dir}\n"
         f"OUTPUT_DIRECTORY: {output_dir}\n"
-        f"OUTPUT_DIRECTORY_YEAR_MONTH: {output_dir_year_month}\n"
         f"ARCHIVE_DIRECTORY: {archive_dir}\n"
+        f"OUTPUT_DIRECTORY_YEAR_MONTH: {output_dir_year_month}\n"
         f"ON_SUCCESS_DELETE: {on_success_delete}\n"
         f"ON_SUCCESS_ARCHIVE: {on_success_archive}\n"
         f"DESKEW: {deskew}\n"

pyproject.toml

@@ -16,8 +16,8 @@ dependencies = [
   "img2pdf>=0.5",
   "packaging>=20",
   "pdfminer.six>=20220319",
-  "pi-heif",                   # Heif image format - maintainers: if this is removed, it will NOT break
-  "pikepdf>=8.10.1, != 9.8.0",
+  "pi-heif",                # Heif image format - maintainers: if this is removed, it will NOT break
+  "pikepdf>=10",
   "Pillow>=10.0.1",
   "pluggy>=1",
   "rich>=13",
@@ -102,7 +102,6 @@ exclude_lines = [
 
 [tool.pytest.ini_options]
 minversion = "6.0"
-norecursedirs = ["lib", ".pc", ".git", "venv", "output", "cache", "resources"]
 testpaths = ["tests"]
 addopts = "-n auto"
 markers = ["slow"]

src/ocrmypdf/__main__.py

@@ -78,6 +78,7 @@ def run(args=None):
 
 if __name__ == '__main__':
     multiprocessing.freeze_support()
-    if os.name == 'posix':
-        multiprocessing.set_start_method('forkserver')
+    if sys.platform not in ('win32', 'darwin'):
+        with suppress(RuntimeError):
+            multiprocessing.set_start_method('forkserver')
     sys.exit(run())

src/ocrmypdf/_exec/ghostscript.py

@@ -286,6 +286,7 @@ def generate_pdfa(
         + compression_args
         + [
             "-dJPEGQ=95",
+            "-dSubsetFonts=false",  # Prevents GS from messing up some encodings
             f"-dPDFA={pdfa_part}",
             "-dPDFACompatibilityPolicy=1",
             "-o",

src/ocrmypdf/_exec/tesseract.py

@@ -186,6 +186,18 @@ def get_orientation(
     return orient_conf
 
 
+def _is_empty_page_error(exc):
+    if b'Empty page!!' in exc.output:  # Tesseract 4.x
+        return True
+
+    return exc.returncode == 1 and (
+        # Tesseract 5.0-5.4 or so
+        exc.output == b''
+        # Tesseract 5.5+
+        or exc.output.startswith(b"Error in boxClipToRectangle: box outside rectangle")
+    )
+
+
 def get_deskew(
     input_file: Path, languages: list[str], engine_mode: int | None, timeout: float
 ) -> float:
@@ -204,11 +216,9 @@ def get_deskew(
     except CalledProcessError as e:
         tesseract_log_output(e.stdout)
         tesseract_log_output(e.stderr)
-        if b'Empty page!!' in e.output or (
-            e.output == b'' and e.returncode == 1
-        ):  # Not enough info for a skew angle - Tess 4 and 5 return different errors
+        if _is_empty_page_error(e):
+            # Not enough info for a skew angle
             return 0.0
-
         raise SubprocessOutputError() from e
 
     parsed = _parse_tesseract_output(p.stdout)

src/ocrmypdf/_plugin_manager.py

@@ -73,19 +73,10 @@ class OcrmypdfPluginManager(pluggy.PluginManager):
                 module = importlib.import_module(name)
                 self.register(module)
 
-        # 2. Install semfree if needed
-        try:
-            # pylint: disable=import-outside-toplevel
-            from multiprocessing.synchronize import SemLock
-
-            del SemLock
-        except ImportError:
-            self.register(importlib.import_module('ocrmypdf.extra_plugins.semfree'))
-
-        # 3. Register setuptools plugins
+        # 2. Register setuptools plugins
         self.load_setuptools_entrypoints('ocrmypdf')
 
-        # 4. Register plugins specified on command line
+        # 3. Register plugins specified on command line
         for name in self.__plugins:
             if isinstance(name, Path) or name.endswith('.py'):
                 # Import by filename

src/ocrmypdf/api.py

@@ -1,7 +1,41 @@
 # SPDX-FileCopyrightText: 2022 James R. Barlow
 # SPDX-License-Identifier: MPL-2.0
 
-"""Functions for using ocrmypdf as an API."""
+"""Python API for OCRmyPDF.
+
+This module provides the main Python API for OCRmyPDF, allowing you to perform
+OCR operations programmatically without using the command line interface.
+
+Main Functions:
+    ocr(): The primary function for OCR processing. Takes an input PDF or image
+        file and produces an OCR'd PDF with searchable text.
+
+    configure_logging(): Set up logging to match the command line interface
+        behavior, with support for progress bars and colored output.
+
+Experimental Functions:
+    _pdf_to_hocr(): Extract text from PDF pages and save as hOCR files for
+        manual editing before final PDF generation.
+
+    _hocr_to_ocr_pdf(): Convert hOCR files back to a searchable PDF after
+        manual text corrections.
+
+The API maintains thread safety through internal locking since OCRmyPDF uses
+global state for plugins. Only one OCR operation can run per Python process
+at a time. For parallel processing, use multiple Python processes.
+
+Example:
+    import ocrmypdf
+
+    # Configure logging (optional)
+    ocrmypdf.configure_logging(ocrmypdf.Verbosity.default)
+
+    # Perform OCR
+    ocrmypdf.ocr('input.pdf', 'output.pdf', language='eng')
+
+For detailed parameter documentation, see the ocr() function docstring and
+the equivalent command line parameters in the OCRmyPDF documentation.
+"""
 
 from __future__ import annotations
 
@@ -357,7 +391,7 @@ def ocr(  # noqa: D417
     create_options_kwargs = {
         k: v
         for k, v in locals().items()
-        if k not in {'input_file', 'output_file', 'kwargs'}
+        if k not in {'input_file', 'output_file', 'kwargs', 'plugin_manager'}
     }
     create_options_kwargs.update(kwargs)
 

src/ocrmypdf/builtin_plugins/concurrency.py

@@ -96,6 +96,31 @@ def thread_init(q: Queue, user_init: UserInit, loglevel) -> None:
     return
 
 
+def setup_executor(use_threads: bool) -> tuple[Queue, Executor, WorkerInit]:
+    if not use_threads:
+        # Some execution environments like AWS Lambda and Termux do not support
+        # semaphores. Check if semaphore support is available, and if not, fall back
+        # to using threads.
+        try:
+            # pylint: disable=import-outside-toplevel
+            from multiprocessing.synchronize import SemLock
+
+            del SemLock
+        except ImportError:
+            use_threads = True
+
+    if use_threads:
+        loq_queue = queue.Queue(-1)
+        executor_class = ThreadPoolExecutor
+        initializer = thread_init
+    else:
+        loq_queue = multiprocessing.Queue(-1)
+        executor_class = ProcessPoolExecutor
+        initializer = process_init
+
+    return loq_queue, executor_class, initializer
+
+
 class StandardExecutor(Executor):
     """Standard OCRmyPDF concurrent task executor."""
 
@@ -110,14 +135,7 @@ class StandardExecutor(Executor):
         task_arguments: Iterable,
         task_finished: Callable,
     ):
-        if use_threads:
-            log_queue: Queue = queue.Queue(-1)
-            executor_class: FuturesExecutorClass = ThreadPoolExecutor
-            initializer: WorkerInit = thread_init
-        else:
-            log_queue = multiprocessing.Queue(-1)
-            executor_class = ProcessPoolExecutor
-            initializer = process_init
+        log_queue, executor_class, initializer = setup_executor(use_threads)
 
         # Regardless of whether we use_threads for worker processes, the log_listener
         # must be a thread. Make sure we create the listener after the worker pool,

src/ocrmypdf/cli.py

@@ -196,8 +196,8 @@ Online documentation is located at:
         "for users who want their file altered as little as possible. 'pdfa' "
         "also has problems with full Unicode text. 'pdf' minimizes changes "
         "to the input file. 'pdf-a1' creates a "
-        "PDF/A1-b file. 'pdf-a2' is equivalent to 'pdfa'. 'pdf-a3' creates a "
-        "PDF/A3-b file. 'none' will produce no output, which may be helpful if "
+        "PDF/A-1b file. 'pdf-a2' is equivalent to 'pdfa'. 'pdf-a3' creates a "
+        "PDF/A-3b file. 'none' will produce no output, which may be helpful if "
         "only the --sidecar is desired.",
     )
 

src/ocrmypdf/extra_plugins/__init__.py

@@ -2,8 +2,4 @@
 #
 # SPDX-License-Identifier: MPL-2.0
 
-"""Extra plugins. These are not automatically inserted when ocrmypdf is run.
-
-You can use these plugins by specifying them on the command line, e.g.:
-ocrmypdf --plugin ocrmypdf.extra_plugins.semfree ...
-"""
+"""Extra plugins. These are not automatically inserted when ocrmypdf is run."""

src/ocrmypdf/extra_plugins/semfree.py

@@ -13,6 +13,9 @@ worker communicates only with the main process.
 This is not without drawbacks. If the tasks are not "even" in size, which cannot
 be guaranteed, some workers may end up with too much work while others are idle.
 It is less efficient than the standard implementation, so not the default.
+
+This module is deprecated and will be removed in a future release. The standard
+executor will fall back to threads in these environments.
 """
 
 from __future__ import annotations
@@ -20,6 +23,7 @@ from __future__ import annotations
 import logging
 import logging.handlers
 import signal
+import warnings
 from collections.abc import Callable, Iterable, Iterator
 from contextlib import suppress
 from enum import Enum, auto
@@ -32,6 +36,11 @@ from ocrmypdf._concurrent import NullProgressBar
 from ocrmypdf.exceptions import InputFileError
 from ocrmypdf.helpers import remove_all_log_handlers
 
+warnings.warn(
+    "semfree.py is deprecated and will be removed in a future release.",
+    DeprecationWarning,
+)
+
 
 class MessageType(Enum):
     """Implement basic IPC messaging."""

src/ocrmypdf/helpers.py

@@ -270,7 +270,7 @@ def check_pdf(input_file: Path) -> bool:
         with pdf:
             with warnings.catch_warnings():
                 warnings.filterwarnings('ignore', message=r'pikepdf.*JBIG2.*')
-                messages = pdf.check()
+                messages = pdf.check_pdf_syntax()
             success = True
             for msg in messages:
                 if 'error' in msg.lower():

src/ocrmypdf/optimize.py

@@ -17,6 +17,7 @@ from typing import Any, NamedTuple, NewType
 from zlib import compress
 
 import img2pdf
+from packaging.version import Version
 from pikepdf import (
     Dictionary,
     Name,
@@ -32,7 +33,7 @@ from pikepdf.models.image import HifiPrintImageNotTranscodableError
 from PIL import Image
 
 from ocrmypdf._concurrent import Executor, SerialExecutor
-from ocrmypdf._exec import jbig2enc, pngquant
+from ocrmypdf._exec import ghostscript, jbig2enc, pngquant
 from ocrmypdf._jobcontext import PdfContext
 from ocrmypdf._progressbar import ProgressBar
 from ocrmypdf.exceptions import OutputFileAccessError
@@ -42,6 +43,7 @@ log = logging.getLogger(__name__)
 
 DEFAULT_JPEG_QUALITY = 75
 DEFAULT_PNG_QUALITY = 70
+FLATE_JPEG_THRESHOLD = 10000
 
 
 Xref = NewType('Xref', int)
@@ -126,6 +128,13 @@ def extract_image_filter(
     if Name.Decode in image:
         log.debug(f"xref {xref}: skipping image with Decode table")
         return None  # Don't mess with custom Decode tables
+    if image.get(Name.SMask, Dictionary()).get(Name.Matte, None) is not None:
+        # https://github.com/ocrmypdf/OCRmyPDF/issues/1536
+        # Do not attempt to optimize images that have a SMask with a Matte.
+        # That means alpha channel pre-blending is used, and we're not prepared
+        # to deal with the complexities of that.
+        log.debug(f"xref {xref}: skipping image whose SMask has Matte")
+        return None
 
     return pim, filtdp
 
@@ -182,6 +191,16 @@ def extract_image_jbig2(
     return None
 
 
+def _should_optimize_jpeg(options, filtdp):
+    if options.optimize >= 2:
+        return True
+    if options.optimize < 2 and ghostscript.version() >= Version('10.6.0'):
+        # Ghostscript 10.6.0+ introduced some sort of JPEG encoding issue.
+        # To resolve this, re-optimize the JPEG anyway.
+        return True
+    return False
+
+
 def extract_image_generic(
     *, pdf: Pdf, root: Path, image: Stream, xref: Xref, options
 ) -> XrefExt | None:
@@ -195,15 +214,7 @@ def extract_image_generic(
     if pim.bits_per_component == 1:
         return None
 
-    if filtdp[0] == Name.DCTDecode and options.optimize >= 2:
-        # This is a simple heuristic derived from some training data, that has
-        # about a 70% chance of guessing whether the JPEG is high quality,
-        # and possibly recompressible, or not. The number itself doesn't mean
-        # anything.
-        # bytes_per_pixel = int(raw_jpeg.Length) / (w * h)
-        # jpeg_quality_estimate = 117.0 * (bytes_per_pixel ** 0.213)
-        # if jpeg_quality_estimate < 65:
-        #     return None
+    if filtdp[0] == Name.DCTDecode and _should_optimize_jpeg(options, filtdp):
         try:
             imgname = root / f'{xref:08d}'
             with imgname.open('wb') as f:
@@ -521,7 +532,19 @@ def _find_deflatable_jpeg(
         return None
     _pim, filtdp = result
 
-    if filtdp[0] == Name.DCTDecode and not filtdp[1] and options.optimize >= 1:
+    if (
+        filtdp[0] == Name.DCTDecode
+        and not filtdp[1]
+        and (
+            (
+                # Don't flate very large images because it will slow down PDF viewers
+                1 <= options.optimize <= 2
+                and image.get(Name.Width, 0) < FLATE_JPEG_THRESHOLD
+                and image.get(Name.Height, 0) < FLATE_JPEG_THRESHOLD
+            )
+            or options.optimize == 3
+        )
+    ):
         return XrefExt(xref, '.memory')
 
     return None

src/ocrmypdf/pdfa.py

@@ -120,10 +120,13 @@ def file_claims_pdfa(filename: Path):
                 'output': 'pdf',
                 'conformance': 'No PDF/A metadata in XMP',
             }
-        valid_part_conforms = {'1A', '1B', '2A', '2B', '2U', '3A', '3B', '3U'}
-        conformance = f'PDF/A-{pdfmeta.pdfa_status}'
+        valid_part_conforms = {'1a', '1b', '2a', '2b', '2u', '3a', '3b', '3u'}
+        # Raw value in XMP metadata returned by pikepdf is uppercase, but ISO
+        # uses lower case for conformance levels.
+        pdfa_status_iso = pdfmeta.pdfa_status.lower()
+        conformance = f'PDF/A-{pdfa_status_iso}'
         pdfa_dict: dict[str, str | bool] = {}
-        if pdfmeta.pdfa_status in valid_part_conforms:
+        if pdfa_status_iso in valid_part_conforms:
             pdfa_dict['pass'] = True
             pdfa_dict['output'] = 'pdfa'
         pdfa_dict['conformance'] = conformance

src/ocrmypdf/pluginspec.py

@@ -492,7 +492,7 @@ def generate_pdfa(
         pdf_version: The minimum PDF version that the output file should be.
             At its own discretion, the PDF/A generator may raise the version,
             but should not lower it.
-        pdfa_part: The desired PDF/A compliance level, such as ``'2B'``.
+        pdfa_part: The desired PDF/A compliance level, such as ``'2b'``.
         progressbar_class: The class of a progress bar, which must implement
             the ProgressBar protocol. If None, no progress is reported.
         stop_on_soft_error: If there is an "soft error" such that PDF/A generation

tests/test_ghostscript.py

@@ -6,6 +6,7 @@ from __future__ import annotations
 import logging
 import secrets
 import subprocess
+import sys
 from decimal import Decimal
 from unittest.mock import patch
 
@@ -165,6 +166,10 @@ class TestDuplicateFilter:
         logger.addFilter(DuplicateFilter(logger))
         return logger
 
+    @pytest.mark.xfail(
+        (3, 13, 3) <= sys.version_info[:3] <= (3, 13, 5),
+        reason="https://github.com/python/cpython/pull/135858",
+    )
     def test_filter_duplicate_messages(self, duplicate_filter_logger, caplog):
         log = duplicate_filter_logger
         log.error("test error message")
@@ -194,6 +199,10 @@ class TestDuplicateFilter:
         assert caplog.records[1].msg == "another error message"
         assert caplog.records[2].msg == "yet another error message"
 
+    @pytest.mark.xfail(
+        (3, 13, 3) <= sys.version_info[:3] <= (3, 13, 5),
+        reason="https://github.com/python/cpython/pull/135858",
+    )
     def test_filter_alt_messages(self, duplicate_filter_logger, caplog):
         log = duplicate_filter_logger
         log.error("test error message")

tests/test_main.py

@@ -801,7 +801,7 @@ def test_pdfa_n(pdfa_level, resources, outpdf):
     )
 
     pdfa_info = file_claims_pdfa(outpdf)
-    assert pdfa_info['conformance'] == f'PDF/A-{pdfa_level}B'
+    assert pdfa_info['conformance'] == f'PDF/A-{pdfa_level}b'
 
 
 def test_decompression_bomb_error(resources, outpdf, caplog):

tests/test_optimize.py

@@ -6,7 +6,7 @@ from __future__ import annotations
 from io import BytesIO
 from os import fspath
 from pathlib import Path
-from unittest.mock import MagicMock, patch
+from unittest.mock import patch
 
 import img2pdf
 import pikepdf
@@ -220,7 +220,7 @@ def test_find_formx(resources):
 
 
 def test_extract_image_filter_with_pdf_image():
-    image = MagicMock()
+    image = Dictionary()
     image.Subtype = Name.Image
     image.Length = 200
     image.Width = 10
@@ -235,20 +235,20 @@ def test_extract_image_filter_with_pdf_image():
 
 
 def test_extract_image_filter_with_non_image():
-    image = MagicMock()
+    image = Dictionary()
     image.Subtype = Name.Form
     assert extract_image_filter(image, None) is None
 
 
 def test_extract_image_filter_with_small_stream_size():
-    image = MagicMock()
+    image = Dictionary()
     image.Subtype = Name.Image
     image.Length = 50
     assert extract_image_filter(image, None) is None
 
 
 def test_extract_image_filter_with_small_dimensions():
-    image = MagicMock()
+    image = Dictionary()
     image.Subtype = Name.Image
     image.Length = 200
     image.Width = 5
@@ -257,7 +257,7 @@ def test_extract_image_filter_with_small_dimensions():
 
 
 def test_extract_image_filter_with_multiple_compression_filters():
-    image = MagicMock()
+    image = Dictionary()
     image.Subtype = Name.Image
     image.Length = 200
     image.Width = 10
@@ -268,7 +268,7 @@ def test_extract_image_filter_with_multiple_compression_filters():
 
 
 def test_extract_image_filter_with_wide_gamut_image():
-    image = MagicMock()
+    image = Dictionary()
     image.Subtype = Name.Image
     image.Length = 200
     image.Width = 10
@@ -296,7 +296,7 @@ def test_extract_image_filter_with_jpeg2000_image():
 
 
 def test_extract_image_filter_with_ccitt_group_3_image():
-    image = MagicMock()
+    image = Dictionary()
     image.Subtype = Name.Image
     image.Length = 200
     image.Width = 10
@@ -309,7 +309,7 @@ def test_extract_image_filter_with_ccitt_group_3_image():
 
 # Triggers pikepdf bug
 # def test_extract_image_filter_with_decode_table():
-#     image = MagicMock()
+#     image = Dictionary()
 #     image.Subtype = Name.Image
 #     image.Length = 200
 #     image.Width = 10
@@ -319,3 +319,26 @@ def test_extract_image_filter_with_ccitt_group_3_image():
 #     image.ColorSpace = Name.DeviceGray
 #     image.Decode = [42, 0]
 #     assert extract_image_filter(image, None) is None
+
+
+def test_extract_image_filter_with_rgb_smask_matte():
+    image = Dictionary()
+    image.Subtype = Name.Image
+    image.Length = 200
+    image.Width = 10
+    image.Height = 10
+    image.Filter = Name.FlateDecode
+    image.BitsPerComponent = 8
+    image.ColorSpace = Name.DeviceRGB
+    image.SMask = Dictionary(
+        Type=Name.Image,
+        Subtype=Name.Image,
+        Length=200,
+        Width=10,
+        Height=10,
+        Filter=Name.FlateDecode,
+        BitsPerComponent=8,
+        ColorSpace=Name.DeviceGray,
+        Matte=Array([1, 2, 3]),
+    )
+    assert extract_image_filter(image, None) is None

tests/test_semfree.py

@@ -3,6 +3,8 @@
 
 from __future__ import annotations
 
+import sys
+
 import pytest
 
 from ocrmypdf.exceptions import ExitCode
@@ -11,16 +13,21 @@ from .conftest import is_linux, run_ocrmypdf_api
 
 
 @pytest.mark.skipif(not is_linux(), reason='semfree plugin only works on Linux')
+@pytest.mark.skipif(
+    sys.version_info >= (3, 14),
+    reason='semfree plugin only works on Python 3.13 or earlier',
+)
 def test_semfree(resources, outpdf):
-    exitcode = run_ocrmypdf_api(
-        resources / 'multipage.pdf',
-        outpdf,
-        '--skip-text',
-        '--skip-big',
-        '2',
-        '--plugin',
-        'ocrmypdf.extra_plugins.semfree',
-        '--plugin',
-        'tests/plugins/tesseract_noop.py',
-    )
-    assert exitcode in (ExitCode.ok, ExitCode.pdfa_conversion_failed)
+    with pytest.warns(DeprecationWarning, match="semfree.py is deprecated"):
+        exitcode = run_ocrmypdf_api(
+            resources / 'multipage.pdf',
+            outpdf,
+            '--skip-text',
+            '--skip-big',
+            '2',
+            '--plugin',
+            'ocrmypdf.extra_plugins.semfree',
+            '--plugin',
+            'tests/plugins/tesseract_noop.py',
+        )
+        assert exitcode in (ExitCode.ok, ExitCode.pdfa_conversion_failed)

tests/test_stdio.py

@@ -48,6 +48,7 @@ def test_stdout(ocrmypdf_exec, resources, outpdf):
     assert check_pdf(output_file)
 
 
+@pytest.mark.skipif(os.name == 'nt', reason='Windows does not support /dev/null')
 def test_dev_null(resources):
     if 'COV_CORE_DATAFILE' in os.environ:
         pytest.skip("Coverage uses stdout")

tests/test_watcher.py

@@ -0,0 +1,54 @@
+import datetime
+import os
+import shutil
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+import pytest
+
+watchdog = pytest.importorskip('watchdog')
+
+
+@pytest.mark.parametrize('year_month', [True, False])
+def test_watcher(tmp_path, resources, year_month):
+    input_dir = tmp_path / 'input'
+    input_dir.mkdir()
+    output_dir = tmp_path / 'output'
+    output_dir.mkdir()
+    processed_dir = tmp_path / 'processed'
+    processed_dir.mkdir()
+
+    if year_month:
+        env_extra = {'OCR_OUTPUT_DIRECTORY_YEAR_MONTH': '1'}
+    else:
+        env_extra = {}
+    proc = subprocess.Popen(
+        [
+            sys.executable,
+            Path(__file__).parent.parent / 'misc' / 'watcher.py',
+            str(input_dir),
+            str(output_dir),
+            str(processed_dir),
+        ],
+        cwd=str(tmp_path),
+        env=os.environ.copy() | env_extra,
+    )
+    time.sleep(5)
+
+    shutil.copy(resources / 'trivial.pdf', input_dir / 'trivial.pdf')
+    time.sleep(5)
+
+    if year_month:
+        assert (
+            output_dir
+            / f'{datetime.date.today().year}'
+            / f'{datetime.date.today().month:02d}'
+            / 'trivial.pdf'
+        ).exists()
+    else:
+        assert (output_dir / 'trivial.pdf').exists()
+
+    proc.terminate()
+    proc.wait()