v16.2.0 release notes

Fixes #1262

.docker/Dockerfile

@@ -64,6 +64,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   img2pdf \
   libsm6 libxext6 libxrender-dev \
   pngquant \
+  python-is-python3 \
   tesseract-ocr \
   tesseract-ocr-chi-sim \
   tesseract-ocr-deu \

.docker/Dockerfile.alpine

@@ -1,7 +1,7 @@
 # SPDX-FileCopyrightText: 2023 James R. Barlow
 # SPDX-License-Identifier: MPL-2.0
 
-FROM alpine:3.18 as base
+FROM alpine:3.19 as base
 
 ENV LANG=C.UTF-8
 ENV TZ=UTC

.github/workflows/build.yml

@@ -32,8 +32,8 @@ jobs:
           - os: ubuntu-latest
             python: "3.12"
             tesseract5: true
-          # - os: ubuntu-latest
-          #   python: "pypy3.10"
+          - os: ubuntu-latest
+            python: "pypy3.10"
 
     env:
       OS: ${{ matrix.os }}

README.md

@@ -70,6 +70,7 @@ Linux, Windows, macOS and FreeBSD are supported. Docker images are also availabl
 | Windows Subsystem for Linux   | ``apt install ocrmypdf``      |
 | Fedora                        | ``dnf install ocrmypdf``      |
 | macOS (Homebrew)              | ``brew install ocrmypdf``     |
+| macOS (MacPorts)              | ``port install ocrmypdf``     |
 | macOS (nix)                   | ``nix-env -i ocrmypdf``       |
 | LinuxBrew                     | ``brew install ocrmypdf``     |
 | FreeBSD                       | ``pkg install py-ocrmypdf``   |

docs/advanced.rst

@@ -118,15 +118,16 @@ OCR for huge images
 -------------------
 
 Tesseract has internal limits on the size
-of images it will process. If you issue
-``--tesseract-downsample-large-images``, OCRmyPDF will downsample images
-to fit Tesseract limits. (The limits are usually entered only for scanned
-images of oversized media, such as large maps or blueprints exceeding
-110 cm or 43 inches in either dimension, and at high DPI.)
+of images it will process. By default,
+``--tesseract-downsample-large-images`` is enabled, and OCRmyPDF will
+downsample images to fit Tesseract limits. (The limits are usually encountered
+only for scanned images of oversized media, such as large maps or blueprints exceeding
+110 cm or 43 inches in either dimension, and at high DPI.) This feature can disabled
+using ``--no-tesseract-downsample-large-images``.
 
 ``--tesseract-downsample-above Npixels`` adjusts the threshold at which images
 will be downsampled. By default, only images that exceed any of Tesseract's
-internal limits are downsampled.
+internal limits are downsampled (32767 pixels on either dimension).
 
 You will also need to set ``--tesseract-timeout`` high enough to allow
 for processing.

docs/installation.rst

@@ -23,7 +23,9 @@ These platforms have one-liner installs:
 +-------------------------------+-----------------------------------------+
 | Fedora                        | ``dnf install ocrmypdf tesseract-osd``  |
 +-------------------------------+-----------------------------------------+
-| macOS                         | ``brew install ocrmypdf``               |
+| macOS (Homebrew)              | ``brew install ocrmypdf``               |
++-------------------------------+-----------------------------------------+
+| macOS (MacPorts)              | ``port install ocrmypdf``               |
 +-------------------------------+-----------------------------------------+
 | LinuxBrew                     | ``brew install ocrmypdf``               |
 +-------------------------------+-----------------------------------------+
@@ -99,12 +101,12 @@ For full details on version availability for your platform, check the
 Fedora
 ------
 
-.. |fedora-37| image:: https://repology.org/badge/version-for-repo/fedora_37/ocrmypdf.svg
-    :alt: Fedora 37
-
 .. |fedora-38| image:: https://repology.org/badge/version-for-repo/fedora_38/ocrmypdf.svg
     :alt: Fedora 38
 
+.. |fedora-39| image:: https://repology.org/badge/version-for-repo/fedora_39/ocrmypdf.svg
+    :alt: Fedora 39
+
 .. |fedora-rawhide| image:: https://repology.org/badge/version-for-repo/fedora_rawhide/ocrmypdf.svg
     :alt: Fedore Rawhide
 
@@ -113,7 +115,7 @@ Fedora
 +-----------------------------------------------+
 | |latest|                                      |
 +-----------------------------------------------+
-| |fedora-37| |fedora-38| |fedora-rawhide|      |
+| |fedora-38| |fedora-39| |fedora-rawhide|      |
 +-----------------------------------------------+
 
 Users of Fedora may simply
@@ -123,7 +125,7 @@ Users of Fedora may simply
     dnf install ocrmypdf tesseract-osd
 
 For full details on version availability, check the `Fedora Package
-Tracker <https://apps.fedoraproject.org/packages/ocrmypdf>`__.
+Tracker <https://packages.fedoraproject.org/pkgs/ocrmypdf/ocrmypdf/>`__.
 
 If the version available for your platform is out of date, you could opt
 to install the latest version from source. See `Installing HEAD revision
@@ -135,7 +137,7 @@ from sources <#installing-head-revision-from-sources>`__.
    issues. OCRmyPDF works fine without it but will produce larger output
    files. If you build jbig2enc from source, ocrmypdf 7.0.0 and later
    will automatically detect it on the ``PATH``. To add JBIG2 encoding,
-   see `Installing the JBIG2 encoder <jbig2>`__.
+   see :ref:`Installing the JBIG2 encoder <jbig2>`.
 
 .. _ubuntu-lts-latest:
 
@@ -160,7 +162,7 @@ and build ocrmypdf in virtual environment:
 
     python3.11 -m venv .venv
 
-To add JBIG2 encoding, see `Installing the JBIG2 encoder <jbig2>`__.
+To add JBIG2 encoding, see :ref:`Installing the JBIG2 encoder <jbig2>`.
 
 Note Fedora packages for language data haven't been branched for RHEL/EPEL, but you can get traineddata files directly from `tesseract
 <https://github.com/tesseract-ocr/tessdata/>`__ and place them in ``/usr/share/tesseract/tessdata``.
@@ -217,7 +219,7 @@ you are using a VM image, such as `the official Vagrant image
 be completed for you.
 
 Next you should install the `base-devel package group
-<https://www.archlinux.org/groups/x86_64/base-devel/>`__. This includes the
+<https://archlinux.org/packages/core/any/base-devel/>`__. This includes the
 standard tooling needed to build packages, such as a compiler and binary tools.
 
 .. code-block:: bash
@@ -260,7 +262,7 @@ page.
     <https://aur.archlinux.org/packages/jbig2enc-git/>`__ and may be installed
     using the same series of steps as for the installation OCRmyPDF AUR
     package. Alternatively, it may be built manually from source following the
-    instructions in `Installing the JBIG2 encoder <jbig2>`__.  If JBIG2 is
+    instructions in :ref:`Installing the JBIG2 encoder <jbig2>`.  If JBIG2 is
     installed, OCRmyPDF 7.0.0 and later will automatically detect it.
 
 Alpine Linux
@@ -325,6 +327,22 @@ languages you can optionally install them all:
 
     brew install tesseract-lang  # Optional: Install all language packs
 
+MacPorts
+--------
+
+.. image:: https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fports.macports.org%2Fapi%2Fv1%2Fports%2Focrmypdf%2F%3Fformat%3Djson&query=version&label=MacPorts
+   :alt: Macports Version Information
+   :target: https://ports.macports.org/port/ocrmypdf
+
+OCRmyPDF is includes in MacPorts:
+
+.. code-block:: bash
+
+    sudo port install ocrmypdf
+
+Note that while this will install tesseract you will need to install
+the appropriate tesseract `language ports <https://ports.macports.org/search/?selected_facets=categories_exact%3Atextproc&installed_file=&q=tesseract&name=on>`__. 
+
 Manual installation on macOS
 ----------------------------
 
@@ -623,7 +641,7 @@ environment:
     pip install git+https://github.com/ocrmypdf/OCRmyPDF.git
 
 Or, to install in `development
-mode <https://pythonhosted.org/setuptools/setuptools.html#development-mode>`__,
+mode <https://packaging.python.org/en/latest/guides/distributing-packages-using-setuptools/#working-in-development-mode>`__,
 allowing customization of OCRmyPDF, use the ``-e`` flag:
 
 .. code-block:: bash

docs/maintainers.rst

@@ -64,4 +64,13 @@ installation documentation.
 
 If you maintain a Linux distribution that supports 32-bit x86 or ARM, OCRmyPDF
 should continue to work as long as all of its dependencies continue to be
-available in 32-bit form. Please note we do not test on 32-bit platforms.
+available in 32-bit form. Please note we do not test on 32-bit platforms.
+
+HEIF/HEIC
+---------
+
+OCRmyPDF defaults to installing the pi-heif PyPI package, which supports converting
+HEIF (High Efficiency Image File Format) images to PDF from the command line.
+If your distribution does not have this library available, you can exclude it and
+OCRmyPDF will gracefully degrade automatically, losing only support for this
+feature.

docs/release_notes.rst

@@ -30,6 +30,36 @@ OCRmyPDF typically supports the three most recent Python versions.
 
 .. |OCRmyPDF PyPI| image:: https://img.shields.io/pypi/v/ocrmypdf.svg
 
+
+v16.2.0
+=======
+
+-  Fixed issue 'NoneType' object has no attribute 'get' when optimizing certain PDFs.
+   :issue:`1293,1271`
+-  Switched formatting from black to ruff.
+-  Added support for sending sidecar output to io.BytesIO.
+-  Added support for converting HEIF/HEIC images (the native image of iPhones and
+   some other devices) to PDFs, when the appropriate pi-hief library is installed.
+   This library is marked as a dependency, but maintainers may opt out if needed.
+-  We now default to downsampling large images that would exceed Tesseract's internal
+   limits, but only if it cause processing to fail. Previously, this behavior only
+   occurred if specifically requested on command line. It can still be configured
+   and disabled. See the --tesseract command line options.
+-  Added Macports install instructions. Thanks @akierig.
+-  Improved logging output when an unexpected error occurs while trying to obtain
+   the version of a third party program.
+
+v16.1.2
+=======
+
+-  Fixed test suite failure when using Ghostscript 10.3.
+-  Other minor corrections.
+
+v16.1.1
+=======
+
+-  Fixed PyPy 3.10 support.
+
 v16.1.0
 =======
 

misc/batch.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 # SPDX-FileCopyrightText: 2016 findingorder <https://github.com/findingorder>
+# SPDX-FileCopyrightText: 2024 nilsro <https://github.com/nilsro>
 # SPDX-License-Identifier: MIT
 
 """Example of using ocrmypdf as a library in a script.
@@ -15,6 +16,10 @@ from __future__ import annotations
 
 import logging
 import sys
+import os
+import posixpath
+import shutil
+import filecmp
 from pathlib import Path
 
 import ocrmypdf
@@ -22,32 +27,65 @@ import ocrmypdf
 # pylint: disable=logging-format-interpolation
 # pylint: disable=logging-not-lazy
 
+
+def filecompare(a, b):
+    try:
+        return filecmp.cmp(a, b, shallow=True)
+    except FileNotFoundError:
+        return False
+
+
 script_dir = Path(__file__).parent
+# set archive_dir to a path for backup original documents. Leave empty if not required.
+archive_dir = "/pdfbak"
 
 if len(sys.argv) > 1:
     start_dir = Path(sys.argv[1])
 else:
-    start_dir = Path('.')
+    start_dir = Path(".")
 
 if len(sys.argv) > 2:
     log_file = Path(sys.argv[2])
 else:
-    log_file = script_dir.with_name('ocr-tree.log')
+    log_file = script_dir.with_name("ocr-tree.log")
 
 logging.basicConfig(
     level=logging.INFO,
-    format='%(asctime)s %(message)s',
+    format="%(asctime)s %(message)s",
     filename=log_file,
-    filemode='a',
+    filemode="a",
 )
 
+logging.info(f"Start directory {start_dir}")
+
 ocrmypdf.configure_logging(ocrmypdf.Verbosity.default)
 
-for filename in start_dir.glob("**/*.py"):
+for filename in start_dir.glob("**/*.pdf"):
     logging.info(f"Processing {filename}")
-    result = ocrmypdf.ocr(filename, filename, deskew=True)
-    if result == ocrmypdf.ExitCode.already_done_ocr:
-        logging.error("Skipped document because it already contained text")
-    elif result == ocrmypdf.ExitCode.ok:
+    if ocrmypdf.pdfa.file_claims_pdfa(filename)["pass"]:
+        logging.info("Skipped document because it already contained text")
+    else:
+        archive_filename = archive_dir + str(filename)
+        if len(archive_dir) > 0 and not filecompare(filename, archive_filename):
+            logging.info(f"Archiving document to {archive_filename}")
+            try:
+                shutil.copy2(filename, posixpath.dirname(archive_filename))
+            except IOError as io_err:
+                os.makedirs(posixpath.dirname(archive_filename))
+                shutil.copy2(filename, posixpath.dirname(archive_filename))
+        try:
+            result = ocrmypdf.ocr(filename, filename, deskew=True)
+            logging.info(result)
+        except ocrmypdf.exceptions.EncryptedPdfError:
+            logging.info("Skipped document because it is encrypted")
+        except ocrmypdf.exceptions.PriorOcrFoundError:
+            logging.info("Skipped document because it already contained text")
+        except ocrmypdf.exceptions.DigitalSignatureError:
+            logging.info("Skipped document because it has a digital signature")
+        except ocrmypdf.exceptions.TaggedPDFError:
+            logging.info(
+                "Skipped document because it does not need ocr as it is tagged"
+            )
+        except:
+            logging.error("Unhandled error occured")
         logging.info("OCR complete")
-    logging.info(result)

misc/synology.py

@@ -53,9 +53,10 @@ for dir_name, _subdirs, file_list in os.walk(start_dir):
         ]
         logging.info(cmd)
         full_path_ocr = os.path.join(dir_name, filename_ocr)
-        with open(filename, 'rb') as input_file, open(
-            full_path_ocr, 'wb'
-        ) as output_file:
+        with (
+            open(filename, 'rb') as input_file,
+            open(full_path_ocr, 'wb') as output_file,
+        ):
             proc = subprocess.run(
                 cmd,
                 stdin=input_file,

misc/watcher.py

@@ -7,7 +7,6 @@
 
 # Do not enable annotations!
 # https://github.com/tiangolo/typer/discussions/598
-# from __future__ import annotations
 
 import json
 import logging
@@ -131,7 +130,7 @@ def execute_ocrmypdf(
 
 
 class HandleObserverEvent(PatternMatchingEventHandler):
-    def __init__(
+    def __init__(  # noqa: D107
         self,
         patterns=None,
         ignore_patterns=None,
@@ -191,7 +190,7 @@ def main(
         bool,
         typer.Option(
             envvar='OCR_OUTPUT_DIRECTORY_YEAR_MONTH',
-            help='Create a subdirectory in the output directory for each year and month',
+            help='Create a subdirectory in the output directory for each year/month',
         ),
     ] = False,
     on_success_delete: Annotated[

pyproject.toml

@@ -12,12 +12,13 @@ readme = "README.md"
 license = { text = "MPL-2.0" }
 requires-python = ">=3.10"
 dependencies = [
-  "Pillow>=10.0.1",
   "deprecation>=2.1.0",
   "img2pdf>=0.5",
   "packaging>=20",
   "pdfminer.six>=20220319",
+  "pi-heif",                # Heif image format - maintainers: if this is removed, it will NOT break
   "pikepdf>=8.10.1",
+  "Pillow>=10.0.1",
   "pluggy>=1",
   "rich>=13",
 ]
@@ -60,7 +61,7 @@ test = [
   "types-Pillow",
   "types-humanfriendly",
 ]
-watcher = ["watchdog>=1.0.2", "typer[all]", "python-dotenv"]
+watcher = ["watchdog>=1.0.2", "typer-slim[standard]", "python-dotenv"]
 webservice = ["Flask>=2.0.1"]
 
 [project.scripts]
@@ -78,29 +79,6 @@ namespaces = false
 [tool.distutils.bdist_wheel]
 python-tag = "py310"
 
-[tool.black]
-line-length = 88
-target-version = ["py310", "py311", "py312"]
-skip-string-normalization = true
-include = '\.pyi?$'
-exclude = '''
-/(
-    \.eggs
-  | \.git
-  | \.hg
-  | \.mypy_cache
-  | \.tox
-  | \.venv
-  | _build
-  | buck-out
-  | build
-  | dist
-  | docs
-  | misc
-  | \.egg-info
-)/
-'''
-
 [tool.coverage.run]
 branch = true
 parallel = true
@@ -153,7 +131,10 @@ module = [
 ignore_missing_imports = true
 
 [tool.ruff]
-select = [
+target-version = "py310"
+
+[tool.ruff.lint]
+"select" = [
   "D",    # pydocstyle
   "E",    # pycodestyle
   "W",    # pycodestyle
@@ -161,17 +142,18 @@ select = [
   "I001", # isort
   "UP",   # pyupgrade
 ]
-target-version = "py310"
 
-[tool.ruff.isort]
+[tool.ruff.lint.isort]
 known-first-party = ["ocrmypdf"]
-required-imports = ["from __future__ import annotations"]
 
-[tool.ruff.pydocstyle]
+[tool.ruff.lint.pydocstyle]
 convention = "google"
 
-[tool.ruff.per-file-ignores]
+[tool.ruff.lint.per-file-ignores]
 "docs/conf.py" = ["D100", "D101", "D105"]
 "tests/*.py" = ["D100", "D101", "D102", "D103", "D105"]
 "misc/*.py" = ["D103", "D101", "D102"]
 "src/ocrmypdf/builtin_plugins/*.py" = ["D103", "D102", "D105"]
+
+[tool.ruff.format]
+quote-style = "preserve"

src/ocrmypdf/_concurrent.py

@@ -7,8 +7,8 @@ from __future__ import annotations
 
 import threading
 from abc import ABC, abstractmethod
-from collections.abc import Iterable
-from typing import Any, Callable, TypeVar
+from collections.abc import Callable, Iterable
+from typing import Any, TypeVar
 
 from ocrmypdf._progressbar import NullProgressBar, ProgressBar
 

src/ocrmypdf/_exec/tesseract.py

@@ -220,7 +220,8 @@ def get_deskew(
 
 def tesseract_log_output(stream: bytes) -> None:
     tlog = TesseractLoggerAdapter(
-        log, extra=log.extra if hasattr(log, 'extra') else None  # type: ignore
+        log,
+        extra=log.extra if hasattr(log, 'extra') else None,  # type: ignore
     )
 
     if not stream:

src/ocrmypdf/_exec/unpaper.py

@@ -8,19 +8,17 @@ from __future__ import annotations
 import logging
 import os
 import shlex
-import sys
 from collections.abc import Iterator
 from contextlib import contextmanager
 from decimal import Decimal
 from pathlib import Path
 from subprocess import PIPE, STDOUT
 from tempfile import TemporaryDirectory
-from typing import Union
 
 from packaging.version import Version
 from PIL import Image
 
-from ocrmypdf.exceptions import MissingDependencyError, SubprocessOutputError
+from ocrmypdf.exceptions import SubprocessOutputError
 from ocrmypdf.subprocess import get_version, run
 
 # unpaper documentation:
@@ -29,7 +27,7 @@ from ocrmypdf.subprocess import get_version, run
 
 UNPAPER_IMAGE_PIXEL_LIMIT = 256 * 1024 * 1024
 
-DecFloat = Union[Decimal, float]
+DecFloat = Decimal | float
 
 log = logging.getLogger(__name__)
 

src/ocrmypdf/_jobcontext.py

@@ -93,8 +93,8 @@ class PageContext:
         state = self.__dict__.copy()
 
         state['options'] = copy(self.options)
-        if not isinstance(state['options'].input_file, (str, bytes, os.PathLike)):
+        if not isinstance(state['options'].input_file, str | bytes | os.PathLike):
             state['options'].input_file = 'stream'
-        if not isinstance(state['options'].output_file, (str, bytes, os.PathLike)):
+        if not isinstance(state['options'].output_file, str | bytes | os.PathLike):
             state['options'].output_file = 'stream'
         return state

src/ocrmypdf/_metadata.py

@@ -166,9 +166,12 @@ def metadata_fixup(
 
     with Pdf.open(context.origin) as original, Pdf.open(working_file) as pdf:
         docinfo = get_docinfo(original, context)
-        with original.open_metadata(
-            set_pikepdf_as_editor=False, update_docinfo=False, strict=False
-        ) as meta_original, pdf.open_metadata() as meta_pdf:
+        with (
+            original.open_metadata(
+                set_pikepdf_as_editor=False, update_docinfo=False, strict=False
+            ) as meta_original,
+            pdf.open_metadata() as meta_pdf,
+        ):
             meta_pdf.load_from_docinfo(
                 docinfo, delete_missing=False, raise_failure=False
             )

src/ocrmypdf/_pipeline.py

@@ -14,7 +14,7 @@ from collections.abc import Iterable, Iterator, Sequence
 from contextlib import suppress
 from io import BytesIO
 from pathlib import Path
-from shutil import copyfileobj, copystat
+from shutil import copyfileobj
 from typing import Any, BinaryIO, TypeVar, cast
 
 import img2pdf
@@ -41,12 +41,23 @@ from ocrmypdf.pdfa import generate_pdfa_ps
 from ocrmypdf.pdfinfo import Colorspace, Encoding, PageInfo, PdfInfo
 from ocrmypdf.pluginspec import OrientationConfidence
 
+try:
+    from pi_heif import register_heif_opener
+except ImportError:
+
+    def register_heif_opener():
+        pass
+
+
 T = TypeVar("T")
 log = logging.getLogger(__name__)
 
 VECTOR_PAGE_DPI = 400
 
 
+register_heif_opener()
+
+
 def triage_image_file(input_file: Path, output_file: Path, options) -> None:
     """Triage the input image file.
 
@@ -131,7 +142,7 @@ def _pdf_guess_version(input_file: Path, search_window=1024) -> str:
     """
     with open(input_file, 'rb') as f:
         signature = f.read(search_window)
-    m = re.search(br'%PDF-(\d\.\d)', signature)
+    m = re.search(rb'%PDF-(\d\.\d)', signature)
     if m:
         return m.group(1).decode('ascii')
     return ''
@@ -767,7 +778,9 @@ def render_hocr_page(hocr: Path, page_context: PageContext) -> Path:
             font=Courier(),
         )
     HocrTransform(
-        hocr_filename=hocr, dpi=dpi.to_scalar(), **debug_kwargs  # square
+        hocr_filename=hocr,
+        dpi=dpi.to_scalar(),
+        **debug_kwargs,  # square
     ).to_pdf(
         out_filename=output_file,
         image_filename=None,

src/ocrmypdf/_pipelines/_common.py

@@ -11,13 +11,13 @@ import os
 import shutil
 import sys
 import threading
-from collections.abc import Sequence
+from collections.abc import Callable, Sequence
 from concurrent.futures.process import BrokenProcessPool
 from concurrent.futures.thread import BrokenThreadPool
 from contextlib import contextmanager
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Callable, NamedTuple, cast
+from typing import NamedTuple, cast
 
 import PIL
 

src/ocrmypdf/_pipelines/hocr_to_ocr_pdf.py

@@ -4,7 +4,6 @@
 
 """Implements the concurrent and page synchronous parts of the pipeline."""
 
-
 from __future__ import annotations
 
 import argparse

src/ocrmypdf/_pipelines/ocr.py

@@ -4,7 +4,6 @@
 
 """Implements the concurrent and page synchronous parts of the pipeline."""
 
-
 from __future__ import annotations
 
 import argparse
@@ -155,12 +154,13 @@ def _run_pipeline(
     options: argparse.Namespace,
     plugin_manager: OcrmypdfPluginManager,
 ) -> ExitCode:
-    with manage_work_folder(
-        work_folder=Path(mkdtemp(prefix="ocrmypdf.io.")),
-        retain=options.keep_temporary_files,
-        print_location=options.keep_temporary_files,
-    ) as work_folder, manage_debug_log_handler(
-        options=options, work_folder=work_folder
+    with (
+        manage_work_folder(
+            work_folder=Path(mkdtemp(prefix="ocrmypdf.io.")),
+            retain=options.keep_temporary_files,
+            print_location=options.keep_temporary_files,
+        ) as work_folder,
+        manage_debug_log_handler(options=options, work_folder=work_folder),
     ):
         executor = setup_pipeline(options, plugin_manager)
         check_requested_output_file(options)

src/ocrmypdf/_pipelines/pdf_to_hocr.py

@@ -4,7 +4,6 @@
 
 """Implements the concurrent and page synchronous parts of the pipeline."""
 
-
 from __future__ import annotations
 
 import argparse

src/ocrmypdf/api.py

@@ -14,7 +14,7 @@ from collections.abc import Iterable, Sequence
 from enum import IntEnum
 from io import IOBase
 from pathlib import Path
-from typing import AnyStr, BinaryIO, Union
+from typing import AnyStr, BinaryIO
 from warnings import warn
 
 import pluggy
@@ -28,8 +28,8 @@ from ocrmypdf._validation import check_options
 from ocrmypdf.cli import ArgumentParser, get_parser
 from ocrmypdf.helpers import is_iterable_notstr
 
-StrPath = Union[Path, AnyStr]
-PathOrIO = Union[BinaryIO, StrPath]
+StrPath = Path | AnyStr
+PathOrIO = BinaryIO | StrPath
 
 # Installing plugins affects the global state of the Python interpreter,
 # so we need to use a lock to prevent multiple threads from installing
@@ -169,7 +169,7 @@ def _kwargs_to_cmdline(
 
         # We have a parameter
         cmdline.append(f"--{cmd_style_arg}")
-        if isinstance(val, (int, float)):
+        if isinstance(val, int | float):
             cmdline.append(str(val))
         elif isinstance(val, str):
             cmdline.append(val)
@@ -201,14 +201,17 @@ def create_options(
         defer_kwargs={'progress_bar', 'plugins', 'parser', 'input_file', 'output_file'},
         **kwargs,
     )
-    if isinstance(input_file, (BinaryIO, IOBase)):
+    if isinstance(input_file, BinaryIO | IOBase):
         cmdline.append('stream://input_file')
     else:
         cmdline.append(os.fspath(input_file))
-    if isinstance(output_file, (BinaryIO, IOBase)):
+    if isinstance(output_file, BinaryIO | IOBase):
         cmdline.append('stream://output_file')
     else:
         cmdline.append(os.fspath(output_file))
+    if 'sidecar' in kwargs and isinstance(kwargs['sidecar'], BinaryIO | IOBase):
+        cmdline.append('--sidecar')
+        cmdline.append('stream://sidecar')
 
     parser.enable_api_mode()
     options = parser.parse_args(cmdline)
@@ -219,6 +222,8 @@ def create_options(
         options.input_file = input_file
     if options.output_file == 'stream://output_file':
         options.output_file = output_file
+    if options.sidecar == 'stream://sidecar':
+        options.sidecar = kwargs['sidecar']
 
     return options
 
@@ -230,7 +235,7 @@ def ocr(  # noqa: D417
     language: Iterable[str] | None = None,
     image_dpi: int | None = None,
     output_type: str | None = None,
-    sidecar: StrPath | None = None,
+    sidecar: PathOrIO | None = None,
     jobs: int | None = None,
     use_threads: bool | None = None,
     title: str | None = None,
@@ -343,7 +348,7 @@ def ocr(  # noqa: D417
 
     if not plugins:
         plugins = []
-    elif isinstance(plugins, (str, Path)):
+    elif isinstance(plugins, str | Path):
         plugins = [plugins]
     else:
         plugins = list(plugins)

src/ocrmypdf/builtin_plugins/concurrency.py

@@ -12,10 +12,10 @@ import queue
 import signal
 import sys
 import threading
-from collections.abc import Iterable
+from collections.abc import Callable, Iterable
 from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed
 from contextlib import suppress
-from typing import Callable, Union
+from typing import Union
 
 from rich.console import Console as RichConsole
 
@@ -25,8 +25,10 @@ from ocrmypdf._progressbar import RichProgressBar
 from ocrmypdf.exceptions import InputFileError
 from ocrmypdf.helpers import remove_all_log_handlers
 
-FuturesExecutorClass = Union[type[ThreadPoolExecutor], type[ProcessPoolExecutor]]
-Queue = Union[multiprocessing.Queue, queue.Queue]
+FuturesExecutorClass = Union[  # noqa: UP007
+    type[ThreadPoolExecutor], type[ProcessPoolExecutor]
+]
+Queue = Union[multiprocessing.Queue, queue.Queue]  # noqa: UP007
 UserInit = Callable[[], None]
 WorkerInit = Callable[[Queue, UserInit, int], None]
 
@@ -128,11 +130,14 @@ class StandardExecutor(Executor):
         listener = threading.Thread(target=log_listener, args=(log_queue,))
         listener.start()
 
-        with self.pbar_class(**progress_kwargs) as pbar, executor_class(
-            max_workers=max_workers,
-            initializer=initializer,
-            initargs=(log_queue, worker_initializer, logging.getLogger("").level),
-        ) as executor:
+        with (
+            self.pbar_class(**progress_kwargs) as pbar,
+            executor_class(
+                max_workers=max_workers,
+                initializer=initializer,
+                initargs=(log_queue, worker_initializer, logging.getLogger("").level),
+            ) as executor,
+        ):
             futures = [executor.submit(task, *args) for args in task_arguments]
             try:
                 for future in as_completed(futures):

src/ocrmypdf/builtin_plugins/default_filters.py

@@ -8,7 +8,5 @@ from ocrmypdf import hookimpl
 
 
 @hookimpl
-def filter_pdf_page(
-    page, image_filename, output_pdf
-):  # pylint: disable=unused-argument
+def filter_pdf_page(page, image_filename, output_pdf):  # pylint: disable=unused-argument
     return output_pdf

src/ocrmypdf/builtin_plugins/tesseract_ocr.py

@@ -2,9 +2,9 @@
 # SPDX-License-Identifier: MPL-2.0
 """Built-in plugin to implement OCR using Tesseract."""
 
-
 from __future__ import annotations
 
+import argparse
 import logging
 import os
 
@@ -94,7 +94,8 @@ def add_options(parser):
     )
     tess.add_argument(
         '--tesseract-downsample-large-images',
-        action='store_true',
+        action=argparse.BooleanOptionalAction,
+        default=True,
         help=(
             "Downsample large images before OCR. Tesseract has an upper limit on the "
             "size images it will support. If this argument is given, OCRmyPDF will "
@@ -220,7 +221,7 @@ class TesseractOcrEngine(OcrEngine):
 
     @staticmethod
     def creator_tag(options):
-        tag = '-PDF' if options.pdf_renderer == 'sandwich' else 'hOCR'
+        tag = '-PDF' if options.pdf_renderer == 'sandwich' else '-hOCR'
         return f"Tesseract OCR{tag} {TesseractOcrEngine.version()}"
 
     def __str__(self):

src/ocrmypdf/cli.py

@@ -6,8 +6,8 @@
 from __future__ import annotations
 
 import argparse
-from collections.abc import Mapping
-from typing import Any, Callable, TypeVar
+from collections.abc import Callable, Mapping
+from typing import Any, TypeVar
 
 from ocrmypdf._version import PROGRAM_NAME as _PROGRAM_NAME
 from ocrmypdf._version import __version__ as _VERSION
@@ -390,7 +390,7 @@ Online documentation is located at:
         action='store',
         type=numeric(float, 0),
         metavar='MPixels',
-        help="Set maximum number of pixels to unpack before treating an image as a "
+        help="Set maximum number of megapixels to unpack before treating an image as a "
         "decompression bomb",
         default=250.0,
     )

src/ocrmypdf/extra_plugins/semfree.py

@@ -20,13 +20,12 @@ from __future__ import annotations
 import logging
 import logging.handlers
 import signal
-from collections.abc import Iterable, Iterator
+from collections.abc import Callable, Iterable, Iterator
 from contextlib import suppress
 from enum import Enum, auto
 from itertools import islice, repeat, takewhile, zip_longest
 from multiprocessing import Pipe, Process
 from multiprocessing.connection import Connection, wait
-from typing import Callable
 
 from ocrmypdf import Executor, hookimpl
 from ocrmypdf._concurrent import NullProgressBar

src/ocrmypdf/helpers.py

@@ -10,7 +10,7 @@ import multiprocessing
 import os
 import shutil
 import warnings
-from collections.abc import Iterable, Sequence
+from collections.abc import Callable, Iterable, Sequence
 from contextlib import suppress
 from decimal import Decimal
 from io import StringIO
@@ -19,7 +19,6 @@ from pathlib import Path
 from statistics import harmonic_mean
 from typing import (
     Any,
-    Callable,
     Generic,
     TypeVar,
 )

src/ocrmypdf/hocrtransform/_hocr.py

@@ -84,7 +84,6 @@ class HocrTransform:
         debug_render_options: DebugRenderOptions | None = None,
     ):
         """Initialize the HocrTransform object."""
-
         if debug:
             log.warning("Use debug_render_options instead", DeprecationWarning)
             self.render_options = DebugRenderOptions(

src/ocrmypdf/languages.py

@@ -7,12 +7,12 @@ Derived from
 https://www.loc.gov/standards/iso639-2/ascii_8bits.html
 """
 
-
 from typing import NamedTuple
 
 
 class ISOCodeData(NamedTuple):
     """Data for a single ISO 639 code."""
+
     alt: str
     alpha_2: str
     english: str
@@ -168,8 +168,10 @@ ISO_639_3 = {
     'chu': ISOCodeData(
         '',
         'cu',
-        ('Church Slavic; Old Slavonic; Church Slavonic;'
-         ' Old Bulgarian; Old Church Slavonic'),
+        (
+            'Church Slavic; Old Slavonic; Church Slavonic;'
+            ' Old Bulgarian; Old Church Slavonic'
+        ),
         "slavon d'église; vieux slave; slavon liturgique; vieux bulgare",
     ),
     'chv': ISOCodeData('', 'cv', 'Chuvash', 'tchouvache'),

src/ocrmypdf/optimize.py

@@ -3,7 +3,6 @@
 
 """Post-processing image optimization of OCR PDFs."""
 
-
 from __future__ import annotations
 
 import logging
@@ -11,11 +10,10 @@ import sys
 import tempfile
 import threading
 from collections import defaultdict
-from collections.abc import Iterator, MutableSet, Sequence
+from collections.abc import Callable, Iterator, MutableSet, Sequence
 from os import fspath
 from pathlib import Path
-from typing import Any, Callable, NamedTuple, NewType
-from warnings import warn
+from typing import Any, NamedTuple, NewType
 from zlib import compress
 
 import img2pdf
@@ -91,6 +89,7 @@ def extract_image_filter(
         if (
             len(pim.filter_decodeparms) == 2
             and first_filtdp[0] == Name.FlateDecode
+            and first_filtdp[1] is not None
             and first_filtdp[1].get(Name.Predictor, 1) == 1
             and second_filtdp[0] == Name.DCTDecode
             and not second_filtdp[1]

src/ocrmypdf/pdfa.py

@@ -90,7 +90,7 @@ def generate_pdfa_ps(target_filename: Path, icc: str = 'sRGB'):
         icc: ICC identifier such as 'sRGB'
     References:
         Adobe PDFMARK Reference:
-        https://www.adobe.com/content/dam/acom/en/devnet/acrobat/pdfs/pdfmark_reference.pdf
+        https://opensource.adobe.com/dc-acrobat-sdk-docs/library/pdfmark/
     """
     if icc != 'sRGB':
         raise NotImplementedError("Only supporting sRGB")

src/ocrmypdf/pdfinfo/info.py

@@ -10,9 +10,8 @@ import atexit
 import logging
 import re
 import statistics
-import sys
 from collections import defaultdict
-from collections.abc import Container, Iterable, Iterator, Mapping, Sequence
+from collections.abc import Callable, Container, Iterable, Iterator, Mapping, Sequence
 from contextlib import contextmanager
 from decimal import Decimal
 from enum import Enum, auto
@@ -20,7 +19,7 @@ from functools import partial
 from math import hypot, inf, isclose
 from os import PathLike
 from pathlib import Path
-from typing import Callable, NamedTuple
+from typing import NamedTuple
 from warnings import warn
 
 from pdfminer.layout import LTPage, LTTextBox
@@ -1060,12 +1059,7 @@ class PageInfo:
 
         weights = [area / total_drawn_area for area in image_areas]
         # Calculate harmonic mean of DPIs weighted by area
-        if sys.version_info >= (3, 10):
-            weighted_dpi = statistics.harmonic_mean(image_dpis, weights)
-        else:
-            weighted_dpi = sum(weights) / sum(
-                weight / dpi for weight, dpi in zip(weights, image_dpis)
-            )
+        weighted_dpi = statistics.harmonic_mean(image_dpis, weights)
         max_dpi = max(image_dpis)
         dpi_average_max_ratio = weighted_dpi / max_dpi
 
@@ -1176,7 +1170,7 @@ class PdfInfo:
     @property
     def filename(self) -> str | Path:
         """Return filename of PDF."""
-        if not isinstance(self._infile, (str, Path)):
+        if not isinstance(self._infile, str | Path):
             raise NotImplementedError("can't get filename from stream")
         return self._infile
 

src/ocrmypdf/pdfinfo/layout.py

@@ -5,12 +5,12 @@
 from __future__ import annotations
 
 import re
-from collections.abc import Mapping
+from collections.abc import Iterator, Mapping
 from contextlib import contextmanager
 from math import copysign
 from os import PathLike
 from pathlib import Path
-from typing import Any, Iterator
+from typing import Any
 from unittest.mock import patch
 
 import pdfminer

src/ocrmypdf/quality.py

@@ -3,7 +3,6 @@
 
 """Utilities to measure OCR quality."""
 
-
 from __future__ import annotations
 
 import re

src/ocrmypdf/subprocess/__init__.py

@@ -8,12 +8,11 @@ import logging
 import os
 import re
 import sys
-from collections.abc import Mapping, Sequence
+from collections.abc import Callable, Mapping, Sequence
 from contextlib import suppress
 from pathlib import Path
 from subprocess import PIPE, STDOUT, CalledProcessError, CompletedProcess, Popen
 from subprocess import run as subprocess_run
-from typing import Callable, Union
 
 from packaging.version import Version
 
@@ -23,7 +22,7 @@ from ocrmypdf.exceptions import MissingDependencyError
 
 log = logging.getLogger(__name__)
 
-Args = Sequence[Union[Path, str]]
+Args = Sequence[Path | str]
 OsEnviron = os._Environ  # pylint: disable=protected-access
 
 
@@ -172,6 +171,7 @@ def get_version(
         ) from e
     except CalledProcessError as e:
         if e.returncode != 0:
+            log.exception(e)
             raise MissingDependencyError(
                 f"Ran program '{program}' but it exited with an error:\n{e.output}"
             ) from e

src/ocrmypdf/subprocess/_windows.py

@@ -9,18 +9,13 @@ import os
 import re
 import shutil
 import sys
-from collections.abc import Iterable, Iterator
+from collections.abc import Callable, Iterable, Iterator
 from itertools import chain
 from pathlib import Path
-from typing import Any, Callable, TypeVar
+from typing import Any, TypeAlias, TypeVar
 
 from packaging.version import InvalidVersion, Version
 
-if sys.version_info >= (3, 10):
-    from typing import TypeAlias
-else:
-    from typing_extensions import TypeAlias  # pragma: no cover
-
 if sys.platform == 'win32':
     # mypy understands 'if sys.platform' better than try/except ModuleNotFoundError
     import winreg  # pylint: disable=import-error
@@ -84,7 +79,7 @@ def registry_path_ghostscript(env=None) -> Iterator[Path]:
                 registry_subkeys(k), key=ghostscript_version_key, default=(0, 0, 0)
             )
         with winreg.OpenKey(
-            winreg.HKEY_LOCAL_MACHINE, fr"SOFTWARE\Artifex\GPL Ghostscript\{latest_gs}"
+            winreg.HKEY_LOCAL_MACHINE, rf"SOFTWARE\Artifex\GPL Ghostscript\{latest_gs}"
         ) as k:
             for _, gs_path, _ in registry_values(k):
                 yield Path(gs_path) / 'bin'

tests/plugins/gs_render_soft_error.py

@@ -3,7 +3,6 @@
 
 from __future__ import annotations
 
-from pathlib import Path
 from subprocess import CalledProcessError
 from unittest.mock import patch
 

tests/plugins/tesseract_cache.py

@@ -169,22 +169,25 @@ class CacheOcrEngine(TesseractOcrEngine):
 
     @staticmethod
     def get_orientation(input_file, options):
-        with CacheOcrEngine.lock, patch(
-            'ocrmypdf._exec.tesseract.run', new=partial(cached_run, options)
+        with (
+            CacheOcrEngine.lock,
+            patch('ocrmypdf._exec.tesseract.run', new=partial(cached_run, options)),
         ):
             return TesseractOcrEngine.get_orientation(input_file, options)
 
     @staticmethod
     def get_deskew(input_file, options) -> float:
-        with CacheOcrEngine.lock, patch(
-            'ocrmypdf._exec.tesseract.run', new=partial(cached_run, options)
+        with (
+            CacheOcrEngine.lock,
+            patch('ocrmypdf._exec.tesseract.run', new=partial(cached_run, options)),
         ):
             return TesseractOcrEngine.get_deskew(input_file, options)
 
     @staticmethod
     def generate_hocr(input_file, output_hocr, output_text, options):
-        with CacheOcrEngine.lock, patch(
-            'ocrmypdf._exec.tesseract.run', new=partial(cached_run, options)
+        with (
+            CacheOcrEngine.lock,
+            patch('ocrmypdf._exec.tesseract.run', new=partial(cached_run, options)),
         ):
             TesseractOcrEngine.generate_hocr(
                 input_file, output_hocr, output_text, options
@@ -192,8 +195,9 @@ class CacheOcrEngine(TesseractOcrEngine):
 
     @staticmethod
     def generate_pdf(input_file, output_pdf, output_text, options):
-        with CacheOcrEngine.lock, patch(
-            'ocrmypdf._exec.tesseract.run', new=partial(cached_run, options)
+        with (
+            CacheOcrEngine.lock,
+            patch('ocrmypdf._exec.tesseract.run', new=partial(cached_run, options)),
         ):
             TesseractOcrEngine.generate_pdf(
                 input_file, output_pdf, output_text, options

tests/plugins/tesseract_debug_rotate.py

@@ -72,9 +72,10 @@ class FixedRotateNoopOcrEngine(OcrEngine):
 
     @staticmethod
     def generate_hocr(input_file, output_hocr, output_text, options):
-        with Image.open(input_file) as im, open(
-            output_hocr, 'w', encoding='utf-8'
-        ) as f:
+        with (
+            Image.open(input_file) as im,
+            open(output_hocr, 'w', encoding='utf-8') as f,
+        ):
             w, h = im.size
             f.write(HOCR_TEMPLATE.format(str(w), str(h)))
         with open(output_text, 'w') as f:

tests/plugins/tesseract_noop.py

@@ -70,9 +70,10 @@ class NoopOcrEngine(OcrEngine):
 
     @staticmethod
     def generate_hocr(input_file, output_hocr, output_text, options):
-        with Image.open(input_file) as im, open(
-            output_hocr, 'w', encoding='utf-8'
-        ) as f:
+        with (
+            Image.open(input_file) as im,
+            open(output_hocr, 'w', encoding='utf-8') as f,
+        ):
             w, h = im.size
             f.write(HOCR_TEMPLATE.format(str(w), str(h)))
         with open(output_text, 'w') as f:

tests/plugins/tesseract_simulate_oom_killer.py

@@ -9,6 +9,7 @@ ensure we fail with an error rather than deadlock in such cases.
 Page 4 was chosen because of this number's association with bad luck
 in many East Asian cultures.
 """
+
 # type: ignore
 from __future__ import annotations
 

tests/test_api.py

@@ -29,6 +29,18 @@ def test_stream_api(resources: Path):
     assert b'%PDF' in out.read(1024)
 
 
+def test_sidecar_stringio(resources: Path, outdir: Path, outpdf: Path):
+    s = BytesIO()
+    ocrmypdf.ocr(
+        resources / 'ccitt.pdf',
+        outpdf,
+        plugins=['tests/plugins/tesseract_cache.py'],
+        sidecar=s
+    )
+    s.seek(0)
+    assert b'the' in s.getvalue()
+
+
 def test_hocr_api_multipage(resources: Path, outdir: Path, outpdf: Path):
     ocrmypdf.api._pdf_to_hocr(
         resources / 'multipage.pdf',
@@ -62,3 +74,4 @@ def test_hocr_to_pdf_api(resources: Path, outdir: Path, outpdf: Path):
 
     text = extract_text(outpdf)
     assert 'hocr' in text and 'the' not in text
+

tests/test_graft.py

@@ -11,9 +11,11 @@ import ocrmypdf
 
 
 def test_no_glyphless_graft(resources, outdir):
-    with pikepdf.open(resources / 'francais.pdf') as pdf, pikepdf.open(
-        resources / 'aspect.pdf'
-    ) as pdf_aspect, pikepdf.open(resources / 'cmyk.pdf') as pdf_cmyk:
+    with (
+        pikepdf.open(resources / 'francais.pdf') as pdf,
+        pikepdf.open(resources / 'aspect.pdf') as pdf_aspect,
+        pikepdf.open(resources / 'cmyk.pdf') as pdf_cmyk,
+    ):
         pdf.pages.extend(pdf_aspect.pages)
         pdf.pages.extend(pdf_cmyk.pages)
         pdf.save(outdir / 'test.pdf')

tests/test_main.py

@@ -17,7 +17,7 @@ from PIL import Image
 
 import ocrmypdf
 from ocrmypdf._exec import tesseract
-from ocrmypdf.exceptions import ExitCode, MissingDependencyError, OutputFileAccessError
+from ocrmypdf.exceptions import ExitCode, MissingDependencyError
 from ocrmypdf.pdfa import file_claims_pdfa
 from ocrmypdf.pdfinfo import Colorspace, Encoding, PdfInfo
 from ocrmypdf.subprocess import get_version

tests/test_metadata.py

@@ -35,9 +35,10 @@ def test_preserve_docinfo(output_type, resources, outpdf):
         '--plugin',
         'tests/plugins/tesseract_noop.py',
     )
-    with pikepdf.open(resources / 'graph.pdf') as pdf_before, pikepdf.open(
-        output
-    ) as pdf_after:
+    with (
+        pikepdf.open(resources / 'graph.pdf') as pdf_before,
+        pikepdf.open(output) as pdf_after,
+    ):
         for key in ('/Title', '/Author'):
             assert pdf_before.docinfo[key] == pdf_after.docinfo[key]
         pdfa_info = file_claims_pdfa(str(output))

tests/test_preprocessing.py

@@ -9,10 +9,11 @@ import pytest
 from PIL import Image
 
 from ocrmypdf._exec import ghostscript, tesseract
+from ocrmypdf.exceptions import ExitCode
 from ocrmypdf.helpers import Resolution
 from ocrmypdf.pdfinfo import PdfInfo
 
-from .conftest import check_ocrmypdf, have_unpaper
+from .conftest import check_ocrmypdf, have_unpaper, run_ocrmypdf
 
 RENDERERS = ['hocr', 'sandwich']
 
@@ -107,7 +108,7 @@ def test_non_square_resolution(renderer, resources, outpdf):
     in_pageinfo = PdfInfo(resources / 'aspect.pdf')
     assert in_pageinfo[0].dpi.x != in_pageinfo[0].dpi.y
 
-    check_ocrmypdf(
+    proc = run_ocrmypdf(
         resources / 'aspect.pdf',
         outpdf,
         '--pdf-renderer',
@@ -115,6 +116,10 @@ def test_non_square_resolution(renderer, resources, outpdf):
         '--plugin',
         'tests/plugins/tesseract_cache.py',
     )
+    # PDF/A conversion can fail for this file if Ghostscript >= 10.3, so don't test
+    # exit code in that case
+    if proc.returncode != ExitCode.pdfa_conversion_failed:
+        proc.check_returncode()
 
     out_pageinfo = PdfInfo(outpdf)
 

tests/test_userunit.py

@@ -7,10 +7,9 @@ from math import isclose
 
 import pytest
 
-from ocrmypdf.exceptions import ExitCode
 from ocrmypdf.pdfinfo import PdfInfo
 
-from .conftest import check_ocrmypdf, run_ocrmypdf_api
+from .conftest import check_ocrmypdf
 
 # pylint: disable=redefined-outer-name