v16.1.2 release notes

Ghostscript is more picky about a specific case with SMask that cannot be converted to PDF/A

Details here
https://github.com/ArtifexSoftware/ghostpdl/commit/4dcfae36bb4dcbc4ef3b5e5afc98bcde0d6b9ddc

.docker/Dockerfile

@@ -64,6 +64,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   img2pdf \
   libsm6 libxext6 libxrender-dev \
   pngquant \
+  python-is-python3 \
   tesseract-ocr \
   tesseract-ocr-chi-sim \
   tesseract-ocr-deu \

.docker/Dockerfile.alpine

@@ -1,7 +1,7 @@
 # SPDX-FileCopyrightText: 2023 James R. Barlow
 # SPDX-License-Identifier: MPL-2.0
 
-FROM alpine:3.18 as base
+FROM alpine:3.19 as base
 
 ENV LANG=C.UTF-8
 ENV TZ=UTC

.github/workflows/build.yml

@@ -32,8 +32,8 @@ jobs:
           - os: ubuntu-latest
             python: "3.12"
             tesseract5: true
-          # - os: ubuntu-latest
-          #   python: "pypy3.10"
+          - os: ubuntu-latest
+            python: "pypy3.10"
 
     env:
       OS: ${{ matrix.os }}
@@ -100,7 +100,7 @@ jobs:
           python -m pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
 
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           files: ./coverage.xml
           env_vars: OS,PYTHON
@@ -158,7 +158,7 @@ jobs:
           python -m pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
 
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           files: ./coverage.xml
           env_vars: OS,PYTHON
@@ -201,7 +201,7 @@ jobs:
           python -m pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
 
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           files: ./coverage.xml
           env_vars: OS,PYTHON

docs/api.rst

@@ -112,7 +112,9 @@ OCRmyPDF is strict about not writing to standard output so that
 users can safely use it in a pipeline and produce a valid output
 file. A caller application will have to ensure it does not write to
 standard output either, if it wants to be compatible with this
-behavior and support piping to a file.
+behavior and support piping to a file. Another benefit of running
+OCRmyPDF in a child process, as recommended above, is that it will
+not interfere with the parent process's standard output.
 
 Exceptions
 ----------

docs/cookbook.rst

@@ -187,7 +187,7 @@ might remove desirable content, especially from poor quality scans.
    background from grayscale or color images. Monochrome images are
    ignored. This should not be used on documents that contain color
    photos as it may remove them.
--  ``--deskew`` will correct pages were scanned at a skewed angle by
+-  ``--deskew`` will correct pages that were scanned at a skewed angle by
    rotating them back into place.
 -  ``--clean`` uses
    `unpaper <https://www.flameeyes.eu/projects/unpaper>`__ to clean up
@@ -245,6 +245,20 @@ if all you want to is to apply image processing or PDF/A conversion.
     the case. Use ``--tesseract-non-ocr-timeout`` to control the timeout
     for non-OCR operations, if needed.
 
+Remove all text or OCR from my PDF
+----------------------------------
+
+This is getting ridiculous, but OCRmyPDF can complete strip all textual
+information from a PDF and reconstruct it as a "bag of images" PDF.
+
+.. code-block::
+
+    ocrmypdf --tesseract-timeout 0 --force-ocr input.pdf output.pdf
+
+Why would you want to do this? Perhaps you have a PDF where OCR
+fails to produce useful results, and just want to get rid of all OCR information.
+This command also removes OCR generated by third party tools.
+
 Optimize images without performing OCR
 --------------------------------------
 
@@ -393,4 +407,4 @@ any digital signatures will be invalidated.
 OCRmyPDF cannot open documents that are encrypted with a digital certificate.
 
 Versions of OCRmyPDF prior to 14.4.0 would invalidate existing digital signatures
-without warning.
+without warning.

docs/installation.rst

@@ -139,6 +139,32 @@ from sources <#installing-head-revision-from-sources>`__.
 
 .. _ubuntu-lts-latest:
 
+RHEL 9
+------
+
+Prepare the environment by getting Python 3.11:
+
+.. code-block:: bash
+
+    dnf install python3.11 python3.11-pip
+
+Then, follow `Requirements for pip and HEAD install <#requirements-for-pip-and-head-install>`__ to instal dependencies:
+
+.. code-block:: bash
+
+    dnf install ghostscript tesseract
+
+and build ocrmypdf in virtual environment:
+
+.. code-block:: bash
+
+    python3.11 -m venv .venv
+
+To add JBIG2 encoding, see `Installing the JBIG2 encoder <jbig2>`__.
+
+Note Fedora packages for language data haven't been branched for RHEL/EPEL, but you can get traineddata files directly from `tesseract
+<https://github.com/tesseract-ocr/tessdata/>`__ and place them in ``/usr/share/tesseract/tessdata``.
+
 Installing the latest version on Ubuntu 22.04 LTS
 -------------------------------------------------
 
@@ -162,37 +188,13 @@ To add JBIG2 encoding, see :ref:`jbig2`.
 Ubuntu 20.04 LTS
 ----------------
 
-Ubuntu 20.04 includes ocrmypdf 9.6.0 - you can install that with ``apt``. To
-install a more recent version, uninstall the system-provided version of
-ocrmypdf, and install the following dependencies:
+Ubuntu 20.04 includes ocrmypdf 9.6.0 - you can install that with ``apt``. The
+most convenient way to install recent OCRmyPDF on older Ubuntu is to use
+Homebrew on Linux (Linuxbrew).
 
 .. code-block:: bash
 
-    sudo apt-get -y remove ocrmypdf  # remove system ocrmypdf, if installed
-    sudo apt-get -y update
-    sudo apt-get -y install \
-        ghostscript \
-        icc-profiles-free \
-        libxml2 \
-        pngquant \
-        python3-pip \
-        tesseract-ocr \
-        zlib1g
-
-To install ocrmypdf for the system:
-
-.. code-block:: bash
-
-    pip3 install ocrmypdf
-
-To install for the current user only:
-
-.. code-block:: bash
-
-    export PATH=$HOME/.local/bin:$PATH
-    pip3 install --user ocrmypdf
-
-To add JBIG2 encoding, see :ref:`jbig2`.
+    brew install ocrmypdf
 
 Arch Linux (AUR)
 ----------------
@@ -540,9 +542,25 @@ try:
 
     pip install --user ocrmypdf
 
+(If the message appears ``Requirement already satisfied: ocrmypdf in...``,
+you will need to use ``pip install --user --upgrade ocrmypdf``.)
+
 You should then be able to run ``ocrmypdf --version`` and see that the
 latest version was located.
 
+Installing with pipx
+====================
+
+Some users may prefer pipx. As with the method above, you will need to
+satisfy all non-Python dependencies. Then if pipx is installed, you
+can use
+
+.. code-block:: bash
+
+    pipx run ocrmypdf
+
+(If not installed, pipx will install first.)
+
 Requirements for pip and HEAD install
 -------------------------------------
 
@@ -553,7 +571,7 @@ manager. ``pip`` cannot provide them.
 The following versions are required:
 
 -  Python 3.10 or newer
--  Ghostscript 9.55 or newer
+-  Ghostscript 9.54 or newer
 -  Tesseract 4.1.1 or newer
 -  jbig2enc 0.29 or newer
 -  pngquant 2.5 or newer
@@ -671,7 +689,7 @@ To manually install the ``fish`` completion, copy
 Note on 32-bit support
 ======================
 
-Many Python libraries no longer 32-bit binary wheels for Linux. This
+Many Python libraries no longer provide 32-bit binary wheels for Linux. This
 includes many of the libraries that OCRmyPDF depends on, such as
 Pillow. The easiest way to express this to end users is to say we don't
 support 32-bit Linux.

docs/jbig2.rst

@@ -41,7 +41,15 @@ For all other platforms, you would need to build the JBIG2 encoder from source:
 
 Dependencies include libtoolize and libleptonica, which on Ubuntu systems
 are packaged as libtool and libleptonica-dev. On Fedora (35) they are packaged
-as libtool and leptonica-devel.
+as libtool and leptonica-devel. For this to work, please make sure to install
+``autotools``, ``automake``, ``libtool`` and ``leptonica`` first if not already
+installed.
+
+.. code-block:: bash
+
+    [sudo] apt install autotools-dev automake libtool libleptonica-dev
+..
+
 
 Lossy mode JBIG2
 ================

docs/release_notes.rst

@@ -30,6 +30,29 @@ OCRmyPDF typically supports the three most recent Python versions.
 
 .. |OCRmyPDF PyPI| image:: https://img.shields.io/pypi/v/ocrmypdf.svg
 
+v16.1.2
+=======
+
+-  Fixed test suite failure when using Ghostscript 10.3.
+-  Other minor corrections.
+
+v16.1.1
+=======
+
+-  Fixed PyPy 3.10 support.
+
+v16.1.0
+=======
+
+-  Improved hOCR renderer is now default for left to right languages.
+-  Improved handling of rotated pages. Previously, OCR text might be missing for
+   pages that were rotated with a /Rotate tag on the page entry.
+-  Improved handling of cropped pages. Previously, in some cases a page with a
+   crop box would not have its OCR applied correctly and misalignment between
+   OCR text and visible text coudl occur.
+-  Documentation improvements, especially installation instructions for less
+   common platforms.
+
 v16.0.4
 =======
 

misc/batch.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 # SPDX-FileCopyrightText: 2016 findingorder <https://github.com/findingorder>
+# SPDX-FileCopyrightText: 2024 nilsro <https://github.com/nilsro>
 # SPDX-License-Identifier: MIT
 
 """Example of using ocrmypdf as a library in a script.
@@ -15,6 +16,10 @@ from __future__ import annotations
 
 import logging
 import sys
+import os
+import posixpath
+import shutil
+import filecmp
 from pathlib import Path
 
 import ocrmypdf
@@ -22,32 +27,62 @@ import ocrmypdf
 # pylint: disable=logging-format-interpolation
 # pylint: disable=logging-not-lazy
 
+def filecompare(a,b):
+    try:
+        return filecmp.cmp(a, b, shallow=True)
+    except FileNotFoundError:
+        return False
+
+
 script_dir = Path(__file__).parent
+# set archive_dir to a path for backup original documents. Leave empty if not required.
+archive_dir = "/pdfbak"
 
 if len(sys.argv) > 1:
     start_dir = Path(sys.argv[1])
 else:
-    start_dir = Path('.')
+    start_dir = Path(".")
 
 if len(sys.argv) > 2:
     log_file = Path(sys.argv[2])
 else:
-    log_file = script_dir.with_name('ocr-tree.log')
+    log_file = script_dir.with_name("ocr-tree.log")
 
 logging.basicConfig(
     level=logging.INFO,
-    format='%(asctime)s %(message)s',
+    format="%(asctime)s %(message)s",
     filename=log_file,
-    filemode='a',
+    filemode="a",
 )
 
+logging.info(f"Start directory {start_dir}")
+
 ocrmypdf.configure_logging(ocrmypdf.Verbosity.default)
 
-for filename in start_dir.glob("**/*.py"):
+for filename in start_dir.glob("**/*.pdf"):
     logging.info(f"Processing {filename}")
-    result = ocrmypdf.ocr(filename, filename, deskew=True)
-    if result == ocrmypdf.ExitCode.already_done_ocr:
-        logging.error("Skipped document because it already contained text")
-    elif result == ocrmypdf.ExitCode.ok:
+    if ocrmypdf.pdfa.file_claims_pdfa(filename)["pass"]:
+        logging.info("Skipped document because it already contained text")
+    else:
+        archive_filename = archive_dir + str(filename)
+        if len(archive_dir) > 0 and not filecompare(filename, archive_filename):
+            logging.info(f"Archiving document to {archive_filename}")
+            try:
+                shutil.copy2(filename, posixpath.dirname(archive_filename))
+            except IOError as io_err:
+                os.makedirs(posixpath.dirname(archive_filename))
+                shutil.copy2(filename, posixpath.dirname(archive_filename))
+        try:
+            result = ocrmypdf.ocr(filename, filename, deskew=True)
+            logging.info(result)
+        except ocrmypdf.exceptions.EncryptedPdfError:
+            logging.info("Skipped document because it is encrypted")
+        except ocrmypdf.exceptions.PriorOcrFoundError:
+            logging.info("Skipped document because it already contained text")
+        except ocrmypdf.exceptions.DigitalSignatureError:
+            logging.info("Skipped document because it has a digital signature")
+        except ocrmypdf.exceptions.TaggedPDFError:
+            logging.info("Skipped document because it does not need ocr as it is tagged")
+        except:
+            logging.error("Unhandled error occured")
         logging.info("OCR complete")
-    logging.info(result)

misc/watcher.py

@@ -7,7 +7,6 @@
 
 # Do not enable annotations!
 # https://github.com/tiangolo/typer/discussions/598
-# from __future__ import annotations
 
 import json
 import logging
@@ -131,7 +130,7 @@ def execute_ocrmypdf(
 
 
 class HandleObserverEvent(PatternMatchingEventHandler):
-    def __init__(
+    def __init__(  # noqa: D107
         self,
         patterns=None,
         ignore_patterns=None,
@@ -191,7 +190,7 @@ def main(
         bool,
         typer.Option(
             envvar='OCR_OUTPUT_DIRECTORY_YEAR_MONTH',
-            help='Create a subdirectory in the output directory for each year and month',
+            help='Create a subdirectory in the output directory for each year/month',
         ),
     ] = False,
     on_success_delete: Annotated[

pyproject.toml

@@ -131,7 +131,11 @@ norecursedirs = ["lib", ".pc", ".git", "venv", "output", "cache", "resources"]
 testpaths = ["tests"]
 addopts = "-n auto"
 markers = ["slow"]
-filterwarnings = ["ignore:.*XMLParser.*:DeprecationWarning"]
+filterwarnings = [
+  "ignore:.*XMLParser.*:DeprecationWarning",
+  "ignore:.*ast.NameConstant.*:DeprecationWarning:reportlab",
+  "ignore:.*distutils.*:DeprecationWarning:libxmp",
+]
 
 [tool.mypy]
 
@@ -149,7 +153,10 @@ module = [
 ignore_missing_imports = true
 
 [tool.ruff]
-select = [
+target-version = "py310"
+
+[tool.ruff.lint]
+"select" = [
   "D",    # pydocstyle
   "E",    # pycodestyle
   "W",    # pycodestyle
@@ -157,16 +164,14 @@ select = [
   "I001", # isort
   "UP",   # pyupgrade
 ]
-target-version = "py310"
 
-[tool.ruff.isort]
+[tool.ruff.lint.isort]
 known-first-party = ["ocrmypdf"]
-required-imports = ["from __future__ import annotations"]
 
-[tool.ruff.pydocstyle]
+[tool.ruff.lint.pydocstyle]
 convention = "google"
 
-[tool.ruff.per-file-ignores]
+[tool.ruff.lint.per-file-ignores]
 "docs/conf.py" = ["D100", "D101", "D105"]
 "tests/*.py" = ["D100", "D101", "D102", "D103", "D105"]
 "misc/*.py" = ["D103", "D101", "D102"]

src/ocrmypdf/_concurrent.py

@@ -7,8 +7,8 @@ from __future__ import annotations
 
 import threading
 from abc import ABC, abstractmethod
-from collections.abc import Iterable
-from typing import Any, Callable, TypeVar
+from collections.abc import Callable, Iterable
+from typing import Any, TypeVar
 
 from ocrmypdf._progressbar import NullProgressBar, ProgressBar
 

src/ocrmypdf/_exec/unpaper.py

@@ -8,19 +8,17 @@ from __future__ import annotations
 import logging
 import os
 import shlex
-import sys
 from collections.abc import Iterator
 from contextlib import contextmanager
 from decimal import Decimal
 from pathlib import Path
 from subprocess import PIPE, STDOUT
 from tempfile import TemporaryDirectory
-from typing import Union
 
 from packaging.version import Version
 from PIL import Image
 
-from ocrmypdf.exceptions import MissingDependencyError, SubprocessOutputError
+from ocrmypdf.exceptions import SubprocessOutputError
 from ocrmypdf.subprocess import get_version, run
 
 # unpaper documentation:
@@ -29,7 +27,7 @@ from ocrmypdf.subprocess import get_version, run
 
 UNPAPER_IMAGE_PIXEL_LIMIT = 256 * 1024 * 1024
 
-DecFloat = Union[Decimal, float]
+DecFloat = Decimal | float
 
 log = logging.getLogger(__name__)
 

src/ocrmypdf/_graft.py

@@ -289,9 +289,8 @@ class OcrGrafter:
 
             # Translate the text so it is centered at (0, 0), rotate it there, adjust
             # for a size different between initial and text PDF, then untranslate, and
-            # finally move the lower left corner to match the mediabox. All transforms
-            # must be premultiplied so they are applied in reverse order here.
-            ctm = corner @ untranslate @ scale @ rotate @ translate
+            # finally move the lower left corner to match the mediabox.
+            ctm = translate @ rotate @ scale @ untranslate @ corner
             log.debug("Grafting with ctm %r", ctm)
 
             base_resources = _ensure_dictionary(base_page.obj, Name.Resources)

src/ocrmypdf/_jobcontext.py

@@ -93,8 +93,8 @@ class PageContext:
         state = self.__dict__.copy()
 
         state['options'] = copy(self.options)
-        if not isinstance(state['options'].input_file, (str, bytes, os.PathLike)):
+        if not isinstance(state['options'].input_file, str | bytes | os.PathLike):
             state['options'].input_file = 'stream'
-        if not isinstance(state['options'].output_file, (str, bytes, os.PathLike)):
+        if not isinstance(state['options'].output_file, str | bytes | os.PathLike):
             state['options'].output_file = 'stream'
         return state

src/ocrmypdf/_pipeline.py

@@ -12,8 +12,9 @@ import re
 import sys
 from collections.abc import Iterable, Iterator, Sequence
 from contextlib import suppress
+from io import BytesIO
 from pathlib import Path
-from shutil import copyfileobj, copystat
+from shutil import copyfileobj
 from typing import Any, BinaryIO, TypeVar, cast
 
 import img2pdf
@@ -713,19 +714,29 @@ def create_pdf_page_from_image(
     pageinfo = page_context.pageinfo
     pagesize = 72.0 * float(pageinfo.width_inches), 72.0 * float(pageinfo.height_inches)
     effective_rotation = (pageinfo.rotation - orientation_correction) % 360
-    if effective_rotation % 180 == 90:
+    swap_axis = effective_rotation % 180 == 90
+    if swap_axis:
         pagesize = pagesize[1], pagesize[0]
 
-    # This create a single page PDF
-    with open(image, 'rb') as imfile, open(output_file, 'wb') as pdf:
+    # Create a new single page PDF to hold
+    bio = BytesIO()
+    with open(image, 'rb') as imfile:
         log.debug('convert')
 
         layout_fun = img2pdf.get_layout_fun(pagesize)
         img2pdf.convert(
-            imfile, layout_fun=layout_fun, outputstream=pdf, **IMG2PDF_KWARGS
+            imfile,
+            layout_fun=layout_fun,
+            outputstream=bio,
+            engine=img2pdf.Engine.pikepdf,
+            rotation=img2pdf.Rotation.ifvalid,
         )
         log.debug('convert done')
 
+    # img2pdf does not generate boxes correctly, so we fix them
+    bio.seek(0)
+    fix_pagepdf_boxes(bio, output_file, page_context, swap_axis=swap_axis)
+
     output_file = page_context.plugin_manager.hook.filter_pdf_page(
         page=page_context, image_filename=image, output_pdf=output_file
     )
@@ -780,7 +791,57 @@ def ocr_engine_textonly_pdf(
         output_text=output_text,
         options=options,
     )
-    return (output_pdf, output_text)
+    return output_pdf, output_text
+
+
+def _offset_rect(rect: tuple[float, float, float, float], offset: tuple[float, float]):
+    """Offset a rectangle by a given amount."""
+    return (
+        rect[0] + offset[0],
+        rect[1] + offset[1],
+        rect[2] + offset[0],
+        rect[3] + offset[1],
+    )
+
+
+def fix_pagepdf_boxes(
+    infile: Path | BinaryIO,
+    out_file: Path,
+    page_context: PageContext,
+    swap_axis: bool = False,
+) -> Path:
+    """Fix the bounding boxes in a single page PDF.
+
+    The single page PDF is created with a normal MediaBox with its lower left corner
+    at (0, 0). infile is the single page PDF. page_context.mediabox has the original
+    file's mediabox, which may have a different origin. We needto adjust the other
+    boxes in the single page PDF to match the effect they had on the original page.
+
+    When correcting page rotation, we create a single page PDF that is correctly
+    rotated instead of an incorrectly rotated and then setting page.Rotate on it.
+    If rotation is either 90 or 270 degrees, then this function can be called
+    with swap_axis to swap the X and Y coordinates of all the boxes.
+
+    We are not concerned with solving degenerate cases where the boxes overlap or
+    or express invalid rectangles. We merely pass the boxes, producing a
+    transformation equivalent to the change made by constructing a new page image.
+    """
+    with pikepdf.open(infile) as pdf:
+        for page in pdf.pages:
+            # page.BleedBox = page_context.pageinfo.bleedbox
+            # page.ArtBox = page_context.pageinfo.artbox
+            mediabox = page_context.pageinfo.mediabox
+            offset = mediabox[0], mediabox[1]
+            cropbox = _offset_rect(page_context.pageinfo.cropbox, offset)
+            trimbox = _offset_rect(page_context.pageinfo.trimbox, offset)
+
+            if swap_axis:
+                cropbox = cropbox[1], cropbox[0], cropbox[3], cropbox[2]
+                trimbox = trimbox[1], trimbox[0], trimbox[3], trimbox[2]
+            page.CropBox = cropbox
+            page.TrimBox = trimbox
+        pdf.save(out_file)
+    return pdf
 
 
 def generate_postscript_stub(context: PdfContext) -> Path:

src/ocrmypdf/_pipelines/_common.py

@@ -11,13 +11,13 @@ import os
 import shutil
 import sys
 import threading
-from collections.abc import Sequence
+from collections.abc import Callable, Sequence
 from concurrent.futures.process import BrokenProcessPool
 from concurrent.futures.thread import BrokenThreadPool
 from contextlib import contextmanager
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Callable, NamedTuple, cast
+from typing import NamedTuple, cast
 
 import PIL
 

src/ocrmypdf/_pipelines/ocr.py

@@ -80,7 +80,6 @@ def _exec_page_sync(page_context: PageContext) -> PageResult:
         page_context
     )
     ocr_out, text_out = _image_to_ocr_text(page_context, ocr_image_out)
-
     return PageResult(
         pageno=page_context.pageno,
         pdf_page_from_image=pdf_page_from_image_out,

src/ocrmypdf/api.py

@@ -14,7 +14,7 @@ from collections.abc import Iterable, Sequence
 from enum import IntEnum
 from io import IOBase
 from pathlib import Path
-from typing import AnyStr, BinaryIO, Union
+from typing import AnyStr, BinaryIO
 from warnings import warn
 
 import pluggy
@@ -28,8 +28,8 @@ from ocrmypdf._validation import check_options
 from ocrmypdf.cli import ArgumentParser, get_parser
 from ocrmypdf.helpers import is_iterable_notstr
 
-StrPath = Union[Path, AnyStr]
-PathOrIO = Union[BinaryIO, StrPath]
+StrPath = Path | AnyStr
+PathOrIO = BinaryIO | StrPath
 
 # Installing plugins affects the global state of the Python interpreter,
 # so we need to use a lock to prevent multiple threads from installing
@@ -169,7 +169,7 @@ def _kwargs_to_cmdline(
 
         # We have a parameter
         cmdline.append(f"--{cmd_style_arg}")
-        if isinstance(val, (int, float)):
+        if isinstance(val, int | float):
             cmdline.append(str(val))
         elif isinstance(val, str):
             cmdline.append(val)
@@ -201,11 +201,11 @@ def create_options(
         defer_kwargs={'progress_bar', 'plugins', 'parser', 'input_file', 'output_file'},
         **kwargs,
     )
-    if isinstance(input_file, (BinaryIO, IOBase)):
+    if isinstance(input_file, BinaryIO | IOBase):
         cmdline.append('stream://input_file')
     else:
         cmdline.append(os.fspath(input_file))
-    if isinstance(output_file, (BinaryIO, IOBase)):
+    if isinstance(output_file, BinaryIO | IOBase):
         cmdline.append('stream://output_file')
     else:
         cmdline.append(os.fspath(output_file))
@@ -343,7 +343,7 @@ def ocr(  # noqa: D417
 
     if not plugins:
         plugins = []
-    elif isinstance(plugins, (str, Path)):
+    elif isinstance(plugins, str | Path):
         plugins = [plugins]
     else:
         plugins = list(plugins)

src/ocrmypdf/builtin_plugins/concurrency.py

@@ -12,10 +12,10 @@ import queue
 import signal
 import sys
 import threading
-from collections.abc import Iterable
+from collections.abc import Callable, Iterable
 from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed
 from contextlib import suppress
-from typing import Callable, Union
+from typing import Union
 
 from rich.console import Console as RichConsole
 
@@ -25,8 +25,10 @@ from ocrmypdf._progressbar import RichProgressBar
 from ocrmypdf.exceptions import InputFileError
 from ocrmypdf.helpers import remove_all_log_handlers
 
-FuturesExecutorClass = Union[type[ThreadPoolExecutor], type[ProcessPoolExecutor]]
-Queue = Union[multiprocessing.Queue, queue.Queue]
+FuturesExecutorClass = Union[  # noqa: UP007
+    type[ThreadPoolExecutor], type[ProcessPoolExecutor]
+]
+Queue = Union[multiprocessing.Queue, queue.Queue]  # noqa: UP007
 UserInit = Callable[[], None]
 WorkerInit = Callable[[Queue, UserInit, int], None]
 
@@ -128,11 +130,14 @@ class StandardExecutor(Executor):
         listener = threading.Thread(target=log_listener, args=(log_queue,))
         listener.start()
 
-        with self.pbar_class(**progress_kwargs) as pbar, executor_class(
-            max_workers=max_workers,
-            initializer=initializer,
-            initargs=(log_queue, worker_initializer, logging.getLogger("").level),
-        ) as executor:
+        with (
+            self.pbar_class(**progress_kwargs) as pbar,
+            executor_class(
+                max_workers=max_workers,
+                initializer=initializer,
+                initargs=(log_queue, worker_initializer, logging.getLogger("").level),
+            ) as executor,
+        ):
             futures = [executor.submit(task, *args) for args in task_arguments]
             try:
                 for future in as_completed(futures):

src/ocrmypdf/builtin_plugins/tesseract_ocr.py

@@ -146,7 +146,11 @@ def check_options(options):
 
     # Decide on what renderer to use
     if options.pdf_renderer == 'auto':
-        options.pdf_renderer = 'sandwich'
+        if {'ara', 'heb', 'fas', 'per'} & set(options.languages):
+            log.info("Using sandwich renderer since there is an RTL language")
+            options.pdf_renderer = 'sandwich'
+        else:
+            options.pdf_renderer = 'hocr'
 
     if not tesseract.has_thresholding() and options.tesseract_thresholding != 0:
         log.warning(

src/ocrmypdf/cli.py

@@ -6,8 +6,8 @@
 from __future__ import annotations
 
 import argparse
-from collections.abc import Mapping
-from typing import Any, Callable, TypeVar
+from collections.abc import Callable, Mapping
+from typing import Any, TypeVar
 
 from ocrmypdf._version import PROGRAM_NAME as _PROGRAM_NAME
 from ocrmypdf._version import __version__ as _VERSION
@@ -390,7 +390,7 @@ Online documentation is located at:
         action='store',
         type=numeric(float, 0),
         metavar='MPixels',
-        help="Set maximum number of pixels to unpack before treating an image as a "
+        help="Set maximum number of megapixels to unpack before treating an image as a "
         "decompression bomb",
         default=250.0,
     )

src/ocrmypdf/extra_plugins/semfree.py

@@ -20,13 +20,12 @@ from __future__ import annotations
 import logging
 import logging.handlers
 import signal
-from collections.abc import Iterable, Iterator
+from collections.abc import Callable, Iterable, Iterator
 from contextlib import suppress
 from enum import Enum, auto
 from itertools import islice, repeat, takewhile, zip_longest
 from multiprocessing import Pipe, Process
 from multiprocessing.connection import Connection, wait
-from typing import Callable
 
 from ocrmypdf import Executor, hookimpl
 from ocrmypdf._concurrent import NullProgressBar

src/ocrmypdf/helpers.py

@@ -10,7 +10,7 @@ import multiprocessing
 import os
 import shutil
 import warnings
-from collections.abc import Iterable, Sequence
+from collections.abc import Callable, Iterable, Sequence
 from contextlib import suppress
 from decimal import Decimal
 from io import StringIO
@@ -19,13 +19,13 @@ from pathlib import Path
 from statistics import harmonic_mean
 from typing import (
     Any,
-    Callable,
     Generic,
     TypeVar,
 )
 
 import img2pdf
 import pikepdf
+from deprecation import deprecated
 
 log = logging.getLogger(__name__)
 
@@ -136,6 +136,7 @@ class Resolution(Generic[T]):
         return self._isclose(self.x, other.x) and self._isclose(self.y, other.y)
 
 
+@deprecated(deprecated_in='15.4.0')
 class NeverRaise(Exception):
     """An exception that is never raised."""
 

src/ocrmypdf/hocrtransform/_hocr.py

@@ -84,7 +84,6 @@ class HocrTransform:
         debug_render_options: DebugRenderOptions | None = None,
     ):
         """Initialize the HocrTransform object."""
-
         if debug:
             log.warning("Use debug_render_options instead", DeprecationWarning)
             self.render_options = DebugRenderOptions(

src/ocrmypdf/optimize.py

@@ -11,11 +11,10 @@ import sys
 import tempfile
 import threading
 from collections import defaultdict
-from collections.abc import Iterator, MutableSet, Sequence
+from collections.abc import Callable, Iterator, MutableSet, Sequence
 from os import fspath
 from pathlib import Path
-from typing import Any, Callable, NamedTuple, NewType
-from warnings import warn
+from typing import Any, NamedTuple, NewType
 from zlib import compress
 
 import img2pdf
@@ -155,6 +154,13 @@ def extract_image_jbig2(
                 with imgname.open('wb') as f:
                     ext = pim.extract_to(stream=f)
                 imgname.rename(imgname.with_suffix(ext))
+            except NotImplementedError as e:
+                if '/Decode' in str(e):
+                    log.debug(
+                        f"xref {xref}: skipping image with unsupported Decode table"
+                    )
+                    return None
+                raise
             except UnsupportedImageTypeError:
                 return None
             finally:

src/ocrmypdf/pdfinfo/info.py

@@ -10,9 +10,8 @@ import atexit
 import logging
 import re
 import statistics
-import sys
 from collections import defaultdict
-from collections.abc import Container, Iterable, Iterator, Mapping, Sequence
+from collections.abc import Callable, Container, Iterable, Iterator, Mapping, Sequence
 from contextlib import contextmanager
 from decimal import Decimal
 from enum import Enum, auto
@@ -20,7 +19,7 @@ from functools import partial
 from math import hypot, inf, isclose
 from os import PathLike
 from pathlib import Path
-from typing import Callable, NamedTuple
+from typing import NamedTuple
 from warnings import warn
 
 from pdfminer.layout import LTPage, LTTextBox
@@ -854,6 +853,12 @@ class PageInfo:
         width_pt = mediabox[2] - mediabox[0]
         height_pt = mediabox[3] - mediabox[1]
 
+        # self._artbox = [float(d) for d in page.artbox.as_list()]
+        # self._bleedbox = [float(d) for d in page.bleedbox.as_list()]
+        self._cropbox = [float(d) for d in page.cropbox.as_list()]
+        self._mediabox = [float(d) for d in page.mediabox.as_list()]
+        self._trimbox = [float(d) for d in page.trimbox.as_list()]
+
         check_this_page = pageno in check_pages
 
         if check_this_page and detailed_analysis:
@@ -970,6 +975,21 @@ class PageInfo:
         else:
             raise ValueError("rotation must be a cardinal angle")
 
+    @property
+    def cropbox(self) -> FloatRect:
+        """Return cropbox of page in PDF coordinates."""
+        return self._cropbox
+
+    @property
+    def mediabox(self) -> FloatRect:
+        """Return mediabox of page in PDF coordinates."""
+        return self._mediabox
+
+    @property
+    def trimbox(self) -> FloatRect:
+        """Return trimbox of page in PDF coordinates."""
+        return self._trimbox
+
     @property
     def images(self) -> list[ImageInfo]:
         """Return images."""
@@ -1039,12 +1059,7 @@ class PageInfo:
 
         weights = [area / total_drawn_area for area in image_areas]
         # Calculate harmonic mean of DPIs weighted by area
-        if sys.version_info >= (3, 10):
-            weighted_dpi = statistics.harmonic_mean(image_dpis, weights)
-        else:
-            weighted_dpi = sum(weights) / sum(
-                weight / dpi for weight, dpi in zip(weights, image_dpis)
-            )
+        weighted_dpi = statistics.harmonic_mean(image_dpis, weights)
         max_dpi = max(image_dpis)
         dpi_average_max_ratio = weighted_dpi / max_dpi
 
@@ -1155,7 +1170,7 @@ class PdfInfo:
     @property
     def filename(self) -> str | Path:
         """Return filename of PDF."""
-        if not isinstance(self._infile, (str, Path)):
+        if not isinstance(self._infile, str | Path):
             raise NotImplementedError("can't get filename from stream")
         return self._infile
 

src/ocrmypdf/pdfinfo/layout.py

@@ -5,12 +5,12 @@
 from __future__ import annotations
 
 import re
-from collections.abc import Mapping
+from collections.abc import Iterator, Mapping
 from contextlib import contextmanager
 from math import copysign
 from os import PathLike
 from pathlib import Path
-from typing import Any, Iterator
+from typing import Any
 from unittest.mock import patch
 
 import pdfminer

src/ocrmypdf/subprocess/__init__.py

@@ -8,12 +8,11 @@ import logging
 import os
 import re
 import sys
-from collections.abc import Mapping, Sequence
+from collections.abc import Callable, Mapping, Sequence
 from contextlib import suppress
 from pathlib import Path
 from subprocess import PIPE, STDOUT, CalledProcessError, CompletedProcess, Popen
 from subprocess import run as subprocess_run
-from typing import Callable, Union
 
 from packaging.version import Version
 
@@ -23,7 +22,7 @@ from ocrmypdf.exceptions import MissingDependencyError
 
 log = logging.getLogger(__name__)
 
-Args = Sequence[Union[Path, str]]
+Args = Sequence[Path | str]
 OsEnviron = os._Environ  # pylint: disable=protected-access
 
 

src/ocrmypdf/subprocess/_windows.py

@@ -9,18 +9,13 @@ import os
 import re
 import shutil
 import sys
-from collections.abc import Iterable, Iterator
+from collections.abc import Callable, Iterable, Iterator
 from itertools import chain
 from pathlib import Path
-from typing import Any, Callable, TypeVar
+from typing import Any, TypeAlias, TypeVar
 
 from packaging.version import InvalidVersion, Version
 
-if sys.version_info >= (3, 10):
-    from typing import TypeAlias
-else:
-    from typing_extensions import TypeAlias  # pragma: no cover
-
 if sys.platform == 'win32':
     # mypy understands 'if sys.platform' better than try/except ModuleNotFoundError
     import winreg  # pylint: disable=import-error

tests/plugins/gs_render_soft_error.py

@@ -3,7 +3,6 @@
 
 from __future__ import annotations
 
-from pathlib import Path
 from subprocess import CalledProcessError
 from unittest.mock import patch
 

tests/test_main.py

@@ -17,7 +17,7 @@ from PIL import Image
 
 import ocrmypdf
 from ocrmypdf._exec import tesseract
-from ocrmypdf.exceptions import ExitCode, MissingDependencyError, OutputFileAccessError
+from ocrmypdf.exceptions import ExitCode, MissingDependencyError
 from ocrmypdf.pdfa import file_claims_pdfa
 from ocrmypdf.pdfinfo import Colorspace, Encoding, PdfInfo
 from ocrmypdf.subprocess import get_version

tests/test_optimize.py

@@ -176,9 +176,10 @@ def test_multiple_pngs(resources, outdir):
         )
         mock.assert_called()
 
-    with pikepdf.open(outdir / 'in.pdf') as inpdf, pikepdf.open(
-        outdir / 'out.pdf'
-    ) as outpdf:
+    with (
+        pikepdf.open(outdir / 'in.pdf') as inpdf,
+        pikepdf.open(outdir / 'out.pdf') as outpdf,
+    ):
         for n in range(len(inpdf.pages)):
             inim = next(iter(inpdf.pages[n].images.values()))
             outim = next(iter(outpdf.pages[n].images.values()))

tests/test_preprocessing.py

@@ -9,10 +9,11 @@ import pytest
 from PIL import Image
 
 from ocrmypdf._exec import ghostscript, tesseract
+from ocrmypdf.exceptions import ExitCode
 from ocrmypdf.helpers import Resolution
 from ocrmypdf.pdfinfo import PdfInfo
 
-from .conftest import check_ocrmypdf, have_unpaper
+from .conftest import check_ocrmypdf, have_unpaper, run_ocrmypdf
 
 RENDERERS = ['hocr', 'sandwich']
 
@@ -107,7 +108,7 @@ def test_non_square_resolution(renderer, resources, outpdf):
     in_pageinfo = PdfInfo(resources / 'aspect.pdf')
     assert in_pageinfo[0].dpi.x != in_pageinfo[0].dpi.y
 
-    check_ocrmypdf(
+    proc = run_ocrmypdf(
         resources / 'aspect.pdf',
         outpdf,
         '--pdf-renderer',
@@ -115,6 +116,10 @@ def test_non_square_resolution(renderer, resources, outpdf):
         '--plugin',
         'tests/plugins/tesseract_cache.py',
     )
+    # PDF/A conversion can fail for this file if Ghostscript >= 10.3, so don't test
+    # exit code in that case
+    if proc.returncode != ExitCode.pdfa_conversion_failed:
+        proc.check_returncode()
 
     out_pageinfo = PdfInfo(outpdf)
 

tests/test_rotation.py

@@ -4,10 +4,10 @@
 from __future__ import annotations
 
 import operator
-import warnings
 from io import BytesIO
 from math import cos, pi, sin
 from os import fspath
+from subprocess import run
 
 import img2pdf
 import pikepdf
@@ -22,10 +22,6 @@ from ocrmypdf.pdfinfo import PdfInfo
 
 from .conftest import check_ocrmypdf, run_ocrmypdf_api
 
-warnings.filterwarnings(
-    "ignore", category=DeprecationWarning, module="reportlab.lib.rl_safe_eval"
-)
-
 # pylintx: disable=unused-variable
 
 RENDERERS = ['hocr', 'sandwich']
@@ -215,34 +211,35 @@ def test_rotate_deskew_ocr_timeout(resources, outdir):
     assert cmp > 0.95
 
 
+def make_rotate_test(imagefile, outdir, prefix, image_angle, page_angle):
+    memimg = BytesIO()
+    with Image.open(fspath(imagefile)) as im:
+        if image_angle != 0:
+            ccw_angle = -image_angle % 360
+            im = im.transpose(getattr(Image.Transpose, f'ROTATE_{ccw_angle}'))
+        im.save(memimg, format='PNG')
+    memimg.seek(0)
+    mempdf = BytesIO()
+    img2pdf.convert(
+        memimg.read(),
+        layout_fun=img2pdf.get_fixed_dpi_layout_fun((200, 200)),
+        outputstream=mempdf,
+        **IMG2PDF_KWARGS,
+    )
+    mempdf.seek(0)
+    with pikepdf.open(mempdf) as pdf:
+        pdf.pages[0].Rotate = page_angle
+        target = outdir / f'{prefix}_{image_angle}_{page_angle}.pdf'
+        pdf.save(target)
+        return target
+
+
 @pytest.mark.slow
 @pytest.mark.parametrize('page_angle', (0, 90, 180, 270))
 @pytest.mark.parametrize('image_angle', (0, 90, 180, 270))
 def test_rotate_page_level(image_angle, page_angle, resources, outdir, caplog):
-    def make_rotate_test(prefix, image_angle, page_angle):
-        memimg = BytesIO()
-        with Image.open(fspath(resources / 'typewriter.png')) as im:
-            if image_angle != 0:
-                ccw_angle = -image_angle % 360
-                im = im.transpose(getattr(Image.Transpose, f'ROTATE_{ccw_angle}'))
-            im.save(memimg, format='PNG')
-        memimg.seek(0)
-        mempdf = BytesIO()
-        img2pdf.convert(
-            memimg.read(),
-            layout_fun=img2pdf.get_fixed_dpi_layout_fun((200, 200)),
-            outputstream=mempdf,
-            **IMG2PDF_KWARGS,
-        )
-        mempdf.seek(0)
-        with pikepdf.open(mempdf) as pdf:
-            pdf.pages[0].Rotate = page_angle
-            target = outdir / f'{prefix}_{image_angle}_{page_angle}.pdf'
-            pdf.save(target)
-            return target
-
-    reference = make_rotate_test('ref', 0, 0)
-    test = make_rotate_test('test', image_angle, page_angle)
+    reference = make_rotate_test(resources / 'typewriter.png', outdir, 'ref', 0, 0)
+    test = make_rotate_test(resources, outdir, 'test', image_angle, page_angle)
     out = test.with_suffix('.out.pdf')
 
     exitcode = run_ocrmypdf_api(
@@ -258,6 +255,33 @@ def test_rotate_page_level(image_angle, page_angle, resources, outdir, caplog):
     assert compare_images_monochrome(outdir, reference, 1, out, 1) > 0.2
 
 
+@pytest.mark.slow
+@pytest.mark.parametrize('page_rotate_angle', (0, 90, 180, 270))
+def test_page_rotate_tag(page_rotate_angle, resources, outdir, caplog):
+    # Check that pages that have an image that is misrotated but restored to
+    # correct rotation with a /Rotate will be processed correct and yield text.
+    test = make_rotate_test(
+        resources / 'crom.png', outdir, 'test', -page_rotate_angle, page_rotate_angle
+    )
+    out = test.with_suffix('.out.pdf')
+    exitcode = run_ocrmypdf_api(
+        test,
+        out,
+        '-O0',
+    )
+    assert exitcode == 0, caplog.text
+
+    def pdftotext(filename):
+        return (
+            run(['pdftotext', '-enc', 'UTF-8', filename, '-'], capture_output=True)
+            .stdout.strip()
+            .decode('utf-8')
+        )
+
+    test_text = pdftotext(out)
+    assert 'is a' in test_text, test_text
+
+
 def test_rasterize_rotates(resources, tmp_path):
     pm = get_plugin_manager([])
 

tests/test_userunit.py

@@ -7,10 +7,9 @@ from math import isclose
 
 import pytest
 
-from ocrmypdf.exceptions import ExitCode
 from ocrmypdf.pdfinfo import PdfInfo
 
-from .conftest import check_ocrmypdf, run_ocrmypdf_api
+from .conftest import check_ocrmypdf
 
 # pylint: disable=redefined-outer-name