Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3336d67e77 | ||
|
|
73e16e7821 | ||
|
|
6f1d37d78f | ||
|
|
2ed82de2e0 | ||
|
|
c43903fa14 | ||
|
|
1c89cacfef | ||
|
|
75714fe43e | ||
|
|
e371ce95ca | ||
|
|
716a2e22c3 | ||
|
|
10e6019ada | ||
|
|
89c76b5145 | ||
|
|
83c04e6399 | ||
|
|
7fdeeb3635 | ||
|
|
5be368fe75 | ||
|
|
91c5b1e480 | ||
|
|
73154b97ba | ||
|
|
76a40759ae | ||
|
|
12ce565e98 | ||
|
|
9f46126859 | ||
|
|
11849e5a70 | ||
|
|
e30c00cc26 | ||
|
|
001b403657 | ||
|
|
851c61ee85 | ||
|
|
f5ebd23b8f | ||
|
|
81118c6195 | ||
|
|
834b60a02a | ||
|
|
47e3b5b4d2 | ||
|
|
d9346cc3d8 | ||
|
|
a35dd1f9ee | ||
|
|
bf46f4fe35 | ||
|
|
55b76338a8 | ||
|
|
2af7b1c179 | ||
|
|
69f4cca9b6 | ||
|
|
4babdfcfbf |
+10
-10
@@ -87,7 +87,7 @@ jobs:
|
||||
uv run --no-dev pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
|
||||
|
||||
- name: Upload coverage to Codecov
|
||||
uses: codecov/codecov-action@v5
|
||||
uses: codecov/codecov-action@v6
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
@@ -149,7 +149,7 @@ jobs:
|
||||
uv run --no-dev pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
|
||||
|
||||
- name: Upload coverage to Codecov
|
||||
uses: codecov/codecov-action@v5
|
||||
uses: codecov/codecov-action@v6
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
@@ -196,7 +196,7 @@ jobs:
|
||||
uv run --no-dev pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
|
||||
|
||||
- name: Upload coverage to Codecov
|
||||
uses: codecov/codecov-action@v5
|
||||
uses: codecov/codecov-action@v6
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
@@ -218,7 +218,7 @@ jobs:
|
||||
run: |
|
||||
uv build --sdist --wheel
|
||||
|
||||
- uses: actions/upload-artifact@v6
|
||||
- uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: artifact
|
||||
path: |
|
||||
@@ -235,7 +235,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- uses: actions/download-artifact@v7
|
||||
- uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: artifact
|
||||
path: dist
|
||||
@@ -284,17 +284,17 @@ jobs:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: jbarlow83
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
uses: docker/setup-qemu-action@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Print image tag
|
||||
run: echo "Building image ${DOCKER_REPOSITORY}/${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}"
|
||||
@@ -330,14 +330,14 @@ jobs:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: jbarlow83
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Print image tag
|
||||
run: echo "Building image ${DOCKER_REPOSITORY}/${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}"
|
||||
|
||||
@@ -34,7 +34,7 @@ jobs:
|
||||
|
||||
# PyPI doesn't support sigstore publishing, so generate after publishing to PyPI
|
||||
- name: Sign the dists with Sigstore
|
||||
uses: sigstore/gh-action-sigstore-python@v3.2.0
|
||||
uses: sigstore/gh-action-sigstore-python@v3.3.0
|
||||
with:
|
||||
inputs: |
|
||||
./dist/*.tar.gz
|
||||
@@ -87,13 +87,13 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: jbarlow83
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Tag ocrmypdf (Ubuntu) image
|
||||
run: |
|
||||
|
||||
@@ -3,6 +3,27 @@
|
||||
|
||||
# v17
|
||||
|
||||
## v17.4.2
|
||||
|
||||
- Fixed Python API unconditionally overriding ``PIL.Image.MAX_IMAGE_PIXELS``
|
||||
when the caller did not explicitly set ``max_image_mpixels``. Host
|
||||
applications (e.g. Paperless-NGX) that configure the PIL limit before
|
||||
invoking ``ocrmypdf.ocr()`` now have their setting respected. The CLI
|
||||
default of 250 megapixels is unchanged. {issue}`1665`
|
||||
- Updated uv.lock to avoid pinning a vulnerable version of Pillow. {issue}`1666`
|
||||
|
||||
## v17.4.1
|
||||
|
||||
- Fixed RTL text extraction order in the fpdf2 renderer. Arabic lam-alef
|
||||
ligatures and other multi-character CMap entries were garbled by the bidi
|
||||
algorithm during text extraction. {issue}`1655`
|
||||
- Fixed ``work_folder`` not being set in ``PdfContext`` options when using
|
||||
the Python API. Thanks @bluebox-steven. {issue}`1613`
|
||||
- Updated Ghostscript JPEG corruption warning to include the detected version
|
||||
number, confirming the bug persists in Ghostscript 10.7.0.
|
||||
- Internal refactoring.
|
||||
- CI dependency updates.
|
||||
|
||||
## v17.4.0
|
||||
|
||||
- Added ``--no-overwrite`` / ``-n`` option to prevent overwriting output files.
|
||||
|
||||
+2
-1
@@ -6,7 +6,7 @@ build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "ocrmypdf"
|
||||
version = "17.4.0"
|
||||
version = "17.4.2"
|
||||
description = "OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched"
|
||||
readme = "README.md"
|
||||
license = "MPL-2.0"
|
||||
@@ -161,6 +161,7 @@ dev = [
|
||||
"ipykernel>=6.29.5",
|
||||
"reportlab>=4.4.4",
|
||||
"cyclopts>=4.5.1",
|
||||
"pygithub>=2.9.1",
|
||||
]
|
||||
test = [
|
||||
# Core testing framework
|
||||
|
||||
@@ -36,6 +36,7 @@ class PdfContext:
|
||||
plugin_manager,
|
||||
):
|
||||
self.options = options
|
||||
self.options.work_folder = work_folder
|
||||
self.work_folder = work_folder
|
||||
self.origin = origin
|
||||
self.pdfinfo = pdfinfo
|
||||
|
||||
@@ -192,7 +192,7 @@ class OcrOptions(BaseModel):
|
||||
no_overwrite: bool = False
|
||||
|
||||
# Advanced options
|
||||
max_image_mpixels: float = 250.0
|
||||
max_image_mpixels: float | None = None
|
||||
pdf_renderer: str = 'auto'
|
||||
ocr_engine: str = 'auto'
|
||||
rasterizer: str = 'auto'
|
||||
@@ -301,7 +301,7 @@ class OcrOptions(BaseModel):
|
||||
@classmethod
|
||||
def validate_max_image_mpixels(cls, v):
|
||||
"""Validate max image megapixels."""
|
||||
if v < 0:
|
||||
if v is not None and v < 0:
|
||||
raise ValueError("max_image_mpixels must be non-negative")
|
||||
return v
|
||||
|
||||
|
||||
@@ -329,10 +329,13 @@ def setup_pipeline(
|
||||
# Note: OcrOptions is immutable, so we can't modify options.jobs directly
|
||||
# The jobs field should already be set correctly during OcrOptions creation
|
||||
|
||||
# Apply PIL max image pixels side effect
|
||||
PIL.Image.MAX_IMAGE_PIXELS = int(options.max_image_mpixels * 1_000_000)
|
||||
if PIL.Image.MAX_IMAGE_PIXELS == 0:
|
||||
PIL.Image.MAX_IMAGE_PIXELS = None # type: ignore
|
||||
# Apply PIL max image pixels side effect only when explicitly requested.
|
||||
# When None, leave PIL.Image.MAX_IMAGE_PIXELS as the host application
|
||||
# configured it. The CLI passes its own default (250.0) via argparse.
|
||||
if options.max_image_mpixels is not None:
|
||||
PIL.Image.MAX_IMAGE_PIXELS = int(options.max_image_mpixels * 1_000_000)
|
||||
if PIL.Image.MAX_IMAGE_PIXELS == 0:
|
||||
PIL.Image.MAX_IMAGE_PIXELS = None # type: ignore
|
||||
|
||||
pikepdf_enable_mmap()
|
||||
executor = setup_executor(plugin_manager)
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
# SPDX-FileCopyrightText: 2022 James R. Barlow
|
||||
# SPDX-License-Identifier: MPL-2.0
|
||||
__version__ = "17.4.0"
|
||||
__version__ = "17.4.2"
|
||||
|
||||
@@ -131,10 +131,10 @@ def check_options(options):
|
||||
)
|
||||
if gs_version >= Version('10.6.0'):
|
||||
log.warning(
|
||||
"Ghostscript 10.6.x contains JPEG encoding errors that may corrupt "
|
||||
"images. OCRmyPDF will attempt to mitigate, but this version is "
|
||||
"strongly not recommended. Please upgrade to a newer version. "
|
||||
"As of 2025-12, 10.6.0 is the latest version of Ghostscript."
|
||||
"Ghostscript %s contains JPEG encoding errors that may corrupt "
|
||||
"images. OCRmyPDF will attempt to mitigate, but versions 10.6.0+ "
|
||||
"are strongly not recommended until this is fixed upstream.",
|
||||
gs_version,
|
||||
)
|
||||
if options.output_type == 'pdfa':
|
||||
options.output_type = 'pdfa-2'
|
||||
|
||||
@@ -10,6 +10,7 @@ OCR text layers.
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import unicodedata
|
||||
from dataclasses import dataclass
|
||||
from math import atan, cos, degrees, radians, sin, sqrt
|
||||
from pathlib import Path
|
||||
@@ -24,6 +25,21 @@ from ocrmypdf.models.ocr_element import OcrClass, OcrElement
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _is_rtl_text(text: str) -> bool:
|
||||
"""Check if text is right-to-left based on Unicode bidi properties.
|
||||
|
||||
Looks for the first character with a strong directional type
|
||||
(R, AL, or L) to determine the text's base direction.
|
||||
"""
|
||||
for char in text:
|
||||
bidi = unicodedata.bidirectional(char)
|
||||
if bidi in ('R', 'AL'):
|
||||
return True
|
||||
if bidi == 'L':
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
def transform_point(matrix: Matrix, x: float, y: float) -> tuple[float, float]:
|
||||
"""Transform a point (x, y) by a matrix.
|
||||
|
||||
@@ -66,6 +82,17 @@ def transform_box(
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class WordRenderData:
|
||||
"""Rendering parameters for a single word on a line."""
|
||||
|
||||
text: str
|
||||
x_baseline: float
|
||||
font_family: str
|
||||
word_tz: float
|
||||
is_rtl: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
class DebugRenderOptions:
|
||||
"""Options for debug visualization during rendering.
|
||||
@@ -426,8 +453,7 @@ class Fpdf2PdfRenderer:
|
||||
):
|
||||
return
|
||||
|
||||
# Collect word rendering data: (text, x_baseline, font_family, word_tz)
|
||||
word_render_data: list[tuple[str, float, str, float]] = []
|
||||
word_render_data: list[WordRenderData] = []
|
||||
for word in words:
|
||||
if word is None or not word.text or word.bbox is None:
|
||||
continue
|
||||
@@ -459,13 +485,35 @@ class Fpdf2PdfRenderer:
|
||||
)
|
||||
font_family = self._register_font(pdf, font_manager)
|
||||
pdf.set_font(font_family, size=font_size)
|
||||
natural_width = pdf.get_string_width(word.text)
|
||||
|
||||
# For RTL words with invisible text, we use encode_text()
|
||||
# (which maps characters 1:1 in logical order) combined with
|
||||
# a -1 x-scale text matrix. This avoids an fpdf2 issue where
|
||||
# shaped RTL ligature glyphs (e.g. lam-alef) get multi-char
|
||||
# CMap entries whose character order is reversed by the bidi
|
||||
# algorithm during text extraction.
|
||||
# Since the text is invisible, glyph mirroring is harmless.
|
||||
# Compute Tz using unshaped widths to match encode_text().
|
||||
word_is_rtl = self.invisible_text and _is_rtl_text(word.text)
|
||||
if word_is_rtl:
|
||||
saved_shaping = pdf.text_shaping
|
||||
pdf.text_shaping = None
|
||||
natural_width = pdf.get_string_width(word.text)
|
||||
pdf.text_shaping = saved_shaping
|
||||
else:
|
||||
natural_width = pdf.get_string_width(word.text)
|
||||
if natural_width > 0 and word_width_pt > 0:
|
||||
word_tz = (word_width_pt / natural_width) * 100
|
||||
else:
|
||||
word_tz = 100.0
|
||||
|
||||
word_render_data.append((word.text, box_llx, font_family, word_tz))
|
||||
word_render_data.append(WordRenderData(
|
||||
text=word.text,
|
||||
x_baseline=box_llx,
|
||||
font_family=font_family,
|
||||
word_tz=word_tz,
|
||||
is_rtl=word_is_rtl,
|
||||
))
|
||||
|
||||
if not word_render_data:
|
||||
return
|
||||
@@ -564,7 +612,7 @@ class Fpdf2PdfRenderer:
|
||||
def _emit_line_bt_block(
|
||||
self,
|
||||
pdf: FPDF,
|
||||
word_render_data: list[tuple[str, float, str, float]],
|
||||
word_render_data: list[WordRenderData],
|
||||
baseline_matrix: Matrix,
|
||||
font_size: float,
|
||||
total_rotation_deg: float,
|
||||
@@ -580,8 +628,7 @@ class Fpdf2PdfRenderer:
|
||||
|
||||
Args:
|
||||
pdf: FPDF instance
|
||||
word_render_data: List of (text, x_baseline, font_family, word_tz)
|
||||
tuples, one per word on this line
|
||||
word_render_data: List of WordRenderData, one per word on this line
|
||||
baseline_matrix: Transform from baseline coords to page coords
|
||||
font_size: Font size in points
|
||||
total_rotation_deg: Total rotation angle (textangle + slope)
|
||||
@@ -626,7 +673,7 @@ class Fpdf2PdfRenderer:
|
||||
ops.append(f'{tr} Tr')
|
||||
|
||||
# Initial text position
|
||||
first_x_baseline = word_render_data[0][1]
|
||||
first_x_baseline = word_render_data[0].x_baseline
|
||||
if has_rotation:
|
||||
# In the cm-transformed space, origin is at the baseline start
|
||||
ops.append(f'{first_x_baseline:.2f} 0 Td')
|
||||
@@ -641,14 +688,12 @@ class Fpdf2PdfRenderer:
|
||||
prev_font_family: str | None = None
|
||||
prev_x_baseline = first_x_baseline
|
||||
|
||||
for i, (text, x_baseline, font_family, word_tz) in enumerate(
|
||||
word_render_data
|
||||
):
|
||||
for i, word in enumerate(word_render_data):
|
||||
is_last = i == len(word_render_data) - 1
|
||||
|
||||
# Set font if changed
|
||||
if font_family != prev_font_family:
|
||||
pdf.set_font(font_family, size=font_size)
|
||||
if word.font_family != prev_font_family:
|
||||
pdf.set_font(word.font_family, size=font_size)
|
||||
# Register font resource on this page
|
||||
pdf._resource_catalog.add(
|
||||
PDFResourceType.FONT, pdf.current_font.i, pdf.page
|
||||
@@ -656,13 +701,13 @@ class Fpdf2PdfRenderer:
|
||||
ops.append(
|
||||
f'/F{pdf.current_font.i} {pdf.font_size_pt:.2f} Tf'
|
||||
)
|
||||
prev_font_family = font_family
|
||||
prev_font_family = word.font_family
|
||||
|
||||
# Relative positioning (for words after the first)
|
||||
if i > 0:
|
||||
if has_rotation:
|
||||
# In rotated space, advance is purely along x-axis
|
||||
dx_baseline = x_baseline - prev_x_baseline
|
||||
dx_baseline = word.x_baseline - prev_x_baseline
|
||||
ops.append(f'{dx_baseline:.2f} 0 Td')
|
||||
else:
|
||||
# Non-rotated: compute delta in PDF coordinates
|
||||
@@ -670,7 +715,7 @@ class Fpdf2PdfRenderer:
|
||||
baseline_matrix, prev_x_baseline, 0
|
||||
)
|
||||
px_curr, py_curr_f = transform_point(
|
||||
baseline_matrix, x_baseline, 0
|
||||
baseline_matrix, word.x_baseline, 0
|
||||
)
|
||||
dx_pdf = px_curr - px_prev
|
||||
# Flip y delta for PDF coordinates (y-up)
|
||||
@@ -679,31 +724,31 @@ class Fpdf2PdfRenderer:
|
||||
|
||||
# Determine text to render
|
||||
if not is_last:
|
||||
next_text, next_x_baseline, _, _ = word_render_data[i + 1]
|
||||
advance = next_x_baseline - x_baseline
|
||||
next_word = word_render_data[i + 1]
|
||||
advance = next_word.x_baseline - word.x_baseline
|
||||
|
||||
# Add trailing space for text extraction unless both are CJK
|
||||
if (
|
||||
advance > 0
|
||||
and not (
|
||||
self._is_cjk_only(text)
|
||||
and self._is_cjk_only(next_text)
|
||||
self._is_cjk_only(word.text)
|
||||
and self._is_cjk_only(next_word.text)
|
||||
)
|
||||
):
|
||||
text_to_render = text + ' '
|
||||
text_to_render = word.text + ' '
|
||||
else:
|
||||
text_to_render = text
|
||||
text_to_render = word.text
|
||||
else:
|
||||
text_to_render = text
|
||||
text_to_render = word.text
|
||||
|
||||
# Use word_tz (fits word into its hOCR bbox) — Td handles
|
||||
# inter-word gaps, so Tz should not stretch to fill them.
|
||||
render_tz = word_tz
|
||||
ops.append(f'{word.word_tz:.2f} Tz')
|
||||
ops.append(
|
||||
self._encode_shaped_text(pdf, text_to_render, word.is_rtl)
|
||||
)
|
||||
|
||||
ops.append(f'{render_tz:.2f} Tz')
|
||||
ops.append(self._encode_shaped_text(pdf, text_to_render))
|
||||
|
||||
prev_x_baseline = x_baseline
|
||||
prev_x_baseline = word.x_baseline
|
||||
|
||||
# End text object
|
||||
ops.append('ET')
|
||||
@@ -717,15 +762,35 @@ class Fpdf2PdfRenderer:
|
||||
# don't think Tz is still set from our raw operators
|
||||
pdf.font_stretching = 100
|
||||
|
||||
def _encode_shaped_text(self, pdf: FPDF, text: str) -> str:
|
||||
def _encode_shaped_text(
|
||||
self, pdf: FPDF, text: str, is_rtl: bool = False
|
||||
) -> str:
|
||||
"""Encode text using HarfBuzz text shaping for complex script support.
|
||||
|
||||
Unlike font.encode_text() which maps unicode characters one-by-one to
|
||||
glyph IDs, this uses HarfBuzz to handle BiDi reordering, Arabic joining
|
||||
forms, Devanagari conjuncts, and other complex script shaping. Falls
|
||||
back to encode_text() when text shaping is not enabled.
|
||||
|
||||
For RTL words with invisible text, we use encode_text() instead of
|
||||
shape_text(). fpdf2's shape_text() produces RTL ligature glyphs
|
||||
(e.g. lam-alef) with multi-character CMap entries whose character
|
||||
order gets reversed by the bidi algorithm during text extraction,
|
||||
producing garbled output (e.g. "سالح" instead of "سلاح").
|
||||
encode_text() maps characters 1:1 in logical order, giving correct
|
||||
extraction. Since the text is invisible (Tr=3), the lack of proper
|
||||
joining forms and ligature shaping is harmless.
|
||||
"""
|
||||
font = pdf.current_font
|
||||
if is_rtl:
|
||||
# Reverse the text so that after bidi reversal by the text
|
||||
# extractor, the characters end up in correct logical order.
|
||||
# The text cursor advances left-to-right from the word's left
|
||||
# edge (set by Td), so characters are positioned left-to-right
|
||||
# in the PDF. The extractor sees RTL characters in L-to-R
|
||||
# positions and applies bidi reversal, which reverses them.
|
||||
# By pre-reversing, the double reversal yields the original.
|
||||
return font.encode_text(text[::-1])
|
||||
if pdf.text_shaping and pdf.text_shaping.get("use_shaping_engine"):
|
||||
shaped = font.shape_text(text, pdf.font_size_pt, pdf.text_shaping)
|
||||
if shaped:
|
||||
|
||||
+3
-6
@@ -19,14 +19,11 @@ from ocrmypdf.exceptions import ExitCode
|
||||
|
||||
|
||||
class Gs106WarningFilter(logging.Filter):
|
||||
"""Filter out expected Ghostscript 10.6.x warning from test logs."""
|
||||
"""Filter out expected Ghostscript 10.6+ JPEG encoding warning from test logs."""
|
||||
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
# Allow all records except the expected Ghostscript 10.6.x warning
|
||||
return (
|
||||
"Ghostscript 10.6.x contains JPEG encoding errors"
|
||||
not in record.getMessage()
|
||||
)
|
||||
# Allow all records except the expected Ghostscript 10.6+ warning
|
||||
return "contains JPEG encoding errors" not in record.getMessage()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
|
||||
@@ -5,6 +5,8 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
@@ -37,6 +39,26 @@ def resources():
|
||||
return Path(__file__).parent / "resources"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def pdftotext():
|
||||
"""Return a function to extract text from PDF using pdftotext.
|
||||
|
||||
Skips the test if pdftotext is not available.
|
||||
"""
|
||||
pdftotext_path = shutil.which('pdftotext')
|
||||
if pdftotext_path is None:
|
||||
pytest.skip("pdftotext not available")
|
||||
|
||||
def extract_text(pdf_path: Path) -> str:
|
||||
return subprocess.check_output(
|
||||
['pdftotext', '-enc', 'UTF-8', str(pdf_path), '-'],
|
||||
text=True,
|
||||
encoding='utf-8',
|
||||
)
|
||||
|
||||
return extract_text
|
||||
|
||||
|
||||
class TestFpdf2RendererImports:
|
||||
"""Test that all fpdf2 renderer modules can be imported."""
|
||||
|
||||
@@ -443,14 +465,16 @@ class TestWordSegmentation:
|
||||
assert "World" in words_found
|
||||
assert "Test" in words_found
|
||||
|
||||
def test_cjk_no_spurious_spaces(self, multi_font_manager, tmp_path):
|
||||
def test_cjk_no_spurious_spaces(self, multi_font_manager, tmp_path, pdftotext):
|
||||
"""Test that CJK text does not get spurious spaces inserted.
|
||||
|
||||
CJK scripts don't use spaces between characters/words, so we should
|
||||
not insert spaces between adjacent CJK words.
|
||||
"""
|
||||
from pdfminer.high_level import extract_text
|
||||
|
||||
Uses pdftotext (poppler) instead of pdfminer.six because the latter
|
||||
cannot decode the custom Encoding CMap that fpdf2 >= 2.8.7 emits for
|
||||
subsetted CFF-based CID fonts (e.g. NotoSansCJK).
|
||||
"""
|
||||
from ocrmypdf.models.ocr_element import BoundingBox, OcrElement
|
||||
|
||||
# Create a page with CJK words (Chinese characters)
|
||||
@@ -487,15 +511,15 @@ class TestWordSegmentation:
|
||||
output_path = tmp_path / "test_cjk_segmentation.pdf"
|
||||
renderer.render(output_path)
|
||||
|
||||
# Extract text using pdfminer.six
|
||||
extracted_text = extract_text(str(output_path))
|
||||
extracted_text = pdftotext(output_path)
|
||||
|
||||
# CJK text should be present
|
||||
assert "你好" in extracted_text
|
||||
assert "世界" in extracted_text
|
||||
|
||||
# There should NOT be spaces between CJK characters
|
||||
# (but pdfminer may add some whitespace, so we check the raw chars)
|
||||
# (a space between the two words is acceptable, since they are
|
||||
# separated horizontally on the rendered page)
|
||||
extracted_chars = extracted_text.replace(" ", "").replace("\n", "")
|
||||
assert "你好世界" in extracted_chars or (
|
||||
"你好" in extracted_chars and "世界" in extracted_chars
|
||||
|
||||
@@ -5,9 +5,11 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from io import StringIO
|
||||
from pathlib import Path
|
||||
|
||||
import pikepdf
|
||||
import pytest
|
||||
from pdfminer.converter import TextConverter
|
||||
from pdfminer.layout import LAParams
|
||||
@@ -597,3 +599,312 @@ class TestFpdf2PdfRendererLineTypes:
|
||||
check_pdf(str(output_pdf))
|
||||
extracted_text = text_from_pdf(output_pdf)
|
||||
assert "Caption" in extracted_text
|
||||
|
||||
|
||||
def create_rtl_page(
|
||||
words: list[tuple[str, tuple[float, float, float, float]]],
|
||||
language: str = "ara",
|
||||
width: float = 1000,
|
||||
height: float = 500,
|
||||
) -> OcrElement:
|
||||
"""Create an OcrElement page with a single RTL paragraph/line.
|
||||
|
||||
Args:
|
||||
words: List of (text, (left, top, right, bottom)) tuples.
|
||||
language: Language code for the paragraph.
|
||||
width: Page width in pixels.
|
||||
height: Page height in pixels.
|
||||
|
||||
Returns:
|
||||
OcrElement page.
|
||||
"""
|
||||
word_elements = [
|
||||
OcrElement(
|
||||
ocr_class=OcrClass.WORD,
|
||||
text=text,
|
||||
bbox=BoundingBox(
|
||||
left=bbox[0], top=bbox[1], right=bbox[2], bottom=bbox[3]
|
||||
),
|
||||
)
|
||||
for text, bbox in words
|
||||
]
|
||||
line = OcrElement(
|
||||
ocr_class=OcrClass.LINE,
|
||||
bbox=BoundingBox(left=50, top=100, right=950, bottom=200),
|
||||
baseline=Baseline(slope=0.0, intercept=0),
|
||||
direction="rtl",
|
||||
children=word_elements,
|
||||
)
|
||||
paragraph = OcrElement(
|
||||
ocr_class=OcrClass.PARAGRAPH,
|
||||
bbox=BoundingBox(left=50, top=100, right=950, bottom=200),
|
||||
direction="rtl",
|
||||
language=language,
|
||||
children=[line],
|
||||
)
|
||||
return OcrElement(
|
||||
ocr_class=OcrClass.PAGE,
|
||||
bbox=BoundingBox(left=0, top=0, right=width, bottom=height),
|
||||
children=[paragraph],
|
||||
)
|
||||
|
||||
|
||||
def _tounicode_map(pdf_path: Path) -> dict[int, str]:
|
||||
"""Extract all ToUnicode CMap entries from the first page's OCR overlay.
|
||||
|
||||
Returns a dict mapping subset glyph index -> unicode string.
|
||||
"""
|
||||
pdf = pikepdf.open(pdf_path)
|
||||
page = pdf.pages[0]
|
||||
resources = page.get('/Resources', {})
|
||||
|
||||
# Collect fonts from the page and from any Form XObjects (OCR overlay)
|
||||
fonts: dict[str, pikepdf.Object] = {}
|
||||
if '/Font' in resources:
|
||||
for name, obj in resources['/Font'].items():
|
||||
fonts[str(name)] = obj
|
||||
for xobj in resources.get('/XObject', {}).values():
|
||||
if xobj.get('/Subtype') == '/Form':
|
||||
for name, obj in xobj.get('/Resources', {}).get('/Font', {}).items():
|
||||
fonts[str(name)] = obj
|
||||
|
||||
result: dict[int, str] = {}
|
||||
for fobj in fonts.values():
|
||||
tounicode = fobj.get('/ToUnicode')
|
||||
if tounicode is None:
|
||||
continue
|
||||
cmap = bytes(tounicode.read_bytes()).decode('latin-1', errors='replace')
|
||||
for m in re.finditer(r'<([0-9A-Fa-f]+)>\s*<([0-9A-Fa-f]+)>', cmap):
|
||||
src_int = int(m.group(1), 16)
|
||||
dst_hex = m.group(2)
|
||||
chars = ''.join(
|
||||
chr(int(dst_hex[i : i + 4], 16))
|
||||
for i in range(0, len(dst_hex), 4)
|
||||
if int(dst_hex[i : i + 4], 16) > 0
|
||||
)
|
||||
if src_int > 0 and chars:
|
||||
result[src_int] = chars
|
||||
return result
|
||||
|
||||
|
||||
def _decode_tounicode_stream(
|
||||
pdf_path: Path,
|
||||
) -> tuple[dict[int, str], list[int]]:
|
||||
"""Extract ToUnicode CMap and Tj glyph stream from a test PDF.
|
||||
|
||||
Searches the page content stream and any Form XObjects for fonts
|
||||
and Tj operations.
|
||||
|
||||
Returns:
|
||||
(cmap, glyph_ids) where *cmap* maps subset index -> Unicode string
|
||||
and *glyph_ids* is the flat list of 2-byte glyph indices found in
|
||||
the first Tj string.
|
||||
"""
|
||||
pdf = pikepdf.open(pdf_path)
|
||||
page = pdf.pages[0]
|
||||
resources = page.get('/Resources', {})
|
||||
|
||||
# Collect fonts from page and from Form XObjects
|
||||
cmap: dict[int, str] = {}
|
||||
for font_dict in [resources.get('/Font', {})]:
|
||||
for fobj in font_dict.values():
|
||||
tounicode = fobj.get('/ToUnicode')
|
||||
if tounicode is None:
|
||||
continue
|
||||
raw = bytes(tounicode.read_bytes()).decode('latin-1', errors='replace')
|
||||
for m in re.finditer(r'<([0-9A-Fa-f]+)>\s*<([0-9A-Fa-f]+)>', raw):
|
||||
src = int(m.group(1), 16)
|
||||
dst_hex = m.group(2)
|
||||
chars = ''.join(
|
||||
chr(int(dst_hex[i : i + 4], 16))
|
||||
for i in range(0, len(dst_hex), 4)
|
||||
if int(dst_hex[i : i + 4], 16) > 0
|
||||
)
|
||||
if src > 0 and chars:
|
||||
cmap[src] = chars
|
||||
for xobj in resources.get('/XObject', {}).values():
|
||||
if xobj.get('/Subtype') != '/Form':
|
||||
continue
|
||||
for fobj in xobj.get('/Resources', {}).get('/Font', {}).values():
|
||||
tounicode = fobj.get('/ToUnicode')
|
||||
if tounicode is None:
|
||||
continue
|
||||
raw = bytes(tounicode.read_bytes()).decode('latin-1', errors='replace')
|
||||
for m in re.finditer(r'<([0-9A-Fa-f]+)>\s*<([0-9A-Fa-f]+)>', raw):
|
||||
src = int(m.group(1), 16)
|
||||
dst_hex = m.group(2)
|
||||
chars = ''.join(
|
||||
chr(int(dst_hex[i : i + 4], 16))
|
||||
for i in range(0, len(dst_hex), 4)
|
||||
if int(dst_hex[i : i + 4], 16) > 0
|
||||
)
|
||||
if src > 0 and chars:
|
||||
cmap[src] = chars
|
||||
|
||||
# Find first Tj glyph IDs from page content or XObject streams
|
||||
glyph_ids: list[int] = []
|
||||
streams: list[bytes] = []
|
||||
contents = page.get('/Contents')
|
||||
if contents:
|
||||
streams.append(bytes(contents.read_bytes()))
|
||||
for xobj in resources.get('/XObject', {}).values():
|
||||
if xobj.get('/Subtype') == '/Form':
|
||||
streams.append(bytes(xobj.read_bytes()))
|
||||
for data in streams:
|
||||
if glyph_ids:
|
||||
break
|
||||
tj = re.search(rb'\(([^\)]+)\)\s*Tj', data)
|
||||
if tj:
|
||||
raw_bytes = tj.group(1)
|
||||
for j in range(0, len(raw_bytes) - 1, 2):
|
||||
glyph_ids.append((raw_bytes[j] << 8) | raw_bytes[j + 1])
|
||||
return cmap, glyph_ids
|
||||
|
||||
|
||||
class TestRtlTextExtraction:
|
||||
"""Verify that RTL text is extracted in correct logical order.
|
||||
|
||||
The fpdf2 renderer must produce PDF text layers where text extractors
|
||||
(pdftotext, pdfminer) return characters in correct logical (reading)
|
||||
order for Arabic, Hebrew, and Farsi scripts.
|
||||
|
||||
These tests exercise invisible_text=True (the production path) to
|
||||
catch issues like the lam-alef ligature CMap ordering bug (issue #1655).
|
||||
"""
|
||||
|
||||
def test_arabic_lam_alef_extraction_order(self, tmp_path, multi_font_manager):
|
||||
"""Arabic words with lam-alef ligature extract in correct order.
|
||||
|
||||
The lam-alef (لا) ligature was the primary trigger for issue #1655:
|
||||
fpdf2's shape_text() produced a multi-char CMap entry whose
|
||||
character order was reversed by the bidi algorithm during
|
||||
extraction, giving "سالم" instead of "سلام".
|
||||
"""
|
||||
# سلام contains lam-alef: sin(س) lam(ل) alef(ا) meem(م)
|
||||
page = create_rtl_page(
|
||||
[("سلام", (600, 100, 900, 200))],
|
||||
language="fas",
|
||||
)
|
||||
output_pdf = tmp_path / "rtl_lam_alef.pdf"
|
||||
renderer = Fpdf2PdfRenderer(
|
||||
page=page,
|
||||
dpi=72.0,
|
||||
multi_font_manager=multi_font_manager,
|
||||
invisible_text=True,
|
||||
)
|
||||
renderer.render(output_pdf)
|
||||
|
||||
cmap, glyph_ids = _decode_tounicode_stream(output_pdf)
|
||||
# Decode the glyph stream via the CMap
|
||||
decoded = ''.join(cmap.get(g, '') for g in glyph_ids)
|
||||
# The stream is pre-reversed for RTL, so reversing it back
|
||||
# must yield the original logical text
|
||||
logical = decoded[::-1]
|
||||
assert logical == 'سلام', (
|
||||
f"Expected logical text 'سلام', got {logical!r} "
|
||||
f"(stream: {decoded!r}, glyph_ids: {glyph_ids})"
|
||||
)
|
||||
|
||||
def test_arabic_multiple_words_extraction(self, tmp_path, multi_font_manager):
|
||||
"""Multiple Arabic words produce correct Unicode mappings."""
|
||||
page = create_rtl_page(
|
||||
[
|
||||
("مرحبا", (600, 100, 900, 200)),
|
||||
("بالعالم", (100, 100, 500, 200)),
|
||||
],
|
||||
language="ara",
|
||||
)
|
||||
output_pdf = tmp_path / "rtl_arabic_words.pdf"
|
||||
renderer = Fpdf2PdfRenderer(
|
||||
page=page,
|
||||
dpi=72.0,
|
||||
multi_font_manager=multi_font_manager,
|
||||
invisible_text=True,
|
||||
)
|
||||
renderer.render(output_pdf)
|
||||
|
||||
cmap, _ = _decode_tounicode_stream(output_pdf)
|
||||
# Every CMap value should contain valid Arabic characters
|
||||
arabic_chars = {c for chars in cmap.values() for c in chars}
|
||||
expected = set('مرحبابالعالم')
|
||||
assert expected.issubset(arabic_chars | {' '}), (
|
||||
f"CMap missing Arabic characters; got {arabic_chars}"
|
||||
)
|
||||
|
||||
def test_hebrew_extraction_order(self, tmp_path, multi_font_manager):
|
||||
"""Hebrew text produces correct stream order for extraction."""
|
||||
page = create_rtl_page(
|
||||
[("שלום", (600, 100, 900, 200))],
|
||||
language="heb",
|
||||
)
|
||||
output_pdf = tmp_path / "rtl_hebrew.pdf"
|
||||
renderer = Fpdf2PdfRenderer(
|
||||
page=page,
|
||||
dpi=72.0,
|
||||
multi_font_manager=multi_font_manager,
|
||||
invisible_text=True,
|
||||
)
|
||||
renderer.render(output_pdf)
|
||||
|
||||
cmap, glyph_ids = _decode_tounicode_stream(output_pdf)
|
||||
decoded = ''.join(cmap.get(g, '') for g in glyph_ids)
|
||||
logical = decoded[::-1]
|
||||
assert logical == 'שלום', (
|
||||
f"Expected logical text 'שלום', got {logical!r} "
|
||||
f"(stream: {decoded!r})"
|
||||
)
|
||||
|
||||
def test_rtl_tounicode_one_to_one(self, tmp_path, multi_font_manager):
|
||||
"""RTL invisible text produces 1:1 glyph-to-Unicode CMap entries.
|
||||
|
||||
When using encode_text() for RTL words, each glyph maps to exactly
|
||||
one Unicode character. Multi-char ligature CMap entries (produced by
|
||||
shape_text()) are the root cause of the extraction order bug, so
|
||||
their absence confirms the fix.
|
||||
"""
|
||||
page = create_rtl_page(
|
||||
[("سلام", (600, 100, 900, 200))],
|
||||
language="ara",
|
||||
)
|
||||
output_pdf = tmp_path / "rtl_tounicode.pdf"
|
||||
renderer = Fpdf2PdfRenderer(
|
||||
page=page,
|
||||
dpi=72.0,
|
||||
multi_font_manager=multi_font_manager,
|
||||
invisible_text=True,
|
||||
)
|
||||
renderer.render(output_pdf)
|
||||
|
||||
cmap, _ = _decode_tounicode_stream(output_pdf)
|
||||
# Every CMap entry should map to exactly one Unicode character
|
||||
for glyph_id, chars in cmap.items():
|
||||
assert len(chars) == 1, (
|
||||
f"Glyph {glyph_id} maps to {len(chars)} chars {chars!r}; "
|
||||
f"expected 1:1 mapping for RTL invisible text"
|
||||
)
|
||||
|
||||
def test_visible_rtl_still_uses_shaping(self, tmp_path, multi_font_manager):
|
||||
"""Visible RTL text (debug mode) still uses text shaping.
|
||||
|
||||
The encode_text() bypass is only for invisible text. When
|
||||
invisible_text=False, shaping must remain active for correct
|
||||
glyph rendering (joining forms, ligatures).
|
||||
"""
|
||||
page = create_rtl_page(
|
||||
[("سلام", (600, 100, 900, 200))],
|
||||
language="ara",
|
||||
)
|
||||
output_pdf = tmp_path / "rtl_visible.pdf"
|
||||
renderer = Fpdf2PdfRenderer(
|
||||
page=page,
|
||||
dpi=72.0,
|
||||
multi_font_manager=multi_font_manager,
|
||||
invisible_text=False,
|
||||
)
|
||||
renderer.render(output_pdf)
|
||||
|
||||
# Shaped text may have multi-char CMap entries (ligatures);
|
||||
# just verify the PDF is valid and non-empty
|
||||
check_pdf(str(output_pdf))
|
||||
text = text_from_pdf(output_pdf)
|
||||
assert len(text.strip()) > 0, "Visible RTL should produce extractable text"
|
||||
|
||||
@@ -102,6 +102,45 @@ def test_pillow_options():
|
||||
with pytest.raises(ValueError, match="max_image_mpixels must be non-negative"):
|
||||
make_ocr_opts(max_image_mpixels=-1)
|
||||
|
||||
# Default is None, meaning "do not override host-set PIL.Image.MAX_IMAGE_PIXELS"
|
||||
opts = make_ocr_opts()
|
||||
assert opts.max_image_mpixels is None
|
||||
|
||||
|
||||
def test_pillow_max_image_pixels_not_overridden_when_unset():
|
||||
"""Issue #1665: respect host-set PIL.Image.MAX_IMAGE_PIXELS.
|
||||
|
||||
API callers (e.g. Paperless-NGX) that set PIL.Image.MAX_IMAGE_PIXELS
|
||||
before invoking ocrmypdf should not have their setting clobbered when
|
||||
max_image_mpixels is not explicitly passed.
|
||||
"""
|
||||
import PIL.Image
|
||||
|
||||
from ocrmypdf._pipelines._common import setup_pipeline
|
||||
|
||||
parser = get_parser()
|
||||
pm = setup_plugin_infrastructure(plugins=[])
|
||||
pm.add_options(parser=parser)
|
||||
|
||||
saved = PIL.Image.MAX_IMAGE_PIXELS
|
||||
try:
|
||||
PIL.Image.MAX_IMAGE_PIXELS = None # host disables the limit
|
||||
opts = make_ocr_opts()
|
||||
assert opts.max_image_mpixels is None
|
||||
setup_pipeline(opts, pm)
|
||||
assert PIL.Image.MAX_IMAGE_PIXELS is None
|
||||
|
||||
PIL.Image.MAX_IMAGE_PIXELS = 1_000_000_000 # host sets a high limit
|
||||
setup_pipeline(opts, pm)
|
||||
assert PIL.Image.MAX_IMAGE_PIXELS == 1_000_000_000
|
||||
|
||||
# When explicitly passed, it still takes effect.
|
||||
opts = make_ocr_opts(max_image_mpixels=100)
|
||||
setup_pipeline(opts, pm)
|
||||
assert PIL.Image.MAX_IMAGE_PIXELS == 100_000_000
|
||||
finally:
|
||||
PIL.Image.MAX_IMAGE_PIXELS = saved
|
||||
|
||||
|
||||
def test_output_tty():
|
||||
with patch('sys.stdout.isatty', return_value=True), pytest.raises(BadArgsError):
|
||||
|
||||
Reference in New Issue
Block a user