Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3336d67e77 | ||
|
|
73e16e7821 | ||
|
|
6f1d37d78f | ||
|
|
2ed82de2e0 | ||
|
|
c43903fa14 | ||
|
|
1c89cacfef | ||
|
|
75714fe43e | ||
|
|
e371ce95ca | ||
|
|
716a2e22c3 | ||
|
|
10e6019ada | ||
|
|
89c76b5145 | ||
|
|
83c04e6399 | ||
|
|
7fdeeb3635 | ||
|
|
5be368fe75 | ||
|
|
91c5b1e480 | ||
|
|
73154b97ba | ||
|
|
76a40759ae | ||
|
|
12ce565e98 | ||
|
|
9f46126859 | ||
|
|
11849e5a70 | ||
|
|
e30c00cc26 | ||
|
|
001b403657 | ||
|
|
851c61ee85 | ||
|
|
f5ebd23b8f | ||
|
|
81118c6195 | ||
|
|
834b60a02a | ||
|
|
47e3b5b4d2 | ||
|
|
d9346cc3d8 | ||
|
|
4e974ebd46 | ||
|
|
6f2b8408c1 | ||
|
|
1dba941261 | ||
|
|
ef76625abb | ||
|
|
57bb554a70 | ||
|
|
5b9d6f979e | ||
|
|
b588e3bfd7 | ||
|
|
a35dd1f9ee | ||
|
|
bf46f4fe35 | ||
|
|
55b76338a8 | ||
|
|
2af7b1c179 | ||
|
|
69f4cca9b6 | ||
|
|
4babdfcfbf |
+10
-10
@@ -87,7 +87,7 @@ jobs:
|
|||||||
uv run --no-dev pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
|
uv run --no-dev pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
|
||||||
|
|
||||||
- name: Upload coverage to Codecov
|
- name: Upload coverage to Codecov
|
||||||
uses: codecov/codecov-action@v5
|
uses: codecov/codecov-action@v6
|
||||||
env:
|
env:
|
||||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||||
with:
|
with:
|
||||||
@@ -149,7 +149,7 @@ jobs:
|
|||||||
uv run --no-dev pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
|
uv run --no-dev pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
|
||||||
|
|
||||||
- name: Upload coverage to Codecov
|
- name: Upload coverage to Codecov
|
||||||
uses: codecov/codecov-action@v5
|
uses: codecov/codecov-action@v6
|
||||||
env:
|
env:
|
||||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||||
with:
|
with:
|
||||||
@@ -196,7 +196,7 @@ jobs:
|
|||||||
uv run --no-dev pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
|
uv run --no-dev pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
|
||||||
|
|
||||||
- name: Upload coverage to Codecov
|
- name: Upload coverage to Codecov
|
||||||
uses: codecov/codecov-action@v5
|
uses: codecov/codecov-action@v6
|
||||||
env:
|
env:
|
||||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||||
with:
|
with:
|
||||||
@@ -218,7 +218,7 @@ jobs:
|
|||||||
run: |
|
run: |
|
||||||
uv build --sdist --wheel
|
uv build --sdist --wheel
|
||||||
|
|
||||||
- uses: actions/upload-artifact@v6
|
- uses: actions/upload-artifact@v7
|
||||||
with:
|
with:
|
||||||
name: artifact
|
name: artifact
|
||||||
path: |
|
path: |
|
||||||
@@ -235,7 +235,7 @@ jobs:
|
|||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v6
|
- uses: actions/checkout@v6
|
||||||
|
|
||||||
- uses: actions/download-artifact@v7
|
- uses: actions/download-artifact@v8
|
||||||
with:
|
with:
|
||||||
name: artifact
|
name: artifact
|
||||||
path: dist
|
path: dist
|
||||||
@@ -284,17 +284,17 @@ jobs:
|
|||||||
- uses: actions/checkout@v6
|
- uses: actions/checkout@v6
|
||||||
|
|
||||||
- name: Login to Docker Hub
|
- name: Login to Docker Hub
|
||||||
uses: docker/login-action@v3
|
uses: docker/login-action@v4
|
||||||
with:
|
with:
|
||||||
username: jbarlow83
|
username: jbarlow83
|
||||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||||
|
|
||||||
- name: Set up QEMU
|
- name: Set up QEMU
|
||||||
uses: docker/setup-qemu-action@v3
|
uses: docker/setup-qemu-action@v4
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
- name: Set up Docker Buildx
|
||||||
id: buildx
|
id: buildx
|
||||||
uses: docker/setup-buildx-action@v3
|
uses: docker/setup-buildx-action@v4
|
||||||
|
|
||||||
- name: Print image tag
|
- name: Print image tag
|
||||||
run: echo "Building image ${DOCKER_REPOSITORY}/${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}"
|
run: echo "Building image ${DOCKER_REPOSITORY}/${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}"
|
||||||
@@ -330,14 +330,14 @@ jobs:
|
|||||||
- uses: actions/checkout@v6
|
- uses: actions/checkout@v6
|
||||||
|
|
||||||
- name: Login to Docker Hub
|
- name: Login to Docker Hub
|
||||||
uses: docker/login-action@v3
|
uses: docker/login-action@v4
|
||||||
with:
|
with:
|
||||||
username: jbarlow83
|
username: jbarlow83
|
||||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
- name: Set up Docker Buildx
|
||||||
id: buildx
|
id: buildx
|
||||||
uses: docker/setup-buildx-action@v3
|
uses: docker/setup-buildx-action@v4
|
||||||
|
|
||||||
- name: Print image tag
|
- name: Print image tag
|
||||||
run: echo "Building image ${DOCKER_REPOSITORY}/${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}"
|
run: echo "Building image ${DOCKER_REPOSITORY}/${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}"
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ jobs:
|
|||||||
|
|
||||||
# PyPI doesn't support sigstore publishing, so generate after publishing to PyPI
|
# PyPI doesn't support sigstore publishing, so generate after publishing to PyPI
|
||||||
- name: Sign the dists with Sigstore
|
- name: Sign the dists with Sigstore
|
||||||
uses: sigstore/gh-action-sigstore-python@v3.2.0
|
uses: sigstore/gh-action-sigstore-python@v3.3.0
|
||||||
with:
|
with:
|
||||||
inputs: |
|
inputs: |
|
||||||
./dist/*.tar.gz
|
./dist/*.tar.gz
|
||||||
@@ -87,13 +87,13 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Login to Docker Hub
|
- name: Login to Docker Hub
|
||||||
uses: docker/login-action@v3
|
uses: docker/login-action@v4
|
||||||
with:
|
with:
|
||||||
username: jbarlow83
|
username: jbarlow83
|
||||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
- name: Set up Docker Buildx
|
||||||
uses: docker/setup-buildx-action@v3
|
uses: docker/setup-buildx-action@v4
|
||||||
|
|
||||||
- name: Tag ocrmypdf (Ubuntu) image
|
- name: Tag ocrmypdf (Ubuntu) image
|
||||||
run: |
|
run: |
|
||||||
|
|||||||
@@ -3,6 +3,41 @@
|
|||||||
|
|
||||||
# v17
|
# v17
|
||||||
|
|
||||||
|
## v17.4.2
|
||||||
|
|
||||||
|
- Fixed Python API unconditionally overriding ``PIL.Image.MAX_IMAGE_PIXELS``
|
||||||
|
when the caller did not explicitly set ``max_image_mpixels``. Host
|
||||||
|
applications (e.g. Paperless-NGX) that configure the PIL limit before
|
||||||
|
invoking ``ocrmypdf.ocr()`` now have their setting respected. The CLI
|
||||||
|
default of 250 megapixels is unchanged. {issue}`1665`
|
||||||
|
- Updated uv.lock to avoid pinning a vulnerable version of Pillow. {issue}`1666`
|
||||||
|
|
||||||
|
## v17.4.1
|
||||||
|
|
||||||
|
- Fixed RTL text extraction order in the fpdf2 renderer. Arabic lam-alef
|
||||||
|
ligatures and other multi-character CMap entries were garbled by the bidi
|
||||||
|
algorithm during text extraction. {issue}`1655`
|
||||||
|
- Fixed ``work_folder`` not being set in ``PdfContext`` options when using
|
||||||
|
the Python API. Thanks @bluebox-steven. {issue}`1613`
|
||||||
|
- Updated Ghostscript JPEG corruption warning to include the detected version
|
||||||
|
number, confirming the bug persists in Ghostscript 10.7.0.
|
||||||
|
- Internal refactoring.
|
||||||
|
- CI dependency updates.
|
||||||
|
|
||||||
|
## v17.4.0
|
||||||
|
|
||||||
|
- Added ``--no-overwrite`` / ``-n`` option to prevent overwriting output files.
|
||||||
|
If the destination file already exists, OCRmyPDF exits with code 5
|
||||||
|
(``OutputFileAccessError``). {issue}`1642`
|
||||||
|
- Fixed text layer stretching in the fpdf2 renderer for widely-spaced words.
|
||||||
|
The horizontal scaling (Tz) was incorrectly stretched to fill inter-word gaps
|
||||||
|
instead of relying on Td positioning, causing text selection to highlight far
|
||||||
|
beyond the actual word boundaries. {issue}`1635`
|
||||||
|
- Fixed ``optimize=2`` or ``optimize=3`` crash when using the Python API without
|
||||||
|
explicitly setting ``jpg_quality`` or ``png_quality``. {issue}`1641`
|
||||||
|
- Fixed ``verapdf`` availability check crashing with ``NotADirectoryError`` on
|
||||||
|
some platforms. {issue}`1638`
|
||||||
|
|
||||||
## v17.3.0
|
## v17.3.0
|
||||||
|
|
||||||
- Fixed Python API ignoring the ``language`` parameter, always defaulting to
|
- Fixed Python API ignoring the ``language`` parameter, always defaulting to
|
||||||
|
|||||||
+8
-2
@@ -6,7 +6,7 @@ build-backend = "hatchling.build"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "ocrmypdf"
|
name = "ocrmypdf"
|
||||||
version = "17.3.0"
|
version = "17.4.2"
|
||||||
description = "OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched"
|
description = "OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
license = "MPL-2.0"
|
license = "MPL-2.0"
|
||||||
@@ -156,7 +156,13 @@ quote-style = "preserve"
|
|||||||
|
|
||||||
[dependency-groups]
|
[dependency-groups]
|
||||||
# Developer-only tools - use `uv sync --group <name>`
|
# Developer-only tools - use `uv sync --group <name>`
|
||||||
dev = ["mypy>=1.13.0", "ipykernel>=6.29.5", "reportlab>=4.4.4"]
|
dev = [
|
||||||
|
"mypy>=1.13.0",
|
||||||
|
"ipykernel>=6.29.5",
|
||||||
|
"reportlab>=4.4.4",
|
||||||
|
"cyclopts>=4.5.1",
|
||||||
|
"pygithub>=2.9.1",
|
||||||
|
]
|
||||||
test = [
|
test = [
|
||||||
# Core testing framework
|
# Core testing framework
|
||||||
"coverage[toml]>=6.2",
|
"coverage[toml]>=6.2",
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ def available() -> bool:
|
|||||||
"""Check if verapdf is available."""
|
"""Check if verapdf is available."""
|
||||||
try:
|
try:
|
||||||
version()
|
version()
|
||||||
except MissingDependencyError:
|
except (MissingDependencyError, OSError):
|
||||||
return False
|
return False
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|||||||
@@ -36,6 +36,7 @@ class PdfContext:
|
|||||||
plugin_manager,
|
plugin_manager,
|
||||||
):
|
):
|
||||||
self.options = options
|
self.options = options
|
||||||
|
self.options.work_folder = work_folder
|
||||||
self.work_folder = work_folder
|
self.work_folder = work_folder
|
||||||
self.origin = origin
|
self.origin = origin
|
||||||
self.pdfinfo = pdfinfo
|
self.pdfinfo = pdfinfo
|
||||||
|
|||||||
@@ -188,8 +188,11 @@ class OcrOptions(BaseModel):
|
|||||||
"""Compatibility alias for jpg_quality."""
|
"""Compatibility alias for jpg_quality."""
|
||||||
self.jpg_quality = value
|
self.jpg_quality = value
|
||||||
|
|
||||||
|
# Output behavior
|
||||||
|
no_overwrite: bool = False
|
||||||
|
|
||||||
# Advanced options
|
# Advanced options
|
||||||
max_image_mpixels: float = 250.0
|
max_image_mpixels: float | None = None
|
||||||
pdf_renderer: str = 'auto'
|
pdf_renderer: str = 'auto'
|
||||||
ocr_engine: str = 'auto'
|
ocr_engine: str = 'auto'
|
||||||
rasterizer: str = 'auto'
|
rasterizer: str = 'auto'
|
||||||
@@ -298,7 +301,7 @@ class OcrOptions(BaseModel):
|
|||||||
@classmethod
|
@classmethod
|
||||||
def validate_max_image_mpixels(cls, v):
|
def validate_max_image_mpixels(cls, v):
|
||||||
"""Validate max image megapixels."""
|
"""Validate max image megapixels."""
|
||||||
if v < 0:
|
if v is not None and v < 0:
|
||||||
raise ValueError("max_image_mpixels must be non-negative")
|
raise ValueError("max_image_mpixels must be non-negative")
|
||||||
return v
|
return v
|
||||||
|
|
||||||
|
|||||||
@@ -329,10 +329,13 @@ def setup_pipeline(
|
|||||||
# Note: OcrOptions is immutable, so we can't modify options.jobs directly
|
# Note: OcrOptions is immutable, so we can't modify options.jobs directly
|
||||||
# The jobs field should already be set correctly during OcrOptions creation
|
# The jobs field should already be set correctly during OcrOptions creation
|
||||||
|
|
||||||
# Apply PIL max image pixels side effect
|
# Apply PIL max image pixels side effect only when explicitly requested.
|
||||||
PIL.Image.MAX_IMAGE_PIXELS = int(options.max_image_mpixels * 1_000_000)
|
# When None, leave PIL.Image.MAX_IMAGE_PIXELS as the host application
|
||||||
if PIL.Image.MAX_IMAGE_PIXELS == 0:
|
# configured it. The CLI passes its own default (250.0) via argparse.
|
||||||
PIL.Image.MAX_IMAGE_PIXELS = None # type: ignore
|
if options.max_image_mpixels is not None:
|
||||||
|
PIL.Image.MAX_IMAGE_PIXELS = int(options.max_image_mpixels * 1_000_000)
|
||||||
|
if PIL.Image.MAX_IMAGE_PIXELS == 0:
|
||||||
|
PIL.Image.MAX_IMAGE_PIXELS = None # type: ignore
|
||||||
|
|
||||||
pikepdf_enable_mmap()
|
pikepdf_enable_mmap()
|
||||||
executor = setup_executor(plugin_manager)
|
executor = setup_executor(plugin_manager)
|
||||||
|
|||||||
@@ -217,6 +217,17 @@ def check_requested_output_file(options: OcrOptions) -> None:
|
|||||||
f"Output file location ({options.output_file}) is not a writable file."
|
f"Output file location ({options.output_file}) is not a writable file."
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if (
|
||||||
|
options.no_overwrite
|
||||||
|
and not hasattr(options.output_file, 'writable')
|
||||||
|
and options.output_file != '-'
|
||||||
|
and Path(str(options.output_file)).exists()
|
||||||
|
):
|
||||||
|
raise OutputFileAccessError(
|
||||||
|
f"Output file already exists: {options.output_file}\n"
|
||||||
|
"To overwrite it, omit the --no-overwrite / -n option."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def report_output_file_size(
|
def report_output_file_size(
|
||||||
options: OcrOptions,
|
options: OcrOptions,
|
||||||
|
|||||||
@@ -1,3 +1,3 @@
|
|||||||
# SPDX-FileCopyrightText: 2022 James R. Barlow
|
# SPDX-FileCopyrightText: 2022 James R. Barlow
|
||||||
# SPDX-License-Identifier: MPL-2.0
|
# SPDX-License-Identifier: MPL-2.0
|
||||||
__version__ = "17.3.0"
|
__version__ = "17.4.2"
|
||||||
|
|||||||
@@ -440,6 +440,7 @@ def ocr(
|
|||||||
continue_on_soft_render_error: bool | None = None,
|
continue_on_soft_render_error: bool | None = None,
|
||||||
invalidate_digital_signatures: bool | None = None,
|
invalidate_digital_signatures: bool | None = None,
|
||||||
tagged_pdf_mode: str | None = None,
|
tagged_pdf_mode: str | None = None,
|
||||||
|
no_overwrite: bool | None = None,
|
||||||
plugins: Iterable[Path | str] | None = None,
|
plugins: Iterable[Path | str] | None = None,
|
||||||
plugin_manager: OcrmypdfPluginManager | None = None,
|
plugin_manager: OcrmypdfPluginManager | None = None,
|
||||||
keep_temporary_files: bool | None = None,
|
keep_temporary_files: bool | None = None,
|
||||||
@@ -502,6 +503,7 @@ def ocr( # noqa: D417
|
|||||||
continue_on_soft_render_error: bool | None = None,
|
continue_on_soft_render_error: bool | None = None,
|
||||||
invalidate_digital_signatures: bool | None = None,
|
invalidate_digital_signatures: bool | None = None,
|
||||||
tagged_pdf_mode: str | None = None,
|
tagged_pdf_mode: str | None = None,
|
||||||
|
no_overwrite: bool | None = None,
|
||||||
plugins: Iterable[Path | str] | None = None,
|
plugins: Iterable[Path | str] | None = None,
|
||||||
plugin_manager: OcrmypdfPluginManager | None = None,
|
plugin_manager: OcrmypdfPluginManager | None = None,
|
||||||
keep_temporary_files: bool | None = None,
|
keep_temporary_files: bool | None = None,
|
||||||
|
|||||||
@@ -131,10 +131,10 @@ def check_options(options):
|
|||||||
)
|
)
|
||||||
if gs_version >= Version('10.6.0'):
|
if gs_version >= Version('10.6.0'):
|
||||||
log.warning(
|
log.warning(
|
||||||
"Ghostscript 10.6.x contains JPEG encoding errors that may corrupt "
|
"Ghostscript %s contains JPEG encoding errors that may corrupt "
|
||||||
"images. OCRmyPDF will attempt to mitigate, but this version is "
|
"images. OCRmyPDF will attempt to mitigate, but versions 10.6.0+ "
|
||||||
"strongly not recommended. Please upgrade to a newer version. "
|
"are strongly not recommended until this is fixed upstream.",
|
||||||
"As of 2025-12, 10.6.0 is the latest version of Ghostscript."
|
gs_version,
|
||||||
)
|
)
|
||||||
if options.output_type == 'pdfa':
|
if options.output_type == 'pdfa':
|
||||||
options.output_type = 'pdfa-2'
|
options.output_type = 'pdfa-2'
|
||||||
|
|||||||
+12
-2
@@ -137,8 +137,9 @@ Online documentation is located at:
|
|||||||
'output_file',
|
'output_file',
|
||||||
metavar="output_pdf",
|
metavar="output_pdf",
|
||||||
help="Output searchable PDF file (or '-' to write to standard output). "
|
help="Output searchable PDF file (or '-' to write to standard output). "
|
||||||
"Existing files will be overwritten. If same as input file, the "
|
"Existing files will be overwritten (use --no-overwrite to prevent this). "
|
||||||
"input file will be updated only if processing is successful.",
|
"If same as input file, the input file will be updated only if "
|
||||||
|
"processing is successful.",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'-l',
|
'-l',
|
||||||
@@ -190,6 +191,15 @@ Online documentation is located at:
|
|||||||
"may not both use stdout at the same time.",
|
"may not both use stdout at the same time.",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
'-n',
|
||||||
|
'--no-overwrite',
|
||||||
|
action='store_true',
|
||||||
|
default=False,
|
||||||
|
help="If the output file already exists, exit with an error instead of "
|
||||||
|
"overwriting it.",
|
||||||
|
)
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
'--version',
|
'--version',
|
||||||
action='version',
|
action='version',
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ OCR text layers.
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import unicodedata
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from math import atan, cos, degrees, radians, sin, sqrt
|
from math import atan, cos, degrees, radians, sin, sqrt
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -24,6 +25,21 @@ from ocrmypdf.models.ocr_element import OcrClass, OcrElement
|
|||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _is_rtl_text(text: str) -> bool:
|
||||||
|
"""Check if text is right-to-left based on Unicode bidi properties.
|
||||||
|
|
||||||
|
Looks for the first character with a strong directional type
|
||||||
|
(R, AL, or L) to determine the text's base direction.
|
||||||
|
"""
|
||||||
|
for char in text:
|
||||||
|
bidi = unicodedata.bidirectional(char)
|
||||||
|
if bidi in ('R', 'AL'):
|
||||||
|
return True
|
||||||
|
if bidi == 'L':
|
||||||
|
return False
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
def transform_point(matrix: Matrix, x: float, y: float) -> tuple[float, float]:
|
def transform_point(matrix: Matrix, x: float, y: float) -> tuple[float, float]:
|
||||||
"""Transform a point (x, y) by a matrix.
|
"""Transform a point (x, y) by a matrix.
|
||||||
|
|
||||||
@@ -66,6 +82,17 @@ def transform_box(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class WordRenderData:
|
||||||
|
"""Rendering parameters for a single word on a line."""
|
||||||
|
|
||||||
|
text: str
|
||||||
|
x_baseline: float
|
||||||
|
font_family: str
|
||||||
|
word_tz: float
|
||||||
|
is_rtl: bool
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class DebugRenderOptions:
|
class DebugRenderOptions:
|
||||||
"""Options for debug visualization during rendering.
|
"""Options for debug visualization during rendering.
|
||||||
@@ -426,8 +453,7 @@ class Fpdf2PdfRenderer:
|
|||||||
):
|
):
|
||||||
return
|
return
|
||||||
|
|
||||||
# Collect word rendering data: (text, x_baseline, font_family, word_tz)
|
word_render_data: list[WordRenderData] = []
|
||||||
word_render_data: list[tuple[str, float, str, float]] = []
|
|
||||||
for word in words:
|
for word in words:
|
||||||
if word is None or not word.text or word.bbox is None:
|
if word is None or not word.text or word.bbox is None:
|
||||||
continue
|
continue
|
||||||
@@ -459,13 +485,35 @@ class Fpdf2PdfRenderer:
|
|||||||
)
|
)
|
||||||
font_family = self._register_font(pdf, font_manager)
|
font_family = self._register_font(pdf, font_manager)
|
||||||
pdf.set_font(font_family, size=font_size)
|
pdf.set_font(font_family, size=font_size)
|
||||||
natural_width = pdf.get_string_width(word.text)
|
|
||||||
|
# For RTL words with invisible text, we use encode_text()
|
||||||
|
# (which maps characters 1:1 in logical order) combined with
|
||||||
|
# a -1 x-scale text matrix. This avoids an fpdf2 issue where
|
||||||
|
# shaped RTL ligature glyphs (e.g. lam-alef) get multi-char
|
||||||
|
# CMap entries whose character order is reversed by the bidi
|
||||||
|
# algorithm during text extraction.
|
||||||
|
# Since the text is invisible, glyph mirroring is harmless.
|
||||||
|
# Compute Tz using unshaped widths to match encode_text().
|
||||||
|
word_is_rtl = self.invisible_text and _is_rtl_text(word.text)
|
||||||
|
if word_is_rtl:
|
||||||
|
saved_shaping = pdf.text_shaping
|
||||||
|
pdf.text_shaping = None
|
||||||
|
natural_width = pdf.get_string_width(word.text)
|
||||||
|
pdf.text_shaping = saved_shaping
|
||||||
|
else:
|
||||||
|
natural_width = pdf.get_string_width(word.text)
|
||||||
if natural_width > 0 and word_width_pt > 0:
|
if natural_width > 0 and word_width_pt > 0:
|
||||||
word_tz = (word_width_pt / natural_width) * 100
|
word_tz = (word_width_pt / natural_width) * 100
|
||||||
else:
|
else:
|
||||||
word_tz = 100.0
|
word_tz = 100.0
|
||||||
|
|
||||||
word_render_data.append((word.text, box_llx, font_family, word_tz))
|
word_render_data.append(WordRenderData(
|
||||||
|
text=word.text,
|
||||||
|
x_baseline=box_llx,
|
||||||
|
font_family=font_family,
|
||||||
|
word_tz=word_tz,
|
||||||
|
is_rtl=word_is_rtl,
|
||||||
|
))
|
||||||
|
|
||||||
if not word_render_data:
|
if not word_render_data:
|
||||||
return
|
return
|
||||||
@@ -564,7 +612,7 @@ class Fpdf2PdfRenderer:
|
|||||||
def _emit_line_bt_block(
|
def _emit_line_bt_block(
|
||||||
self,
|
self,
|
||||||
pdf: FPDF,
|
pdf: FPDF,
|
||||||
word_render_data: list[tuple[str, float, str, float]],
|
word_render_data: list[WordRenderData],
|
||||||
baseline_matrix: Matrix,
|
baseline_matrix: Matrix,
|
||||||
font_size: float,
|
font_size: float,
|
||||||
total_rotation_deg: float,
|
total_rotation_deg: float,
|
||||||
@@ -580,8 +628,7 @@ class Fpdf2PdfRenderer:
|
|||||||
|
|
||||||
Args:
|
Args:
|
||||||
pdf: FPDF instance
|
pdf: FPDF instance
|
||||||
word_render_data: List of (text, x_baseline, font_family, word_tz)
|
word_render_data: List of WordRenderData, one per word on this line
|
||||||
tuples, one per word on this line
|
|
||||||
baseline_matrix: Transform from baseline coords to page coords
|
baseline_matrix: Transform from baseline coords to page coords
|
||||||
font_size: Font size in points
|
font_size: Font size in points
|
||||||
total_rotation_deg: Total rotation angle (textangle + slope)
|
total_rotation_deg: Total rotation angle (textangle + slope)
|
||||||
@@ -626,7 +673,7 @@ class Fpdf2PdfRenderer:
|
|||||||
ops.append(f'{tr} Tr')
|
ops.append(f'{tr} Tr')
|
||||||
|
|
||||||
# Initial text position
|
# Initial text position
|
||||||
first_x_baseline = word_render_data[0][1]
|
first_x_baseline = word_render_data[0].x_baseline
|
||||||
if has_rotation:
|
if has_rotation:
|
||||||
# In the cm-transformed space, origin is at the baseline start
|
# In the cm-transformed space, origin is at the baseline start
|
||||||
ops.append(f'{first_x_baseline:.2f} 0 Td')
|
ops.append(f'{first_x_baseline:.2f} 0 Td')
|
||||||
@@ -641,14 +688,12 @@ class Fpdf2PdfRenderer:
|
|||||||
prev_font_family: str | None = None
|
prev_font_family: str | None = None
|
||||||
prev_x_baseline = first_x_baseline
|
prev_x_baseline = first_x_baseline
|
||||||
|
|
||||||
for i, (text, x_baseline, font_family, word_tz) in enumerate(
|
for i, word in enumerate(word_render_data):
|
||||||
word_render_data
|
|
||||||
):
|
|
||||||
is_last = i == len(word_render_data) - 1
|
is_last = i == len(word_render_data) - 1
|
||||||
|
|
||||||
# Set font if changed
|
# Set font if changed
|
||||||
if font_family != prev_font_family:
|
if word.font_family != prev_font_family:
|
||||||
pdf.set_font(font_family, size=font_size)
|
pdf.set_font(word.font_family, size=font_size)
|
||||||
# Register font resource on this page
|
# Register font resource on this page
|
||||||
pdf._resource_catalog.add(
|
pdf._resource_catalog.add(
|
||||||
PDFResourceType.FONT, pdf.current_font.i, pdf.page
|
PDFResourceType.FONT, pdf.current_font.i, pdf.page
|
||||||
@@ -656,13 +701,13 @@ class Fpdf2PdfRenderer:
|
|||||||
ops.append(
|
ops.append(
|
||||||
f'/F{pdf.current_font.i} {pdf.font_size_pt:.2f} Tf'
|
f'/F{pdf.current_font.i} {pdf.font_size_pt:.2f} Tf'
|
||||||
)
|
)
|
||||||
prev_font_family = font_family
|
prev_font_family = word.font_family
|
||||||
|
|
||||||
# Relative positioning (for words after the first)
|
# Relative positioning (for words after the first)
|
||||||
if i > 0:
|
if i > 0:
|
||||||
if has_rotation:
|
if has_rotation:
|
||||||
# In rotated space, advance is purely along x-axis
|
# In rotated space, advance is purely along x-axis
|
||||||
dx_baseline = x_baseline - prev_x_baseline
|
dx_baseline = word.x_baseline - prev_x_baseline
|
||||||
ops.append(f'{dx_baseline:.2f} 0 Td')
|
ops.append(f'{dx_baseline:.2f} 0 Td')
|
||||||
else:
|
else:
|
||||||
# Non-rotated: compute delta in PDF coordinates
|
# Non-rotated: compute delta in PDF coordinates
|
||||||
@@ -670,44 +715,40 @@ class Fpdf2PdfRenderer:
|
|||||||
baseline_matrix, prev_x_baseline, 0
|
baseline_matrix, prev_x_baseline, 0
|
||||||
)
|
)
|
||||||
px_curr, py_curr_f = transform_point(
|
px_curr, py_curr_f = transform_point(
|
||||||
baseline_matrix, x_baseline, 0
|
baseline_matrix, word.x_baseline, 0
|
||||||
)
|
)
|
||||||
dx_pdf = px_curr - px_prev
|
dx_pdf = px_curr - px_prev
|
||||||
# Flip y delta for PDF coordinates (y-up)
|
# Flip y delta for PDF coordinates (y-up)
|
||||||
dy_pdf = -(py_curr_f - py_prev_f)
|
dy_pdf = -(py_curr_f - py_prev_f)
|
||||||
ops.append(f'{dx_pdf:.2f} {dy_pdf:.2f} Td')
|
ops.append(f'{dx_pdf:.2f} {dy_pdf:.2f} Td')
|
||||||
|
|
||||||
# Determine text to render and compute Tz
|
# Determine text to render
|
||||||
if not is_last:
|
if not is_last:
|
||||||
next_text, next_x_baseline, _, _ = word_render_data[i + 1]
|
next_word = word_render_data[i + 1]
|
||||||
advance = next_x_baseline - x_baseline
|
advance = next_word.x_baseline - word.x_baseline
|
||||||
|
|
||||||
# Add trailing space unless both words are CJK-only
|
# Add trailing space for text extraction unless both are CJK
|
||||||
if (
|
if (
|
||||||
advance > 0
|
advance > 0
|
||||||
and not (
|
and not (
|
||||||
self._is_cjk_only(text)
|
self._is_cjk_only(word.text)
|
||||||
and self._is_cjk_only(next_text)
|
and self._is_cjk_only(next_word.text)
|
||||||
)
|
)
|
||||||
):
|
):
|
||||||
text_to_render = text + ' '
|
text_to_render = word.text + ' '
|
||||||
natural_w = pdf.get_string_width(text_to_render)
|
|
||||||
render_tz = (
|
|
||||||
(advance / natural_w) * 100
|
|
||||||
if natural_w > 0
|
|
||||||
else word_tz
|
|
||||||
)
|
|
||||||
else:
|
else:
|
||||||
text_to_render = text
|
text_to_render = word.text
|
||||||
render_tz = word_tz
|
|
||||||
else:
|
else:
|
||||||
text_to_render = text
|
text_to_render = word.text
|
||||||
render_tz = word_tz
|
|
||||||
|
|
||||||
ops.append(f'{render_tz:.2f} Tz')
|
# Use word_tz (fits word into its hOCR bbox) — Td handles
|
||||||
ops.append(self._encode_shaped_text(pdf, text_to_render))
|
# inter-word gaps, so Tz should not stretch to fill them.
|
||||||
|
ops.append(f'{word.word_tz:.2f} Tz')
|
||||||
|
ops.append(
|
||||||
|
self._encode_shaped_text(pdf, text_to_render, word.is_rtl)
|
||||||
|
)
|
||||||
|
|
||||||
prev_x_baseline = x_baseline
|
prev_x_baseline = word.x_baseline
|
||||||
|
|
||||||
# End text object
|
# End text object
|
||||||
ops.append('ET')
|
ops.append('ET')
|
||||||
@@ -721,15 +762,35 @@ class Fpdf2PdfRenderer:
|
|||||||
# don't think Tz is still set from our raw operators
|
# don't think Tz is still set from our raw operators
|
||||||
pdf.font_stretching = 100
|
pdf.font_stretching = 100
|
||||||
|
|
||||||
def _encode_shaped_text(self, pdf: FPDF, text: str) -> str:
|
def _encode_shaped_text(
|
||||||
|
self, pdf: FPDF, text: str, is_rtl: bool = False
|
||||||
|
) -> str:
|
||||||
"""Encode text using HarfBuzz text shaping for complex script support.
|
"""Encode text using HarfBuzz text shaping for complex script support.
|
||||||
|
|
||||||
Unlike font.encode_text() which maps unicode characters one-by-one to
|
Unlike font.encode_text() which maps unicode characters one-by-one to
|
||||||
glyph IDs, this uses HarfBuzz to handle BiDi reordering, Arabic joining
|
glyph IDs, this uses HarfBuzz to handle BiDi reordering, Arabic joining
|
||||||
forms, Devanagari conjuncts, and other complex script shaping. Falls
|
forms, Devanagari conjuncts, and other complex script shaping. Falls
|
||||||
back to encode_text() when text shaping is not enabled.
|
back to encode_text() when text shaping is not enabled.
|
||||||
|
|
||||||
|
For RTL words with invisible text, we use encode_text() instead of
|
||||||
|
shape_text(). fpdf2's shape_text() produces RTL ligature glyphs
|
||||||
|
(e.g. lam-alef) with multi-character CMap entries whose character
|
||||||
|
order gets reversed by the bidi algorithm during text extraction,
|
||||||
|
producing garbled output (e.g. "سالح" instead of "سلاح").
|
||||||
|
encode_text() maps characters 1:1 in logical order, giving correct
|
||||||
|
extraction. Since the text is invisible (Tr=3), the lack of proper
|
||||||
|
joining forms and ligature shaping is harmless.
|
||||||
"""
|
"""
|
||||||
font = pdf.current_font
|
font = pdf.current_font
|
||||||
|
if is_rtl:
|
||||||
|
# Reverse the text so that after bidi reversal by the text
|
||||||
|
# extractor, the characters end up in correct logical order.
|
||||||
|
# The text cursor advances left-to-right from the word's left
|
||||||
|
# edge (set by Td), so characters are positioned left-to-right
|
||||||
|
# in the PDF. The extractor sees RTL characters in L-to-R
|
||||||
|
# positions and applies bidi reversal, which reverses them.
|
||||||
|
# By pre-reversing, the double reversal yields the original.
|
||||||
|
return font.encode_text(text[::-1])
|
||||||
if pdf.text_shaping and pdf.text_shaping.get("use_shaping_engine"):
|
if pdf.text_shaping and pdf.text_shaping.get("use_shaping_engine"):
|
||||||
shaped = font.shape_text(text, pdf.font_size_pt, pdf.text_shaping)
|
shaped = font.shape_text(text, pdf.font_size_pt, pdf.text_shaping)
|
||||||
if shaped:
|
if shaped:
|
||||||
|
|||||||
@@ -681,9 +681,9 @@ def optimize(
|
|||||||
safe_symlink(input_file, output_file)
|
safe_symlink(input_file, output_file)
|
||||||
return output_file
|
return output_file
|
||||||
|
|
||||||
if options.jpg_quality == 0:
|
if not options.jpg_quality:
|
||||||
options.jpg_quality = DEFAULT_JPEG_QUALITY if options.optimize < 3 else 40
|
options.jpg_quality = DEFAULT_JPEG_QUALITY if options.optimize < 3 else 40
|
||||||
if options.png_quality == 0:
|
if not options.png_quality:
|
||||||
options.png_quality = DEFAULT_PNG_QUALITY if options.optimize < 3 else 30
|
options.png_quality = DEFAULT_PNG_QUALITY if options.optimize < 3 else 30
|
||||||
|
|
||||||
with Pdf.open(input_file) as pdf:
|
with Pdf.open(input_file) as pdf:
|
||||||
|
|||||||
+3
-6
@@ -19,14 +19,11 @@ from ocrmypdf.exceptions import ExitCode
|
|||||||
|
|
||||||
|
|
||||||
class Gs106WarningFilter(logging.Filter):
|
class Gs106WarningFilter(logging.Filter):
|
||||||
"""Filter out expected Ghostscript 10.6.x warning from test logs."""
|
"""Filter out expected Ghostscript 10.6+ JPEG encoding warning from test logs."""
|
||||||
|
|
||||||
def filter(self, record: logging.LogRecord) -> bool:
|
def filter(self, record: logging.LogRecord) -> bool:
|
||||||
# Allow all records except the expected Ghostscript 10.6.x warning
|
# Allow all records except the expected Ghostscript 10.6+ warning
|
||||||
return (
|
return "contains JPEG encoding errors" not in record.getMessage()
|
||||||
"Ghostscript 10.6.x contains JPEG encoding errors"
|
|
||||||
not in record.getMessage()
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(autouse=True)
|
@pytest.fixture(autouse=True)
|
||||||
|
|||||||
@@ -5,6 +5,8 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
@@ -37,6 +39,26 @@ def resources():
|
|||||||
return Path(__file__).parent / "resources"
|
return Path(__file__).parent / "resources"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def pdftotext():
|
||||||
|
"""Return a function to extract text from PDF using pdftotext.
|
||||||
|
|
||||||
|
Skips the test if pdftotext is not available.
|
||||||
|
"""
|
||||||
|
pdftotext_path = shutil.which('pdftotext')
|
||||||
|
if pdftotext_path is None:
|
||||||
|
pytest.skip("pdftotext not available")
|
||||||
|
|
||||||
|
def extract_text(pdf_path: Path) -> str:
|
||||||
|
return subprocess.check_output(
|
||||||
|
['pdftotext', '-enc', 'UTF-8', str(pdf_path), '-'],
|
||||||
|
text=True,
|
||||||
|
encoding='utf-8',
|
||||||
|
)
|
||||||
|
|
||||||
|
return extract_text
|
||||||
|
|
||||||
|
|
||||||
class TestFpdf2RendererImports:
|
class TestFpdf2RendererImports:
|
||||||
"""Test that all fpdf2 renderer modules can be imported."""
|
"""Test that all fpdf2 renderer modules can be imported."""
|
||||||
|
|
||||||
@@ -443,14 +465,16 @@ class TestWordSegmentation:
|
|||||||
assert "World" in words_found
|
assert "World" in words_found
|
||||||
assert "Test" in words_found
|
assert "Test" in words_found
|
||||||
|
|
||||||
def test_cjk_no_spurious_spaces(self, multi_font_manager, tmp_path):
|
def test_cjk_no_spurious_spaces(self, multi_font_manager, tmp_path, pdftotext):
|
||||||
"""Test that CJK text does not get spurious spaces inserted.
|
"""Test that CJK text does not get spurious spaces inserted.
|
||||||
|
|
||||||
CJK scripts don't use spaces between characters/words, so we should
|
CJK scripts don't use spaces between characters/words, so we should
|
||||||
not insert spaces between adjacent CJK words.
|
not insert spaces between adjacent CJK words.
|
||||||
"""
|
|
||||||
from pdfminer.high_level import extract_text
|
|
||||||
|
|
||||||
|
Uses pdftotext (poppler) instead of pdfminer.six because the latter
|
||||||
|
cannot decode the custom Encoding CMap that fpdf2 >= 2.8.7 emits for
|
||||||
|
subsetted CFF-based CID fonts (e.g. NotoSansCJK).
|
||||||
|
"""
|
||||||
from ocrmypdf.models.ocr_element import BoundingBox, OcrElement
|
from ocrmypdf.models.ocr_element import BoundingBox, OcrElement
|
||||||
|
|
||||||
# Create a page with CJK words (Chinese characters)
|
# Create a page with CJK words (Chinese characters)
|
||||||
@@ -487,15 +511,15 @@ class TestWordSegmentation:
|
|||||||
output_path = tmp_path / "test_cjk_segmentation.pdf"
|
output_path = tmp_path / "test_cjk_segmentation.pdf"
|
||||||
renderer.render(output_path)
|
renderer.render(output_path)
|
||||||
|
|
||||||
# Extract text using pdfminer.six
|
extracted_text = pdftotext(output_path)
|
||||||
extracted_text = extract_text(str(output_path))
|
|
||||||
|
|
||||||
# CJK text should be present
|
# CJK text should be present
|
||||||
assert "你好" in extracted_text
|
assert "你好" in extracted_text
|
||||||
assert "世界" in extracted_text
|
assert "世界" in extracted_text
|
||||||
|
|
||||||
# There should NOT be spaces between CJK characters
|
# There should NOT be spaces between CJK characters
|
||||||
# (but pdfminer may add some whitespace, so we check the raw chars)
|
# (a space between the two words is acceptable, since they are
|
||||||
|
# separated horizontally on the rendered page)
|
||||||
extracted_chars = extracted_text.replace(" ", "").replace("\n", "")
|
extracted_chars = extracted_text.replace(" ", "").replace("\n", "")
|
||||||
assert "你好世界" in extracted_chars or (
|
assert "你好世界" in extracted_chars or (
|
||||||
"你好" in extracted_chars and "世界" in extracted_chars
|
"你好" in extracted_chars and "世界" in extracted_chars
|
||||||
|
|||||||
@@ -5,9 +5,11 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
from io import StringIO
|
from io import StringIO
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pikepdf
|
||||||
import pytest
|
import pytest
|
||||||
from pdfminer.converter import TextConverter
|
from pdfminer.converter import TextConverter
|
||||||
from pdfminer.layout import LAParams
|
from pdfminer.layout import LAParams
|
||||||
@@ -597,3 +599,312 @@ class TestFpdf2PdfRendererLineTypes:
|
|||||||
check_pdf(str(output_pdf))
|
check_pdf(str(output_pdf))
|
||||||
extracted_text = text_from_pdf(output_pdf)
|
extracted_text = text_from_pdf(output_pdf)
|
||||||
assert "Caption" in extracted_text
|
assert "Caption" in extracted_text
|
||||||
|
|
||||||
|
|
||||||
|
def create_rtl_page(
|
||||||
|
words: list[tuple[str, tuple[float, float, float, float]]],
|
||||||
|
language: str = "ara",
|
||||||
|
width: float = 1000,
|
||||||
|
height: float = 500,
|
||||||
|
) -> OcrElement:
|
||||||
|
"""Create an OcrElement page with a single RTL paragraph/line.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
words: List of (text, (left, top, right, bottom)) tuples.
|
||||||
|
language: Language code for the paragraph.
|
||||||
|
width: Page width in pixels.
|
||||||
|
height: Page height in pixels.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
OcrElement page.
|
||||||
|
"""
|
||||||
|
word_elements = [
|
||||||
|
OcrElement(
|
||||||
|
ocr_class=OcrClass.WORD,
|
||||||
|
text=text,
|
||||||
|
bbox=BoundingBox(
|
||||||
|
left=bbox[0], top=bbox[1], right=bbox[2], bottom=bbox[3]
|
||||||
|
),
|
||||||
|
)
|
||||||
|
for text, bbox in words
|
||||||
|
]
|
||||||
|
line = OcrElement(
|
||||||
|
ocr_class=OcrClass.LINE,
|
||||||
|
bbox=BoundingBox(left=50, top=100, right=950, bottom=200),
|
||||||
|
baseline=Baseline(slope=0.0, intercept=0),
|
||||||
|
direction="rtl",
|
||||||
|
children=word_elements,
|
||||||
|
)
|
||||||
|
paragraph = OcrElement(
|
||||||
|
ocr_class=OcrClass.PARAGRAPH,
|
||||||
|
bbox=BoundingBox(left=50, top=100, right=950, bottom=200),
|
||||||
|
direction="rtl",
|
||||||
|
language=language,
|
||||||
|
children=[line],
|
||||||
|
)
|
||||||
|
return OcrElement(
|
||||||
|
ocr_class=OcrClass.PAGE,
|
||||||
|
bbox=BoundingBox(left=0, top=0, right=width, bottom=height),
|
||||||
|
children=[paragraph],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _tounicode_map(pdf_path: Path) -> dict[int, str]:
|
||||||
|
"""Extract all ToUnicode CMap entries from the first page's OCR overlay.
|
||||||
|
|
||||||
|
Returns a dict mapping subset glyph index -> unicode string.
|
||||||
|
"""
|
||||||
|
pdf = pikepdf.open(pdf_path)
|
||||||
|
page = pdf.pages[0]
|
||||||
|
resources = page.get('/Resources', {})
|
||||||
|
|
||||||
|
# Collect fonts from the page and from any Form XObjects (OCR overlay)
|
||||||
|
fonts: dict[str, pikepdf.Object] = {}
|
||||||
|
if '/Font' in resources:
|
||||||
|
for name, obj in resources['/Font'].items():
|
||||||
|
fonts[str(name)] = obj
|
||||||
|
for xobj in resources.get('/XObject', {}).values():
|
||||||
|
if xobj.get('/Subtype') == '/Form':
|
||||||
|
for name, obj in xobj.get('/Resources', {}).get('/Font', {}).items():
|
||||||
|
fonts[str(name)] = obj
|
||||||
|
|
||||||
|
result: dict[int, str] = {}
|
||||||
|
for fobj in fonts.values():
|
||||||
|
tounicode = fobj.get('/ToUnicode')
|
||||||
|
if tounicode is None:
|
||||||
|
continue
|
||||||
|
cmap = bytes(tounicode.read_bytes()).decode('latin-1', errors='replace')
|
||||||
|
for m in re.finditer(r'<([0-9A-Fa-f]+)>\s*<([0-9A-Fa-f]+)>', cmap):
|
||||||
|
src_int = int(m.group(1), 16)
|
||||||
|
dst_hex = m.group(2)
|
||||||
|
chars = ''.join(
|
||||||
|
chr(int(dst_hex[i : i + 4], 16))
|
||||||
|
for i in range(0, len(dst_hex), 4)
|
||||||
|
if int(dst_hex[i : i + 4], 16) > 0
|
||||||
|
)
|
||||||
|
if src_int > 0 and chars:
|
||||||
|
result[src_int] = chars
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _decode_tounicode_stream(
|
||||||
|
pdf_path: Path,
|
||||||
|
) -> tuple[dict[int, str], list[int]]:
|
||||||
|
"""Extract ToUnicode CMap and Tj glyph stream from a test PDF.
|
||||||
|
|
||||||
|
Searches the page content stream and any Form XObjects for fonts
|
||||||
|
and Tj operations.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
(cmap, glyph_ids) where *cmap* maps subset index -> Unicode string
|
||||||
|
and *glyph_ids* is the flat list of 2-byte glyph indices found in
|
||||||
|
the first Tj string.
|
||||||
|
"""
|
||||||
|
pdf = pikepdf.open(pdf_path)
|
||||||
|
page = pdf.pages[0]
|
||||||
|
resources = page.get('/Resources', {})
|
||||||
|
|
||||||
|
# Collect fonts from page and from Form XObjects
|
||||||
|
cmap: dict[int, str] = {}
|
||||||
|
for font_dict in [resources.get('/Font', {})]:
|
||||||
|
for fobj in font_dict.values():
|
||||||
|
tounicode = fobj.get('/ToUnicode')
|
||||||
|
if tounicode is None:
|
||||||
|
continue
|
||||||
|
raw = bytes(tounicode.read_bytes()).decode('latin-1', errors='replace')
|
||||||
|
for m in re.finditer(r'<([0-9A-Fa-f]+)>\s*<([0-9A-Fa-f]+)>', raw):
|
||||||
|
src = int(m.group(1), 16)
|
||||||
|
dst_hex = m.group(2)
|
||||||
|
chars = ''.join(
|
||||||
|
chr(int(dst_hex[i : i + 4], 16))
|
||||||
|
for i in range(0, len(dst_hex), 4)
|
||||||
|
if int(dst_hex[i : i + 4], 16) > 0
|
||||||
|
)
|
||||||
|
if src > 0 and chars:
|
||||||
|
cmap[src] = chars
|
||||||
|
for xobj in resources.get('/XObject', {}).values():
|
||||||
|
if xobj.get('/Subtype') != '/Form':
|
||||||
|
continue
|
||||||
|
for fobj in xobj.get('/Resources', {}).get('/Font', {}).values():
|
||||||
|
tounicode = fobj.get('/ToUnicode')
|
||||||
|
if tounicode is None:
|
||||||
|
continue
|
||||||
|
raw = bytes(tounicode.read_bytes()).decode('latin-1', errors='replace')
|
||||||
|
for m in re.finditer(r'<([0-9A-Fa-f]+)>\s*<([0-9A-Fa-f]+)>', raw):
|
||||||
|
src = int(m.group(1), 16)
|
||||||
|
dst_hex = m.group(2)
|
||||||
|
chars = ''.join(
|
||||||
|
chr(int(dst_hex[i : i + 4], 16))
|
||||||
|
for i in range(0, len(dst_hex), 4)
|
||||||
|
if int(dst_hex[i : i + 4], 16) > 0
|
||||||
|
)
|
||||||
|
if src > 0 and chars:
|
||||||
|
cmap[src] = chars
|
||||||
|
|
||||||
|
# Find first Tj glyph IDs from page content or XObject streams
|
||||||
|
glyph_ids: list[int] = []
|
||||||
|
streams: list[bytes] = []
|
||||||
|
contents = page.get('/Contents')
|
||||||
|
if contents:
|
||||||
|
streams.append(bytes(contents.read_bytes()))
|
||||||
|
for xobj in resources.get('/XObject', {}).values():
|
||||||
|
if xobj.get('/Subtype') == '/Form':
|
||||||
|
streams.append(bytes(xobj.read_bytes()))
|
||||||
|
for data in streams:
|
||||||
|
if glyph_ids:
|
||||||
|
break
|
||||||
|
tj = re.search(rb'\(([^\)]+)\)\s*Tj', data)
|
||||||
|
if tj:
|
||||||
|
raw_bytes = tj.group(1)
|
||||||
|
for j in range(0, len(raw_bytes) - 1, 2):
|
||||||
|
glyph_ids.append((raw_bytes[j] << 8) | raw_bytes[j + 1])
|
||||||
|
return cmap, glyph_ids
|
||||||
|
|
||||||
|
|
||||||
|
class TestRtlTextExtraction:
|
||||||
|
"""Verify that RTL text is extracted in correct logical order.
|
||||||
|
|
||||||
|
The fpdf2 renderer must produce PDF text layers where text extractors
|
||||||
|
(pdftotext, pdfminer) return characters in correct logical (reading)
|
||||||
|
order for Arabic, Hebrew, and Farsi scripts.
|
||||||
|
|
||||||
|
These tests exercise invisible_text=True (the production path) to
|
||||||
|
catch issues like the lam-alef ligature CMap ordering bug (issue #1655).
|
||||||
|
"""
|
||||||
|
|
||||||
|
def test_arabic_lam_alef_extraction_order(self, tmp_path, multi_font_manager):
|
||||||
|
"""Arabic words with lam-alef ligature extract in correct order.
|
||||||
|
|
||||||
|
The lam-alef (لا) ligature was the primary trigger for issue #1655:
|
||||||
|
fpdf2's shape_text() produced a multi-char CMap entry whose
|
||||||
|
character order was reversed by the bidi algorithm during
|
||||||
|
extraction, giving "سالم" instead of "سلام".
|
||||||
|
"""
|
||||||
|
# سلام contains lam-alef: sin(س) lam(ل) alef(ا) meem(م)
|
||||||
|
page = create_rtl_page(
|
||||||
|
[("سلام", (600, 100, 900, 200))],
|
||||||
|
language="fas",
|
||||||
|
)
|
||||||
|
output_pdf = tmp_path / "rtl_lam_alef.pdf"
|
||||||
|
renderer = Fpdf2PdfRenderer(
|
||||||
|
page=page,
|
||||||
|
dpi=72.0,
|
||||||
|
multi_font_manager=multi_font_manager,
|
||||||
|
invisible_text=True,
|
||||||
|
)
|
||||||
|
renderer.render(output_pdf)
|
||||||
|
|
||||||
|
cmap, glyph_ids = _decode_tounicode_stream(output_pdf)
|
||||||
|
# Decode the glyph stream via the CMap
|
||||||
|
decoded = ''.join(cmap.get(g, '') for g in glyph_ids)
|
||||||
|
# The stream is pre-reversed for RTL, so reversing it back
|
||||||
|
# must yield the original logical text
|
||||||
|
logical = decoded[::-1]
|
||||||
|
assert logical == 'سلام', (
|
||||||
|
f"Expected logical text 'سلام', got {logical!r} "
|
||||||
|
f"(stream: {decoded!r}, glyph_ids: {glyph_ids})"
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_arabic_multiple_words_extraction(self, tmp_path, multi_font_manager):
|
||||||
|
"""Multiple Arabic words produce correct Unicode mappings."""
|
||||||
|
page = create_rtl_page(
|
||||||
|
[
|
||||||
|
("مرحبا", (600, 100, 900, 200)),
|
||||||
|
("بالعالم", (100, 100, 500, 200)),
|
||||||
|
],
|
||||||
|
language="ara",
|
||||||
|
)
|
||||||
|
output_pdf = tmp_path / "rtl_arabic_words.pdf"
|
||||||
|
renderer = Fpdf2PdfRenderer(
|
||||||
|
page=page,
|
||||||
|
dpi=72.0,
|
||||||
|
multi_font_manager=multi_font_manager,
|
||||||
|
invisible_text=True,
|
||||||
|
)
|
||||||
|
renderer.render(output_pdf)
|
||||||
|
|
||||||
|
cmap, _ = _decode_tounicode_stream(output_pdf)
|
||||||
|
# Every CMap value should contain valid Arabic characters
|
||||||
|
arabic_chars = {c for chars in cmap.values() for c in chars}
|
||||||
|
expected = set('مرحبابالعالم')
|
||||||
|
assert expected.issubset(arabic_chars | {' '}), (
|
||||||
|
f"CMap missing Arabic characters; got {arabic_chars}"
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_hebrew_extraction_order(self, tmp_path, multi_font_manager):
|
||||||
|
"""Hebrew text produces correct stream order for extraction."""
|
||||||
|
page = create_rtl_page(
|
||||||
|
[("שלום", (600, 100, 900, 200))],
|
||||||
|
language="heb",
|
||||||
|
)
|
||||||
|
output_pdf = tmp_path / "rtl_hebrew.pdf"
|
||||||
|
renderer = Fpdf2PdfRenderer(
|
||||||
|
page=page,
|
||||||
|
dpi=72.0,
|
||||||
|
multi_font_manager=multi_font_manager,
|
||||||
|
invisible_text=True,
|
||||||
|
)
|
||||||
|
renderer.render(output_pdf)
|
||||||
|
|
||||||
|
cmap, glyph_ids = _decode_tounicode_stream(output_pdf)
|
||||||
|
decoded = ''.join(cmap.get(g, '') for g in glyph_ids)
|
||||||
|
logical = decoded[::-1]
|
||||||
|
assert logical == 'שלום', (
|
||||||
|
f"Expected logical text 'שלום', got {logical!r} "
|
||||||
|
f"(stream: {decoded!r})"
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_rtl_tounicode_one_to_one(self, tmp_path, multi_font_manager):
|
||||||
|
"""RTL invisible text produces 1:1 glyph-to-Unicode CMap entries.
|
||||||
|
|
||||||
|
When using encode_text() for RTL words, each glyph maps to exactly
|
||||||
|
one Unicode character. Multi-char ligature CMap entries (produced by
|
||||||
|
shape_text()) are the root cause of the extraction order bug, so
|
||||||
|
their absence confirms the fix.
|
||||||
|
"""
|
||||||
|
page = create_rtl_page(
|
||||||
|
[("سلام", (600, 100, 900, 200))],
|
||||||
|
language="ara",
|
||||||
|
)
|
||||||
|
output_pdf = tmp_path / "rtl_tounicode.pdf"
|
||||||
|
renderer = Fpdf2PdfRenderer(
|
||||||
|
page=page,
|
||||||
|
dpi=72.0,
|
||||||
|
multi_font_manager=multi_font_manager,
|
||||||
|
invisible_text=True,
|
||||||
|
)
|
||||||
|
renderer.render(output_pdf)
|
||||||
|
|
||||||
|
cmap, _ = _decode_tounicode_stream(output_pdf)
|
||||||
|
# Every CMap entry should map to exactly one Unicode character
|
||||||
|
for glyph_id, chars in cmap.items():
|
||||||
|
assert len(chars) == 1, (
|
||||||
|
f"Glyph {glyph_id} maps to {len(chars)} chars {chars!r}; "
|
||||||
|
f"expected 1:1 mapping for RTL invisible text"
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_visible_rtl_still_uses_shaping(self, tmp_path, multi_font_manager):
|
||||||
|
"""Visible RTL text (debug mode) still uses text shaping.
|
||||||
|
|
||||||
|
The encode_text() bypass is only for invisible text. When
|
||||||
|
invisible_text=False, shaping must remain active for correct
|
||||||
|
glyph rendering (joining forms, ligatures).
|
||||||
|
"""
|
||||||
|
page = create_rtl_page(
|
||||||
|
[("سلام", (600, 100, 900, 200))],
|
||||||
|
language="ara",
|
||||||
|
)
|
||||||
|
output_pdf = tmp_path / "rtl_visible.pdf"
|
||||||
|
renderer = Fpdf2PdfRenderer(
|
||||||
|
page=page,
|
||||||
|
dpi=72.0,
|
||||||
|
multi_font_manager=multi_font_manager,
|
||||||
|
invisible_text=False,
|
||||||
|
)
|
||||||
|
renderer.render(output_pdf)
|
||||||
|
|
||||||
|
# Shaped text may have multi-char CMap entries (ligatures);
|
||||||
|
# just verify the PDF is valid and non-empty
|
||||||
|
check_pdf(str(output_pdf))
|
||||||
|
text = text_from_pdf(output_pdf)
|
||||||
|
assert len(text.strip()) > 0, "Visible RTL should produce extractable text"
|
||||||
|
|||||||
@@ -102,6 +102,45 @@ def test_pillow_options():
|
|||||||
with pytest.raises(ValueError, match="max_image_mpixels must be non-negative"):
|
with pytest.raises(ValueError, match="max_image_mpixels must be non-negative"):
|
||||||
make_ocr_opts(max_image_mpixels=-1)
|
make_ocr_opts(max_image_mpixels=-1)
|
||||||
|
|
||||||
|
# Default is None, meaning "do not override host-set PIL.Image.MAX_IMAGE_PIXELS"
|
||||||
|
opts = make_ocr_opts()
|
||||||
|
assert opts.max_image_mpixels is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_pillow_max_image_pixels_not_overridden_when_unset():
|
||||||
|
"""Issue #1665: respect host-set PIL.Image.MAX_IMAGE_PIXELS.
|
||||||
|
|
||||||
|
API callers (e.g. Paperless-NGX) that set PIL.Image.MAX_IMAGE_PIXELS
|
||||||
|
before invoking ocrmypdf should not have their setting clobbered when
|
||||||
|
max_image_mpixels is not explicitly passed.
|
||||||
|
"""
|
||||||
|
import PIL.Image
|
||||||
|
|
||||||
|
from ocrmypdf._pipelines._common import setup_pipeline
|
||||||
|
|
||||||
|
parser = get_parser()
|
||||||
|
pm = setup_plugin_infrastructure(plugins=[])
|
||||||
|
pm.add_options(parser=parser)
|
||||||
|
|
||||||
|
saved = PIL.Image.MAX_IMAGE_PIXELS
|
||||||
|
try:
|
||||||
|
PIL.Image.MAX_IMAGE_PIXELS = None # host disables the limit
|
||||||
|
opts = make_ocr_opts()
|
||||||
|
assert opts.max_image_mpixels is None
|
||||||
|
setup_pipeline(opts, pm)
|
||||||
|
assert PIL.Image.MAX_IMAGE_PIXELS is None
|
||||||
|
|
||||||
|
PIL.Image.MAX_IMAGE_PIXELS = 1_000_000_000 # host sets a high limit
|
||||||
|
setup_pipeline(opts, pm)
|
||||||
|
assert PIL.Image.MAX_IMAGE_PIXELS == 1_000_000_000
|
||||||
|
|
||||||
|
# When explicitly passed, it still takes effect.
|
||||||
|
opts = make_ocr_opts(max_image_mpixels=100)
|
||||||
|
setup_pipeline(opts, pm)
|
||||||
|
assert PIL.Image.MAX_IMAGE_PIXELS == 100_000_000
|
||||||
|
finally:
|
||||||
|
PIL.Image.MAX_IMAGE_PIXELS = saved
|
||||||
|
|
||||||
|
|
||||||
def test_output_tty():
|
def test_output_tty():
|
||||||
with patch('sys.stdout.isatty', return_value=True), pytest.raises(BadArgsError):
|
with patch('sys.stdout.isatty', return_value=True), pytest.raises(BadArgsError):
|
||||||
|
|||||||
Reference in New Issue
Block a user