Add explicit word spacing for pdfminer.six compatibility
Insert space characters between words in the fpdf2 renderer so PDF readers like pdfminer.six can properly segment words during text extraction. Some PDF readers rely on explicit space characters rather than inferring word boundaries from positioning. - Use itertools.pairwise to iterate consecutive word pairs - Render space immediately after each word (content stream order matters) - Skip space insertion between CJK words (no spaces in CJK text) - Use 5% line height threshold to filter OCR noise - Support RTL text direction
This commit is contained in:
James R. Barlow
@@ -11,6 +11,7 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from itertools import pairwise
|
||||
from math import atan, degrees
|
||||
from pathlib import Path
|
||||
|
||||
@@ -377,18 +378,38 @@ class Fpdf2PdfRenderer:
|
||||
# Get inverse of baseline_matrix for transforming word bboxes
|
||||
inv_baseline_matrix = baseline_matrix.inverse()
|
||||
|
||||
# Render each word
|
||||
for word in line.children:
|
||||
if word.ocr_class == OcrClass.WORD and word.text:
|
||||
# Collect words to render
|
||||
words: list[OcrElement | None] = [
|
||||
w for w in line.children if w.ocr_class == OcrClass.WORD and w.text
|
||||
]
|
||||
|
||||
# Render each word followed by space (except last)
|
||||
# Use pairwise to iterate over consecutive word pairs, pairing the last
|
||||
# word with a None to signal the end of the line.
|
||||
for current_word, next_word in pairwise(words + [None]):
|
||||
if current_word: # Don't render EOL sentinel
|
||||
# Render the current word
|
||||
self._render_word(
|
||||
pdf,
|
||||
word,
|
||||
current_word,
|
||||
baseline_matrix,
|
||||
inv_baseline_matrix,
|
||||
font_size,
|
||||
total_rotation_deg,
|
||||
line_language,
|
||||
)
|
||||
if next_word: # Don't render EOL sentinel
|
||||
self._maybe_render_space(
|
||||
pdf,
|
||||
current_word,
|
||||
next_word,
|
||||
baseline_matrix,
|
||||
inv_baseline_matrix,
|
||||
font_size,
|
||||
total_rotation_deg,
|
||||
line_language,
|
||||
line.direction,
|
||||
)
|
||||
|
||||
def _render_word(
|
||||
self,
|
||||
@@ -500,6 +521,195 @@ class Fpdf2PdfRenderer:
|
||||
# Reset stretching
|
||||
pdf.set_stretching(100)
|
||||
|
||||
def _is_cjk_only(self, text: str) -> bool:
|
||||
"""Check if text contains only CJK characters.
|
||||
|
||||
CJK scripts don't use spaces between words, so we should not insert
|
||||
spaces between adjacent CJK words.
|
||||
|
||||
Args:
|
||||
text: Text to check
|
||||
|
||||
Returns:
|
||||
True if text contains only CJK characters
|
||||
"""
|
||||
for char in text:
|
||||
cp = ord(char)
|
||||
# Check if character is in CJK ranges
|
||||
if not (
|
||||
0x4E00 <= cp <= 0x9FFF # CJK Unified Ideographs
|
||||
or 0x3400 <= cp <= 0x4DBF # CJK Extension A
|
||||
or 0x20000 <= cp <= 0x2A6DF # CJK Extension B
|
||||
or 0x2A700 <= cp <= 0x2B73F # CJK Extension C
|
||||
or 0x2B740 <= cp <= 0x2B81F # CJK Extension D
|
||||
or 0x2B820 <= cp <= 0x2CEAF # CJK Extension E
|
||||
or 0x2CEB0 <= cp <= 0x2EBEF # CJK Extension F
|
||||
or 0x30000 <= cp <= 0x3134F # CJK Extension G
|
||||
or 0x3040 <= cp <= 0x309F # Hiragana
|
||||
or 0x30A0 <= cp <= 0x30FF # Katakana
|
||||
or 0x31F0 <= cp <= 0x31FF # Katakana Phonetic Extensions
|
||||
or 0xAC00 <= cp <= 0xD7AF # Hangul Syllables
|
||||
or 0x1100 <= cp <= 0x11FF # Hangul Jamo
|
||||
or 0x3130 <= cp <= 0x318F # Hangul Compatibility Jamo
|
||||
or 0xA960 <= cp <= 0xA97F # Hangul Jamo Extended-A
|
||||
or 0xD7B0 <= cp <= 0xD7FF # Hangul Jamo Extended-B
|
||||
or 0x3000 <= cp <= 0x303F # CJK Symbols and Punctuation
|
||||
or 0xFF00 <= cp <= 0xFFEF # Halfwidth and Fullwidth Forms
|
||||
):
|
||||
return False
|
||||
return True
|
||||
|
||||
def _maybe_render_space(
|
||||
self,
|
||||
pdf: FPDF,
|
||||
current_word: OcrElement,
|
||||
next_word: OcrElement,
|
||||
baseline_matrix: Matrix,
|
||||
inv_baseline_matrix: Matrix,
|
||||
font_size: float,
|
||||
rotation_deg: float,
|
||||
line_language: str | None,
|
||||
direction: str | None,
|
||||
) -> None:
|
||||
"""Render a space character between two words if a gap exists.
|
||||
|
||||
This ensures that PDF readers like pdfminer.six can properly segment
|
||||
words during text extraction. Some PDF readers rely on explicit space
|
||||
characters rather than inferring word boundaries from positioning.
|
||||
|
||||
Args:
|
||||
pdf: FPDF instance
|
||||
current_word: The word that was just rendered
|
||||
next_word: The next word to be rendered
|
||||
baseline_matrix: Transform from baseline coords to page coords
|
||||
inv_baseline_matrix: Transform from page coords to baseline coords
|
||||
font_size: Font size in points
|
||||
rotation_deg: Total rotation angle for text
|
||||
line_language: Language code from line for font selection
|
||||
direction: Text direction ("ltr" or "rtl")
|
||||
"""
|
||||
if current_word.bbox is None or next_word.bbox is None:
|
||||
return
|
||||
|
||||
# Skip if both words are CJK-only (no spaces in CJK text)
|
||||
if self._is_cjk_only(current_word.text) and self._is_cjk_only(next_word.text):
|
||||
return
|
||||
|
||||
# Calculate gap between words
|
||||
if direction == "rtl":
|
||||
gap_left = next_word.bbox.right
|
||||
gap_right = current_word.bbox.left
|
||||
else:
|
||||
gap_left = current_word.bbox.right
|
||||
gap_right = next_word.bbox.left
|
||||
|
||||
gap_width_px = gap_right - gap_left
|
||||
|
||||
# Use word height as proxy for line height
|
||||
line_height_px = current_word.bbox.height
|
||||
|
||||
# Skip if gap is too small (noise) or words are overlapping
|
||||
if gap_width_px <= line_height_px * 0.05:
|
||||
return
|
||||
|
||||
# Render space in the gap
|
||||
self._render_space(
|
||||
pdf,
|
||||
gap_left,
|
||||
gap_right,
|
||||
current_word.bbox.top,
|
||||
current_word.bbox.bottom,
|
||||
baseline_matrix,
|
||||
inv_baseline_matrix,
|
||||
font_size,
|
||||
rotation_deg,
|
||||
line_language,
|
||||
)
|
||||
|
||||
def _render_space(
|
||||
self,
|
||||
pdf: FPDF,
|
||||
gap_left_px: float,
|
||||
gap_right_px: float,
|
||||
gap_top_px: float,
|
||||
gap_bottom_px: float,
|
||||
baseline_matrix: Matrix,
|
||||
inv_baseline_matrix: Matrix,
|
||||
font_size: float,
|
||||
rotation_deg: float,
|
||||
line_language: str | None,
|
||||
) -> None:
|
||||
"""Render a space character in a gap between words.
|
||||
|
||||
Uses the same baseline transformation logic as word rendering to ensure
|
||||
proper alignment on rotated or sloped baselines.
|
||||
|
||||
Args:
|
||||
pdf: FPDF instance
|
||||
gap_left_px: Left edge of gap in pixels
|
||||
gap_right_px: Right edge of gap in pixels
|
||||
gap_top_px: Top edge of gap in pixels
|
||||
gap_bottom_px: Bottom edge of gap in pixels
|
||||
baseline_matrix: Transform from baseline coords to page coords
|
||||
inv_baseline_matrix: Transform from page coords to baseline coords
|
||||
font_size: Font size in points
|
||||
rotation_deg: Total rotation angle for text
|
||||
line_language: Language code from line for font selection
|
||||
"""
|
||||
# Convert gap to PDF points
|
||||
gap_left_pt = self.coord_transform.px_to_pt(gap_left_px)
|
||||
gap_top_pt = self.coord_transform.px_to_pt(gap_top_px)
|
||||
gap_right_pt = self.coord_transform.px_to_pt(gap_right_px)
|
||||
gap_bottom_pt = self.coord_transform.px_to_pt(gap_bottom_px)
|
||||
gap_width_pt = gap_right_pt - gap_left_pt
|
||||
|
||||
# Transform gap bbox into baseline coordinate system to get x position
|
||||
box_llx, _, _, _ = transform_box(
|
||||
inv_baseline_matrix,
|
||||
gap_left_pt,
|
||||
gap_top_pt,
|
||||
gap_right_pt,
|
||||
gap_bottom_pt,
|
||||
)
|
||||
|
||||
# Select font (use default font for space)
|
||||
font_manager = self.multi_font_manager.select_font_for_word(" ", line_language)
|
||||
font_family = self._register_font(pdf, font_manager)
|
||||
|
||||
# Set font
|
||||
pdf.set_font(font_family, size=font_size)
|
||||
|
||||
# Calculate natural space width and scaling
|
||||
natural_width = pdf.get_string_width(" ")
|
||||
if natural_width > 0 and gap_width_pt > 0:
|
||||
scale_x = (gap_width_pt / natural_width) * 100
|
||||
else:
|
||||
scale_x = 100
|
||||
|
||||
# Apply horizontal stretching
|
||||
pdf.set_stretching(scale_x)
|
||||
|
||||
# Transform the baseline-relative x position back to page coordinates
|
||||
page_x, page_y = transform_point(baseline_matrix, box_llx, 0)
|
||||
|
||||
# Calculate y position based on baseline (same as _render_word)
|
||||
ascent, descent, _ = font_manager.get_font_metrics()
|
||||
total_height = ascent + abs(descent)
|
||||
baseline_offset_ratio = ascent / total_height
|
||||
adjusted_y = page_y - font_size * baseline_offset_ratio
|
||||
|
||||
# Position and draw space with rotation
|
||||
if abs(rotation_deg) > 0.1:
|
||||
with pdf.rotation(-rotation_deg, x=page_x, y=page_y):
|
||||
pdf.set_xy(page_x, adjusted_y)
|
||||
pdf.cell(text=" ")
|
||||
else:
|
||||
pdf.set_xy(page_x, adjusted_y)
|
||||
pdf.cell(text=" ")
|
||||
|
||||
# Reset stretching
|
||||
pdf.set_stretching(100)
|
||||
|
||||
def _render_debug_line_bbox(
|
||||
self,
|
||||
pdf: FPDF,
|
||||
|
||||
@@ -364,3 +364,169 @@ class TestFpdf2RendererWithHocr:
|
||||
|
||||
assert output_path.exists()
|
||||
assert output_path.stat().st_size > 0
|
||||
|
||||
|
||||
class TestWordSegmentation:
|
||||
"""Test that rendered PDFs have proper word segmentation for pdfminer.six."""
|
||||
|
||||
def test_word_segmentation_with_pdfminer(self, multi_font_manager, tmp_path):
|
||||
"""Test that pdfminer.six can extract words with proper spacing.
|
||||
|
||||
This test verifies that explicit space characters are inserted between
|
||||
words so that pdfminer.six (and similar PDF readers) can properly
|
||||
segment words during text extraction.
|
||||
"""
|
||||
from pdfminer.high_level import extract_text
|
||||
|
||||
from ocrmypdf.hocrtransform.ocr_element import BoundingBox, OcrElement
|
||||
|
||||
# Create a page with multiple words on one line
|
||||
word1 = OcrElement(
|
||||
ocr_class=OcrClass.WORD,
|
||||
text="Hello",
|
||||
bbox=BoundingBox(left=100, top=100, right=200, bottom=130),
|
||||
)
|
||||
word2 = OcrElement(
|
||||
ocr_class=OcrClass.WORD,
|
||||
text="World",
|
||||
bbox=BoundingBox(left=220, top=100, right=320, bottom=130),
|
||||
)
|
||||
word3 = OcrElement(
|
||||
ocr_class=OcrClass.WORD,
|
||||
text="Test",
|
||||
bbox=BoundingBox(left=340, top=100, right=420, bottom=130),
|
||||
)
|
||||
line = OcrElement(
|
||||
ocr_class=OcrClass.LINE,
|
||||
bbox=BoundingBox(left=100, top=100, right=420, bottom=130),
|
||||
children=[word1, word2, word3],
|
||||
)
|
||||
page = OcrElement(
|
||||
ocr_class=OcrClass.PAGE,
|
||||
bbox=BoundingBox(left=0, top=0, right=612, bottom=792),
|
||||
children=[line],
|
||||
)
|
||||
|
||||
renderer = Fpdf2PdfRenderer(
|
||||
page=page,
|
||||
dpi=72, # 1:1 mapping to PDF points
|
||||
multi_font_manager=multi_font_manager,
|
||||
invisible_text=False,
|
||||
)
|
||||
|
||||
output_path = tmp_path / "test_word_segmentation.pdf"
|
||||
renderer.render(output_path)
|
||||
|
||||
# Extract text using pdfminer.six
|
||||
extracted_text = extract_text(str(output_path))
|
||||
|
||||
# Verify words are separated by spaces
|
||||
assert "Hello" in extracted_text
|
||||
assert "World" in extracted_text
|
||||
assert "Test" in extracted_text
|
||||
|
||||
# The text should NOT be run together like "HelloWorldTest"
|
||||
assert "HelloWorld" not in extracted_text
|
||||
assert "WorldTest" not in extracted_text
|
||||
|
||||
# Verify proper word segmentation - words should be separated
|
||||
# (allowing for whitespace variations)
|
||||
words_found = extracted_text.split()
|
||||
assert "Hello" in words_found
|
||||
assert "World" in words_found
|
||||
assert "Test" in words_found
|
||||
|
||||
def test_cjk_no_spurious_spaces(self, multi_font_manager, tmp_path):
|
||||
"""Test that CJK text does not get spurious spaces inserted.
|
||||
|
||||
CJK scripts don't use spaces between characters/words, so we should
|
||||
not insert spaces between adjacent CJK words.
|
||||
"""
|
||||
from pdfminer.high_level import extract_text
|
||||
|
||||
from ocrmypdf.hocrtransform.ocr_element import BoundingBox, OcrElement
|
||||
|
||||
# Create a page with CJK words (Chinese characters)
|
||||
# 你好 = "Hello" in Chinese
|
||||
# 世界 = "World" in Chinese
|
||||
word1 = OcrElement(
|
||||
ocr_class=OcrClass.WORD,
|
||||
text="你好",
|
||||
bbox=BoundingBox(left=100, top=100, right=160, bottom=130),
|
||||
)
|
||||
word2 = OcrElement(
|
||||
ocr_class=OcrClass.WORD,
|
||||
text="世界",
|
||||
bbox=BoundingBox(left=170, top=100, right=230, bottom=130),
|
||||
)
|
||||
line = OcrElement(
|
||||
ocr_class=OcrClass.LINE,
|
||||
bbox=BoundingBox(left=100, top=100, right=230, bottom=130),
|
||||
children=[word1, word2],
|
||||
)
|
||||
page = OcrElement(
|
||||
ocr_class=OcrClass.PAGE,
|
||||
bbox=BoundingBox(left=0, top=0, right=612, bottom=792),
|
||||
children=[line],
|
||||
)
|
||||
|
||||
renderer = Fpdf2PdfRenderer(
|
||||
page=page,
|
||||
dpi=72,
|
||||
multi_font_manager=multi_font_manager,
|
||||
invisible_text=False,
|
||||
)
|
||||
|
||||
output_path = tmp_path / "test_cjk_segmentation.pdf"
|
||||
renderer.render(output_path)
|
||||
|
||||
# Extract text using pdfminer.six
|
||||
extracted_text = extract_text(str(output_path))
|
||||
|
||||
# CJK text should be present
|
||||
assert "你好" in extracted_text
|
||||
assert "世界" in extracted_text
|
||||
|
||||
# There should NOT be spaces between CJK characters
|
||||
# (but pdfminer may add some whitespace, so we check the raw chars)
|
||||
extracted_chars = extracted_text.replace(" ", "").replace("\n", "")
|
||||
assert "你好世界" in extracted_chars or (
|
||||
"你好" in extracted_chars and "世界" in extracted_chars
|
||||
)
|
||||
|
||||
def test_latin_hocr_word_segmentation(
|
||||
self, resources, multi_font_manager, tmp_path
|
||||
):
|
||||
"""Test word segmentation with real Latin hOCR file."""
|
||||
from pdfminer.high_level import extract_text
|
||||
|
||||
hocr_path = resources / "latin.hocr"
|
||||
if not hocr_path.exists():
|
||||
pytest.skip("latin.hocr not found")
|
||||
|
||||
parser = HocrParser(hocr_path)
|
||||
page = parser.parse()
|
||||
|
||||
renderer = Fpdf2PdfRenderer(
|
||||
page=page,
|
||||
dpi=300,
|
||||
multi_font_manager=multi_font_manager,
|
||||
invisible_text=False,
|
||||
)
|
||||
|
||||
output_path = tmp_path / "latin_segmentation.pdf"
|
||||
renderer.render(output_path)
|
||||
|
||||
# Extract text using pdfminer.six
|
||||
extracted_text = extract_text(str(output_path))
|
||||
|
||||
# The Latin text should have proper word segmentation
|
||||
# Words should be separable
|
||||
words = extracted_text.split()
|
||||
assert len(words) > 0
|
||||
|
||||
# Check that common English words are properly segmented
|
||||
# (not stuck together)
|
||||
text_no_newlines = extracted_text.replace("\n", " ")
|
||||
# There should be spaces in the extracted text
|
||||
assert " " in text_no_newlines
|
||||
|
||||
Reference in New Issue
Block a user