From c85c8941d34d137f72b34701929539fb96eefae8 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Wed, 11 Feb 2026 00:38:38 -0800 Subject: [PATCH] Fix pdftotext word spacing by emitting single BT block per line poppler/pdftotext does not carry Tz (horizontal scaling) across BT/ET boundaries, causing words to appear on separate lines. Replace per-word BT blocks (via fpdf2's cell/set_stretching API) with a single BT block per line using raw PDF operators. Each non-last word gets a trailing space with Tz calculated to span exactly to the next word's start position. --- docs/release_notes.md | 4 + src/ocrmypdf/fpdf_renderer/renderer.py | 452 +++++++++++-------------- src/ocrmypdf/hocrtransform/__main__.py | 3 +- 3 files changed, 196 insertions(+), 263 deletions(-) diff --git a/docs/release_notes.md b/docs/release_notes.md index 65089105..ee77c203 100644 --- a/docs/release_notes.md +++ b/docs/release_notes.md @@ -28,6 +28,10 @@ official when it's tagged and posted to PyPI. ## v17.2.0 +- Fixed incorrect word spacing in poppler-based PDF viewers and tools (Evince, + pdftotext, and others) where words on the same line appeared separated by + double newlines. This works around a poppler bug where Tz (horizontal scaling) + is not carried across BT/ET boundaries. {issue}`1632` - Fixed OCR text layer being visible instead of invisible due to incorrect fpdf2 text rendering mode attribute. This caused OCR text to appear when images were removed from the PDF. {issue}`1631` diff --git a/src/ocrmypdf/fpdf_renderer/renderer.py b/src/ocrmypdf/fpdf_renderer/renderer.py index 3d68b76b..f751d3b9 100644 --- a/src/ocrmypdf/fpdf_renderer/renderer.py +++ b/src/ocrmypdf/fpdf_renderer/renderer.py @@ -11,12 +11,11 @@ from __future__ import annotations import logging from dataclasses import dataclass -from itertools import pairwise -from math import atan, cos, degrees, radians, sin +from math import atan, cos, degrees, radians, sin, sqrt from pathlib import Path from fpdf import FPDF -from fpdf.enums import TextMode +from fpdf.enums import PDFResourceType, TextMode from pikepdf import Matrix, Rectangle from ocrmypdf.font import FontManager, MultiFontManager @@ -413,34 +412,64 @@ class Fpdf2PdfRenderer: ): return - # Render each word followed by space (except last) - # Use pairwise to iterate over consecutive word pairs, pairing the last - # word with a None to signal the end of the line. - for current_word, next_word in pairwise(words + [None]): - if current_word: # Don't render EOL sentinel - # Render the current word - self._render_word( - pdf, - current_word, - baseline_matrix, - inv_baseline_matrix, - font_size, - total_rotation_deg, - line_language, - ) - if next_word: # Don't render EOL sentinel - self._maybe_render_space( - pdf, - current_word, - next_word, - baseline_matrix, - inv_baseline_matrix, - font_size, - total_rotation_deg, - line_language, - line.direction, + # Collect word rendering data: (text, x_baseline, font_family, word_tz) + word_render_data: list[tuple[str, float, str, float]] = [] + for word in words: + if word is None or not word.text or word.bbox is None: + continue + + word_left_pt = self.coord_transform.px_to_pt(word.bbox.left) + word_top_pt = self.coord_transform.px_to_pt(word.bbox.top) + word_right_pt = self.coord_transform.px_to_pt(word.bbox.right) + word_bottom_pt = self.coord_transform.px_to_pt(word.bbox.bottom) + word_width_pt = word_right_pt - word_left_pt + + # Debug rendering: draw word bbox (in page coordinates) + if self.debug_options.render_word_bbox: + self._render_debug_word_bbox( + pdf, word_left_pt, word_top_pt, word_right_pt, word_bottom_pt ) + # Get x position in baseline coordinate system + box_llx, _, _, _ = transform_box( + inv_baseline_matrix, + word_left_pt, + word_top_pt, + word_right_pt, + word_bottom_pt, + ) + + # Select font and compute word-only Tz + font_manager = self.multi_font_manager.select_font_for_word( + word.text, line_language + ) + font_family = self._register_font(pdf, font_manager) + pdf.set_font(font_family, size=font_size) + natural_width = pdf.get_string_width(word.text) + if natural_width > 0 and word_width_pt > 0: + word_tz = (word_width_pt / natural_width) * 100 + else: + word_tz = 100.0 + + word_render_data.append((word.text, box_llx, font_family, word_tz)) + + if not word_render_data: + return + + # Emit single BT block for the entire line using raw PDF operators. + # This avoids a poppler bug where Tz (horizontal scaling) is not + # carried across BT/ET boundaries, affecting all poppler-based tools + # and viewers (Evince, pdftotext, etc.). By keeping all words in a + # single BT block with relative Td positioning and per-word Tz, we + # ensure correct inter-word spacing. + self._emit_line_bt_block( + pdf, + word_render_data, + baseline_matrix, + font_size, + total_rotation_deg, + ) + def _check_aspect_ratio_plausible( self, pdf: FPDF, @@ -518,115 +547,165 @@ class Fpdf2PdfRenderer: self._logged_aspect_ratio_suppression = True return False - def _render_word( + def _emit_line_bt_block( self, pdf: FPDF, - word: OcrElement, + word_render_data: list[tuple[str, float, str, float]], baseline_matrix: Matrix, - inv_baseline_matrix: Matrix, font_size: float, - rotation_deg: float, - line_language: str | None, + total_rotation_deg: float, ) -> None: - """Render a word using word bbox positioning. + """Emit a single BT block for the entire line using raw PDF operators. - Position text so its visual bounding box matches the hOCR word bbox. - This provides more accurate placement than baseline-relative positioning - because we match the actual glyph bounds rather than relying on font - metrics which may not exactly match the OCR'd text appearance. + Writes all words in a single BT..ET block with relative Td positioning + and per-word Tz. Each non-last word gets a trailing space appended, with + Tz calculated so the rendered width of "word " spans from the current + word's start to the next word's start. This works around a poppler bug + where Tz is not carried across BT/ET boundaries, which affects all + poppler-based viewers and tools (Evince, pdftotext, etc.). Args: pdf: FPDF instance - word: Word OCR element + word_render_data: List of (text, x_baseline, font_family, word_tz) + tuples, one per word on this line baseline_matrix: Transform from baseline coords to page coords - inv_baseline_matrix: Transform from page coords to baseline coords - font_size: Font size in points (from line calculation) - rotation_deg: Total rotation angle for text - line_language: Language code from line for font selection + font_size: Font size in points + total_rotation_deg: Total rotation angle (textangle + slope) """ - if not word.text or word.bbox is None: - return + page_height = self.coord_transform.page_height_pt - # Select appropriate font for this word - font_manager = self.multi_font_manager.select_font_for_word( - word.text, line_language - ) + # Compute baseline direction in PDF coordinates for rotation + has_rotation = abs(total_rotation_deg) > 0.01 + bx0, by0_fpdf = transform_point(baseline_matrix, 0, 0) + by0_pdf = page_height - by0_fpdf - # Register font with fpdf2 - font_family = self._register_font(pdf, font_manager) + ops: list[str] = [] - # Convert word bbox to PDF points - word_left_pt = self.coord_transform.px_to_pt(word.bbox.left) - word_top_pt = self.coord_transform.px_to_pt(word.bbox.top) - word_right_pt = self.coord_transform.px_to_pt(word.bbox.right) - word_bottom_pt = self.coord_transform.px_to_pt(word.bbox.bottom) - word_width_pt = word_right_pt - word_left_pt + if has_rotation: + # Compute direction vector along the baseline in PDF coordinates + bx1, by1_fpdf = transform_point(baseline_matrix, 100, 0) + by1_pdf = page_height - by1_fpdf + dx = bx1 - bx0 + dy = by1_pdf - by0_pdf + length = sqrt(dx * dx + dy * dy) + if length > 0: + cos_a = dx / length + sin_a = dy / length + else: + cos_a = 1.0 + sin_a = 0.0 - # Transform word bbox into baseline coordinate system to get x position - box_llx, _, _, _ = transform_box( - inv_baseline_matrix, - word_left_pt, - word_top_pt, - word_right_pt, - word_bottom_pt, - ) - - # Debug rendering: draw word bbox (in page coordinates) - if self.debug_options.render_word_bbox: - self._render_debug_word_bbox( - pdf, word_left_pt, word_top_pt, word_right_pt, word_bottom_pt + # Save graphics state, apply rotation+translation via cm. + # The cm maps local coordinates (baseline-aligned, x along text) + # to PDF page coordinates. + ops.append('q') + ops.append( + f'{cos_a:.6f} {sin_a:.6f} {-sin_a:.6f} {cos_a:.6f} ' + f'{bx0:.2f} {by0_pdf:.2f} cm' ) - # Use line-based font_size for consistent vertical sizing - word_font_size = font_size + # Begin text object + ops.append('BT') - # Set font - pdf.set_font(font_family, size=word_font_size) + # Text render mode: 3 = invisible, 0 = fill + tr = 3 if self.invisible_text else 0 + ops.append(f'{tr} Tr') - # Calculate natural text width at this font size - natural_width = pdf.get_string_width(word.text) - - # Calculate horizontal scale to fit word bbox width - if natural_width > 0 and word_width_pt > 0: - scale_x = (word_width_pt / natural_width) * 100 + # Initial text position + first_x_baseline = word_render_data[0][1] + if has_rotation: + # In the cm-transformed space, origin is at the baseline start + ops.append(f'{first_x_baseline:.2f} 0 Td') else: - scale_x = 100 + # Direct PDF coordinates + page_x, page_y_fpdf = transform_point( + baseline_matrix, first_x_baseline, 0 + ) + page_y_pdf = page_height - page_y_fpdf + ops.append(f'{page_x:.2f} {page_y_pdf:.2f} Td') - # Apply horizontal stretching - pdf.set_stretching(scale_x) + prev_font_family: str | None = None + prev_x_baseline = first_x_baseline - # Get left side bearing of first character to compensate for glyph offset - lsb_pt = font_manager.get_left_side_bearing(word.text[0], word_font_size) + for i, (text, x_baseline, font_family, word_tz) in enumerate( + word_render_data + ): + is_last = i == len(word_render_data) - 1 - # Transform the baseline-relative x position back to page coordinates - # The word sits at (box_llx, 0) in baseline coords (on the baseline) - page_x, page_y = transform_point(baseline_matrix, box_llx, 0) + # Set font if changed + if font_family != prev_font_family: + pdf.set_font(font_family, size=font_size) + # Register font resource on this page + pdf._resource_catalog.add( + PDFResourceType.FONT, pdf.current_font.i, pdf.page + ) + ops.append( + f'/F{pdf.current_font.i} {pdf.font_size_pt:.2f} Tf' + ) + prev_font_family = font_family - # Adjust x position to account for lsb (scaled by horizontal stretch) - adjusted_x = page_x - lsb_pt * (scale_x / 100) + # Relative positioning (for words after the first) + if i > 0: + if has_rotation: + # In rotated space, advance is purely along x-axis + dx_baseline = x_baseline - prev_x_baseline + ops.append(f'{dx_baseline:.2f} 0 Td') + else: + # Non-rotated: compute delta in PDF coordinates + px_prev, py_prev_f = transform_point( + baseline_matrix, prev_x_baseline, 0 + ) + px_curr, py_curr_f = transform_point( + baseline_matrix, x_baseline, 0 + ) + dx_pdf = px_curr - px_prev + # Flip y delta for PDF coordinates (y-up) + dy_pdf = -(py_curr_f - py_prev_f) + ops.append(f'{dx_pdf:.2f} {dy_pdf:.2f} Td') - # Calculate y position based on baseline - # In fpdf2, set_xy(x, y) positions text such that the baseline is at: - # baseline_y = set_y + font_size * (ascent / (ascent + |descent|)) - # We want baseline at page_y, so: - # page_y = set_y + font_size * (ascent / (ascent + |descent|)) - # set_y = page_y - font_size * (ascent / (ascent + |descent|)) - ascent, descent, _ = font_manager.get_font_metrics() - total_height = ascent + abs(descent) - baseline_offset_ratio = ascent / total_height - adjusted_y = page_y - word_font_size * baseline_offset_ratio + # Determine text to render and compute Tz + if not is_last: + next_text, next_x_baseline, _, _ = word_render_data[i + 1] + advance = next_x_baseline - x_baseline - # Position and draw text with rotation - if abs(rotation_deg) > 0.1: - with pdf.rotation(-rotation_deg, x=page_x, y=page_y): - pdf.set_xy(adjusted_x, adjusted_y) - pdf.cell(text=word.text) - else: - pdf.set_xy(adjusted_x, adjusted_y) - pdf.cell(text=word.text) + # Add trailing space unless both words are CJK-only + if ( + advance > 0 + and not ( + self._is_cjk_only(text) + and self._is_cjk_only(next_text) + ) + ): + text_to_render = text + ' ' + natural_w = pdf.get_string_width(text_to_render) + render_tz = ( + (advance / natural_w) * 100 + if natural_w > 0 + else word_tz + ) + else: + text_to_render = text + render_tz = word_tz + else: + text_to_render = text + render_tz = word_tz - # Reset stretching - pdf.set_stretching(100) + ops.append(f'{render_tz:.2f} Tz') + ops.append(pdf.current_font.encode_text(text_to_render)) + + prev_x_baseline = x_baseline + + # End text object + ops.append('ET') + + if has_rotation: + ops.append('Q') + + pdf._out('\n'.join(ops)) + + # Reset fpdf2's internal stretching tracking so subsequent API calls + # don't think Tz is still set from our raw operators + pdf.font_stretching = 100 def _is_cjk_only(self, text: str) -> bool: """Check if text contains only CJK characters. @@ -666,157 +745,6 @@ class Fpdf2PdfRenderer: return False return True - def _maybe_render_space( - self, - pdf: FPDF, - current_word: OcrElement, - next_word: OcrElement, - baseline_matrix: Matrix, - inv_baseline_matrix: Matrix, - font_size: float, - rotation_deg: float, - line_language: str | None, - direction: str | None, - ) -> None: - """Render a space character between two words if a gap exists. - - This ensures that PDF readers like pdfminer.six can properly segment - words during text extraction. Some PDF readers rely on explicit space - characters rather than inferring word boundaries from positioning. - - Args: - pdf: FPDF instance - current_word: The word that was just rendered - next_word: The next word to be rendered - baseline_matrix: Transform from baseline coords to page coords - inv_baseline_matrix: Transform from page coords to baseline coords - font_size: Font size in points - rotation_deg: Total rotation angle for text - line_language: Language code from line for font selection - direction: Text direction ("ltr" or "rtl") - """ - if current_word.bbox is None or next_word.bbox is None: - return - - # Skip if both words are CJK-only (no spaces in CJK text) - if self._is_cjk_only(current_word.text) and self._is_cjk_only(next_word.text): - return - - # Calculate gap between words - if direction == "rtl": - gap_left = next_word.bbox.right - gap_right = current_word.bbox.left - else: - gap_left = current_word.bbox.right - gap_right = next_word.bbox.left - - gap_width_px = gap_right - gap_left - - # Use word height as proxy for line height - line_height_px = current_word.bbox.height - - # Skip if gap is too small (noise) or words are overlapping - if gap_width_px <= line_height_px * 0.05: - return - - # Render space in the gap - self._render_space( - pdf, - gap_left, - gap_right, - current_word.bbox.top, - current_word.bbox.bottom, - baseline_matrix, - inv_baseline_matrix, - font_size, - rotation_deg, - line_language, - ) - - def _render_space( - self, - pdf: FPDF, - gap_left_px: float, - gap_right_px: float, - gap_top_px: float, - gap_bottom_px: float, - baseline_matrix: Matrix, - inv_baseline_matrix: Matrix, - font_size: float, - rotation_deg: float, - line_language: str | None, - ) -> None: - """Render a space character in a gap between words. - - Uses the same baseline transformation logic as word rendering to ensure - proper alignment on rotated or sloped baselines. - - Args: - pdf: FPDF instance - gap_left_px: Left edge of gap in pixels - gap_right_px: Right edge of gap in pixels - gap_top_px: Top edge of gap in pixels - gap_bottom_px: Bottom edge of gap in pixels - baseline_matrix: Transform from baseline coords to page coords - inv_baseline_matrix: Transform from page coords to baseline coords - font_size: Font size in points - rotation_deg: Total rotation angle for text - line_language: Language code from line for font selection - """ - # Convert gap to PDF points - gap_left_pt = self.coord_transform.px_to_pt(gap_left_px) - gap_top_pt = self.coord_transform.px_to_pt(gap_top_px) - gap_right_pt = self.coord_transform.px_to_pt(gap_right_px) - gap_bottom_pt = self.coord_transform.px_to_pt(gap_bottom_px) - gap_width_pt = gap_right_pt - gap_left_pt - - # Transform gap bbox into baseline coordinate system to get x position - box_llx, _, _, _ = transform_box( - inv_baseline_matrix, - gap_left_pt, - gap_top_pt, - gap_right_pt, - gap_bottom_pt, - ) - - # Select font (use default font for space) - font_manager = self.multi_font_manager.select_font_for_word(" ", line_language) - font_family = self._register_font(pdf, font_manager) - - # Set font - pdf.set_font(font_family, size=font_size) - - # Calculate natural space width and scaling - natural_width = pdf.get_string_width(" ") - if natural_width > 0 and gap_width_pt > 0: - scale_x = (gap_width_pt / natural_width) * 100 - else: - scale_x = 100 - - # Apply horizontal stretching - pdf.set_stretching(scale_x) - - # Transform the baseline-relative x position back to page coordinates - page_x, page_y = transform_point(baseline_matrix, box_llx, 0) - - # Calculate y position based on baseline (same as _render_word) - ascent, descent, _ = font_manager.get_font_metrics() - total_height = ascent + abs(descent) - baseline_offset_ratio = ascent / total_height - adjusted_y = page_y - font_size * baseline_offset_ratio - - # Position and draw space with rotation - if abs(rotation_deg) > 0.1: - with pdf.rotation(-rotation_deg, x=page_x, y=page_y): - pdf.set_xy(page_x, adjusted_y) - pdf.cell(text=" ") - else: - pdf.set_xy(page_x, adjusted_y) - pdf.cell(text=" ") - - # Reset stretching - pdf.set_stretching(100) - def _render_debug_line_bbox( self, pdf: FPDF, diff --git a/src/ocrmypdf/hocrtransform/__main__.py b/src/ocrmypdf/hocrtransform/__main__.py index 76375293..e75b3f70 100644 --- a/src/ocrmypdf/hocrtransform/__main__.py +++ b/src/ocrmypdf/hocrtransform/__main__.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: MIT """Simple CLI for testing HOCR to PDF conversion using fpdf2 renderer.""" + from __future__ import annotations import argparse @@ -62,7 +63,7 @@ if __name__ == "__main__": page=ocr_page, dpi=dpi, multi_font_manager=multi_font_manager, - invisible_text=not args.boundingboxes, # Visible text in debug mode + invisible_text=False, debug_render_options=debug_options, ) renderer.render(Path(args.outputfile))