Fix CJK test broken by fpdf2 2.8.7 CFF font encoding change

fpdf2 >= 2.8.7 emits a custom begincidchar Encoding CMap for CFF-based
CID fonts (e.g. NotoSansCJK). pdfminer.six returns <CMap: None> for such
CMaps, so text extraction yields empty output. Switch to pdftotext (poppler)
which handles the new encoding correctly.
This commit is contained in:
James R. Barlow
2026-04-19 23:26:30 -07:00
parent 73e16e7821
commit 3336d67e77
+30 -6
View File
@@ -5,6 +5,8 @@
from __future__ import annotations
import shutil
import subprocess
from pathlib import Path
import pytest
@@ -37,6 +39,26 @@ def resources():
return Path(__file__).parent / "resources"
@pytest.fixture
def pdftotext():
"""Return a function to extract text from PDF using pdftotext.
Skips the test if pdftotext is not available.
"""
pdftotext_path = shutil.which('pdftotext')
if pdftotext_path is None:
pytest.skip("pdftotext not available")
def extract_text(pdf_path: Path) -> str:
return subprocess.check_output(
['pdftotext', '-enc', 'UTF-8', str(pdf_path), '-'],
text=True,
encoding='utf-8',
)
return extract_text
class TestFpdf2RendererImports:
"""Test that all fpdf2 renderer modules can be imported."""
@@ -443,14 +465,16 @@ class TestWordSegmentation:
assert "World" in words_found
assert "Test" in words_found
def test_cjk_no_spurious_spaces(self, multi_font_manager, tmp_path):
def test_cjk_no_spurious_spaces(self, multi_font_manager, tmp_path, pdftotext):
"""Test that CJK text does not get spurious spaces inserted.
CJK scripts don't use spaces between characters/words, so we should
not insert spaces between adjacent CJK words.
"""
from pdfminer.high_level import extract_text
Uses pdftotext (poppler) instead of pdfminer.six because the latter
cannot decode the custom Encoding CMap that fpdf2 >= 2.8.7 emits for
subsetted CFF-based CID fonts (e.g. NotoSansCJK).
"""
from ocrmypdf.models.ocr_element import BoundingBox, OcrElement
# Create a page with CJK words (Chinese characters)
@@ -487,15 +511,15 @@ class TestWordSegmentation:
output_path = tmp_path / "test_cjk_segmentation.pdf"
renderer.render(output_path)
# Extract text using pdfminer.six
extracted_text = extract_text(str(output_path))
extracted_text = pdftotext(output_path)
# CJK text should be present
assert "你好" in extracted_text
assert "世界" in extracted_text
# There should NOT be spaces between CJK characters
# (but pdfminer may add some whitespace, so we check the raw chars)
# (a space between the two words is acceptable, since they are
# separated horizontally on the rendered page)
extracted_chars = extracted_text.replace(" ", "").replace("\n", "")
assert "你好世界" in extracted_chars or (
"你好" in extracted_chars and "世界" in extracted_chars