Compare commits

...
4 Commits
Author SHA1 Message Date
James R. Barlow de403f6d5e Bump version: v17.7.1 2026-06-19 16:44:31 -07:00
James R. Barlow 86b6f2c907 v17.7.1 release notes 2026-06-19 16:44:10 -07:00
e6fab76918 Fix Windows redo-ocr performance regression; drop pdfminer BUFSIZ workaround (#1706)
Since v16.4.3, OCRmyPDF forced pdfminer's read buffer to 256 MiB to work
around a pdfminer bug that mishandled tokens split across the buffer
boundary (gh #1361). On Windows this caused a severe performance
regression (gh #1662): CPython's BufferedReader.read(n) eagerly allocates
an n-byte buffer on every read, so pdfminer's thousands of seek+read
cycles each paid a ~30 ms 256 MiB allocation (this allocation is lazy and
effectively free on Linux). For a typical PDF the "Scanning contents"
phase went from ~5s on Linux to ~60s on Windows.

The underlying pdfminer bug was fixed upstream in pdfminer.six 20250327
(pdfminer/pdfminer.six#1030), with a follow-up for tokens split across
streams in 20260107 (pdfminer/pdfminer.six#1158). Remove the monkeypatch
entirely and raise the minimum pdfminer.six to 20260107 so we rely on the
upstream fix instead.

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-18 10:34:20 -07:00
jbarlowandGitHub 334918d0f7 Discover variable and per-language Noto fonts for the OCR text layer (#1652) (#1707)
* Bump uv lock, fix bump_version

* Discover variable and per-language Noto fonts for the OCR text layer (#1652)

SystemFontProvider only matched static "-Regular.ttf/.otf" filenames, so
the variable fonts shipped by Homebrew casks and current Google Fonts
(e.g. NotoSansArabic[wdth,wght].ttf) were never found. Users who had
installed the font still got the glyphless Occulta fallback and a cryptic
"No font found" warning.

- Match variable fonts (Base[...]), -VF, and bare-family filenames via a
  boundary-aware flexible search, escaping the glob-special brackets.
- Make CJK language-aware: the modern per-language Noto fonts (NotoSansSC
  /TC/HK/JP/KR) are region subsets, so map each CJK language to its own
  family and keep the full-coverage pan-CJK super font as a shared
  fallback. Glyph coverage, not shape, is what matters for the invisible
  text layer.
- Reword the missing-font warning to explain the consequence (searchable
  but blank when highlighted) and name the language-specific font.
2026-06-18 10:34:08 -07:00
10 changed files with 485 additions and 36 deletions
+4 -3
View File
@@ -18,8 +18,9 @@ import cyclopts
from packaging.version import InvalidVersion, Version
try:
from github import Github, GithubException
from github import Auth, Github, GithubException
except ImportError:
Auth = None # type: ignore
Github = None # type: ignore
GithubException = Exception # type: ignore
@@ -68,7 +69,7 @@ def validate_release_notes(new_version: str) -> bool:
def get_github_client():
"""Get an authenticated GitHub client."""
if Github is None:
if Github is None or Auth is None:
print(f"{RED}error:{OFF} PyGithub is not installed")
print(" Install with: pip install PyGithub")
return None
@@ -92,7 +93,7 @@ def get_github_client():
return None
try:
return Github(token)
return Github(auth=Auth.Token(token))
except GithubException as e:
print(f"{RED}error:{OFF} Failed to authenticate with GitHub: {e}")
return None
+30
View File
@@ -3,6 +3,36 @@
# v17
## v17.7.1
- Fixed a severe, Windows-specific performance regression in the "Scanning
contents" phase, most visible with `--redo-ocr` ({issue}`1662`). Since
v16.4.3, OCRmyPDF forced pdfminer's read buffer to 256 MiB to work around a
pdfminer bug that mishandled tokens split across the buffer boundary
({issue}`1361`). On Windows, CPython's `BufferedReader.read()` eagerly
allocates a buffer of the requested size on every read, so the oversized
buffer made each of pdfminer's thousands of reads cost tens of milliseconds
(this allocation is lazy, and effectively free, on Linux). The underlying
pdfminer bug was fixed upstream in pdfminer.six 20250327
([#1030](https://github.com/pdfminer/pdfminer.six/pull/1030)), with a
follow-up for tokens split across streams in 20260107
([#1158](https://github.com/pdfminer/pdfminer.six/pull/1158)), so the
workaround has been removed and the minimum pdfminer.six version raised to
20260107.
- The font discovery used to build the OCR text layer now finds variable fonts
such as `NotoSansArabic[wdth,wght].ttf`, the form shipped by Homebrew casks
and current Google Fonts releases. Previously only static `-Regular.ttf`/`.otf`
files were matched, so users who had installed the correct Noto font still got
the glyphless fallback and a "No font found" warning ({issue}`1652`).
- Font discovery is now language-aware for CJK: each Chinese, Japanese, and
Korean language maps to its own per-language Noto family (NotoSansSC, TC, HK,
JP, KR), with the pan-CJK super font kept as a shared fallback, since the
per-language fonts are region subsets that may lack glyphs from other scripts.
- The warning shown when no installed font has glyphs for some text was reworded
to explain the consequence — the text is still added as a searchable, copyable
layer but appears blank when highlighted in a viewer — and to name the specific
font family to install.
## v17.7.0
- The Docker images now run as a non-root user (`app`, uid/gid 1000) by default
+2 -2
View File
@@ -6,7 +6,7 @@ build-backend = "hatchling.build"
[project]
name = "ocrmypdf"
version = "17.7.0"
version = "17.7.1"
description = "OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched"
readme = "README.md"
license = "MPL-2.0"
@@ -16,7 +16,7 @@ dependencies = [
"fpdf2>=2.8.0",
"img2pdf>=0.5",
"packaging>=20",
"pdfminer.six>=20220319",
"pdfminer.six>=20260107", # fixes parsing of tokens split across the read buffer/streams (gh #1361)
"pi-heif", # Heif image format - maintainers: if this is removed, it will NOT break
"pikepdf>=10",
"Pillow>=10.0.1",
+1 -1
View File
@@ -1,3 +1,3 @@
# SPDX-FileCopyrightText: 2022 James R. Barlow
# SPDX-License-Identifier: MPL-2.0
__version__ = "17.7.0"
__version__ = "17.7.1"
+28 -15
View File
@@ -54,13 +54,15 @@ class MultiFontManager:
'kok': 'NotoSansDevanagari-Regular', # Konkani
'bho': 'NotoSansDevanagari-Regular', # Bhojpuri
'mai': 'NotoSansDevanagari-Regular', # Maithili
# CJK
'chi': 'NotoSansCJK-Regular', # Chinese (generic)
'zho': 'NotoSansCJK-Regular', # Chinese (ISO 639-3)
'chi_sim': 'NotoSansCJK-Regular', # Chinese Simplified (Tesseract)
'chi_tra': 'NotoSansCJK-Regular', # Chinese Traditional (Tesseract)
'jpn': 'NotoSansCJK-Regular', # Japanese
'kor': 'NotoSansCJK-Regular', # Korean
# CJK — prefer the family matching the document language, because the
# modern per-language Noto fonts are region subsets (e.g. NotoSansSC
# lacks Japanese kana). The pan-CJK super font is a shared fallback.
'chi': 'NotoSansSC-Regular', # Chinese (generic → Simplified)
'zho': 'NotoSansSC-Regular', # Chinese (ISO 639-3)
'chi_sim': 'NotoSansSC-Regular', # Chinese Simplified (Tesseract)
'chi_tra': 'NotoSansTC-Regular', # Chinese Traditional (Tesseract)
'jpn': 'NotoSansJP-Regular', # Japanese
'kor': 'NotoSansKR-Regular', # Korean
# Thai
'tha': 'NotoSansThai-Regular', # Thai
# Hebrew
@@ -113,7 +115,14 @@ class MultiFontManager:
'NotoSans-Regular', # Latin, Greek, Cyrillic
'NotoSansArabic-Regular',
'NotoSansDevanagari-Regular',
# Pan-CJK super font first (full coverage), then the per-language
# subsets so a glyph missing from one CJK family is found in another.
'NotoSansCJK-Regular',
'NotoSansSC-Regular',
'NotoSansTC-Regular',
'NotoSansHK-Regular',
'NotoSansJP-Regular',
'NotoSansKR-Regular',
'NotoSansThai-Regular',
'NotoSansHebrew-Regular',
'NotoSansBengali-Regular',
@@ -256,19 +265,23 @@ class MultiFontManager:
self._warned_scripts.add(warn_key)
if line_language and line_language in self.LANGUAGE_FONT_MAP:
font_name = self.LANGUAGE_FONT_MAP[line_language]
font_family = self.LANGUAGE_FONT_MAP[line_language].removesuffix('-Regular')
log.warning(
"No font found with glyphs for '%s' text. "
"Install %s for better rendering. "
"See https://fonts.google.com/noto",
"No installed font has glyphs for the detected '%s' text, so "
"it was added as an invisible text layer: it stays searchable "
"and copyable, but appears blank when highlighted in a PDF "
"viewer. Install the %s font family (via your OS package "
"manager or https://fonts.google.com/noto) for full rendering.",
line_language,
font_name,
font_family,
)
else:
log.warning(
"No font found with glyphs for some text. "
"Install Noto fonts for better rendering. "
"See https://fonts.google.com/noto"
"No installed font has glyphs for some of the detected text, "
"so it was added as an invisible text layer: it stays "
"searchable and copyable, but appears blank when highlighted "
"in a PDF viewer. Install the matching Noto fonts "
"(https://fonts.google.com/noto) for full rendering."
)
def _has_all_glyphs(self, font: FontManager, text: str) -> bool:
+122
View File
@@ -9,6 +9,7 @@ Linux, macOS, and Windows platforms.
from __future__ import annotations
import glob
import logging
import os
import sys
@@ -75,6 +76,35 @@ class SystemFontProvider:
# Variable fonts
'NotoSansCJKsc-VF.otf',
],
# Per-language CJK families. Modern Google Fonts / Homebrew ship these
# as region subset variable fonts ('NotoSansJP[wght].ttf'), matched by
# the flexible base search; the legacy per-region super OTFs (full
# coverage) are listed here so they also satisfy the logical name.
'NotoSansSC-Regular': [
'NotoSansSC-Regular.otf',
'NotoSansSC-Regular.ttf',
'NotoSansCJKsc-Regular.otf',
],
'NotoSansTC-Regular': [
'NotoSansTC-Regular.otf',
'NotoSansTC-Regular.ttf',
'NotoSansCJKtc-Regular.otf',
],
'NotoSansHK-Regular': [
'NotoSansHK-Regular.otf',
'NotoSansHK-Regular.ttf',
'NotoSansCJKhk-Regular.otf',
],
'NotoSansJP-Regular': [
'NotoSansJP-Regular.otf',
'NotoSansJP-Regular.ttf',
'NotoSansCJKjp-Regular.otf',
],
'NotoSansKR-Regular': [
'NotoSansKR-Regular.otf',
'NotoSansKR-Regular.ttf',
'NotoSansCJKkr-Regular.otf',
],
'NotoSansThai-Regular': [
'NotoSansThai-Regular.ttf',
'NotoSansThai-Regular.otf',
@@ -149,6 +179,28 @@ class SystemFontProvider:
],
}
# Font file extensions we know how to load.
_FONT_EXTENSIONS = ('.ttf', '.otf', '.ttc')
# Acceptable filename variants for a font family, ranked best-first.
# Lower rank wins when multiple variants of the same family are present.
_VARIANT_RANK = {'regular': 0, 'variable': 1, 'vf': 2, 'plain': 3}
# Extra family bases that can satisfy a logical font, tried after its own
# base (so the listed order is the preference). CJK is the case that needs
# this: the legacy Adobe-style 'NotoSansCJKsc-Regular.otf' is handled by
# NOTO_FONT_PATTERNS, but Homebrew casks and current Google Fonts ship the
# per-language families as variable fonts (e.g. 'NotoSansSC[wght].ttf').
_ALTERNATE_BASES: dict[str, list[str]] = {
'NotoSansCJK-Regular': [
'NotoSansSC', # Simplified Chinese
'NotoSansTC', # Traditional Chinese
'NotoSansHK', # Hong Kong
'NotoSansJP', # Japanese
'NotoSansKR', # Korean
],
}
def __init__(self) -> None:
"""Initialize system font provider with empty caches."""
# Cache: font_name -> FontManager (successfully loaded fonts)
@@ -230,6 +282,76 @@ class SystemFontProvider:
# Skip directories we can't read
continue
# No exact static '-Regular' file. Many distributors (Homebrew casks,
# current Google Fonts releases) ship Noto fonts as variable fonts with
# bracketed axis filenames such as 'NotoSansArabic[wdth,wght].ttf'.
# Fall back to a flexible search that also accepts those. See #1652.
return self._find_variant_font_file(font_name)
@staticmethod
def _classify_variant(stem: str, base: str) -> str | None:
"""Classify a font filename stem as a usable variant of ``base``.
Args:
stem: Filename without extension (e.g. 'NotoSansArabic[wdth,wght]')
base: Family base name (e.g. 'NotoSansArabic')
Returns:
The variant kind ('regular', 'variable', 'vf', 'plain') or None if
the stem is not an acceptable representative of the family. The
boundary after ``base`` is required so that 'NotoSans' does not
match 'NotoSansArabic', and 'NotoSansArabicUI'/'NotoSansArabic-Bold'
do not match a request for 'NotoSansArabic'.
"""
if stem == f'{base}-Regular':
return 'regular'
if stem.startswith(f'{base}['): # variable font, e.g. Base[wdth,wght]
return 'variable'
if stem == f'{base}-VF': # alternate variable-font naming
return 'vf'
if stem == base: # bare family name
return 'plain'
return None
def _find_variant_font_file(self, font_name: str) -> Path | None:
"""Search for a variable font or other acceptable filename variant.
Tries the font's own family base first, then any alternate bases (used
for the modern per-language CJK families). Within that, a static Regular
is preferred over a variable font. See issue #1652.
Args:
font_name: Logical font name (e.g. 'NotoSansArabic-Regular')
Returns:
Path to the best-ranked matching font file, or None.
"""
bases = [font_name.removesuffix('-Regular')]
bases.extend(self._ALTERNATE_BASES.get(font_name, []))
# Selection key (base_index, variant_rank): earlier base wins, then the
# better variant. Path is carried along but not part of the comparison.
best: tuple[tuple[int, int], Path] | None = None
for base_index, base in enumerate(bases):
for font_dir in self._get_font_dirs():
if not font_dir.exists():
continue
try:
for path in font_dir.rglob(glob.escape(base) + '*'):
if path.suffix.lower() not in self._FONT_EXTENSIONS:
continue
kind = self._classify_variant(path.stem, base)
if kind is None:
continue
key = (base_index, self._VARIANT_RANK[kind])
if best is None or key < best[0]:
best = (key, path)
except PermissionError:
# Skip directories we can't read
continue
if best is not None:
log.debug("Found system font %s at %s (variant match)", font_name, best[1])
return best[1]
return None
def get_font(self, font_name: str) -> FontManager | None:
-7
View File
@@ -17,7 +17,6 @@ import pdfminer
import pdfminer.encodingdb
import pdfminer.pdfdevice
import pdfminer.pdfinterp
import pdfminer.psparser
from deprecation import deprecated
from pdfminer.converter import PDFLayoutAnalyzer
from pdfminer.layout import LAParams, LTChar, LTPage, LTTextBox
@@ -60,12 +59,6 @@ def pdfsimplefont__init__(
PDFSimpleFont.__init__ = pdfsimplefont__init__
# Patch pdfminer.six buffer size
# The parser doesn't properly handle keyword tokens are split across the end of the
# buffer, so increase the buffer size something far larger than will ever be seen.
pdfminer.psparser.PSBaseParser.BUFSIZ = 256 * 1024 * 1024
def pdftype3font__pscript5_get_height(self):
"""Monkeypatch for PScript5.dll PDFs.
+134 -7
View File
@@ -149,7 +149,9 @@ def test_select_font_for_chinese_language(multi_font_manager):
if not has_cjk_font(multi_font_manager):
pytest.skip("CJK font not available")
font_manager = multi_font_manager.select_font_for_word("你好", "zho")
assert font_manager == multi_font_manager.fonts['NotoSansCJK-Regular']
# A real, glyph-covering CJK font is selected (which specific family
# depends on what is installed: pan-CJK super font or a per-language subset).
assert font_manager.font_path.name != 'Occulta.ttf'
def test_select_font_for_chinese_generic(multi_font_manager):
@@ -157,7 +159,9 @@ def test_select_font_for_chinese_generic(multi_font_manager):
if not has_cjk_font(multi_font_manager):
pytest.skip("CJK font not available")
font_manager = multi_font_manager.select_font_for_word("中文", "chi")
assert font_manager == multi_font_manager.fonts['NotoSansCJK-Regular']
# A real, glyph-covering CJK font is selected (which specific family
# depends on what is installed: pan-CJK super font or a per-language subset).
assert font_manager.font_path.name != 'Occulta.ttf'
def test_select_font_for_chinese_simplified(multi_font_manager):
@@ -165,7 +169,9 @@ def test_select_font_for_chinese_simplified(multi_font_manager):
if not has_cjk_font(multi_font_manager):
pytest.skip("CJK font not available")
font_manager = multi_font_manager.select_font_for_word("简体字", "chi_sim")
assert font_manager == multi_font_manager.fonts['NotoSansCJK-Regular']
# A real, glyph-covering CJK font is selected (which specific family
# depends on what is installed: pan-CJK super font or a per-language subset).
assert font_manager.font_path.name != 'Occulta.ttf'
def test_select_font_for_chinese_traditional(multi_font_manager):
@@ -173,7 +179,9 @@ def test_select_font_for_chinese_traditional(multi_font_manager):
if not has_cjk_font(multi_font_manager):
pytest.skip("CJK font not available")
font_manager = multi_font_manager.select_font_for_word("漢字", "chi_tra")
assert font_manager == multi_font_manager.fonts['NotoSansCJK-Regular']
# A real, glyph-covering CJK font is selected (which specific family
# depends on what is installed: pan-CJK super font or a per-language subset).
assert font_manager.font_path.name != 'Occulta.ttf'
def test_select_font_for_japanese_language(multi_font_manager):
@@ -181,7 +189,9 @@ def test_select_font_for_japanese_language(multi_font_manager):
if not has_cjk_font(multi_font_manager):
pytest.skip("CJK font not available")
font_manager = multi_font_manager.select_font_for_word("こんにちは", "jpn")
assert font_manager == multi_font_manager.fonts['NotoSansCJK-Regular']
# A real, glyph-covering CJK font is selected (which specific family
# depends on what is installed: pan-CJK super font or a per-language subset).
assert font_manager.font_path.name != 'Occulta.ttf'
def test_select_font_for_korean_language(multi_font_manager):
@@ -189,7 +199,9 @@ def test_select_font_for_korean_language(multi_font_manager):
if not has_cjk_font(multi_font_manager):
pytest.skip("CJK font not available")
font_manager = multi_font_manager.select_font_for_word("안녕하세요", "kor")
assert font_manager == multi_font_manager.fonts['NotoSansCJK-Regular']
# A real, glyph-covering CJK font is selected (which specific family
# depends on what is installed: pan-CJK super font or a per-language subset).
assert font_manager.font_path.name != 'Occulta.ttf'
# --- Latin/English Tests ---
@@ -232,7 +244,9 @@ def test_cjk_text_without_language_hint(multi_font_manager):
if not has_cjk_font(multi_font_manager):
pytest.skip("CJK font not available")
font_manager = multi_font_manager.select_font_for_word("你好", None)
assert font_manager == multi_font_manager.fonts['NotoSansCJK-Regular']
# A real, glyph-covering CJK font is selected (which specific family
# depends on what is installed: pan-CJK super font or a per-language subset).
assert font_manager.font_path.name != 'Occulta.ttf'
def test_fallback_to_occulta_font(multi_font_manager):
@@ -444,3 +458,116 @@ def test_builtin_font_provider_missing_occulta_raises(tmp_path):
"""Test that missing Occulta.ttf raises FileNotFoundError."""
with pytest.raises(FileNotFoundError, match="Required fallback font"):
BuiltinFontProvider(tmp_path)
class _StubHbFont:
"""Minimal uharfbuzz Font stand-in with controllable glyph coverage."""
def __init__(self, covered_codepoints: set[int]):
self._covered = covered_codepoints
def get_nominal_glyph(self, codepoint: int) -> int:
return 1 if codepoint in self._covered else 0
class _FakeFontManager:
"""FontManager stand-in whose glyph coverage is fixed per test."""
def __init__(self, name: str, covered_chars: str):
self.font_path = Path(name)
self._hb = _StubHbFont({ord(c) for c in covered_chars})
def get_hb_font(self) -> _StubHbFont:
return self._hb
class _FakeFontProvider:
"""FontProvider returning controlled fonts by logical name."""
def __init__(self, fonts: dict[str, _FakeFontManager]):
self._fonts = fonts
self._fallback = _FakeFontManager('Occulta.ttf', '')
def get_font(self, name: str) -> _FakeFontManager | None:
return self._fonts.get(name)
def get_available_fonts(self) -> list[str]:
return list(self._fonts)
def get_fallback_font(self) -> _FakeFontManager:
return self._fallback
def test_japanese_prefers_jp_family_over_other_cjk():
"""A Japanese language hint selects NotoSansJP, not another CJK family."""
fonts = {
'NotoSansSC-Regular': _FakeFontManager('NotoSansSC.ttf', ''),
'NotoSansJP-Regular': _FakeFontManager('NotoSansJP.ttf', '中こ'),
}
manager = MultiFontManager(font_provider=_FakeFontProvider(fonts))
# 'こ' (kana) is only covered by JP; both cover the kanji '中'.
font = manager.select_font_for_word('中こ', 'jpn')
assert font.font_path.name == 'NotoSansJP.ttf'
def test_chinese_simplified_prefers_sc_family():
"""A Simplified Chinese hint selects NotoSansSC over the pan-CJK font."""
fonts = {
'NotoSansSC-Regular': _FakeFontManager('NotoSansSC.ttf', ''),
'NotoSansCJK-Regular': _FakeFontManager('NotoSansCJK.ttc', ''),
}
manager = MultiFontManager(font_provider=_FakeFontProvider(fonts))
font = manager.select_font_for_word('', 'chi_sim')
assert font.font_path.name == 'NotoSansSC.ttf'
def test_cjk_fails_over_when_preferred_subset_lacks_glyph():
"""If the language's subset font lacks a glyph, another CJK family is used."""
fonts = {
# Simplified Chinese subset cannot render Japanese kana.
'NotoSansSC-Regular': _FakeFontManager('NotoSansSC.ttf', ''),
'NotoSansJP-Regular': _FakeFontManager('NotoSansJP.ttf', '中こ'),
}
manager = MultiFontManager(font_provider=_FakeFontProvider(fonts))
# Tagged Simplified Chinese, but the text needs kana only JP covers.
font = manager.select_font_for_word('', 'chi_sim')
assert font.font_path.name == 'NotoSansJP.ttf'
def test_cjk_falls_back_to_pan_cjk_super_font():
"""When only the full-coverage pan-CJK font exists, it serves any CJK lang."""
fonts = {
'NotoSansCJK-Regular': _FakeFontManager('NotoSansCJK.ttc', '中こ안'),
}
manager = MultiFontManager(font_provider=_FakeFontProvider(fonts))
assert manager.select_font_for_word('', 'jpn').font_path.name == 'NotoSansCJK.ttc'
def test_missing_cjk_font_warning_names_language_font(font_dir, caplog):
"""The missing-font warning names the language-specific CJK family (#1652)."""
manager = MultiFontManager(font_provider=BuiltinFontProvider(font_dir))
with caplog.at_level(logging.WARNING):
manager.select_font_for_word("こんにちは", "jpn")
assert 'NotoSansJP' in caplog.text
def test_missing_font_warning_explains_consequences(font_dir, caplog):
"""The glyphless-fallback warning should be actionable, not cryptic (#1652).
With only builtin fonts available, Arabic text cannot be covered, so the
manager falls back to glyphless Occulta and must warn helpfully.
"""
# Builtin-only provider: NotoSansArabic is never available, forcing fallback.
manager = MultiFontManager(font_provider=BuiltinFontProvider(font_dir))
with caplog.at_level(logging.WARNING):
manager.select_font_for_word("سلام", "fas")
msg = caplog.text
# Identifies the affected language and the font family to install.
assert 'fas' in msg
assert 'NotoSansArabic' in msg
# Explains the user-visible consequence so the message is not cryptic:
# the text stays searchable but renders blank when highlighted.
assert 'searchable' in msg.lower()
assert 'highlight' in msg.lower() or 'select' in msg.lower()
+163
View File
@@ -190,6 +190,9 @@ class TestSystemFontProviderAvailableFonts:
assert 'NotoSansCJK-Regular' in fonts
assert 'NotoSansArabic-Regular' in fonts
assert 'NotoSansThai-Regular' in fonts
# Per-language CJK families (modern Google Fonts / Homebrew naming)
assert 'NotoSansSC-Regular' in fonts
assert 'NotoSansJP-Regular' in fonts
def test_fallback_font_raises(self):
"""Test that get_fallback_font raises NotImplementedError."""
@@ -198,6 +201,166 @@ class TestSystemFontProviderAvailableFonts:
provider.get_fallback_font()
class TestSystemFontProviderVariableFonts:
"""Test discovery of variable fonts and non-static filename variants.
Homebrew casks and Google Fonts ship Noto fonts as variable fonts with
bracketed axis filenames (e.g. ``NotoSansArabic[wdth,wght].ttf``) rather
than the static ``NotoSansArabic-Regular.ttf``. See issue #1652.
"""
@pytest.fixture
def real_font_bytes(self):
"""Bytes of a real, loadable font (content is irrelevant to the test)."""
font_path = (
Path(__file__).parent.parent
/ "src"
/ "ocrmypdf"
/ "data"
/ "NotoSans-Regular.ttf"
)
if not font_path.exists():
pytest.skip("Builtin font not available")
return font_path.read_bytes()
def _provider_for(self, tmp_path, filenames, real_font_bytes):
"""Build a provider whose only font dir is tmp_path with given files."""
for name in filenames:
(tmp_path / name).write_bytes(real_font_bytes)
provider = SystemFontProvider()
provider._font_dirs = [tmp_path]
return provider
def test_finds_variable_font_with_axes(self, tmp_path, real_font_bytes):
"""A bracketed variable font satisfies a request for the static name."""
provider = self._provider_for(
tmp_path, ['NotoSansArabic[wdth,wght].ttf'], real_font_bytes
)
font = provider.get_font('NotoSansArabic-Regular')
assert font is not None
assert font.font_path.name == 'NotoSansArabic[wdth,wght].ttf'
def test_finds_weight_only_variable_font(self, tmp_path, real_font_bytes):
"""A variable font with only a weight axis is also discovered."""
provider = self._provider_for(
tmp_path, ['NotoSansHebrew[wght].ttf'], real_font_bytes
)
assert provider.get_font('NotoSansHebrew-Regular') is not None
def test_variable_font_does_not_cross_match_other_script(
self, tmp_path, real_font_bytes
):
"""The generic NotoSans request must not match a script-specific font."""
provider = self._provider_for(
tmp_path, ['NotoSansArabic[wdth,wght].ttf'], real_font_bytes
)
# NotoSans (Latin) must NOT be satisfied by NotoSansArabic.
assert provider.get_font('NotoSans-Regular') is None
def test_does_not_match_ui_or_bold_variants(self, tmp_path, real_font_bytes):
"""Width/UI and weight variants must not satisfy the Regular request."""
provider = self._provider_for(
tmp_path,
['NotoSansArabicUI-Regular.ttf', 'NotoSansArabic-Bold.ttf'],
real_font_bytes,
)
assert provider.get_font('NotoSansArabic-Regular') is None
def test_prefers_static_regular_over_variable(self, tmp_path, real_font_bytes):
"""When both exist, the static Regular is preferred for predictability."""
provider = self._provider_for(
tmp_path,
['NotoSansArabic[wdth,wght].ttf', 'NotoSansArabic-Regular.ttf'],
real_font_bytes,
)
font = provider.get_font('NotoSansArabic-Regular')
assert font is not None
assert font.font_path.name == 'NotoSansArabic-Regular.ttf'
# --- Modern per-language CJK families (NotoSansSC/TC/HK/JP/KR) ---
# Homebrew casks (font-noto-sans-sc, ...) and Google Fonts ship CJK as
# variable fonts under these bases rather than the legacy NotoSansCJK*.
@pytest.mark.parametrize(
'filename',
[
'NotoSansSC[wght].ttf', # Simplified Chinese (Homebrew/Google)
'NotoSansTC[wght].ttf', # Traditional Chinese
'NotoSansHK[wght].ttf', # Hong Kong
'NotoSansJP[wght].ttf', # Japanese
'NotoSansKR[wght].ttf', # Korean
],
)
def test_finds_modern_cjk_variable_font(self, tmp_path, real_font_bytes, filename):
"""A modern per-language CJK variable font satisfies NotoSansCJK."""
provider = self._provider_for(tmp_path, [filename], real_font_bytes)
font = provider.get_font('NotoSansCJK-Regular')
assert font is not None
assert font.font_path.name == filename
def test_finds_static_cjk_language_variant(self, tmp_path, real_font_bytes):
"""A static per-language CJK Regular also satisfies NotoSansCJK."""
provider = self._provider_for(
tmp_path, ['NotoSansTC-Regular.otf'], real_font_bytes
)
assert provider.get_font('NotoSansCJK-Regular') is not None
def test_prefers_pan_cjk_over_language_variant(self, tmp_path, real_font_bytes):
"""The pan-CJK family is preferred over a single-language variant."""
provider = self._provider_for(
tmp_path,
['NotoSansSC[wght].ttf', 'NotoSansCJK[wght].ttf'],
real_font_bytes,
)
font = provider.get_font('NotoSansCJK-Regular')
assert font is not None
assert font.font_path.name == 'NotoSansCJK[wght].ttf'
def test_modern_cjk_does_not_cross_match_latin(self, tmp_path, real_font_bytes):
"""A CJK variable font must not satisfy the generic NotoSans request."""
provider = self._provider_for(
tmp_path, ['NotoSansSC[wght].ttf'], real_font_bytes
)
assert provider.get_font('NotoSans-Regular') is None
# --- Per-language CJK families reachable by their own logical name ---
# Needed so MultiFontManager can prefer the family matching the document
# language (NotoSansJP for Japanese, NotoSansSC for Simplified Chinese, ...).
@pytest.mark.parametrize(
'logical,filename',
[
('NotoSansSC-Regular', 'NotoSansSC[wght].ttf'),
('NotoSansTC-Regular', 'NotoSansTC[wght].ttf'),
('NotoSansHK-Regular', 'NotoSansHK[wght].ttf'),
('NotoSansJP-Regular', 'NotoSansJP[wght].ttf'),
('NotoSansKR-Regular', 'NotoSansKR[wght].ttf'),
],
)
def test_per_language_cjk_logical_name_resolves(
self, tmp_path, real_font_bytes, logical, filename
):
"""Each per-language CJK family is reachable by its own logical name."""
provider = self._provider_for(tmp_path, [filename], real_font_bytes)
font = provider.get_font(logical)
assert font is not None
assert font.font_path.name == filename
def test_per_language_cjk_static_resolves(self, tmp_path, real_font_bytes):
"""A static per-language Regular also resolves by logical name."""
provider = self._provider_for(
tmp_path, ['NotoSansJP-Regular.otf'], real_font_bytes
)
assert provider.get_font('NotoSansJP-Regular') is not None
def test_per_language_cjk_does_not_cross_match(self, tmp_path, real_font_bytes):
"""A JP font must not satisfy an SC request (distinct families)."""
provider = self._provider_for(
tmp_path, ['NotoSansJP[wght].ttf'], real_font_bytes
)
assert provider.get_font('NotoSansSC-Regular') is None
# --- ChainedFontProvider Tests ---
Generated
+1 -1
View File
@@ -1434,7 +1434,7 @@ wheels = [
[[package]]
name = "ocrmypdf"
version = "17.6.0"
version = "17.7.0"
source = { editable = "." }
dependencies = [
{ name = "deprecation" },