Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
de403f6d5e | ||
|
|
86b6f2c907 | ||
|
|
e6fab76918 | ||
|
|
334918d0f7 |
+4
-3
@@ -18,8 +18,9 @@ import cyclopts
|
||||
from packaging.version import InvalidVersion, Version
|
||||
|
||||
try:
|
||||
from github import Github, GithubException
|
||||
from github import Auth, Github, GithubException
|
||||
except ImportError:
|
||||
Auth = None # type: ignore
|
||||
Github = None # type: ignore
|
||||
GithubException = Exception # type: ignore
|
||||
|
||||
@@ -68,7 +69,7 @@ def validate_release_notes(new_version: str) -> bool:
|
||||
|
||||
def get_github_client():
|
||||
"""Get an authenticated GitHub client."""
|
||||
if Github is None:
|
||||
if Github is None or Auth is None:
|
||||
print(f"{RED}error:{OFF} PyGithub is not installed")
|
||||
print(" Install with: pip install PyGithub")
|
||||
return None
|
||||
@@ -92,7 +93,7 @@ def get_github_client():
|
||||
return None
|
||||
|
||||
try:
|
||||
return Github(token)
|
||||
return Github(auth=Auth.Token(token))
|
||||
except GithubException as e:
|
||||
print(f"{RED}error:{OFF} Failed to authenticate with GitHub: {e}")
|
||||
return None
|
||||
|
||||
@@ -3,6 +3,36 @@
|
||||
|
||||
# v17
|
||||
|
||||
## v17.7.1
|
||||
|
||||
- Fixed a severe, Windows-specific performance regression in the "Scanning
|
||||
contents" phase, most visible with `--redo-ocr` ({issue}`1662`). Since
|
||||
v16.4.3, OCRmyPDF forced pdfminer's read buffer to 256 MiB to work around a
|
||||
pdfminer bug that mishandled tokens split across the buffer boundary
|
||||
({issue}`1361`). On Windows, CPython's `BufferedReader.read()` eagerly
|
||||
allocates a buffer of the requested size on every read, so the oversized
|
||||
buffer made each of pdfminer's thousands of reads cost tens of milliseconds
|
||||
(this allocation is lazy, and effectively free, on Linux). The underlying
|
||||
pdfminer bug was fixed upstream in pdfminer.six 20250327
|
||||
([#1030](https://github.com/pdfminer/pdfminer.six/pull/1030)), with a
|
||||
follow-up for tokens split across streams in 20260107
|
||||
([#1158](https://github.com/pdfminer/pdfminer.six/pull/1158)), so the
|
||||
workaround has been removed and the minimum pdfminer.six version raised to
|
||||
20260107.
|
||||
- The font discovery used to build the OCR text layer now finds variable fonts
|
||||
such as `NotoSansArabic[wdth,wght].ttf`, the form shipped by Homebrew casks
|
||||
and current Google Fonts releases. Previously only static `-Regular.ttf`/`.otf`
|
||||
files were matched, so users who had installed the correct Noto font still got
|
||||
the glyphless fallback and a "No font found" warning ({issue}`1652`).
|
||||
- Font discovery is now language-aware for CJK: each Chinese, Japanese, and
|
||||
Korean language maps to its own per-language Noto family (NotoSansSC, TC, HK,
|
||||
JP, KR), with the pan-CJK super font kept as a shared fallback, since the
|
||||
per-language fonts are region subsets that may lack glyphs from other scripts.
|
||||
- The warning shown when no installed font has glyphs for some text was reworded
|
||||
to explain the consequence — the text is still added as a searchable, copyable
|
||||
layer but appears blank when highlighted in a viewer — and to name the specific
|
||||
font family to install.
|
||||
|
||||
## v17.7.0
|
||||
|
||||
- The Docker images now run as a non-root user (`app`, uid/gid 1000) by default
|
||||
|
||||
+2
-2
@@ -6,7 +6,7 @@ build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "ocrmypdf"
|
||||
version = "17.7.0"
|
||||
version = "17.7.1"
|
||||
description = "OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched"
|
||||
readme = "README.md"
|
||||
license = "MPL-2.0"
|
||||
@@ -16,7 +16,7 @@ dependencies = [
|
||||
"fpdf2>=2.8.0",
|
||||
"img2pdf>=0.5",
|
||||
"packaging>=20",
|
||||
"pdfminer.six>=20220319",
|
||||
"pdfminer.six>=20260107", # fixes parsing of tokens split across the read buffer/streams (gh #1361)
|
||||
"pi-heif", # Heif image format - maintainers: if this is removed, it will NOT break
|
||||
"pikepdf>=10",
|
||||
"Pillow>=10.0.1",
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
# SPDX-FileCopyrightText: 2022 James R. Barlow
|
||||
# SPDX-License-Identifier: MPL-2.0
|
||||
__version__ = "17.7.0"
|
||||
__version__ = "17.7.1"
|
||||
|
||||
@@ -54,13 +54,15 @@ class MultiFontManager:
|
||||
'kok': 'NotoSansDevanagari-Regular', # Konkani
|
||||
'bho': 'NotoSansDevanagari-Regular', # Bhojpuri
|
||||
'mai': 'NotoSansDevanagari-Regular', # Maithili
|
||||
# CJK
|
||||
'chi': 'NotoSansCJK-Regular', # Chinese (generic)
|
||||
'zho': 'NotoSansCJK-Regular', # Chinese (ISO 639-3)
|
||||
'chi_sim': 'NotoSansCJK-Regular', # Chinese Simplified (Tesseract)
|
||||
'chi_tra': 'NotoSansCJK-Regular', # Chinese Traditional (Tesseract)
|
||||
'jpn': 'NotoSansCJK-Regular', # Japanese
|
||||
'kor': 'NotoSansCJK-Regular', # Korean
|
||||
# CJK — prefer the family matching the document language, because the
|
||||
# modern per-language Noto fonts are region subsets (e.g. NotoSansSC
|
||||
# lacks Japanese kana). The pan-CJK super font is a shared fallback.
|
||||
'chi': 'NotoSansSC-Regular', # Chinese (generic → Simplified)
|
||||
'zho': 'NotoSansSC-Regular', # Chinese (ISO 639-3)
|
||||
'chi_sim': 'NotoSansSC-Regular', # Chinese Simplified (Tesseract)
|
||||
'chi_tra': 'NotoSansTC-Regular', # Chinese Traditional (Tesseract)
|
||||
'jpn': 'NotoSansJP-Regular', # Japanese
|
||||
'kor': 'NotoSansKR-Regular', # Korean
|
||||
# Thai
|
||||
'tha': 'NotoSansThai-Regular', # Thai
|
||||
# Hebrew
|
||||
@@ -113,7 +115,14 @@ class MultiFontManager:
|
||||
'NotoSans-Regular', # Latin, Greek, Cyrillic
|
||||
'NotoSansArabic-Regular',
|
||||
'NotoSansDevanagari-Regular',
|
||||
# Pan-CJK super font first (full coverage), then the per-language
|
||||
# subsets so a glyph missing from one CJK family is found in another.
|
||||
'NotoSansCJK-Regular',
|
||||
'NotoSansSC-Regular',
|
||||
'NotoSansTC-Regular',
|
||||
'NotoSansHK-Regular',
|
||||
'NotoSansJP-Regular',
|
||||
'NotoSansKR-Regular',
|
||||
'NotoSansThai-Regular',
|
||||
'NotoSansHebrew-Regular',
|
||||
'NotoSansBengali-Regular',
|
||||
@@ -256,19 +265,23 @@ class MultiFontManager:
|
||||
self._warned_scripts.add(warn_key)
|
||||
|
||||
if line_language and line_language in self.LANGUAGE_FONT_MAP:
|
||||
font_name = self.LANGUAGE_FONT_MAP[line_language]
|
||||
font_family = self.LANGUAGE_FONT_MAP[line_language].removesuffix('-Regular')
|
||||
log.warning(
|
||||
"No font found with glyphs for '%s' text. "
|
||||
"Install %s for better rendering. "
|
||||
"See https://fonts.google.com/noto",
|
||||
"No installed font has glyphs for the detected '%s' text, so "
|
||||
"it was added as an invisible text layer: it stays searchable "
|
||||
"and copyable, but appears blank when highlighted in a PDF "
|
||||
"viewer. Install the %s font family (via your OS package "
|
||||
"manager or https://fonts.google.com/noto) for full rendering.",
|
||||
line_language,
|
||||
font_name,
|
||||
font_family,
|
||||
)
|
||||
else:
|
||||
log.warning(
|
||||
"No font found with glyphs for some text. "
|
||||
"Install Noto fonts for better rendering. "
|
||||
"See https://fonts.google.com/noto"
|
||||
"No installed font has glyphs for some of the detected text, "
|
||||
"so it was added as an invisible text layer: it stays "
|
||||
"searchable and copyable, but appears blank when highlighted "
|
||||
"in a PDF viewer. Install the matching Noto fonts "
|
||||
"(https://fonts.google.com/noto) for full rendering."
|
||||
)
|
||||
|
||||
def _has_all_glyphs(self, font: FontManager, text: str) -> bool:
|
||||
|
||||
@@ -9,6 +9,7 @@ Linux, macOS, and Windows platforms.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import glob
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
@@ -75,6 +76,35 @@ class SystemFontProvider:
|
||||
# Variable fonts
|
||||
'NotoSansCJKsc-VF.otf',
|
||||
],
|
||||
# Per-language CJK families. Modern Google Fonts / Homebrew ship these
|
||||
# as region subset variable fonts ('NotoSansJP[wght].ttf'), matched by
|
||||
# the flexible base search; the legacy per-region super OTFs (full
|
||||
# coverage) are listed here so they also satisfy the logical name.
|
||||
'NotoSansSC-Regular': [
|
||||
'NotoSansSC-Regular.otf',
|
||||
'NotoSansSC-Regular.ttf',
|
||||
'NotoSansCJKsc-Regular.otf',
|
||||
],
|
||||
'NotoSansTC-Regular': [
|
||||
'NotoSansTC-Regular.otf',
|
||||
'NotoSansTC-Regular.ttf',
|
||||
'NotoSansCJKtc-Regular.otf',
|
||||
],
|
||||
'NotoSansHK-Regular': [
|
||||
'NotoSansHK-Regular.otf',
|
||||
'NotoSansHK-Regular.ttf',
|
||||
'NotoSansCJKhk-Regular.otf',
|
||||
],
|
||||
'NotoSansJP-Regular': [
|
||||
'NotoSansJP-Regular.otf',
|
||||
'NotoSansJP-Regular.ttf',
|
||||
'NotoSansCJKjp-Regular.otf',
|
||||
],
|
||||
'NotoSansKR-Regular': [
|
||||
'NotoSansKR-Regular.otf',
|
||||
'NotoSansKR-Regular.ttf',
|
||||
'NotoSansCJKkr-Regular.otf',
|
||||
],
|
||||
'NotoSansThai-Regular': [
|
||||
'NotoSansThai-Regular.ttf',
|
||||
'NotoSansThai-Regular.otf',
|
||||
@@ -149,6 +179,28 @@ class SystemFontProvider:
|
||||
],
|
||||
}
|
||||
|
||||
# Font file extensions we know how to load.
|
||||
_FONT_EXTENSIONS = ('.ttf', '.otf', '.ttc')
|
||||
|
||||
# Acceptable filename variants for a font family, ranked best-first.
|
||||
# Lower rank wins when multiple variants of the same family are present.
|
||||
_VARIANT_RANK = {'regular': 0, 'variable': 1, 'vf': 2, 'plain': 3}
|
||||
|
||||
# Extra family bases that can satisfy a logical font, tried after its own
|
||||
# base (so the listed order is the preference). CJK is the case that needs
|
||||
# this: the legacy Adobe-style 'NotoSansCJKsc-Regular.otf' is handled by
|
||||
# NOTO_FONT_PATTERNS, but Homebrew casks and current Google Fonts ship the
|
||||
# per-language families as variable fonts (e.g. 'NotoSansSC[wght].ttf').
|
||||
_ALTERNATE_BASES: dict[str, list[str]] = {
|
||||
'NotoSansCJK-Regular': [
|
||||
'NotoSansSC', # Simplified Chinese
|
||||
'NotoSansTC', # Traditional Chinese
|
||||
'NotoSansHK', # Hong Kong
|
||||
'NotoSansJP', # Japanese
|
||||
'NotoSansKR', # Korean
|
||||
],
|
||||
}
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""Initialize system font provider with empty caches."""
|
||||
# Cache: font_name -> FontManager (successfully loaded fonts)
|
||||
@@ -230,6 +282,76 @@ class SystemFontProvider:
|
||||
# Skip directories we can't read
|
||||
continue
|
||||
|
||||
# No exact static '-Regular' file. Many distributors (Homebrew casks,
|
||||
# current Google Fonts releases) ship Noto fonts as variable fonts with
|
||||
# bracketed axis filenames such as 'NotoSansArabic[wdth,wght].ttf'.
|
||||
# Fall back to a flexible search that also accepts those. See #1652.
|
||||
return self._find_variant_font_file(font_name)
|
||||
|
||||
@staticmethod
|
||||
def _classify_variant(stem: str, base: str) -> str | None:
|
||||
"""Classify a font filename stem as a usable variant of ``base``.
|
||||
|
||||
Args:
|
||||
stem: Filename without extension (e.g. 'NotoSansArabic[wdth,wght]')
|
||||
base: Family base name (e.g. 'NotoSansArabic')
|
||||
|
||||
Returns:
|
||||
The variant kind ('regular', 'variable', 'vf', 'plain') or None if
|
||||
the stem is not an acceptable representative of the family. The
|
||||
boundary after ``base`` is required so that 'NotoSans' does not
|
||||
match 'NotoSansArabic', and 'NotoSansArabicUI'/'NotoSansArabic-Bold'
|
||||
do not match a request for 'NotoSansArabic'.
|
||||
"""
|
||||
if stem == f'{base}-Regular':
|
||||
return 'regular'
|
||||
if stem.startswith(f'{base}['): # variable font, e.g. Base[wdth,wght]
|
||||
return 'variable'
|
||||
if stem == f'{base}-VF': # alternate variable-font naming
|
||||
return 'vf'
|
||||
if stem == base: # bare family name
|
||||
return 'plain'
|
||||
return None
|
||||
|
||||
def _find_variant_font_file(self, font_name: str) -> Path | None:
|
||||
"""Search for a variable font or other acceptable filename variant.
|
||||
|
||||
Tries the font's own family base first, then any alternate bases (used
|
||||
for the modern per-language CJK families). Within that, a static Regular
|
||||
is preferred over a variable font. See issue #1652.
|
||||
|
||||
Args:
|
||||
font_name: Logical font name (e.g. 'NotoSansArabic-Regular')
|
||||
|
||||
Returns:
|
||||
Path to the best-ranked matching font file, or None.
|
||||
"""
|
||||
bases = [font_name.removesuffix('-Regular')]
|
||||
bases.extend(self._ALTERNATE_BASES.get(font_name, []))
|
||||
|
||||
# Selection key (base_index, variant_rank): earlier base wins, then the
|
||||
# better variant. Path is carried along but not part of the comparison.
|
||||
best: tuple[tuple[int, int], Path] | None = None
|
||||
for base_index, base in enumerate(bases):
|
||||
for font_dir in self._get_font_dirs():
|
||||
if not font_dir.exists():
|
||||
continue
|
||||
try:
|
||||
for path in font_dir.rglob(glob.escape(base) + '*'):
|
||||
if path.suffix.lower() not in self._FONT_EXTENSIONS:
|
||||
continue
|
||||
kind = self._classify_variant(path.stem, base)
|
||||
if kind is None:
|
||||
continue
|
||||
key = (base_index, self._VARIANT_RANK[kind])
|
||||
if best is None or key < best[0]:
|
||||
best = (key, path)
|
||||
except PermissionError:
|
||||
# Skip directories we can't read
|
||||
continue
|
||||
if best is not None:
|
||||
log.debug("Found system font %s at %s (variant match)", font_name, best[1])
|
||||
return best[1]
|
||||
return None
|
||||
|
||||
def get_font(self, font_name: str) -> FontManager | None:
|
||||
|
||||
@@ -17,7 +17,6 @@ import pdfminer
|
||||
import pdfminer.encodingdb
|
||||
import pdfminer.pdfdevice
|
||||
import pdfminer.pdfinterp
|
||||
import pdfminer.psparser
|
||||
from deprecation import deprecated
|
||||
from pdfminer.converter import PDFLayoutAnalyzer
|
||||
from pdfminer.layout import LAParams, LTChar, LTPage, LTTextBox
|
||||
@@ -60,12 +59,6 @@ def pdfsimplefont__init__(
|
||||
|
||||
PDFSimpleFont.__init__ = pdfsimplefont__init__
|
||||
|
||||
# Patch pdfminer.six buffer size
|
||||
# The parser doesn't properly handle keyword tokens are split across the end of the
|
||||
# buffer, so increase the buffer size something far larger than will ever be seen.
|
||||
pdfminer.psparser.PSBaseParser.BUFSIZ = 256 * 1024 * 1024
|
||||
|
||||
|
||||
def pdftype3font__pscript5_get_height(self):
|
||||
"""Monkeypatch for PScript5.dll PDFs.
|
||||
|
||||
|
||||
@@ -149,7 +149,9 @@ def test_select_font_for_chinese_language(multi_font_manager):
|
||||
if not has_cjk_font(multi_font_manager):
|
||||
pytest.skip("CJK font not available")
|
||||
font_manager = multi_font_manager.select_font_for_word("你好", "zho")
|
||||
assert font_manager == multi_font_manager.fonts['NotoSansCJK-Regular']
|
||||
# A real, glyph-covering CJK font is selected (which specific family
|
||||
# depends on what is installed: pan-CJK super font or a per-language subset).
|
||||
assert font_manager.font_path.name != 'Occulta.ttf'
|
||||
|
||||
|
||||
def test_select_font_for_chinese_generic(multi_font_manager):
|
||||
@@ -157,7 +159,9 @@ def test_select_font_for_chinese_generic(multi_font_manager):
|
||||
if not has_cjk_font(multi_font_manager):
|
||||
pytest.skip("CJK font not available")
|
||||
font_manager = multi_font_manager.select_font_for_word("中文", "chi")
|
||||
assert font_manager == multi_font_manager.fonts['NotoSansCJK-Regular']
|
||||
# A real, glyph-covering CJK font is selected (which specific family
|
||||
# depends on what is installed: pan-CJK super font or a per-language subset).
|
||||
assert font_manager.font_path.name != 'Occulta.ttf'
|
||||
|
||||
|
||||
def test_select_font_for_chinese_simplified(multi_font_manager):
|
||||
@@ -165,7 +169,9 @@ def test_select_font_for_chinese_simplified(multi_font_manager):
|
||||
if not has_cjk_font(multi_font_manager):
|
||||
pytest.skip("CJK font not available")
|
||||
font_manager = multi_font_manager.select_font_for_word("简体字", "chi_sim")
|
||||
assert font_manager == multi_font_manager.fonts['NotoSansCJK-Regular']
|
||||
# A real, glyph-covering CJK font is selected (which specific family
|
||||
# depends on what is installed: pan-CJK super font or a per-language subset).
|
||||
assert font_manager.font_path.name != 'Occulta.ttf'
|
||||
|
||||
|
||||
def test_select_font_for_chinese_traditional(multi_font_manager):
|
||||
@@ -173,7 +179,9 @@ def test_select_font_for_chinese_traditional(multi_font_manager):
|
||||
if not has_cjk_font(multi_font_manager):
|
||||
pytest.skip("CJK font not available")
|
||||
font_manager = multi_font_manager.select_font_for_word("漢字", "chi_tra")
|
||||
assert font_manager == multi_font_manager.fonts['NotoSansCJK-Regular']
|
||||
# A real, glyph-covering CJK font is selected (which specific family
|
||||
# depends on what is installed: pan-CJK super font or a per-language subset).
|
||||
assert font_manager.font_path.name != 'Occulta.ttf'
|
||||
|
||||
|
||||
def test_select_font_for_japanese_language(multi_font_manager):
|
||||
@@ -181,7 +189,9 @@ def test_select_font_for_japanese_language(multi_font_manager):
|
||||
if not has_cjk_font(multi_font_manager):
|
||||
pytest.skip("CJK font not available")
|
||||
font_manager = multi_font_manager.select_font_for_word("こんにちは", "jpn")
|
||||
assert font_manager == multi_font_manager.fonts['NotoSansCJK-Regular']
|
||||
# A real, glyph-covering CJK font is selected (which specific family
|
||||
# depends on what is installed: pan-CJK super font or a per-language subset).
|
||||
assert font_manager.font_path.name != 'Occulta.ttf'
|
||||
|
||||
|
||||
def test_select_font_for_korean_language(multi_font_manager):
|
||||
@@ -189,7 +199,9 @@ def test_select_font_for_korean_language(multi_font_manager):
|
||||
if not has_cjk_font(multi_font_manager):
|
||||
pytest.skip("CJK font not available")
|
||||
font_manager = multi_font_manager.select_font_for_word("안녕하세요", "kor")
|
||||
assert font_manager == multi_font_manager.fonts['NotoSansCJK-Regular']
|
||||
# A real, glyph-covering CJK font is selected (which specific family
|
||||
# depends on what is installed: pan-CJK super font or a per-language subset).
|
||||
assert font_manager.font_path.name != 'Occulta.ttf'
|
||||
|
||||
|
||||
# --- Latin/English Tests ---
|
||||
@@ -232,7 +244,9 @@ def test_cjk_text_without_language_hint(multi_font_manager):
|
||||
if not has_cjk_font(multi_font_manager):
|
||||
pytest.skip("CJK font not available")
|
||||
font_manager = multi_font_manager.select_font_for_word("你好", None)
|
||||
assert font_manager == multi_font_manager.fonts['NotoSansCJK-Regular']
|
||||
# A real, glyph-covering CJK font is selected (which specific family
|
||||
# depends on what is installed: pan-CJK super font or a per-language subset).
|
||||
assert font_manager.font_path.name != 'Occulta.ttf'
|
||||
|
||||
|
||||
def test_fallback_to_occulta_font(multi_font_manager):
|
||||
@@ -444,3 +458,116 @@ def test_builtin_font_provider_missing_occulta_raises(tmp_path):
|
||||
"""Test that missing Occulta.ttf raises FileNotFoundError."""
|
||||
with pytest.raises(FileNotFoundError, match="Required fallback font"):
|
||||
BuiltinFontProvider(tmp_path)
|
||||
|
||||
|
||||
class _StubHbFont:
|
||||
"""Minimal uharfbuzz Font stand-in with controllable glyph coverage."""
|
||||
|
||||
def __init__(self, covered_codepoints: set[int]):
|
||||
self._covered = covered_codepoints
|
||||
|
||||
def get_nominal_glyph(self, codepoint: int) -> int:
|
||||
return 1 if codepoint in self._covered else 0
|
||||
|
||||
|
||||
class _FakeFontManager:
|
||||
"""FontManager stand-in whose glyph coverage is fixed per test."""
|
||||
|
||||
def __init__(self, name: str, covered_chars: str):
|
||||
self.font_path = Path(name)
|
||||
self._hb = _StubHbFont({ord(c) for c in covered_chars})
|
||||
|
||||
def get_hb_font(self) -> _StubHbFont:
|
||||
return self._hb
|
||||
|
||||
|
||||
class _FakeFontProvider:
|
||||
"""FontProvider returning controlled fonts by logical name."""
|
||||
|
||||
def __init__(self, fonts: dict[str, _FakeFontManager]):
|
||||
self._fonts = fonts
|
||||
self._fallback = _FakeFontManager('Occulta.ttf', '')
|
||||
|
||||
def get_font(self, name: str) -> _FakeFontManager | None:
|
||||
return self._fonts.get(name)
|
||||
|
||||
def get_available_fonts(self) -> list[str]:
|
||||
return list(self._fonts)
|
||||
|
||||
def get_fallback_font(self) -> _FakeFontManager:
|
||||
return self._fallback
|
||||
|
||||
|
||||
def test_japanese_prefers_jp_family_over_other_cjk():
|
||||
"""A Japanese language hint selects NotoSansJP, not another CJK family."""
|
||||
fonts = {
|
||||
'NotoSansSC-Regular': _FakeFontManager('NotoSansSC.ttf', '中'),
|
||||
'NotoSansJP-Regular': _FakeFontManager('NotoSansJP.ttf', '中こ'),
|
||||
}
|
||||
manager = MultiFontManager(font_provider=_FakeFontProvider(fonts))
|
||||
# 'こ' (kana) is only covered by JP; both cover the kanji '中'.
|
||||
font = manager.select_font_for_word('中こ', 'jpn')
|
||||
assert font.font_path.name == 'NotoSansJP.ttf'
|
||||
|
||||
|
||||
def test_chinese_simplified_prefers_sc_family():
|
||||
"""A Simplified Chinese hint selects NotoSansSC over the pan-CJK font."""
|
||||
fonts = {
|
||||
'NotoSansSC-Regular': _FakeFontManager('NotoSansSC.ttf', '简'),
|
||||
'NotoSansCJK-Regular': _FakeFontManager('NotoSansCJK.ttc', '简'),
|
||||
}
|
||||
manager = MultiFontManager(font_provider=_FakeFontProvider(fonts))
|
||||
font = manager.select_font_for_word('简', 'chi_sim')
|
||||
assert font.font_path.name == 'NotoSansSC.ttf'
|
||||
|
||||
|
||||
def test_cjk_fails_over_when_preferred_subset_lacks_glyph():
|
||||
"""If the language's subset font lacks a glyph, another CJK family is used."""
|
||||
fonts = {
|
||||
# Simplified Chinese subset cannot render Japanese kana.
|
||||
'NotoSansSC-Regular': _FakeFontManager('NotoSansSC.ttf', '中'),
|
||||
'NotoSansJP-Regular': _FakeFontManager('NotoSansJP.ttf', '中こ'),
|
||||
}
|
||||
manager = MultiFontManager(font_provider=_FakeFontProvider(fonts))
|
||||
# Tagged Simplified Chinese, but the text needs kana only JP covers.
|
||||
font = manager.select_font_for_word('こ', 'chi_sim')
|
||||
assert font.font_path.name == 'NotoSansJP.ttf'
|
||||
|
||||
|
||||
def test_cjk_falls_back_to_pan_cjk_super_font():
|
||||
"""When only the full-coverage pan-CJK font exists, it serves any CJK lang."""
|
||||
fonts = {
|
||||
'NotoSansCJK-Regular': _FakeFontManager('NotoSansCJK.ttc', '中こ안'),
|
||||
}
|
||||
manager = MultiFontManager(font_provider=_FakeFontProvider(fonts))
|
||||
assert manager.select_font_for_word('こ', 'jpn').font_path.name == 'NotoSansCJK.ttc'
|
||||
|
||||
|
||||
def test_missing_cjk_font_warning_names_language_font(font_dir, caplog):
|
||||
"""The missing-font warning names the language-specific CJK family (#1652)."""
|
||||
manager = MultiFontManager(font_provider=BuiltinFontProvider(font_dir))
|
||||
with caplog.at_level(logging.WARNING):
|
||||
manager.select_font_for_word("こんにちは", "jpn")
|
||||
assert 'NotoSansJP' in caplog.text
|
||||
|
||||
|
||||
def test_missing_font_warning_explains_consequences(font_dir, caplog):
|
||||
"""The glyphless-fallback warning should be actionable, not cryptic (#1652).
|
||||
|
||||
With only builtin fonts available, Arabic text cannot be covered, so the
|
||||
manager falls back to glyphless Occulta and must warn helpfully.
|
||||
"""
|
||||
# Builtin-only provider: NotoSansArabic is never available, forcing fallback.
|
||||
manager = MultiFontManager(font_provider=BuiltinFontProvider(font_dir))
|
||||
|
||||
with caplog.at_level(logging.WARNING):
|
||||
manager.select_font_for_word("سلام", "fas")
|
||||
|
||||
msg = caplog.text
|
||||
# Identifies the affected language and the font family to install.
|
||||
assert 'fas' in msg
|
||||
assert 'NotoSansArabic' in msg
|
||||
# Explains the user-visible consequence so the message is not cryptic:
|
||||
# the text stays searchable but renders blank when highlighted.
|
||||
assert 'searchable' in msg.lower()
|
||||
assert 'highlight' in msg.lower() or 'select' in msg.lower()
|
||||
|
||||
@@ -190,6 +190,9 @@ class TestSystemFontProviderAvailableFonts:
|
||||
assert 'NotoSansCJK-Regular' in fonts
|
||||
assert 'NotoSansArabic-Regular' in fonts
|
||||
assert 'NotoSansThai-Regular' in fonts
|
||||
# Per-language CJK families (modern Google Fonts / Homebrew naming)
|
||||
assert 'NotoSansSC-Regular' in fonts
|
||||
assert 'NotoSansJP-Regular' in fonts
|
||||
|
||||
def test_fallback_font_raises(self):
|
||||
"""Test that get_fallback_font raises NotImplementedError."""
|
||||
@@ -198,6 +201,166 @@ class TestSystemFontProviderAvailableFonts:
|
||||
provider.get_fallback_font()
|
||||
|
||||
|
||||
class TestSystemFontProviderVariableFonts:
|
||||
"""Test discovery of variable fonts and non-static filename variants.
|
||||
|
||||
Homebrew casks and Google Fonts ship Noto fonts as variable fonts with
|
||||
bracketed axis filenames (e.g. ``NotoSansArabic[wdth,wght].ttf``) rather
|
||||
than the static ``NotoSansArabic-Regular.ttf``. See issue #1652.
|
||||
"""
|
||||
|
||||
@pytest.fixture
|
||||
def real_font_bytes(self):
|
||||
"""Bytes of a real, loadable font (content is irrelevant to the test)."""
|
||||
font_path = (
|
||||
Path(__file__).parent.parent
|
||||
/ "src"
|
||||
/ "ocrmypdf"
|
||||
/ "data"
|
||||
/ "NotoSans-Regular.ttf"
|
||||
)
|
||||
if not font_path.exists():
|
||||
pytest.skip("Builtin font not available")
|
||||
return font_path.read_bytes()
|
||||
|
||||
def _provider_for(self, tmp_path, filenames, real_font_bytes):
|
||||
"""Build a provider whose only font dir is tmp_path with given files."""
|
||||
for name in filenames:
|
||||
(tmp_path / name).write_bytes(real_font_bytes)
|
||||
provider = SystemFontProvider()
|
||||
provider._font_dirs = [tmp_path]
|
||||
return provider
|
||||
|
||||
def test_finds_variable_font_with_axes(self, tmp_path, real_font_bytes):
|
||||
"""A bracketed variable font satisfies a request for the static name."""
|
||||
provider = self._provider_for(
|
||||
tmp_path, ['NotoSansArabic[wdth,wght].ttf'], real_font_bytes
|
||||
)
|
||||
font = provider.get_font('NotoSansArabic-Regular')
|
||||
assert font is not None
|
||||
assert font.font_path.name == 'NotoSansArabic[wdth,wght].ttf'
|
||||
|
||||
def test_finds_weight_only_variable_font(self, tmp_path, real_font_bytes):
|
||||
"""A variable font with only a weight axis is also discovered."""
|
||||
provider = self._provider_for(
|
||||
tmp_path, ['NotoSansHebrew[wght].ttf'], real_font_bytes
|
||||
)
|
||||
assert provider.get_font('NotoSansHebrew-Regular') is not None
|
||||
|
||||
def test_variable_font_does_not_cross_match_other_script(
|
||||
self, tmp_path, real_font_bytes
|
||||
):
|
||||
"""The generic NotoSans request must not match a script-specific font."""
|
||||
provider = self._provider_for(
|
||||
tmp_path, ['NotoSansArabic[wdth,wght].ttf'], real_font_bytes
|
||||
)
|
||||
# NotoSans (Latin) must NOT be satisfied by NotoSansArabic.
|
||||
assert provider.get_font('NotoSans-Regular') is None
|
||||
|
||||
def test_does_not_match_ui_or_bold_variants(self, tmp_path, real_font_bytes):
|
||||
"""Width/UI and weight variants must not satisfy the Regular request."""
|
||||
provider = self._provider_for(
|
||||
tmp_path,
|
||||
['NotoSansArabicUI-Regular.ttf', 'NotoSansArabic-Bold.ttf'],
|
||||
real_font_bytes,
|
||||
)
|
||||
assert provider.get_font('NotoSansArabic-Regular') is None
|
||||
|
||||
def test_prefers_static_regular_over_variable(self, tmp_path, real_font_bytes):
|
||||
"""When both exist, the static Regular is preferred for predictability."""
|
||||
provider = self._provider_for(
|
||||
tmp_path,
|
||||
['NotoSansArabic[wdth,wght].ttf', 'NotoSansArabic-Regular.ttf'],
|
||||
real_font_bytes,
|
||||
)
|
||||
font = provider.get_font('NotoSansArabic-Regular')
|
||||
assert font is not None
|
||||
assert font.font_path.name == 'NotoSansArabic-Regular.ttf'
|
||||
|
||||
# --- Modern per-language CJK families (NotoSansSC/TC/HK/JP/KR) ---
|
||||
# Homebrew casks (font-noto-sans-sc, ...) and Google Fonts ship CJK as
|
||||
# variable fonts under these bases rather than the legacy NotoSansCJK*.
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'filename',
|
||||
[
|
||||
'NotoSansSC[wght].ttf', # Simplified Chinese (Homebrew/Google)
|
||||
'NotoSansTC[wght].ttf', # Traditional Chinese
|
||||
'NotoSansHK[wght].ttf', # Hong Kong
|
||||
'NotoSansJP[wght].ttf', # Japanese
|
||||
'NotoSansKR[wght].ttf', # Korean
|
||||
],
|
||||
)
|
||||
def test_finds_modern_cjk_variable_font(self, tmp_path, real_font_bytes, filename):
|
||||
"""A modern per-language CJK variable font satisfies NotoSansCJK."""
|
||||
provider = self._provider_for(tmp_path, [filename], real_font_bytes)
|
||||
font = provider.get_font('NotoSansCJK-Regular')
|
||||
assert font is not None
|
||||
assert font.font_path.name == filename
|
||||
|
||||
def test_finds_static_cjk_language_variant(self, tmp_path, real_font_bytes):
|
||||
"""A static per-language CJK Regular also satisfies NotoSansCJK."""
|
||||
provider = self._provider_for(
|
||||
tmp_path, ['NotoSansTC-Regular.otf'], real_font_bytes
|
||||
)
|
||||
assert provider.get_font('NotoSansCJK-Regular') is not None
|
||||
|
||||
def test_prefers_pan_cjk_over_language_variant(self, tmp_path, real_font_bytes):
|
||||
"""The pan-CJK family is preferred over a single-language variant."""
|
||||
provider = self._provider_for(
|
||||
tmp_path,
|
||||
['NotoSansSC[wght].ttf', 'NotoSansCJK[wght].ttf'],
|
||||
real_font_bytes,
|
||||
)
|
||||
font = provider.get_font('NotoSansCJK-Regular')
|
||||
assert font is not None
|
||||
assert font.font_path.name == 'NotoSansCJK[wght].ttf'
|
||||
|
||||
def test_modern_cjk_does_not_cross_match_latin(self, tmp_path, real_font_bytes):
|
||||
"""A CJK variable font must not satisfy the generic NotoSans request."""
|
||||
provider = self._provider_for(
|
||||
tmp_path, ['NotoSansSC[wght].ttf'], real_font_bytes
|
||||
)
|
||||
assert provider.get_font('NotoSans-Regular') is None
|
||||
|
||||
# --- Per-language CJK families reachable by their own logical name ---
|
||||
# Needed so MultiFontManager can prefer the family matching the document
|
||||
# language (NotoSansJP for Japanese, NotoSansSC for Simplified Chinese, ...).
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'logical,filename',
|
||||
[
|
||||
('NotoSansSC-Regular', 'NotoSansSC[wght].ttf'),
|
||||
('NotoSansTC-Regular', 'NotoSansTC[wght].ttf'),
|
||||
('NotoSansHK-Regular', 'NotoSansHK[wght].ttf'),
|
||||
('NotoSansJP-Regular', 'NotoSansJP[wght].ttf'),
|
||||
('NotoSansKR-Regular', 'NotoSansKR[wght].ttf'),
|
||||
],
|
||||
)
|
||||
def test_per_language_cjk_logical_name_resolves(
|
||||
self, tmp_path, real_font_bytes, logical, filename
|
||||
):
|
||||
"""Each per-language CJK family is reachable by its own logical name."""
|
||||
provider = self._provider_for(tmp_path, [filename], real_font_bytes)
|
||||
font = provider.get_font(logical)
|
||||
assert font is not None
|
||||
assert font.font_path.name == filename
|
||||
|
||||
def test_per_language_cjk_static_resolves(self, tmp_path, real_font_bytes):
|
||||
"""A static per-language Regular also resolves by logical name."""
|
||||
provider = self._provider_for(
|
||||
tmp_path, ['NotoSansJP-Regular.otf'], real_font_bytes
|
||||
)
|
||||
assert provider.get_font('NotoSansJP-Regular') is not None
|
||||
|
||||
def test_per_language_cjk_does_not_cross_match(self, tmp_path, real_font_bytes):
|
||||
"""A JP font must not satisfy an SC request (distinct families)."""
|
||||
provider = self._provider_for(
|
||||
tmp_path, ['NotoSansJP[wght].ttf'], real_font_bytes
|
||||
)
|
||||
assert provider.get_font('NotoSansSC-Regular') is None
|
||||
|
||||
|
||||
# --- ChainedFontProvider Tests ---
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user