menekis/OCRmyPDF
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add font infrastructure and glyphless font

Browse Source 
- Add font module with FontManager, FontProvider, MultiFontManager,
  and SystemFontProvider for multilingual font support
- Add NotoSans-Regular.ttf for Latin text rendering
- Replace pdf.ttf with Occulta.ttf glyphless font
- Add script to generate new Occulta glyphless font
- System font discovery for CJK, Arabic, Devanagari scripts
This commit is contained in:
James R. Barlow
2026-01-06 13:44:54 -08:00
parent 83a43408c2
commit 64726f97b3
11 changed files with 1188 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitattributes
+1
View File
@@ -13,5 +13,6 @@
*.jpg binary
*.bin binary
*.afdesign binary
*.ttf binary
.git_archival.txt export-subst
REUSE.toml
+2 -8
View File
@@ -167,15 +167,9 @@ SPDX-FileCopyrightText = [
SPDX-License-Identifier = "Zlib"
[[annotations]]
path = "src/ocrmypdf/data/pdf.ttf"
path = "src/ocrmypdf/data/Occulta.ttf"
precedence = "aggregate"
SPDX-FileCopyrightText = [
"(C) 2014 Ray Smith",
"(C) 2015 Ken Sharp",
"(C) 2016 James R. Barlow",
"(C) 2016 Jeff Breidenbach",
"(C) 2017 Zdenko Podobný",
]
SPDX-FileCopyrightText = ["(C) 2026 James R. Barlow"]
SPDX-License-Identifier = "Apache-2.0"
[[annotations]]
scripts/generate_glyphless_font.py
+231
View File
@@ -0,0 +1,231 @@
#!/usr/bin/env python3
# SPDX-FileCopyrightText: 2024 James R. Barlow
# SPDX-License-Identifier: MPL-2.0
"""Generate the Occulta glyphless font for OCRmyPDF.
Occulta (Latin for "hidden") is a glyphless font designed for invisible text layers
in searchable PDFs. It has proper Unicode cmap coverage using format 13 (many-to-one)
for efficient mapping of all BMP codepoints to a small set of width-specific glyphs.
Features:
- Full BMP coverage (U+0000 to U+FFFF)
- Width-aware glyphs for proper text selection:
- Zero-width for combining marks and invisible characters
- Regular width (500 units) for Latin, Greek, Cyrillic, Arabic, Hebrew, etc.
- Double width (1000 units) for CJK and fullwidth characters
- Uses cmap format 13 (many-to-one) for ~12KB size vs ~780KB with format 12
- Compatible with fpdf2 and other modern PDF libraries
Usage:
python scripts/generate_glyphless_font.py
Output:
src/ocrmypdf/data/Occulta.ttf
"""
from __future__ import annotations
import unicodedata
from pathlib import Path
from fontTools.fontBuilder import FontBuilder
from fontTools.ttLib import TTFont
from fontTools.ttLib.tables._c_m_a_p import CmapSubtable
from fontTools.ttLib.tables._g_l_y_f import Glyph
# Output path relative to this script
OUTPUT_PATH = Path(__file__).parent.parent / "src" / "ocrmypdf" / "data" / "Occulta.ttf"
# Font metrics (units per em = 1000)
UNITS_PER_EM = 1000
ASCENT = 800
DESCENT = -200
# Glyph definitions: (name, advance_width, left_side_bearing)
GLYPHS = [
(".notdef", 500, 0), # Required, used for unmapped characters
("space", 500, 0), # U+0020 SPACE
("nbspace", 500, 0), # U+00A0 NO-BREAK SPACE
("blank0", 0, 0), # Zero-width (combining marks, ZWNJ, ZWJ, BOM)
("blank1", 500, 0), # Regular width (most scripts)
("blank2", 1000, 0), # Double width (CJK, fullwidth)
]
# Explicit zero-width character codepoints
ZERO_WIDTH_CHARS = frozenset(
[
0x200B, # ZERO WIDTH SPACE
0x200C, # ZERO WIDTH NON-JOINER
0x200D, # ZERO WIDTH JOINER
0xFEFF, # ZERO WIDTH NO-BREAK SPACE (BOM)
0x200E, # LEFT-TO-RIGHT MARK
0x200F, # RIGHT-TO-LEFT MARK
0x202A, # LEFT-TO-RIGHT EMBEDDING
0x202B, # RIGHT-TO-LEFT EMBEDDING
0x202C, # POP DIRECTIONAL FORMATTING
0x202D, # LEFT-TO-RIGHT OVERRIDE
0x202E, # RIGHT-TO-LEFT OVERRIDE
0x2060, # WORD JOINER
0x2061, # FUNCTION APPLICATION
0x2062, # INVISIBLE TIMES
0x2063, # INVISIBLE SEPARATOR
0x2064, # INVISIBLE PLUS
]
)
def classify_codepoint(codepoint: int) -> str:
"""Classify a Unicode codepoint into one of our glyph categories.
Args:
codepoint: Unicode codepoint (0x0000 to 0xFFFF)
Returns:
Glyph name to map this codepoint to
"""
# Special cases first
if codepoint == 0x0020:
return "space"
if codepoint == 0x00A0:
return "nbspace"
if codepoint in ZERO_WIDTH_CHARS:
return "blank0"
# Use Unicode properties for the rest
char = chr(codepoint)
try:
category = unicodedata.category(char)
east_asian_width = unicodedata.east_asian_width(char)
# Combining marks are zero-width
if category.startswith("M"):
return "blank0"
# Wide and Fullwidth characters are double-width
if east_asian_width in ("W", "F"):
return "blank2"
# Everything else is regular width
return "blank1"
except (ValueError, TypeError):
# Fallback for any edge cases
return "blank1"
def build_cmap() -> dict[int, str]:
"""Build the Unicode to glyph name mapping for the entire BMP.
Returns:
Dictionary mapping codepoints to glyph names
"""
return {cp: classify_codepoint(cp) for cp in range(0x10000)}
def create_font() -> TTFont:
"""Create the Occulta glyphless font.
Returns:
TTFont object ready to be saved
"""
glyph_names = [g[0] for g in GLYPHS]
# Start building the font
fb = FontBuilder(UNITS_PER_EM, isTTF=True)
fb.setupGlyphOrder(glyph_names)
# Create empty (invisible) glyphs
glyphs = {}
for name, _, _ in GLYPHS:
glyph = Glyph()
glyph.numberOfContours = 0
glyphs[name] = glyph
fb.setupGlyf(glyphs)
# Set up horizontal metrics
metrics = {name: (width, lsb) for name, width, lsb in GLYPHS}
fb.setupHorizontalMetrics(metrics)
# Minimal cmap to satisfy FontBuilder (we'll replace it later)
fb.setupCharacterMap({0x0020: "space", 0x00A0: "nbspace"})
# Set up other required tables
fb.setupHorizontalHeader(ascent=ASCENT, descent=DESCENT)
fb.setupOS2(
sTypoAscender=ASCENT,
sTypoDescender=DESCENT,
sTypoLineGap=0,
usWinAscent=UNITS_PER_EM,
usWinDescent=abs(DESCENT),
sxHeight=500,
sCapHeight=700,
)
import time
# Use current time for font timestamps
now = int(time.time())
fb.setupHead(unitsPerEm=UNITS_PER_EM, created=now, modified=now)
fb.setupPost()
fb.setupNameTable(
{
"familyName": "Occulta",
"styleName": "Regular",
"uniqueFontIdentifier": "OCRmyPDF;Occulta-Regular;2026",
"fullName": "Occulta Regular",
"version": "Version 2.0",
"psName": "Occulta-Regular",
}
)
# Build the font
font = fb.font
# Now replace the cmap with format 13 for efficient many-to-one mapping
char_to_glyph = build_cmap()
cmap13 = CmapSubtable.newSubtable(13)
cmap13.platformID = 3 # Windows
cmap13.platEncID = 10 # Unicode full repertoire
cmap13.language = 0
cmap13.cmap = char_to_glyph
font["cmap"].tables = [cmap13]
return font
def main() -> None:
"""Generate the Occulta font and save it."""
print("Generating Occulta glyphless font...")
font = create_font()
# Create output directory if needed
OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True)
# Save the font
font.save(str(OUTPUT_PATH))
font.close()
# Report statistics
size = OUTPUT_PATH.stat().st_size
print(f"Saved to: {OUTPUT_PATH}")
print(f"Size: {size:,} bytes")
# Verify cmap
font = TTFont(str(OUTPUT_PATH))
for table in font["cmap"].tables:
print(
f"cmap: Platform {table.platformID}, "
f"Encoding {table.platEncID}, "
f"Format {table.format}, "
f"{len(table.cmap)} mappings"
)
font.close()
print("Done!")
if __name__ == "__main__":
main()
src/ocrmypdf/data/NotoSans-Regular.ttf
BIN
View File
Binary file not shown.
src/ocrmypdf/data/Occulta.ttf
BIN
View File
Binary file not shown.
src/ocrmypdf/data/pdf.ttf
BIN
View File
Binary file not shown.
src/ocrmypdf/font/__init__.py
+30
View File
@@ -0,0 +1,30 @@
# SPDX-FileCopyrightText: 2025 James R. Barlow
# SPDX-License-Identifier: MPL-2.0
"""Font management for OCRmyPDF PDF rendering.
This module provides font infrastructure for the fpdf2 PDF renderer. It includes:
- FontManager: Base class for font loading and glyph checking
- FontProvider: Protocol and implementations for font discovery
- MultiFontManager: Automatic font selection for multilingual documents
- SystemFontProvider: System font discovery
"""
from ocrmypdf.font.font_manager import FontManager
from ocrmypdf.font.font_provider import (
BuiltinFontProvider,
ChainedFontProvider,
FontProvider,
)
from ocrmypdf.font.multi_font_manager import MultiFontManager
from ocrmypdf.font.system_font_provider import SystemFontProvider
__all__ = [
"FontManager",
"FontProvider",
"BuiltinFontProvider",
"ChainedFontProvider",
"MultiFontManager",
"SystemFontProvider",
]
src/ocrmypdf/font/font_manager.py
+115
View File
@@ -0,0 +1,115 @@
# SPDX-FileCopyrightText: 2025 James R. Barlow
# SPDX-License-Identifier: MPL-2.0
"""Base font management for PDF rendering.
This module provides the base FontManager class that handles font loading
and glyph checking using uharfbuzz.
"""
from __future__ import annotations
from pathlib import Path
import uharfbuzz as hb
class FontManager:
"""Manages font loading and glyph checking for PDF rendering.
This base class handles loading fonts with uharfbuzz for glyph checking
and text shaping. Renderer-specific subclasses should extend this to
add their own font objects.
Attributes:
font_path: Path to the font file
font_data: Raw font file bytes
font_index: Index within TTC collection (0 for single-font files)
hb_face: uharfbuzz Face object
hb_font: uharfbuzz Font object
"""
def __init__(self, font_path: Path, font_index: int = 0):
"""Initialize font manager.
Args:
font_path: Path to TrueType/OpenType font file
font_index: Index of font within a TTC collection (default 0).
For single-font files (.ttf, .otf), use 0.
"""
self.font_path = font_path
self.font_index = font_index
# Load font data
self.font_data = font_path.read_bytes()
# Load font with uharfbuzz for glyph checking and text measurement
# Note: uharfbuzz Face also supports font_index for TTC files
self.hb_face = hb.Face(self.font_data, font_index)
self.hb_font = hb.Font(self.hb_face)
def get_hb_font(self) -> hb.Font:
"""Get uharfbuzz Font object for text measurement.
Returns:
UHarfBuzz Font instance
"""
return self.hb_font
def has_glyph(self, codepoint: int) -> bool:
"""Check if font has a glyph for given codepoint.
Args:
codepoint: Unicode codepoint
Returns:
True if font has a real glyph (not .notdef)
"""
glyph_id = self.hb_font.get_nominal_glyph(codepoint)
return glyph_id is not None and glyph_id != 0
def get_font_metrics(self) -> tuple[float, float, float]:
"""Get normalized font metrics (ascent, descent, units_per_em).
Returns:
Tuple of (ascent, descent, units_per_em) where ascent and descent
are in font units. Ascent is positive (above baseline), descent
is typically negative (below baseline).
"""
extents = self.hb_font.get_font_extents('ltr')
units_per_em = self.hb_face.upem
return (extents.ascender, extents.descender, units_per_em)
def get_left_side_bearing(self, char: str, font_size: float) -> float:
"""Get the left side bearing of a character at a given font size.
The left side bearing (lsb) is the horizontal distance from the glyph
origin (x=0) to the leftmost pixel of the glyph. A positive lsb means
there's whitespace before the glyph starts.
Args:
char: Single character to get lsb for
font_size: Font size in points
Returns:
Left side bearing in points. Returns 0 if character not found.
"""
if not char:
return 0.0
codepoint = ord(char)
glyph_id = self.hb_font.get_nominal_glyph(codepoint)
if glyph_id is None or glyph_id == 0:
return 0.0
# Get glyph extents which include left/right bearing info
extents = self.hb_font.get_glyph_extents(glyph_id)
if extents is None:
return 0.0
# x_bearing is the left side bearing in font units
units_per_em = self.hb_face.upem
lsb_units = extents.x_bearing
lsb_pt = lsb_units * font_size / units_per_em
return lsb_pt
src/ocrmypdf/font/font_provider.py
+189
View File
@@ -0,0 +1,189 @@
# SPDX-FileCopyrightText: 2025 James R. Barlow
# SPDX-License-Identifier: MPL-2.0
"""Font provider protocol and implementations for PDF rendering."""
from __future__ import annotations
import logging
from pathlib import Path
from typing import Protocol
from ocrmypdf.font.font_manager import FontManager
log = logging.getLogger(__name__)
class FontProvider(Protocol):
"""Protocol for providing fonts to MultiFontManager.
Implementations are responsible for knowing where fonts are located
and loading them. MultiFontManager asks for fonts by name and uses
them for glyph coverage checking.
"""
def get_font(self, font_name: str) -> FontManager | None:
"""Get a FontManager for the named font.
Args:
font_name: Logical font name (e.g., 'NotoSans-Regular')
Returns:
FontManager if font is available, None otherwise
"""
...
def get_available_fonts(self) -> list[str]:
"""Get list of available font names.
Returns:
List of font names that can be retrieved with get_font()
"""
...
def get_fallback_font(self) -> FontManager:
"""Get the glyphless fallback font.
This font must always be available and handles any codepoint.
Returns:
FontManager for the glyphless fallback font (Occulta.ttf)
"""
...
class BuiltinFontProvider:
"""Font provider using builtin fonts from ocrmypdf/data directory."""
# Mapping of logical font names to filenames
# Only Latin (NotoSans) and the glyphless fallback (Occulta.ttf) are bundled.
# All other scripts (Arabic, Devanagari, CJK, etc.) are discovered from
# system fonts by SystemFontProvider to reduce package size.
FONT_FILES = {
'NotoSans-Regular': 'NotoSans-Regular.ttf',
'Occulta': 'Occulta.ttf',
}
def __init__(self, font_dir: Path | None = None):
"""Initialize builtin font provider.
Args:
font_dir: Directory containing font files. If None, uses
the default ocrmypdf/data directory.
"""
if font_dir is None:
font_dir = Path(__file__).parent.parent / "data"
self.font_dir = font_dir
self._fonts: dict[str, FontManager] = {}
self._load_fonts()
def _load_fonts(self) -> None:
"""Load available fonts, logging warnings for missing ones."""
for font_name, font_file in self.FONT_FILES.items():
font_path = self.font_dir / font_file
if not font_path.exists():
if font_name == 'Occulta':
raise FileNotFoundError(
f"Required fallback font not found: {font_path}"
)
log.warning(
"Font %s not found at %s - OCR output quality for some "
"scripts may be affected",
font_name,
font_path,
)
continue
try:
self._fonts[font_name] = FontManager(font_path)
except Exception as e:
if font_name == 'Occulta':
raise ValueError(
f"Failed to load required fallback font {font_file}: {e}"
) from e
log.warning(
"Failed to load font %s: %s - OCR output quality may be affected",
font_name,
e,
)
def get_font(self, font_name: str) -> FontManager | None:
"""Get a FontManager for the named font."""
return self._fonts.get(font_name)
def get_available_fonts(self) -> list[str]:
"""Get list of available font names."""
return list(self._fonts.keys())
def get_fallback_font(self) -> FontManager:
"""Get the glyphless fallback font."""
return self._fonts['Occulta']
class ChainedFontProvider:
"""Font provider that tries multiple providers in order.
This allows combining builtin fonts with system fonts, trying
the builtin provider first and falling back to system fonts
for fonts not bundled with the package.
"""
def __init__(self, providers: list[FontProvider]):
"""Initialize chained font provider.
Args:
providers: List of font providers to try in order.
The first provider that returns a font wins.
"""
if not providers:
raise ValueError("At least one provider is required")
self.providers = providers
def get_font(self, font_name: str) -> FontManager | None:
"""Get a FontManager for the named font.
Tries each provider in order until one returns a font.
Args:
font_name: Logical font name (e.g., 'NotoSans-Regular')
Returns:
FontManager if any provider has the font, None otherwise
"""
for provider in self.providers:
if font := provider.get_font(font_name):
return font
return None
def get_available_fonts(self) -> list[str]:
"""Get list of available font names from all providers.
Returns:
Combined list of font names (deduplicated, order preserved)
"""
seen: set[str] = set()
result: list[str] = []
for provider in self.providers:
for name in provider.get_available_fonts():
if name not in seen:
seen.add(name)
result.append(name)
return result
def get_fallback_font(self) -> FontManager:
"""Get the glyphless fallback font.
Tries each provider until one provides a fallback font.
Returns:
FontManager for the fallback font
Raises:
RuntimeError: If no provider can provide a fallback font
"""
for provider in self.providers:
try:
return provider.get_fallback_font()
except (NotImplementedError, AttributeError, KeyError):
continue
raise RuntimeError("No fallback font available from any provider")
src/ocrmypdf/font/multi_font_manager.py
+323
View File
@@ -0,0 +1,323 @@
# SPDX-FileCopyrightText: 2025 James R. Barlow
# SPDX-License-Identifier: MPL-2.0
"""Multi-font management for PDF rendering.
Provides automatic font selection for multilingual documents based on
language hints and glyph coverage analysis.
"""
from __future__ import annotations
import logging
from pathlib import Path
from ocrmypdf.font.font_manager import FontManager
from ocrmypdf.font.font_provider import (
BuiltinFontProvider,
ChainedFontProvider,
FontProvider,
)
from ocrmypdf.font.system_font_provider import SystemFontProvider
log = logging.getLogger(__name__)
class MultiFontManager:
"""Manages multiple fonts with automatic selection and fallback.
This class orchestrates multiple FontManager instances to provide
word-level font selection for multilingual documents. It uses a hybrid
approach combining language hints from hOCR with glyph coverage analysis.
Font selection strategy:
1. Try language-preferred font (if language hint available)
2. Try fallback fonts in order by glyph coverage
3. Fall back to Occulta.ttf (glyphless fallback)
"""
# Language to font mapping
# Keys are ISO 639-2/3 codes or Tesseract language codes
LANGUAGE_FONT_MAP = {
# Arabic script
'ara': 'NotoSansArabic-Regular', # Arabic
'per': 'NotoSansArabic-Regular', # Persian (uses Arabic script)
'fas': 'NotoSansArabic-Regular', # Farsi (alternative code for Persian)
'urd': 'NotoSansArabic-Regular', # Urdu (uses Arabic script)
'pus': 'NotoSansArabic-Regular', # Pashto
'kur': 'NotoSansArabic-Regular', # Kurdish (Arabic script variant)
# Devanagari script
'hin': 'NotoSansDevanagari-Regular', # Hindi
'san': 'NotoSansDevanagari-Regular', # Sanskrit
'mar': 'NotoSansDevanagari-Regular', # Marathi
'nep': 'NotoSansDevanagari-Regular', # Nepali
'kok': 'NotoSansDevanagari-Regular', # Konkani
'bho': 'NotoSansDevanagari-Regular', # Bhojpuri
'mai': 'NotoSansDevanagari-Regular', # Maithili
# CJK
'chi': 'NotoSansCJK-Regular', # Chinese (generic)
'zho': 'NotoSansCJK-Regular', # Chinese (ISO 639-3)
'chi_sim': 'NotoSansCJK-Regular', # Chinese Simplified (Tesseract)
'chi_tra': 'NotoSansCJK-Regular', # Chinese Traditional (Tesseract)
'jpn': 'NotoSansCJK-Regular', # Japanese
'kor': 'NotoSansCJK-Regular', # Korean
# Thai
'tha': 'NotoSansThai-Regular', # Thai
# Hebrew
'heb': 'NotoSansHebrew-Regular', # Hebrew
'yid': 'NotoSansHebrew-Regular', # Yiddish (uses Hebrew script)
# Bengali script
'ben': 'NotoSansBengali-Regular', # Bengali
'asm': 'NotoSansBengali-Regular', # Assamese (uses Bengali script)
# Tamil
'tam': 'NotoSansTamil-Regular', # Tamil
# Gujarati
'guj': 'NotoSansGujarati-Regular', # Gujarati
# Telugu
'tel': 'NotoSansTelugu-Regular', # Telugu
# Kannada
'kan': 'NotoSansKannada-Regular', # Kannada
# Malayalam
'mal': 'NotoSansMalayalam-Regular', # Malayalam
# Myanmar (Burmese)
'mya': 'NotoSansMyanmar-Regular', # Myanmar
# Khmer (Cambodian)
'khm': 'NotoSansKhmer-Regular', # Khmer
# Lao
'lao': 'NotoSansLao-Regular', # Lao
# Georgian
'kat': 'NotoSansGeorgian-Regular', # Georgian
'geo': 'NotoSansGeorgian-Regular', # Georgian (alternative)
# Armenian
'hye': 'NotoSansArmenian-Regular', # Armenian
'arm': 'NotoSansArmenian-Regular', # Armenian (alternative)
# Ethiopic
'amh': 'NotoSansEthiopic-Regular', # Amharic
'tir': 'NotoSansEthiopic-Regular', # Tigrinya
# Sinhala
'sin': 'NotoSansSinhala-Regular', # Sinhala
# Gurmukhi (Punjabi)
'pan': 'NotoSansGurmukhi-Regular', # Punjabi
'pnb': 'NotoSansGurmukhi-Regular', # Western Punjabi
# Oriya
'ori': 'NotoSansOriya-Regular', # Oriya
'ory': 'NotoSansOriya-Regular', # Oriya (alternative)
# Tibetan
'bod': 'NotoSansTibetan-Regular', # Tibetan
'tib': 'NotoSansTibetan-Regular', # Tibetan (alternative)
}
# Ordered fallback chain for fonts (after language-preferred font)
# Order matters: most common scripts first for faster matching
FALLBACK_FONTS = [
'NotoSans-Regular', # Latin, Greek, Cyrillic
'NotoSansArabic-Regular',
'NotoSansDevanagari-Regular',
'NotoSansCJK-Regular',
'NotoSansThai-Regular',
'NotoSansHebrew-Regular',
'NotoSansBengali-Regular',
'NotoSansTamil-Regular',
'NotoSansGujarati-Regular',
'NotoSansTelugu-Regular',
'NotoSansKannada-Regular',
'NotoSansMalayalam-Regular',
'NotoSansMyanmar-Regular',
'NotoSansKhmer-Regular',
'NotoSansLao-Regular',
'NotoSansGeorgian-Regular',
'NotoSansArmenian-Regular',
'NotoSansEthiopic-Regular',
'NotoSansSinhala-Regular',
'NotoSansGurmukhi-Regular',
'NotoSansOriya-Regular',
'NotoSansTibetan-Regular',
]
def __init__(
self,
font_dir: Path | None = None,
*,
font_provider: FontProvider | None = None,
):
"""Initialize multi-font manager.
Args:
font_dir: Directory containing font files. If font_provider is
not specified, this is passed to BuiltinFontProvider.
font_provider: Provider for loading fonts. If None, uses a
ChainedFontProvider that tries builtin fonts first,
then searches system fonts.
"""
if font_provider is not None:
self.font_provider = font_provider
else:
# Use chained provider: try builtin fonts first, then system fonts
self.font_provider = ChainedFontProvider([
BuiltinFontProvider(font_dir),
SystemFontProvider(),
])
# Font selection cache: (word_text, language) -> font_name
self._selection_cache: dict[tuple[str, str | None], str] = {}
# Track whether we've warned about missing fonts (warn once per script)
self._warned_scripts: set[str] = set()
@property
def fonts(self) -> dict[str, FontManager]:
"""Get all loaded fonts (backward compatibility)."""
return self.get_all_fonts()
def _try_font(
self, font_name: str, word_text: str, cache_key: tuple[str, str | None]
) -> FontManager | None:
"""Try to use a font for the given word.
Args:
font_name: Name of font to try
word_text: Text content to check
cache_key: Cache key for storing successful result
Returns:
FontManager if font exists and has all glyphs, None otherwise
"""
font = self.font_provider.get_font(font_name)
if font is None:
return None
if self._has_all_glyphs(font, word_text):
self._selection_cache[cache_key] = font_name
return font
return None
def select_font_for_word(
self, word_text: str, line_language: str | None
) -> FontManager:
"""Select appropriate font for a word.
Uses a hybrid approach:
1. Language-based selection (if language hint available)
2. Ordered fallback through available fonts by glyph coverage
3. Final fallback to Occulta.ttf (glyphless)
Args:
word_text: The text content of the word
line_language: Language code from hOCR (e.g., 'ara', 'eng')
Returns:
FontManager instance to use for rendering this word
"""
cache_key = (word_text, line_language)
if cache_key in self._selection_cache:
cached_name = self._selection_cache[cache_key]
font = self.font_provider.get_font(cached_name)
if font:
return font
tried_fonts: set[str] = set()
# Phase 1: Try language-preferred font
if line_language and line_language in self.LANGUAGE_FONT_MAP:
preferred = self.LANGUAGE_FONT_MAP[line_language]
tried_fonts.add(preferred)
if result := self._try_font(preferred, word_text, cache_key):
return result
# Phase 2: Try fallback fonts in order
for font_name in self.FALLBACK_FONTS:
if font_name in tried_fonts:
continue
if result := self._try_font(font_name, word_text, cache_key):
return result
# Phase 3: Glyphless fallback (always succeeds)
# Warn if we're falling back for non-ASCII text (likely missing font)
self._warn_missing_font(word_text, line_language)
self._selection_cache[cache_key] = 'Occulta'
return self.font_provider.get_fallback_font()
def _warn_missing_font(
self, word_text: str, line_language: str | None
) -> None:
"""Warn user about missing font for non-Latin text.
Only warns once per language/script to avoid log spam.
"""
# Determine a key for deduplication (language or 'non-ascii')
warn_key = line_language if line_language else 'unknown'
# Only warn for non-ASCII text and only once per key
if warn_key in self._warned_scripts:
return
# Check if text contains non-ASCII characters
if not any(ord(c) > 127 for c in word_text):
return
self._warned_scripts.add(warn_key)
if line_language and line_language in self.LANGUAGE_FONT_MAP:
font_name = self.LANGUAGE_FONT_MAP[line_language]
log.warning(
"No font found with glyphs for '%s' text. "
"Install %s for better rendering. "
"See https://fonts.google.com/noto",
line_language,
font_name,
)
else:
log.warning(
"No font found with glyphs for some text. "
"Install Noto fonts for better rendering. "
"See https://fonts.google.com/noto"
)
def _has_all_glyphs(self, font: FontManager, text: str) -> bool:
"""Check if a font has glyphs for all characters in text.
Args:
font: FontManager instance to check
text: Text to verify coverage for
Returns:
True if font has real glyphs for all characters (not .notdef)
"""
if not text:
return True
hb_font = font.get_hb_font()
for char in text:
codepoint = ord(char)
glyph_id = hb_font.get_nominal_glyph(codepoint)
if glyph_id is None or glyph_id == 0: # 0 = .notdef glyph
return False
return True
def has_all_glyphs(self, font_name: str, text: str) -> bool:
"""Check if a named font has glyphs for all characters in text.
Args:
font_name: Name of font to check
text: Text to verify coverage for
Returns:
True if font has real glyphs for all characters (not .notdef)
"""
font = self.font_provider.get_font(font_name)
if font is None:
return False
return self._has_all_glyphs(font, text)
def get_all_fonts(self) -> dict[str, FontManager]:
"""Get all loaded font managers.
Returns:
Dictionary mapping font names to FontManager instances
"""
result = {}
for name in self.font_provider.get_available_fonts():
font = self.font_provider.get_font(name)
if font is not None:
result[name] = font
return result
src/ocrmypdf/font/system_font_provider.py
+297
View File
@@ -0,0 +1,297 @@
# SPDX-FileCopyrightText: 2025 James R. Barlow
# SPDX-License-Identifier: MPL-2.0
"""System font discovery for PDF rendering.
Provides lazy discovery of Noto fonts installed on the system across
Linux, macOS, and Windows platforms.
"""
from __future__ import annotations
import logging
import os
import sys
from pathlib import Path
from ocrmypdf.font.font_manager import FontManager
log = logging.getLogger(__name__)
class SystemFontProvider:
"""Discovers and provides system-installed Noto fonts with lazy scanning.
This provider searches standard system font directories for Noto fonts.
Scanning is performed lazily - only when a font is actually requested
and not found in the builtin fonts. Results are cached for the lifetime
of the provider instance.
"""
# System font directories by platform
SYSTEM_FONT_DIRS: dict[str, list[Path]] = {
'linux': [
Path('/usr/share/fonts'),
Path('/usr/local/share/fonts'),
Path.home() / '.fonts',
Path.home() / '.local/share/fonts',
],
'freebsd': [
Path('/usr/local/share/fonts'),
Path.home() / '.fonts',
],
'darwin': [
Path('/Library/Fonts'),
Path('/System/Library/Fonts'),
Path.home() / 'Library/Fonts',
],
# Windows is handled dynamically in _get_font_dirs()
}
# Noto font logical names → possible filenames (priority order)
# The first match found will be used
NOTO_FONT_PATTERNS: dict[str, list[str]] = {
'NotoSans-Regular': [
'NotoSans-Regular.ttf',
'NotoSans-Regular.otf',
],
'NotoSansArabic-Regular': [
'NotoSansArabic-Regular.ttf',
'NotoSansArabic-Regular.otf',
],
'NotoSansDevanagari-Regular': [
'NotoSansDevanagari-Regular.ttf',
'NotoSansDevanagari-Regular.otf',
],
'NotoSansCJK-Regular': [
# Language-specific variants (any will work for CJK)
'NotoSansCJKsc-Regular.otf', # Simplified Chinese
'NotoSansCJKtc-Regular.otf', # Traditional Chinese
'NotoSansCJKjp-Regular.otf', # Japanese
'NotoSansCJKkr-Regular.otf', # Korean
# TTC collections (common on Linux distros)
'NotoSansCJK-Regular.ttc',
'NotoSansCJKsc-Regular.ttc',
# Variable fonts
'NotoSansCJKsc-VF.otf',
],
'NotoSansThai-Regular': [
'NotoSansThai-Regular.ttf',
'NotoSansThai-Regular.otf',
],
'NotoSansHebrew-Regular': [
'NotoSansHebrew-Regular.ttf',
'NotoSansHebrew-Regular.otf',
],
'NotoSansBengali-Regular': [
'NotoSansBengali-Regular.ttf',
'NotoSansBengali-Regular.otf',
],
'NotoSansTamil-Regular': [
'NotoSansTamil-Regular.ttf',
'NotoSansTamil-Regular.otf',
],
'NotoSansGujarati-Regular': [
'NotoSansGujarati-Regular.ttf',
'NotoSansGujarati-Regular.otf',
],
'NotoSansTelugu-Regular': [
'NotoSansTelugu-Regular.ttf',
'NotoSansTelugu-Regular.otf',
],
'NotoSansKannada-Regular': [
'NotoSansKannada-Regular.ttf',
'NotoSansKannada-Regular.otf',
],
'NotoSansMalayalam-Regular': [
'NotoSansMalayalam-Regular.ttf',
'NotoSansMalayalam-Regular.otf',
],
'NotoSansMyanmar-Regular': [
'NotoSansMyanmar-Regular.ttf',
'NotoSansMyanmar-Regular.otf',
],
'NotoSansKhmer-Regular': [
'NotoSansKhmer-Regular.ttf',
'NotoSansKhmer-Regular.otf',
],
'NotoSansLao-Regular': [
'NotoSansLao-Regular.ttf',
'NotoSansLao-Regular.otf',
],
'NotoSansGeorgian-Regular': [
'NotoSansGeorgian-Regular.ttf',
'NotoSansGeorgian-Regular.otf',
],
'NotoSansArmenian-Regular': [
'NotoSansArmenian-Regular.ttf',
'NotoSansArmenian-Regular.otf',
],
'NotoSansEthiopic-Regular': [
'NotoSansEthiopic-Regular.ttf',
'NotoSansEthiopic-Regular.otf',
],
'NotoSansSinhala-Regular': [
'NotoSansSinhala-Regular.ttf',
'NotoSansSinhala-Regular.otf',
],
'NotoSansGurmukhi-Regular': [
'NotoSansGurmukhi-Regular.ttf',
'NotoSansGurmukhi-Regular.otf',
],
'NotoSansOriya-Regular': [
'NotoSansOriya-Regular.ttf',
'NotoSansOriya-Regular.otf',
],
'NotoSansTibetan-Regular': [
'NotoSansTibetan-Regular.ttf',
'NotoSansTibetan-Regular.otf',
],
}
def __init__(self) -> None:
"""Initialize system font provider with empty caches."""
# Cache: font_name -> FontManager (successfully loaded fonts)
self._font_cache: dict[str, FontManager] = {}
# Negative cache: font names we've searched for but not found
self._not_found: set[str] = set()
# Cached font directories (computed lazily)
self._font_dirs: list[Path] | None = None
def _get_platform(self) -> str:
"""Get the current platform identifier.
Returns:
Platform string: 'linux', 'darwin', 'windows', or 'freebsd'
"""
if sys.platform == 'win32':
return 'windows'
elif sys.platform == 'darwin':
return 'darwin'
elif 'freebsd' in sys.platform:
return 'freebsd'
else:
return 'linux'
def _get_font_dirs(self) -> list[Path]:
"""Get font directories for the current platform.
Returns:
List of paths to search for fonts (may include non-existent paths)
"""
if self._font_dirs is not None:
return self._font_dirs
platform = self._get_platform()
if platform == 'windows':
# Get Windows font directories from environment
windir = os.environ.get('WINDIR', r'C:\Windows')
self._font_dirs = [Path(windir) / 'Fonts']
# User-installed fonts (Windows 10+)
localappdata = os.environ.get('LOCALAPPDATA')
if localappdata:
self._font_dirs.append(
Path(localappdata) / 'Microsoft' / 'Windows' / 'Fonts'
)
else:
self._font_dirs = list(self.SYSTEM_FONT_DIRS.get(platform, []))
return self._font_dirs
def _find_font_file(self, font_name: str) -> Path | None:
"""Search system directories for a font file.
Args:
font_name: Logical font name (e.g., 'NotoSansCJK-Regular')
Returns:
Path to font file if found, None otherwise
"""
if font_name not in self.NOTO_FONT_PATTERNS:
return None
patterns = self.NOTO_FONT_PATTERNS[font_name]
for font_dir in self._get_font_dirs():
if not font_dir.exists():
continue
for pattern in patterns:
# Search recursively for the font file
try:
matches = list(font_dir.rglob(pattern))
if matches:
log.debug(
"Found system font %s at %s", font_name, matches[0]
)
return matches[0]
except PermissionError:
# Skip directories we can't read
continue
return None
def get_font(self, font_name: str) -> FontManager | None:
"""Get a FontManager for the named font (lazy loading).
This method implements lazy scanning: fonts are only searched for
when first requested. Results (both positive and negative) are
cached for subsequent calls.
Args:
font_name: Logical font name (e.g., 'NotoSansCJK-Regular')
Returns:
FontManager if font is found and loadable, None otherwise
"""
# Check positive cache first
if font_name in self._font_cache:
return self._font_cache[font_name]
# Check negative cache (already searched, not found)
if font_name in self._not_found:
return None
# Lazy scan for this specific font
font_path = self._find_font_file(font_name)
if font_path is not None:
try:
fm = FontManager(font_path)
self._font_cache[font_name] = fm
return fm
except Exception as e:
log.warning(
"Found font %s at %s but failed to load: %s",
font_name,
font_path,
e,
)
# Cache negative result
self._not_found.add(font_name)
return None
def get_available_fonts(self) -> list[str]:
"""Get list of font names this provider can potentially find.
Note: This returns all font names we know patterns for, not
necessarily fonts that are actually installed. Use get_font()
to check if a specific font is available.
Returns:
List of logical font names
"""
return list(self.NOTO_FONT_PATTERNS.keys())
def get_fallback_font(self) -> FontManager:
"""Get the glyphless fallback font.
Raises:
NotImplementedError: System provider doesn't provide fallback.
Use BuiltinFontProvider for the fallback font.
"""
raise NotImplementedError(
"SystemFontProvider does not provide a fallback font. "
"Use BuiltinFontProvider for Occulta.ttf fallback."
)