Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
aa6a32e7d1 | ||
|
|
ea99758747 | ||
|
|
4942751a1b | ||
|
|
be06e3184a | ||
|
|
39bf09f1eb | ||
|
|
aaffc46f73 | ||
|
|
0277b3b3ba | ||
|
|
0817542883 | ||
|
|
6f4744dd20 | ||
|
|
5d49f75c56 | ||
|
|
5a824ddd8c | ||
|
|
54bf03a454 | ||
|
|
009754d137 | ||
|
|
f0a3a74374 | ||
|
|
178d339c8e | ||
|
|
d3f8d01227 | ||
|
|
b60df59c62 | ||
|
|
640b3062b2 | ||
|
|
ef903db360 | ||
|
|
92a2fe880a |
@@ -26,7 +26,7 @@ jobs:
|
|||||||
version: "0.9.x"
|
version: "0.9.x"
|
||||||
|
|
||||||
- name: "Set up Python"
|
- name: "Set up Python"
|
||||||
uses: actions/setup-python@v6
|
uses: actions/setup-python@v7
|
||||||
with:
|
with:
|
||||||
python-version: "3.11"
|
python-version: "3.11"
|
||||||
|
|
||||||
@@ -60,7 +60,7 @@ jobs:
|
|||||||
version: "0.9.x"
|
version: "0.9.x"
|
||||||
|
|
||||||
- name: "Set up Python"
|
- name: "Set up Python"
|
||||||
uses: actions/setup-python@v6
|
uses: actions/setup-python@v7
|
||||||
with:
|
with:
|
||||||
python-version: ${{ matrix.python }}
|
python-version: ${{ matrix.python }}
|
||||||
|
|
||||||
@@ -151,7 +151,7 @@ jobs:
|
|||||||
version: "0.9.x"
|
version: "0.9.x"
|
||||||
|
|
||||||
- name: "Set up Python"
|
- name: "Set up Python"
|
||||||
uses: actions/setup-python@v6
|
uses: actions/setup-python@v7
|
||||||
with:
|
with:
|
||||||
python-version: ${{ matrix.python }}
|
python-version: ${{ matrix.python }}
|
||||||
|
|
||||||
@@ -200,7 +200,7 @@ jobs:
|
|||||||
version: "0.9.x"
|
version: "0.9.x"
|
||||||
|
|
||||||
- name: "Set up Python"
|
- name: "Set up Python"
|
||||||
uses: actions/setup-python@v6
|
uses: actions/setup-python@v7
|
||||||
with:
|
with:
|
||||||
python-version: ${{ matrix.python }}
|
python-version: ${{ matrix.python }}
|
||||||
|
|
||||||
@@ -275,6 +275,14 @@ jobs:
|
|||||||
run: |
|
run: |
|
||||||
TAG="v${{ steps.version.outputs.version }}"
|
TAG="v${{ steps.version.outputs.version }}"
|
||||||
|
|
||||||
|
# If release.yml already published this version, _version.py may
|
||||||
|
# still reflect it until the next version bump commit. Don't
|
||||||
|
# re-draft an already-published release on later pushes to main.
|
||||||
|
if [[ "$(gh release view "$TAG" --json isDraft --jq .isDraft 2>/dev/null)" == "false" ]]; then
|
||||||
|
echo "Release $TAG is already published; skipping."
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
# Delete existing draft release if it exists (ignore errors)
|
# Delete existing draft release if it exists (ignore errors)
|
||||||
gh release delete "$TAG" --yes 2>/dev/null || true
|
gh release delete "$TAG" --yes 2>/dev/null || true
|
||||||
|
|
||||||
|
|||||||
@@ -331,6 +331,16 @@ def bump_version() -> None:
|
|||||||
contents = contents.replace(find, replace)
|
contents = contents.replace(find, replace)
|
||||||
path.write_text(contents, encoding="utf8")
|
path.write_text(contents, encoding="utf8")
|
||||||
|
|
||||||
|
# Format only after every file (including pyproject.toml) reflects the new
|
||||||
|
# version. Running `uv run` while pyproject.toml still had the old version
|
||||||
|
# would leave its post-bump environment/lockfile resync to happen for the
|
||||||
|
# first time during the commit's pre-commit hooks instead of here, which
|
||||||
|
# then aborts the commit with a spurious "files were modified by this
|
||||||
|
# hook" error.
|
||||||
|
for path, _find, _replace in actions:
|
||||||
|
if path.suffix == ".py":
|
||||||
|
subprocess.run(["uv", "run", "ruff", "format", str(path)], check=True)
|
||||||
|
|
||||||
print("Files updated.")
|
print("Files updated.")
|
||||||
print()
|
print()
|
||||||
|
|
||||||
|
|||||||
@@ -95,10 +95,12 @@ from multiprocessing import Process
|
|||||||
import ocrmypdf
|
import ocrmypdf
|
||||||
from ocrmypdf import OcrOptions
|
from ocrmypdf import OcrOptions
|
||||||
|
|
||||||
|
|
||||||
def ocrmypdf_process():
|
def ocrmypdf_process():
|
||||||
options = OcrOptions(input_file='input.pdf', output_file='output.pdf')
|
options = OcrOptions(input_file='input.pdf', output_file='output.pdf')
|
||||||
ocrmypdf.ocr(options)
|
ocrmypdf.ocr(options)
|
||||||
|
|
||||||
|
|
||||||
def call_ocrmypdf_from_my_app():
|
def call_ocrmypdf_from_my_app():
|
||||||
p = Process(target=ocrmypdf_process)
|
p = Process(target=ocrmypdf_process)
|
||||||
p.start()
|
p.start()
|
||||||
|
|||||||
+19
-3
@@ -663,9 +663,25 @@ provides text shaping for proper multilingual support. These replace the
|
|||||||
legacy hOCR-based renderer. Install with: `pip install fpdf2 uharfbuzz`
|
legacy hOCR-based renderer. Install with: `pip install fpdf2 uharfbuzz`
|
||||||
|
|
||||||
**fonts-noto** (or an equivalent comprehensive font package) is recommended
|
**fonts-noto** (or an equivalent comprehensive font package) is recommended
|
||||||
for proper text rendering, especially for non-Latin scripts. On Debian/Ubuntu:
|
for proper text rendering, especially for non-Latin scripts. OCRmyPDF bundles
|
||||||
`apt install fonts-noto`. On Fedora: `dnf install google-noto-fonts-common`.
|
a Latin font only, and discovers the rest from the fonts installed on your
|
||||||
On macOS with Homebrew: `brew install font-noto`.
|
system.
|
||||||
|
|
||||||
|
- Debian/Ubuntu: `apt install fonts-noto`
|
||||||
|
- Fedora: `dnf install google-noto-fonts-all`
|
||||||
|
- macOS with Homebrew: Homebrew has no single Noto package; each family is a
|
||||||
|
separate cask. Install at least
|
||||||
|
`brew install --cask font-noto-sans font-noto-serif`, plus a cask per
|
||||||
|
additional script you OCR, for example
|
||||||
|
`brew install --cask font-noto-sans-arabic font-noto-sans-cjk`. Run
|
||||||
|
`brew search font-noto` to list them all.
|
||||||
|
|
||||||
|
If OCRmyPDF warns that no installed font has glyphs for some of the text, the
|
||||||
|
message names the characters it could not render, for example
|
||||||
|
`'Ꮳ' U+13E3 CHEROKEE LETTER TSA`. Install the Noto font for that script — here,
|
||||||
|
`fonts-noto-core` on Debian or `font-noto-sans-cherokee` on Homebrew. The text
|
||||||
|
layer remains searchable and copyable either way; only its appearance when
|
||||||
|
highlighted in a PDF viewer is affected.
|
||||||
|
|
||||||
**pypdfium2**, if present, provides fast PDF page rasterization using
|
**pypdfium2**, if present, provides fast PDF page rasterization using
|
||||||
the pdfium library (the same library used by Google Chrome). It is
|
the pdfium library (the same library used by Google Chrome). It is
|
||||||
|
|||||||
+3
-1
@@ -47,7 +47,9 @@ OCRmyPDF has the following runtime dependencies:
|
|||||||
**For text rendering** (expressing OCR results in PDF):
|
**For text rendering** (expressing OCR results in PDF):
|
||||||
- `fpdf2` (Python package) - Required for text layer rendering
|
- `fpdf2` (Python package) - Required for text layer rendering
|
||||||
- `uharfbuzz` (Python package) - Required for text layer rendering
|
- `uharfbuzz` (Python package) - Required for text layer rendering
|
||||||
- `font-noto` (system package) - Recommended for text layer rendering
|
- Noto fonts (system package) - Recommended for text layer rendering.
|
||||||
|
`fonts-noto` on Debian/Ubuntu, `google-noto-fonts-all` on Fedora; Homebrew
|
||||||
|
has no single Noto package, only per-family casks such as `font-noto-sans`.
|
||||||
|
|
||||||
**Other dependencies**:
|
**Other dependencies**:
|
||||||
- `unpaper` (system binary) - Optional, enables `--clean` and `--clean-final`
|
- `unpaper` (system binary) - Optional, enables `--clean` and `--clean-final`
|
||||||
|
|||||||
+13
-13
@@ -120,6 +120,7 @@ A plugin may provide the following hooks. Hooks must be decorated with
|
|||||||
```python
|
```python
|
||||||
from ocrmypdf import hookimpl
|
from ocrmypdf import hookimpl
|
||||||
|
|
||||||
|
|
||||||
@hookimpl
|
@hookimpl
|
||||||
def add_options(parser):
|
def add_options(parser):
|
||||||
pass
|
pass
|
||||||
@@ -205,12 +206,11 @@ from ocrmypdf._options import OcrOptions
|
|||||||
|
|
||||||
```python
|
```python
|
||||||
# Before (v16 and earlier)
|
# Before (v16 and earlier)
|
||||||
def check_options(options: argparse.Namespace) -> None:
|
def check_options(options: argparse.Namespace) -> None: ...
|
||||||
...
|
|
||||||
|
|
||||||
# After (v17+)
|
# After (v17+)
|
||||||
def check_options(options: OcrOptions) -> None:
|
def check_options(options: OcrOptions) -> None: ...
|
||||||
...
|
|
||||||
```
|
```
|
||||||
|
|
||||||
**Attribute access unchanged:**
|
**Attribute access unchanged:**
|
||||||
@@ -229,6 +229,7 @@ options.tesseract_timeout
|
|||||||
def check_options(options):
|
def check_options(options):
|
||||||
options.some_computed_value = compute_value(options)
|
options.some_computed_value = compute_value(options)
|
||||||
|
|
||||||
|
|
||||||
# After (v17 pattern - compute at point of use)
|
# After (v17 pattern - compute at point of use)
|
||||||
def some_function(options):
|
def some_function(options):
|
||||||
computed = compute_value(options)
|
computed = compute_value(options)
|
||||||
@@ -336,19 +337,17 @@ from ocrmypdf import OcrElement, OcrClass, BoundingBox
|
|||||||
|
|
||||||
# OcrElement - represents any OCR structural unit
|
# OcrElement - represents any OCR structural unit
|
||||||
page = OcrElement(
|
page = OcrElement(
|
||||||
ocr_class=OcrClass.PAGE,
|
ocr_class=OcrClass.PAGE, bbox=BoundingBox(0, 0, 612, 792), children=[...]
|
||||||
bbox=BoundingBox(0, 0, 612, 792),
|
|
||||||
children=[...]
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# BoundingBox - axis-aligned bounding box (left, top, right, bottom)
|
# BoundingBox - axis-aligned bounding box (left, top, right, bottom)
|
||||||
bbox = BoundingBox(left=100, top=50, right=300, bottom=80)
|
bbox = BoundingBox(left=100, top=50, right=300, bottom=80)
|
||||||
|
|
||||||
# OcrClass - constants for element types
|
# OcrClass - constants for element types
|
||||||
OcrClass.PAGE # "ocr_page"
|
OcrClass.PAGE # "ocr_page"
|
||||||
OcrClass.LINE # "ocr_line"
|
OcrClass.LINE # "ocr_line"
|
||||||
OcrClass.WORD # "ocrx_word"
|
OcrClass.WORD # "ocrx_word"
|
||||||
OcrClass.PARAGRAPH # "ocr_par"
|
OcrClass.PARAGRAPH # "ocr_par"
|
||||||
```
|
```
|
||||||
|
|
||||||
**Navigating the tree:**
|
**Navigating the tree:**
|
||||||
@@ -378,6 +377,7 @@ from pathlib import Path
|
|||||||
from ocrmypdf.pluginspec import OcrEngine
|
from ocrmypdf.pluginspec import OcrEngine
|
||||||
from ocrmypdf import OcrElement, OcrClass, BoundingBox
|
from ocrmypdf import OcrElement, OcrClass, BoundingBox
|
||||||
|
|
||||||
|
|
||||||
class MyOcrEngine(OcrEngine):
|
class MyOcrEngine(OcrEngine):
|
||||||
def generate_ocr(
|
def generate_ocr(
|
||||||
self,
|
self,
|
||||||
@@ -402,10 +402,10 @@ class MyOcrEngine(OcrEngine):
|
|||||||
text="Hello",
|
text="Hello",
|
||||||
),
|
),
|
||||||
# ... more words
|
# ... more words
|
||||||
]
|
],
|
||||||
),
|
),
|
||||||
# ... more lines
|
# ... more lines
|
||||||
]
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
def supports_generate_ocr(self) -> bool:
|
def supports_generate_ocr(self) -> bool:
|
||||||
|
|||||||
@@ -3,6 +3,42 @@
|
|||||||
|
|
||||||
# v17
|
# v17
|
||||||
|
|
||||||
|
## v17.9.0
|
||||||
|
|
||||||
|
- OCRmyPDF now uses any Noto font installed on the system, not just the two
|
||||||
|
dozen script families it knows by name ({issue}`1722`). Previously a document
|
||||||
|
in, say, Cherokee or Vai was rendered with the glyphless fallback font even
|
||||||
|
though the matching font was installed — a common situation on macOS, which
|
||||||
|
ships around a hundred script-specific Noto faces. When the named fonts
|
||||||
|
cannot cover a word, OCRmyPDF now searches the installed fonts for one that
|
||||||
|
can.
|
||||||
|
- The "no installed font has glyphs" warning now names the characters it could
|
||||||
|
not render, with their codepoints and Unicode names, so it is clear which
|
||||||
|
font to install. Text that mixes scripts no single font covers is now
|
||||||
|
reported as such, instead of advising the user to install fonts they may
|
||||||
|
already have.
|
||||||
|
- Fixed the macOS font installation instructions, which recommended a Homebrew
|
||||||
|
package (`font-noto`) that does not exist ({issue}`1722`). Homebrew has no
|
||||||
|
single Noto package; each family is a separate cask. The Fedora package name
|
||||||
|
was also corrected to `google-noto-fonts-all`.
|
||||||
|
- Font providers may now implement the optional `GlyphSearchingFontProvider`
|
||||||
|
protocol to participate in coverage-based font search.
|
||||||
|
- Fixed `--jpeg-quality`/`--jpg-quality` having no effect on the CLI: the
|
||||||
|
value was silently dropped before reaching the optimizer, which then
|
||||||
|
always used its own built-in default JPEG quality regardless of what was
|
||||||
|
requested ({issue}`1723`). The same bug affected the Python API's
|
||||||
|
`jpg_quality` parameter. `ocrmypdf.ocr()` now accepts `jpeg_quality`
|
||||||
|
(matching the CLI flag name) as the canonical parameter; `jpg_quality`
|
||||||
|
still works but is deprecated.
|
||||||
|
- Hardened PDF parsing against malformed (non-dictionary) `/Resources`,
|
||||||
|
`/XObject`, and `/FontDescriptor` entries, which previously crashed
|
||||||
|
`ocrmypdf.ocr()` with `AttributeError`/`TypeError`/`ValueError` on
|
||||||
|
otherwise-processable files, both during PDF/A font scanning and general
|
||||||
|
image scanning ({issue}`1713`). Thanks @mvanhorn for the initial fix.
|
||||||
|
- Release process improvements: fixed a CI bug where every push to main
|
||||||
|
after a release was tagged would incorrectly revert the just-published
|
||||||
|
GitHub release back to draft status.
|
||||||
|
|
||||||
## v17.8.1
|
## v17.8.1
|
||||||
|
|
||||||
- Improved the `--tesseract-pagesegmode` help text to point to
|
- Improved the `--tesseract-pagesegmode` help text to point to
|
||||||
|
|||||||
+1
-1
@@ -6,7 +6,7 @@ build-backend = "hatchling.build"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "ocrmypdf"
|
name = "ocrmypdf"
|
||||||
version = "17.8.1"
|
version = "17.9.0"
|
||||||
description = "OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched"
|
description = "OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
license = "MPL-2.0"
|
license = "MPL-2.0"
|
||||||
|
|||||||
@@ -207,19 +207,19 @@ class OcrOptions(BaseModel):
|
|||||||
|
|
||||||
# Optimization
|
# Optimization
|
||||||
optimize: int = 1
|
optimize: int = 1
|
||||||
jpg_quality: int | None = None
|
jpeg_quality: int | None = None
|
||||||
png_quality: int | None = None
|
png_quality: int | None = None
|
||||||
|
|
||||||
# Compatibility alias for plugins that expect jpeg_quality
|
# Deprecated compatibility alias for code that still uses the old field name
|
||||||
@property
|
@property
|
||||||
def jpeg_quality(self):
|
def jpg_quality(self):
|
||||||
"""Compatibility alias for jpg_quality."""
|
"""Deprecated compatibility alias for jpeg_quality."""
|
||||||
return self.jpg_quality
|
return self.jpeg_quality
|
||||||
|
|
||||||
@jpeg_quality.setter
|
@jpg_quality.setter
|
||||||
def jpeg_quality(self, value):
|
def jpg_quality(self, value):
|
||||||
"""Compatibility alias for jpg_quality."""
|
"""Deprecated compatibility alias for jpeg_quality."""
|
||||||
self.jpg_quality = value
|
self.jpeg_quality = value
|
||||||
|
|
||||||
# Output behavior
|
# Output behavior
|
||||||
no_overwrite: bool = False
|
no_overwrite: bool = False
|
||||||
@@ -642,12 +642,6 @@ class OcrOptions(BaseModel):
|
|||||||
value = self.optimize
|
value = self.optimize
|
||||||
if value is not None:
|
if value is not None:
|
||||||
kwargs[field_name] = _convert_value(value)
|
kwargs[field_name] = _convert_value(value)
|
||||||
elif namespace == 'optimize' and field_name == 'jpeg_quality':
|
|
||||||
# jpg_quality maps to jpeg_quality
|
|
||||||
if 'jpg_quality' in OcrOptions.model_fields:
|
|
||||||
value = self.jpg_quality
|
|
||||||
if value is not None:
|
|
||||||
kwargs[field_name] = _convert_value(value)
|
|
||||||
|
|
||||||
# Create and cache the plugin options instance
|
# Create and cache the plugin options instance
|
||||||
instance = model_class(**kwargs)
|
instance = model_class(**kwargs)
|
||||||
|
|||||||
@@ -1,3 +1,3 @@
|
|||||||
# SPDX-FileCopyrightText: 2022 James R. Barlow
|
# SPDX-FileCopyrightText: 2022 James R. Barlow
|
||||||
# SPDX-License-Identifier: MPL-2.0
|
# SPDX-License-Identifier: MPL-2.0
|
||||||
__version__ = "17.8.1"
|
__version__ = "17.9.0"
|
||||||
|
|||||||
+25
-3
@@ -345,6 +345,23 @@ def _remap_language_to_languages(options_kwargs: dict) -> None:
|
|||||||
del options_kwargs['language']
|
del options_kwargs['language']
|
||||||
|
|
||||||
|
|
||||||
|
def _remap_jpg_quality_to_jpeg_quality(options_kwargs: dict) -> None:
|
||||||
|
"""Map the deprecated 'jpg_quality' parameter to 'jpeg_quality'.
|
||||||
|
|
||||||
|
'jpg_quality' was the original API parameter name. 'jpeg_quality' is the
|
||||||
|
canonical OcrOptions field, matching the primary --jpeg-quality CLI flag.
|
||||||
|
Prefer an explicitly-given 'jpeg_quality' if both are set.
|
||||||
|
"""
|
||||||
|
if 'jpg_quality' not in options_kwargs:
|
||||||
|
return
|
||||||
|
old_value = options_kwargs.pop('jpg_quality')
|
||||||
|
if old_value is None:
|
||||||
|
return
|
||||||
|
warn("ocrmypdf.ocr(jpg_quality=...) is deprecated, use jpeg_quality= instead.")
|
||||||
|
if options_kwargs.get('jpeg_quality') is None:
|
||||||
|
options_kwargs['jpeg_quality'] = old_value
|
||||||
|
|
||||||
|
|
||||||
def create_options(
|
def create_options(
|
||||||
*, input_file: PathOrIO, output_file: PathOrIO, parser: ArgumentParser, **kwargs
|
*, input_file: PathOrIO, output_file: PathOrIO, parser: ArgumentParser, **kwargs
|
||||||
) -> OcrOptions:
|
) -> OcrOptions:
|
||||||
@@ -369,6 +386,9 @@ def create_options(
|
|||||||
# Map API parameter 'language' to OcrOptions field 'languages'
|
# Map API parameter 'language' to OcrOptions field 'languages'
|
||||||
_remap_language_to_languages(options_kwargs)
|
_remap_language_to_languages(options_kwargs)
|
||||||
|
|
||||||
|
# Map deprecated 'jpg_quality' parameter to 'jpeg_quality'
|
||||||
|
_remap_jpg_quality_to_jpeg_quality(options_kwargs)
|
||||||
|
|
||||||
# Set input and output files
|
# Set input and output files
|
||||||
options_kwargs['input_file'] = input_file
|
options_kwargs['input_file'] = input_file
|
||||||
options_kwargs['output_file'] = output_file
|
options_kwargs['output_file'] = output_file
|
||||||
@@ -448,7 +468,8 @@ def ocr(
|
|||||||
redo_ocr: bool | None = None,
|
redo_ocr: bool | None = None,
|
||||||
skip_big: float | None = None,
|
skip_big: float | None = None,
|
||||||
optimize: int | None = None,
|
optimize: int | None = None,
|
||||||
jpg_quality: int | None = None,
|
jpeg_quality: int | None = None,
|
||||||
|
jpg_quality: int | None = None, # Deprecated, use jpeg_quality instead
|
||||||
png_quality: int | None = None,
|
png_quality: int | None = None,
|
||||||
jbig2_lossy: bool | None = None,
|
jbig2_lossy: bool | None = None,
|
||||||
jbig2_page_group_size: int | None = None,
|
jbig2_page_group_size: int | None = None,
|
||||||
@@ -511,7 +532,8 @@ def ocr( # noqa: D417
|
|||||||
redo_ocr: bool | None = None, # Legacy, use mode='redo' instead
|
redo_ocr: bool | None = None, # Legacy, use mode='redo' instead
|
||||||
skip_big: float | None = None,
|
skip_big: float | None = None,
|
||||||
optimize: int | None = None,
|
optimize: int | None = None,
|
||||||
jpg_quality: int | None = None,
|
jpeg_quality: int | None = None,
|
||||||
|
jpg_quality: int | None = None, # Deprecated, use jpeg_quality instead
|
||||||
png_quality: int | None = None,
|
png_quality: int | None = None,
|
||||||
jbig2_lossy: bool | None = None, # Deprecated, ignored
|
jbig2_lossy: bool | None = None, # Deprecated, ignored
|
||||||
jbig2_page_group_size: int | None = None, # Deprecated, ignored
|
jbig2_page_group_size: int | None = None, # Deprecated, ignored
|
||||||
@@ -883,7 +905,7 @@ def _hocr_to_ocr_pdf( # noqa: D417
|
|||||||
jobs: int | None = None,
|
jobs: int | None = None,
|
||||||
use_threads: bool | None = None,
|
use_threads: bool | None = None,
|
||||||
optimize: int | None = None,
|
optimize: int | None = None,
|
||||||
jpg_quality: int | None = None,
|
jpeg_quality: int | None = None,
|
||||||
png_quality: int | None = None,
|
png_quality: int | None = None,
|
||||||
jbig2_lossy: bool | None = None, # Deprecated, ignored
|
jbig2_lossy: bool | None = None, # Deprecated, ignored
|
||||||
jbig2_page_group_size: int | None = None, # Deprecated, ignored
|
jbig2_page_group_size: int | None = None, # Deprecated, ignored
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ from ocrmypdf.font.font_provider import (
|
|||||||
BuiltinFontProvider,
|
BuiltinFontProvider,
|
||||||
ChainedFontProvider,
|
ChainedFontProvider,
|
||||||
FontProvider,
|
FontProvider,
|
||||||
|
GlyphSearchingFontProvider,
|
||||||
)
|
)
|
||||||
from ocrmypdf.font.multi_font_manager import MultiFontManager
|
from ocrmypdf.font.multi_font_manager import MultiFontManager
|
||||||
from ocrmypdf.font.system_font_provider import SystemFontProvider
|
from ocrmypdf.font.system_font_provider import SystemFontProvider
|
||||||
@@ -25,6 +26,7 @@ from ocrmypdf.font.system_font_provider import SystemFontProvider
|
|||||||
__all__ = [
|
__all__ = [
|
||||||
"FontManager",
|
"FontManager",
|
||||||
"FontProvider",
|
"FontProvider",
|
||||||
|
"GlyphSearchingFontProvider",
|
||||||
"BuiltinFontProvider",
|
"BuiltinFontProvider",
|
||||||
"ChainedFontProvider",
|
"ChainedFontProvider",
|
||||||
"MultiFontManager",
|
"MultiFontManager",
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import logging
|
import logging
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Protocol
|
from typing import Protocol, runtime_checkable
|
||||||
|
|
||||||
from ocrmypdf.font.font_manager import FontManager
|
from ocrmypdf.font.font_manager import FontManager
|
||||||
|
|
||||||
@@ -52,6 +52,34 @@ class FontProvider(Protocol):
|
|||||||
...
|
...
|
||||||
|
|
||||||
|
|
||||||
|
@runtime_checkable
|
||||||
|
class GlyphSearchingFontProvider(Protocol):
|
||||||
|
"""Optional capability: find a font by glyph coverage rather than by name.
|
||||||
|
|
||||||
|
A provider only knows a limited set of logical font names, but it may have
|
||||||
|
access to many more fonts than it can name (e.g. the ~100 script-specific
|
||||||
|
Noto faces macOS installs). Implementing this lets MultiFontManager use
|
||||||
|
them as a last resort instead of falling back to glyphless rendering.
|
||||||
|
|
||||||
|
Providers that do not implement this are used as-is; the capability is
|
||||||
|
detected at runtime with ``isinstance``.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def find_font_with_glyphs(self, text: str) -> tuple[str, FontManager] | None:
|
||||||
|
"""Find a font that has glyphs for every character in text.
|
||||||
|
|
||||||
|
The returned name must subsequently resolve through ``get_font()``, so
|
||||||
|
that callers can cache the selection by name.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
text: Text the font must fully cover
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
(logical font name, FontManager), or None if no font covers text
|
||||||
|
"""
|
||||||
|
...
|
||||||
|
|
||||||
|
|
||||||
class BuiltinFontProvider:
|
class BuiltinFontProvider:
|
||||||
"""Font provider using builtin fonts from ocrmypdf/data directory."""
|
"""Font provider using builtin fonts from ocrmypdf/data directory."""
|
||||||
|
|
||||||
@@ -119,6 +147,18 @@ class BuiltinFontProvider:
|
|||||||
"""Get the glyphless fallback font."""
|
"""Get the glyphless fallback font."""
|
||||||
return self._fonts['Occulta']
|
return self._fonts['Occulta']
|
||||||
|
|
||||||
|
def find_font_with_glyphs(self, text: str) -> tuple[str, FontManager] | None:
|
||||||
|
"""Find a bundled font that covers text, ignoring glyphless Occulta."""
|
||||||
|
if not text:
|
||||||
|
return None
|
||||||
|
codepoints = {ord(c) for c in text}
|
||||||
|
for name, font in self._fonts.items():
|
||||||
|
if name == 'Occulta':
|
||||||
|
continue
|
||||||
|
if all(font.has_glyph(cp) for cp in codepoints):
|
||||||
|
return name, font
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
class ChainedFontProvider:
|
class ChainedFontProvider:
|
||||||
"""Font provider that tries multiple providers in order.
|
"""Font provider that tries multiple providers in order.
|
||||||
@@ -170,6 +210,25 @@ class ChainedFontProvider:
|
|||||||
result.append(name)
|
result.append(name)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
def find_font_with_glyphs(self, text: str) -> tuple[str, FontManager] | None:
|
||||||
|
"""Ask each capable provider in turn for a font that covers text.
|
||||||
|
|
||||||
|
Providers that don't implement the search are skipped.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
text: Text the font must fully cover
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
(logical font name, FontManager) from the first provider with a
|
||||||
|
match, or None if no provider found one
|
||||||
|
"""
|
||||||
|
for provider in self.providers:
|
||||||
|
if not isinstance(provider, GlyphSearchingFontProvider):
|
||||||
|
continue
|
||||||
|
if found := provider.find_font_with_glyphs(text):
|
||||||
|
return found
|
||||||
|
return None
|
||||||
|
|
||||||
def get_fallback_font(self) -> FontManager:
|
def get_fallback_font(self) -> FontManager:
|
||||||
"""Get the glyphless fallback font.
|
"""Get the glyphless fallback font.
|
||||||
|
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ language hints and glyph coverage analysis.
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import unicodedata
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from ocrmypdf.font.font_manager import FontManager
|
from ocrmypdf.font.font_manager import FontManager
|
||||||
@@ -17,6 +18,7 @@ from ocrmypdf.font.font_provider import (
|
|||||||
BuiltinFontProvider,
|
BuiltinFontProvider,
|
||||||
ChainedFontProvider,
|
ChainedFontProvider,
|
||||||
FontProvider,
|
FontProvider,
|
||||||
|
GlyphSearchingFontProvider,
|
||||||
)
|
)
|
||||||
from ocrmypdf.font.system_font_provider import SystemFontProvider
|
from ocrmypdf.font.system_font_provider import SystemFontProvider
|
||||||
|
|
||||||
@@ -33,9 +35,17 @@ class MultiFontManager:
|
|||||||
Font selection strategy:
|
Font selection strategy:
|
||||||
1. Try language-preferred font (if language hint available)
|
1. Try language-preferred font (if language hint available)
|
||||||
2. Try fallback fonts in order by glyph coverage
|
2. Try fallback fonts in order by glyph coverage
|
||||||
3. Fall back to Occulta.ttf (glyphless fallback)
|
3. Ask the provider for any installed font that covers the text
|
||||||
|
4. Fall back to Occulta.ttf (glyphless fallback)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
# How many uncoverable characters to name in the missing-font warning
|
||||||
|
MAX_REPORTED_CHARS = 3
|
||||||
|
|
||||||
|
# How many characters of a word to look up individually when composing that
|
||||||
|
# warning; each lookup may scan every font installed on the system
|
||||||
|
MAX_EXAMINED_CHARS = 8
|
||||||
|
|
||||||
# Language to font mapping
|
# Language to font mapping
|
||||||
# Keys are ISO 639-2/3 codes or Tesseract language codes
|
# Keys are ISO 639-2/3 codes or Tesseract language codes
|
||||||
LANGUAGE_FONT_MAP = {
|
LANGUAGE_FONT_MAP = {
|
||||||
@@ -173,6 +183,9 @@ class MultiFontManager:
|
|||||||
self._selection_cache: dict[tuple[str, str | None], str] = {}
|
self._selection_cache: dict[tuple[str, str | None], str] = {}
|
||||||
# Track whether we've warned about missing fonts (warn once per script)
|
# Track whether we've warned about missing fonts (warn once per script)
|
||||||
self._warned_scripts: set[str] = set()
|
self._warned_scripts: set[str] = set()
|
||||||
|
# Fonts found by glyph coverage rather than by name, tried before
|
||||||
|
# repeating the (expensive) provider search
|
||||||
|
self._discovered_fonts: list[str] = []
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def fonts(self) -> dict[str, FontManager]:
|
def fonts(self) -> dict[str, FontManager]:
|
||||||
@@ -208,7 +221,8 @@ class MultiFontManager:
|
|||||||
Uses a hybrid approach:
|
Uses a hybrid approach:
|
||||||
1. Language-based selection (if language hint available)
|
1. Language-based selection (if language hint available)
|
||||||
2. Ordered fallback through available fonts by glyph coverage
|
2. Ordered fallback through available fonts by glyph coverage
|
||||||
3. Final fallback to Occulta.ttf (glyphless)
|
3. Provider search over every installed font, by glyph coverage
|
||||||
|
4. Final fallback to Occulta.ttf (glyphless)
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
word_text: The text content of the word
|
word_text: The text content of the word
|
||||||
@@ -233,19 +247,50 @@ class MultiFontManager:
|
|||||||
if result := self._try_font(preferred, word_text, cache_key):
|
if result := self._try_font(preferred, word_text, cache_key):
|
||||||
return result
|
return result
|
||||||
|
|
||||||
# Phase 2: Try fallback fonts in order
|
# Phase 2: Try fallback fonts in order, then anything a previous
|
||||||
for font_name in self.FALLBACK_FONTS:
|
# coverage search turned up
|
||||||
|
for font_name in [*self.FALLBACK_FONTS, *self._discovered_fonts]:
|
||||||
if font_name in tried_fonts:
|
if font_name in tried_fonts:
|
||||||
continue
|
continue
|
||||||
|
tried_fonts.add(font_name)
|
||||||
if result := self._try_font(font_name, word_text, cache_key):
|
if result := self._try_font(font_name, word_text, cache_key):
|
||||||
return result
|
return result
|
||||||
|
|
||||||
# Phase 3: Glyphless fallback (always succeeds)
|
# Phase 3: Ask the provider to search every installed font. The named
|
||||||
|
# families cover common scripts only, but systems ship many more (macOS
|
||||||
|
# installs ~100 Noto faces), and those should be used before giving up
|
||||||
|
# on rendering the text at all. See issue #1722.
|
||||||
|
if found := self._search_font_by_coverage(word_text):
|
||||||
|
font_name, font = found
|
||||||
|
self._selection_cache[cache_key] = font_name
|
||||||
|
return font
|
||||||
|
|
||||||
|
# Phase 4: Glyphless fallback (always succeeds)
|
||||||
# Warn if we're falling back for non-ASCII text (likely missing font)
|
# Warn if we're falling back for non-ASCII text (likely missing font)
|
||||||
self._warn_missing_font(word_text, line_language)
|
self._warn_missing_font(word_text, line_language)
|
||||||
self._selection_cache[cache_key] = 'Occulta'
|
self._selection_cache[cache_key] = 'Occulta'
|
||||||
return self.font_provider.get_fallback_font()
|
return self.font_provider.get_fallback_font()
|
||||||
|
|
||||||
|
def _search_font_by_coverage(self, text: str) -> tuple[str, FontManager] | None:
|
||||||
|
"""Search the provider for any font covering text, if it supports it.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
text: Text the font must fully cover
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
(font name, FontManager), or None if unsupported or nothing matched
|
||||||
|
"""
|
||||||
|
provider = self.font_provider
|
||||||
|
if not isinstance(provider, GlyphSearchingFontProvider):
|
||||||
|
return None
|
||||||
|
found = provider.find_font_with_glyphs(text)
|
||||||
|
if found is None:
|
||||||
|
return None
|
||||||
|
font_name, _font = found
|
||||||
|
if font_name not in self._discovered_fonts:
|
||||||
|
self._discovered_fonts.append(font_name)
|
||||||
|
return found
|
||||||
|
|
||||||
def _warn_missing_font(self, word_text: str, line_language: str | None) -> None:
|
def _warn_missing_font(self, word_text: str, line_language: str | None) -> None:
|
||||||
"""Warn user about missing font for non-Latin text.
|
"""Warn user about missing font for non-Latin text.
|
||||||
|
|
||||||
@@ -264,26 +309,97 @@ class MultiFontManager:
|
|||||||
|
|
||||||
self._warned_scripts.add(warn_key)
|
self._warned_scripts.add(warn_key)
|
||||||
|
|
||||||
|
uncoverable = self._uncoverable_characters(word_text)
|
||||||
|
if not uncoverable:
|
||||||
|
# Every character has a font, but no single font has them all.
|
||||||
|
# Telling the user to install fonts would be wrong advice here.
|
||||||
|
log.warning(
|
||||||
|
"Text mixing scripts that no single installed font covers (%r) "
|
||||||
|
"was added as an invisible text layer: it stays searchable and "
|
||||||
|
"copyable, but appears blank when highlighted in a PDF viewer. "
|
||||||
|
"Installing more fonts will not help; OCRmyPDF uses one font "
|
||||||
|
"per word.",
|
||||||
|
word_text,
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
missing = self._describe_characters(uncoverable)
|
||||||
if line_language and line_language in self.LANGUAGE_FONT_MAP:
|
if line_language and line_language in self.LANGUAGE_FONT_MAP:
|
||||||
font_family = self.LANGUAGE_FONT_MAP[line_language].removesuffix('-Regular')
|
font_family = self.LANGUAGE_FONT_MAP[line_language].removesuffix('-Regular')
|
||||||
log.warning(
|
log.warning(
|
||||||
"No installed font has glyphs for the detected '%s' text, so "
|
"No installed font has glyphs for the detected '%s' text (%s), "
|
||||||
"it was added as an invisible text layer: it stays searchable "
|
"so it was added as an invisible text layer: it stays searchable "
|
||||||
"and copyable, but appears blank when highlighted in a PDF "
|
"and copyable, but appears blank when highlighted in a PDF "
|
||||||
"viewer. Install the %s font family (via your OS package "
|
"viewer. Install the %s font family (via your OS package "
|
||||||
"manager or https://fonts.google.com/noto) for full rendering.",
|
"manager or https://fonts.google.com/noto) for full rendering.",
|
||||||
line_language,
|
line_language,
|
||||||
|
missing,
|
||||||
font_family,
|
font_family,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
log.warning(
|
log.warning(
|
||||||
"No installed font has glyphs for some of the detected text, "
|
"No installed font has glyphs for some of the detected text "
|
||||||
"so it was added as an invisible text layer: it stays "
|
"(%s), so it was added as an invisible text layer: it stays "
|
||||||
"searchable and copyable, but appears blank when highlighted "
|
"searchable and copyable, but appears blank when highlighted "
|
||||||
"in a PDF viewer. Install the matching Noto fonts "
|
"in a PDF viewer. Install a Noto font covering that script "
|
||||||
"(https://fonts.google.com/noto) for full rendering."
|
"(https://fonts.google.com/noto) for full rendering.",
|
||||||
|
missing,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _uncoverable_characters(self, word_text: str) -> list[str]:
|
||||||
|
"""Find the characters of word_text that no installed font can render.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
word_text: The word that fell back to glyphless rendering
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The distinct uncoverable characters, in order of first appearance,
|
||||||
|
considering at most MAX_EXAMINED_CHARS of them
|
||||||
|
"""
|
||||||
|
candidates = [
|
||||||
|
char
|
||||||
|
for char in dict.fromkeys(word_text) # de-duplicate, keep order
|
||||||
|
if not char.isspace() and not self._is_char_renderable(char)
|
||||||
|
]
|
||||||
|
# The named fonts missed these, but the provider may still have a font
|
||||||
|
# for them, so confirm before telling the user to install anything. The
|
||||||
|
# search walks every installed font, hence the cap on how many
|
||||||
|
# characters we are willing to look up for one warning.
|
||||||
|
return [
|
||||||
|
char
|
||||||
|
for char in candidates[: self.MAX_EXAMINED_CHARS]
|
||||||
|
if self._search_font_by_coverage(char) is None
|
||||||
|
]
|
||||||
|
|
||||||
|
def _describe_characters(self, chars: list[str]) -> str:
|
||||||
|
"""Describe characters by codepoint and Unicode name.
|
||||||
|
|
||||||
|
Naming the codepoints tells the user which font to install even for
|
||||||
|
scripts OCRmyPDF has no language mapping for, which the generic
|
||||||
|
"install the matching Noto fonts" advice did not. See issue #1722.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
chars: Characters to describe
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Human-readable description, truncated to MAX_REPORTED_CHARS
|
||||||
|
"""
|
||||||
|
described = ", ".join(
|
||||||
|
f"{char!r} U+{ord(char):04X} {unicodedata.name(char, 'unnamed character')}"
|
||||||
|
for char in chars[: self.MAX_REPORTED_CHARS]
|
||||||
|
)
|
||||||
|
if len(chars) > self.MAX_REPORTED_CHARS:
|
||||||
|
described += f", and {len(chars) - self.MAX_REPORTED_CHARS} more"
|
||||||
|
return described
|
||||||
|
|
||||||
|
def _is_char_renderable(self, char: str) -> bool:
|
||||||
|
"""Check whether any font already known to us has a glyph for char."""
|
||||||
|
for font_name in [*self.FALLBACK_FONTS, *self._discovered_fonts]:
|
||||||
|
font = self.font_provider.get_font(font_name)
|
||||||
|
if font is not None and self._has_all_glyphs(font, char):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
def _has_all_glyphs(self, font: FontManager, text: str) -> bool:
|
def _has_all_glyphs(self, font: FontManager, text: str) -> bool:
|
||||||
"""Check if a font has glyphs for all characters in text.
|
"""Check if a font has glyphs for all characters in text.
|
||||||
|
|
||||||
|
|||||||
@@ -209,6 +209,13 @@ class SystemFontProvider:
|
|||||||
self._not_found: set[str] = set()
|
self._not_found: set[str] = set()
|
||||||
# Cached font directories (computed lazily)
|
# Cached font directories (computed lazily)
|
||||||
self._font_dirs: list[Path] | None = None
|
self._font_dirs: list[Path] | None = None
|
||||||
|
# Cached (logical name, path) of every Noto face on the system, in the
|
||||||
|
# order the coverage search should try them (computed lazily)
|
||||||
|
self._noto_candidates: list[tuple[str, Path]] | None = None
|
||||||
|
# Memoized results of find_font_with_glyphs(), keyed by codepoint set
|
||||||
|
self._coverage_cache: dict[frozenset[int], str | None] = {}
|
||||||
|
# Font files that failed to load, so we only complain about them once
|
||||||
|
self._unloadable: set[Path] = set()
|
||||||
|
|
||||||
def _get_platform(self) -> str:
|
def _get_platform(self) -> str:
|
||||||
"""Get the current platform identifier.
|
"""Get the current platform identifier.
|
||||||
@@ -352,6 +359,145 @@ class SystemFontProvider:
|
|||||||
return best[1]
|
return best[1]
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _family_base(stem: str) -> str | None:
|
||||||
|
"""Get the Noto family base a filename stem is the Regular face of.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
stem: Filename without extension, e.g. 'NotoSansCherokee-Regular'
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The family base ('NotoSansCherokee') or None if the stem is not a
|
||||||
|
Noto font, or is a weight/slope variant such as '-Bold' or
|
||||||
|
'-Italic' that should not stand in for the family.
|
||||||
|
"""
|
||||||
|
head = stem.split('[', 1)[0] # drop variable-font axes, e.g. '[wght]'
|
||||||
|
if head.endswith('-Regular'):
|
||||||
|
head = head[: -len('-Regular')]
|
||||||
|
elif head.endswith('-VF'):
|
||||||
|
head = head[: -len('-VF')]
|
||||||
|
elif '-' in head:
|
||||||
|
return None
|
||||||
|
return head if head.startswith('Noto') else None
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _candidate_sort_key(cls, base: str) -> tuple[int, int, str]:
|
||||||
|
"""Rank a family base for the coverage search.
|
||||||
|
|
||||||
|
Sans comes before serif before everything else, and plain families come
|
||||||
|
ahead of their narrower UI and Mono cousins.
|
||||||
|
"""
|
||||||
|
if base.startswith('NotoSans'):
|
||||||
|
family_rank = 0
|
||||||
|
elif base.startswith('NotoSerif'):
|
||||||
|
family_rank = 1
|
||||||
|
else:
|
||||||
|
family_rank = 2
|
||||||
|
narrow_use = base.endswith('UI') or base.startswith('NotoSansMono')
|
||||||
|
return (family_rank, int(narrow_use), base)
|
||||||
|
|
||||||
|
def _get_noto_candidates(self) -> list[tuple[str, Path]]:
|
||||||
|
"""Enumerate every Noto family installed on the system.
|
||||||
|
|
||||||
|
Scans each font directory once and keeps the best-ranked file per
|
||||||
|
family, so a family present in several directories or in several
|
||||||
|
variants contributes a single candidate.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of (logical font name, path) in the order to try them.
|
||||||
|
"""
|
||||||
|
if self._noto_candidates is not None:
|
||||||
|
return self._noto_candidates
|
||||||
|
|
||||||
|
best: dict[str, tuple[int, Path]] = {}
|
||||||
|
for font_dir in self._get_font_dirs():
|
||||||
|
if not font_dir.exists():
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
paths = sorted(font_dir.rglob('Noto*'))
|
||||||
|
except OSError:
|
||||||
|
# Skip directories we can't read
|
||||||
|
continue
|
||||||
|
for path in paths:
|
||||||
|
if path.suffix.lower() not in self._FONT_EXTENSIONS:
|
||||||
|
continue
|
||||||
|
base = self._family_base(path.stem)
|
||||||
|
if base is None:
|
||||||
|
continue
|
||||||
|
kind = self._classify_variant(path.stem, base)
|
||||||
|
if kind is None:
|
||||||
|
continue
|
||||||
|
rank = self._VARIANT_RANK[kind]
|
||||||
|
if base not in best or rank < best[base][0]:
|
||||||
|
best[base] = (rank, path)
|
||||||
|
|
||||||
|
self._noto_candidates = [
|
||||||
|
(f'{base}-Regular', path)
|
||||||
|
for base, (_rank, path) in sorted(
|
||||||
|
best.items(), key=lambda item: self._candidate_sort_key(item[0])
|
||||||
|
)
|
||||||
|
]
|
||||||
|
return self._noto_candidates
|
||||||
|
|
||||||
|
def find_font_with_glyphs(self, text: str) -> tuple[str, FontManager] | None:
|
||||||
|
"""Find any installed Noto font that covers every character in text.
|
||||||
|
|
||||||
|
``NOTO_FONT_PATTERNS`` enumerates the couple dozen scripts OCRmyPDF
|
||||||
|
knows by name, but systems ship far more: macOS alone installs around a
|
||||||
|
hundred script-specific Noto faces in
|
||||||
|
``/System/Library/Fonts/Supplemental``. This is the last resort that
|
||||||
|
makes those usable, so a document is only rendered glyphless when no
|
||||||
|
installed font can actually cover it. See issue #1722.
|
||||||
|
|
||||||
|
This walks every Noto face on the system and is therefore expensive;
|
||||||
|
results are memoized, and callers should only reach it after the named
|
||||||
|
fonts have failed.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
text: Text that the returned font must fully cover
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
(logical font name, FontManager) of the first covering font, or
|
||||||
|
None if nothing installed covers the text.
|
||||||
|
"""
|
||||||
|
if not text:
|
||||||
|
return None
|
||||||
|
needed = frozenset(ord(c) for c in text)
|
||||||
|
|
||||||
|
if needed in self._coverage_cache:
|
||||||
|
cached_name = self._coverage_cache[needed]
|
||||||
|
if cached_name is None:
|
||||||
|
return None
|
||||||
|
if cached := self._font_cache.get(cached_name):
|
||||||
|
return cached_name, cached
|
||||||
|
|
||||||
|
for font_name, path in self._get_noto_candidates():
|
||||||
|
font = self._font_cache.get(font_name)
|
||||||
|
if font is None:
|
||||||
|
if path in self._unloadable:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
font = FontManager(path)
|
||||||
|
except Exception as e:
|
||||||
|
log.debug("Skipping unreadable font %s: %s", path, e)
|
||||||
|
self._unloadable.add(path)
|
||||||
|
continue
|
||||||
|
if all(font.has_glyph(cp) for cp in needed):
|
||||||
|
# Keep only fonts we actually use; the rest are released so a
|
||||||
|
# full scan doesn't retain every font file on the system.
|
||||||
|
self._font_cache[font_name] = font
|
||||||
|
self._not_found.discard(font_name)
|
||||||
|
self._coverage_cache[needed] = font_name
|
||||||
|
log.debug(
|
||||||
|
"Found system font %s at %s (glyph coverage match)",
|
||||||
|
font_name,
|
||||||
|
path,
|
||||||
|
)
|
||||||
|
return font_name, font
|
||||||
|
|
||||||
|
self._coverage_cache[needed] = None
|
||||||
|
return None
|
||||||
|
|
||||||
def get_font(self, font_name: str) -> FontManager | None:
|
def get_font(self, font_name: str) -> FontManager | None:
|
||||||
"""Get a FontManager for the named font (lazy loading).
|
"""Get a FontManager for the named font (lazy loading).
|
||||||
|
|
||||||
|
|||||||
@@ -449,12 +449,12 @@ def convert_to_jbig2(
|
|||||||
|
|
||||||
|
|
||||||
def _optimize_jpeg(
|
def _optimize_jpeg(
|
||||||
xref: Xref, in_jpg: Path, opt_jpg: Path, jpg_quality: int
|
xref: Xref, in_jpg: Path, opt_jpg: Path, jpeg_quality: int
|
||||||
) -> tuple[Xref, Path | None]:
|
) -> tuple[Xref, Path | None]:
|
||||||
with Image.open(in_jpg) as im:
|
with Image.open(in_jpg) as im:
|
||||||
save_kwargs: dict[str, Any] = {'optimize': True}
|
save_kwargs: dict[str, Any] = {'optimize': True}
|
||||||
if isinstance(jpg_quality, int) and 0 < jpg_quality <= 100:
|
if isinstance(jpeg_quality, int) and 0 < jpeg_quality <= 100:
|
||||||
save_kwargs['quality'] = jpg_quality
|
save_kwargs['quality'] = jpeg_quality
|
||||||
im.save(opt_jpg, **save_kwargs)
|
im.save(opt_jpg, **save_kwargs)
|
||||||
|
|
||||||
if opt_jpg.stat().st_size > in_jpg.stat().st_size:
|
if opt_jpg.stat().st_size > in_jpg.stat().st_size:
|
||||||
@@ -473,7 +473,7 @@ def transcode_jpegs(
|
|||||||
for xref in jpegs:
|
for xref in jpegs:
|
||||||
in_jpg = jpg_name(root, xref)
|
in_jpg = jpg_name(root, xref)
|
||||||
opt_jpg = in_jpg.with_suffix('.opt.jpg')
|
opt_jpg = in_jpg.with_suffix('.opt.jpg')
|
||||||
yield xref, in_jpg, opt_jpg, options.jpg_quality
|
yield xref, in_jpg, opt_jpg, options.jpeg_quality
|
||||||
|
|
||||||
def finish_jpeg(result: tuple[Xref, Path | None], pbar: ProgressBar):
|
def finish_jpeg(result: tuple[Xref, Path | None], pbar: ProgressBar):
|
||||||
xref, opt_jpg = result
|
xref, opt_jpg = result
|
||||||
@@ -703,8 +703,8 @@ def optimize(
|
|||||||
safe_symlink(input_file, output_file)
|
safe_symlink(input_file, output_file)
|
||||||
return output_file
|
return output_file
|
||||||
|
|
||||||
if not options.jpg_quality:
|
if not options.jpeg_quality:
|
||||||
options.jpg_quality = DEFAULT_JPEG_QUALITY if options.optimize < 3 else 40
|
options.jpeg_quality = DEFAULT_JPEG_QUALITY if options.optimize < 3 else 40
|
||||||
if not options.png_quality:
|
if not options.png_quality:
|
||||||
options.png_quality = DEFAULT_PNG_QUALITY if options.optimize < 3 else 30
|
options.png_quality = DEFAULT_PNG_QUALITY if options.optimize < 3 else 30
|
||||||
|
|
||||||
@@ -766,7 +766,7 @@ def main(infile, outfile, level, jobs=1):
|
|||||||
output_file=outfile, # Required field
|
output_file=outfile, # Required field
|
||||||
jobs=jobs,
|
jobs=jobs,
|
||||||
optimize=int(level),
|
optimize=int(level),
|
||||||
jpg_quality=0, # Use default
|
jpeg_quality=0, # Use default
|
||||||
png_quality=0,
|
png_quality=0,
|
||||||
jbig2_threshold=0.85,
|
jbig2_threshold=0.85,
|
||||||
quiet=True,
|
quiet=True,
|
||||||
|
|||||||
+13
-7
@@ -12,7 +12,7 @@ from importlib.resources import files as package_files
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import pikepdf
|
import pikepdf
|
||||||
from pikepdf import Array, Dictionary, Name, Pdf, Stream
|
from pikepdf import Array, Dictionary, Name, Object, Pdf, Stream
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -137,11 +137,13 @@ def file_claims_pdfa(filename: Path):
|
|||||||
return pdfa_dict
|
return pdfa_dict
|
||||||
|
|
||||||
|
|
||||||
def _cid_font_is_embedded(type0_font: Dictionary) -> bool:
|
def _cid_font_is_embedded(type0_font: Object) -> bool:
|
||||||
"""Return True if a Type0 font's CID descendant carries embedded glyphs."""
|
"""Return True if a Type0 font's CID descendant carries embedded glyphs."""
|
||||||
for descendant in type0_font.get(Name.DescendantFonts, []):
|
for descendant in type0_font.get(Name.DescendantFonts, []):
|
||||||
descriptor = descendant.get(Name.FontDescriptor, None)
|
descriptor = descendant.get(Name.FontDescriptor, None)
|
||||||
if descriptor is not None and any(
|
# A malformed PDF may store a non-dictionary here; `key in descriptor`
|
||||||
|
# raises on those, so require a real dictionary before probing it.
|
||||||
|
if isinstance(descriptor, Dictionary) and any(
|
||||||
key in descriptor for key in (Name.FontFile, Name.FontFile2, Name.FontFile3)
|
key in descriptor for key in (Name.FontFile, Name.FontFile2, Name.FontFile3)
|
||||||
):
|
):
|
||||||
return True
|
return True
|
||||||
@@ -174,9 +176,13 @@ def find_nonembedded_cid_fonts(pdf: Pdf) -> set[str]:
|
|||||||
def scan_resources(resources, depth: int = 0) -> None:
|
def scan_resources(resources, depth: int = 0) -> None:
|
||||||
if resources is None or depth > 10:
|
if resources is None or depth > 10:
|
||||||
return
|
return
|
||||||
|
# A well-formed PDF stores dictionaries under /Font and /XObject, but a
|
||||||
|
# malformed one (common in OCR workloads) may store an array, a name, or
|
||||||
|
# another non-dictionary object. Only such dictionaries have .values(),
|
||||||
|
# so guard with isinstance rather than let the scan crash (issue #1713).
|
||||||
fonts = resources.get(Name.Font, None)
|
fonts = resources.get(Name.Font, None)
|
||||||
if fonts is not None:
|
if isinstance(fonts, Dictionary):
|
||||||
for font in fonts.values():
|
for font in fonts.as_dict().values():
|
||||||
try:
|
try:
|
||||||
if font.get(Name.Subtype) != Name.Type0:
|
if font.get(Name.Subtype) != Name.Type0:
|
||||||
continue
|
continue
|
||||||
@@ -186,8 +192,8 @@ def find_nonembedded_cid_fonts(pdf: Pdf) -> set[str]:
|
|||||||
except (AttributeError, TypeError, KeyError):
|
except (AttributeError, TypeError, KeyError):
|
||||||
continue
|
continue
|
||||||
xobjects = resources.get(Name.XObject, None)
|
xobjects = resources.get(Name.XObject, None)
|
||||||
if xobjects is not None:
|
if isinstance(xobjects, Dictionary):
|
||||||
for xobj in xobjects.values():
|
for xobj in xobjects.as_dict().values():
|
||||||
if xobj.get(Name.Subtype) == Name.Form and Name.Resources in xobj:
|
if xobj.get(Name.Subtype) == Name.Form and Name.Resources in xobj:
|
||||||
scan_resources(xobj[Name.Resources], depth + 1)
|
scan_resources(xobj[Name.Resources], depth + 1)
|
||||||
|
|
||||||
|
|||||||
@@ -287,9 +287,15 @@ def _image_xobjects(container) -> Iterator[tuple[Object, str]]:
|
|||||||
if Name.Resources not in container:
|
if Name.Resources not in container:
|
||||||
return
|
return
|
||||||
resources = container[Name.Resources]
|
resources = container[Name.Resources]
|
||||||
if Name.XObject not in resources:
|
# A malformed PDF may store a non-dictionary at /Resources or
|
||||||
|
# /Resources /XObject; treat that as "no image XObjects" instead of
|
||||||
|
# crashing when we try to iterate it.
|
||||||
|
if not isinstance(resources, Dictionary):
|
||||||
return
|
return
|
||||||
for key, candidate in resources[Name.XObject].items():
|
xobjects = resources.get(Name.XObject)
|
||||||
|
if not isinstance(xobjects, Dictionary):
|
||||||
|
return
|
||||||
|
for key, candidate in xobjects.items():
|
||||||
if candidate is None or Name.Subtype not in candidate:
|
if candidate is None or Name.Subtype not in candidate:
|
||||||
continue
|
continue
|
||||||
if candidate[Name.Subtype] == Name.Image:
|
if candidate[Name.Subtype] == Name.Image:
|
||||||
@@ -336,9 +342,14 @@ def _find_form_xobject_images(pdf: Pdf, container: Object, contentsinfo: Content
|
|||||||
if Name.Resources not in container:
|
if Name.Resources not in container:
|
||||||
return
|
return
|
||||||
resources = container[Name.Resources]
|
resources = container[Name.Resources]
|
||||||
if Name.XObject not in resources:
|
# As in _image_xobjects, tolerate a non-dictionary /Resources or
|
||||||
|
# /Resources /XObject in a malformed PDF rather than crashing.
|
||||||
|
if not isinstance(resources, Dictionary):
|
||||||
return
|
return
|
||||||
xobjs = resources[Name.XObject].as_dict()
|
xobject = resources.get(Name.XObject)
|
||||||
|
if not isinstance(xobject, Dictionary):
|
||||||
|
return
|
||||||
|
xobjs = xobject.as_dict()
|
||||||
for xobj in xobjs:
|
for xobj in xobjs:
|
||||||
candidate = xobjs[xobj]
|
candidate = xobjs[xobj]
|
||||||
if candidate is None or candidate.get(Name.Subtype) != Name.Form:
|
if candidate is None or candidate.get(Name.Subtype) != Name.Form:
|
||||||
|
|||||||
@@ -79,6 +79,45 @@ def test_language_parameter_mapped_to_languages():
|
|||||||
assert options.languages == ['eng', 'spa']
|
assert options.languages == ['eng', 'spa']
|
||||||
|
|
||||||
|
|
||||||
|
def test_jpeg_quality_parameter_reaches_options():
|
||||||
|
"""The canonical 'jpeg_quality' API parameter must reach OcrOptions.
|
||||||
|
|
||||||
|
Regression test for GitHub issue #1723: --jpeg-quality was silently
|
||||||
|
dropped by the CLI's namespace_to_options() because the OcrOptions field
|
||||||
|
was named jpg_quality. create_options(), used by the Python API, has the
|
||||||
|
same field-name matching logic and is affected the same way when passed
|
||||||
|
the alias name.
|
||||||
|
"""
|
||||||
|
from ocrmypdf.api import create_options, setup_plugin_infrastructure
|
||||||
|
from ocrmypdf.cli import get_parser
|
||||||
|
|
||||||
|
setup_plugin_infrastructure()
|
||||||
|
parser = get_parser()
|
||||||
|
|
||||||
|
options = create_options(
|
||||||
|
input_file='test.pdf', output_file='output.pdf', parser=parser, jpeg_quality=10
|
||||||
|
)
|
||||||
|
assert options.jpeg_quality == 10
|
||||||
|
|
||||||
|
|
||||||
|
def test_jpg_quality_parameter_deprecated_alias():
|
||||||
|
"""The old 'jpg_quality' API parameter still works but warns."""
|
||||||
|
from ocrmypdf.api import create_options, setup_plugin_infrastructure
|
||||||
|
from ocrmypdf.cli import get_parser
|
||||||
|
|
||||||
|
setup_plugin_infrastructure()
|
||||||
|
parser = get_parser()
|
||||||
|
|
||||||
|
with pytest.warns(UserWarning, match='jpg_quality'):
|
||||||
|
options = create_options(
|
||||||
|
input_file='test.pdf',
|
||||||
|
output_file='output.pdf',
|
||||||
|
parser=parser,
|
||||||
|
jpg_quality=42,
|
||||||
|
)
|
||||||
|
assert options.jpeg_quality == 42
|
||||||
|
|
||||||
|
|
||||||
def test_stream_api(resources: Path):
|
def test_stream_api(resources: Path):
|
||||||
in_ = (resources / 'graph.pdf').open('rb')
|
in_ = (resources / 'graph.pdf').open('rb')
|
||||||
out = BytesIO()
|
out = BytesIO()
|
||||||
|
|||||||
@@ -569,3 +569,117 @@ def test_missing_font_warning_explains_consequences(font_dir, caplog):
|
|||||||
# the text stays searchable but renders blank when highlighted.
|
# the text stays searchable but renders blank when highlighted.
|
||||||
assert 'searchable' in msg.lower()
|
assert 'searchable' in msg.lower()
|
||||||
assert 'highlight' in msg.lower() or 'select' in msg.lower()
|
assert 'highlight' in msg.lower() or 'select' in msg.lower()
|
||||||
|
|
||||||
|
|
||||||
|
# --- Coverage-driven last-resort font search (#1722) ---
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeProviderWithSearch(_FakeFontProvider):
|
||||||
|
"""FontProvider that can also search unlisted fonts by glyph coverage."""
|
||||||
|
|
||||||
|
def __init__(self, fonts, searchable):
|
||||||
|
super().__init__(fonts)
|
||||||
|
self._searchable = searchable
|
||||||
|
self.search_calls: list[str] = []
|
||||||
|
|
||||||
|
def find_font_with_glyphs(self, text):
|
||||||
|
self.search_calls.append(text)
|
||||||
|
for name, font in self._searchable.items():
|
||||||
|
hb = font.get_hb_font()
|
||||||
|
if all(hb.get_nominal_glyph(ord(c)) for c in text):
|
||||||
|
self._fonts[name] = font # discovered fonts become resolvable
|
||||||
|
return name, font
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def test_unlisted_font_found_by_coverage_search():
|
||||||
|
"""A script outside FALLBACK_FONTS is rendered if the font is installed."""
|
||||||
|
searchable = {'NotoSansCherokee-Regular': _FakeFontManager('Cherokee.ttf', 'ᏣᎳᎩ')}
|
||||||
|
provider = _FakeProviderWithSearch({}, searchable)
|
||||||
|
manager = MultiFontManager(font_provider=provider)
|
||||||
|
|
||||||
|
font = manager.select_font_for_word('ᏣᎳᎩ', None)
|
||||||
|
|
||||||
|
assert font.font_path.name == 'Cherokee.ttf'
|
||||||
|
assert provider.search_calls == ['ᏣᎳᎩ']
|
||||||
|
|
||||||
|
|
||||||
|
def test_coverage_search_result_is_cached():
|
||||||
|
"""The expensive coverage search runs once per distinct word."""
|
||||||
|
searchable = {'NotoSansCherokee-Regular': _FakeFontManager('Cherokee.ttf', 'ᏣᎳᎩ')}
|
||||||
|
provider = _FakeProviderWithSearch({}, searchable)
|
||||||
|
manager = MultiFontManager(font_provider=provider)
|
||||||
|
|
||||||
|
manager.select_font_for_word('ᏣᎳᎩ', None)
|
||||||
|
manager.select_font_for_word('ᏣᎳᎩ', None)
|
||||||
|
|
||||||
|
assert len(provider.search_calls) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_named_fonts_take_precedence_over_coverage_search():
|
||||||
|
"""The coverage search is a last resort, not a substitute for named fonts."""
|
||||||
|
fonts = {'NotoSans-Regular': _FakeFontManager('NotoSans.ttf', 'abc')}
|
||||||
|
searchable = {'NotoSansMono-Regular': _FakeFontManager('NotoSansMono.ttf', 'abc')}
|
||||||
|
provider = _FakeProviderWithSearch(fonts, searchable)
|
||||||
|
manager = MultiFontManager(font_provider=provider)
|
||||||
|
|
||||||
|
font = manager.select_font_for_word('abc', None)
|
||||||
|
|
||||||
|
assert font.font_path.name == 'NotoSans.ttf'
|
||||||
|
assert provider.search_calls == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_provider_without_coverage_search_still_falls_back():
|
||||||
|
"""Providers predating find_font_with_glyphs() keep working (duck-typed)."""
|
||||||
|
manager = MultiFontManager(font_provider=_FakeFontProvider({}))
|
||||||
|
font = manager.select_font_for_word('ᏣᎳᎩ', None)
|
||||||
|
assert font.font_path.name == 'Occulta.ttf'
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_font_warning_names_the_missing_characters(font_dir, caplog):
|
||||||
|
"""The warning must identify what could not be rendered (#1722).
|
||||||
|
|
||||||
|
The user's real question is "which font package do I install?" — naming the
|
||||||
|
offending codepoints and their Unicode names answers it even for scripts
|
||||||
|
OCRmyPDF has no language mapping for.
|
||||||
|
"""
|
||||||
|
manager = MultiFontManager(font_provider=BuiltinFontProvider(font_dir))
|
||||||
|
|
||||||
|
with caplog.at_level(logging.WARNING):
|
||||||
|
manager.select_font_for_word('ᏣᎳᎩ', None)
|
||||||
|
|
||||||
|
msg = caplog.text
|
||||||
|
assert 'U+13E3' in msg # CHEROKEE LETTER TSA
|
||||||
|
assert 'CHEROKEE' in msg.upper()
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_character_warning_ignores_covered_characters(font_dir, caplog):
|
||||||
|
"""Only the uncoverable characters are reported, not the whole word."""
|
||||||
|
manager = MultiFontManager(font_provider=BuiltinFontProvider(font_dir))
|
||||||
|
|
||||||
|
with caplog.at_level(logging.WARNING):
|
||||||
|
manager.select_font_for_word('aᏣb', None)
|
||||||
|
|
||||||
|
msg = caplog.text
|
||||||
|
assert 'U+13E3' in msg
|
||||||
|
assert 'U+0061' not in msg # 'a' is covered by the builtin Latin font
|
||||||
|
|
||||||
|
|
||||||
|
def test_mixed_script_word_warning_does_not_advise_installing_fonts(caplog):
|
||||||
|
"""A word no single font covers is reported as such, not as a missing font.
|
||||||
|
|
||||||
|
Every character here has an installed font; telling the user to install
|
||||||
|
more would be wrong advice and is exactly the confusion behind #1722.
|
||||||
|
"""
|
||||||
|
fonts = {'NotoSans-Regular': _FakeFontManager('NotoSans.ttf', 'ab')}
|
||||||
|
searchable = {'NotoSansCherokee-Regular': _FakeFontManager('Cherokee.ttf', 'Ꮳ')}
|
||||||
|
manager = MultiFontManager(font_provider=_FakeProviderWithSearch(fonts, searchable))
|
||||||
|
|
||||||
|
with caplog.at_level(logging.WARNING):
|
||||||
|
font = manager.select_font_for_word('aᏣb', None)
|
||||||
|
|
||||||
|
assert font.font_path.name == 'Occulta.ttf'
|
||||||
|
msg = caplog.text
|
||||||
|
assert 'mixing scripts' in msg
|
||||||
|
assert 'will not help' in msg
|
||||||
|
assert 'U+' not in msg # no codepoints to blame; nothing to install
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ from PIL import Image, ImageDraw
|
|||||||
from ocrmypdf import optimize as opt
|
from ocrmypdf import optimize as opt
|
||||||
from ocrmypdf._exec import jbig2enc, pngquant
|
from ocrmypdf._exec import jbig2enc, pngquant
|
||||||
from ocrmypdf._exec.ghostscript import rasterize_pdf
|
from ocrmypdf._exec.ghostscript import rasterize_pdf
|
||||||
|
from ocrmypdf.cli import get_options_and_plugins
|
||||||
from ocrmypdf.helpers import IMG2PDF_KWARGS, Resolution
|
from ocrmypdf.helpers import IMG2PDF_KWARGS, Resolution
|
||||||
from ocrmypdf.optimize import PdfImage, extract_image_filter
|
from ocrmypdf.optimize import PdfImage, extract_image_filter
|
||||||
from ocrmypdf.pluginspec import GhostscriptRasterDevice
|
from ocrmypdf.pluginspec import GhostscriptRasterDevice
|
||||||
@@ -81,6 +82,28 @@ def test_jpg_png_params(resources, outpdf):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_jpeg_quality_cli_flag_reaches_options(resources, outpdf):
|
||||||
|
# Regression test for #1723: --jpeg-quality was silently dropped by
|
||||||
|
# namespace_to_options() because the argparse dest ('jpeg_quality') did
|
||||||
|
# not match the OcrOptions field it was checked against.
|
||||||
|
input_ = fspath(resources / 'c02-22.pdf')
|
||||||
|
options, _pm = get_options_and_plugins(
|
||||||
|
['--jpeg-quality', '10', input_, fspath(outpdf)]
|
||||||
|
)
|
||||||
|
assert options.jpeg_quality == 10
|
||||||
|
|
||||||
|
|
||||||
|
def test_jpg_quality_cli_alias_reaches_options(resources, outpdf):
|
||||||
|
# --jpg-quality is a hidden alias for --jpeg-quality (same argparse dest).
|
||||||
|
input_ = fspath(resources / 'c02-22.pdf')
|
||||||
|
options, _pm = get_options_and_plugins(
|
||||||
|
['--jpg-quality', '42', input_, fspath(outpdf)]
|
||||||
|
)
|
||||||
|
assert options.jpeg_quality == 42
|
||||||
|
# The old field name is still readable as a deprecated compatibility alias.
|
||||||
|
assert options.jpg_quality == 42
|
||||||
|
|
||||||
|
|
||||||
@needs_jbig2enc
|
@needs_jbig2enc
|
||||||
def test_jbig2_lossless(resources, outpdf):
|
def test_jbig2_lossless(resources, outpdf):
|
||||||
"""Test that JBIG2 lossless encoding works without JBIG2Globals."""
|
"""Test that JBIG2 lossless encoding works without JBIG2Globals."""
|
||||||
|
|||||||
@@ -94,6 +94,52 @@ class TestFindNonembeddedCidFonts:
|
|||||||
with pikepdf.open(path) as pdf:
|
with pikepdf.open(path) as pdf:
|
||||||
assert find_nonembedded_cid_fonts(pdf) == {'ZZZ+Hidden'}
|
assert find_nonembedded_cid_fonts(pdf) == {'ZZZ+Hidden'}
|
||||||
|
|
||||||
|
def test_non_dictionary_font_and_xobject_resources_are_ignored(self, tmp_path):
|
||||||
|
# A malformed PDF may carry a /Font or /XObject resource that is not a
|
||||||
|
# dictionary (an array, a name, an empty value). Scanning must skip it
|
||||||
|
# rather than raise when iterating its values (regression test for the
|
||||||
|
# crash reported in issue #1713).
|
||||||
|
path = tmp_path / 'malformed_resources.pdf'
|
||||||
|
with pikepdf.new() as pdf:
|
||||||
|
page = pdf.add_blank_page()
|
||||||
|
page.Resources = pikepdf.Dictionary(
|
||||||
|
Font=pikepdf.Array([]),
|
||||||
|
XObject=pikepdf.Array([]),
|
||||||
|
)
|
||||||
|
pdf.save(path)
|
||||||
|
with pikepdf.open(path) as pdf:
|
||||||
|
assert find_nonembedded_cid_fonts(pdf) == set()
|
||||||
|
|
||||||
|
def test_non_dictionary_font_descriptor_is_reported(self, tmp_path):
|
||||||
|
# A Type0 font whose descendant carries a non-dictionary /FontDescriptor
|
||||||
|
# has no embedded glyph data, so it must be reported -- not crash. This
|
||||||
|
# is the same malformed-resource bug class as #1713, one level deeper:
|
||||||
|
# `key in descriptor` raises ValueError on a non-dictionary.
|
||||||
|
path = tmp_path / 'bad_descriptor.pdf'
|
||||||
|
with pikepdf.new() as pdf:
|
||||||
|
page = pdf.add_blank_page()
|
||||||
|
cidfont = pdf.make_indirect(
|
||||||
|
pikepdf.Dictionary(
|
||||||
|
Type=Name.Font,
|
||||||
|
Subtype=Name.CIDFontType2,
|
||||||
|
BaseFont=Name('/BOGUS+CID'),
|
||||||
|
FontDescriptor=Name.NotADictionary,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
type0 = pdf.make_indirect(
|
||||||
|
pikepdf.Dictionary(
|
||||||
|
Type=Name.Font,
|
||||||
|
Subtype=Name.Type0,
|
||||||
|
BaseFont=Name('/BOGUS+CID'),
|
||||||
|
Encoding=Name.Identity_H,
|
||||||
|
DescendantFonts=pikepdf.Array([cidfont]),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
page.Resources = pikepdf.Dictionary(Font=pikepdf.Dictionary(F0=type0))
|
||||||
|
pdf.save(path)
|
||||||
|
with pikepdf.open(path) as pdf:
|
||||||
|
assert find_nonembedded_cid_fonts(pdf) == {'BOGUS+CID'}
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def nonembedded_cid_pdf(tmp_path):
|
def nonembedded_cid_pdf(tmp_path):
|
||||||
|
|||||||
@@ -448,6 +448,55 @@ def test_fill_ink_cs_resets_color_to_black():
|
|||||||
assert _ink_of_first_xobject(b"0.8 0.2 0.2 rg /DeviceGray cs /Im0 Do") is Ink.mono
|
assert _ink_of_first_xobject(b"0.8 0.2 0.2 rg /DeviceGray cs /Im0 Do") is Ink.mono
|
||||||
|
|
||||||
|
|
||||||
|
def test_nondict_xobject_tolerated(outdir):
|
||||||
|
# A malformed PDF may store a non-dictionary object (here an Array) at
|
||||||
|
# /Resources /XObject. Scanning for images must tolerate this rather than
|
||||||
|
# crash on .items(); OCRmyPDF's domain is messy machine-generated PDFs.
|
||||||
|
# Same robustness class as the pdfa.py find_nonembedded_cid_fonts fix.
|
||||||
|
pdf = pikepdf.Pdf.new()
|
||||||
|
page = pdf.add_blank_page(page_size=(612, 792))
|
||||||
|
page.Resources = pikepdf.Dictionary(
|
||||||
|
Font=pikepdf.Array([]), XObject=pikepdf.Array([])
|
||||||
|
)
|
||||||
|
out = outdir / 'malformed_xobj.pdf'
|
||||||
|
pdf.save(out)
|
||||||
|
|
||||||
|
info = pdfinfo.PdfInfo(out)
|
||||||
|
assert len(info) == 1
|
||||||
|
assert len(info[0].images) == 0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
'resources',
|
||||||
|
[
|
||||||
|
pikepdf.Array([]), # non-dict /Resources
|
||||||
|
pikepdf.Name.Foo, # non-dict /Resources (name)
|
||||||
|
pikepdf.Dictionary(XObject=pikepdf.Array([])), # non-dict /XObject
|
||||||
|
pikepdf.Dictionary(XObject=pikepdf.Name.Foo), # non-dict /XObject (name)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_image_scanners_tolerate_nondict_resources(resources):
|
||||||
|
# Exercise the image scanners directly on an in-memory container whose
|
||||||
|
# /Resources or /Resources /XObject is not a dictionary. (pikepdf
|
||||||
|
# normalizes a non-dict /Resources assigned to a page on save, so these
|
||||||
|
# cases must be built in memory to reach the scanner unmodified.)
|
||||||
|
from ocrmypdf.pdfinfo._contentstream import ContentsInfo
|
||||||
|
from ocrmypdf.pdfinfo._image import _find_form_xobject_images, _image_xobjects
|
||||||
|
|
||||||
|
container = pikepdf.Dictionary(Type=pikepdf.Name.Page, Resources=resources)
|
||||||
|
empty = ContentsInfo(
|
||||||
|
xobject_settings=[],
|
||||||
|
inline_images=[],
|
||||||
|
found_vector=False,
|
||||||
|
found_text=False,
|
||||||
|
name_index={},
|
||||||
|
)
|
||||||
|
pdf = pikepdf.Pdf.new()
|
||||||
|
|
||||||
|
assert list(_image_xobjects(container)) == []
|
||||||
|
assert list(_find_form_xobject_images(pdf, container, empty)) == []
|
||||||
|
|
||||||
|
|
||||||
def test_imageinfo_ink_inherited_in_form_xobject(outdir):
|
def test_imageinfo_ink_inherited_in_form_xobject(outdir):
|
||||||
# A mask drawn inside a Form XObject inherits the fill color set before the
|
# A mask drawn inside a Form XObject inherits the fill color set before the
|
||||||
# Do that paints the form; the gray classification must reach the mask.
|
# Do that paints the form; the gray classification must reach the mask.
|
||||||
|
|||||||
@@ -363,6 +363,140 @@ class TestSystemFontProviderVariableFonts:
|
|||||||
assert provider.get_font('NotoSansSC-Regular') is None
|
assert provider.get_font('NotoSansSC-Regular') is None
|
||||||
|
|
||||||
|
|
||||||
|
class TestSystemFontProviderUnlistedFamilies:
|
||||||
|
"""Test the coverage-driven search over Noto families we don't enumerate.
|
||||||
|
|
||||||
|
``NOTO_FONT_PATTERNS`` names only the couple dozen most common scripts, but
|
||||||
|
macOS ships ~100 script-specific Noto fonts and Homebrew/Linux distros offer
|
||||||
|
even more. Those fonts must still be usable when the enumerated families
|
||||||
|
cannot cover the text. See issue #1722.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def real_font_bytes(self):
|
||||||
|
"""Bytes of a real, loadable font covering ASCII."""
|
||||||
|
font_path = (
|
||||||
|
Path(__file__).parent.parent
|
||||||
|
/ "src"
|
||||||
|
/ "ocrmypdf"
|
||||||
|
/ "data"
|
||||||
|
/ "NotoSans-Regular.ttf"
|
||||||
|
)
|
||||||
|
if not font_path.exists():
|
||||||
|
pytest.skip("Builtin font not available")
|
||||||
|
return font_path.read_bytes()
|
||||||
|
|
||||||
|
def _provider_for(self, tmp_path, filenames, real_font_bytes):
|
||||||
|
"""Build a provider whose only font dir is tmp_path with given files."""
|
||||||
|
for name in filenames:
|
||||||
|
(tmp_path / name).write_bytes(real_font_bytes)
|
||||||
|
provider = SystemFontProvider()
|
||||||
|
provider._font_dirs = [tmp_path]
|
||||||
|
return provider
|
||||||
|
|
||||||
|
def test_finds_unlisted_family_by_coverage(self, tmp_path, real_font_bytes):
|
||||||
|
"""A Noto family absent from NOTO_FONT_PATTERNS is still usable."""
|
||||||
|
provider = self._provider_for(
|
||||||
|
tmp_path, ['NotoSansCherokee-Regular.ttf'], real_font_bytes
|
||||||
|
)
|
||||||
|
assert 'NotoSansCherokee-Regular' not in provider.NOTO_FONT_PATTERNS
|
||||||
|
found = provider.find_font_with_glyphs('A')
|
||||||
|
assert found is not None
|
||||||
|
name, font = found
|
||||||
|
assert name == 'NotoSansCherokee-Regular'
|
||||||
|
assert font.font_path.name == 'NotoSansCherokee-Regular.ttf'
|
||||||
|
|
||||||
|
def test_finds_unlisted_variable_family(self, tmp_path, real_font_bytes):
|
||||||
|
"""Bracketed variable filenames are eligible for the coverage search."""
|
||||||
|
provider = self._provider_for(
|
||||||
|
tmp_path, ['NotoSansVithkuqi[wght].ttf'], real_font_bytes
|
||||||
|
)
|
||||||
|
found = provider.find_font_with_glyphs('A')
|
||||||
|
assert found is not None
|
||||||
|
assert found[0] == 'NotoSansVithkuqi-Regular'
|
||||||
|
|
||||||
|
def test_discovered_font_resolves_by_logical_name(self, tmp_path, real_font_bytes):
|
||||||
|
"""A font found by coverage is afterwards reachable via get_font()."""
|
||||||
|
provider = self._provider_for(
|
||||||
|
tmp_path, ['NotoSansCherokee-Regular.ttf'], real_font_bytes
|
||||||
|
)
|
||||||
|
name, font = provider.find_font_with_glyphs('A')
|
||||||
|
assert provider.get_font(name) is font
|
||||||
|
|
||||||
|
def test_negative_cache_does_not_block_discovery(self, tmp_path, real_font_bytes):
|
||||||
|
"""A prior failed get_font() must not hide a later coverage match."""
|
||||||
|
provider = self._provider_for(
|
||||||
|
tmp_path, ['NotoSansCherokee-Regular.ttf'], real_font_bytes
|
||||||
|
)
|
||||||
|
assert provider.get_font('NotoSansCherokee-Regular') is None # not listed
|
||||||
|
name, font = provider.find_font_with_glyphs('A')
|
||||||
|
assert provider.get_font(name) is font
|
||||||
|
|
||||||
|
def test_skips_bold_and_italic_styles(self, tmp_path, real_font_bytes):
|
||||||
|
"""Only Regular/variable faces are candidates, never Bold or Italic."""
|
||||||
|
provider = self._provider_for(
|
||||||
|
tmp_path,
|
||||||
|
[
|
||||||
|
'NotoSansCherokee-Bold.ttf',
|
||||||
|
'NotoSansCherokee-Italic.ttf',
|
||||||
|
'NotoSans-Italic[wdth,wght].ttf',
|
||||||
|
],
|
||||||
|
real_font_bytes,
|
||||||
|
)
|
||||||
|
assert provider.find_font_with_glyphs('A') is None
|
||||||
|
|
||||||
|
def test_ignores_non_noto_fonts(self, tmp_path, real_font_bytes):
|
||||||
|
"""Non-Noto system fonts are not enlisted by the coverage search."""
|
||||||
|
provider = self._provider_for(
|
||||||
|
tmp_path, ['DejaVuSans.ttf', 'Arial.ttf'], real_font_bytes
|
||||||
|
)
|
||||||
|
assert provider.find_font_with_glyphs('A') is None
|
||||||
|
|
||||||
|
def test_returns_none_when_no_font_covers_text(self, tmp_path, real_font_bytes):
|
||||||
|
"""Text no installed font covers yields no match rather than a wrong one."""
|
||||||
|
provider = self._provider_for(
|
||||||
|
tmp_path, ['NotoSansCherokee-Regular.ttf'], real_font_bytes
|
||||||
|
)
|
||||||
|
# U+13A3 CHEROKEE LETTER O is absent from the Latin font's cmap.
|
||||||
|
assert provider.find_font_with_glyphs('Ꭳ') is None
|
||||||
|
|
||||||
|
def test_empty_text_does_not_match(self, tmp_path, real_font_bytes):
|
||||||
|
"""Empty text has nothing to cover, so no font is claimed for it."""
|
||||||
|
provider = self._provider_for(
|
||||||
|
tmp_path, ['NotoSansCherokee-Regular.ttf'], real_font_bytes
|
||||||
|
)
|
||||||
|
assert provider.find_font_with_glyphs('') is None
|
||||||
|
|
||||||
|
def test_unloadable_font_file_is_skipped(self, tmp_path, real_font_bytes):
|
||||||
|
"""A corrupt font file does not abort the search for a usable one."""
|
||||||
|
(tmp_path / 'NotoSansBroken-Regular.ttf').write_bytes(b'not a font')
|
||||||
|
provider = self._provider_for(
|
||||||
|
tmp_path, ['NotoSansCherokee-Regular.ttf'], real_font_bytes
|
||||||
|
)
|
||||||
|
found = provider.find_font_with_glyphs('A')
|
||||||
|
assert found is not None
|
||||||
|
assert found[0] == 'NotoSansCherokee-Regular'
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
'stem,expected',
|
||||||
|
[
|
||||||
|
('NotoSansCherokee-Regular', 'NotoSansCherokee'),
|
||||||
|
('NotoSansCherokee[wght]', 'NotoSansCherokee'),
|
||||||
|
('NotoSansArabic[wdth,wght]', 'NotoSansArabic'),
|
||||||
|
('NotoSansCJKsc-VF', 'NotoSansCJKsc'),
|
||||||
|
('NotoMusic', 'NotoMusic'),
|
||||||
|
('NotoSansCJK-Regular', 'NotoSansCJK'),
|
||||||
|
('NotoSans-Bold', None),
|
||||||
|
('NotoSans-Italic[wdth,wght]', None),
|
||||||
|
('NotoSans-SemiCondensedBlackItalic', None),
|
||||||
|
('DejaVuSans-Regular', None),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_family_base_parsing(self, stem, expected):
|
||||||
|
"""Filename stems map to family bases, rejecting non-Regular styles."""
|
||||||
|
assert SystemFontProvider._family_base(stem) == expected
|
||||||
|
|
||||||
|
|
||||||
# --- ChainedFontProvider Tests ---
|
# --- ChainedFontProvider Tests ---
|
||||||
|
|
||||||
|
|
||||||
@@ -507,3 +641,49 @@ class TestChainedFontProviderIntegration:
|
|||||||
|
|
||||||
# Chain should have at least as many fonts as builtin
|
# Chain should have at least as many fonts as builtin
|
||||||
assert chain_fonts >= builtin_fonts
|
assert chain_fonts >= builtin_fonts
|
||||||
|
|
||||||
|
|
||||||
|
class TestChainedFontProviderCoverageSearch:
|
||||||
|
"""Test that the chain delegates the coverage search to its members."""
|
||||||
|
|
||||||
|
class _Searchable:
|
||||||
|
"""Provider stub that reports one findable font."""
|
||||||
|
|
||||||
|
def __init__(self, result):
|
||||||
|
self.result = result
|
||||||
|
self.calls = 0
|
||||||
|
|
||||||
|
def get_font(self, name):
|
||||||
|
return None
|
||||||
|
|
||||||
|
def get_available_fonts(self):
|
||||||
|
return []
|
||||||
|
|
||||||
|
def get_fallback_font(self):
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def find_font_with_glyphs(self, text):
|
||||||
|
self.calls += 1
|
||||||
|
return self.result
|
||||||
|
|
||||||
|
def test_delegates_to_first_provider_that_finds_a_font(self):
|
||||||
|
"""The first provider with a match wins; later ones are not consulted."""
|
||||||
|
first = self._Searchable(('NotoSansX-Regular', MagicMock()))
|
||||||
|
second = self._Searchable(('NotoSansY-Regular', MagicMock()))
|
||||||
|
chain = ChainedFontProvider([first, second])
|
||||||
|
|
||||||
|
assert chain.find_font_with_glyphs('x')[0] == 'NotoSansX-Regular'
|
||||||
|
assert second.calls == 0
|
||||||
|
|
||||||
|
def test_skips_providers_without_the_capability(self):
|
||||||
|
"""Providers lacking find_font_with_glyphs() are skipped, not fatal."""
|
||||||
|
legacy = MagicMock(spec=['get_font', 'get_available_fonts'])
|
||||||
|
searchable = self._Searchable(('NotoSansX-Regular', MagicMock()))
|
||||||
|
chain = ChainedFontProvider([legacy, searchable])
|
||||||
|
|
||||||
|
assert chain.find_font_with_glyphs('x')[0] == 'NotoSansX-Regular'
|
||||||
|
|
||||||
|
def test_returns_none_when_nothing_matches(self):
|
||||||
|
"""No provider matching yields None so the caller can use Occulta."""
|
||||||
|
chain = ChainedFontProvider([self._Searchable(None)])
|
||||||
|
assert chain.find_font_with_glyphs('x') is None
|
||||||
|
|||||||
@@ -89,8 +89,15 @@ def test_mutex_options():
|
|||||||
make_ocr_opts(redo_ocr=True, force_ocr=True)
|
make_ocr_opts(redo_ocr=True, force_ocr=True)
|
||||||
|
|
||||||
|
|
||||||
def test_optimizing(caplog):
|
def test_optimizing_png_quality_warns(caplog):
|
||||||
vd.check_options(*make_opts_pm(optimize=0, png_quality=18, jpeg_quality=10))
|
vd.check_options(*make_opts_pm(optimize=0, png_quality=18))
|
||||||
|
assert 'will be ignored because' in caplog.text
|
||||||
|
|
||||||
|
|
||||||
|
def test_optimizing_jpeg_quality_warns(caplog):
|
||||||
|
# Isolated from png_quality so this actually exercises the jpeg_quality
|
||||||
|
# path rather than being confounded by png_quality also being set.
|
||||||
|
vd.check_options(*make_opts_pm(optimize=0, jpeg_quality=10))
|
||||||
assert 'will be ignored because' in caplog.text
|
assert 'will be ignored because' in caplog.text
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user