Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
aa6a32e7d1 | ||
|
|
ea99758747 | ||
|
|
4942751a1b | ||
|
|
be06e3184a | ||
|
|
39bf09f1eb | ||
|
|
aaffc46f73 | ||
|
|
0277b3b3ba | ||
|
|
0817542883 | ||
|
|
6f4744dd20 | ||
|
|
5d49f75c56 | ||
|
|
5a824ddd8c | ||
|
|
54bf03a454 | ||
|
|
009754d137 | ||
|
|
f0a3a74374 | ||
|
|
178d339c8e | ||
|
|
d3f8d01227 | ||
|
|
b60df59c62 | ||
|
|
640b3062b2 | ||
|
|
ef903db360 | ||
|
|
9cda02317b | ||
|
|
92a2fe880a | ||
|
|
089f46690a | ||
|
|
e45c40b063 | ||
|
|
bbac5307f2 | ||
|
|
6167783696 | ||
|
|
3d291e72c0 | ||
|
|
efebe9ca2e | ||
|
|
12ec97f732 | ||
|
|
3f1aceade2 | ||
|
|
212b28e602 | ||
|
|
dfdb32995e | ||
|
|
273826377e | ||
|
|
5569d4db07 |
@@ -14,8 +14,29 @@ on:
|
|||||||
pull_request:
|
pull_request:
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
|
lint:
|
||||||
|
name: Lint (prek)
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v7
|
||||||
|
|
||||||
|
- name: Install uv
|
||||||
|
uses: astral-sh/setup-uv@v7
|
||||||
|
with:
|
||||||
|
version: "0.9.x"
|
||||||
|
|
||||||
|
- name: "Set up Python"
|
||||||
|
uses: actions/setup-python@v7
|
||||||
|
with:
|
||||||
|
python-version: "3.11"
|
||||||
|
|
||||||
|
- name: Run prek
|
||||||
|
run: |
|
||||||
|
uv run prek run --all-files
|
||||||
|
|
||||||
test_linux:
|
test_linux:
|
||||||
name: Test ${{ matrix.os }} with Python ${{ matrix.python }}
|
name: Test ${{ matrix.os }} with Python ${{ matrix.python }}
|
||||||
|
needs: lint
|
||||||
runs-on: ${{ matrix.os }}
|
runs-on: ${{ matrix.os }}
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
@@ -39,7 +60,7 @@ jobs:
|
|||||||
version: "0.9.x"
|
version: "0.9.x"
|
||||||
|
|
||||||
- name: "Set up Python"
|
- name: "Set up Python"
|
||||||
uses: actions/setup-python@v6
|
uses: actions/setup-python@v7
|
||||||
with:
|
with:
|
||||||
python-version: ${{ matrix.python }}
|
python-version: ${{ matrix.python }}
|
||||||
|
|
||||||
@@ -96,6 +117,7 @@ jobs:
|
|||||||
|
|
||||||
test_macos:
|
test_macos:
|
||||||
name: Test macOS
|
name: Test macOS
|
||||||
|
needs: lint
|
||||||
runs-on: ${{ matrix.os }}
|
runs-on: ${{ matrix.os }}
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
@@ -129,7 +151,7 @@ jobs:
|
|||||||
version: "0.9.x"
|
version: "0.9.x"
|
||||||
|
|
||||||
- name: "Set up Python"
|
- name: "Set up Python"
|
||||||
uses: actions/setup-python@v6
|
uses: actions/setup-python@v7
|
||||||
with:
|
with:
|
||||||
python-version: ${{ matrix.python }}
|
python-version: ${{ matrix.python }}
|
||||||
|
|
||||||
@@ -158,6 +180,7 @@ jobs:
|
|||||||
|
|
||||||
test_windows:
|
test_windows:
|
||||||
name: Test Windows
|
name: Test Windows
|
||||||
|
needs: lint
|
||||||
runs-on: ${{ matrix.os }}
|
runs-on: ${{ matrix.os }}
|
||||||
strategy:
|
strategy:
|
||||||
matrix:
|
matrix:
|
||||||
@@ -177,7 +200,7 @@ jobs:
|
|||||||
version: "0.9.x"
|
version: "0.9.x"
|
||||||
|
|
||||||
- name: "Set up Python"
|
- name: "Set up Python"
|
||||||
uses: actions/setup-python@v6
|
uses: actions/setup-python@v7
|
||||||
with:
|
with:
|
||||||
python-version: ${{ matrix.python }}
|
python-version: ${{ matrix.python }}
|
||||||
|
|
||||||
@@ -252,6 +275,14 @@ jobs:
|
|||||||
run: |
|
run: |
|
||||||
TAG="v${{ steps.version.outputs.version }}"
|
TAG="v${{ steps.version.outputs.version }}"
|
||||||
|
|
||||||
|
# If release.yml already published this version, _version.py may
|
||||||
|
# still reflect it until the next version bump commit. Don't
|
||||||
|
# re-draft an already-published release on later pushes to main.
|
||||||
|
if [[ "$(gh release view "$TAG" --json isDraft --jq .isDraft 2>/dev/null)" == "false" ]]; then
|
||||||
|
echo "Release $TAG is already published; skipping."
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
# Delete existing draft release if it exists (ignore errors)
|
# Delete existing draft release if it exists (ignore errors)
|
||||||
gh release delete "$TAG" --yes 2>/dev/null || true
|
gh release delete "$TAG" --yes 2>/dev/null || true
|
||||||
|
|
||||||
|
|||||||
@@ -1,27 +0,0 @@
|
|||||||
# SPDX-FileCopyrightText: 2022 James R. Barlow
|
|
||||||
# SPDX-License-Identifier: MPL-2.0
|
|
||||||
|
|
||||||
repos:
|
|
||||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
|
||||||
rev: v4.4.0
|
|
||||||
hooks:
|
|
||||||
- id: check-case-conflict
|
|
||||||
- id: check-merge-conflict
|
|
||||||
- id: check-toml
|
|
||||||
- id: check-yaml
|
|
||||||
- id: debug-statements
|
|
||||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
||||||
rev: "v0.14.11"
|
|
||||||
hooks:
|
|
||||||
- id: ruff-check
|
|
||||||
args: [--fix]
|
|
||||||
- id: ruff-format
|
|
||||||
- repo: https://github.com/pre-commit/mirrors-mypy
|
|
||||||
rev: v1.2.0
|
|
||||||
hooks:
|
|
||||||
- id: mypy
|
|
||||||
additional_dependencies:
|
|
||||||
- types-toml
|
|
||||||
- types-setuptools
|
|
||||||
- types-requests
|
|
||||||
- types-Pillow
|
|
||||||
+13
-3
@@ -6,7 +6,6 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import glob
|
|
||||||
import os
|
import os
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
@@ -282,7 +281,7 @@ def bump_version() -> None:
|
|||||||
actions = []
|
actions = []
|
||||||
|
|
||||||
for path_pattern, version_pattern in config:
|
for path_pattern, version_pattern in config:
|
||||||
paths = [Path(p) for p in glob.glob(path_pattern)]
|
paths = list(Path().glob(path_pattern))
|
||||||
|
|
||||||
if not paths:
|
if not paths:
|
||||||
print(f"error: Pattern {path_pattern} didn't match any files")
|
print(f"error: Pattern {path_pattern} didn't match any files")
|
||||||
@@ -306,7 +305,8 @@ def bump_version() -> None:
|
|||||||
|
|
||||||
if not found_at_least_one_file_needing_update:
|
if not found_at_least_one_file_needing_update:
|
||||||
print(
|
print(
|
||||||
f'''error: Didn't find any occurrences of "{find_pattern}" in "{path_pattern}"'''
|
f'''error: Didn't find any occurrences of "{find_pattern}" '''
|
||||||
|
f'''in "{path_pattern}"'''
|
||||||
)
|
)
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
@@ -331,6 +331,16 @@ def bump_version() -> None:
|
|||||||
contents = contents.replace(find, replace)
|
contents = contents.replace(find, replace)
|
||||||
path.write_text(contents, encoding="utf8")
|
path.write_text(contents, encoding="utf8")
|
||||||
|
|
||||||
|
# Format only after every file (including pyproject.toml) reflects the new
|
||||||
|
# version. Running `uv run` while pyproject.toml still had the old version
|
||||||
|
# would leave its post-bump environment/lockfile resync to happen for the
|
||||||
|
# first time during the commit's pre-commit hooks instead of here, which
|
||||||
|
# then aborts the commit with a spurious "files were modified by this
|
||||||
|
# hook" error.
|
||||||
|
for path, _find, _replace in actions:
|
||||||
|
if path.suffix == ".py":
|
||||||
|
subprocess.run(["uv", "run", "ruff", "format", str(path)], check=True)
|
||||||
|
|
||||||
print("Files updated.")
|
print("Files updated.")
|
||||||
print()
|
print()
|
||||||
|
|
||||||
|
|||||||
@@ -95,10 +95,12 @@ from multiprocessing import Process
|
|||||||
import ocrmypdf
|
import ocrmypdf
|
||||||
from ocrmypdf import OcrOptions
|
from ocrmypdf import OcrOptions
|
||||||
|
|
||||||
|
|
||||||
def ocrmypdf_process():
|
def ocrmypdf_process():
|
||||||
options = OcrOptions(input_file='input.pdf', output_file='output.pdf')
|
options = OcrOptions(input_file='input.pdf', output_file='output.pdf')
|
||||||
ocrmypdf.ocr(options)
|
ocrmypdf.ocr(options)
|
||||||
|
|
||||||
|
|
||||||
def call_ocrmypdf_from_my_app():
|
def call_ocrmypdf_from_my_app():
|
||||||
p = Process(target=ocrmypdf_process)
|
p = Process(target=ocrmypdf_process)
|
||||||
p.start()
|
p.start()
|
||||||
|
|||||||
+19
-3
@@ -663,9 +663,25 @@ provides text shaping for proper multilingual support. These replace the
|
|||||||
legacy hOCR-based renderer. Install with: `pip install fpdf2 uharfbuzz`
|
legacy hOCR-based renderer. Install with: `pip install fpdf2 uharfbuzz`
|
||||||
|
|
||||||
**fonts-noto** (or an equivalent comprehensive font package) is recommended
|
**fonts-noto** (or an equivalent comprehensive font package) is recommended
|
||||||
for proper text rendering, especially for non-Latin scripts. On Debian/Ubuntu:
|
for proper text rendering, especially for non-Latin scripts. OCRmyPDF bundles
|
||||||
`apt install fonts-noto`. On Fedora: `dnf install google-noto-fonts-common`.
|
a Latin font only, and discovers the rest from the fonts installed on your
|
||||||
On macOS with Homebrew: `brew install font-noto`.
|
system.
|
||||||
|
|
||||||
|
- Debian/Ubuntu: `apt install fonts-noto`
|
||||||
|
- Fedora: `dnf install google-noto-fonts-all`
|
||||||
|
- macOS with Homebrew: Homebrew has no single Noto package; each family is a
|
||||||
|
separate cask. Install at least
|
||||||
|
`brew install --cask font-noto-sans font-noto-serif`, plus a cask per
|
||||||
|
additional script you OCR, for example
|
||||||
|
`brew install --cask font-noto-sans-arabic font-noto-sans-cjk`. Run
|
||||||
|
`brew search font-noto` to list them all.
|
||||||
|
|
||||||
|
If OCRmyPDF warns that no installed font has glyphs for some of the text, the
|
||||||
|
message names the characters it could not render, for example
|
||||||
|
`'Ꮳ' U+13E3 CHEROKEE LETTER TSA`. Install the Noto font for that script — here,
|
||||||
|
`fonts-noto-core` on Debian or `font-noto-sans-cherokee` on Homebrew. The text
|
||||||
|
layer remains searchable and copyable either way; only its appearance when
|
||||||
|
highlighted in a PDF viewer is affected.
|
||||||
|
|
||||||
**pypdfium2**, if present, provides fast PDF page rasterization using
|
**pypdfium2**, if present, provides fast PDF page rasterization using
|
||||||
the pdfium library (the same library used by Google Chrome). It is
|
the pdfium library (the same library used by Google Chrome). It is
|
||||||
|
|||||||
+3
-1
@@ -47,7 +47,9 @@ OCRmyPDF has the following runtime dependencies:
|
|||||||
**For text rendering** (expressing OCR results in PDF):
|
**For text rendering** (expressing OCR results in PDF):
|
||||||
- `fpdf2` (Python package) - Required for text layer rendering
|
- `fpdf2` (Python package) - Required for text layer rendering
|
||||||
- `uharfbuzz` (Python package) - Required for text layer rendering
|
- `uharfbuzz` (Python package) - Required for text layer rendering
|
||||||
- `font-noto` (system package) - Recommended for text layer rendering
|
- Noto fonts (system package) - Recommended for text layer rendering.
|
||||||
|
`fonts-noto` on Debian/Ubuntu, `google-noto-fonts-all` on Fedora; Homebrew
|
||||||
|
has no single Noto package, only per-family casks such as `font-noto-sans`.
|
||||||
|
|
||||||
**Other dependencies**:
|
**Other dependencies**:
|
||||||
- `unpaper` (system binary) - Optional, enables `--clean` and `--clean-final`
|
- `unpaper` (system binary) - Optional, enables `--clean` and `--clean-final`
|
||||||
|
|||||||
+13
-13
@@ -120,6 +120,7 @@ A plugin may provide the following hooks. Hooks must be decorated with
|
|||||||
```python
|
```python
|
||||||
from ocrmypdf import hookimpl
|
from ocrmypdf import hookimpl
|
||||||
|
|
||||||
|
|
||||||
@hookimpl
|
@hookimpl
|
||||||
def add_options(parser):
|
def add_options(parser):
|
||||||
pass
|
pass
|
||||||
@@ -205,12 +206,11 @@ from ocrmypdf._options import OcrOptions
|
|||||||
|
|
||||||
```python
|
```python
|
||||||
# Before (v16 and earlier)
|
# Before (v16 and earlier)
|
||||||
def check_options(options: argparse.Namespace) -> None:
|
def check_options(options: argparse.Namespace) -> None: ...
|
||||||
...
|
|
||||||
|
|
||||||
# After (v17+)
|
# After (v17+)
|
||||||
def check_options(options: OcrOptions) -> None:
|
def check_options(options: OcrOptions) -> None: ...
|
||||||
...
|
|
||||||
```
|
```
|
||||||
|
|
||||||
**Attribute access unchanged:**
|
**Attribute access unchanged:**
|
||||||
@@ -229,6 +229,7 @@ options.tesseract_timeout
|
|||||||
def check_options(options):
|
def check_options(options):
|
||||||
options.some_computed_value = compute_value(options)
|
options.some_computed_value = compute_value(options)
|
||||||
|
|
||||||
|
|
||||||
# After (v17 pattern - compute at point of use)
|
# After (v17 pattern - compute at point of use)
|
||||||
def some_function(options):
|
def some_function(options):
|
||||||
computed = compute_value(options)
|
computed = compute_value(options)
|
||||||
@@ -336,19 +337,17 @@ from ocrmypdf import OcrElement, OcrClass, BoundingBox
|
|||||||
|
|
||||||
# OcrElement - represents any OCR structural unit
|
# OcrElement - represents any OCR structural unit
|
||||||
page = OcrElement(
|
page = OcrElement(
|
||||||
ocr_class=OcrClass.PAGE,
|
ocr_class=OcrClass.PAGE, bbox=BoundingBox(0, 0, 612, 792), children=[...]
|
||||||
bbox=BoundingBox(0, 0, 612, 792),
|
|
||||||
children=[...]
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# BoundingBox - axis-aligned bounding box (left, top, right, bottom)
|
# BoundingBox - axis-aligned bounding box (left, top, right, bottom)
|
||||||
bbox = BoundingBox(left=100, top=50, right=300, bottom=80)
|
bbox = BoundingBox(left=100, top=50, right=300, bottom=80)
|
||||||
|
|
||||||
# OcrClass - constants for element types
|
# OcrClass - constants for element types
|
||||||
OcrClass.PAGE # "ocr_page"
|
OcrClass.PAGE # "ocr_page"
|
||||||
OcrClass.LINE # "ocr_line"
|
OcrClass.LINE # "ocr_line"
|
||||||
OcrClass.WORD # "ocrx_word"
|
OcrClass.WORD # "ocrx_word"
|
||||||
OcrClass.PARAGRAPH # "ocr_par"
|
OcrClass.PARAGRAPH # "ocr_par"
|
||||||
```
|
```
|
||||||
|
|
||||||
**Navigating the tree:**
|
**Navigating the tree:**
|
||||||
@@ -378,6 +377,7 @@ from pathlib import Path
|
|||||||
from ocrmypdf.pluginspec import OcrEngine
|
from ocrmypdf.pluginspec import OcrEngine
|
||||||
from ocrmypdf import OcrElement, OcrClass, BoundingBox
|
from ocrmypdf import OcrElement, OcrClass, BoundingBox
|
||||||
|
|
||||||
|
|
||||||
class MyOcrEngine(OcrEngine):
|
class MyOcrEngine(OcrEngine):
|
||||||
def generate_ocr(
|
def generate_ocr(
|
||||||
self,
|
self,
|
||||||
@@ -402,10 +402,10 @@ class MyOcrEngine(OcrEngine):
|
|||||||
text="Hello",
|
text="Hello",
|
||||||
),
|
),
|
||||||
# ... more words
|
# ... more words
|
||||||
]
|
],
|
||||||
),
|
),
|
||||||
# ... more lines
|
# ... more lines
|
||||||
]
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
def supports_generate_ocr(self) -> bool:
|
def supports_generate_ocr(self) -> bool:
|
||||||
|
|||||||
@@ -3,6 +3,55 @@
|
|||||||
|
|
||||||
# v17
|
# v17
|
||||||
|
|
||||||
|
## v17.9.0
|
||||||
|
|
||||||
|
- OCRmyPDF now uses any Noto font installed on the system, not just the two
|
||||||
|
dozen script families it knows by name ({issue}`1722`). Previously a document
|
||||||
|
in, say, Cherokee or Vai was rendered with the glyphless fallback font even
|
||||||
|
though the matching font was installed — a common situation on macOS, which
|
||||||
|
ships around a hundred script-specific Noto faces. When the named fonts
|
||||||
|
cannot cover a word, OCRmyPDF now searches the installed fonts for one that
|
||||||
|
can.
|
||||||
|
- The "no installed font has glyphs" warning now names the characters it could
|
||||||
|
not render, with their codepoints and Unicode names, so it is clear which
|
||||||
|
font to install. Text that mixes scripts no single font covers is now
|
||||||
|
reported as such, instead of advising the user to install fonts they may
|
||||||
|
already have.
|
||||||
|
- Fixed the macOS font installation instructions, which recommended a Homebrew
|
||||||
|
package (`font-noto`) that does not exist ({issue}`1722`). Homebrew has no
|
||||||
|
single Noto package; each family is a separate cask. The Fedora package name
|
||||||
|
was also corrected to `google-noto-fonts-all`.
|
||||||
|
- Font providers may now implement the optional `GlyphSearchingFontProvider`
|
||||||
|
protocol to participate in coverage-based font search.
|
||||||
|
- Fixed `--jpeg-quality`/`--jpg-quality` having no effect on the CLI: the
|
||||||
|
value was silently dropped before reaching the optimizer, which then
|
||||||
|
always used its own built-in default JPEG quality regardless of what was
|
||||||
|
requested ({issue}`1723`). The same bug affected the Python API's
|
||||||
|
`jpg_quality` parameter. `ocrmypdf.ocr()` now accepts `jpeg_quality`
|
||||||
|
(matching the CLI flag name) as the canonical parameter; `jpg_quality`
|
||||||
|
still works but is deprecated.
|
||||||
|
- Hardened PDF parsing against malformed (non-dictionary) `/Resources`,
|
||||||
|
`/XObject`, and `/FontDescriptor` entries, which previously crashed
|
||||||
|
`ocrmypdf.ocr()` with `AttributeError`/`TypeError`/`ValueError` on
|
||||||
|
otherwise-processable files, both during PDF/A font scanning and general
|
||||||
|
image scanning ({issue}`1713`). Thanks @mvanhorn for the initial fix.
|
||||||
|
- Release process improvements: fixed a CI bug where every push to main
|
||||||
|
after a release was tagged would incorrectly revert the just-published
|
||||||
|
GitHub release back to draft status.
|
||||||
|
|
||||||
|
## v17.8.1
|
||||||
|
|
||||||
|
- Improved the `--tesseract-pagesegmode` help text to point to
|
||||||
|
`tesseract --help-extra`, since Tesseract 5.5.2 moved the page segmentation
|
||||||
|
mode documentation there from `tesseract --help`. Thanks @sokai.
|
||||||
|
- Internal refactoring: completed a project-wide mypy type-checking pass
|
||||||
|
(`--check-untyped-defs` is now enabled, and the mypy pre-commit hook is now
|
||||||
|
blocking rather than advisory), fixing several latent edge-case bugs
|
||||||
|
surfaced along the way.
|
||||||
|
- Release process improvements: migrated from pre-commit to prek for local
|
||||||
|
git hooks, and added a dedicated lint job to CI.
|
||||||
|
- Improved typing strictness for `Path`.
|
||||||
|
|
||||||
## v17.8.0
|
## v17.8.0
|
||||||
|
|
||||||
- `--output-type auto` (the default) again produces PDF/A whenever it can,
|
- `--output-type auto` (the default) again produces PDF/A whenever it can,
|
||||||
|
|||||||
+2
-3
@@ -16,7 +16,6 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import filecmp
|
import filecmp
|
||||||
import logging
|
import logging
|
||||||
import os
|
|
||||||
import posixpath
|
import posixpath
|
||||||
import shutil
|
import shutil
|
||||||
import sys
|
import sys
|
||||||
@@ -39,7 +38,7 @@ script_dir = Path(__file__).parent
|
|||||||
# set archive_dir to a path for backup original documents. Leave empty if not required.
|
# set archive_dir to a path for backup original documents. Leave empty if not required.
|
||||||
archive_dir = "/pdfbak"
|
archive_dir = "/pdfbak"
|
||||||
|
|
||||||
start_dir = Path(sys.argv[1]) if len(sys.argv) > 1 else Path(".")
|
start_dir = Path(sys.argv[1]) if len(sys.argv) > 1 else Path()
|
||||||
|
|
||||||
if len(sys.argv) > 2:
|
if len(sys.argv) > 2:
|
||||||
log_file = Path(sys.argv[2])
|
log_file = Path(sys.argv[2])
|
||||||
@@ -68,7 +67,7 @@ for filename in start_dir.glob("**/*.pdf"):
|
|||||||
try:
|
try:
|
||||||
shutil.copy2(filename, posixpath.dirname(archive_filename))
|
shutil.copy2(filename, posixpath.dirname(archive_filename))
|
||||||
except OSError:
|
except OSError:
|
||||||
os.makedirs(posixpath.dirname(archive_filename))
|
Path(posixpath.dirname(archive_filename)).mkdir(parents=True)
|
||||||
shutil.copy2(filename, posixpath.dirname(archive_filename))
|
shutil.copy2(filename, posixpath.dirname(archive_filename))
|
||||||
try:
|
try:
|
||||||
result = ocrmypdf.ocr(filename, filename, deskew=True)
|
result = ocrmypdf.ocr(filename, filename, deskew=True)
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
# SPDX-License-Identifier: MIT
|
# SPDX-License-Identifier: MIT
|
||||||
|
|
||||||
"""Helper script for bisecting PDFs to find a page with an issue."""
|
"""Helper script for bisecting PDFs to find a page with an issue."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import sys
|
import sys
|
||||||
|
|||||||
@@ -37,8 +37,8 @@ def do_column(label, suffix, d):
|
|||||||
env[k] = v
|
env[k] = v
|
||||||
args = shlex.split(
|
args = shlex.split(
|
||||||
cli.format(
|
cli.format(
|
||||||
in_=os.path.join(d, "input.pdf"),
|
in_=Path(d) / "input.pdf",
|
||||||
out=os.path.join(d, f"output{suffix}.pdf"),
|
out=Path(d) / f"output{suffix}.pdf",
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
with st.expander("Environment variables", expanded=bool(env_text.strip())):
|
with st.expander("Environment variables", expanded=bool(env_text.strip())):
|
||||||
@@ -106,10 +106,10 @@ def main():
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
doc1 = pymupdf.open(os.path.join(d, "output1.pdf"))
|
doc1 = pymupdf.open(Path(d, "output1.pdf"))
|
||||||
doc2 = pymupdf.open(os.path.join(d, "output2.pdf"))
|
doc2 = pymupdf.open(Path(d, "output2.pdf"))
|
||||||
for i, page1_2 in enumerate(zip(doc1, doc2, strict=False)):
|
for i, page1_2 in enumerate(zip(doc1, doc2, strict=False)):
|
||||||
st.write(f"Page {i+1}")
|
st.write(f"Page {i + 1}")
|
||||||
page1, page2 = page1_2
|
page1, page2 = page1_2
|
||||||
col1, col2 = st.columns(2)
|
col1, col2 = st.columns(2)
|
||||||
with col1, st.container(border=True):
|
with col1, st.container(border=True):
|
||||||
|
|||||||
+3
-4
@@ -5,7 +5,6 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import os
|
|
||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from tempfile import TemporaryDirectory
|
from tempfile import TemporaryDirectory
|
||||||
@@ -60,10 +59,10 @@ def main():
|
|||||||
Path(d, "2.pdf").write_bytes(pdf_bytes2)
|
Path(d, "2.pdf").write_bytes(pdf_bytes2)
|
||||||
|
|
||||||
with st.expander("Text"):
|
with st.expander("Text"):
|
||||||
doc1 = pymupdf.open(os.path.join(d, "1.pdf"))
|
doc1 = pymupdf.open(Path(d, "1.pdf"))
|
||||||
doc2 = pymupdf.open(os.path.join(d, "2.pdf"))
|
doc2 = pymupdf.open(Path(d, "2.pdf"))
|
||||||
for i, page1_2 in enumerate(zip(doc1, doc2, strict=False)):
|
for i, page1_2 in enumerate(zip(doc1, doc2, strict=False)):
|
||||||
st.write(f"Page {i+1}")
|
st.write(f"Page {i + 1}")
|
||||||
page1, page2 = page1_2
|
page1, page2 = page1_2
|
||||||
col1, col2 = st.columns(2)
|
col1, col2 = st.columns(2)
|
||||||
with col1, st.container(border=True):
|
with col1, st.container(border=True):
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ def main(
|
|||||||
engine: Annotated[str, cyclopts.Parameter()] = 'pdftotext',
|
engine: Annotated[str, cyclopts.Parameter()] = 'pdftotext',
|
||||||
):
|
):
|
||||||
"""Compare text in PDFs."""
|
"""Compare text in PDFs."""
|
||||||
with open(pdf1, 'rb') as f1, open(pdf2, 'rb') as f2:
|
with pdf1.open('rb') as f1, pdf2.open('rb') as f2:
|
||||||
text1 = run(
|
text1 = run(
|
||||||
['pdftotext', '-layout', '-', '-'],
|
['pdftotext', '-layout', '-', '-'],
|
||||||
stdin=f1,
|
stdin=f1,
|
||||||
|
|||||||
+10
-9
@@ -13,13 +13,14 @@ import shutil
|
|||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
# pylint: disable=logging-format-interpolation
|
# pylint: disable=logging-format-interpolation
|
||||||
# pylint: disable=logging-not-lazy
|
# pylint: disable=logging-not-lazy
|
||||||
|
|
||||||
script_dir = os.path.dirname(os.path.realpath(__file__))
|
script_dir = Path(os.path.realpath(__file__)).parent
|
||||||
timestamp = time.strftime("%Y-%m-%d-%H%M_")
|
timestamp = time.strftime("%Y-%m-%d-%H%M_")
|
||||||
log_file = script_dir + '/' + timestamp + 'ocrmypdf.log'
|
log_file = script_dir / (timestamp + 'ocrmypdf.log')
|
||||||
logging.basicConfig(
|
logging.basicConfig(
|
||||||
level=logging.INFO,
|
level=logging.INFO,
|
||||||
format='%(asctime)s %(message)s',
|
format='%(asctime)s %(message)s',
|
||||||
@@ -33,10 +34,10 @@ for dir_name, _subdirs, file_list in os.walk(start_dir):
|
|||||||
logging.info(dir_name)
|
logging.info(dir_name)
|
||||||
os.chdir(dir_name)
|
os.chdir(dir_name)
|
||||||
for filename in file_list:
|
for filename in file_list:
|
||||||
file_stem, file_ext = os.path.splitext(filename)
|
file_stem, file_ext = Path(filename).stem, Path(filename).suffix
|
||||||
if file_ext != '.pdf':
|
if file_ext != '.pdf':
|
||||||
continue
|
continue
|
||||||
full_path = os.path.join(dir_name, filename)
|
full_path = Path(dir_name, filename)
|
||||||
timestamp_ocr = time.strftime("%Y-%m-%d-%H%M_OCR_")
|
timestamp_ocr = time.strftime("%Y-%m-%d-%H%M_OCR_")
|
||||||
filename_ocr = timestamp_ocr + file_stem + '.pdf'
|
filename_ocr = timestamp_ocr + file_stem + '.pdf'
|
||||||
# create string for pdf processing
|
# create string for pdf processing
|
||||||
@@ -52,10 +53,10 @@ for dir_name, _subdirs, file_list in os.walk(start_dir):
|
|||||||
'-',
|
'-',
|
||||||
]
|
]
|
||||||
logging.info(cmd)
|
logging.info(cmd)
|
||||||
full_path_ocr = os.path.join(dir_name, filename_ocr)
|
full_path_ocr = Path(dir_name, filename_ocr)
|
||||||
with (
|
with (
|
||||||
open(filename, 'rb') as input_file,
|
Path(filename).open('rb') as input_file,
|
||||||
open(full_path_ocr, 'wb') as output_file,
|
full_path_ocr.open('wb') as output_file,
|
||||||
):
|
):
|
||||||
proc = subprocess.run(
|
proc = subprocess.run(
|
||||||
cmd,
|
cmd,
|
||||||
@@ -67,8 +68,8 @@ for dir_name, _subdirs, file_list in os.walk(start_dir):
|
|||||||
errors='ignore',
|
errors='ignore',
|
||||||
)
|
)
|
||||||
logging.info(proc.stderr)
|
logging.info(proc.stderr)
|
||||||
os.chmod(full_path_ocr, 0o664)
|
full_path_ocr.chmod(0o664)
|
||||||
os.chmod(full_path, 0o664)
|
full_path.chmod(0o664)
|
||||||
full_path_ocr_archive = sys.argv[2]
|
full_path_ocr_archive = sys.argv[2]
|
||||||
full_path_archive = sys.argv[2] + '/no_ocr'
|
full_path_archive = sys.argv[2] + '/no_ocr'
|
||||||
shutil.move(full_path_ocr, full_path_ocr_archive)
|
shutil.move(full_path_ocr, full_path_ocr_archive)
|
||||||
|
|||||||
+2
-2
@@ -13,7 +13,7 @@ import logging
|
|||||||
import shutil
|
import shutil
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
from enum import Enum
|
from enum import StrEnum
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Annotated, Any
|
from typing import Annotated, Any
|
||||||
|
|
||||||
@@ -35,7 +35,7 @@ app = cyclopts.App(name="ocrmypdf-watcher")
|
|||||||
log = logging.getLogger('ocrmypdf-watcher')
|
log = logging.getLogger('ocrmypdf-watcher')
|
||||||
|
|
||||||
|
|
||||||
class LoggingLevelEnum(str, Enum):
|
class LoggingLevelEnum(StrEnum):
|
||||||
"""Enum for logging levels."""
|
"""Enum for logging levels."""
|
||||||
|
|
||||||
DEBUG = "DEBUG"
|
DEBUG = "DEBUG"
|
||||||
|
|||||||
@@ -0,0 +1,64 @@
|
|||||||
|
# prek pre-commit configuration — https://prek.j178.dev
|
||||||
|
#
|
||||||
|
# The local/system hooks below invoke the project's OWN pinned tools (ruff/mypy
|
||||||
|
# from uv.lock) and mirror .github/workflows/build.yml's lint job exactly, so
|
||||||
|
# they can never drift from CI's versions or rules. prek installs nothing of
|
||||||
|
# its own for them — "system" language just execs whatever `uv run` resolves.
|
||||||
|
#
|
||||||
|
# The pre-commit/pre-commit-hooks repo hooks below are generic file checks with
|
||||||
|
# no project-local tool equivalent, so they're kept as a normal (non-local) repo.
|
||||||
|
#
|
||||||
|
# Run all checks manually: `uv run prek run --all-files`
|
||||||
|
# Install the git hooks: `uv run prek install`
|
||||||
|
|
||||||
|
default_install_hook_types = ["pre-commit", "pre-push"]
|
||||||
|
default_stages = ["pre-commit"]
|
||||||
|
|
||||||
|
[[repos]]
|
||||||
|
repo = "https://github.com/pre-commit/pre-commit-hooks"
|
||||||
|
rev = "v4.4.0"
|
||||||
|
|
||||||
|
[[repos.hooks]]
|
||||||
|
id = "check-case-conflict"
|
||||||
|
|
||||||
|
[[repos.hooks]]
|
||||||
|
id = "check-merge-conflict"
|
||||||
|
|
||||||
|
[[repos.hooks]]
|
||||||
|
id = "check-toml"
|
||||||
|
|
||||||
|
[[repos.hooks]]
|
||||||
|
id = "check-yaml"
|
||||||
|
|
||||||
|
[[repos.hooks]]
|
||||||
|
id = "debug-statements"
|
||||||
|
|
||||||
|
[[repos]]
|
||||||
|
repo = "local"
|
||||||
|
|
||||||
|
[[repos.hooks]]
|
||||||
|
id = "ruff-format"
|
||||||
|
name = "ruff format (check)"
|
||||||
|
language = "system"
|
||||||
|
entry = "uv run ruff format --check ."
|
||||||
|
types = ["python"]
|
||||||
|
pass_filenames = false
|
||||||
|
require_serial = true
|
||||||
|
|
||||||
|
[[repos.hooks]]
|
||||||
|
id = "ruff-check"
|
||||||
|
name = "ruff check"
|
||||||
|
language = "system"
|
||||||
|
entry = "uv run ruff check ."
|
||||||
|
types = ["python"]
|
||||||
|
pass_filenames = false
|
||||||
|
require_serial = true
|
||||||
|
|
||||||
|
[[repos.hooks]]
|
||||||
|
id = "mypy"
|
||||||
|
name = "mypy"
|
||||||
|
language = "system"
|
||||||
|
entry = "uv run mypy src/ocrmypdf"
|
||||||
|
types = ["python"]
|
||||||
|
pass_filenames = false
|
||||||
|
require_serial = true
|
||||||
+16
-4
@@ -6,13 +6,12 @@ build-backend = "hatchling.build"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "ocrmypdf"
|
name = "ocrmypdf"
|
||||||
version = "17.8.0"
|
version = "17.9.0"
|
||||||
description = "OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched"
|
description = "OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
license = "MPL-2.0"
|
license = "MPL-2.0"
|
||||||
requires-python = ">=3.11"
|
requires-python = ">=3.11"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"deprecation>=2.1.0",
|
|
||||||
"fpdf2>=2.8.0",
|
"fpdf2>=2.8.0",
|
||||||
"img2pdf>=0.5",
|
"img2pdf>=0.5",
|
||||||
"packaging>=20",
|
"packaging>=20",
|
||||||
@@ -24,6 +23,7 @@ dependencies = [
|
|||||||
"pydantic>=2.12.5",
|
"pydantic>=2.12.5",
|
||||||
"pypdfium2>=5.0.0",
|
"pypdfium2>=5.0.0",
|
||||||
"rich>=13",
|
"rich>=13",
|
||||||
|
"typing-extensions>=4.12; python_version < '3.13'",
|
||||||
"uharfbuzz>=0.53.2",
|
"uharfbuzz>=0.53.2",
|
||||||
]
|
]
|
||||||
authors = [{ name = "James R. Barlow", email = "james@purplerock.ca" }]
|
authors = [{ name = "James R. Barlow", email = "james@purplerock.ca" }]
|
||||||
@@ -98,18 +98,28 @@ filterwarnings = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
[tool.mypy]
|
[tool.mypy]
|
||||||
|
check_untyped_defs = true
|
||||||
|
|
||||||
[[tool.mypy.overrides]]
|
[[tool.mypy.overrides]]
|
||||||
module = [
|
module = [
|
||||||
'pluggy',
|
|
||||||
'img2pdf',
|
'img2pdf',
|
||||||
'pdfminer.*',
|
'pdfminer.*',
|
||||||
'reportlab.*',
|
'reportlab.*',
|
||||||
'fitz',
|
'fitz',
|
||||||
'libxmp.utils',
|
'libxmp.utils',
|
||||||
|
'pypdfium2',
|
||||||
|
'uharfbuzz',
|
||||||
|
'pi_heif',
|
||||||
]
|
]
|
||||||
ignore_missing_imports = true
|
ignore_missing_imports = true
|
||||||
|
|
||||||
|
[[tool.mypy.overrides]]
|
||||||
|
# Test functions are not required to annotate their return type (almost
|
||||||
|
# always None); it's a low-value hint that would just be noise here.
|
||||||
|
module = 'tests.*'
|
||||||
|
disallow_untyped_defs = false
|
||||||
|
disallow_incomplete_defs = false
|
||||||
|
|
||||||
[tool.ruff]
|
[tool.ruff]
|
||||||
target-version = "py311"
|
target-version = "py311"
|
||||||
exclude = ["src/ocrmypdf/_version.py"] # Autogenerated
|
exclude = ["src/ocrmypdf/_version.py"] # Autogenerated
|
||||||
@@ -125,6 +135,7 @@ exclude = ["src/ocrmypdf/_version.py"] # Autogenerated
|
|||||||
"SIM", # simplify
|
"SIM", # simplify
|
||||||
"B", # flake8-bugbear
|
"B", # flake8-bugbear
|
||||||
"ICN", # flake8-import-conventions
|
"ICN", # flake8-import-conventions
|
||||||
|
"PTH", # flake8-use-pathlib
|
||||||
]
|
]
|
||||||
ignore = [
|
ignore = [
|
||||||
"B028", # warning with no explicit stacklevel
|
"B028", # warning with no explicit stacklevel
|
||||||
@@ -158,6 +169,8 @@ quote-style = "preserve"
|
|||||||
# Developer-only tools - use `uv sync --group <name>`
|
# Developer-only tools - use `uv sync --group <name>`
|
||||||
dev = [
|
dev = [
|
||||||
"mypy>=1.13.0",
|
"mypy>=1.13.0",
|
||||||
|
"ruff>=0.14.11",
|
||||||
|
"prek>=0.4.8",
|
||||||
"ipykernel>=6.29.5",
|
"ipykernel>=6.29.5",
|
||||||
"reportlab>=4.4.4",
|
"reportlab>=4.4.4",
|
||||||
"cyclopts>=4.5.1",
|
"cyclopts>=4.5.1",
|
||||||
@@ -174,7 +187,6 @@ test = [
|
|||||||
"python-xmp-toolkit==2.0.1", # also requires apt-get install libexempi3
|
"python-xmp-toolkit==2.0.1", # also requires apt-get install libexempi3
|
||||||
"reportlab>=3.6.8",
|
"reportlab>=3.6.8",
|
||||||
# Type stubs for testing
|
# Type stubs for testing
|
||||||
"types-Pillow",
|
|
||||||
"types-humanfriendly",
|
"types-humanfriendly",
|
||||||
# Extended test capabilities (merged from extended_test)
|
# Extended test capabilities (merged from extended_test)
|
||||||
"pymupdf>=1.24.14",
|
"pymupdf>=1.24.14",
|
||||||
|
|||||||
@@ -49,7 +49,7 @@ def run(args=None):
|
|||||||
with suppress(AttributeError, PermissionError):
|
with suppress(AttributeError, PermissionError):
|
||||||
os.nice(5)
|
os.nice(5)
|
||||||
|
|
||||||
verbosity = options.verbose
|
verbosity = Verbosity(options.verbose)
|
||||||
if not os.isatty(sys.stderr.fileno()):
|
if not os.isatty(sys.stderr.fileno()):
|
||||||
options.progress_bar = False
|
options.progress_bar = False
|
||||||
if options.quiet:
|
if options.quiet:
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ from __future__ import annotations
|
|||||||
import threading
|
import threading
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from collections.abc import Callable, Iterable
|
from collections.abc import Callable, Iterable
|
||||||
from typing import Any, TypeVar
|
from typing import Any, TypeVar, cast
|
||||||
|
|
||||||
from ocrmypdf._progressbar import NullProgressBar, ProgressBar
|
from ocrmypdf._progressbar import NullProgressBar, ProgressBar
|
||||||
|
|
||||||
@@ -72,7 +72,10 @@ class Executor(ABC):
|
|||||||
if not task_finished:
|
if not task_finished:
|
||||||
task_finished = _task_finished_noop
|
task_finished = _task_finished_noop
|
||||||
if not task:
|
if not task:
|
||||||
task = _task_noop
|
# _task_noop always returns None, but T is unbound here (it's
|
||||||
|
# only meaningful when a real task is supplied); task_finished's
|
||||||
|
# own no-op default accepts Any, so this is safe.
|
||||||
|
task = cast('Callable[..., T]', _task_noop)
|
||||||
|
|
||||||
with self.pool_lock:
|
with self.pool_lock:
|
||||||
self._execute(
|
self._execute(
|
||||||
|
|||||||
@@ -105,8 +105,8 @@ def _gs_devicen_reported(stream) -> bool:
|
|||||||
|
|
||||||
|
|
||||||
def rasterize_pdf(
|
def rasterize_pdf(
|
||||||
input_file: os.PathLike,
|
input_file: Path,
|
||||||
output_file: os.PathLike,
|
output_file: Path,
|
||||||
*,
|
*,
|
||||||
raster_device: GhostscriptRasterDevice,
|
raster_device: GhostscriptRasterDevice,
|
||||||
raster_dpi: Resolution,
|
raster_dpi: Resolution,
|
||||||
@@ -209,6 +209,7 @@ def rasterize_pdf(
|
|||||||
)
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
im: Image.Image
|
||||||
with Image.open(output_file) as im:
|
with Image.open(output_file) as im:
|
||||||
if needs_low_dpi_resize:
|
if needs_low_dpi_resize:
|
||||||
# Resize to the dimensions that would have resulted from the
|
# Resize to the dimensions that would have resulted from the
|
||||||
@@ -288,7 +289,7 @@ class GhostscriptFollower:
|
|||||||
|
|
||||||
def generate_pdfa(
|
def generate_pdfa(
|
||||||
pdf_pages,
|
pdf_pages,
|
||||||
output_file: os.PathLike,
|
output_file: Path,
|
||||||
*,
|
*,
|
||||||
compression: str,
|
compression: str,
|
||||||
color_conversion_strategy: str,
|
color_conversion_strategy: str,
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ def available() -> bool:
|
|||||||
|
|
||||||
def convert_single(cwd, infile, outfile, threshold):
|
def convert_single(cwd, infile, outfile, threshold):
|
||||||
args = ['jbig2', '--pdf', '-t', str(threshold), infile]
|
args = ['jbig2', '--pdf', '-t', str(threshold), infile]
|
||||||
with open(outfile, 'wb') as fstdout:
|
with outfile.open('wb') as fstdout:
|
||||||
proc = run(args, cwd=cwd, stdout=fstdout, stderr=PIPE)
|
proc = run(args, cwd=cwd, stdout=fstdout, stderr=PIPE)
|
||||||
proc.check_returncode()
|
proc.check_returncode()
|
||||||
return proc
|
return proc
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ def quantize(input_file: Path, output_file: Path, quality_min: int, quality_max:
|
|||||||
quality_min: Minimum quality to use
|
quality_min: Minimum quality to use
|
||||||
quality_max: Maximum quality to use
|
quality_max: Maximum quality to use
|
||||||
"""
|
"""
|
||||||
with open(input_file, 'rb') as input_stream:
|
with input_file.open('rb') as input_stream:
|
||||||
args = [
|
args = [
|
||||||
'pngquant',
|
'pngquant',
|
||||||
'--force',
|
'--force',
|
||||||
|
|||||||
+15
-5
@@ -6,11 +6,12 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
from collections.abc import Collection
|
||||||
from contextlib import suppress
|
from contextlib import suppress
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING, cast
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from ocrmypdf.hocrtransform import OcrElement
|
from ocrmypdf.hocrtransform import OcrElement
|
||||||
@@ -18,6 +19,7 @@ if TYPE_CHECKING:
|
|||||||
from pikepdf import (
|
from pikepdf import (
|
||||||
Dictionary,
|
Dictionary,
|
||||||
Name,
|
Name,
|
||||||
|
Object,
|
||||||
Operator,
|
Operator,
|
||||||
Page,
|
Page,
|
||||||
Pdf,
|
Pdf,
|
||||||
@@ -181,9 +183,13 @@ def strip_invisible_text(pdf: Pdf, page: Page):
|
|||||||
render_mode_stack = []
|
render_mode_stack = []
|
||||||
text_objects = []
|
text_objects = []
|
||||||
|
|
||||||
for operands, operator in parse_content_stream(page, ''):
|
for instruction in parse_content_stream(page, ''):
|
||||||
|
operands, operator = instruction.operands, instruction.operator
|
||||||
if operator == Operator('Tr'):
|
if operator == Operator('Tr'):
|
||||||
render_mode = operands[0]
|
# operands[0] is already a plain int under pikepdf's default
|
||||||
|
# (implicit) conversion mode, or a pikepdf.Object under explicit
|
||||||
|
# conversion mode; int() handles both.
|
||||||
|
render_mode = int(operands[0])
|
||||||
|
|
||||||
if operator == Operator('q'):
|
if operator == Operator('q'):
|
||||||
render_mode_stack.append(render_mode)
|
render_mode_stack.append(render_mode)
|
||||||
@@ -207,7 +213,11 @@ def strip_invisible_text(pdf: Pdf, page: Page):
|
|||||||
stream.extend(text_objects)
|
stream.extend(text_objects)
|
||||||
text_objects.clear()
|
text_objects.clear()
|
||||||
|
|
||||||
content_stream = unparse_content_stream(stream)
|
# pikepdf's Collection[...] parameter doesn't structurally match our
|
||||||
|
# _ObjectList-based tuples even though it works fine at runtime.
|
||||||
|
content_stream = unparse_content_stream(
|
||||||
|
cast('list[tuple[Collection[Object], Operator]]', stream)
|
||||||
|
)
|
||||||
page.Contents = Stream(pdf, content_stream)
|
page.Contents = Stream(pdf, content_stream)
|
||||||
|
|
||||||
|
|
||||||
@@ -436,7 +446,7 @@ class OcrGrafter:
|
|||||||
self.pdf_base.close()
|
self.pdf_base.close()
|
||||||
return self.output_file
|
return self.output_file
|
||||||
|
|
||||||
def _parse_hocr_pages(self):
|
def _parse_hocr_pages(self) -> list[Fpdf2ParsedPage]:
|
||||||
"""Render all pages to multi-page PDF with shared fonts, then graft."""
|
"""Render all pages to multi-page PDF with shared fonts, then graft."""
|
||||||
from ocrmypdf.hocrtransform.hocr_parser import HocrParser
|
from ocrmypdf.hocrtransform.hocr_parser import HocrParser
|
||||||
|
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import datetime as dt
|
import datetime as dt
|
||||||
import logging
|
import logging
|
||||||
import os
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
@@ -103,7 +102,7 @@ def should_linearize(working_file: Path, context: PdfContext) -> bool:
|
|||||||
|
|
||||||
For smaller files, linearization is not worth the effort.
|
For smaller files, linearization is not worth the effort.
|
||||||
"""
|
"""
|
||||||
filesize = os.stat(working_file).st_size
|
filesize = working_file.stat().st_size
|
||||||
return filesize > (context.options.fast_web_view * 1_000_000)
|
return filesize > (context.options.fast_web_view * 1_000_000)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+12
-19
@@ -29,7 +29,7 @@ log = logging.getLogger(__name__)
|
|||||||
|
|
||||||
# Module-level registry for plugin option models
|
# Module-level registry for plugin option models
|
||||||
# This is populated by setup_plugin_infrastructure() after plugins are loaded
|
# This is populated by setup_plugin_infrastructure() after plugins are loaded
|
||||||
_plugin_option_models: dict[str, type] = {}
|
_plugin_option_models: dict[str, type[BaseModel]] = {}
|
||||||
|
|
||||||
PathOrIO = BinaryIO | IOBase | Path | str | bytes
|
PathOrIO = BinaryIO | IOBase | Path | str | bytes
|
||||||
|
|
||||||
@@ -207,20 +207,19 @@ class OcrOptions(BaseModel):
|
|||||||
|
|
||||||
# Optimization
|
# Optimization
|
||||||
optimize: int = 1
|
optimize: int = 1
|
||||||
jpg_quality: int | None = None
|
jpeg_quality: int | None = None
|
||||||
png_quality: int | None = None
|
png_quality: int | None = None
|
||||||
jbig2_threshold: float = 0.85
|
|
||||||
|
|
||||||
# Compatibility alias for plugins that expect jpeg_quality
|
# Deprecated compatibility alias for code that still uses the old field name
|
||||||
@property
|
@property
|
||||||
def jpeg_quality(self):
|
def jpg_quality(self):
|
||||||
"""Compatibility alias for jpg_quality."""
|
"""Deprecated compatibility alias for jpeg_quality."""
|
||||||
return self.jpg_quality
|
return self.jpeg_quality
|
||||||
|
|
||||||
@jpeg_quality.setter
|
@jpg_quality.setter
|
||||||
def jpeg_quality(self, value):
|
def jpg_quality(self, value):
|
||||||
"""Compatibility alias for jpg_quality."""
|
"""Deprecated compatibility alias for jpeg_quality."""
|
||||||
self.jpg_quality = value
|
self.jpeg_quality = value
|
||||||
|
|
||||||
# Output behavior
|
# Output behavior
|
||||||
no_overwrite: bool = False
|
no_overwrite: bool = False
|
||||||
@@ -465,7 +464,7 @@ class OcrOptions(BaseModel):
|
|||||||
):
|
):
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
"Since you specified `--output-type none`, the output file "
|
"Since you specified `--output-type none`, the output file "
|
||||||
f"{self.output_file} cannot be produced. Set the output file to "
|
f"{str(self.output_file)} cannot be produced. Set the output file to "
|
||||||
f"`-` to suppress this message."
|
f"`-` to suppress this message."
|
||||||
)
|
)
|
||||||
return self
|
return self
|
||||||
@@ -576,7 +575,7 @@ class OcrOptions(BaseModel):
|
|||||||
)
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def register_plugin_models(cls, models: dict[str, type]) -> None:
|
def register_plugin_models(cls, models: dict[str, type[BaseModel]]) -> None:
|
||||||
"""Register plugin option model classes for nested access.
|
"""Register plugin option model classes for nested access.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -643,12 +642,6 @@ class OcrOptions(BaseModel):
|
|||||||
value = self.optimize
|
value = self.optimize
|
||||||
if value is not None:
|
if value is not None:
|
||||||
kwargs[field_name] = _convert_value(value)
|
kwargs[field_name] = _convert_value(value)
|
||||||
elif namespace == 'optimize' and field_name == 'jpeg_quality':
|
|
||||||
# jpg_quality maps to jpeg_quality
|
|
||||||
if 'jpg_quality' in OcrOptions.model_fields:
|
|
||||||
value = self.jpg_quality
|
|
||||||
if value is not None:
|
|
||||||
kwargs[field_name] = _convert_value(value)
|
|
||||||
|
|
||||||
# Create and cache the plugin options instance
|
# Create and cache the plugin options instance
|
||||||
instance = model_class(**kwargs)
|
instance = model_class(**kwargs)
|
||||||
|
|||||||
+18
-17
@@ -28,7 +28,7 @@ from ocrmypdf._concurrent import Executor
|
|||||||
from ocrmypdf._exec import unpaper
|
from ocrmypdf._exec import unpaper
|
||||||
from ocrmypdf._jobcontext import PageContext, PdfContext
|
from ocrmypdf._jobcontext import PageContext, PdfContext
|
||||||
from ocrmypdf._metadata import repair_docinfo_nuls
|
from ocrmypdf._metadata import repair_docinfo_nuls
|
||||||
from ocrmypdf._options import OcrOptions, ProcessingMode, TaggedPdfMode
|
from ocrmypdf._options import OcrOptions, PathOrIO, ProcessingMode, TaggedPdfMode
|
||||||
from ocrmypdf._pageboxes import log_box_repairs, repair_page_boxes
|
from ocrmypdf._pageboxes import log_box_repairs, repair_page_boxes
|
||||||
from ocrmypdf._stdoutprotect import get_protected_stdout_fd
|
from ocrmypdf._stdoutprotect import get_protected_stdout_fd
|
||||||
from ocrmypdf.exceptions import (
|
from ocrmypdf.exceptions import (
|
||||||
@@ -140,7 +140,7 @@ def triage_image_file(input_file: Path, output_file: Path, options: OcrOptions)
|
|||||||
layout_fun = img2pdf.get_fixed_dpi_layout_fun(
|
layout_fun = img2pdf.get_fixed_dpi_layout_fun(
|
||||||
Resolution(options.image_dpi, options.image_dpi)
|
Resolution(options.image_dpi, options.image_dpi)
|
||||||
)
|
)
|
||||||
with open(output_file, 'wb') as outf:
|
with output_file.open('wb') as outf:
|
||||||
img2pdf.convert(
|
img2pdf.convert(
|
||||||
os.fspath(input_file),
|
os.fspath(input_file),
|
||||||
layout_fun=layout_fun,
|
layout_fun=layout_fun,
|
||||||
@@ -159,7 +159,7 @@ def _pdf_guess_version(input_file: Path, search_window=1024) -> str:
|
|||||||
|
|
||||||
Returns empty string if not found, indicating file is probably not PDF.
|
Returns empty string if not found, indicating file is probably not PDF.
|
||||||
"""
|
"""
|
||||||
with open(input_file, 'rb') as f:
|
with input_file.open('rb') as f:
|
||||||
signature = f.read(search_window)
|
signature = f.read(search_window)
|
||||||
m = re.search(rb'%PDF-(\d\.\d)', signature)
|
m = re.search(rb'%PDF-(\d\.\d)', signature)
|
||||||
if m:
|
if m:
|
||||||
@@ -683,6 +683,7 @@ def create_ocr_image(image: Path, page_context: PageContext) -> Path:
|
|||||||
"""
|
"""
|
||||||
output_file = page_context.get_path('ocr.png')
|
output_file = page_context.get_path('ocr.png')
|
||||||
options = page_context.options
|
options = page_context.options
|
||||||
|
im: Image.Image
|
||||||
with Image.open(image) as im:
|
with Image.open(image) as im:
|
||||||
log.debug('resolution %r', im.info['dpi'])
|
log.debug('resolution %r', im.info['dpi'])
|
||||||
|
|
||||||
@@ -832,7 +833,7 @@ def create_pdf_page_from_image(
|
|||||||
|
|
||||||
# Create a new single page PDF to hold
|
# Create a new single page PDF to hold
|
||||||
bio = BytesIO()
|
bio = BytesIO()
|
||||||
with open(image, 'rb') as imfile:
|
with image.open('rb') as imfile:
|
||||||
log.debug('convert')
|
log.debug('convert')
|
||||||
|
|
||||||
layout_fun = img2pdf.get_layout_fun(pagesize)
|
layout_fun = img2pdf.get_layout_fun(pagesize)
|
||||||
@@ -1198,7 +1199,7 @@ def should_linearize(working_file: Path, context: PdfContext) -> bool:
|
|||||||
|
|
||||||
For smaller files, linearization is not worth the effort.
|
For smaller files, linearization is not worth the effort.
|
||||||
"""
|
"""
|
||||||
filesize = os.stat(working_file).st_size
|
filesize = working_file.stat().st_size
|
||||||
return filesize > (context.options.fast_web_view * 1_000_000)
|
return filesize > (context.options.fast_web_view * 1_000_000)
|
||||||
|
|
||||||
|
|
||||||
@@ -1284,7 +1285,8 @@ def enumerate_compress_ranges(
|
|||||||
A tuple containing a range of indices and the corresponding element.
|
A tuple containing a range of indices and the corresponding element.
|
||||||
If the element is None, the range represents a skipped range of indices.
|
If the element is None, the range represents a skipped range of indices.
|
||||||
"""
|
"""
|
||||||
skipped_from, index = None, None
|
skipped_from: int | None = None
|
||||||
|
index: int | None = None
|
||||||
for index, txt_file in enumerate(iterable):
|
for index, txt_file in enumerate(iterable):
|
||||||
index += 1
|
index += 1
|
||||||
if txt_file:
|
if txt_file:
|
||||||
@@ -1296,6 +1298,9 @@ def enumerate_compress_ranges(
|
|||||||
if skipped_from is None:
|
if skipped_from is None:
|
||||||
skipped_from = index
|
skipped_from = index
|
||||||
if skipped_from is not None:
|
if skipped_from is not None:
|
||||||
|
# skipped_from can only be set inside the loop above, so the loop
|
||||||
|
# must have run at least once and index is guaranteed to be an int.
|
||||||
|
assert index is not None
|
||||||
yield (skipped_from, index), None
|
yield (skipped_from, index), None
|
||||||
|
|
||||||
|
|
||||||
@@ -1307,7 +1312,7 @@ def merge_sidecars(txt_files: Iterable[Path | None], context: PdfContext) -> Pat
|
|||||||
and returns the path to the merged file.
|
and returns the path to the merged file.
|
||||||
"""
|
"""
|
||||||
output_file = context.get_path('sidecar.txt')
|
output_file = context.get_path('sidecar.txt')
|
||||||
with open(output_file, 'w', encoding="utf-8") as stream:
|
with output_file.open('w', encoding="utf-8") as stream:
|
||||||
for (from_, to_), txt_file in enumerate_compress_ranges(txt_files):
|
for (from_, to_), txt_file in enumerate_compress_ranges(txt_files):
|
||||||
if from_ != 1:
|
if from_ != 1:
|
||||||
stream.write('\f') # Form feed between pages for all pages after first
|
stream.write('\f') # Form feed between pages for all pages after first
|
||||||
@@ -1322,18 +1327,12 @@ def merge_sidecars(txt_files: Iterable[Path | None], context: PdfContext) -> Pat
|
|||||||
return output_file
|
return output_file
|
||||||
|
|
||||||
|
|
||||||
def copy_final(
|
def copy_final(input_file: Path, output_file: PathOrIO) -> None:
|
||||||
input_file: Path, output_file: str | Path | BinaryIO, original_file: Path | None
|
|
||||||
) -> None:
|
|
||||||
"""Copy the final temporary file to the output destination.
|
"""Copy the final temporary file to the output destination.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
input_file (Path): The intermediate input file to copy.
|
input_file: The intermediate input file to copy.
|
||||||
output_file (str | Path | BinaryIO): The output file to copy to.
|
output_file: The output file to copy to.
|
||||||
original_file: The original file to copy attributes from.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
None
|
|
||||||
"""
|
"""
|
||||||
log.debug('%s -> %s', input_file, output_file)
|
log.debug('%s -> %s', input_file, output_file)
|
||||||
with input_file.open('rb') as input_stream:
|
with input_file.open('rb') as input_stream:
|
||||||
@@ -1359,5 +1358,7 @@ def copy_final(
|
|||||||
# At this point we overwrite the output_file specified by the user
|
# At this point we overwrite the output_file specified by the user
|
||||||
# use copyfileobj because then we use open() to create the file and
|
# use copyfileobj because then we use open() to create the file and
|
||||||
# get the appropriate umask, ownership, etc.
|
# get the appropriate umask, ownership, etc.
|
||||||
with open(output_file, 'w+b') as output_stream:
|
# The `hasattr` check above already ruled out stream-like objects.
|
||||||
|
assert isinstance(output_file, str | bytes | os.PathLike)
|
||||||
|
with Path(os.fsdecode(output_file)).open('w+b') as output_stream:
|
||||||
copyfileobj(input_stream, output_stream)
|
copyfileobj(input_stream, output_stream)
|
||||||
|
|||||||
@@ -98,8 +98,8 @@ def exec_hocr_to_ocr_pdf(context: PdfContext, executor: Executor) -> Sequence[st
|
|||||||
log.info("Postprocessing...")
|
log.info("Postprocessing...")
|
||||||
pdf, messages = postprocess(pdf, context, executor)
|
pdf, messages = postprocess(pdf, context, executor)
|
||||||
|
|
||||||
# Copy PDF file to destination (we don't know the input PDF file name)
|
# Copy PDF file to destination
|
||||||
copy_final(pdf, options.output_file, None)
|
copy_final(pdf, options.output_file)
|
||||||
return messages
|
return messages
|
||||||
|
|
||||||
|
|
||||||
@@ -109,6 +109,9 @@ def run_hocr_to_ocr_pdf_pipeline(
|
|||||||
plugin_manager: OcrmypdfPluginManager,
|
plugin_manager: OcrmypdfPluginManager,
|
||||||
) -> ExitCode:
|
) -> ExitCode:
|
||||||
"""Run pipeline to convert hOCR to final output PDF."""
|
"""Run pipeline to convert hOCR to final output PDF."""
|
||||||
|
# The _hocr_to_ocr_pdf() API requires work_folder: Path and stores it on
|
||||||
|
# options before this pipeline runs, so it is always set at this point.
|
||||||
|
assert options.work_folder is not None
|
||||||
with manage_work_folder(
|
with manage_work_folder(
|
||||||
work_folder=options.work_folder, retain=True, print_location=False
|
work_folder=options.work_folder, retain=True, print_location=False
|
||||||
) as work_folder:
|
) as work_folder:
|
||||||
|
|||||||
@@ -145,7 +145,7 @@ def exec_concurrent(context: PdfContext, executor: Executor) -> Sequence[str]:
|
|||||||
if options.sidecar:
|
if options.sidecar:
|
||||||
text = merge_sidecars(sidecars, context)
|
text = merge_sidecars(sidecars, context)
|
||||||
# Copy text file to destination
|
# Copy text file to destination
|
||||||
copy_final(text, options.sidecar, options.input_file)
|
copy_final(text, options.sidecar)
|
||||||
|
|
||||||
# Merge layers to one single pdf
|
# Merge layers to one single pdf
|
||||||
pdf = ocrgraft.finalize()
|
pdf = ocrgraft.finalize()
|
||||||
@@ -157,7 +157,7 @@ def exec_concurrent(context: PdfContext, executor: Executor) -> Sequence[str]:
|
|||||||
pdf, messages = postprocess(pdf, context, executor)
|
pdf, messages = postprocess(pdf, context, executor)
|
||||||
|
|
||||||
# Copy PDF file to destination
|
# Copy PDF file to destination
|
||||||
copy_final(pdf, options.output_file, options.input_file)
|
copy_final(pdf, options.output_file)
|
||||||
return messages
|
return messages
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import logging
|
import logging
|
||||||
import logging.handlers
|
import logging.handlers
|
||||||
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
from functools import partial
|
from functools import partial
|
||||||
|
|
||||||
@@ -91,6 +92,9 @@ def run_hocr_pipeline(
|
|||||||
"""Run pipeline to output hOCR."""
|
"""Run pipeline to output hOCR."""
|
||||||
if options.output_folder is None:
|
if options.output_folder is None:
|
||||||
raise ValueError("output_folder must be specified for hOCR pipeline")
|
raise ValueError("output_folder must be specified for hOCR pipeline")
|
||||||
|
# This pipeline is only reachable via the _pdf_to_hocr() API, which
|
||||||
|
# declares input_pdf: Path - streams and raw bytes paths are not supported.
|
||||||
|
assert isinstance(options.input_file, str | os.PathLike)
|
||||||
with manage_work_folder(
|
with manage_work_folder(
|
||||||
work_folder=options.output_folder, retain=True, print_location=False
|
work_folder=options.output_folder, retain=True, print_location=False
|
||||||
) as work_folder:
|
) as work_folder:
|
||||||
@@ -100,9 +104,7 @@ def run_hocr_pipeline(
|
|||||||
|
|
||||||
# Gather pdfinfo and create context
|
# Gather pdfinfo and create context
|
||||||
pdfinfo = do_get_pdfinfo(origin_pdf, executor, options)
|
pdfinfo = do_get_pdfinfo(origin_pdf, executor, options)
|
||||||
context = PdfContext(
|
context = PdfContext(options, work_folder, origin_pdf, pdfinfo, plugin_manager)
|
||||||
options, work_folder, options.input_file, pdfinfo, plugin_manager
|
|
||||||
)
|
|
||||||
# Validate options are okay for this pdf
|
# Validate options are okay for this pdf
|
||||||
validate_pdfinfo_options(context)
|
validate_pdfinfo_options(context)
|
||||||
exec_pdf_to_hocr(context, executor)
|
exec_pdf_to_hocr(context, executor)
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ from pydantic import BaseModel
|
|||||||
import ocrmypdf.builtin_plugins
|
import ocrmypdf.builtin_plugins
|
||||||
from ocrmypdf import Executor, PdfContext, pluginspec
|
from ocrmypdf import Executor, PdfContext, pluginspec
|
||||||
from ocrmypdf._options import OcrOptions
|
from ocrmypdf._options import OcrOptions
|
||||||
|
from ocrmypdf._plugin_registry import PluginOptionRegistry
|
||||||
from ocrmypdf._progressbar import ProgressBar
|
from ocrmypdf._progressbar import ProgressBar
|
||||||
from ocrmypdf.helpers import Resolution
|
from ocrmypdf.helpers import Resolution
|
||||||
from ocrmypdf.pluginspec import OcrEngine
|
from ocrmypdf.pluginspec import OcrEngine
|
||||||
@@ -53,10 +54,11 @@ class OcrmypdfPluginManager:
|
|||||||
self._plugins = plugins
|
self._plugins = plugins
|
||||||
self._builtins = builtins
|
self._builtins = builtins
|
||||||
self._pm = pluggy.PluginManager(*args, **kwargs)
|
self._pm = pluggy.PluginManager(*args, **kwargs)
|
||||||
|
self._option_registry: PluginOptionRegistry | None = None
|
||||||
self._setup_plugins()
|
self._setup_plugins()
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def pluggy(self) -> pluggy.PluginManager:
|
def pluggy_manager(self) -> pluggy.PluginManager:
|
||||||
"""Access the underlying pluggy.PluginManager for advanced use cases.
|
"""Access the underlying pluggy.PluginManager for advanced use cases.
|
||||||
|
|
||||||
This is useful for plugins that need to call methods like set_blocked()
|
This is useful for plugins that need to call methods like set_blocked()
|
||||||
@@ -74,7 +76,8 @@ class OcrmypdfPluginManager:
|
|||||||
return state
|
return state
|
||||||
|
|
||||||
def __setstate__(self, state):
|
def __setstate__(self, state):
|
||||||
self.__init__(
|
OcrmypdfPluginManager.__init__(
|
||||||
|
self,
|
||||||
*state['init_args'],
|
*state['init_args'],
|
||||||
plugins=state['plugins'],
|
plugins=state['plugins'],
|
||||||
builtins=state['builtins'],
|
builtins=state['builtins'],
|
||||||
@@ -86,10 +89,10 @@ class OcrmypdfPluginManager:
|
|||||||
|
|
||||||
# 1. Register builtins
|
# 1. Register builtins
|
||||||
if self._builtins:
|
if self._builtins:
|
||||||
for module in sorted(
|
for module_info in sorted(
|
||||||
pkgutil.iter_modules(ocrmypdf.builtin_plugins.__path__)
|
pkgutil.iter_modules(ocrmypdf.builtin_plugins.__path__)
|
||||||
):
|
):
|
||||||
name = f'ocrmypdf.builtin_plugins.{module.name}'
|
name = f'ocrmypdf.builtin_plugins.{module_info.name}'
|
||||||
module = importlib.import_module(name)
|
module = importlib.import_module(name)
|
||||||
self._pm.register(module)
|
self._pm.register(module)
|
||||||
|
|
||||||
@@ -97,17 +100,20 @@ class OcrmypdfPluginManager:
|
|||||||
self._pm.load_setuptools_entrypoints('ocrmypdf')
|
self._pm.load_setuptools_entrypoints('ocrmypdf')
|
||||||
|
|
||||||
# 3. Register plugins specified on command line
|
# 3. Register plugins specified on command line
|
||||||
for name in self._plugins:
|
for plugin in self._plugins:
|
||||||
if isinstance(name, Path) or name.endswith('.py'):
|
if isinstance(plugin, Path) or plugin.endswith('.py'):
|
||||||
# Import by filename
|
# Import by filename
|
||||||
module_name = Path(name).stem
|
plugin_path = Path(plugin)
|
||||||
spec = importlib.util.spec_from_file_location(module_name, name)
|
module_name = plugin_path.stem
|
||||||
|
spec = importlib.util.spec_from_file_location(module_name, plugin_path)
|
||||||
|
if spec is None or spec.loader is None:
|
||||||
|
raise ImportError(f'Could not load plugin from {plugin_path}')
|
||||||
module = importlib.util.module_from_spec(spec)
|
module = importlib.util.module_from_spec(spec)
|
||||||
sys.modules[module_name] = module
|
sys.modules[module_name] = module
|
||||||
spec.loader.exec_module(module)
|
spec.loader.exec_module(module)
|
||||||
else:
|
else:
|
||||||
# Import by dotted module name
|
# Import by dotted module name
|
||||||
module = importlib.import_module(name)
|
module = importlib.import_module(plugin)
|
||||||
self._pm.register(module)
|
self._pm.register(module)
|
||||||
|
|
||||||
# =========================================================================
|
# =========================================================================
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ class PluginOptionRegistry:
|
|||||||
compatibility (e.g., options.tesseract_timeout).
|
compatibility (e.g., options.tesseract_timeout).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self) -> None:
|
||||||
self._option_models: dict[str, type[BaseModel]] = {}
|
self._option_models: dict[str, type[BaseModel]] = {}
|
||||||
|
|
||||||
def register_option_model(
|
def register_option_model(
|
||||||
|
|||||||
+25
-10
@@ -10,8 +10,10 @@ import logging
|
|||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
from collections.abc import Sequence
|
from collections.abc import Sequence
|
||||||
|
from collections.abc import Set as AbstractSet
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from shutil import copyfileobj
|
from shutil import copyfileobj
|
||||||
|
from typing import BinaryIO, cast
|
||||||
|
|
||||||
import pikepdf
|
import pikepdf
|
||||||
|
|
||||||
@@ -48,7 +50,7 @@ def check_platform() -> None:
|
|||||||
|
|
||||||
|
|
||||||
def check_options_languages(
|
def check_options_languages(
|
||||||
options: OcrOptions, ocr_engine_languages: list[str]
|
options: OcrOptions, ocr_engine_languages: AbstractSet[str]
|
||||||
) -> None:
|
) -> None:
|
||||||
# Check for blocked languages first, before checking if they're installed
|
# Check for blocked languages first, before checking if they're installed
|
||||||
DENIED_LANGUAGES = {'equ', 'osd'}
|
DENIED_LANGUAGES = {'equ', 'osd'}
|
||||||
@@ -92,7 +94,15 @@ def check_options_sidecar(options: OcrOptions) -> None:
|
|||||||
raise BadArgsError(
|
raise BadArgsError(
|
||||||
"--sidecar filename needed when output file is /dev/null or NUL."
|
"--sidecar filename needed when output file is /dev/null or NUL."
|
||||||
)
|
)
|
||||||
options.sidecar = options.output_file + '.txt'
|
elif not isinstance(options.output_file, str | Path):
|
||||||
|
# The '\0' sentinel is only ever set by the CLI, which always
|
||||||
|
# supplies output_file as a plain path - not a stream. If this
|
||||||
|
# somehow fires, the caller mixed a CLI-only sentinel with the
|
||||||
|
# stream-based API.
|
||||||
|
raise BadArgsError(
|
||||||
|
"--sidecar filename needed when output file is not a path."
|
||||||
|
)
|
||||||
|
options.sidecar = os.fspath(options.output_file) + '.txt'
|
||||||
if options.sidecar == options.input_file or options.sidecar == options.output_file:
|
if options.sidecar == options.input_file or options.sidecar == options.output_file:
|
||||||
raise BadArgsError(
|
raise BadArgsError(
|
||||||
"--sidecar file must be different from the input and output files"
|
"--sidecar file must be different from the input and output files"
|
||||||
@@ -190,24 +200,28 @@ def create_input_file(options: OcrOptions, work_folder: Path) -> tuple[Path, str
|
|||||||
# stdin
|
# stdin
|
||||||
log.info('reading file from standard input')
|
log.info('reading file from standard input')
|
||||||
target = work_folder / 'stdin'
|
target = work_folder / 'stdin'
|
||||||
with open(target, 'wb') as stream_buffer:
|
with target.open('wb') as stream_buffer:
|
||||||
copyfileobj(sys.stdin.buffer, stream_buffer)
|
copyfileobj(sys.stdin.buffer, stream_buffer)
|
||||||
return target, "stdin"
|
return target, "stdin"
|
||||||
elif hasattr(options.input_file, 'readable'):
|
elif hasattr(options.input_file, 'readable'):
|
||||||
if not options.input_file.readable():
|
input_stream = cast(BinaryIO, options.input_file)
|
||||||
|
if not input_stream.readable():
|
||||||
raise InputFileError("Input file stream is not readable")
|
raise InputFileError("Input file stream is not readable")
|
||||||
log.info('reading file from input stream')
|
log.info('reading file from input stream')
|
||||||
target = work_folder / 'stream'
|
target = work_folder / 'stream'
|
||||||
with open(target, 'wb') as stream_buffer:
|
with target.open('wb') as stream_buffer:
|
||||||
copyfileobj(options.input_file, stream_buffer)
|
copyfileobj(input_stream, stream_buffer)
|
||||||
return target, "stream"
|
return target, "stream"
|
||||||
else:
|
else:
|
||||||
|
# The branches above already ruled out the stdin sentinel and
|
||||||
|
# stream-like objects, so this must be a filesystem path.
|
||||||
|
assert isinstance(options.input_file, str | bytes | os.PathLike)
|
||||||
try:
|
try:
|
||||||
target = work_folder / 'origin'
|
target = work_folder / 'origin'
|
||||||
safe_symlink(options.input_file, target)
|
safe_symlink(options.input_file, target)
|
||||||
return target, os.fspath(options.input_file)
|
return target, os.fsdecode(options.input_file)
|
||||||
except FileNotFoundError as e:
|
except FileNotFoundError as e:
|
||||||
msg = f"File not found - {options.input_file}"
|
msg = f"File not found - {os.fsdecode(options.input_file)}"
|
||||||
if running_in_docker(): # pragma: no cover
|
if running_in_docker(): # pragma: no cover
|
||||||
msg += (
|
msg += (
|
||||||
"\nDocker cannot access your working directory unless you "
|
"\nDocker cannot access your working directory unless you "
|
||||||
@@ -249,7 +263,8 @@ def check_requested_output_file(options: OcrOptions) -> None:
|
|||||||
raise OutputFileAccessError("Output stream is not writable")
|
raise OutputFileAccessError("Output stream is not writable")
|
||||||
elif not is_file_writable(options.output_file):
|
elif not is_file_writable(options.output_file):
|
||||||
raise OutputFileAccessError(
|
raise OutputFileAccessError(
|
||||||
f"Output file location ({options.output_file}) is not a writable file."
|
f"Output file location ({os.fsdecode(options.output_file)}) is not a "
|
||||||
|
"writable file."
|
||||||
)
|
)
|
||||||
|
|
||||||
if (
|
if (
|
||||||
@@ -259,7 +274,7 @@ def check_requested_output_file(options: OcrOptions) -> None:
|
|||||||
and Path(str(options.output_file)).exists()
|
and Path(str(options.output_file)).exists()
|
||||||
):
|
):
|
||||||
raise OutputFileAccessError(
|
raise OutputFileAccessError(
|
||||||
f"Output file already exists: {options.output_file}\n"
|
f"Output file already exists: {os.fsdecode(options.output_file)}\n"
|
||||||
"To overwrite it, omit the --no-overwrite / -n option."
|
"To overwrite it, omit the --no-overwrite / -n option."
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -10,9 +10,8 @@ import os
|
|||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
import pluggy
|
|
||||||
|
|
||||||
from ocrmypdf._options import OcrOptions
|
from ocrmypdf._options import OcrOptions
|
||||||
|
from ocrmypdf._plugin_manager import OcrmypdfPluginManager
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -20,9 +19,8 @@ log = logging.getLogger(__name__)
|
|||||||
class ValidationCoordinator:
|
class ValidationCoordinator:
|
||||||
"""Coordinates validation across plugin models and core options."""
|
"""Coordinates validation across plugin models and core options."""
|
||||||
|
|
||||||
def __init__(self, plugin_manager: pluggy.PluginManager):
|
def __init__(self, plugin_manager: OcrmypdfPluginManager):
|
||||||
self.plugin_manager = plugin_manager
|
self.plugin_manager = plugin_manager
|
||||||
self.registry = getattr(plugin_manager, '_option_registry', None)
|
|
||||||
|
|
||||||
def validate_all_options(self, options: OcrOptions) -> None:
|
def validate_all_options(self, options: OcrOptions) -> None:
|
||||||
"""Run comprehensive validation on all options.
|
"""Run comprehensive validation on all options.
|
||||||
@@ -110,13 +108,18 @@ class ValidationCoordinator:
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Validate output type compatibility
|
# Validate output type compatibility
|
||||||
if options.output_type == 'none' and str(options.output_file) not in (
|
output_file_display = (
|
||||||
|
os.fsdecode(options.output_file)
|
||||||
|
if isinstance(options.output_file, bytes)
|
||||||
|
else str(options.output_file)
|
||||||
|
)
|
||||||
|
if options.output_type == 'none' and output_file_display not in (
|
||||||
os.devnull,
|
os.devnull,
|
||||||
'-',
|
'-',
|
||||||
):
|
):
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
"Since you specified `--output-type none`, the output file "
|
"Since you specified `--output-type none`, the output file "
|
||||||
f"{options.output_file} cannot be produced. Set the output file to "
|
f"{output_file_display} cannot be produced. Set the output file to "
|
||||||
"`-` to suppress this message."
|
"`-` to suppress this message."
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -1,3 +1,3 @@
|
|||||||
# SPDX-FileCopyrightText: 2022 James R. Barlow
|
# SPDX-FileCopyrightText: 2022 James R. Barlow
|
||||||
# SPDX-License-Identifier: MPL-2.0
|
# SPDX-License-Identifier: MPL-2.0
|
||||||
__version__ = "17.8.0"
|
__version__ = "17.9.0"
|
||||||
|
|||||||
+29
-5
@@ -50,6 +50,8 @@ from pathlib import Path
|
|||||||
from typing import BinaryIO, overload
|
from typing import BinaryIO, overload
|
||||||
from warnings import warn
|
from warnings import warn
|
||||||
|
|
||||||
|
from pydantic import BaseModel
|
||||||
|
|
||||||
from ocrmypdf._logging import PageNumberFilter
|
from ocrmypdf._logging import PageNumberFilter
|
||||||
from ocrmypdf._options import OcrOptions
|
from ocrmypdf._options import OcrOptions
|
||||||
from ocrmypdf._pipelines.hocr_to_ocr_pdf import run_hocr_to_ocr_pdf_pipeline
|
from ocrmypdf._pipelines.hocr_to_ocr_pdf import run_hocr_to_ocr_pdf_pipeline
|
||||||
@@ -106,7 +108,7 @@ def setup_plugin_infrastructure(
|
|||||||
plugin_manager = get_plugin_manager(plugins)
|
plugin_manager = get_plugin_manager(plugins)
|
||||||
|
|
||||||
# Initialize plugins (pass the underlying pluggy manager)
|
# Initialize plugins (pass the underlying pluggy manager)
|
||||||
plugin_manager.initialize(plugin_manager=plugin_manager.pluggy)
|
plugin_manager.initialize(plugin_manager=plugin_manager.pluggy_manager)
|
||||||
|
|
||||||
# Initialize plugin option registry
|
# Initialize plugin option registry
|
||||||
from ocrmypdf._plugin_registry import PluginOptionRegistry
|
from ocrmypdf._plugin_registry import PluginOptionRegistry
|
||||||
@@ -115,7 +117,7 @@ def setup_plugin_infrastructure(
|
|||||||
|
|
||||||
# Let plugins register their option models
|
# Let plugins register their option models
|
||||||
option_models = plugin_manager.register_options()
|
option_models = plugin_manager.register_options()
|
||||||
all_plugin_models: dict[str, type] = {}
|
all_plugin_models: dict[str, type[BaseModel]] = {}
|
||||||
for plugin_options in option_models:
|
for plugin_options in option_models:
|
||||||
if plugin_options: # Skip None returns
|
if plugin_options: # Skip None returns
|
||||||
for namespace, model_class in plugin_options.items():
|
for namespace, model_class in plugin_options.items():
|
||||||
@@ -343,6 +345,23 @@ def _remap_language_to_languages(options_kwargs: dict) -> None:
|
|||||||
del options_kwargs['language']
|
del options_kwargs['language']
|
||||||
|
|
||||||
|
|
||||||
|
def _remap_jpg_quality_to_jpeg_quality(options_kwargs: dict) -> None:
|
||||||
|
"""Map the deprecated 'jpg_quality' parameter to 'jpeg_quality'.
|
||||||
|
|
||||||
|
'jpg_quality' was the original API parameter name. 'jpeg_quality' is the
|
||||||
|
canonical OcrOptions field, matching the primary --jpeg-quality CLI flag.
|
||||||
|
Prefer an explicitly-given 'jpeg_quality' if both are set.
|
||||||
|
"""
|
||||||
|
if 'jpg_quality' not in options_kwargs:
|
||||||
|
return
|
||||||
|
old_value = options_kwargs.pop('jpg_quality')
|
||||||
|
if old_value is None:
|
||||||
|
return
|
||||||
|
warn("ocrmypdf.ocr(jpg_quality=...) is deprecated, use jpeg_quality= instead.")
|
||||||
|
if options_kwargs.get('jpeg_quality') is None:
|
||||||
|
options_kwargs['jpeg_quality'] = old_value
|
||||||
|
|
||||||
|
|
||||||
def create_options(
|
def create_options(
|
||||||
*, input_file: PathOrIO, output_file: PathOrIO, parser: ArgumentParser, **kwargs
|
*, input_file: PathOrIO, output_file: PathOrIO, parser: ArgumentParser, **kwargs
|
||||||
) -> OcrOptions:
|
) -> OcrOptions:
|
||||||
@@ -367,6 +386,9 @@ def create_options(
|
|||||||
# Map API parameter 'language' to OcrOptions field 'languages'
|
# Map API parameter 'language' to OcrOptions field 'languages'
|
||||||
_remap_language_to_languages(options_kwargs)
|
_remap_language_to_languages(options_kwargs)
|
||||||
|
|
||||||
|
# Map deprecated 'jpg_quality' parameter to 'jpeg_quality'
|
||||||
|
_remap_jpg_quality_to_jpeg_quality(options_kwargs)
|
||||||
|
|
||||||
# Set input and output files
|
# Set input and output files
|
||||||
options_kwargs['input_file'] = input_file
|
options_kwargs['input_file'] = input_file
|
||||||
options_kwargs['output_file'] = output_file
|
options_kwargs['output_file'] = output_file
|
||||||
@@ -446,7 +468,8 @@ def ocr(
|
|||||||
redo_ocr: bool | None = None,
|
redo_ocr: bool | None = None,
|
||||||
skip_big: float | None = None,
|
skip_big: float | None = None,
|
||||||
optimize: int | None = None,
|
optimize: int | None = None,
|
||||||
jpg_quality: int | None = None,
|
jpeg_quality: int | None = None,
|
||||||
|
jpg_quality: int | None = None, # Deprecated, use jpeg_quality instead
|
||||||
png_quality: int | None = None,
|
png_quality: int | None = None,
|
||||||
jbig2_lossy: bool | None = None,
|
jbig2_lossy: bool | None = None,
|
||||||
jbig2_page_group_size: int | None = None,
|
jbig2_page_group_size: int | None = None,
|
||||||
@@ -509,7 +532,8 @@ def ocr( # noqa: D417
|
|||||||
redo_ocr: bool | None = None, # Legacy, use mode='redo' instead
|
redo_ocr: bool | None = None, # Legacy, use mode='redo' instead
|
||||||
skip_big: float | None = None,
|
skip_big: float | None = None,
|
||||||
optimize: int | None = None,
|
optimize: int | None = None,
|
||||||
jpg_quality: int | None = None,
|
jpeg_quality: int | None = None,
|
||||||
|
jpg_quality: int | None = None, # Deprecated, use jpeg_quality instead
|
||||||
png_quality: int | None = None,
|
png_quality: int | None = None,
|
||||||
jbig2_lossy: bool | None = None, # Deprecated, ignored
|
jbig2_lossy: bool | None = None, # Deprecated, ignored
|
||||||
jbig2_page_group_size: int | None = None, # Deprecated, ignored
|
jbig2_page_group_size: int | None = None, # Deprecated, ignored
|
||||||
@@ -881,7 +905,7 @@ def _hocr_to_ocr_pdf( # noqa: D417
|
|||||||
jobs: int | None = None,
|
jobs: int | None = None,
|
||||||
use_threads: bool | None = None,
|
use_threads: bool | None = None,
|
||||||
optimize: int | None = None,
|
optimize: int | None = None,
|
||||||
jpg_quality: int | None = None,
|
jpeg_quality: int | None = None,
|
||||||
png_quality: int | None = None,
|
png_quality: int | None = None,
|
||||||
jbig2_lossy: bool | None = None, # Deprecated, ignored
|
jbig2_lossy: bool | None = None, # Deprecated, ignored
|
||||||
jbig2_page_group_size: int | None = None, # Deprecated, ignored
|
jbig2_page_group_size: int | None = None, # Deprecated, ignored
|
||||||
|
|||||||
@@ -27,9 +27,12 @@ from ocrmypdf.exceptions import InputFileError
|
|||||||
from ocrmypdf.helpers import remove_all_log_handlers
|
from ocrmypdf.helpers import remove_all_log_handlers
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
|
from logging import LogRecord
|
||||||
from typing import TypeAlias
|
from typing import TypeAlias
|
||||||
|
|
||||||
Queue: TypeAlias = multiprocessing.queues.Queue | queue.Queue
|
Queue: TypeAlias = (
|
||||||
|
multiprocessing.queues.Queue[LogRecord | None] | queue.Queue[LogRecord | None]
|
||||||
|
)
|
||||||
UserInit: TypeAlias = Callable[[], None]
|
UserInit: TypeAlias = Callable[[], None]
|
||||||
WorkerInit: TypeAlias = Callable[[Queue, UserInit, int], None]
|
WorkerInit: TypeAlias = Callable[[Queue, UserInit, int], None]
|
||||||
|
|
||||||
@@ -99,7 +102,9 @@ def thread_init(q: Queue, user_init: UserInit, loglevel) -> None:
|
|||||||
return
|
return
|
||||||
|
|
||||||
|
|
||||||
def setup_executor(use_threads: bool) -> tuple[Queue, Executor, WorkerInit]:
|
def setup_executor(
|
||||||
|
use_threads: bool,
|
||||||
|
) -> tuple[Queue, FuturesExecutorClass, WorkerInit]:
|
||||||
if not use_threads:
|
if not use_threads:
|
||||||
# Some execution environments like AWS Lambda and Termux do not support
|
# Some execution environments like AWS Lambda and Termux do not support
|
||||||
# semaphores. Check if semaphore support is available, and if not, fall back
|
# semaphores. Check if semaphore support is available, and if not, fall back
|
||||||
@@ -112,6 +117,8 @@ def setup_executor(use_threads: bool) -> tuple[Queue, Executor, WorkerInit]:
|
|||||||
except ImportError:
|
except ImportError:
|
||||||
use_threads = True
|
use_threads = True
|
||||||
|
|
||||||
|
loq_queue: Queue
|
||||||
|
executor_class: FuturesExecutorClass
|
||||||
if use_threads:
|
if use_threads:
|
||||||
loq_queue = queue.Queue(-1)
|
loq_queue = queue.Queue(-1)
|
||||||
executor_class = ThreadPoolExecutor
|
executor_class = ThreadPoolExecutor
|
||||||
|
|||||||
@@ -198,6 +198,7 @@ def _process_image_for_output(
|
|||||||
'png16m',
|
'png16m',
|
||||||
'pngalpha',
|
'pngalpha',
|
||||||
)
|
)
|
||||||
|
format_name: Literal['PNG', 'TIFF', 'JPEG']
|
||||||
if raster_device_lower in png_devices:
|
if raster_device_lower in png_devices:
|
||||||
format_name = 'PNG'
|
format_name = 'PNG'
|
||||||
elif raster_device_lower in ('jpeg', 'jpeggray', 'jpg'):
|
elif raster_device_lower in ('jpeg', 'jpeggray', 'jpg'):
|
||||||
|
|||||||
@@ -130,7 +130,7 @@ class TesseractOptions(BaseModel):
|
|||||||
metavar='PSM',
|
metavar='PSM',
|
||||||
choices=range(0, 14),
|
choices=range(0, 14),
|
||||||
dest=f'{namespace}_pagesegmode',
|
dest=f'{namespace}_pagesegmode',
|
||||||
help="Set Tesseract page segmentation mode (see tesseract --help).",
|
help="Set Tesseract page segmentation mode (see tesseract --help-extra).",
|
||||||
)
|
)
|
||||||
|
|
||||||
tess.add_argument(
|
tess.add_argument(
|
||||||
@@ -168,7 +168,7 @@ class TesseractOptions(BaseModel):
|
|||||||
tess.add_argument(
|
tess.add_argument(
|
||||||
f'--{namespace}-timeout',
|
f'--{namespace}-timeout',
|
||||||
default=180.0,
|
default=180.0,
|
||||||
type=numeric(float, 0),
|
type=numeric(float, 0.0),
|
||||||
metavar='SECONDS',
|
metavar='SECONDS',
|
||||||
dest=f'{namespace}_timeout',
|
dest=f'{namespace}_timeout',
|
||||||
help=(
|
help=(
|
||||||
@@ -183,7 +183,7 @@ class TesseractOptions(BaseModel):
|
|||||||
tess.add_argument(
|
tess.add_argument(
|
||||||
f'--{namespace}-non-ocr-timeout',
|
f'--{namespace}-non-ocr-timeout',
|
||||||
default=180.0,
|
default=180.0,
|
||||||
type=numeric(float, 0),
|
type=numeric(float, 0.0),
|
||||||
metavar='SECONDS',
|
metavar='SECONDS',
|
||||||
dest=f'{namespace}_non_ocr_timeout',
|
dest=f'{namespace}_non_ocr_timeout',
|
||||||
help=(
|
help=(
|
||||||
|
|||||||
+4
-4
@@ -362,7 +362,7 @@ Online documentation is located at:
|
|||||||
)
|
)
|
||||||
ocrsettings.add_argument(
|
ocrsettings.add_argument(
|
||||||
'--skip-big',
|
'--skip-big',
|
||||||
type=numeric(float, 0, 5000),
|
type=numeric(float, 0.0, 5000.0),
|
||||||
metavar='MPixels',
|
metavar='MPixels',
|
||||||
help="Skip OCR on pages larger than the specified amount of megapixels, "
|
help="Skip OCR on pages larger than the specified amount of megapixels, "
|
||||||
"but include skipped pages in final output",
|
"but include skipped pages in final output",
|
||||||
@@ -398,7 +398,7 @@ Online documentation is located at:
|
|||||||
advanced.add_argument(
|
advanced.add_argument(
|
||||||
'--max-image-mpixels',
|
'--max-image-mpixels',
|
||||||
action='store',
|
action='store',
|
||||||
type=numeric(float, 0),
|
type=numeric(float, 0.0),
|
||||||
metavar='MPixels',
|
metavar='MPixels',
|
||||||
help="Set maximum number of megapixels to unpack before treating an image as a "
|
help="Set maximum number of megapixels to unpack before treating an image as a "
|
||||||
"decompression bomb",
|
"decompression bomb",
|
||||||
@@ -438,14 +438,14 @@ Online documentation is located at:
|
|||||||
advanced.add_argument(
|
advanced.add_argument(
|
||||||
'--rotate-pages-threshold',
|
'--rotate-pages-threshold',
|
||||||
default=DEFAULT_ROTATE_PAGES_THRESHOLD,
|
default=DEFAULT_ROTATE_PAGES_THRESHOLD,
|
||||||
type=numeric(float, 0, 1000),
|
type=numeric(float, 0.0, 1000.0),
|
||||||
metavar='CONFIDENCE',
|
metavar='CONFIDENCE',
|
||||||
help="Only rotate pages when confidence is above this value (arbitrary "
|
help="Only rotate pages when confidence is above this value (arbitrary "
|
||||||
"units reported by tesseract)",
|
"units reported by tesseract)",
|
||||||
)
|
)
|
||||||
advanced.add_argument(
|
advanced.add_argument(
|
||||||
'--fast-web-view',
|
'--fast-web-view',
|
||||||
type=numeric(float, 0),
|
type=numeric(float, 0.0),
|
||||||
default=1.0,
|
default=1.0,
|
||||||
metavar="MEGABYTES",
|
metavar="MEGABYTES",
|
||||||
help="If the size of file is more than this threshold (in MB), then "
|
help="If the size of file is more than this threshold (in MB), then "
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ This module provides font infrastructure for the fpdf2 PDF renderer. It includes
|
|||||||
- MultiFontManager: Automatic font selection for multilingual documents
|
- MultiFontManager: Automatic font selection for multilingual documents
|
||||||
- SystemFontProvider: System font discovery
|
- SystemFontProvider: System font discovery
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from ocrmypdf.font.font_manager import FontManager
|
from ocrmypdf.font.font_manager import FontManager
|
||||||
@@ -17,6 +18,7 @@ from ocrmypdf.font.font_provider import (
|
|||||||
BuiltinFontProvider,
|
BuiltinFontProvider,
|
||||||
ChainedFontProvider,
|
ChainedFontProvider,
|
||||||
FontProvider,
|
FontProvider,
|
||||||
|
GlyphSearchingFontProvider,
|
||||||
)
|
)
|
||||||
from ocrmypdf.font.multi_font_manager import MultiFontManager
|
from ocrmypdf.font.multi_font_manager import MultiFontManager
|
||||||
from ocrmypdf.font.system_font_provider import SystemFontProvider
|
from ocrmypdf.font.system_font_provider import SystemFontProvider
|
||||||
@@ -24,6 +26,7 @@ from ocrmypdf.font.system_font_provider import SystemFontProvider
|
|||||||
__all__ = [
|
__all__ = [
|
||||||
"FontManager",
|
"FontManager",
|
||||||
"FontProvider",
|
"FontProvider",
|
||||||
|
"GlyphSearchingFontProvider",
|
||||||
"BuiltinFontProvider",
|
"BuiltinFontProvider",
|
||||||
"ChainedFontProvider",
|
"ChainedFontProvider",
|
||||||
"MultiFontManager",
|
"MultiFontManager",
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import logging
|
import logging
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Protocol
|
from typing import Protocol, runtime_checkable
|
||||||
|
|
||||||
from ocrmypdf.font.font_manager import FontManager
|
from ocrmypdf.font.font_manager import FontManager
|
||||||
|
|
||||||
@@ -52,6 +52,34 @@ class FontProvider(Protocol):
|
|||||||
...
|
...
|
||||||
|
|
||||||
|
|
||||||
|
@runtime_checkable
|
||||||
|
class GlyphSearchingFontProvider(Protocol):
|
||||||
|
"""Optional capability: find a font by glyph coverage rather than by name.
|
||||||
|
|
||||||
|
A provider only knows a limited set of logical font names, but it may have
|
||||||
|
access to many more fonts than it can name (e.g. the ~100 script-specific
|
||||||
|
Noto faces macOS installs). Implementing this lets MultiFontManager use
|
||||||
|
them as a last resort instead of falling back to glyphless rendering.
|
||||||
|
|
||||||
|
Providers that do not implement this are used as-is; the capability is
|
||||||
|
detected at runtime with ``isinstance``.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def find_font_with_glyphs(self, text: str) -> tuple[str, FontManager] | None:
|
||||||
|
"""Find a font that has glyphs for every character in text.
|
||||||
|
|
||||||
|
The returned name must subsequently resolve through ``get_font()``, so
|
||||||
|
that callers can cache the selection by name.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
text: Text the font must fully cover
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
(logical font name, FontManager), or None if no font covers text
|
||||||
|
"""
|
||||||
|
...
|
||||||
|
|
||||||
|
|
||||||
class BuiltinFontProvider:
|
class BuiltinFontProvider:
|
||||||
"""Font provider using builtin fonts from ocrmypdf/data directory."""
|
"""Font provider using builtin fonts from ocrmypdf/data directory."""
|
||||||
|
|
||||||
@@ -119,6 +147,18 @@ class BuiltinFontProvider:
|
|||||||
"""Get the glyphless fallback font."""
|
"""Get the glyphless fallback font."""
|
||||||
return self._fonts['Occulta']
|
return self._fonts['Occulta']
|
||||||
|
|
||||||
|
def find_font_with_glyphs(self, text: str) -> tuple[str, FontManager] | None:
|
||||||
|
"""Find a bundled font that covers text, ignoring glyphless Occulta."""
|
||||||
|
if not text:
|
||||||
|
return None
|
||||||
|
codepoints = {ord(c) for c in text}
|
||||||
|
for name, font in self._fonts.items():
|
||||||
|
if name == 'Occulta':
|
||||||
|
continue
|
||||||
|
if all(font.has_glyph(cp) for cp in codepoints):
|
||||||
|
return name, font
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
class ChainedFontProvider:
|
class ChainedFontProvider:
|
||||||
"""Font provider that tries multiple providers in order.
|
"""Font provider that tries multiple providers in order.
|
||||||
@@ -170,6 +210,25 @@ class ChainedFontProvider:
|
|||||||
result.append(name)
|
result.append(name)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
def find_font_with_glyphs(self, text: str) -> tuple[str, FontManager] | None:
|
||||||
|
"""Ask each capable provider in turn for a font that covers text.
|
||||||
|
|
||||||
|
Providers that don't implement the search are skipped.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
text: Text the font must fully cover
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
(logical font name, FontManager) from the first provider with a
|
||||||
|
match, or None if no provider found one
|
||||||
|
"""
|
||||||
|
for provider in self.providers:
|
||||||
|
if not isinstance(provider, GlyphSearchingFontProvider):
|
||||||
|
continue
|
||||||
|
if found := provider.find_font_with_glyphs(text):
|
||||||
|
return found
|
||||||
|
return None
|
||||||
|
|
||||||
def get_fallback_font(self) -> FontManager:
|
def get_fallback_font(self) -> FontManager:
|
||||||
"""Get the glyphless fallback font.
|
"""Get the glyphless fallback font.
|
||||||
|
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ language hints and glyph coverage analysis.
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import unicodedata
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from ocrmypdf.font.font_manager import FontManager
|
from ocrmypdf.font.font_manager import FontManager
|
||||||
@@ -17,6 +18,7 @@ from ocrmypdf.font.font_provider import (
|
|||||||
BuiltinFontProvider,
|
BuiltinFontProvider,
|
||||||
ChainedFontProvider,
|
ChainedFontProvider,
|
||||||
FontProvider,
|
FontProvider,
|
||||||
|
GlyphSearchingFontProvider,
|
||||||
)
|
)
|
||||||
from ocrmypdf.font.system_font_provider import SystemFontProvider
|
from ocrmypdf.font.system_font_provider import SystemFontProvider
|
||||||
|
|
||||||
@@ -33,9 +35,17 @@ class MultiFontManager:
|
|||||||
Font selection strategy:
|
Font selection strategy:
|
||||||
1. Try language-preferred font (if language hint available)
|
1. Try language-preferred font (if language hint available)
|
||||||
2. Try fallback fonts in order by glyph coverage
|
2. Try fallback fonts in order by glyph coverage
|
||||||
3. Fall back to Occulta.ttf (glyphless fallback)
|
3. Ask the provider for any installed font that covers the text
|
||||||
|
4. Fall back to Occulta.ttf (glyphless fallback)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
# How many uncoverable characters to name in the missing-font warning
|
||||||
|
MAX_REPORTED_CHARS = 3
|
||||||
|
|
||||||
|
# How many characters of a word to look up individually when composing that
|
||||||
|
# warning; each lookup may scan every font installed on the system
|
||||||
|
MAX_EXAMINED_CHARS = 8
|
||||||
|
|
||||||
# Language to font mapping
|
# Language to font mapping
|
||||||
# Keys are ISO 639-2/3 codes or Tesseract language codes
|
# Keys are ISO 639-2/3 codes or Tesseract language codes
|
||||||
LANGUAGE_FONT_MAP = {
|
LANGUAGE_FONT_MAP = {
|
||||||
@@ -173,6 +183,9 @@ class MultiFontManager:
|
|||||||
self._selection_cache: dict[tuple[str, str | None], str] = {}
|
self._selection_cache: dict[tuple[str, str | None], str] = {}
|
||||||
# Track whether we've warned about missing fonts (warn once per script)
|
# Track whether we've warned about missing fonts (warn once per script)
|
||||||
self._warned_scripts: set[str] = set()
|
self._warned_scripts: set[str] = set()
|
||||||
|
# Fonts found by glyph coverage rather than by name, tried before
|
||||||
|
# repeating the (expensive) provider search
|
||||||
|
self._discovered_fonts: list[str] = []
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def fonts(self) -> dict[str, FontManager]:
|
def fonts(self) -> dict[str, FontManager]:
|
||||||
@@ -208,7 +221,8 @@ class MultiFontManager:
|
|||||||
Uses a hybrid approach:
|
Uses a hybrid approach:
|
||||||
1. Language-based selection (if language hint available)
|
1. Language-based selection (if language hint available)
|
||||||
2. Ordered fallback through available fonts by glyph coverage
|
2. Ordered fallback through available fonts by glyph coverage
|
||||||
3. Final fallback to Occulta.ttf (glyphless)
|
3. Provider search over every installed font, by glyph coverage
|
||||||
|
4. Final fallback to Occulta.ttf (glyphless)
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
word_text: The text content of the word
|
word_text: The text content of the word
|
||||||
@@ -233,19 +247,50 @@ class MultiFontManager:
|
|||||||
if result := self._try_font(preferred, word_text, cache_key):
|
if result := self._try_font(preferred, word_text, cache_key):
|
||||||
return result
|
return result
|
||||||
|
|
||||||
# Phase 2: Try fallback fonts in order
|
# Phase 2: Try fallback fonts in order, then anything a previous
|
||||||
for font_name in self.FALLBACK_FONTS:
|
# coverage search turned up
|
||||||
|
for font_name in [*self.FALLBACK_FONTS, *self._discovered_fonts]:
|
||||||
if font_name in tried_fonts:
|
if font_name in tried_fonts:
|
||||||
continue
|
continue
|
||||||
|
tried_fonts.add(font_name)
|
||||||
if result := self._try_font(font_name, word_text, cache_key):
|
if result := self._try_font(font_name, word_text, cache_key):
|
||||||
return result
|
return result
|
||||||
|
|
||||||
# Phase 3: Glyphless fallback (always succeeds)
|
# Phase 3: Ask the provider to search every installed font. The named
|
||||||
|
# families cover common scripts only, but systems ship many more (macOS
|
||||||
|
# installs ~100 Noto faces), and those should be used before giving up
|
||||||
|
# on rendering the text at all. See issue #1722.
|
||||||
|
if found := self._search_font_by_coverage(word_text):
|
||||||
|
font_name, font = found
|
||||||
|
self._selection_cache[cache_key] = font_name
|
||||||
|
return font
|
||||||
|
|
||||||
|
# Phase 4: Glyphless fallback (always succeeds)
|
||||||
# Warn if we're falling back for non-ASCII text (likely missing font)
|
# Warn if we're falling back for non-ASCII text (likely missing font)
|
||||||
self._warn_missing_font(word_text, line_language)
|
self._warn_missing_font(word_text, line_language)
|
||||||
self._selection_cache[cache_key] = 'Occulta'
|
self._selection_cache[cache_key] = 'Occulta'
|
||||||
return self.font_provider.get_fallback_font()
|
return self.font_provider.get_fallback_font()
|
||||||
|
|
||||||
|
def _search_font_by_coverage(self, text: str) -> tuple[str, FontManager] | None:
|
||||||
|
"""Search the provider for any font covering text, if it supports it.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
text: Text the font must fully cover
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
(font name, FontManager), or None if unsupported or nothing matched
|
||||||
|
"""
|
||||||
|
provider = self.font_provider
|
||||||
|
if not isinstance(provider, GlyphSearchingFontProvider):
|
||||||
|
return None
|
||||||
|
found = provider.find_font_with_glyphs(text)
|
||||||
|
if found is None:
|
||||||
|
return None
|
||||||
|
font_name, _font = found
|
||||||
|
if font_name not in self._discovered_fonts:
|
||||||
|
self._discovered_fonts.append(font_name)
|
||||||
|
return found
|
||||||
|
|
||||||
def _warn_missing_font(self, word_text: str, line_language: str | None) -> None:
|
def _warn_missing_font(self, word_text: str, line_language: str | None) -> None:
|
||||||
"""Warn user about missing font for non-Latin text.
|
"""Warn user about missing font for non-Latin text.
|
||||||
|
|
||||||
@@ -264,26 +309,97 @@ class MultiFontManager:
|
|||||||
|
|
||||||
self._warned_scripts.add(warn_key)
|
self._warned_scripts.add(warn_key)
|
||||||
|
|
||||||
|
uncoverable = self._uncoverable_characters(word_text)
|
||||||
|
if not uncoverable:
|
||||||
|
# Every character has a font, but no single font has them all.
|
||||||
|
# Telling the user to install fonts would be wrong advice here.
|
||||||
|
log.warning(
|
||||||
|
"Text mixing scripts that no single installed font covers (%r) "
|
||||||
|
"was added as an invisible text layer: it stays searchable and "
|
||||||
|
"copyable, but appears blank when highlighted in a PDF viewer. "
|
||||||
|
"Installing more fonts will not help; OCRmyPDF uses one font "
|
||||||
|
"per word.",
|
||||||
|
word_text,
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
missing = self._describe_characters(uncoverable)
|
||||||
if line_language and line_language in self.LANGUAGE_FONT_MAP:
|
if line_language and line_language in self.LANGUAGE_FONT_MAP:
|
||||||
font_family = self.LANGUAGE_FONT_MAP[line_language].removesuffix('-Regular')
|
font_family = self.LANGUAGE_FONT_MAP[line_language].removesuffix('-Regular')
|
||||||
log.warning(
|
log.warning(
|
||||||
"No installed font has glyphs for the detected '%s' text, so "
|
"No installed font has glyphs for the detected '%s' text (%s), "
|
||||||
"it was added as an invisible text layer: it stays searchable "
|
"so it was added as an invisible text layer: it stays searchable "
|
||||||
"and copyable, but appears blank when highlighted in a PDF "
|
"and copyable, but appears blank when highlighted in a PDF "
|
||||||
"viewer. Install the %s font family (via your OS package "
|
"viewer. Install the %s font family (via your OS package "
|
||||||
"manager or https://fonts.google.com/noto) for full rendering.",
|
"manager or https://fonts.google.com/noto) for full rendering.",
|
||||||
line_language,
|
line_language,
|
||||||
|
missing,
|
||||||
font_family,
|
font_family,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
log.warning(
|
log.warning(
|
||||||
"No installed font has glyphs for some of the detected text, "
|
"No installed font has glyphs for some of the detected text "
|
||||||
"so it was added as an invisible text layer: it stays "
|
"(%s), so it was added as an invisible text layer: it stays "
|
||||||
"searchable and copyable, but appears blank when highlighted "
|
"searchable and copyable, but appears blank when highlighted "
|
||||||
"in a PDF viewer. Install the matching Noto fonts "
|
"in a PDF viewer. Install a Noto font covering that script "
|
||||||
"(https://fonts.google.com/noto) for full rendering."
|
"(https://fonts.google.com/noto) for full rendering.",
|
||||||
|
missing,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _uncoverable_characters(self, word_text: str) -> list[str]:
|
||||||
|
"""Find the characters of word_text that no installed font can render.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
word_text: The word that fell back to glyphless rendering
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The distinct uncoverable characters, in order of first appearance,
|
||||||
|
considering at most MAX_EXAMINED_CHARS of them
|
||||||
|
"""
|
||||||
|
candidates = [
|
||||||
|
char
|
||||||
|
for char in dict.fromkeys(word_text) # de-duplicate, keep order
|
||||||
|
if not char.isspace() and not self._is_char_renderable(char)
|
||||||
|
]
|
||||||
|
# The named fonts missed these, but the provider may still have a font
|
||||||
|
# for them, so confirm before telling the user to install anything. The
|
||||||
|
# search walks every installed font, hence the cap on how many
|
||||||
|
# characters we are willing to look up for one warning.
|
||||||
|
return [
|
||||||
|
char
|
||||||
|
for char in candidates[: self.MAX_EXAMINED_CHARS]
|
||||||
|
if self._search_font_by_coverage(char) is None
|
||||||
|
]
|
||||||
|
|
||||||
|
def _describe_characters(self, chars: list[str]) -> str:
|
||||||
|
"""Describe characters by codepoint and Unicode name.
|
||||||
|
|
||||||
|
Naming the codepoints tells the user which font to install even for
|
||||||
|
scripts OCRmyPDF has no language mapping for, which the generic
|
||||||
|
"install the matching Noto fonts" advice did not. See issue #1722.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
chars: Characters to describe
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Human-readable description, truncated to MAX_REPORTED_CHARS
|
||||||
|
"""
|
||||||
|
described = ", ".join(
|
||||||
|
f"{char!r} U+{ord(char):04X} {unicodedata.name(char, 'unnamed character')}"
|
||||||
|
for char in chars[: self.MAX_REPORTED_CHARS]
|
||||||
|
)
|
||||||
|
if len(chars) > self.MAX_REPORTED_CHARS:
|
||||||
|
described += f", and {len(chars) - self.MAX_REPORTED_CHARS} more"
|
||||||
|
return described
|
||||||
|
|
||||||
|
def _is_char_renderable(self, char: str) -> bool:
|
||||||
|
"""Check whether any font already known to us has a glyph for char."""
|
||||||
|
for font_name in [*self.FALLBACK_FONTS, *self._discovered_fonts]:
|
||||||
|
font = self.font_provider.get_font(font_name)
|
||||||
|
if font is not None and self._has_all_glyphs(font, char):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
def _has_all_glyphs(self, font: FontManager, text: str) -> bool:
|
def _has_all_glyphs(self, font: FontManager, text: str) -> bool:
|
||||||
"""Check if a font has glyphs for all characters in text.
|
"""Check if a font has glyphs for all characters in text.
|
||||||
|
|
||||||
|
|||||||
@@ -209,6 +209,13 @@ class SystemFontProvider:
|
|||||||
self._not_found: set[str] = set()
|
self._not_found: set[str] = set()
|
||||||
# Cached font directories (computed lazily)
|
# Cached font directories (computed lazily)
|
||||||
self._font_dirs: list[Path] | None = None
|
self._font_dirs: list[Path] | None = None
|
||||||
|
# Cached (logical name, path) of every Noto face on the system, in the
|
||||||
|
# order the coverage search should try them (computed lazily)
|
||||||
|
self._noto_candidates: list[tuple[str, Path]] | None = None
|
||||||
|
# Memoized results of find_font_with_glyphs(), keyed by codepoint set
|
||||||
|
self._coverage_cache: dict[frozenset[int], str | None] = {}
|
||||||
|
# Font files that failed to load, so we only complain about them once
|
||||||
|
self._unloadable: set[Path] = set()
|
||||||
|
|
||||||
def _get_platform(self) -> str:
|
def _get_platform(self) -> str:
|
||||||
"""Get the current platform identifier.
|
"""Get the current platform identifier.
|
||||||
@@ -274,9 +281,7 @@ class SystemFontProvider:
|
|||||||
try:
|
try:
|
||||||
matches = list(font_dir.rglob(pattern))
|
matches = list(font_dir.rglob(pattern))
|
||||||
if matches:
|
if matches:
|
||||||
log.debug(
|
log.debug("Found system font %s at %s", font_name, matches[0])
|
||||||
"Found system font %s at %s", font_name, matches[0]
|
|
||||||
)
|
|
||||||
return matches[0]
|
return matches[0]
|
||||||
except PermissionError:
|
except PermissionError:
|
||||||
# Skip directories we can't read
|
# Skip directories we can't read
|
||||||
@@ -354,6 +359,145 @@ class SystemFontProvider:
|
|||||||
return best[1]
|
return best[1]
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _family_base(stem: str) -> str | None:
|
||||||
|
"""Get the Noto family base a filename stem is the Regular face of.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
stem: Filename without extension, e.g. 'NotoSansCherokee-Regular'
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The family base ('NotoSansCherokee') or None if the stem is not a
|
||||||
|
Noto font, or is a weight/slope variant such as '-Bold' or
|
||||||
|
'-Italic' that should not stand in for the family.
|
||||||
|
"""
|
||||||
|
head = stem.split('[', 1)[0] # drop variable-font axes, e.g. '[wght]'
|
||||||
|
if head.endswith('-Regular'):
|
||||||
|
head = head[: -len('-Regular')]
|
||||||
|
elif head.endswith('-VF'):
|
||||||
|
head = head[: -len('-VF')]
|
||||||
|
elif '-' in head:
|
||||||
|
return None
|
||||||
|
return head if head.startswith('Noto') else None
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _candidate_sort_key(cls, base: str) -> tuple[int, int, str]:
|
||||||
|
"""Rank a family base for the coverage search.
|
||||||
|
|
||||||
|
Sans comes before serif before everything else, and plain families come
|
||||||
|
ahead of their narrower UI and Mono cousins.
|
||||||
|
"""
|
||||||
|
if base.startswith('NotoSans'):
|
||||||
|
family_rank = 0
|
||||||
|
elif base.startswith('NotoSerif'):
|
||||||
|
family_rank = 1
|
||||||
|
else:
|
||||||
|
family_rank = 2
|
||||||
|
narrow_use = base.endswith('UI') or base.startswith('NotoSansMono')
|
||||||
|
return (family_rank, int(narrow_use), base)
|
||||||
|
|
||||||
|
def _get_noto_candidates(self) -> list[tuple[str, Path]]:
|
||||||
|
"""Enumerate every Noto family installed on the system.
|
||||||
|
|
||||||
|
Scans each font directory once and keeps the best-ranked file per
|
||||||
|
family, so a family present in several directories or in several
|
||||||
|
variants contributes a single candidate.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of (logical font name, path) in the order to try them.
|
||||||
|
"""
|
||||||
|
if self._noto_candidates is not None:
|
||||||
|
return self._noto_candidates
|
||||||
|
|
||||||
|
best: dict[str, tuple[int, Path]] = {}
|
||||||
|
for font_dir in self._get_font_dirs():
|
||||||
|
if not font_dir.exists():
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
paths = sorted(font_dir.rglob('Noto*'))
|
||||||
|
except OSError:
|
||||||
|
# Skip directories we can't read
|
||||||
|
continue
|
||||||
|
for path in paths:
|
||||||
|
if path.suffix.lower() not in self._FONT_EXTENSIONS:
|
||||||
|
continue
|
||||||
|
base = self._family_base(path.stem)
|
||||||
|
if base is None:
|
||||||
|
continue
|
||||||
|
kind = self._classify_variant(path.stem, base)
|
||||||
|
if kind is None:
|
||||||
|
continue
|
||||||
|
rank = self._VARIANT_RANK[kind]
|
||||||
|
if base not in best or rank < best[base][0]:
|
||||||
|
best[base] = (rank, path)
|
||||||
|
|
||||||
|
self._noto_candidates = [
|
||||||
|
(f'{base}-Regular', path)
|
||||||
|
for base, (_rank, path) in sorted(
|
||||||
|
best.items(), key=lambda item: self._candidate_sort_key(item[0])
|
||||||
|
)
|
||||||
|
]
|
||||||
|
return self._noto_candidates
|
||||||
|
|
||||||
|
def find_font_with_glyphs(self, text: str) -> tuple[str, FontManager] | None:
|
||||||
|
"""Find any installed Noto font that covers every character in text.
|
||||||
|
|
||||||
|
``NOTO_FONT_PATTERNS`` enumerates the couple dozen scripts OCRmyPDF
|
||||||
|
knows by name, but systems ship far more: macOS alone installs around a
|
||||||
|
hundred script-specific Noto faces in
|
||||||
|
``/System/Library/Fonts/Supplemental``. This is the last resort that
|
||||||
|
makes those usable, so a document is only rendered glyphless when no
|
||||||
|
installed font can actually cover it. See issue #1722.
|
||||||
|
|
||||||
|
This walks every Noto face on the system and is therefore expensive;
|
||||||
|
results are memoized, and callers should only reach it after the named
|
||||||
|
fonts have failed.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
text: Text that the returned font must fully cover
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
(logical font name, FontManager) of the first covering font, or
|
||||||
|
None if nothing installed covers the text.
|
||||||
|
"""
|
||||||
|
if not text:
|
||||||
|
return None
|
||||||
|
needed = frozenset(ord(c) for c in text)
|
||||||
|
|
||||||
|
if needed in self._coverage_cache:
|
||||||
|
cached_name = self._coverage_cache[needed]
|
||||||
|
if cached_name is None:
|
||||||
|
return None
|
||||||
|
if cached := self._font_cache.get(cached_name):
|
||||||
|
return cached_name, cached
|
||||||
|
|
||||||
|
for font_name, path in self._get_noto_candidates():
|
||||||
|
font = self._font_cache.get(font_name)
|
||||||
|
if font is None:
|
||||||
|
if path in self._unloadable:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
font = FontManager(path)
|
||||||
|
except Exception as e:
|
||||||
|
log.debug("Skipping unreadable font %s: %s", path, e)
|
||||||
|
self._unloadable.add(path)
|
||||||
|
continue
|
||||||
|
if all(font.has_glyph(cp) for cp in needed):
|
||||||
|
# Keep only fonts we actually use; the rest are released so a
|
||||||
|
# full scan doesn't retain every font file on the system.
|
||||||
|
self._font_cache[font_name] = font
|
||||||
|
self._not_found.discard(font_name)
|
||||||
|
self._coverage_cache[needed] = font_name
|
||||||
|
log.debug(
|
||||||
|
"Found system font %s at %s (glyph coverage match)",
|
||||||
|
font_name,
|
||||||
|
path,
|
||||||
|
)
|
||||||
|
return font_name, font
|
||||||
|
|
||||||
|
self._coverage_cache[needed] = None
|
||||||
|
return None
|
||||||
|
|
||||||
def get_font(self, font_name: str) -> FontManager | None:
|
def get_font(self, font_name: str) -> FontManager | None:
|
||||||
"""Get a FontManager for the named font (lazy loading).
|
"""Get a FontManager for the named font (lazy loading).
|
||||||
|
|
||||||
|
|||||||
@@ -6,6 +6,7 @@
|
|||||||
This module provides the PDF renderer using fpdf2 for creating
|
This module provides the PDF renderer using fpdf2 for creating
|
||||||
searchable OCR text layers.
|
searchable OCR text layers.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from ocrmypdf.fpdf_renderer.renderer import (
|
from ocrmypdf.fpdf_renderer.renderer import (
|
||||||
|
|||||||
@@ -14,9 +14,11 @@ import unicodedata
|
|||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from math import atan, cos, degrees, radians, sin, sqrt
|
from math import atan, cos, degrees, radians, sin, sqrt
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from typing import cast
|
||||||
|
|
||||||
from fpdf import FPDF
|
from fpdf import FPDF
|
||||||
from fpdf.enums import PDFResourceType, TextMode
|
from fpdf.enums import PDFResourceType, TextMode
|
||||||
|
from fpdf.fonts import TTFFont
|
||||||
from pikepdf import Matrix, Rectangle
|
from pikepdf import Matrix, Rectangle
|
||||||
|
|
||||||
from ocrmypdf.font import FontManager, MultiFontManager
|
from ocrmypdf.font import FontManager, MultiFontManager
|
||||||
@@ -241,10 +243,16 @@ class Fpdf2PdfRenderer:
|
|||||||
pdf: FPDF instance to render into
|
pdf: FPDF instance to render into
|
||||||
"""
|
"""
|
||||||
# Add page with correct dimensions
|
# Add page with correct dimensions
|
||||||
|
# fpdf2's add_page() stub says format: str, but its docstring and
|
||||||
|
# get_page_format() helper confirm a (width, height) tuple is
|
||||||
|
# supported too - the annotation on add_page() itself is just wrong.
|
||||||
pdf.add_page(
|
pdf.add_page(
|
||||||
format=(
|
format=cast(
|
||||||
self.coord_transform.page_width_pt,
|
'str',
|
||||||
self.coord_transform.page_height_pt,
|
(
|
||||||
|
self.coord_transform.page_width_pt,
|
||||||
|
self.coord_transform.page_height_pt,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -448,8 +456,13 @@ class Fpdf2PdfRenderer:
|
|||||||
# entirely (slope=0, no textangle) and produced garbage text in a
|
# entirely (slope=0, no textangle) and produced garbage text in a
|
||||||
# bounding box whose shape doesn't match the text content at all.
|
# bounding box whose shape doesn't match the text content at all.
|
||||||
if not self._check_aspect_ratio_plausible(
|
if not self._check_aspect_ratio_plausible(
|
||||||
pdf, words, font_size, slope_angle_deg,
|
pdf,
|
||||||
line_size_width, line_size_height, line_language,
|
words,
|
||||||
|
font_size,
|
||||||
|
slope_angle_deg,
|
||||||
|
line_size_width,
|
||||||
|
line_size_height,
|
||||||
|
line_language,
|
||||||
):
|
):
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -507,13 +520,15 @@ class Fpdf2PdfRenderer:
|
|||||||
else:
|
else:
|
||||||
word_tz = 100.0
|
word_tz = 100.0
|
||||||
|
|
||||||
word_render_data.append(WordRenderData(
|
word_render_data.append(
|
||||||
text=word.text,
|
WordRenderData(
|
||||||
x_baseline=box_llx,
|
text=word.text,
|
||||||
font_family=font_family,
|
x_baseline=box_llx,
|
||||||
word_tz=word_tz,
|
font_family=font_family,
|
||||||
is_rtl=word_is_rtl,
|
word_tz=word_tz,
|
||||||
))
|
is_rtl=word_is_rtl,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
if not word_render_data:
|
if not word_render_data:
|
||||||
return
|
return
|
||||||
@@ -561,9 +576,7 @@ class Fpdf2PdfRenderer:
|
|||||||
if line_size_width >= line_size_height:
|
if line_size_width >= line_size_height:
|
||||||
return True
|
return True
|
||||||
|
|
||||||
line_text = ' '.join(
|
line_text = ' '.join(w.text for w in words if w is not None and w.text)
|
||||||
w.text for w in words if w is not None and w.text
|
|
||||||
)
|
|
||||||
if not line_text:
|
if not line_text:
|
||||||
return True
|
return True
|
||||||
|
|
||||||
@@ -603,9 +616,7 @@ class Fpdf2PdfRenderer:
|
|||||||
line_text[:80],
|
line_text[:80],
|
||||||
)
|
)
|
||||||
if not self._logged_aspect_ratio_suppression:
|
if not self._logged_aspect_ratio_suppression:
|
||||||
log.info(
|
log.info("Suppressing OCR output text with improbable aspect ratio")
|
||||||
"Suppressing OCR output text with improbable aspect ratio"
|
|
||||||
)
|
|
||||||
self._logged_aspect_ratio_suppression = True
|
self._logged_aspect_ratio_suppression = True
|
||||||
return False
|
return False
|
||||||
|
|
||||||
@@ -679,9 +690,7 @@ class Fpdf2PdfRenderer:
|
|||||||
ops.append(f'{first_x_baseline:.2f} 0 Td')
|
ops.append(f'{first_x_baseline:.2f} 0 Td')
|
||||||
else:
|
else:
|
||||||
# Direct PDF coordinates
|
# Direct PDF coordinates
|
||||||
page_x, page_y_fpdf = transform_point(
|
page_x, page_y_fpdf = transform_point(baseline_matrix, first_x_baseline, 0)
|
||||||
baseline_matrix, first_x_baseline, 0
|
|
||||||
)
|
|
||||||
page_y_pdf = page_height - page_y_fpdf
|
page_y_pdf = page_height - page_y_fpdf
|
||||||
ops.append(f'{page_x:.2f} {page_y_pdf:.2f} Td')
|
ops.append(f'{page_x:.2f} {page_y_pdf:.2f} Td')
|
||||||
|
|
||||||
@@ -694,13 +703,15 @@ class Fpdf2PdfRenderer:
|
|||||||
# Set font if changed
|
# Set font if changed
|
||||||
if word.font_family != prev_font_family:
|
if word.font_family != prev_font_family:
|
||||||
pdf.set_font(word.font_family, size=font_size)
|
pdf.set_font(word.font_family, size=font_size)
|
||||||
|
# We only ever register fonts via add_font() with a TTF file
|
||||||
|
# (see _register_font), so set_font() always resolves to a
|
||||||
|
# TTFFont, never a built-in CoreFont or leaves it unset.
|
||||||
|
assert pdf.current_font is not None
|
||||||
# Register font resource on this page
|
# Register font resource on this page
|
||||||
pdf._resource_catalog.add(
|
pdf._resource_catalog.add(
|
||||||
PDFResourceType.FONT, pdf.current_font.i, pdf.page
|
PDFResourceType.FONT, pdf.current_font.i, pdf.page
|
||||||
)
|
)
|
||||||
ops.append(
|
ops.append(f'/F{pdf.current_font.i} {pdf.font_size_pt:.2f} Tf')
|
||||||
f'/F{pdf.current_font.i} {pdf.font_size_pt:.2f} Tf'
|
|
||||||
)
|
|
||||||
prev_font_family = word.font_family
|
prev_font_family = word.font_family
|
||||||
|
|
||||||
# Relative positioning (for words after the first)
|
# Relative positioning (for words after the first)
|
||||||
@@ -728,12 +739,8 @@ class Fpdf2PdfRenderer:
|
|||||||
advance = next_word.x_baseline - word.x_baseline
|
advance = next_word.x_baseline - word.x_baseline
|
||||||
|
|
||||||
# Add trailing space for text extraction unless both are CJK
|
# Add trailing space for text extraction unless both are CJK
|
||||||
if (
|
if advance > 0 and not (
|
||||||
advance > 0
|
self._is_cjk_only(word.text) and self._is_cjk_only(next_word.text)
|
||||||
and not (
|
|
||||||
self._is_cjk_only(word.text)
|
|
||||||
and self._is_cjk_only(next_word.text)
|
|
||||||
)
|
|
||||||
):
|
):
|
||||||
text_to_render = word.text + ' '
|
text_to_render = word.text + ' '
|
||||||
else:
|
else:
|
||||||
@@ -744,9 +751,7 @@ class Fpdf2PdfRenderer:
|
|||||||
# Use word_tz (fits word into its hOCR bbox) — Td handles
|
# Use word_tz (fits word into its hOCR bbox) — Td handles
|
||||||
# inter-word gaps, so Tz should not stretch to fill them.
|
# inter-word gaps, so Tz should not stretch to fill them.
|
||||||
ops.append(f'{word.word_tz:.2f} Tz')
|
ops.append(f'{word.word_tz:.2f} Tz')
|
||||||
ops.append(
|
ops.append(self._encode_shaped_text(pdf, text_to_render, word.is_rtl))
|
||||||
self._encode_shaped_text(pdf, text_to_render, word.is_rtl)
|
|
||||||
)
|
|
||||||
|
|
||||||
prev_x_baseline = word.x_baseline
|
prev_x_baseline = word.x_baseline
|
||||||
|
|
||||||
@@ -762,9 +767,7 @@ class Fpdf2PdfRenderer:
|
|||||||
# don't think Tz is still set from our raw operators
|
# don't think Tz is still set from our raw operators
|
||||||
pdf.font_stretching = 100
|
pdf.font_stretching = 100
|
||||||
|
|
||||||
def _encode_shaped_text(
|
def _encode_shaped_text(self, pdf: FPDF, text: str, is_rtl: bool = False) -> str:
|
||||||
self, pdf: FPDF, text: str, is_rtl: bool = False
|
|
||||||
) -> str:
|
|
||||||
"""Encode text using HarfBuzz text shaping for complex script support.
|
"""Encode text using HarfBuzz text shaping for complex script support.
|
||||||
|
|
||||||
Unlike font.encode_text() which maps unicode characters one-by-one to
|
Unlike font.encode_text() which maps unicode characters one-by-one to
|
||||||
@@ -782,6 +785,10 @@ class Fpdf2PdfRenderer:
|
|||||||
joining forms and ligature shaping is harmless.
|
joining forms and ligature shaping is harmless.
|
||||||
"""
|
"""
|
||||||
font = pdf.current_font
|
font = pdf.current_font
|
||||||
|
# We only ever register fonts via add_font() with a TTF file (see
|
||||||
|
# _register_font), so current_font is always a TTFFont - never the
|
||||||
|
# built-in CoreFont (which lacks shape_text()/escape_text()) or None.
|
||||||
|
assert isinstance(font, TTFFont)
|
||||||
if is_rtl:
|
if is_rtl:
|
||||||
# Reverse the text so that after bidi reversal by the text
|
# Reverse the text so that after bidi reversal by the text
|
||||||
# extractor, the characters end up in correct logical order.
|
# extractor, the characters end up in correct logical order.
|
||||||
|
|||||||
+52
-17
@@ -18,6 +18,7 @@ from math import isclose, isfinite
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from statistics import harmonic_mean
|
from statistics import harmonic_mean
|
||||||
from typing import (
|
from typing import (
|
||||||
|
TYPE_CHECKING,
|
||||||
Any,
|
Any,
|
||||||
Generic,
|
Generic,
|
||||||
TypeVar,
|
TypeVar,
|
||||||
@@ -26,6 +27,9 @@ from typing import (
|
|||||||
import img2pdf
|
import img2pdf
|
||||||
import pikepdf
|
import pikepdf
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from _typeshed import StrOrBytesPath
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
IMG2PDF_KWARGS = dict(engine=img2pdf.Engine.pikepdf, rotation=img2pdf.Rotation.ifvalid)
|
IMG2PDF_KWARGS = dict(engine=img2pdf.Engine.pikepdf, rotation=img2pdf.Rotation.ifvalid)
|
||||||
@@ -135,7 +139,7 @@ class Resolution(Generic[T]):
|
|||||||
return self._isclose(self.x, other.x) and self._isclose(self.y, other.y)
|
return self._isclose(self.x, other.x) and self._isclose(self.y, other.y)
|
||||||
|
|
||||||
|
|
||||||
def safe_symlink(input_file: os.PathLike, soft_link_name: os.PathLike) -> None:
|
def safe_symlink(input_file: StrOrBytesPath, soft_link_name: StrOrBytesPath) -> None:
|
||||||
"""Create a symbolic link at ``soft_link_name``, which references ``input_file``.
|
"""Create a symbolic link at ``soft_link_name``, which references ``input_file``.
|
||||||
|
|
||||||
Think of this as copying ``input_file`` to ``soft_link_name`` with less overhead.
|
Think of this as copying ``input_file`` to ``soft_link_name`` with less overhead.
|
||||||
@@ -144,11 +148,11 @@ def safe_symlink(input_file: os.PathLike, soft_link_name: os.PathLike) -> None:
|
|||||||
used since symlinks may require administrator privileges. An existing link at the
|
used since symlinks may require administrator privileges. An existing link at the
|
||||||
destination is removed.
|
destination is removed.
|
||||||
"""
|
"""
|
||||||
input_file = os.fspath(input_file)
|
input_path = Path(os.fsdecode(input_file))
|
||||||
soft_link_name = os.fspath(soft_link_name)
|
soft_link_path = Path(os.fsdecode(soft_link_name))
|
||||||
|
|
||||||
# Guard against soft linking to oneself
|
# Guard against soft linking to oneself
|
||||||
if input_file == soft_link_name:
|
if input_path == soft_link_path:
|
||||||
log.warning(
|
log.warning(
|
||||||
"No symbolic link created. You are using the original data directory "
|
"No symbolic link created. You are using the original data directory "
|
||||||
"as the working directory."
|
"as the working directory."
|
||||||
@@ -156,24 +160,24 @@ def safe_symlink(input_file: os.PathLike, soft_link_name: os.PathLike) -> None:
|
|||||||
return
|
return
|
||||||
|
|
||||||
# Soft link already exists: delete for relink?
|
# Soft link already exists: delete for relink?
|
||||||
if os.path.lexists(soft_link_name):
|
if os.path.lexists(soft_link_path):
|
||||||
# do not delete or overwrite real (non-soft link) file
|
# do not delete or overwrite real (non-soft link) file
|
||||||
if not os.path.islink(soft_link_name):
|
if not soft_link_path.is_symlink():
|
||||||
raise FileExistsError(f"{soft_link_name} exists and is not a link")
|
raise FileExistsError(f"{soft_link_path} exists and is not a link")
|
||||||
os.unlink(soft_link_name)
|
soft_link_path.unlink()
|
||||||
|
|
||||||
if not os.path.exists(input_file):
|
if not input_path.exists():
|
||||||
raise FileNotFoundError(f"trying to create a broken symlink to {input_file}")
|
raise FileNotFoundError(f"trying to create a broken symlink to {input_path}")
|
||||||
|
|
||||||
if os.name == 'nt':
|
if os.name == 'nt':
|
||||||
# Don't actually use symlinks on Windows due to permission issues
|
# Don't actually use symlinks on Windows due to permission issues
|
||||||
shutil.copyfile(input_file, soft_link_name)
|
shutil.copyfile(input_path, soft_link_path)
|
||||||
return
|
return
|
||||||
|
|
||||||
log.debug("os.symlink(%s, %s)", input_file, soft_link_name)
|
log.debug("os.symlink(%s, %s)", input_path, soft_link_path)
|
||||||
|
|
||||||
# Create symbolic link using absolute path
|
# Create symbolic link using absolute path
|
||||||
os.symlink(os.path.abspath(input_file), soft_link_name)
|
soft_link_path.symlink_to(input_path.resolve())
|
||||||
|
|
||||||
|
|
||||||
def samefile(file1: os.PathLike, file2: os.PathLike) -> bool:
|
def samefile(file1: os.PathLike, file2: os.PathLike) -> bool:
|
||||||
@@ -184,7 +188,7 @@ def samefile(file1: os.PathLike, file2: os.PathLike) -> bool:
|
|||||||
if os.name == 'nt':
|
if os.name == 'nt':
|
||||||
return file1 == file2
|
return file1 == file2
|
||||||
else:
|
else:
|
||||||
return os.path.samefile(file1, file2)
|
return Path(file1).samefile(file2)
|
||||||
|
|
||||||
|
|
||||||
def is_iterable_notstr(thing: Any) -> bool:
|
def is_iterable_notstr(thing: Any) -> bool:
|
||||||
@@ -199,7 +203,7 @@ def monotonic(seq: Sequence) -> bool:
|
|||||||
|
|
||||||
def page_number(input_file: os.PathLike) -> int:
|
def page_number(input_file: os.PathLike) -> int:
|
||||||
"""Get one-based page number implied by filename (000002.pdf -> 2)."""
|
"""Get one-based page number implied by filename (000002.pdf -> 2)."""
|
||||||
return int(os.path.basename(os.fspath(input_file))[0:6])
|
return int(Path(input_file).name[0:6])
|
||||||
|
|
||||||
|
|
||||||
def available_cpu_count() -> int:
|
def available_cpu_count() -> int:
|
||||||
@@ -214,7 +218,7 @@ def available_cpu_count() -> int:
|
|||||||
return 1
|
return 1
|
||||||
|
|
||||||
|
|
||||||
def is_file_writable(test_file: os.PathLike) -> bool:
|
def is_file_writable(test_file: StrOrBytesPath) -> bool:
|
||||||
"""Intentionally racy test if target is writable.
|
"""Intentionally racy test if target is writable.
|
||||||
|
|
||||||
We intend to write to the output file if and only if we succeed and
|
We intend to write to the output file if and only if we succeed and
|
||||||
@@ -222,7 +226,7 @@ def is_file_writable(test_file: os.PathLike) -> bool:
|
|||||||
the location is writable.
|
the location is writable.
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
p = Path(test_file)
|
p = Path(os.fsdecode(test_file))
|
||||||
if p.is_symlink():
|
if p.is_symlink():
|
||||||
p = p.resolve(strict=False)
|
p = p.resolve(strict=False)
|
||||||
|
|
||||||
@@ -329,6 +333,37 @@ def pikepdf_enable_mmap() -> None:
|
|||||||
log.debug("pikepdf mmap not available")
|
log.debug("pikepdf mmap not available")
|
||||||
|
|
||||||
|
|
||||||
|
def pikepdf_get_int(obj: pikepdf.Object, key: pikepdf.Name, default: int = 0) -> int:
|
||||||
|
"""Look up a key on a pikepdf dictionary/stream, returning a plain int.
|
||||||
|
|
||||||
|
``.get(key, default)``'s return type is the ambiguous ``Object | int``,
|
||||||
|
which does not support arithmetic or comparison against a plain int. In
|
||||||
|
pikepdf's default (implicit) conversion mode, a PDF Integer is already
|
||||||
|
unboxed to a native ``int`` by the time we see it here; under explicit
|
||||||
|
conversion mode it would instead be a ``pikepdf.Object``. ``int()``
|
||||||
|
handles both, since ``Object`` implements ``__int__``.
|
||||||
|
"""
|
||||||
|
value = obj.get(key)
|
||||||
|
return int(value) if value is not None else default
|
||||||
|
|
||||||
|
|
||||||
|
def pikepdf_get_bool(
|
||||||
|
obj: pikepdf.Object, key: pikepdf.Name, default: bool = False
|
||||||
|
) -> bool:
|
||||||
|
"""Look up a key on a pikepdf dictionary/stream, returning a plain bool.
|
||||||
|
|
||||||
|
Unlike ``int()``/``float()``, ``bool()`` is not supported on
|
||||||
|
``pikepdf.Object`` (it raises), so both conversion modes must be
|
||||||
|
handled explicitly. See :func:`pikepdf_get_int` for background.
|
||||||
|
"""
|
||||||
|
value = obj.get(key)
|
||||||
|
if value is None:
|
||||||
|
return default
|
||||||
|
if isinstance(value, bool):
|
||||||
|
return value
|
||||||
|
return value.as_bool(default)
|
||||||
|
|
||||||
|
|
||||||
def running_in_docker() -> bool:
|
def running_in_docker() -> bool:
|
||||||
"""Returns True if we seem to be running in a Docker container."""
|
"""Returns True if we seem to be running in a Docker container."""
|
||||||
return Path('/.dockerenv').exists()
|
return Path('/.dockerenv').exists()
|
||||||
|
|||||||
@@ -6,6 +6,7 @@
|
|||||||
Derived from
|
Derived from
|
||||||
https://www.loc.gov/standards/iso639-2/ascii_8bits.html
|
https://www.loc.gov/standards/iso639-2/ascii_8bits.html
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from typing import NamedTuple
|
from typing import NamedTuple
|
||||||
|
|||||||
+19
-14
@@ -12,7 +12,7 @@ import threading
|
|||||||
from collections.abc import Callable, Iterator, MutableSet, Sequence
|
from collections.abc import Callable, Iterator, MutableSet, Sequence
|
||||||
from os import fspath
|
from os import fspath
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, NamedTuple, NewType
|
from typing import Any, NamedTuple, NewType, cast
|
||||||
from zlib import compress
|
from zlib import compress
|
||||||
|
|
||||||
import img2pdf
|
import img2pdf
|
||||||
@@ -37,7 +37,7 @@ from ocrmypdf._exec import ghostscript, jbig2enc, pngquant
|
|||||||
from ocrmypdf._jobcontext import PdfContext
|
from ocrmypdf._jobcontext import PdfContext
|
||||||
from ocrmypdf._progressbar import ProgressBar
|
from ocrmypdf._progressbar import ProgressBar
|
||||||
from ocrmypdf.exceptions import OutputFileAccessError
|
from ocrmypdf.exceptions import OutputFileAccessError
|
||||||
from ocrmypdf.helpers import IMG2PDF_KWARGS, safe_symlink
|
from ocrmypdf.helpers import IMG2PDF_KWARGS, pikepdf_get_int, safe_symlink
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -449,12 +449,12 @@ def convert_to_jbig2(
|
|||||||
|
|
||||||
|
|
||||||
def _optimize_jpeg(
|
def _optimize_jpeg(
|
||||||
xref: Xref, in_jpg: Path, opt_jpg: Path, jpg_quality: int
|
xref: Xref, in_jpg: Path, opt_jpg: Path, jpeg_quality: int
|
||||||
) -> tuple[Xref, Path | None]:
|
) -> tuple[Xref, Path | None]:
|
||||||
with Image.open(in_jpg) as im:
|
with Image.open(in_jpg) as im:
|
||||||
save_kwargs: dict[str, Any] = {'optimize': True}
|
save_kwargs: dict[str, Any] = {'optimize': True}
|
||||||
if isinstance(jpg_quality, int) and 0 < jpg_quality <= 100:
|
if isinstance(jpeg_quality, int) and 0 < jpeg_quality <= 100:
|
||||||
save_kwargs['quality'] = jpg_quality
|
save_kwargs['quality'] = jpeg_quality
|
||||||
im.save(opt_jpg, **save_kwargs)
|
im.save(opt_jpg, **save_kwargs)
|
||||||
|
|
||||||
if opt_jpg.stat().st_size > in_jpg.stat().st_size:
|
if opt_jpg.stat().st_size > in_jpg.stat().st_size:
|
||||||
@@ -473,7 +473,7 @@ def transcode_jpegs(
|
|||||||
for xref in jpegs:
|
for xref in jpegs:
|
||||||
in_jpg = jpg_name(root, xref)
|
in_jpg = jpg_name(root, xref)
|
||||||
opt_jpg = in_jpg.with_suffix('.opt.jpg')
|
opt_jpg = in_jpg.with_suffix('.opt.jpg')
|
||||||
yield xref, in_jpg, opt_jpg, options.jpg_quality
|
yield xref, in_jpg, opt_jpg, options.jpeg_quality
|
||||||
|
|
||||||
def finish_jpeg(result: tuple[Xref, Path | None], pbar: ProgressBar):
|
def finish_jpeg(result: tuple[Xref, Path | None], pbar: ProgressBar):
|
||||||
xref, opt_jpg = result
|
xref, opt_jpg = result
|
||||||
@@ -527,8 +527,8 @@ def _find_deflatable_jpeg(
|
|||||||
(
|
(
|
||||||
# Don't flate very large images because it will slow down PDF viewers
|
# Don't flate very large images because it will slow down PDF viewers
|
||||||
1 <= options.optimize <= 2
|
1 <= options.optimize <= 2
|
||||||
and image.get(Name.Width, 0) < FLATE_JPEG_THRESHOLD
|
and pikepdf_get_int(image, Name.Width) < FLATE_JPEG_THRESHOLD
|
||||||
and image.get(Name.Height, 0) < FLATE_JPEG_THRESHOLD
|
and pikepdf_get_int(image, Name.Height) < FLATE_JPEG_THRESHOLD
|
||||||
)
|
)
|
||||||
or options.optimize == 3
|
or options.optimize == 3
|
||||||
)
|
)
|
||||||
@@ -608,10 +608,13 @@ def _transcode_png(pdf: Pdf, filename: Path, xref: Xref) -> bool:
|
|||||||
local_image = pdf.copy_foreign(foreign_image)
|
local_image = pdf.copy_foreign(foreign_image)
|
||||||
|
|
||||||
im_obj = pdf.get_object(xref, 0)
|
im_obj = pdf.get_object(xref, 0)
|
||||||
|
# pikepdf's Object attribute access can't statically know Filter/
|
||||||
|
# DecodeParms hold these specific subtypes, but a copied image's
|
||||||
|
# stream dictionary always does per the PDF spec.
|
||||||
im_obj.write(
|
im_obj.write(
|
||||||
local_image.read_raw_bytes(),
|
local_image.read_raw_bytes(),
|
||||||
filter=local_image.Filter,
|
filter=cast('Name | Array | list[Name] | None', local_image.Filter),
|
||||||
decode_parms=local_image.DecodeParms,
|
decode_parms=cast('Dictionary | Array | None', local_image.DecodeParms),
|
||||||
)
|
)
|
||||||
|
|
||||||
# Don't copy keys from the new image...
|
# Don't copy keys from the new image...
|
||||||
@@ -700,8 +703,8 @@ def optimize(
|
|||||||
safe_symlink(input_file, output_file)
|
safe_symlink(input_file, output_file)
|
||||||
return output_file
|
return output_file
|
||||||
|
|
||||||
if not options.jpg_quality:
|
if not options.jpeg_quality:
|
||||||
options.jpg_quality = DEFAULT_JPEG_QUALITY if options.optimize < 3 else 40
|
options.jpeg_quality = DEFAULT_JPEG_QUALITY if options.optimize < 3 else 40
|
||||||
if not options.png_quality:
|
if not options.png_quality:
|
||||||
options.png_quality = DEFAULT_PNG_QUALITY if options.optimize < 3 else 30
|
options.png_quality = DEFAULT_PNG_QUALITY if options.optimize < 3 else 30
|
||||||
|
|
||||||
@@ -763,7 +766,7 @@ def main(infile, outfile, level, jobs=1):
|
|||||||
output_file=outfile, # Required field
|
output_file=outfile, # Required field
|
||||||
jobs=jobs,
|
jobs=jobs,
|
||||||
optimize=int(level),
|
optimize=int(level),
|
||||||
jpg_quality=0, # Use default
|
jpeg_quality=0, # Use default
|
||||||
png_quality=0,
|
png_quality=0,
|
||||||
jbig2_threshold=0.85,
|
jbig2_threshold=0.85,
|
||||||
quiet=True,
|
quiet=True,
|
||||||
@@ -771,7 +774,9 @@ def main(infile, outfile, level, jobs=1):
|
|||||||
)
|
)
|
||||||
|
|
||||||
with TemporaryDirectory() as tmpdir:
|
with TemporaryDirectory() as tmpdir:
|
||||||
context = PdfContext(options, Path(tmpdir), infile, None, None)
|
# optimize() only reads context.options on this standalone path, so
|
||||||
|
# pdfinfo and plugin_manager are not needed.
|
||||||
|
context = PdfContext(options, Path(tmpdir), infile, None, None) # type: ignore[arg-type]
|
||||||
tmpout = Path(tmpdir) / 'out.pdf'
|
tmpout = Path(tmpdir) / 'out.pdf'
|
||||||
optimize(
|
optimize(
|
||||||
infile,
|
infile,
|
||||||
|
|||||||
+13
-7
@@ -12,7 +12,7 @@ from importlib.resources import files as package_files
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import pikepdf
|
import pikepdf
|
||||||
from pikepdf import Array, Dictionary, Name, Pdf, Stream
|
from pikepdf import Array, Dictionary, Name, Object, Pdf, Stream
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -137,11 +137,13 @@ def file_claims_pdfa(filename: Path):
|
|||||||
return pdfa_dict
|
return pdfa_dict
|
||||||
|
|
||||||
|
|
||||||
def _cid_font_is_embedded(type0_font: Dictionary) -> bool:
|
def _cid_font_is_embedded(type0_font: Object) -> bool:
|
||||||
"""Return True if a Type0 font's CID descendant carries embedded glyphs."""
|
"""Return True if a Type0 font's CID descendant carries embedded glyphs."""
|
||||||
for descendant in type0_font.get(Name.DescendantFonts, []):
|
for descendant in type0_font.get(Name.DescendantFonts, []):
|
||||||
descriptor = descendant.get(Name.FontDescriptor, None)
|
descriptor = descendant.get(Name.FontDescriptor, None)
|
||||||
if descriptor is not None and any(
|
# A malformed PDF may store a non-dictionary here; `key in descriptor`
|
||||||
|
# raises on those, so require a real dictionary before probing it.
|
||||||
|
if isinstance(descriptor, Dictionary) and any(
|
||||||
key in descriptor for key in (Name.FontFile, Name.FontFile2, Name.FontFile3)
|
key in descriptor for key in (Name.FontFile, Name.FontFile2, Name.FontFile3)
|
||||||
):
|
):
|
||||||
return True
|
return True
|
||||||
@@ -174,9 +176,13 @@ def find_nonembedded_cid_fonts(pdf: Pdf) -> set[str]:
|
|||||||
def scan_resources(resources, depth: int = 0) -> None:
|
def scan_resources(resources, depth: int = 0) -> None:
|
||||||
if resources is None or depth > 10:
|
if resources is None or depth > 10:
|
||||||
return
|
return
|
||||||
|
# A well-formed PDF stores dictionaries under /Font and /XObject, but a
|
||||||
|
# malformed one (common in OCR workloads) may store an array, a name, or
|
||||||
|
# another non-dictionary object. Only such dictionaries have .values(),
|
||||||
|
# so guard with isinstance rather than let the scan crash (issue #1713).
|
||||||
fonts = resources.get(Name.Font, None)
|
fonts = resources.get(Name.Font, None)
|
||||||
if fonts is not None:
|
if isinstance(fonts, Dictionary):
|
||||||
for font in fonts.values():
|
for font in fonts.as_dict().values():
|
||||||
try:
|
try:
|
||||||
if font.get(Name.Subtype) != Name.Type0:
|
if font.get(Name.Subtype) != Name.Type0:
|
||||||
continue
|
continue
|
||||||
@@ -186,8 +192,8 @@ def find_nonembedded_cid_fonts(pdf: Pdf) -> set[str]:
|
|||||||
except (AttributeError, TypeError, KeyError):
|
except (AttributeError, TypeError, KeyError):
|
||||||
continue
|
continue
|
||||||
xobjects = resources.get(Name.XObject, None)
|
xobjects = resources.get(Name.XObject, None)
|
||||||
if xobjects is not None:
|
if isinstance(xobjects, Dictionary):
|
||||||
for xobj in xobjects.values():
|
for xobj in xobjects.as_dict().values():
|
||||||
if xobj.get(Name.Subtype) == Name.Form and Name.Resources in xobj:
|
if xobj.get(Name.Subtype) == Name.Form and Name.Resources in xobj:
|
||||||
scan_resources(xobj[Name.Resources], depth + 1)
|
scan_resources(xobj[Name.Resources], depth + 1)
|
||||||
|
|
||||||
|
|||||||
@@ -172,10 +172,10 @@ def _interpret_contents(
|
|||||||
name_index = defaultdict(lambda: [])
|
name_index = defaultdict(lambda: [])
|
||||||
found_vector = False
|
found_vector = False
|
||||||
found_text = False
|
found_text = False
|
||||||
vector_ops = set('S s f F f* B B* b b*'.split())
|
vector_ops = set(['S', 's', 'f', 'F', 'f*', 'B', 'B*', 'b', 'b*'])
|
||||||
text_showing_ops = set("""TJ Tj " '""".split())
|
text_showing_ops = set(["TJ", "Tj", '"', "'"])
|
||||||
image_ops = set('BI ID EI q Q Do cm'.split())
|
image_ops = set(['BI', 'ID', 'EI', 'q', 'Q', 'Do', 'cm'])
|
||||||
color_ops = set('g rg k cs sc scn'.split())
|
color_ops = set(['g', 'rg', 'k', 'cs', 'sc', 'scn'])
|
||||||
operator_whitelist = ' '.join(vector_ops | text_showing_ops | image_ops | color_ops)
|
operator_whitelist = ' '.join(vector_ops | text_showing_ops | image_ops | color_ops)
|
||||||
|
|
||||||
for n, graphobj in enumerate(
|
for n, graphobj in enumerate(
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ from __future__ import annotations
|
|||||||
import logging
|
import logging
|
||||||
from collections.abc import Iterator
|
from collections.abc import Iterator
|
||||||
from decimal import Decimal
|
from decimal import Decimal
|
||||||
|
from typing import cast
|
||||||
|
|
||||||
from pikepdf import (
|
from pikepdf import (
|
||||||
Dictionary,
|
Dictionary,
|
||||||
@@ -20,7 +21,7 @@ from pikepdf import (
|
|||||||
UnsupportedImageTypeError,
|
UnsupportedImageTypeError,
|
||||||
)
|
)
|
||||||
|
|
||||||
from ocrmypdf.helpers import Resolution
|
from ocrmypdf.helpers import Resolution, pikepdf_get_int
|
||||||
from ocrmypdf.pdfinfo._contentstream import (
|
from ocrmypdf.pdfinfo._contentstream import (
|
||||||
ContentsInfo,
|
ContentsInfo,
|
||||||
TextMarker,
|
TextMarker,
|
||||||
@@ -54,6 +55,7 @@ class ImageInfo:
|
|||||||
|
|
||||||
_comp: int | None
|
_comp: int | None
|
||||||
_name: str
|
_name: str
|
||||||
|
_enc: Encoding | None
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
@@ -90,8 +92,8 @@ class ImageInfo:
|
|||||||
# itself. Some PDF writers use this to create a grayscale stencil
|
# itself. Some PDF writers use this to create a grayscale stencil
|
||||||
# mask. For our purposes, the effective size is the size of the
|
# mask. For our purposes, the effective size is the size of the
|
||||||
# larger component (image or smask).
|
# larger component (image or smask).
|
||||||
self._width = max(smask.get(Name.Width, 0), self._width)
|
self._width = max(pikepdf_get_int(smask, Name.Width), self._width)
|
||||||
self._height = max(smask.get(Name.Height, 0), self._height)
|
self._height = max(pikepdf_get_int(smask, Name.Height), self._height)
|
||||||
if (mask := pim.obj.get(Name.Mask, None)) is not None and isinstance(
|
if (mask := pim.obj.get(Name.Mask, None)) is not None and isinstance(
|
||||||
mask, Stream | Dictionary
|
mask, Stream | Dictionary
|
||||||
):
|
):
|
||||||
@@ -99,8 +101,8 @@ class ImageInfo:
|
|||||||
# /Mask can be a Stream or an Array. If it's a Stream,
|
# /Mask can be a Stream or an Array. If it's a Stream,
|
||||||
# use its /Width and /Height if they are larger than the main
|
# use its /Width and /Height if they are larger than the main
|
||||||
# image's.
|
# image's.
|
||||||
self._width = max(mask.get(Name.Width, 0), self._width)
|
self._width = max(pikepdf_get_int(mask, Name.Width), self._width)
|
||||||
self._height = max(mask.get(Name.Height, 0), self._height)
|
self._height = max(pikepdf_get_int(mask, Name.Height), self._height)
|
||||||
|
|
||||||
# If /ImageMask is true, then this image is a stencil mask
|
# If /ImageMask is true, then this image is a stencil mask
|
||||||
# (Images that draw with this stencil mask will have a reference to
|
# (Images that draw with this stencil mask will have a reference to
|
||||||
@@ -285,9 +287,15 @@ def _image_xobjects(container) -> Iterator[tuple[Object, str]]:
|
|||||||
if Name.Resources not in container:
|
if Name.Resources not in container:
|
||||||
return
|
return
|
||||||
resources = container[Name.Resources]
|
resources = container[Name.Resources]
|
||||||
if Name.XObject not in resources:
|
# A malformed PDF may store a non-dictionary at /Resources or
|
||||||
|
# /Resources /XObject; treat that as "no image XObjects" instead of
|
||||||
|
# crashing when we try to iterate it.
|
||||||
|
if not isinstance(resources, Dictionary):
|
||||||
return
|
return
|
||||||
for key, candidate in resources[Name.XObject].items():
|
xobjects = resources.get(Name.XObject)
|
||||||
|
if not isinstance(xobjects, Dictionary):
|
||||||
|
return
|
||||||
|
for key, candidate in xobjects.items():
|
||||||
if candidate is None or Name.Subtype not in candidate:
|
if candidate is None or Name.Subtype not in candidate:
|
||||||
continue
|
continue
|
||||||
if candidate[Name.Subtype] == Name.Image:
|
if candidate[Name.Subtype] == Name.Image:
|
||||||
@@ -334,9 +342,14 @@ def _find_form_xobject_images(pdf: Pdf, container: Object, contentsinfo: Content
|
|||||||
if Name.Resources not in container:
|
if Name.Resources not in container:
|
||||||
return
|
return
|
||||||
resources = container[Name.Resources]
|
resources = container[Name.Resources]
|
||||||
if Name.XObject not in resources:
|
# As in _image_xobjects, tolerate a non-dictionary /Resources or
|
||||||
|
# /Resources /XObject in a malformed PDF rather than crashing.
|
||||||
|
if not isinstance(resources, Dictionary):
|
||||||
return
|
return
|
||||||
xobjs = resources[Name.XObject].as_dict()
|
xobject = resources.get(Name.XObject)
|
||||||
|
if not isinstance(xobject, Dictionary):
|
||||||
|
return
|
||||||
|
xobjs = xobject.as_dict()
|
||||||
for xobj in xobjs:
|
for xobj in xobjs:
|
||||||
candidate = xobjs[xobj]
|
candidate = xobjs[xobj]
|
||||||
if candidate is None or candidate.get(Name.Subtype) != Name.Form:
|
if candidate is None or candidate.get(Name.Subtype) != Name.Form:
|
||||||
@@ -396,7 +409,9 @@ def _process_content_streams(
|
|||||||
# A Form XObject may provide its own matrix to map form space into
|
# A Form XObject may provide its own matrix to map form space into
|
||||||
# user space. Get this if one exists
|
# user space. Get this if one exists
|
||||||
form_shorthand = container.get(Name.Matrix, Matrix())
|
form_shorthand = container.get(Name.Matrix, Matrix())
|
||||||
form_matrix = Matrix(form_shorthand)
|
# pikepdf's Matrix() stub omits the Object/Array overload, but the
|
||||||
|
# underlying C++ implementation accepts any 6-element numeric array.
|
||||||
|
form_matrix = Matrix(cast(Matrix, form_shorthand))
|
||||||
|
|
||||||
# Concatenate form matrix with CTM to ensure CTM is correct for
|
# Concatenate form matrix with CTM to ensure CTM is correct for
|
||||||
# drawing this instance of the XObject
|
# drawing this instance of the XObject
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import atexit
|
import atexit
|
||||||
import logging
|
import logging
|
||||||
from collections.abc import Container, Sequence
|
from collections.abc import Container
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
from functools import partial
|
from functools import partial
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -28,7 +28,7 @@ logger = logging.getLogger()
|
|||||||
worker_pdf = None # pylint: disable=invalid-name
|
worker_pdf = None # pylint: disable=invalid-name
|
||||||
|
|
||||||
|
|
||||||
def _pdf_pageinfo_sync_init(pdf: Pdf, infile: Path, pdfminer_loglevel):
|
def _pdf_pageinfo_sync_init(pdf: Pdf | None, infile: Path, pdfminer_loglevel):
|
||||||
global worker_pdf # pylint: disable=global-statement,invalid-name
|
global worker_pdf # pylint: disable=global-statement,invalid-name
|
||||||
pikepdf_enable_mmap()
|
pikepdf_enable_mmap()
|
||||||
|
|
||||||
@@ -75,16 +75,16 @@ def _pdf_pageinfo_sync(
|
|||||||
|
|
||||||
|
|
||||||
def _pdf_pageinfo_concurrent(
|
def _pdf_pageinfo_concurrent(
|
||||||
pdf,
|
pdf: Pdf,
|
||||||
executor: Executor,
|
executor: Executor,
|
||||||
max_workers: int,
|
max_workers: int | None,
|
||||||
use_threads: bool,
|
use_threads: bool,
|
||||||
infile,
|
infile: Path,
|
||||||
progbar,
|
progbar: bool,
|
||||||
check_pages,
|
check_pages: Container[int],
|
||||||
detailed_analysis: bool = False,
|
detailed_analysis: bool = False,
|
||||||
miner_state: PdfMinerState | None = None,
|
miner_state: PdfMinerState | None = None,
|
||||||
) -> Sequence[PageInfo | None]:
|
) -> list[PageInfo | None]:
|
||||||
pages: list[PageInfo | None] = [None] * len(pdf.pages)
|
pages: list[PageInfo | None] = [None] * len(pdf.pages)
|
||||||
|
|
||||||
def update_pageinfo(page: PageInfo, pbar: ProgressBar):
|
def update_pageinfo(page: PageInfo, pbar: ProgressBar):
|
||||||
|
|||||||
@@ -16,12 +16,12 @@ from pathlib import Path
|
|||||||
from typing import NamedTuple
|
from typing import NamedTuple
|
||||||
|
|
||||||
from pdfminer.layout import LTPage, LTTextBox
|
from pdfminer.layout import LTPage, LTTextBox
|
||||||
from pikepdf import Name, Page, Pdf
|
from pikepdf import Name, Object, Page, Pdf
|
||||||
|
|
||||||
from ocrmypdf._concurrent import Executor, SerialExecutor
|
from ocrmypdf._concurrent import Executor, SerialExecutor
|
||||||
from ocrmypdf._pageboxes import coerce_box
|
from ocrmypdf._pageboxes import coerce_box
|
||||||
from ocrmypdf.exceptions import EncryptedPdfError
|
from ocrmypdf.exceptions import EncryptedPdfError
|
||||||
from ocrmypdf.helpers import Resolution
|
from ocrmypdf.helpers import Resolution, pikepdf_get_bool, pikepdf_get_int
|
||||||
from ocrmypdf.pdfinfo._contentstream import TextboxInfo, TextMarker, VectorMarker
|
from ocrmypdf.pdfinfo._contentstream import TextboxInfo, TextMarker, VectorMarker
|
||||||
from ocrmypdf.pdfinfo._image import ImageInfo, _process_content_streams
|
from ocrmypdf.pdfinfo._image import ImageInfo, _process_content_streams
|
||||||
from ocrmypdf.pdfinfo._types import FloatRect
|
from ocrmypdf.pdfinfo._types import FloatRect
|
||||||
@@ -160,6 +160,10 @@ class PageInfo:
|
|||||||
check_this_page = pageno in check_pages
|
check_this_page = pageno in check_pages
|
||||||
|
|
||||||
if check_this_page and detailed_analysis:
|
if check_this_page and detailed_analysis:
|
||||||
|
# miner_state is only None when detailed_analysis is False (see
|
||||||
|
# PdfInfo.__init__, which ties the two together), so it must be
|
||||||
|
# set here.
|
||||||
|
assert miner_state is not None
|
||||||
page_analysis = miner_state.get_page_analysis(pageno)
|
page_analysis = miner_state.get_page_analysis(pageno)
|
||||||
if page_analysis is not None:
|
if page_analysis is not None:
|
||||||
self._textboxes = list(
|
self._textboxes = list(
|
||||||
@@ -175,7 +179,11 @@ class PageInfo:
|
|||||||
self._has_text = None # i.e. "no information"
|
self._has_text = None # i.e. "no information"
|
||||||
|
|
||||||
userunit = page.get(Name.UserUnit, Decimal(1.0))
|
userunit = page.get(Name.UserUnit, Decimal(1.0))
|
||||||
if not isinstance(userunit, Decimal):
|
if isinstance(userunit, Object):
|
||||||
|
# Only reachable under pikepdf's explicit conversion mode; the
|
||||||
|
# default (implicit) mode already unboxes to int/float/Decimal.
|
||||||
|
userunit = Decimal(userunit.as_float())
|
||||||
|
elif not isinstance(userunit, Decimal):
|
||||||
userunit = Decimal(userunit)
|
userunit = Decimal(userunit)
|
||||||
self._userunit = userunit
|
self._userunit = userunit
|
||||||
self._width_inches = width_pt * userunit / Decimal(72.0)
|
self._width_inches = width_pt * userunit / Decimal(72.0)
|
||||||
@@ -189,7 +197,7 @@ class PageInfo:
|
|||||||
self._has_text = False
|
self._has_text = False
|
||||||
self._images = []
|
self._images = []
|
||||||
for info in _process_content_streams(
|
for info in _process_content_streams(
|
||||||
pdf=pdf, container=page, shorthand=userunit_shorthand
|
pdf=pdf, container=page.obj, shorthand=userunit_shorthand
|
||||||
):
|
):
|
||||||
if isinstance(info, VectorMarker):
|
if isinstance(info, VectorMarker):
|
||||||
self._has_vector = True
|
self._has_vector = True
|
||||||
@@ -446,16 +454,21 @@ class PdfInfo:
|
|||||||
detailed_analysis=detailed_analysis,
|
detailed_analysis=detailed_analysis,
|
||||||
miner_state=miner_state,
|
miner_state=miner_state,
|
||||||
)
|
)
|
||||||
self._needs_rendering = pdf.Root.get(Name.NeedsRendering, False)
|
self._needs_rendering = pikepdf_get_bool(pdf.Root, Name.NeedsRendering)
|
||||||
if Name.AcroForm in pdf.Root:
|
if Name.AcroForm in pdf.Root:
|
||||||
if (
|
if (
|
||||||
len(pdf.Root.AcroForm.get(Name.Fields, [])) > 0
|
len(pdf.Root.AcroForm.get(Name.Fields, [])) > 0
|
||||||
or Name.XFA in pdf.Root.AcroForm
|
or Name.XFA in pdf.Root.AcroForm
|
||||||
):
|
):
|
||||||
self._has_acroform = True
|
self._has_acroform = True
|
||||||
self._has_signature = bool(pdf.Root.AcroForm.get(Name.SigFlags, 0) & 1)
|
self._has_signature = bool(
|
||||||
self._is_tagged = bool(
|
pikepdf_get_int(pdf.Root.AcroForm, Name.SigFlags) & 1
|
||||||
pdf.Root.get(Name.MarkInfo, {}).get(Name.Marked, False)
|
)
|
||||||
|
mark_info = pdf.Root.get(Name.MarkInfo)
|
||||||
|
self._is_tagged = (
|
||||||
|
pikepdf_get_bool(mark_info, Name.Marked)
|
||||||
|
if mark_info is not None
|
||||||
|
else False
|
||||||
)
|
)
|
||||||
self._has_structure_tree = Name.StructTreeRoot in pdf.Root
|
self._has_structure_tree = Name.StructTreeRoot in pdf.Root
|
||||||
|
|
||||||
@@ -537,6 +550,8 @@ def main(): # pragma: no cover
|
|||||||
pprint(pdfinfo)
|
pprint(pdfinfo)
|
||||||
for page in pdfinfo.pages:
|
for page in pdfinfo.pages:
|
||||||
pprint(page)
|
pprint(page)
|
||||||
|
if page is None:
|
||||||
|
continue
|
||||||
for im in page.images:
|
for im in page.images:
|
||||||
pprint(im)
|
pprint(im)
|
||||||
|
|
||||||
|
|||||||
@@ -5,19 +5,19 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import re
|
import re
|
||||||
|
import sys
|
||||||
from collections.abc import Iterator, Mapping
|
from collections.abc import Iterator, Mapping
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
from math import copysign
|
from math import copysign
|
||||||
from os import PathLike
|
from os import PathLike
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any, BinaryIO
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
import pdfminer
|
import pdfminer
|
||||||
import pdfminer.encodingdb
|
import pdfminer.encodingdb
|
||||||
import pdfminer.pdfdevice
|
import pdfminer.pdfdevice
|
||||||
import pdfminer.pdfinterp
|
import pdfminer.pdfinterp
|
||||||
from deprecation import deprecated
|
|
||||||
from pdfminer.converter import PDFLayoutAnalyzer
|
from pdfminer.converter import PDFLayoutAnalyzer
|
||||||
from pdfminer.layout import LAParams, LTChar, LTPage, LTTextBox
|
from pdfminer.layout import LAParams, LTChar, LTPage, LTTextBox
|
||||||
from pdfminer.pdfcolor import PDFColorSpace
|
from pdfminer.pdfcolor import PDFColorSpace
|
||||||
@@ -30,6 +30,11 @@ from pdfminer.utils import Matrix, bbox2str, matrix2str
|
|||||||
|
|
||||||
from ocrmypdf.exceptions import EncryptedPdfError, InputFileError
|
from ocrmypdf.exceptions import EncryptedPdfError, InputFileError
|
||||||
|
|
||||||
|
if sys.version_info >= (3, 13):
|
||||||
|
from warnings import deprecated
|
||||||
|
else:
|
||||||
|
from typing_extensions import deprecated
|
||||||
|
|
||||||
STRIP_NAME = re.compile(r'[0-9]+')
|
STRIP_NAME = re.compile(r'[0-9]+')
|
||||||
|
|
||||||
|
|
||||||
@@ -57,7 +62,8 @@ def pdfsimplefont__init__(
|
|||||||
return
|
return
|
||||||
|
|
||||||
|
|
||||||
PDFSimpleFont.__init__ = pdfsimplefont__init__
|
PDFSimpleFont.__init__ = pdfsimplefont__init__ # type: ignore[method-assign]
|
||||||
|
|
||||||
|
|
||||||
def pdftype3font__pscript5_get_height(self):
|
def pdftype3font__pscript5_get_height(self):
|
||||||
"""Monkeypatch for PScript5.dll PDFs.
|
"""Monkeypatch for PScript5.dll PDFs.
|
||||||
@@ -283,7 +289,7 @@ def patch_pdfminer(pscript5_mode: bool):
|
|||||||
yield
|
yield
|
||||||
|
|
||||||
|
|
||||||
@deprecated(deprecated_in='16.6.0', details='Use PdfMinerState instead.')
|
@deprecated('Deprecated since 16.6.0; use PdfMinerState instead.')
|
||||||
def get_page_analysis(
|
def get_page_analysis(
|
||||||
infile: PathLike, pageno: int, pscript5_mode: bool
|
infile: PathLike, pageno: int, pscript5_mode: bool
|
||||||
) -> LTPage | None:
|
) -> LTPage | None:
|
||||||
@@ -331,10 +337,10 @@ class PdfMinerState:
|
|||||||
self.infile = infile
|
self.infile = infile
|
||||||
self.rman = pdfminer.pdfinterp.PDFResourceManager(caching=True)
|
self.rman = pdfminer.pdfinterp.PDFResourceManager(caching=True)
|
||||||
self.disable_boxes_flow = None
|
self.disable_boxes_flow = None
|
||||||
self.page_iter = None
|
self.page_iter: Iterator[PDFPage] | None = None
|
||||||
self.page_cache: list[PDFPage] = []
|
self.page_cache: list[PDFPage] = []
|
||||||
self.pscript5_mode = pscript5_mode
|
self.pscript5_mode = pscript5_mode
|
||||||
self.file = None
|
self.file: BinaryIO | None = None
|
||||||
|
|
||||||
def __enter__(self):
|
def __enter__(self):
|
||||||
"""Enter the context manager."""
|
"""Enter the context manager."""
|
||||||
@@ -350,6 +356,7 @@ class PdfMinerState:
|
|||||||
|
|
||||||
def get_page_analysis(self, pageno: int):
|
def get_page_analysis(self, pageno: int):
|
||||||
"""Get the page analysis for a given page."""
|
"""Get the page analysis for a given page."""
|
||||||
|
assert self.page_iter is not None, "must be used as a context manager"
|
||||||
while len(self.page_cache) <= pageno:
|
while len(self.page_cache) <= pageno:
|
||||||
try:
|
try:
|
||||||
self.page_cache.append(next(self.page_iter))
|
self.page_cache.append(next(self.page_iter))
|
||||||
|
|||||||
@@ -94,7 +94,10 @@ def _error_trailer(program: str, package: str | Mapping[str, str], **kwargs) ->
|
|||||||
|
|
||||||
|
|
||||||
def _error_missing_program(
|
def _error_missing_program(
|
||||||
program: str, package: str, required_for: str | None, recommended: bool
|
program: str,
|
||||||
|
package: str | Mapping[str, str],
|
||||||
|
required_for: str | None,
|
||||||
|
recommended: bool,
|
||||||
) -> None:
|
) -> None:
|
||||||
# pylint: disable=unused-argument
|
# pylint: disable=unused-argument
|
||||||
if recommended:
|
if recommended:
|
||||||
@@ -108,7 +111,7 @@ def _error_missing_program(
|
|||||||
|
|
||||||
def _error_old_version(
|
def _error_old_version(
|
||||||
program: str,
|
program: str,
|
||||||
package: str,
|
package: str | Mapping[str, str],
|
||||||
need_version: str,
|
need_version: str,
|
||||||
found_version: str,
|
found_version: str,
|
||||||
required_for: str | None,
|
required_for: str | None,
|
||||||
@@ -124,7 +127,7 @@ def _error_old_version(
|
|||||||
def check_external_program(
|
def check_external_program(
|
||||||
*,
|
*,
|
||||||
program: str,
|
program: str,
|
||||||
package: str,
|
package: str | Mapping[str, str],
|
||||||
version_checker: Callable[[], Version],
|
version_checker: Callable[[], Version],
|
||||||
need_version: str | Version,
|
need_version: str | Version,
|
||||||
required_for: str | None = None,
|
required_for: str | None = None,
|
||||||
|
|||||||
@@ -131,7 +131,7 @@ def _fix_process_args(
|
|||||||
args = fix_windows_args(program, args, env)
|
args = fix_windows_args(program, args, env)
|
||||||
|
|
||||||
log.debug("Running: %s", args)
|
log.debug("Running: %s", args)
|
||||||
process_log = log.getChild(os.path.basename(program))
|
process_log = log.getChild(Path(program).name)
|
||||||
text = bool(kwargs.get('text', False))
|
text = bool(kwargs.get('text', False))
|
||||||
|
|
||||||
return args, env, process_log, text
|
return args, env, process_log, text
|
||||||
|
|||||||
@@ -74,11 +74,11 @@ class FixedRotateNoopOcrEngine(OcrEngine):
|
|||||||
def generate_hocr(input_file, output_hocr, output_text, options):
|
def generate_hocr(input_file, output_hocr, output_text, options):
|
||||||
with (
|
with (
|
||||||
Image.open(input_file) as im,
|
Image.open(input_file) as im,
|
||||||
open(output_hocr, 'w', encoding='utf-8') as f,
|
output_hocr.open('w', encoding='utf-8') as f,
|
||||||
):
|
):
|
||||||
w, h = im.size
|
w, h = im.size
|
||||||
f.write(HOCR_TEMPLATE.format(str(w), str(h)))
|
f.write(HOCR_TEMPLATE.format(str(w), str(h)))
|
||||||
with open(output_text, 'w') as f:
|
with output_text.open('w') as f:
|
||||||
f.write('')
|
f.write('')
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|||||||
@@ -72,11 +72,11 @@ class NoopOcrEngine(OcrEngine):
|
|||||||
def generate_hocr(input_file, output_hocr, output_text, options):
|
def generate_hocr(input_file, output_hocr, output_text, options):
|
||||||
with (
|
with (
|
||||||
Image.open(input_file) as im,
|
Image.open(input_file) as im,
|
||||||
open(output_hocr, 'w', encoding='utf-8') as f,
|
output_hocr.open('w', encoding='utf-8') as f,
|
||||||
):
|
):
|
||||||
w, h = im.size
|
w, h = im.size
|
||||||
f.write(HOCR_TEMPLATE.format(str(w), str(h)))
|
f.write(HOCR_TEMPLATE.format(str(w), str(h)))
|
||||||
with open(output_text, 'w') as f:
|
with output_text.open('w') as f:
|
||||||
f.write('')
|
f.write('')
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|||||||
+39
-1
@@ -28,7 +28,6 @@ def test_language_parameter_mapped_to_languages():
|
|||||||
Regression test for GitHub issue #1640: the Python API ignored the language
|
Regression test for GitHub issue #1640: the Python API ignored the language
|
||||||
parameter, always defaulting to 'eng'.
|
parameter, always defaulting to 'eng'.
|
||||||
"""
|
"""
|
||||||
from ocrmypdf._options import OcrOptions
|
|
||||||
from ocrmypdf.api import create_options, setup_plugin_infrastructure
|
from ocrmypdf.api import create_options, setup_plugin_infrastructure
|
||||||
from ocrmypdf.cli import get_parser
|
from ocrmypdf.cli import get_parser
|
||||||
|
|
||||||
@@ -80,6 +79,45 @@ def test_language_parameter_mapped_to_languages():
|
|||||||
assert options.languages == ['eng', 'spa']
|
assert options.languages == ['eng', 'spa']
|
||||||
|
|
||||||
|
|
||||||
|
def test_jpeg_quality_parameter_reaches_options():
|
||||||
|
"""The canonical 'jpeg_quality' API parameter must reach OcrOptions.
|
||||||
|
|
||||||
|
Regression test for GitHub issue #1723: --jpeg-quality was silently
|
||||||
|
dropped by the CLI's namespace_to_options() because the OcrOptions field
|
||||||
|
was named jpg_quality. create_options(), used by the Python API, has the
|
||||||
|
same field-name matching logic and is affected the same way when passed
|
||||||
|
the alias name.
|
||||||
|
"""
|
||||||
|
from ocrmypdf.api import create_options, setup_plugin_infrastructure
|
||||||
|
from ocrmypdf.cli import get_parser
|
||||||
|
|
||||||
|
setup_plugin_infrastructure()
|
||||||
|
parser = get_parser()
|
||||||
|
|
||||||
|
options = create_options(
|
||||||
|
input_file='test.pdf', output_file='output.pdf', parser=parser, jpeg_quality=10
|
||||||
|
)
|
||||||
|
assert options.jpeg_quality == 10
|
||||||
|
|
||||||
|
|
||||||
|
def test_jpg_quality_parameter_deprecated_alias():
|
||||||
|
"""The old 'jpg_quality' API parameter still works but warns."""
|
||||||
|
from ocrmypdf.api import create_options, setup_plugin_infrastructure
|
||||||
|
from ocrmypdf.cli import get_parser
|
||||||
|
|
||||||
|
setup_plugin_infrastructure()
|
||||||
|
parser = get_parser()
|
||||||
|
|
||||||
|
with pytest.warns(UserWarning, match='jpg_quality'):
|
||||||
|
options = create_options(
|
||||||
|
input_file='test.pdf',
|
||||||
|
output_file='output.pdf',
|
||||||
|
parser=parser,
|
||||||
|
jpg_quality=42,
|
||||||
|
)
|
||||||
|
assert options.jpeg_quality == 42
|
||||||
|
|
||||||
|
|
||||||
def test_stream_api(resources: Path):
|
def test_stream_api(resources: Path):
|
||||||
in_ = (resources / 'graph.pdf').open('rb')
|
in_ = (resources / 'graph.pdf').open('rb')
|
||||||
out = BytesIO()
|
out = BytesIO()
|
||||||
|
|||||||
@@ -219,7 +219,7 @@ class TestFpdf2MultiPageRenderer:
|
|||||||
for i in range(3):
|
for i in range(3):
|
||||||
word = OcrElement(
|
word = OcrElement(
|
||||||
ocr_class=OcrClass.WORD,
|
ocr_class=OcrClass.WORD,
|
||||||
text=f"Page{i+1}",
|
text=f"Page{i + 1}",
|
||||||
bbox=BoundingBox(left=100, top=100, right=200, bottom=130),
|
bbox=BoundingBox(left=100, top=100, right=200, bottom=130),
|
||||||
)
|
)
|
||||||
line = OcrElement(
|
line = OcrElement(
|
||||||
|
|||||||
+6
-6
@@ -78,9 +78,9 @@ def test_redo_ocr_with_offset_mediabox(resources, outdir):
|
|||||||
mediabox = list(page.MediaBox)
|
mediabox = list(page.MediaBox)
|
||||||
|
|
||||||
# MediaBox origin should be preserved
|
# MediaBox origin should be preserved
|
||||||
assert (
|
assert float(mediabox[1]) == y_offset, (
|
||||||
float(mediabox[1]) == y_offset
|
f"MediaBox Y origin should be preserved at {y_offset}, got {mediabox[1]}"
|
||||||
), f"MediaBox Y origin should be preserved at {y_offset}, got {mediabox[1]}"
|
)
|
||||||
|
|
||||||
# The content stream should include a CTM with the Y origin translation.
|
# The content stream should include a CTM with the Y origin translation.
|
||||||
# Without the fix, the CTM was omitted for rotation==0, causing a shift.
|
# Without the fix, the CTM was omitted for rotation==0, causing a shift.
|
||||||
@@ -153,7 +153,7 @@ def test_strip_invisble_text():
|
|||||||
nr_visible_pre = count('visible', page)
|
nr_visible_pre = count('visible', page)
|
||||||
ocrmypdf._graft.strip_invisible_text(pdf, page)
|
ocrmypdf._graft.strip_invisible_text(pdf, page)
|
||||||
nr_visible_post = count('visible', page)
|
nr_visible_post = count('visible', page)
|
||||||
assert (
|
assert nr_visible_pre == nr_visible_post, (
|
||||||
nr_visible_pre == nr_visible_post
|
'Number of visible text elements did not change'
|
||||||
), 'Number of visible text elements did not change'
|
)
|
||||||
assert count('invisible', page) == 0, 'No invisible elems left'
|
assert count('invisible', page) == 0, 'No invisible elems left'
|
||||||
|
|||||||
@@ -121,8 +121,8 @@ def test_shim_paths(tmp_path):
|
|||||||
results = result_str.split(os.pathsep)
|
results = result_str.split(os.pathsep)
|
||||||
assert results[0] == str(syspath), results
|
assert results[0] == str(syspath), results
|
||||||
assert results[-3].endswith('tesseract-ocr'), results
|
assert results[-3].endswith('tesseract-ocr'), results
|
||||||
assert results[-2].endswith(os.path.join('gs9.52.3', 'bin')), results
|
assert results[-2].endswith(str(Path('gs9.52.3', 'bin'))), results
|
||||||
assert results[-1].endswith(os.path.join('gs', '9.51', 'bin')), results
|
assert results[-1].endswith(str(Path('gs', '9.51', 'bin'))), results
|
||||||
|
|
||||||
|
|
||||||
def test_resolution():
|
def test_resolution():
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ from .conftest import check_ocrmypdf
|
|||||||
|
|
||||||
def text_from_pdf(filename):
|
def text_from_pdf(filename):
|
||||||
output_string = StringIO()
|
output_string = StringIO()
|
||||||
with open(filename, 'rb') as in_file:
|
with filename.open('rb') as in_file:
|
||||||
parser = PDFParser(in_file)
|
parser = PDFParser(in_file)
|
||||||
doc = PDFDocument(parser)
|
doc = PDFDocument(parser)
|
||||||
rsrcmgr = PDFResourceManager()
|
rsrcmgr = PDFResourceManager()
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
"""Test JSON serialization of OcrOptions for multiprocessing compatibility."""
|
"""Test JSON serialization of OcrOptions for multiprocessing compatibility."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import multiprocessing
|
import multiprocessing
|
||||||
|
|||||||
+5
-5
@@ -107,7 +107,7 @@ def test_redo_ocr(resources, outpdf):
|
|||||||
|
|
||||||
def test_argsfile(resources, outdir):
|
def test_argsfile(resources, outdir):
|
||||||
path_argsfile = outdir / 'test_argsfile.txt'
|
path_argsfile = outdir / 'test_argsfile.txt'
|
||||||
with open(str(path_argsfile), 'w') as argsfile:
|
with path_argsfile.open('w') as argsfile:
|
||||||
print(
|
print(
|
||||||
'--title',
|
'--title',
|
||||||
'ArgsFile Test',
|
'ArgsFile Test',
|
||||||
@@ -646,7 +646,7 @@ def test_compression_preserved(ocrmypdf_exec, resources, image, outpdf):
|
|||||||
|
|
||||||
im = Image.open(input_file)
|
im = Image.open(input_file)
|
||||||
# Runs: ocrmypdf - output.pdf < testfile
|
# Runs: ocrmypdf - output.pdf < testfile
|
||||||
with open(input_file, 'rb') as input_stream:
|
with Path(input_file).open('rb') as input_stream:
|
||||||
p_args = ocrmypdf_exec + [
|
p_args = ocrmypdf_exec + [
|
||||||
'--optimize',
|
'--optimize',
|
||||||
'0',
|
'0',
|
||||||
@@ -704,7 +704,7 @@ def test_compression_changed(ocrmypdf_exec, resources, image, compression, outpd
|
|||||||
im = Image.open(input_file)
|
im = Image.open(input_file)
|
||||||
|
|
||||||
# Runs: ocrmypdf - output.pdf < testfile
|
# Runs: ocrmypdf - output.pdf < testfile
|
||||||
with open(input_file, 'rb') as input_stream:
|
with Path(input_file).open('rb') as input_stream:
|
||||||
p_args = ocrmypdf_exec + [
|
p_args = ocrmypdf_exec + [
|
||||||
'--image-dpi',
|
'--image-dpi',
|
||||||
'150',
|
'150',
|
||||||
@@ -763,7 +763,7 @@ def test_sidecar_pagecount(resources, outpdf):
|
|||||||
pdfinfo = PdfInfo(resources / '3small.pdf')
|
pdfinfo = PdfInfo(resources / '3small.pdf')
|
||||||
num_pages = len(pdfinfo)
|
num_pages = len(pdfinfo)
|
||||||
|
|
||||||
with open(sidecar, encoding='utf-8') as f:
|
with sidecar.open(encoding='utf-8') as f:
|
||||||
ocr_text = f.read()
|
ocr_text = f.read()
|
||||||
|
|
||||||
# There should a formfeed between each pair of pages, so the count of
|
# There should a formfeed between each pair of pages, so the count of
|
||||||
@@ -784,7 +784,7 @@ def test_sidecar_nonempty(resources, outpdf):
|
|||||||
'tests/plugins/tesseract_cache.py',
|
'tests/plugins/tesseract_cache.py',
|
||||||
)
|
)
|
||||||
|
|
||||||
with open(sidecar, encoding='utf-8') as f:
|
with sidecar.open(encoding='utf-8') as f:
|
||||||
ocr_text = f.read()
|
ocr_text = f.read()
|
||||||
assert 'the' in ocr_text
|
assert 'the' in ocr_text
|
||||||
|
|
||||||
|
|||||||
@@ -137,7 +137,7 @@ def test_unset_metadata(output_type, field, resources, outpdf, caplog):
|
|||||||
# isn't contained anywhere in the output pdf. We'll also check to ensure
|
# isn't contained anywhere in the output pdf. We'll also check to ensure
|
||||||
# it's in the input pdf and that any values not unset are still in the
|
# it's in the input pdf and that any values not unset are still in the
|
||||||
# output pdf.
|
# output pdf.
|
||||||
with open(input_file, 'rb') as before, open(outpdf, 'rb') as after:
|
with input_file.open('rb') as before, outpdf.open('rb') as after:
|
||||||
before_data = before.read()
|
before_data = before.read()
|
||||||
after_data = after.read()
|
after_data = after.read()
|
||||||
|
|
||||||
|
|||||||
@@ -43,7 +43,7 @@ def has_devanagari_font(manager: MultiFontManager) -> bool:
|
|||||||
# Marker for tests that require CJK fonts
|
# Marker for tests that require CJK fonts
|
||||||
requires_cjk = pytest.mark.skipif(
|
requires_cjk = pytest.mark.skipif(
|
||||||
"not has_cjk_font(MultiFontManager())",
|
"not has_cjk_font(MultiFontManager())",
|
||||||
reason="CJK font not available (not installed on system)"
|
reason="CJK font not available (not installed on system)",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -356,9 +356,7 @@ def test_get_all_fonts(multi_font_manager):
|
|||||||
class MockFontProvider:
|
class MockFontProvider:
|
||||||
"""Mock FontProvider for testing missing fonts."""
|
"""Mock FontProvider for testing missing fonts."""
|
||||||
|
|
||||||
def __init__(
|
def __init__(self, available_fonts: dict[str, FontManager], fallback: FontManager):
|
||||||
self, available_fonts: dict[str, FontManager], fallback: FontManager
|
|
||||||
):
|
|
||||||
"""Initialize mock font provider with given fonts."""
|
"""Initialize mock font provider with given fonts."""
|
||||||
self._fonts = available_fonts
|
self._fonts = available_fonts
|
||||||
self._fallback = fallback
|
self._fallback = fallback
|
||||||
@@ -571,3 +569,117 @@ def test_missing_font_warning_explains_consequences(font_dir, caplog):
|
|||||||
# the text stays searchable but renders blank when highlighted.
|
# the text stays searchable but renders blank when highlighted.
|
||||||
assert 'searchable' in msg.lower()
|
assert 'searchable' in msg.lower()
|
||||||
assert 'highlight' in msg.lower() or 'select' in msg.lower()
|
assert 'highlight' in msg.lower() or 'select' in msg.lower()
|
||||||
|
|
||||||
|
|
||||||
|
# --- Coverage-driven last-resort font search (#1722) ---
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeProviderWithSearch(_FakeFontProvider):
|
||||||
|
"""FontProvider that can also search unlisted fonts by glyph coverage."""
|
||||||
|
|
||||||
|
def __init__(self, fonts, searchable):
|
||||||
|
super().__init__(fonts)
|
||||||
|
self._searchable = searchable
|
||||||
|
self.search_calls: list[str] = []
|
||||||
|
|
||||||
|
def find_font_with_glyphs(self, text):
|
||||||
|
self.search_calls.append(text)
|
||||||
|
for name, font in self._searchable.items():
|
||||||
|
hb = font.get_hb_font()
|
||||||
|
if all(hb.get_nominal_glyph(ord(c)) for c in text):
|
||||||
|
self._fonts[name] = font # discovered fonts become resolvable
|
||||||
|
return name, font
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def test_unlisted_font_found_by_coverage_search():
|
||||||
|
"""A script outside FALLBACK_FONTS is rendered if the font is installed."""
|
||||||
|
searchable = {'NotoSansCherokee-Regular': _FakeFontManager('Cherokee.ttf', 'ᏣᎳᎩ')}
|
||||||
|
provider = _FakeProviderWithSearch({}, searchable)
|
||||||
|
manager = MultiFontManager(font_provider=provider)
|
||||||
|
|
||||||
|
font = manager.select_font_for_word('ᏣᎳᎩ', None)
|
||||||
|
|
||||||
|
assert font.font_path.name == 'Cherokee.ttf'
|
||||||
|
assert provider.search_calls == ['ᏣᎳᎩ']
|
||||||
|
|
||||||
|
|
||||||
|
def test_coverage_search_result_is_cached():
|
||||||
|
"""The expensive coverage search runs once per distinct word."""
|
||||||
|
searchable = {'NotoSansCherokee-Regular': _FakeFontManager('Cherokee.ttf', 'ᏣᎳᎩ')}
|
||||||
|
provider = _FakeProviderWithSearch({}, searchable)
|
||||||
|
manager = MultiFontManager(font_provider=provider)
|
||||||
|
|
||||||
|
manager.select_font_for_word('ᏣᎳᎩ', None)
|
||||||
|
manager.select_font_for_word('ᏣᎳᎩ', None)
|
||||||
|
|
||||||
|
assert len(provider.search_calls) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_named_fonts_take_precedence_over_coverage_search():
|
||||||
|
"""The coverage search is a last resort, not a substitute for named fonts."""
|
||||||
|
fonts = {'NotoSans-Regular': _FakeFontManager('NotoSans.ttf', 'abc')}
|
||||||
|
searchable = {'NotoSansMono-Regular': _FakeFontManager('NotoSansMono.ttf', 'abc')}
|
||||||
|
provider = _FakeProviderWithSearch(fonts, searchable)
|
||||||
|
manager = MultiFontManager(font_provider=provider)
|
||||||
|
|
||||||
|
font = manager.select_font_for_word('abc', None)
|
||||||
|
|
||||||
|
assert font.font_path.name == 'NotoSans.ttf'
|
||||||
|
assert provider.search_calls == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_provider_without_coverage_search_still_falls_back():
|
||||||
|
"""Providers predating find_font_with_glyphs() keep working (duck-typed)."""
|
||||||
|
manager = MultiFontManager(font_provider=_FakeFontProvider({}))
|
||||||
|
font = manager.select_font_for_word('ᏣᎳᎩ', None)
|
||||||
|
assert font.font_path.name == 'Occulta.ttf'
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_font_warning_names_the_missing_characters(font_dir, caplog):
|
||||||
|
"""The warning must identify what could not be rendered (#1722).
|
||||||
|
|
||||||
|
The user's real question is "which font package do I install?" — naming the
|
||||||
|
offending codepoints and their Unicode names answers it even for scripts
|
||||||
|
OCRmyPDF has no language mapping for.
|
||||||
|
"""
|
||||||
|
manager = MultiFontManager(font_provider=BuiltinFontProvider(font_dir))
|
||||||
|
|
||||||
|
with caplog.at_level(logging.WARNING):
|
||||||
|
manager.select_font_for_word('ᏣᎳᎩ', None)
|
||||||
|
|
||||||
|
msg = caplog.text
|
||||||
|
assert 'U+13E3' in msg # CHEROKEE LETTER TSA
|
||||||
|
assert 'CHEROKEE' in msg.upper()
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_character_warning_ignores_covered_characters(font_dir, caplog):
|
||||||
|
"""Only the uncoverable characters are reported, not the whole word."""
|
||||||
|
manager = MultiFontManager(font_provider=BuiltinFontProvider(font_dir))
|
||||||
|
|
||||||
|
with caplog.at_level(logging.WARNING):
|
||||||
|
manager.select_font_for_word('aᏣb', None)
|
||||||
|
|
||||||
|
msg = caplog.text
|
||||||
|
assert 'U+13E3' in msg
|
||||||
|
assert 'U+0061' not in msg # 'a' is covered by the builtin Latin font
|
||||||
|
|
||||||
|
|
||||||
|
def test_mixed_script_word_warning_does_not_advise_installing_fonts(caplog):
|
||||||
|
"""A word no single font covers is reported as such, not as a missing font.
|
||||||
|
|
||||||
|
Every character here has an installed font; telling the user to install
|
||||||
|
more would be wrong advice and is exactly the confusion behind #1722.
|
||||||
|
"""
|
||||||
|
fonts = {'NotoSans-Regular': _FakeFontManager('NotoSans.ttf', 'ab')}
|
||||||
|
searchable = {'NotoSansCherokee-Regular': _FakeFontManager('Cherokee.ttf', 'Ꮳ')}
|
||||||
|
manager = MultiFontManager(font_provider=_FakeProviderWithSearch(fonts, searchable))
|
||||||
|
|
||||||
|
with caplog.at_level(logging.WARNING):
|
||||||
|
font = manager.select_font_for_word('aᏣb', None)
|
||||||
|
|
||||||
|
assert font.font_path.name == 'Occulta.ttf'
|
||||||
|
msg = caplog.text
|
||||||
|
assert 'mixing scripts' in msg
|
||||||
|
assert 'will not help' in msg
|
||||||
|
assert 'U+' not in msg # no codepoints to blame; nothing to install
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ This tests the fpdf2 renderer with various language groups:
|
|||||||
- CJK (Chinese Simplified/Traditional, Japanese, Korean)
|
- CJK (Chinese Simplified/Traditional, Japanese, Korean)
|
||||||
- Devanagari (Hindi, Sanskrit)
|
- Devanagari (Hindi, Sanskrit)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import shutil
|
import shutil
|
||||||
@@ -208,9 +209,9 @@ class TestArabicScript:
|
|||||||
for para in page.paragraphs:
|
for para in page.paragraphs:
|
||||||
if para.language in ('ara', 'per'):
|
if para.language in ('ara', 'per'):
|
||||||
# Arabic paragraphs should have RTL direction
|
# Arabic paragraphs should have RTL direction
|
||||||
assert (
|
assert para.direction == 'rtl', (
|
||||||
para.direction == 'rtl'
|
"Arabic paragraph should have RTL direction"
|
||||||
), "Arabic paragraph should have RTL direction"
|
)
|
||||||
|
|
||||||
|
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
@@ -529,9 +530,9 @@ class TestBaselineHandling:
|
|||||||
for line in page.lines:
|
for line in page.lines:
|
||||||
if line.baseline:
|
if line.baseline:
|
||||||
# Baseline should be reasonable
|
# Baseline should be reasonable
|
||||||
assert (
|
assert -1.0 <= line.baseline.slope <= 1.0, (
|
||||||
-1.0 <= line.baseline.slope <= 1.0
|
"Baseline slope should be reasonable"
|
||||||
), "Baseline slope should be reasonable"
|
)
|
||||||
|
|
||||||
|
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
@@ -556,9 +557,9 @@ class TestFontCoverage:
|
|||||||
]
|
]
|
||||||
|
|
||||||
for sample in latin_samples:
|
for sample in latin_samples:
|
||||||
assert multi_font_manager.has_all_glyphs(
|
assert multi_font_manager.has_all_glyphs('NotoSans-Regular', sample), (
|
||||||
'NotoSans-Regular', sample
|
f"NotoSans should cover: {sample}"
|
||||||
), f"NotoSans should cover: {sample}"
|
)
|
||||||
|
|
||||||
def test_noto_sans_arabic_coverage(self, multi_font_manager_arabic):
|
def test_noto_sans_arabic_coverage(self, multi_font_manager_arabic):
|
||||||
"""Test NotoSansArabic covers Arabic characters."""
|
"""Test NotoSansArabic covers Arabic characters."""
|
||||||
@@ -602,9 +603,9 @@ class TestFontCoverage:
|
|||||||
]
|
]
|
||||||
|
|
||||||
for sample in cjk_samples:
|
for sample in cjk_samples:
|
||||||
assert multi_font_manager.has_all_glyphs(
|
assert multi_font_manager.has_all_glyphs('NotoSansCJK-Regular', sample), (
|
||||||
'NotoSansCJK-Regular', sample
|
f"NotoSansCJK should cover: {sample}"
|
||||||
), f"NotoSansCJK should cover: {sample}"
|
)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ from PIL import Image, ImageDraw
|
|||||||
from ocrmypdf import optimize as opt
|
from ocrmypdf import optimize as opt
|
||||||
from ocrmypdf._exec import jbig2enc, pngquant
|
from ocrmypdf._exec import jbig2enc, pngquant
|
||||||
from ocrmypdf._exec.ghostscript import rasterize_pdf
|
from ocrmypdf._exec.ghostscript import rasterize_pdf
|
||||||
|
from ocrmypdf.cli import get_options_and_plugins
|
||||||
from ocrmypdf.helpers import IMG2PDF_KWARGS, Resolution
|
from ocrmypdf.helpers import IMG2PDF_KWARGS, Resolution
|
||||||
from ocrmypdf.optimize import PdfImage, extract_image_filter
|
from ocrmypdf.optimize import PdfImage, extract_image_filter
|
||||||
from ocrmypdf.pluginspec import GhostscriptRasterDevice
|
from ocrmypdf.pluginspec import GhostscriptRasterDevice
|
||||||
@@ -81,6 +82,28 @@ def test_jpg_png_params(resources, outpdf):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_jpeg_quality_cli_flag_reaches_options(resources, outpdf):
|
||||||
|
# Regression test for #1723: --jpeg-quality was silently dropped by
|
||||||
|
# namespace_to_options() because the argparse dest ('jpeg_quality') did
|
||||||
|
# not match the OcrOptions field it was checked against.
|
||||||
|
input_ = fspath(resources / 'c02-22.pdf')
|
||||||
|
options, _pm = get_options_and_plugins(
|
||||||
|
['--jpeg-quality', '10', input_, fspath(outpdf)]
|
||||||
|
)
|
||||||
|
assert options.jpeg_quality == 10
|
||||||
|
|
||||||
|
|
||||||
|
def test_jpg_quality_cli_alias_reaches_options(resources, outpdf):
|
||||||
|
# --jpg-quality is a hidden alias for --jpeg-quality (same argparse dest).
|
||||||
|
input_ = fspath(resources / 'c02-22.pdf')
|
||||||
|
options, _pm = get_options_and_plugins(
|
||||||
|
['--jpg-quality', '42', input_, fspath(outpdf)]
|
||||||
|
)
|
||||||
|
assert options.jpeg_quality == 42
|
||||||
|
# The old field name is still readable as a deprecated compatibility alias.
|
||||||
|
assert options.jpg_quality == 42
|
||||||
|
|
||||||
|
|
||||||
@needs_jbig2enc
|
@needs_jbig2enc
|
||||||
def test_jbig2_lossless(resources, outpdf):
|
def test_jbig2_lossless(resources, outpdf):
|
||||||
"""Test that JBIG2 lossless encoding works without JBIG2Globals."""
|
"""Test that JBIG2 lossless encoding works without JBIG2Globals."""
|
||||||
|
|||||||
+35
-31
@@ -32,7 +32,7 @@ from ocrmypdf.hocrtransform import (
|
|||||||
def text_from_pdf(filename: Path) -> str:
|
def text_from_pdf(filename: Path) -> str:
|
||||||
"""Extract text from a PDF file using pdfminer."""
|
"""Extract text from a PDF file using pdfminer."""
|
||||||
output_string = StringIO()
|
output_string = StringIO()
|
||||||
with open(filename, 'rb') as in_file:
|
with filename.open('rb') as in_file:
|
||||||
parser = PDFParser(in_file)
|
parser = PDFParser(in_file)
|
||||||
doc = PDFDocument(parser)
|
doc = PDFDocument(parser)
|
||||||
rsrcmgr = PDFResourceManager()
|
rsrcmgr = PDFResourceManager()
|
||||||
@@ -511,7 +511,8 @@ class TestFpdf2PdfRendererErrors:
|
|||||||
def test_invalid_ocr_class(self, multi_font_manager):
|
def test_invalid_ocr_class(self, multi_font_manager):
|
||||||
"""Test that non-page elements are rejected."""
|
"""Test that non-page elements are rejected."""
|
||||||
line = OcrElement(
|
line = OcrElement(
|
||||||
ocr_class=OcrClass.LINE, bbox=BoundingBox(left=0, top=0, right=100, bottom=50)
|
ocr_class=OcrClass.LINE,
|
||||||
|
bbox=BoundingBox(left=0, top=0, right=100, bottom=50),
|
||||||
)
|
)
|
||||||
|
|
||||||
with pytest.raises(ValueError, match="ocr_page"):
|
with pytest.raises(ValueError, match="ocr_page"):
|
||||||
@@ -622,9 +623,7 @@ def create_rtl_page(
|
|||||||
OcrElement(
|
OcrElement(
|
||||||
ocr_class=OcrClass.WORD,
|
ocr_class=OcrClass.WORD,
|
||||||
text=text,
|
text=text,
|
||||||
bbox=BoundingBox(
|
bbox=BoundingBox(left=bbox[0], top=bbox[1], right=bbox[2], bottom=bbox[3]),
|
||||||
left=bbox[0], top=bbox[1], right=bbox[2], bottom=bbox[3]
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
for text, bbox in words
|
for text, bbox in words
|
||||||
]
|
]
|
||||||
@@ -656,16 +655,19 @@ def _tounicode_map(pdf_path: Path) -> dict[int, str]:
|
|||||||
"""
|
"""
|
||||||
pdf = pikepdf.open(pdf_path)
|
pdf = pikepdf.open(pdf_path)
|
||||||
page = pdf.pages[0]
|
page = pdf.pages[0]
|
||||||
resources = page.get('/Resources', {})
|
resources = page.get('/Resources', pikepdf.Dictionary()).as_dict()
|
||||||
|
|
||||||
# Collect fonts from the page and from any Form XObjects (OCR overlay)
|
# Collect fonts from the page and from any Form XObjects (OCR overlay)
|
||||||
fonts: dict[str, pikepdf.Object] = {}
|
fonts: dict[str, pikepdf.Object] = {}
|
||||||
if '/Font' in resources:
|
if '/Font' in resources:
|
||||||
for name, obj in resources['/Font'].items():
|
for name, obj in resources['/Font'].as_dict().items():
|
||||||
fonts[str(name)] = obj
|
fonts[str(name)] = obj
|
||||||
for xobj in resources.get('/XObject', {}).values():
|
for xobj in resources.get('/XObject', pikepdf.Dictionary()).as_dict().values():
|
||||||
if xobj.get('/Subtype') == '/Form':
|
if xobj.get('/Subtype') == '/Form':
|
||||||
for name, obj in xobj.get('/Resources', {}).get('/Font', {}).items():
|
xobj_resources = xobj.get('/Resources', pikepdf.Dictionary()).as_dict()
|
||||||
|
for name, obj in (
|
||||||
|
xobj_resources.get('/Font', pikepdf.Dictionary()).as_dict().items()
|
||||||
|
):
|
||||||
fonts[str(name)] = obj
|
fonts[str(name)] = obj
|
||||||
|
|
||||||
result: dict[int, str] = {}
|
result: dict[int, str] = {}
|
||||||
@@ -702,30 +704,33 @@ def _decode_tounicode_stream(
|
|||||||
"""
|
"""
|
||||||
pdf = pikepdf.open(pdf_path)
|
pdf = pikepdf.open(pdf_path)
|
||||||
page = pdf.pages[0]
|
page = pdf.pages[0]
|
||||||
resources = page.get('/Resources', {})
|
resources = page.get('/Resources', pikepdf.Dictionary()).as_dict()
|
||||||
|
|
||||||
# Collect fonts from page and from Form XObjects
|
# Collect fonts from page and from Form XObjects
|
||||||
cmap: dict[int, str] = {}
|
cmap: dict[int, str] = {}
|
||||||
for font_dict in [resources.get('/Font', {})]:
|
font_dict = resources.get('/Font', pikepdf.Dictionary()).as_dict()
|
||||||
for fobj in font_dict.values():
|
for fobj in font_dict.values():
|
||||||
tounicode = fobj.get('/ToUnicode')
|
tounicode = fobj.get('/ToUnicode')
|
||||||
if tounicode is None:
|
if tounicode is None:
|
||||||
continue
|
continue
|
||||||
raw = bytes(tounicode.read_bytes()).decode('latin-1', errors='replace')
|
raw = bytes(tounicode.read_bytes()).decode('latin-1', errors='replace')
|
||||||
for m in re.finditer(r'<([0-9A-Fa-f]+)>\s*<([0-9A-Fa-f]+)>', raw):
|
for m in re.finditer(r'<([0-9A-Fa-f]+)>\s*<([0-9A-Fa-f]+)>', raw):
|
||||||
src = int(m.group(1), 16)
|
src = int(m.group(1), 16)
|
||||||
dst_hex = m.group(2)
|
dst_hex = m.group(2)
|
||||||
chars = ''.join(
|
chars = ''.join(
|
||||||
chr(int(dst_hex[i : i + 4], 16))
|
chr(int(dst_hex[i : i + 4], 16))
|
||||||
for i in range(0, len(dst_hex), 4)
|
for i in range(0, len(dst_hex), 4)
|
||||||
if int(dst_hex[i : i + 4], 16) > 0
|
if int(dst_hex[i : i + 4], 16) > 0
|
||||||
)
|
)
|
||||||
if src > 0 and chars:
|
if src > 0 and chars:
|
||||||
cmap[src] = chars
|
cmap[src] = chars
|
||||||
for xobj in resources.get('/XObject', {}).values():
|
for xobj in resources.get('/XObject', pikepdf.Dictionary()).as_dict().values():
|
||||||
if xobj.get('/Subtype') != '/Form':
|
if xobj.get('/Subtype') != '/Form':
|
||||||
continue
|
continue
|
||||||
for fobj in xobj.get('/Resources', {}).get('/Font', {}).values():
|
xobj_resources = xobj.get('/Resources', pikepdf.Dictionary()).as_dict()
|
||||||
|
for fobj in (
|
||||||
|
xobj_resources.get('/Font', pikepdf.Dictionary()).as_dict().values()
|
||||||
|
):
|
||||||
tounicode = fobj.get('/ToUnicode')
|
tounicode = fobj.get('/ToUnicode')
|
||||||
if tounicode is None:
|
if tounicode is None:
|
||||||
continue
|
continue
|
||||||
@@ -747,7 +752,7 @@ def _decode_tounicode_stream(
|
|||||||
contents = page.get('/Contents')
|
contents = page.get('/Contents')
|
||||||
if contents:
|
if contents:
|
||||||
streams.append(bytes(contents.read_bytes()))
|
streams.append(bytes(contents.read_bytes()))
|
||||||
for xobj in resources.get('/XObject', {}).values():
|
for xobj in resources.get('/XObject', pikepdf.Dictionary()).as_dict().values():
|
||||||
if xobj.get('/Subtype') == '/Form':
|
if xobj.get('/Subtype') == '/Form':
|
||||||
streams.append(bytes(xobj.read_bytes()))
|
streams.append(bytes(xobj.read_bytes()))
|
||||||
for data in streams:
|
for data in streams:
|
||||||
@@ -850,8 +855,7 @@ class TestRtlTextExtraction:
|
|||||||
decoded = ''.join(cmap.get(g, '') for g in glyph_ids)
|
decoded = ''.join(cmap.get(g, '') for g in glyph_ids)
|
||||||
logical = decoded[::-1]
|
logical = decoded[::-1]
|
||||||
assert logical == 'שלום', (
|
assert logical == 'שלום', (
|
||||||
f"Expected logical text 'שלום', got {logical!r} "
|
f"Expected logical text 'שלום', got {logical!r} (stream: {decoded!r})"
|
||||||
f"(stream: {decoded!r})"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_rtl_tounicode_one_to_one(self, tmp_path, multi_font_manager):
|
def test_rtl_tounicode_one_to_one(self, tmp_path, multi_font_manager):
|
||||||
|
|||||||
@@ -94,6 +94,52 @@ class TestFindNonembeddedCidFonts:
|
|||||||
with pikepdf.open(path) as pdf:
|
with pikepdf.open(path) as pdf:
|
||||||
assert find_nonembedded_cid_fonts(pdf) == {'ZZZ+Hidden'}
|
assert find_nonembedded_cid_fonts(pdf) == {'ZZZ+Hidden'}
|
||||||
|
|
||||||
|
def test_non_dictionary_font_and_xobject_resources_are_ignored(self, tmp_path):
|
||||||
|
# A malformed PDF may carry a /Font or /XObject resource that is not a
|
||||||
|
# dictionary (an array, a name, an empty value). Scanning must skip it
|
||||||
|
# rather than raise when iterating its values (regression test for the
|
||||||
|
# crash reported in issue #1713).
|
||||||
|
path = tmp_path / 'malformed_resources.pdf'
|
||||||
|
with pikepdf.new() as pdf:
|
||||||
|
page = pdf.add_blank_page()
|
||||||
|
page.Resources = pikepdf.Dictionary(
|
||||||
|
Font=pikepdf.Array([]),
|
||||||
|
XObject=pikepdf.Array([]),
|
||||||
|
)
|
||||||
|
pdf.save(path)
|
||||||
|
with pikepdf.open(path) as pdf:
|
||||||
|
assert find_nonembedded_cid_fonts(pdf) == set()
|
||||||
|
|
||||||
|
def test_non_dictionary_font_descriptor_is_reported(self, tmp_path):
|
||||||
|
# A Type0 font whose descendant carries a non-dictionary /FontDescriptor
|
||||||
|
# has no embedded glyph data, so it must be reported -- not crash. This
|
||||||
|
# is the same malformed-resource bug class as #1713, one level deeper:
|
||||||
|
# `key in descriptor` raises ValueError on a non-dictionary.
|
||||||
|
path = tmp_path / 'bad_descriptor.pdf'
|
||||||
|
with pikepdf.new() as pdf:
|
||||||
|
page = pdf.add_blank_page()
|
||||||
|
cidfont = pdf.make_indirect(
|
||||||
|
pikepdf.Dictionary(
|
||||||
|
Type=Name.Font,
|
||||||
|
Subtype=Name.CIDFontType2,
|
||||||
|
BaseFont=Name('/BOGUS+CID'),
|
||||||
|
FontDescriptor=Name.NotADictionary,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
type0 = pdf.make_indirect(
|
||||||
|
pikepdf.Dictionary(
|
||||||
|
Type=Name.Font,
|
||||||
|
Subtype=Name.Type0,
|
||||||
|
BaseFont=Name('/BOGUS+CID'),
|
||||||
|
Encoding=Name.Identity_H,
|
||||||
|
DescendantFonts=pikepdf.Array([cidfont]),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
page.Resources = pikepdf.Dictionary(Font=pikepdf.Dictionary(F0=type0))
|
||||||
|
pdf.save(path)
|
||||||
|
with pikepdf.open(path) as pdf:
|
||||||
|
assert find_nonembedded_cid_fonts(pdf) == {'BOGUS+CID'}
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def nonembedded_cid_pdf(tmp_path):
|
def nonembedded_cid_pdf(tmp_path):
|
||||||
|
|||||||
@@ -448,6 +448,55 @@ def test_fill_ink_cs_resets_color_to_black():
|
|||||||
assert _ink_of_first_xobject(b"0.8 0.2 0.2 rg /DeviceGray cs /Im0 Do") is Ink.mono
|
assert _ink_of_first_xobject(b"0.8 0.2 0.2 rg /DeviceGray cs /Im0 Do") is Ink.mono
|
||||||
|
|
||||||
|
|
||||||
|
def test_nondict_xobject_tolerated(outdir):
|
||||||
|
# A malformed PDF may store a non-dictionary object (here an Array) at
|
||||||
|
# /Resources /XObject. Scanning for images must tolerate this rather than
|
||||||
|
# crash on .items(); OCRmyPDF's domain is messy machine-generated PDFs.
|
||||||
|
# Same robustness class as the pdfa.py find_nonembedded_cid_fonts fix.
|
||||||
|
pdf = pikepdf.Pdf.new()
|
||||||
|
page = pdf.add_blank_page(page_size=(612, 792))
|
||||||
|
page.Resources = pikepdf.Dictionary(
|
||||||
|
Font=pikepdf.Array([]), XObject=pikepdf.Array([])
|
||||||
|
)
|
||||||
|
out = outdir / 'malformed_xobj.pdf'
|
||||||
|
pdf.save(out)
|
||||||
|
|
||||||
|
info = pdfinfo.PdfInfo(out)
|
||||||
|
assert len(info) == 1
|
||||||
|
assert len(info[0].images) == 0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
'resources',
|
||||||
|
[
|
||||||
|
pikepdf.Array([]), # non-dict /Resources
|
||||||
|
pikepdf.Name.Foo, # non-dict /Resources (name)
|
||||||
|
pikepdf.Dictionary(XObject=pikepdf.Array([])), # non-dict /XObject
|
||||||
|
pikepdf.Dictionary(XObject=pikepdf.Name.Foo), # non-dict /XObject (name)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_image_scanners_tolerate_nondict_resources(resources):
|
||||||
|
# Exercise the image scanners directly on an in-memory container whose
|
||||||
|
# /Resources or /Resources /XObject is not a dictionary. (pikepdf
|
||||||
|
# normalizes a non-dict /Resources assigned to a page on save, so these
|
||||||
|
# cases must be built in memory to reach the scanner unmodified.)
|
||||||
|
from ocrmypdf.pdfinfo._contentstream import ContentsInfo
|
||||||
|
from ocrmypdf.pdfinfo._image import _find_form_xobject_images, _image_xobjects
|
||||||
|
|
||||||
|
container = pikepdf.Dictionary(Type=pikepdf.Name.Page, Resources=resources)
|
||||||
|
empty = ContentsInfo(
|
||||||
|
xobject_settings=[],
|
||||||
|
inline_images=[],
|
||||||
|
found_vector=False,
|
||||||
|
found_text=False,
|
||||||
|
name_index={},
|
||||||
|
)
|
||||||
|
pdf = pikepdf.Pdf.new()
|
||||||
|
|
||||||
|
assert list(_image_xobjects(container)) == []
|
||||||
|
assert list(_find_form_xobject_images(pdf, container, empty)) == []
|
||||||
|
|
||||||
|
|
||||||
def test_imageinfo_ink_inherited_in_form_xobject(outdir):
|
def test_imageinfo_ink_inherited_in_form_xobject(outdir):
|
||||||
# A mask drawn inside a Form XObject inherits the fill color set before the
|
# A mask drawn inside a Form XObject inherits the fill color set before the
|
||||||
# Do that paints the form; the gray classification must reach the mask.
|
# Do that paints the form; the gray classification must reach the mask.
|
||||||
|
|||||||
+18
-18
@@ -160,9 +160,9 @@ def test_rotated_skew_timeout(resources, outpdf, rasterizer):
|
|||||||
input_file = resources / 'rotated_skew.pdf'
|
input_file = resources / 'rotated_skew.pdf'
|
||||||
in_pageinfo = PdfInfo(input_file)[0]
|
in_pageinfo = PdfInfo(input_file)[0]
|
||||||
|
|
||||||
assert (
|
assert in_pageinfo.height_pixels < in_pageinfo.width_pixels, (
|
||||||
in_pageinfo.height_pixels < in_pageinfo.width_pixels
|
"Expected the input page to be landscape"
|
||||||
), "Expected the input page to be landscape"
|
)
|
||||||
assert in_pageinfo.rotation == 90, "Expected a rotated page"
|
assert in_pageinfo.rotation == 90, "Expected a rotated page"
|
||||||
|
|
||||||
out = check_ocrmypdf(
|
out = check_ocrmypdf(
|
||||||
@@ -184,9 +184,9 @@ def test_rotated_skew_timeout(resources, outpdf, rasterizer):
|
|||||||
|
|
||||||
assert out_pageinfo.rotation == 0, "Expected no page rotation for output"
|
assert out_pageinfo.rotation == 0, "Expected no page rotation for output"
|
||||||
|
|
||||||
assert (
|
assert in_pageinfo.width_pixels == h and in_pageinfo.height_pixels == w, (
|
||||||
in_pageinfo.width_pixels == h and in_pageinfo.height_pixels == w
|
"Expected page rotation to be baked in"
|
||||||
), "Expected page rotation to be baked in"
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize('rasterizer', ['pypdfium', 'ghostscript'])
|
@pytest.mark.parametrize('rasterizer', ['pypdfium', 'ghostscript'])
|
||||||
@@ -411,16 +411,16 @@ def test_simulated_scan(outdir):
|
|||||||
)
|
)
|
||||||
|
|
||||||
with pikepdf.open(outdir / 'out.pdf') as pdf:
|
with pikepdf.open(outdir / 'out.pdf') as pdf:
|
||||||
assert (
|
assert pdf.pages[1].mediabox[2] > pdf.pages[1].mediabox[3], (
|
||||||
pdf.pages[1].mediabox[2] > pdf.pages[1].mediabox[3]
|
"Wrong orientation: not landscape"
|
||||||
), "Wrong orientation: not landscape"
|
)
|
||||||
assert (
|
assert pdf.pages[3].mediabox[2] > pdf.pages[3].mediabox[3], (
|
||||||
pdf.pages[3].mediabox[2] > pdf.pages[3].mediabox[3]
|
"Wrong orientation: Not landscape"
|
||||||
), "Wrong orientation: Not landscape"
|
)
|
||||||
|
|
||||||
assert (
|
assert pdf.pages[0].mediabox[2] < pdf.pages[0].mediabox[3], (
|
||||||
pdf.pages[0].mediabox[2] < pdf.pages[0].mediabox[3]
|
"Wrong orientation: Not portrait"
|
||||||
), "Wrong orientation: Not portrait"
|
)
|
||||||
assert (
|
assert pdf.pages[2].mediabox[2] < pdf.pages[2].mediabox[3], (
|
||||||
pdf.pages[2].mediabox[2] < pdf.pages[2].mediabox[3]
|
"Wrong orientation: Not portrait"
|
||||||
), "Wrong orientation: Not portrait"
|
)
|
||||||
|
|||||||
+5
-4
@@ -4,6 +4,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
from pathlib import Path
|
||||||
from subprocess import DEVNULL, PIPE, run
|
from subprocess import DEVNULL, PIPE, run
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
@@ -18,7 +19,7 @@ def test_stdin(ocrmypdf_exec, resources, outpdf):
|
|||||||
output_file = str(outpdf)
|
output_file = str(outpdf)
|
||||||
|
|
||||||
# Runs: ocrmypdf - output.pdf < testfile.pdf
|
# Runs: ocrmypdf - output.pdf < testfile.pdf
|
||||||
with open(input_file, 'rb') as input_stream:
|
with Path(input_file).open('rb') as input_stream:
|
||||||
p_args = ocrmypdf_exec + [
|
p_args = ocrmypdf_exec + [
|
||||||
'-',
|
'-',
|
||||||
output_file,
|
output_file,
|
||||||
@@ -36,7 +37,7 @@ def test_stdout(ocrmypdf_exec, resources, outpdf):
|
|||||||
output_file = str(outpdf)
|
output_file = str(outpdf)
|
||||||
|
|
||||||
# Runs: ocrmypdf francais.pdf - > test_stdout.pdf
|
# Runs: ocrmypdf francais.pdf - > test_stdout.pdf
|
||||||
with open(output_file, 'wb') as output_stream:
|
with Path(output_file).open('wb') as output_stream:
|
||||||
p_args = ocrmypdf_exec + [
|
p_args = ocrmypdf_exec + [
|
||||||
input_file,
|
input_file,
|
||||||
'-',
|
'-',
|
||||||
@@ -58,7 +59,7 @@ def test_stdout_protected_from_pollution(ocrmypdf_exec, resources, outpdf):
|
|||||||
# A plugin deliberately writes garbage to stdout during the run. With stdout
|
# A plugin deliberately writes garbage to stdout during the run. With stdout
|
||||||
# protection active, that garbage must be diverted to stderr and never reach
|
# protection active, that garbage must be diverted to stderr and never reach
|
||||||
# the PDF we are writing to stdout.
|
# the PDF we are writing to stdout.
|
||||||
with open(output_file, 'wb') as output_stream:
|
with Path(output_file).open('wb') as output_stream:
|
||||||
p_args = ocrmypdf_exec + [
|
p_args = ocrmypdf_exec + [
|
||||||
input_file,
|
input_file,
|
||||||
'-',
|
'-',
|
||||||
@@ -70,7 +71,7 @@ def test_stdout_protected_from_pollution(ocrmypdf_exec, resources, outpdf):
|
|||||||
p = run(p_args, stdout=output_stream, stderr=PIPE, stdin=DEVNULL, check=True)
|
p = run(p_args, stdout=output_stream, stderr=PIPE, stdin=DEVNULL, check=True)
|
||||||
|
|
||||||
assert check_pdf(output_file), "PDF on stdout was corrupted"
|
assert check_pdf(output_file), "PDF on stdout was corrupted"
|
||||||
with open(output_file, 'rb') as f:
|
with Path(output_file).open('rb') as f:
|
||||||
assert b'POLLUTION' not in f.read(), "pollution leaked into the PDF"
|
assert b'POLLUTION' not in f.read(), "pollution leaked into the PDF"
|
||||||
assert b'POLLUTION' in p.stderr, "pollution was not diverted to stderr"
|
assert b'POLLUTION' in p.stderr, "pollution was not diverted to stderr"
|
||||||
|
|
||||||
|
|||||||
@@ -103,7 +103,10 @@ class TestSystemFontProviderDirectories:
|
|||||||
patch.object(sys, 'platform', 'win32'),
|
patch.object(sys, 'platform', 'win32'),
|
||||||
patch.dict(
|
patch.dict(
|
||||||
'os.environ',
|
'os.environ',
|
||||||
{'WINDIR': r'C:\Windows', 'LOCALAPPDATA': r'C:\Users\Test\AppData\Local'},
|
{
|
||||||
|
'WINDIR': r'C:\Windows',
|
||||||
|
'LOCALAPPDATA': r'C:\Users\Test\AppData\Local',
|
||||||
|
},
|
||||||
),
|
),
|
||||||
):
|
):
|
||||||
provider._font_dirs = None # Reset cache
|
provider._font_dirs = None # Reset cache
|
||||||
@@ -113,8 +116,7 @@ class TestSystemFontProviderDirectories:
|
|||||||
assert len(dirs) == 2
|
assert len(dirs) == 2
|
||||||
assert any('Windows' in d and 'Fonts' in d for d in dir_strs)
|
assert any('Windows' in d and 'Fonts' in d for d in dir_strs)
|
||||||
assert any(
|
assert any(
|
||||||
'AppData' in d and 'Local' in d and 'Fonts' in d
|
'AppData' in d and 'Local' in d and 'Fonts' in d for d in dir_strs
|
||||||
for d in dir_strs
|
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_font_dirs_cached(self):
|
def test_font_dirs_cached(self):
|
||||||
@@ -361,6 +363,140 @@ class TestSystemFontProviderVariableFonts:
|
|||||||
assert provider.get_font('NotoSansSC-Regular') is None
|
assert provider.get_font('NotoSansSC-Regular') is None
|
||||||
|
|
||||||
|
|
||||||
|
class TestSystemFontProviderUnlistedFamilies:
|
||||||
|
"""Test the coverage-driven search over Noto families we don't enumerate.
|
||||||
|
|
||||||
|
``NOTO_FONT_PATTERNS`` names only the couple dozen most common scripts, but
|
||||||
|
macOS ships ~100 script-specific Noto fonts and Homebrew/Linux distros offer
|
||||||
|
even more. Those fonts must still be usable when the enumerated families
|
||||||
|
cannot cover the text. See issue #1722.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def real_font_bytes(self):
|
||||||
|
"""Bytes of a real, loadable font covering ASCII."""
|
||||||
|
font_path = (
|
||||||
|
Path(__file__).parent.parent
|
||||||
|
/ "src"
|
||||||
|
/ "ocrmypdf"
|
||||||
|
/ "data"
|
||||||
|
/ "NotoSans-Regular.ttf"
|
||||||
|
)
|
||||||
|
if not font_path.exists():
|
||||||
|
pytest.skip("Builtin font not available")
|
||||||
|
return font_path.read_bytes()
|
||||||
|
|
||||||
|
def _provider_for(self, tmp_path, filenames, real_font_bytes):
|
||||||
|
"""Build a provider whose only font dir is tmp_path with given files."""
|
||||||
|
for name in filenames:
|
||||||
|
(tmp_path / name).write_bytes(real_font_bytes)
|
||||||
|
provider = SystemFontProvider()
|
||||||
|
provider._font_dirs = [tmp_path]
|
||||||
|
return provider
|
||||||
|
|
||||||
|
def test_finds_unlisted_family_by_coverage(self, tmp_path, real_font_bytes):
|
||||||
|
"""A Noto family absent from NOTO_FONT_PATTERNS is still usable."""
|
||||||
|
provider = self._provider_for(
|
||||||
|
tmp_path, ['NotoSansCherokee-Regular.ttf'], real_font_bytes
|
||||||
|
)
|
||||||
|
assert 'NotoSansCherokee-Regular' not in provider.NOTO_FONT_PATTERNS
|
||||||
|
found = provider.find_font_with_glyphs('A')
|
||||||
|
assert found is not None
|
||||||
|
name, font = found
|
||||||
|
assert name == 'NotoSansCherokee-Regular'
|
||||||
|
assert font.font_path.name == 'NotoSansCherokee-Regular.ttf'
|
||||||
|
|
||||||
|
def test_finds_unlisted_variable_family(self, tmp_path, real_font_bytes):
|
||||||
|
"""Bracketed variable filenames are eligible for the coverage search."""
|
||||||
|
provider = self._provider_for(
|
||||||
|
tmp_path, ['NotoSansVithkuqi[wght].ttf'], real_font_bytes
|
||||||
|
)
|
||||||
|
found = provider.find_font_with_glyphs('A')
|
||||||
|
assert found is not None
|
||||||
|
assert found[0] == 'NotoSansVithkuqi-Regular'
|
||||||
|
|
||||||
|
def test_discovered_font_resolves_by_logical_name(self, tmp_path, real_font_bytes):
|
||||||
|
"""A font found by coverage is afterwards reachable via get_font()."""
|
||||||
|
provider = self._provider_for(
|
||||||
|
tmp_path, ['NotoSansCherokee-Regular.ttf'], real_font_bytes
|
||||||
|
)
|
||||||
|
name, font = provider.find_font_with_glyphs('A')
|
||||||
|
assert provider.get_font(name) is font
|
||||||
|
|
||||||
|
def test_negative_cache_does_not_block_discovery(self, tmp_path, real_font_bytes):
|
||||||
|
"""A prior failed get_font() must not hide a later coverage match."""
|
||||||
|
provider = self._provider_for(
|
||||||
|
tmp_path, ['NotoSansCherokee-Regular.ttf'], real_font_bytes
|
||||||
|
)
|
||||||
|
assert provider.get_font('NotoSansCherokee-Regular') is None # not listed
|
||||||
|
name, font = provider.find_font_with_glyphs('A')
|
||||||
|
assert provider.get_font(name) is font
|
||||||
|
|
||||||
|
def test_skips_bold_and_italic_styles(self, tmp_path, real_font_bytes):
|
||||||
|
"""Only Regular/variable faces are candidates, never Bold or Italic."""
|
||||||
|
provider = self._provider_for(
|
||||||
|
tmp_path,
|
||||||
|
[
|
||||||
|
'NotoSansCherokee-Bold.ttf',
|
||||||
|
'NotoSansCherokee-Italic.ttf',
|
||||||
|
'NotoSans-Italic[wdth,wght].ttf',
|
||||||
|
],
|
||||||
|
real_font_bytes,
|
||||||
|
)
|
||||||
|
assert provider.find_font_with_glyphs('A') is None
|
||||||
|
|
||||||
|
def test_ignores_non_noto_fonts(self, tmp_path, real_font_bytes):
|
||||||
|
"""Non-Noto system fonts are not enlisted by the coverage search."""
|
||||||
|
provider = self._provider_for(
|
||||||
|
tmp_path, ['DejaVuSans.ttf', 'Arial.ttf'], real_font_bytes
|
||||||
|
)
|
||||||
|
assert provider.find_font_with_glyphs('A') is None
|
||||||
|
|
||||||
|
def test_returns_none_when_no_font_covers_text(self, tmp_path, real_font_bytes):
|
||||||
|
"""Text no installed font covers yields no match rather than a wrong one."""
|
||||||
|
provider = self._provider_for(
|
||||||
|
tmp_path, ['NotoSansCherokee-Regular.ttf'], real_font_bytes
|
||||||
|
)
|
||||||
|
# U+13A3 CHEROKEE LETTER O is absent from the Latin font's cmap.
|
||||||
|
assert provider.find_font_with_glyphs('Ꭳ') is None
|
||||||
|
|
||||||
|
def test_empty_text_does_not_match(self, tmp_path, real_font_bytes):
|
||||||
|
"""Empty text has nothing to cover, so no font is claimed for it."""
|
||||||
|
provider = self._provider_for(
|
||||||
|
tmp_path, ['NotoSansCherokee-Regular.ttf'], real_font_bytes
|
||||||
|
)
|
||||||
|
assert provider.find_font_with_glyphs('') is None
|
||||||
|
|
||||||
|
def test_unloadable_font_file_is_skipped(self, tmp_path, real_font_bytes):
|
||||||
|
"""A corrupt font file does not abort the search for a usable one."""
|
||||||
|
(tmp_path / 'NotoSansBroken-Regular.ttf').write_bytes(b'not a font')
|
||||||
|
provider = self._provider_for(
|
||||||
|
tmp_path, ['NotoSansCherokee-Regular.ttf'], real_font_bytes
|
||||||
|
)
|
||||||
|
found = provider.find_font_with_glyphs('A')
|
||||||
|
assert found is not None
|
||||||
|
assert found[0] == 'NotoSansCherokee-Regular'
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
'stem,expected',
|
||||||
|
[
|
||||||
|
('NotoSansCherokee-Regular', 'NotoSansCherokee'),
|
||||||
|
('NotoSansCherokee[wght]', 'NotoSansCherokee'),
|
||||||
|
('NotoSansArabic[wdth,wght]', 'NotoSansArabic'),
|
||||||
|
('NotoSansCJKsc-VF', 'NotoSansCJKsc'),
|
||||||
|
('NotoMusic', 'NotoMusic'),
|
||||||
|
('NotoSansCJK-Regular', 'NotoSansCJK'),
|
||||||
|
('NotoSans-Bold', None),
|
||||||
|
('NotoSans-Italic[wdth,wght]', None),
|
||||||
|
('NotoSans-SemiCondensedBlackItalic', None),
|
||||||
|
('DejaVuSans-Regular', None),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_family_base_parsing(self, stem, expected):
|
||||||
|
"""Filename stems map to family bases, rejecting non-Regular styles."""
|
||||||
|
assert SystemFontProvider._family_base(stem) == expected
|
||||||
|
|
||||||
|
|
||||||
# --- ChainedFontProvider Tests ---
|
# --- ChainedFontProvider Tests ---
|
||||||
|
|
||||||
|
|
||||||
@@ -505,3 +641,49 @@ class TestChainedFontProviderIntegration:
|
|||||||
|
|
||||||
# Chain should have at least as many fonts as builtin
|
# Chain should have at least as many fonts as builtin
|
||||||
assert chain_fonts >= builtin_fonts
|
assert chain_fonts >= builtin_fonts
|
||||||
|
|
||||||
|
|
||||||
|
class TestChainedFontProviderCoverageSearch:
|
||||||
|
"""Test that the chain delegates the coverage search to its members."""
|
||||||
|
|
||||||
|
class _Searchable:
|
||||||
|
"""Provider stub that reports one findable font."""
|
||||||
|
|
||||||
|
def __init__(self, result):
|
||||||
|
self.result = result
|
||||||
|
self.calls = 0
|
||||||
|
|
||||||
|
def get_font(self, name):
|
||||||
|
return None
|
||||||
|
|
||||||
|
def get_available_fonts(self):
|
||||||
|
return []
|
||||||
|
|
||||||
|
def get_fallback_font(self):
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def find_font_with_glyphs(self, text):
|
||||||
|
self.calls += 1
|
||||||
|
return self.result
|
||||||
|
|
||||||
|
def test_delegates_to_first_provider_that_finds_a_font(self):
|
||||||
|
"""The first provider with a match wins; later ones are not consulted."""
|
||||||
|
first = self._Searchable(('NotoSansX-Regular', MagicMock()))
|
||||||
|
second = self._Searchable(('NotoSansY-Regular', MagicMock()))
|
||||||
|
chain = ChainedFontProvider([first, second])
|
||||||
|
|
||||||
|
assert chain.find_font_with_glyphs('x')[0] == 'NotoSansX-Regular'
|
||||||
|
assert second.calls == 0
|
||||||
|
|
||||||
|
def test_skips_providers_without_the_capability(self):
|
||||||
|
"""Providers lacking find_font_with_glyphs() are skipped, not fatal."""
|
||||||
|
legacy = MagicMock(spec=['get_font', 'get_available_fonts'])
|
||||||
|
searchable = self._Searchable(('NotoSansX-Regular', MagicMock()))
|
||||||
|
chain = ChainedFontProvider([legacy, searchable])
|
||||||
|
|
||||||
|
assert chain.find_font_with_glyphs('x')[0] == 'NotoSansX-Regular'
|
||||||
|
|
||||||
|
def test_returns_none_when_nothing_matches(self):
|
||||||
|
"""No provider matching yields None so the caller can use Occulta."""
|
||||||
|
chain = ChainedFontProvider([self._Searchable(None)])
|
||||||
|
assert chain.find_font_with_glyphs('x') is None
|
||||||
|
|||||||
+19
-11
@@ -47,10 +47,13 @@ def make_ocr_opts(input_file='a.pdf', output_file='b.pdf', **kwargs):
|
|||||||
|
|
||||||
|
|
||||||
def test_old_tesseract_error():
|
def test_old_tesseract_error():
|
||||||
with patch(
|
with (
|
||||||
'ocrmypdf._exec.tesseract.version',
|
patch(
|
||||||
return_value=TesseractVersion('4.00.00alpha'),
|
'ocrmypdf._exec.tesseract.version',
|
||||||
), pytest.raises(MissingDependencyError):
|
return_value=TesseractVersion('4.00.00alpha'),
|
||||||
|
),
|
||||||
|
pytest.raises(MissingDependencyError),
|
||||||
|
):
|
||||||
vd.check_options(*make_opts_pm(pdf_renderer='sandwich', language='eng'))
|
vd.check_options(*make_opts_pm(pdf_renderer='sandwich', language='eng'))
|
||||||
|
|
||||||
|
|
||||||
@@ -59,9 +62,9 @@ def test_tesseract_not_installed(caplog):
|
|||||||
not_found.side_effect = FileNotFoundError('tesseract')
|
not_found.side_effect = FileNotFoundError('tesseract')
|
||||||
with pytest.raises(MissingDependencyError, match="Could not find program"):
|
with pytest.raises(MissingDependencyError, match="Could not find program"):
|
||||||
vd.check_options(*make_opts_pm())
|
vd.check_options(*make_opts_pm())
|
||||||
assert (
|
assert "'tesseract' could not be executed" in caplog.text, (
|
||||||
"'tesseract' could not be executed" in caplog.text
|
"Error message not printed"
|
||||||
), "Error message not printed"
|
)
|
||||||
assert 'install' in caplog.text, "Install advice not printed"
|
assert 'install' in caplog.text, "Install advice not printed"
|
||||||
not_found.assert_called()
|
not_found.assert_called()
|
||||||
|
|
||||||
@@ -86,10 +89,15 @@ def test_mutex_options():
|
|||||||
make_ocr_opts(redo_ocr=True, force_ocr=True)
|
make_ocr_opts(redo_ocr=True, force_ocr=True)
|
||||||
|
|
||||||
|
|
||||||
def test_optimizing(caplog):
|
def test_optimizing_png_quality_warns(caplog):
|
||||||
vd.check_options(
|
vd.check_options(*make_opts_pm(optimize=0, png_quality=18))
|
||||||
*make_opts_pm(optimize=0, png_quality=18, jpeg_quality=10)
|
assert 'will be ignored because' in caplog.text
|
||||||
)
|
|
||||||
|
|
||||||
|
def test_optimizing_jpeg_quality_warns(caplog):
|
||||||
|
# Isolated from png_quality so this actually exercises the jpeg_quality
|
||||||
|
# path rather than being confounded by png_quality also being set.
|
||||||
|
vd.check_options(*make_opts_pm(optimize=0, jpeg_quality=10))
|
||||||
assert 'will be ignored because' in caplog.text
|
assert 'will be ignored because' in caplog.text
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user