Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5d7b5742e4 | ||
|
|
c391b2b7d0 | ||
|
|
0250929150 | ||
|
|
9748208e68 | ||
|
|
e4b0c04be4 | ||
|
|
efb83ad64f | ||
|
|
08e40f96e8 | ||
|
|
3f6feb1dcc | ||
|
|
ab6553f4ff | ||
|
|
cedca9fa1f | ||
|
|
3f40118022 | ||
|
|
b18b1da6d0 | ||
|
|
14fb9f56e8 | ||
|
|
8709cf506b | ||
|
|
9a92eb40df | ||
|
|
0a59c210f9 | ||
|
|
0b370fdd15 | ||
|
|
1c16dd26f7 | ||
|
|
c355d927ba | ||
|
|
c993857752 | ||
|
|
84f5fe9ee0 | ||
|
|
3336d67e77 | ||
|
|
73e16e7821 | ||
|
|
6f1d37d78f | ||
|
|
2ed82de2e0 | ||
|
|
c43903fa14 | ||
|
|
1c89cacfef | ||
|
|
75714fe43e | ||
|
|
e371ce95ca | ||
|
|
716a2e22c3 | ||
|
|
10e6019ada | ||
|
|
89c76b5145 | ||
|
|
83c04e6399 | ||
|
|
7fdeeb3635 | ||
|
|
5be368fe75 | ||
|
|
91c5b1e480 | ||
|
|
73154b97ba | ||
|
|
76a40759ae | ||
|
|
12ce565e98 | ||
|
|
9f46126859 | ||
|
|
11849e5a70 | ||
|
|
e30c00cc26 | ||
|
|
001b403657 | ||
|
|
851c61ee85 | ||
|
|
f5ebd23b8f | ||
|
|
81118c6195 | ||
|
|
834b60a02a | ||
|
|
47e3b5b4d2 | ||
|
|
d9346cc3d8 | ||
|
|
a35dd1f9ee | ||
|
|
bf46f4fe35 | ||
|
|
55b76338a8 | ||
|
|
2af7b1c179 | ||
|
|
69f4cca9b6 | ||
|
|
4babdfcfbf |
+10
-10
@@ -87,7 +87,7 @@ jobs:
|
||||
uv run --no-dev pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
|
||||
|
||||
- name: Upload coverage to Codecov
|
||||
uses: codecov/codecov-action@v5
|
||||
uses: codecov/codecov-action@v6
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
@@ -149,7 +149,7 @@ jobs:
|
||||
uv run --no-dev pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
|
||||
|
||||
- name: Upload coverage to Codecov
|
||||
uses: codecov/codecov-action@v5
|
||||
uses: codecov/codecov-action@v6
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
@@ -196,7 +196,7 @@ jobs:
|
||||
uv run --no-dev pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
|
||||
|
||||
- name: Upload coverage to Codecov
|
||||
uses: codecov/codecov-action@v5
|
||||
uses: codecov/codecov-action@v6
|
||||
env:
|
||||
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
|
||||
with:
|
||||
@@ -218,7 +218,7 @@ jobs:
|
||||
run: |
|
||||
uv build --sdist --wheel
|
||||
|
||||
- uses: actions/upload-artifact@v6
|
||||
- uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: artifact
|
||||
path: |
|
||||
@@ -235,7 +235,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- uses: actions/download-artifact@v7
|
||||
- uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: artifact
|
||||
path: dist
|
||||
@@ -284,17 +284,17 @@ jobs:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: jbarlow83
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
uses: docker/setup-qemu-action@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Print image tag
|
||||
run: echo "Building image ${DOCKER_REPOSITORY}/${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}"
|
||||
@@ -330,14 +330,14 @@ jobs:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: jbarlow83
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Print image tag
|
||||
run: echo "Building image ${DOCKER_REPOSITORY}/${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}"
|
||||
|
||||
@@ -34,7 +34,7 @@ jobs:
|
||||
|
||||
# PyPI doesn't support sigstore publishing, so generate after publishing to PyPI
|
||||
- name: Sign the dists with Sigstore
|
||||
uses: sigstore/gh-action-sigstore-python@v3.2.0
|
||||
uses: sigstore/gh-action-sigstore-python@v3.3.0
|
||||
with:
|
||||
inputs: |
|
||||
./dist/*.tar.gz
|
||||
@@ -87,13 +87,13 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: jbarlow83
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Tag ocrmypdf (Ubuntu) image
|
||||
run: |
|
||||
|
||||
+90
-76
@@ -1,8 +1,3 @@
|
||||
# OCRmyPDF 中文说明
|
||||
|
||||
我将为您创建一个中文版的 README.md 文件。以下是基于原始英文版本的中文翻译:
|
||||
|
||||
```markdown:/Users/huapai/PycharmProjects/OCRmyPDF/README.md
|
||||
<!-- SPDX-FileCopyrightText: 2014 Julien Pfefferkorn -->
|
||||
<!-- SPDX-FileCopyrightText: 2015 James R. Barlow -->
|
||||
<!-- SPDX-License-Identifier: CC-BY-SA-4.0 -->
|
||||
@@ -16,98 +11,109 @@
|
||||
[docs]: https://readthedocs.org/projects/ocrmypdf/badge/?version=latest "RTD"
|
||||
[pyversions]: https://img.shields.io/pypi/pyversions/ocrmypdf "支持的 Python 版本"
|
||||
|
||||
OCRmyPDF 为扫描的 PDF 文件添加 OCR 文本层,使其可以被搜索或复制粘贴。
|
||||
OCRmyPDF 会为扫描版 PDF 文件添加 OCR 文本层,使其可以搜索或复制粘贴。
|
||||
|
||||
```bash
|
||||
ocrmypdf # 这是一个可脚本化的命令行程序
|
||||
-l eng+fra # 支持多种语言
|
||||
--rotate-pages # 可以修正旋转错误的页面
|
||||
--deskew # 可以校正倾斜的 PDF!
|
||||
--title "My PDF" # 可以更改输出元数据
|
||||
--jobs 4 # 默认使用多核心处理
|
||||
--output-type pdfa # 默认生成 PDF/A 格式
|
||||
ocrmypdf # 它是一个可脚本化的命令行程序
|
||||
-l eng+fra # 它支持多种语言
|
||||
--rotate-pages # 它可以修正旋转方向错误的页面
|
||||
--deskew # 它可以校正歪斜的 PDF!
|
||||
--title "My PDF" # 它可以更改输出元数据
|
||||
--jobs 4 # 它默认使用多个 CPU 核心
|
||||
--output-type pdfa # 它默认生成 PDF/A
|
||||
input_scanned.pdf # 接受 PDF 输入(或图像)
|
||||
output_searchable.pdf # 生成经过验证的 PDF 输出
|
||||
```
|
||||
|
||||
[查看发布说明了解最新变更的详情](https://ocrmypdf.readthedocs.io/en/latest/release_notes.html)。
|
||||
[查看发布说明,了解最新变更详情](https://ocrmypdf.readthedocs.io/en/latest/release_notes.html)。
|
||||
|
||||
## 主要特点
|
||||
## 主要功能
|
||||
|
||||
- 从普通 PDF 生成可搜索的 [PDF/A](https://en.wikipedia.org/?title=PDF/A) 文件
|
||||
- 准确地将 OCR 文本放置在图像下方,便于复制/粘贴
|
||||
- 将 OCR 文本准确放置在图像下方,便于复制/粘贴
|
||||
- 保持原始嵌入图像的精确分辨率
|
||||
- 在可能的情况下,以"无损"操作方式插入 OCR 信息,不破坏任何其他内容
|
||||
- 在可能时,以“无损”操作插入 OCR 信息,不干扰任何其他内容
|
||||
- 优化 PDF 图像,通常生成比输入文件更小的文件
|
||||
- 如果需要,在执行 OCR 前对图像进行校正和/或清理
|
||||
- 按需在执行 OCR 前校正和/或清理图像
|
||||
- 验证输入和输出文件
|
||||
- 在所有可用的 CPU 核心上分配工作
|
||||
- 在所有可用 CPU 核心间分配工作
|
||||
- 使用 [Tesseract OCR](https://github.com/tesseract-ocr/tesseract) 引擎识别超过 [100 种语言](https://github.com/tesseract-ocr/tessdata)
|
||||
- 保护您的私人数据安全
|
||||
- 适当扩展以处理包含数千页的文件
|
||||
- 在数百万 PDF 上经过实战测试
|
||||
- 保护你的私有数据。
|
||||
- 可以妥善扩展,处理包含数千页的文件。
|
||||
- 已在数百万份 PDF 上经过实战检验。
|
||||
|
||||
<img src="misc/screencast/demo.svg" alt="终端会话中的 OCRmyPDF 演示">
|
||||
<img src="misc/screencast/demo.svg" alt="OCRmyPDF 在终端会话中的演示">
|
||||
|
||||
详情请参阅[文档](https://ocrmypdf.readthedocs.io/en/latest/)。
|
||||
|
||||
## 开发动机
|
||||
## 动机
|
||||
|
||||
我在网上搜索免费的命令行工具来对 PDF 文件进行 OCR:我找到了很多,但没有一个真正令人满意:
|
||||
我曾在网上寻找一款免费的命令行工具来对 PDF 文件执行 OCR:我找到了很多,但没有一个真正令人满意:
|
||||
|
||||
- 要么它们生成的 PDF 文件中文本位置错误(使复制/粘贴变得不可能)
|
||||
- 要么它们不处理重音和多语言字符
|
||||
- 要么它们改变了嵌入图像的分辨率
|
||||
- 要么它们生成了体积巨大的 PDF 文件
|
||||
- 要么它们在尝试 OCR 时崩溃
|
||||
- 要么它们不生成有效的 PDF 文件
|
||||
- 最重要的是,它们都不生成 PDF/A 文件(专为长期存储设计的格式)
|
||||
- 要么生成的 PDF 文件中文本位于图像下方的错误位置(导致无法复制/粘贴)
|
||||
- 要么无法处理重音字符和多语言字符
|
||||
- 要么会改变嵌入图像的分辨率
|
||||
- 要么生成的 PDF 文件大得离谱
|
||||
- 要么在尝试 OCR 时崩溃
|
||||
- 要么无法生成有效的 PDF 文件
|
||||
- 除此之外,它们都不能生成 PDF/A 文件(专为长期存储设计的格式)
|
||||
|
||||
...所以我决定开发自己的工具。
|
||||
……所以我决定开发自己的工具。
|
||||
|
||||
## 安装
|
||||
|
||||
支持 Linux、Windows、macOS 和 FreeBSD。Docker 镜像也可用,同时支持 x64 和 ARM。
|
||||
支持 Linux、Windows、macOS 和 FreeBSD。也提供 Docker 镜像,同时支持 x64 和 ARM。
|
||||
|
||||
| 操作系统 | 安装命令 |
|
||||
| --------------------------- | ----------------------------- |
|
||||
| Debian, Ubuntu | ``apt install ocrmypdf`` |
|
||||
| Windows Subsystem for Linux | ``apt install ocrmypdf`` |
|
||||
| Fedora | ``dnf install ocrmypdf`` |
|
||||
| macOS (Homebrew) | ``brew install ocrmypdf`` |
|
||||
| macOS (MacPorts) | ``port install ocrmypdf`` |
|
||||
| macOS (nix) | ``nix-env -i ocrmypdf`` |
|
||||
| LinuxBrew | ``brew install ocrmypdf`` |
|
||||
| FreeBSD | ``pkg install py-ocrmypdf`` |
|
||||
| Ubuntu Snap | ``snap install ocrmypdf`` |
|
||||
| 操作系统 | 安装命令 |
|
||||
| ----------------------------- | ------------------------------ |
|
||||
| Debian, Ubuntu | ``apt install ocrmypdf`` |
|
||||
| Windows Subsystem for Linux | ``apt install ocrmypdf`` |
|
||||
| Fedora | ``dnf install ocrmypdf`` |
|
||||
| macOS (Homebrew) | ``brew install ocrmypdf`` |
|
||||
| macOS (MacPorts) | ``port install ocrmypdf`` |
|
||||
| macOS (nix) | ``nix-env -i ocrmypdf`` |
|
||||
| LinuxBrew | ``brew install ocrmypdf`` |
|
||||
| FreeBSD | ``pkg install py-ocrmypdf`` |
|
||||
| OpenBSD | ``pkg_add ocrmypdf`` |
|
||||
| Ubuntu Snap | ``snap install ocrmypdf`` |
|
||||
|
||||
对于其他用户,[请参阅我们的文档](https://ocrmypdf.readthedocs.io/en/latest/installation.html)了解安装步骤。
|
||||
其他用户请[参阅我们的文档](https://ocrmypdf.readthedocs.io/en/latest/installation.html)了解安装步骤。
|
||||
|
||||
## 语言
|
||||
|
||||
OCRmyPDF 使用 Tesseract 进行 OCR,并依赖其语言包。对于 Linux 用户,您通常可以找到提供语言包的软件包:
|
||||
OCRmyPDF 使用 Tesseract 执行 OCR,并依赖其语言包。对于 Linux 用户,通常可以找到提供语言包的软件包:
|
||||
|
||||
```bash
|
||||
# 显示所有 Tesseract 语言包的列表
|
||||
apt-cache search tesseract-ocr
|
||||
|
||||
# Debian/Ubuntu 用户
|
||||
apt-get install tesseract-ocr-chi-sim # 示例:安装中文简体语言包
|
||||
apt-cache search tesseract-ocr # 显示所有 Tesseract 语言包列表
|
||||
apt-get install tesseract-ocr-chi-sim # 示例:安装简体中文语言包
|
||||
|
||||
|
||||
# Arch Linux 用户
|
||||
pacman -S tesseract-data-eng tesseract-data-deu # 示例:安装英语和德语语言包
|
||||
|
||||
# OpenBSD 用户
|
||||
pkg_info -aQ tesseract # 显示所有 Tesseract 语言包列表
|
||||
pkg_add tesseract-cym # 示例:安装威尔士语语言包
|
||||
|
||||
# brew macOS 用户
|
||||
brew install tesseract-lang
|
||||
|
||||
# Fedora 用户
|
||||
dnf search tesseract-langpack # 显示所有 Tesseract 语言包列表
|
||||
dnf install tesseract-langpack-ita # 示例:安装意大利语语言包
|
||||
|
||||
|
||||
```
|
||||
|
||||
然后,您可以向 OCRmyPDF 传递 `-l LANG` 参数,提示它应该搜索哪些语言。可以请求多种语言。
|
||||
随后可以向 OCRmyPDF 传递 `-l LANG` 参数,提示它应搜索哪些语言。可以同时请求多种语言。
|
||||
|
||||
OCRmyPDF 支持 Tesseract 4.1.1+。它会自动使用在 `PATH` 环境变量中首先找到的版本。在 Windows 上,如果 `PATH` 不提供 Tesseract 二进制文件,我们会根据 Windows 注册表使用已安装的最高版本号。
|
||||
OCRmyPDF 支持 Tesseract 4.1.1+。它会自动使用 `PATH` 环境变量中首先找到的版本。在 Windows 上,如果 `PATH` 中没有 Tesseract 二进制文件,我们会根据 Windows 注册表使用已安装的最高版本号。
|
||||
|
||||
## 文档和支持
|
||||
|
||||
安装 OCRmyPDF 后,可以通过以下方式访问内置帮助,解释命令语法和选项:
|
||||
安装 OCRmyPDF 后,可以通过以下命令访问内置帮助,了解命令语法和选项:
|
||||
|
||||
```bash
|
||||
ocrmypdf --help
|
||||
@@ -115,13 +121,13 @@ ocrmypdf --help
|
||||
|
||||
我们的[文档托管在 Read the Docs 上](https://ocrmypdf.readthedocs.io/en/latest/index.html)。
|
||||
|
||||
请在我们的 [GitHub issues](https://github.com/ocrmypdf/OCRmyPDF/issues) 页面上报告问题,并遵循问题模板以获得快速响应。
|
||||
请在我们的 [GitHub issues](https://github.com/ocrmypdf/OCRmyPDF/issues) 页面报告问题,并遵循 issue 模板以便快速获得响应。
|
||||
|
||||
## 功能演示
|
||||
|
||||
```bash
|
||||
# 添加 OCR 层并转换为 PDF/A
|
||||
ocrmypdf input.pdf output.pdf
|
||||
# 添加 OCR 层并要求输出 PDF/A
|
||||
ocrmypdf --output-type pdfa input.pdf output.pdf
|
||||
|
||||
# 将图像转换为单页 PDF
|
||||
ocrmypdf input.jpg output.pdf
|
||||
@@ -129,45 +135,53 @@ ocrmypdf input.jpg output.pdf
|
||||
# 就地为文件添加 OCR(仅在成功时修改文件)
|
||||
ocrmypdf myfile.pdf myfile.pdf
|
||||
|
||||
# 使用非英语语言进行 OCR(查找您语言的 ISO 639-3 代码)
|
||||
# 使用非英语语言执行 OCR(请查找对应语言的 ISO 639-3 代码)
|
||||
ocrmypdf -l fra LeParisien.pdf LeParisien.pdf
|
||||
|
||||
# OCR 多语言文档
|
||||
ocrmypdf -l eng+fra Bilingual-English-French.pdf Bilingual-English-French.pdf
|
||||
|
||||
# 校正(矫正倾斜的页面)
|
||||
# 校正歪斜页面
|
||||
ocrmypdf --deskew input.pdf output.pdf
|
||||
```
|
||||
|
||||
更多功能,请参阅[文档](https://ocrmypdf.readthedocs.io/en/latest/index.html)。
|
||||
更多功能请参阅[文档](https://ocrmypdf.readthedocs.io/en/latest/index.html)。
|
||||
|
||||
## 要求
|
||||
|
||||
除了所需的 Python 版本外,OCRmyPDF 还需要外部程序安装 Ghostscript 和 Tesseract OCR。OCRmyPDF 是纯 Python 编写的,几乎可以在所有平台上运行:Linux、macOS、Windows 和 FreeBSD。
|
||||
除所需的 Python 版本外,OCRmyPDF 还需要安装 Ghostscript 和 Tesseract OCR 这两个外部程序。OCRmyPDF 是纯 Python 项目,几乎可以在所有平台上运行:Linux、macOS、Windows 和 FreeBSD。
|
||||
|
||||
## 媒体报道
|
||||
## 插件
|
||||
|
||||
- [使用 OCRmyPDF 实现无纸化](https://medium.com/@ikirichenko/going-paperless-with-ocrmypdf-e2f36143f46a)
|
||||
- [将扫描文档转换为带有编辑的压缩可搜索 PDF](https://medium.com/@treyharris/converting-a-scanned-document-into-a-compressed-searchable-pdf-with-redactions-63f61c34fe4c)
|
||||
- [c't 1-2014, 第 59 页](https://heise.de/-2279695):在德国领先的 IT 杂志 c't 中详细介绍 OCRmyPDF v1.0
|
||||
- [heise Open Source, 09/2014: 使用 OCRmyPDF 进行文本识别](https://heise.de/-2356670)
|
||||
- [heise 使用 OCRmyPDF 创建可搜索的 PDF 文档](https://www.heise.de/ratgeber/Durchsuchbare-PDF-Dokumente-mit-OCRmyPDF-erstellen-4607592.html)
|
||||
- [优秀实用工具:OCRmyPDF](https://www.linuxlinks.com/excellent-utilities-ocrmypdf-add-ocr-text-layer-scanned-pdfs/)
|
||||
- [LinuxUser 使用 OCRmyPDF 和 Scanbd 自动化文本识别](https://www.linux-community.de/ausgaben/linuxuser/2021/06/texterkennung-mit-ocrmypdf-und-scanbd-automatisieren/)
|
||||
- [Y Combinator 讨论](https://news.ycombinator.com/item?id=32028752)
|
||||
OCRmyPDF 提供插件接口,允许扩展或替换其能力。以下是我们知道的一些插件:
|
||||
|
||||
## 商业咨询
|
||||
- [OCRmyPDF-AppleOCR](https://github.com/mkyt/ocrmypdf-AppleOCR):用 Apple Vision Framework 替换标准 Tesseract OCR 引擎。需要 macOS。
|
||||
- [OCRmyPDF-EasyOCR](https://github.com/ocrmypdf/OCRmyPDF-EasyOCR):用 EasyOCR 替换标准 Tesseract OCR 引擎;EasyOCR 是基于 PyTorch 的较新 OCR 引擎。强烈建议使用 GPU。
|
||||
- [OCRmyPDF-PaddleOCR](https://github.com/clefru/ocrmypdf-paddleocr):用 PaddleOCR 替换标准 Tesseract OCR 引擎;PaddleOCR 是功能强大的 GPU 加速 OCR 引擎。
|
||||
|
||||
如果没有公司和用户选择为功能开发和咨询提供支持,OCRmyPDF 就不会成为今天的软件。我们很乐意讨论所有咨询,无论是扩展现有功能集,还是将 OCRmyPDF 集成到更大的系统中。
|
||||
[paperless-ngx](https://docs.paperless-ngx.com/) 将 OCRmyPDF 集成到可搜索的文档管理系统中。
|
||||
|
||||
## 新闻与媒体
|
||||
|
||||
- [Going paperless with OCRmyPDF](https://medium.com/@ikirichenko/going-paperless-with-ocrmypdf-e2f36143f46a)
|
||||
- [Converting a scanned document into a compressed searchable PDF with redactions](https://medium.com/@treyharris/converting-a-scanned-document-into-a-compressed-searchable-pdf-with-redactions-63f61c34fe4c)
|
||||
- [c't 1-2014,第 59 页](https://heise.de/-2279695):德国领先 IT 杂志 c't 对 OCRmyPDF v1.0 的详细介绍
|
||||
- [heise Open Source, 09/2014: Texterkennung mit OCRmyPDF](https://heise.de/-2356670)
|
||||
- [heise Durchsuchbare PDF-Dokumente mit OCRmyPDF erstellen](https://www.heise.de/ratgeber/Durchsuchbare-PDF-Dokumente-mit-OCRmyPDF-erstellen-4607592.html)
|
||||
- [Excellent Utilities: OCRmyPDF](https://www.linuxlinks.com/excellent-utilities-ocrmypdf-add-ocr-text-layer-scanned-pdfs/)
|
||||
- [LinuxUser Texterkennung mit OCRmyPDF und Scanbd automatisieren](https://www.linux-community.de/ausgaben/linuxuser/2021/06/texterkennung-mit-ocrmypdf-und-scanbd-automatisieren/)
|
||||
- [Y Combinator discussion](https://news.ycombinator.com/item?id=32028752)
|
||||
|
||||
## 商务咨询
|
||||
|
||||
如果没有公司和用户选择支持功能开发与咨询服务,OCRmyPDF 不会成为今天的软件。无论是扩展现有功能集,还是将 OCRmyPDF 集成到更大的系统中,我们都很乐意讨论各类咨询需求。
|
||||
|
||||
## 许可证
|
||||
|
||||
OCRmyPDF 软件根据 Mozilla 公共许可证 2.0 (MPL-2.0) 授权。此许可证允许将 OCRmyPDF 与其他代码集成,包括商业和闭源代码,但要求您发布对 OCRmyPDF 所做的源代码级修改。
|
||||
OCRmyPDF 软件采用 Mozilla Public License 2.0 (MPL-2.0) 授权。该许可证允许将 OCRmyPDF 与其他代码集成,包括商业代码和闭源代码,但要求你发布对 OCRmyPDF 所做的源代码级修改。
|
||||
|
||||
OCRmyPDF 的某些组件有其他许可证,如标准 SPDX 许可证标识符或 DEP5 版权和许可信息文件所示。一般来说,非核心代码根据 MIT 许可,文档和测试文件根据 Creative Commons ShareAlike 4.0 (CC-BY-SA 4.0) 许可。
|
||||
OCRmyPDF 的某些组件采用其他许可证,具体由标准 SPDX 许可证标识符或 DEP5 版权与许可信息文件标明。一般来说,非核心代码采用 MIT 许可证,文档和测试文件采用 Creative Commons ShareAlike 4.0 (CC-BY-SA 4.0) 许可证。
|
||||
|
||||
## 免责声明
|
||||
|
||||
本软件按"原样"分发,不提供任何明示或暗示的保证或条件。
|
||||
|
||||
这份中文版 README.md 保留了原始文档的所有重要信息,包括功能介绍、安装说明、语言支持、使用示例等内容,同时保持了原始格式和结构。
|
||||
本软件按“原样”分发,不提供任何明示或暗示的保证或条件。
|
||||
|
||||
@@ -419,6 +419,70 @@ curves. In this case, you may want to use a different color conversion
|
||||
strategy. The `--color-conversion-strategy` option allows you to select a
|
||||
different strategy, such as `RGB`.
|
||||
|
||||
## Advanced Ghostscript tuning
|
||||
|
||||
:::{versionadded} 17.5.0
|
||||
:::
|
||||
|
||||
OCRmyPDF intentionally hides most Ghostscript controls because Ghostscript
|
||||
is a legacy code path. The preferred PDF/A pipeline in v17+ uses pypdfium2
|
||||
as the rasterizer and verapdf to validate speculative PDF/A output, with
|
||||
Ghostscript reserved as a fallback for PDFs that cannot be made compliant
|
||||
without it. OCRmyPDF's separate optimizer (controlled by `--optimize`,
|
||||
`--jpeg-quality`, `--png-quality`, etc.) is the supported way to shrink
|
||||
output PDFs: it gives consistent results across input files, and isolates
|
||||
Ghostscript so it can focus on producing a PDF/A with as few image
|
||||
transformations as possible.
|
||||
|
||||
The two options below are exposed for advanced users who want to tune
|
||||
Ghostscript's intermediate PDF/A output directly. Most users will get
|
||||
more predictable results from the optimizer.
|
||||
|
||||
### `--ghostscript-jpeg-quality Q`
|
||||
|
||||
Sets Ghostscript's `-dJPEGQ` switch for images that Ghostscript chooses
|
||||
to recompress to JPEG while building a PDF/A. `Q=0` requests maximum
|
||||
compression and `Q=100` requests best quality; if the flag is omitted,
|
||||
OCRmyPDF passes `95` (the historical default). This only affects images
|
||||
Ghostscript transcodes — existing JPEGs pass through unchanged on modern
|
||||
Ghostscript releases. For end-to-end JPEG quality tuning, prefer
|
||||
`--jpeg-quality`, which is implemented by the OCRmyPDF optimizer and is
|
||||
applied independently of whatever Ghostscript decides to do.
|
||||
|
||||
Note: setting both `--ghostscript-jpeg-quality` and `--jpeg-quality` can
|
||||
result in double JPEG recompression, since the optimizer may re-encode
|
||||
images that Ghostscript already recompressed. This can degrade quality
|
||||
in subtle ways.
|
||||
|
||||
### `--ghostscript-jpeg-maxdpi DPI`
|
||||
|
||||
Enables Ghostscript's image downsampling and caps color, grayscale, and
|
||||
monochrome image resolution to `DPI`. The downsample threshold is set to
|
||||
`1.0`, so any image whose effective DPI exceeds the cap will be
|
||||
downsampled.
|
||||
|
||||
Reducing JPEG quality is almost always a better trade than downsampling
|
||||
at the same compression budget: a 400 DPI JPEG at modest quality usually
|
||||
looks much better than a 200 DPI JPEG, because the JPEG codec can spend
|
||||
bits where they count. Downsampling is also dangerous for PDFs that
|
||||
combine a low-resolution color image with a high-resolution monochrome
|
||||
mask — capping the mask resolution can produce visible quality loss.
|
||||
For these reasons, prefer `--jpeg-quality` over `--ghostscript-jpeg-maxdpi`
|
||||
unless you specifically want to force a hard DPI cap.
|
||||
|
||||
Example:
|
||||
|
||||
```bash
|
||||
ocrmypdf --output-type pdfa \
|
||||
--ghostscript-jpeg-quality 80 \
|
||||
--ghostscript-jpeg-maxdpi 150 \
|
||||
in.pdf out.pdf
|
||||
```
|
||||
|
||||
These options only take effect when Ghostscript is invoked for PDF/A
|
||||
conversion (`--output-type pdfa`, `pdfa-1`, `pdfa-2`, or `pdfa-3`, or
|
||||
when `--output-type auto` falls back to Ghostscript).
|
||||
|
||||
## PDF/A output modes
|
||||
|
||||
:::{versionchanged} 17.0.0
|
||||
|
||||
+25
-5
@@ -31,6 +31,16 @@ ocrmypdf --output-type pdf input.pdf output.pdf
|
||||
ocrmypdf --output-type pdfa --pdfa-image-compression jpeg input.pdf output.pdf
|
||||
```
|
||||
|
||||
### Reduce JPEG quality with the optimizer
|
||||
|
||||
This is the recommended way to shrink JPEG content in the output. The
|
||||
optimizer applies regardless of `--output-type`, so it works on both
|
||||
plain PDFs and Ghostscript-produced PDF/A files.
|
||||
|
||||
```bash
|
||||
ocrmypdf --optimize 2 --jpeg-quality 60 input.pdf output.pdf
|
||||
```
|
||||
|
||||
### Modify a file in place
|
||||
|
||||
The file will only be overwritten if OCRmyPDF is successful.
|
||||
@@ -333,12 +343,22 @@ Hyphens denote a range of pages and commas separate page numbers. If you
|
||||
prefer to use spaces, quote all of the page numbers:
|
||||
`--pages '2, 3, 5, 7'`.
|
||||
|
||||
The token `end` (case-insensitive) is an alias for the last page in the
|
||||
document. For example, `--pages 3-end` OCRs from page 3 through the
|
||||
final page, and `--pages end` OCRs only the last page:
|
||||
|
||||
```bash
|
||||
ocrmypdf --pages 3-end input.pdf output.pdf
|
||||
ocrmypdf --pages end input.pdf output.pdf
|
||||
```
|
||||
|
||||
OCRmyPDF will warn if your list of page numbers contains duplicates or
|
||||
overlapping pages. OCRmyPDF does not currently account for document page
|
||||
numbers, such as an introduction section of a book that uses Roman
|
||||
numerals. It simply counts the number of virtual pieces of paper since
|
||||
the start. If your list of pages is out of numerical order, OCRmyPDF
|
||||
will sort it for you.
|
||||
overlapping pages. (Repeated page numbers are de-duplicated automatically,
|
||||
since the underlying set of pages is what matters.) OCRmyPDF does not
|
||||
currently account for document page numbers, such as an introduction
|
||||
section of a book that uses Roman numerals. It simply counts the number
|
||||
of virtual pieces of paper since the start. If your list of pages is out
|
||||
of numerical order, OCRmyPDF will sort it for you.
|
||||
|
||||
Regardless of the argument to `--pages`, OCRmyPDF will optimize all
|
||||
pages/images in the file and convert it to PDF/A, unless you disable
|
||||
|
||||
@@ -183,6 +183,11 @@ v17 addresses through alternative codepaths. When Ghostscript is used:
|
||||
`jpeg` or `lossless` to set all images to one type or the other.
|
||||
Ghostscript lacks an option to maintain the input image's format.
|
||||
(Modern Ghostscript can copy JPEG images without transcoding them.)
|
||||
Advanced users can also tune Ghostscript's image recompression with
|
||||
`--ghostscript-jpeg-quality` and `--ghostscript-jpeg-maxdpi`; see
|
||||
[Advanced Ghostscript tuning](advanced.md#advanced-ghostscript-tuning).
|
||||
Most users should prefer `--jpeg-quality` (applied by the OCRmyPDF
|
||||
optimizer) over those Ghostscript-scoped controls.
|
||||
- Ghostscript's PDF/A conversion removes any XMP metadata that is not
|
||||
one of the standard XMP metadata namespaces for PDFs. In particular,
|
||||
PRISM Metadata is removed.
|
||||
|
||||
+10
-1
@@ -98,7 +98,16 @@ If `pngquant` is installed, OCRmyPDF will use it to perform quantize
|
||||
paletted images to reduce their size.
|
||||
|
||||
The quality of JPEGs may be lowered, on the assumption that a lower
|
||||
quality image may be suitable for storage after OCR.
|
||||
quality image may be suitable for storage after OCR. Use `--jpeg-quality`
|
||||
to control the optimizer's JPEG quality target. The optimizer is the
|
||||
recommended way to reduce JPEG image sizes: it applies consistently
|
||||
regardless of whether Ghostscript was used to produce a PDF/A.
|
||||
|
||||
If you specifically need to tune Ghostscript's own PDF/A image handling
|
||||
(for example, to force a hard DPI cap), see
|
||||
[Advanced Ghostscript tuning](advanced.md#advanced-ghostscript-tuning)
|
||||
for the separate `--ghostscript-jpeg-quality` and
|
||||
`--ghostscript-jpeg-maxdpi` options.
|
||||
|
||||
It is not possible to optimize all image types. Uncommon image types may
|
||||
be skipped by the optimizer.
|
||||
|
||||
@@ -3,6 +3,49 @@
|
||||
|
||||
# v17
|
||||
|
||||
## v17.5.0
|
||||
|
||||
- Added support for the ``end`` alias in ``--pages``, denoting the last page
|
||||
of the document. For example, ``--pages 3-end`` OCRs from page 3 through
|
||||
the final page. {issue}`1615`
|
||||
- Added ``--ghostscript-jpeg-quality`` and ``--ghostscript-jpeg-maxdpi``
|
||||
advanced options for tuning Ghostscript's PDF/A output. The optimizer's
|
||||
``--jpeg-quality`` remains the recommended file-size control.
|
||||
- Fixed pypdfium2 rasterizer clipping content when the CropBox was smaller
|
||||
than the MediaBox (e.g. JSTOR or cropped PDFs). {issue}`1685`
|
||||
- Fixed Form XObject cycle detection in the optimizer's image xref scan.
|
||||
Self-referential or DAG-shaped Form graphs (notably from PowerPoint
|
||||
exports) previously produced floods of recursion warnings and could hang
|
||||
for minutes. {issue}`1321`
|
||||
- Tesseract config errors are now surfaced as ``TesseractConfigError`` with
|
||||
actionable guidance, instead of crashing later with a confusing
|
||||
``FileNotFoundError`` on the missing hOCR output. {issue}`1687`
|
||||
- Refreshed the Chinese README translation. Thanks @cislunarspace.
|
||||
- Internal refactoring of the ``_exec`` and ``subprocess`` modules to
|
||||
separate probing from execution.
|
||||
- CI dependency updates.
|
||||
|
||||
## v17.4.2
|
||||
|
||||
- Fixed Python API unconditionally overriding ``PIL.Image.MAX_IMAGE_PIXELS``
|
||||
when the caller did not explicitly set ``max_image_mpixels``. Host
|
||||
applications (e.g. Paperless-NGX) that configure the PIL limit before
|
||||
invoking ``ocrmypdf.ocr()`` now have their setting respected. The CLI
|
||||
default of 250 megapixels is unchanged. {issue}`1665`
|
||||
- Updated uv.lock to avoid pinning a vulnerable version of Pillow. {issue}`1666`
|
||||
|
||||
## v17.4.1
|
||||
|
||||
- Fixed RTL text extraction order in the fpdf2 renderer. Arabic lam-alef
|
||||
ligatures and other multi-character CMap entries were garbled by the bidi
|
||||
algorithm during text extraction. {issue}`1655`
|
||||
- Fixed ``work_folder`` not being set in ``PdfContext`` options when using
|
||||
the Python API. Thanks @bluebox-steven. {issue}`1613`
|
||||
- Updated Ghostscript JPEG corruption warning to include the detected version
|
||||
number, confirming the bug persists in Ghostscript 10.7.0.
|
||||
- Internal refactoring.
|
||||
- CI dependency updates.
|
||||
|
||||
## v17.4.0
|
||||
|
||||
- Added ``--no-overwrite`` / ``-n`` option to prevent overwriting output files.
|
||||
|
||||
@@ -46,6 +46,8 @@ __ocrmypdf_arguments()
|
||||
--rasterizer (PDF page rasterizer)
|
||||
--rotate-pages-threshold (page rotation confidence)
|
||||
--pdfa-image-compression (set PDF/A image compression options)
|
||||
--ghostscript-jpeg-quality (Ghostscript JPEG quality during PDF/A [0..100])
|
||||
--ghostscript-jpeg-maxdpi (cap Ghostscript image DPI during PDF/A)
|
||||
--fast-web-view (if file size if above this amount in MB linearize PDF)
|
||||
--continue-on-soft-render-error (continue after recoverable render errors)
|
||||
--plugin (name of plugin to import)
|
||||
@@ -337,6 +339,7 @@ __ocrmypdf_check_previous()
|
||||
|
||||
--title|--author|--subject|--keywords|--unpaper-args|--pages|--plugin|\
|
||||
--jpeg-quality|--png-quality|--image-dpi|--oversample|--skip-big|--max-image-mpixels|\
|
||||
--ghostscript-jpeg-quality|--ghostscript-jpeg-maxdpi|\
|
||||
--tesseract-timeout|--tesseract-non-ocr-timeout|--tesseract-downsample-above|\
|
||||
--rotate-pages-threshold|--fast-web-view)
|
||||
# argument required but no completions available
|
||||
|
||||
@@ -102,6 +102,8 @@ function __fish_ocrmypdf_pdfa_compression
|
||||
echo -e "lossless\t"(_ "convert color and grayscale images to lossless (PNG)")
|
||||
end
|
||||
complete -c ocrmypdf -x -l pdfa-image-compression -a '(__fish_ocrmypdf_pdfa_compression)' -d "set PDF/A image compression options"
|
||||
complete -c ocrmypdf -x -l ghostscript-jpeg-quality -d "Ghostscript JPEG quality during PDF/A [0..100]"
|
||||
complete -c ocrmypdf -x -l ghostscript-jpeg-maxdpi -d "cap Ghostscript image DPI during PDF/A"
|
||||
|
||||
complete -c ocrmypdf -x -s j -l jobs -d "how many worker processes to use"
|
||||
complete -c ocrmypdf -x -l title -d "set metadata"
|
||||
|
||||
+2
-1
@@ -6,7 +6,7 @@ build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "ocrmypdf"
|
||||
version = "17.4.0"
|
||||
version = "17.5.0"
|
||||
description = "OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched"
|
||||
readme = "README.md"
|
||||
license = "MPL-2.0"
|
||||
@@ -161,6 +161,7 @@ dev = [
|
||||
"ipykernel>=6.29.5",
|
||||
"reportlab>=4.4.4",
|
||||
"cyclopts>=4.5.1",
|
||||
"pygithub>=2.9.1",
|
||||
]
|
||||
test = [
|
||||
# Core testing framework
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
# SPDX-FileCopyrightText: 2026 James R. Barlow
|
||||
# SPDX-License-Identifier: MPL-2.0
|
||||
"""Probe helper for external executables.
|
||||
|
||||
Each ``ocrmypdf._exec.<tool>`` module describes its external program with a
|
||||
module-level :class:`ToolProbe` and delegates ``version()`` / ``available()``
|
||||
to it. This separates the "is the tool installed and suitable?" question
|
||||
(probing) from the "run the tool" question (execution). Work functions stay
|
||||
as pure module-level functions so they are trivially picklable for use in
|
||||
subprocess workers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
|
||||
from packaging.version import Version
|
||||
|
||||
from ocrmypdf.exceptions import MissingDependencyError
|
||||
from ocrmypdf.subprocess import get_version
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ToolProbe:
|
||||
"""Describes how to detect an external executable and its version.
|
||||
|
||||
Attributes:
|
||||
program: The program name as it appears on PATH (or a full path).
|
||||
version_arg: The argument that elicits a version string.
|
||||
version_regex: A regex with a capturing group that extracts the
|
||||
version from the program's output.
|
||||
version_cls: A :class:`packaging.version.Version` subclass, used for
|
||||
tools with non-standard version strings (e.g. Tesseract).
|
||||
env: Optional environment overrides applied when probing the version.
|
||||
also_catch: Additional exception types that should be treated as
|
||||
"not available" by :meth:`available`. :class:`OSError` is useful
|
||||
for tools like verapdf whose launcher may fail with non-standard
|
||||
errors when the JVM is missing.
|
||||
"""
|
||||
|
||||
program: str
|
||||
version_arg: str = '--version'
|
||||
version_regex: str = r'(\d+(\.\d+)*)'
|
||||
version_cls: type[Version] = Version
|
||||
env: Mapping[str, str] | None = None
|
||||
also_catch: tuple[type[BaseException], ...] = ()
|
||||
|
||||
def version(self) -> Version:
|
||||
"""Return the installed version of the program.
|
||||
|
||||
Raises:
|
||||
MissingDependencyError: if the program cannot be found or its
|
||||
version string cannot be parsed.
|
||||
"""
|
||||
raw = get_version(
|
||||
self.program,
|
||||
version_arg=self.version_arg,
|
||||
regex=self.version_regex,
|
||||
env=self.env,
|
||||
)
|
||||
return self.version_cls(raw)
|
||||
|
||||
def available(self) -> bool:
|
||||
"""Return whether a usable version of the program is installed."""
|
||||
try:
|
||||
self.version()
|
||||
except MissingDependencyError:
|
||||
return False
|
||||
except self.also_catch:
|
||||
return False
|
||||
return True
|
||||
@@ -16,6 +16,7 @@ from subprocess import PIPE, CalledProcessError
|
||||
from packaging.version import Version
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
|
||||
from ocrmypdf._exec._probe import ToolProbe
|
||||
from ocrmypdf.exceptions import (
|
||||
ColorConversionNeededError,
|
||||
InputFileError,
|
||||
@@ -23,7 +24,7 @@ from ocrmypdf.exceptions import (
|
||||
)
|
||||
from ocrmypdf.helpers import Resolution
|
||||
from ocrmypdf.pluginspec import GhostscriptRasterDevice
|
||||
from ocrmypdf.subprocess import get_version, run, run_polling_stderr
|
||||
from ocrmypdf.subprocess import run, run_polling_stderr
|
||||
|
||||
COLOR_CONVERSION_STRATEGIES = frozenset(
|
||||
[
|
||||
@@ -69,11 +70,19 @@ class DuplicateFilter(logging.Filter):
|
||||
return True
|
||||
|
||||
|
||||
log.addFilter(DuplicateFilter(log))
|
||||
PROBE = ToolProbe(program=GS)
|
||||
version = PROBE.version
|
||||
available = PROBE.available
|
||||
|
||||
|
||||
def version() -> Version:
|
||||
return Version(get_version(GS))
|
||||
def _ensure_log_filter_installed() -> None:
|
||||
"""Idempotently attach the duplicate-suppressing filter to the GS logger.
|
||||
|
||||
Called at the top of each work function so the filter is present in the
|
||||
main process *and* in any subprocess worker that calls Ghostscript.
|
||||
"""
|
||||
if not any(isinstance(f, DuplicateFilter) for f in log.filters):
|
||||
log.addFilter(DuplicateFilter(log))
|
||||
|
||||
|
||||
def _gs_error_reported(stream) -> bool:
|
||||
@@ -123,6 +132,7 @@ def rasterize_pdf(
|
||||
use_cropbox: If True, rasterize the CropBox instead of MediaBox.
|
||||
Default is False (use MediaBox).
|
||||
"""
|
||||
_ensure_log_filter_installed()
|
||||
raster_dpi = raster_dpi.round(6)
|
||||
if not page_dpi:
|
||||
page_dpi = raster_dpi
|
||||
@@ -268,11 +278,14 @@ def generate_pdfa(
|
||||
*,
|
||||
compression: str,
|
||||
color_conversion_strategy: str,
|
||||
jpeg_quality: int | None = None,
|
||||
jpeg_maxdpi: int | None = None,
|
||||
pdf_version: str = '1.5',
|
||||
pdfa_part: str = '2',
|
||||
progressbar_class=None,
|
||||
stop_on_error: bool = False,
|
||||
):
|
||||
_ensure_log_filter_installed()
|
||||
# Ghostscript's compression is all or nothing. We can either force all images
|
||||
# to JPEG, force all to Flate/PNG, or let it decide how to encode the images.
|
||||
# In most case it's best to let it decide.
|
||||
@@ -307,6 +320,35 @@ def generate_pdfa(
|
||||
# Windows has lots of fatal "permission denied" errors
|
||||
stop_on_error = False
|
||||
|
||||
# `-dJPEGQ=N` tells Ghostscript to use a JPEG quality of N, IF it decides
|
||||
# to transcode an image to JPEG. When there are existing JPEG images,
|
||||
# Ghostscript uses passthrough mode, so the quality level is not changed.
|
||||
# OCRmyPDF's optimizer separately uses the `--jpeg-quality` command line
|
||||
# option to potentially re-encode JPEG images, regardless of whether
|
||||
# Ghostscript decided to transcode them to JPEG or not.
|
||||
# `jpeg_quality=0` is meaningful to Ghostscript (maximum compression), so
|
||||
# only fall back to the default when the value is None.
|
||||
effective_jpeg_quality = jpeg_quality if jpeg_quality is not None else 95
|
||||
|
||||
# Downsampling images is a blunt-force way to reduce file size and almost
|
||||
# always degrades quality more than lowering JPEG quality at the original
|
||||
# resolution. We expose this for users with very specific needs (e.g.
|
||||
# producing very small files for screen-only viewing); the optimizer is
|
||||
# usually a better choice.
|
||||
downsample_args: list[str] = []
|
||||
if jpeg_maxdpi is not None:
|
||||
downsample_args = [
|
||||
"-dDownsampleColorImages=true",
|
||||
"-dColorImageDownsampleThreshold=1.0",
|
||||
"-dDownsampleGrayImages=true",
|
||||
"-dGrayImageDownsampleThreshold=1.0",
|
||||
"-dDownsampleMonoImages=true",
|
||||
"-dMonoImageDownsampleThreshold=1.0",
|
||||
f"-dColorImageResolution={jpeg_maxdpi}",
|
||||
f"-dGrayImageResolution={jpeg_maxdpi}",
|
||||
f"-dMonoImageResolution={jpeg_maxdpi}",
|
||||
]
|
||||
|
||||
# nb no need to specify ProcessColorModel when ColorConversionStrategy
|
||||
# is set; see:
|
||||
# https://bugs.ghostscript.com/show_bug.cgi?id=699392
|
||||
@@ -323,8 +365,9 @@ def generate_pdfa(
|
||||
]
|
||||
+ (['-dPDFSTOPONERROR'] if stop_on_error else [])
|
||||
+ compression_args
|
||||
+ downsample_args
|
||||
+ [
|
||||
"-dJPEGQ=95",
|
||||
f"-dJPEGQ={effective_jpeg_quality}", # See note above on JPEG quality
|
||||
"-dSubsetFonts=false", # Prevents GS from messing up some encodings
|
||||
f"-dPDFA={pdfa_part}",
|
||||
"-dPDFACompatibilityPolicy=1",
|
||||
|
||||
@@ -9,21 +9,23 @@ from subprocess import PIPE, CalledProcessError
|
||||
|
||||
from packaging.version import Version
|
||||
|
||||
from ocrmypdf._exec._probe import ToolProbe
|
||||
from ocrmypdf.exceptions import MissingDependencyError
|
||||
from ocrmypdf.subprocess import get_version, run
|
||||
from ocrmypdf.subprocess import run
|
||||
|
||||
_PROBE = ToolProbe(program='jbig2', version_regex=r'jbig2enc (\d+(\.\d+)*).*')
|
||||
|
||||
|
||||
def version() -> Version:
|
||||
try:
|
||||
version = get_version('jbig2', regex=r'jbig2enc (\d+(\.\d+)*).*')
|
||||
return _PROBE.version()
|
||||
except CalledProcessError as e:
|
||||
# TeX Live for Windows provides an incompatible jbig2.EXE which may
|
||||
# be on the PATH.
|
||||
raise MissingDependencyError('jbig2enc') from e
|
||||
return Version(version)
|
||||
|
||||
|
||||
def available():
|
||||
def available() -> bool:
|
||||
try:
|
||||
version()
|
||||
except MissingDependencyError:
|
||||
|
||||
@@ -8,22 +8,12 @@ from __future__ import annotations
|
||||
from pathlib import Path
|
||||
from subprocess import PIPE
|
||||
|
||||
from packaging.version import Version
|
||||
from ocrmypdf._exec._probe import ToolProbe
|
||||
from ocrmypdf.subprocess import run
|
||||
|
||||
from ocrmypdf.exceptions import MissingDependencyError
|
||||
from ocrmypdf.subprocess import get_version, run
|
||||
|
||||
|
||||
def version() -> Version:
|
||||
return Version(get_version('pngquant', regex=r'(\d+(\.\d+)*).*'))
|
||||
|
||||
|
||||
def available():
|
||||
try:
|
||||
version()
|
||||
except MissingDependencyError:
|
||||
return False
|
||||
return True
|
||||
PROBE = ToolProbe(program='pngquant', version_regex=r'(\d+(\.\d+)*).*')
|
||||
version = PROBE.version
|
||||
available = PROBE.available
|
||||
|
||||
|
||||
def quantize(input_file: Path, output_file: Path, quality_min: int, quality_max: int):
|
||||
|
||||
@@ -17,13 +17,14 @@ from subprocess import PIPE, STDOUT, CalledProcessError, TimeoutExpired
|
||||
|
||||
from packaging.version import Version
|
||||
|
||||
from ocrmypdf._exec._probe import ToolProbe
|
||||
from ocrmypdf.exceptions import (
|
||||
MissingDependencyError,
|
||||
SubprocessOutputError,
|
||||
TesseractConfigError,
|
||||
)
|
||||
from ocrmypdf.pluginspec import OrientationConfidence
|
||||
from ocrmypdf.subprocess import get_version, run
|
||||
from ocrmypdf.subprocess import run
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -115,8 +116,13 @@ class TesseractVersion(Version):
|
||||
)
|
||||
|
||||
|
||||
def version() -> Version:
|
||||
return TesseractVersion(get_version('tesseract', regex=r'tesseract\s(.+)'))
|
||||
PROBE = ToolProbe(
|
||||
program='tesseract',
|
||||
version_regex=r'tesseract\s(.+)',
|
||||
version_cls=TesseractVersion,
|
||||
)
|
||||
version = PROBE.version
|
||||
available = PROBE.available
|
||||
|
||||
|
||||
def has_thresholding() -> bool:
|
||||
@@ -287,9 +293,7 @@ def tesseract_log_output(stream: bytes) -> None:
|
||||
|
||||
lines = text.splitlines()
|
||||
for line in lines:
|
||||
if line.startswith(
|
||||
("Tesseract Open Source", "Warning in pixReadMem")
|
||||
):
|
||||
if line.startswith(("Tesseract Open Source", "Warning in pixReadMem")):
|
||||
continue
|
||||
elif 'diacritics' in line:
|
||||
tlog.warning("lots of diacritics - possibly poor OCR")
|
||||
@@ -309,6 +313,23 @@ def tesseract_log_output(stream: bytes) -> None:
|
||||
tlog.warning(line.strip())
|
||||
elif 'read_params_file' in line.lower():
|
||||
tlog.error(line.strip())
|
||||
# Tesseract emits "read_params_file: Can't open <name>" when it
|
||||
# cannot locate a config file (e.g. 'hocr', 'txt') in its
|
||||
# tessdata configs/ directory, then exits 0 without producing
|
||||
# the requested output. Promote to a hard error so the user
|
||||
# sees the root cause instead of a downstream FileNotFoundError.
|
||||
if "Can't open" in line:
|
||||
missing = line.split("Can't open", 1)[1].strip()
|
||||
else:
|
||||
missing = line.strip()
|
||||
raise TesseractConfigError(
|
||||
f"Tesseract cannot open its config file '{missing}'. "
|
||||
"This usually means Tesseract is installed but its config "
|
||||
"files are missing from the tessdata configs/ directory. "
|
||||
"On Debian/Ubuntu, ensure the 'tesseract-ocr' package is "
|
||||
"fully installed. If you set TESSDATA_PREFIX, verify its "
|
||||
"configs/ subdirectory contains the required files."
|
||||
)
|
||||
else:
|
||||
tlog.info(line.strip())
|
||||
|
||||
@@ -389,6 +410,12 @@ def generate_hocr(
|
||||
raise SubprocessOutputError() from e
|
||||
else:
|
||||
tesseract_log_output(stdout)
|
||||
if not output_hocr.exists():
|
||||
raise SubprocessOutputError(
|
||||
"Tesseract exited successfully but did not produce the "
|
||||
f"expected hOCR output at {output_hocr}. Tesseract output:\n"
|
||||
+ (stdout.decode(errors='replace') if stdout else '(empty)')
|
||||
)
|
||||
# The sidecar text file will get the suffix .txt; rename it to
|
||||
# whatever caller wants it named
|
||||
with suppress(FileNotFoundError):
|
||||
@@ -457,6 +484,12 @@ def generate_pdf(
|
||||
stdout = p.stdout
|
||||
with suppress(FileNotFoundError):
|
||||
prefix.with_suffix('.txt').replace(output_text)
|
||||
if not output_pdf.exists():
|
||||
raise SubprocessOutputError(
|
||||
"Tesseract exited successfully but did not produce the "
|
||||
f"expected PDF output at {output_pdf}. Tesseract output:\n"
|
||||
+ (stdout.decode(errors='replace') if stdout else '(empty)')
|
||||
)
|
||||
except TimeoutExpired:
|
||||
page_timedout(timeout)
|
||||
use_skip_page(output_pdf, output_text)
|
||||
|
||||
@@ -14,11 +14,11 @@ from pathlib import Path
|
||||
from subprocess import PIPE, STDOUT
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
from packaging.version import Version
|
||||
from PIL import Image
|
||||
|
||||
from ocrmypdf._exec._probe import ToolProbe
|
||||
from ocrmypdf.exceptions import SubprocessOutputError
|
||||
from ocrmypdf.subprocess import get_version, run
|
||||
from ocrmypdf.subprocess import run
|
||||
|
||||
# unpaper documentation:
|
||||
# https://github.com/Flameeyes/unpaper/blob/main/doc/basic-concepts.md
|
||||
@@ -46,8 +46,9 @@ class UnpaperImageTooLargeError(Exception):
|
||||
super().__init__(self.message)
|
||||
|
||||
|
||||
def version() -> Version:
|
||||
return Version(get_version('unpaper', regex=r'(?m).*?(\d+(\.\d+)(\.\d+)?)'))
|
||||
PROBE = ToolProbe(program='unpaper', version_regex=r'(?m).*?(\d+(\.\d+)(\.\d+)?)')
|
||||
version = PROBE.version
|
||||
available = PROBE.available
|
||||
|
||||
|
||||
@contextmanager
|
||||
|
||||
@@ -11,10 +11,9 @@ from pathlib import Path
|
||||
from subprocess import PIPE
|
||||
from typing import NamedTuple
|
||||
|
||||
from packaging.version import Version
|
||||
|
||||
from ocrmypdf._exec._probe import ToolProbe
|
||||
from ocrmypdf.exceptions import MissingDependencyError
|
||||
from ocrmypdf.subprocess import get_version, run
|
||||
from ocrmypdf.subprocess import run
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -27,18 +26,13 @@ class ValidationResult(NamedTuple):
|
||||
message: str
|
||||
|
||||
|
||||
def version() -> Version:
|
||||
"""Get verapdf version."""
|
||||
return Version(get_version('verapdf', regex=r'veraPDF (\d+(\.\d+)*)'))
|
||||
|
||||
|
||||
def available() -> bool:
|
||||
"""Check if verapdf is available."""
|
||||
try:
|
||||
version()
|
||||
except (MissingDependencyError, OSError):
|
||||
return False
|
||||
return True
|
||||
PROBE = ToolProbe(
|
||||
program='verapdf',
|
||||
version_regex=r'veraPDF (\d+(\.\d+)*)',
|
||||
also_catch=(OSError,),
|
||||
)
|
||||
version = PROBE.version
|
||||
available = PROBE.available
|
||||
|
||||
|
||||
def output_type_to_flavour(output_type: str) -> str:
|
||||
|
||||
@@ -36,6 +36,7 @@ class PdfContext:
|
||||
plugin_manager,
|
||||
):
|
||||
self.options = options
|
||||
self.options.work_folder = work_folder
|
||||
self.work_folder = work_folder
|
||||
self.origin = origin
|
||||
self.pdfinfo = pdfinfo
|
||||
|
||||
@@ -65,8 +65,34 @@ class TaggedPdfMode(StrEnum):
|
||||
ignore = 'ignore'
|
||||
|
||||
|
||||
def _pages_from_ranges(ranges: str) -> set[int]:
|
||||
"""Convert page range string to set of page numbers."""
|
||||
def _has_end_alias(ranges: str) -> bool:
|
||||
"""Return True if the page range string uses the ``end`` alias."""
|
||||
return 'end' in ranges.lower()
|
||||
|
||||
|
||||
def _resolve_page_token(token: str, total_pages: int | None) -> int:
|
||||
"""Convert a single page-number token to a 1-based integer.
|
||||
|
||||
The literal ``end`` (case-insensitive) is resolved to ``total_pages``. If
|
||||
``total_pages`` is None, an error is raised.
|
||||
"""
|
||||
if token.lower() == 'end':
|
||||
if total_pages is None:
|
||||
raise BadArgsError(
|
||||
"'end' was used in --pages but the total page count is not yet "
|
||||
"known"
|
||||
)
|
||||
return total_pages
|
||||
return int(token)
|
||||
|
||||
|
||||
def _pages_from_ranges(ranges: str, total_pages: int | None = None) -> set[int]:
|
||||
"""Convert page range string to set of 0-based page numbers.
|
||||
|
||||
The token ``end`` (case-insensitive) is an alias for the last page of the
|
||||
document. It is resolved using ``total_pages``; if ``end`` appears in the
|
||||
string and ``total_pages`` is None, a :class:`BadArgsError` is raised.
|
||||
"""
|
||||
pages: list[int] = []
|
||||
page_groups = ranges.replace(' ', '').split(',')
|
||||
for group in page_groups:
|
||||
@@ -75,10 +101,15 @@ def _pages_from_ranges(ranges: str) -> set[int]:
|
||||
try:
|
||||
start, end = group.split('-')
|
||||
except ValueError:
|
||||
pages.append(int(group) - 1)
|
||||
try:
|
||||
pages.append(_resolve_page_token(group, total_pages) - 1)
|
||||
except ValueError:
|
||||
raise BadArgsError(f"invalid page number '{group}'") from None
|
||||
else:
|
||||
try:
|
||||
new_pages = list(range(int(start) - 1, int(end)))
|
||||
start_n = _resolve_page_token(start, total_pages)
|
||||
end_n = _resolve_page_token(end, total_pages)
|
||||
new_pages = list(range(start_n - 1, end_n))
|
||||
if not new_pages:
|
||||
raise BadArgsError(
|
||||
f"invalid page subrange '{start}-{end}'"
|
||||
@@ -192,7 +223,7 @@ class OcrOptions(BaseModel):
|
||||
no_overwrite: bool = False
|
||||
|
||||
# Advanced options
|
||||
max_image_mpixels: float = 250.0
|
||||
max_image_mpixels: float | None = None
|
||||
pdf_renderer: str = 'auto'
|
||||
ocr_engine: str = 'auto'
|
||||
rasterizer: str = 'auto'
|
||||
@@ -301,7 +332,7 @@ class OcrOptions(BaseModel):
|
||||
@classmethod
|
||||
def validate_max_image_mpixels(cls, v):
|
||||
"""Validate max image megapixels."""
|
||||
if v < 0:
|
||||
if v is not None and v < 0:
|
||||
raise ValueError("max_image_mpixels must be non-negative")
|
||||
return v
|
||||
|
||||
@@ -332,11 +363,19 @@ class OcrOptions(BaseModel):
|
||||
@field_validator('pages')
|
||||
@classmethod
|
||||
def validate_pages_format(cls, v):
|
||||
"""Convert page ranges string to set of page numbers."""
|
||||
"""Convert page ranges string to set of page numbers.
|
||||
|
||||
If the string uses the ``end`` alias, the original string is preserved
|
||||
so that resolution can happen later, once the document's page count is
|
||||
known.
|
||||
"""
|
||||
if v is None:
|
||||
return v
|
||||
if isinstance(v, set):
|
||||
return v # Already processed
|
||||
if _has_end_alias(v):
|
||||
# Defer resolution until total page count is known
|
||||
return v
|
||||
|
||||
# Convert string ranges to set of page numbers
|
||||
return _pages_from_ranges(v)
|
||||
@@ -582,6 +621,13 @@ class OcrOptions(BaseModel):
|
||||
value = getattr(self, flat_name)
|
||||
if value is not None:
|
||||
kwargs[field_name] = _convert_value(value)
|
||||
# Plugin-scoped fields that aren't in the central OcrOptions
|
||||
# registry: argparse stores them in extra_attrs under the
|
||||
# namespace_field name.
|
||||
elif flat_name in self.extra_attrs:
|
||||
value = self.extra_attrs[flat_name]
|
||||
if value is not None:
|
||||
kwargs[field_name] = _convert_value(value)
|
||||
# Also check direct field name (for fields like jbig2_lossy)
|
||||
elif field_name in OcrOptions.model_fields:
|
||||
value = getattr(self, field_name)
|
||||
|
||||
@@ -329,10 +329,13 @@ def setup_pipeline(
|
||||
# Note: OcrOptions is immutable, so we can't modify options.jobs directly
|
||||
# The jobs field should already be set correctly during OcrOptions creation
|
||||
|
||||
# Apply PIL max image pixels side effect
|
||||
PIL.Image.MAX_IMAGE_PIXELS = int(options.max_image_mpixels * 1_000_000)
|
||||
if PIL.Image.MAX_IMAGE_PIXELS == 0:
|
||||
PIL.Image.MAX_IMAGE_PIXELS = None # type: ignore
|
||||
# Apply PIL max image pixels side effect only when explicitly requested.
|
||||
# When None, leave PIL.Image.MAX_IMAGE_PIXELS as the host application
|
||||
# configured it. The CLI passes its own default (250.0) via argparse.
|
||||
if options.max_image_mpixels is not None:
|
||||
PIL.Image.MAX_IMAGE_PIXELS = int(options.max_image_mpixels * 1_000_000)
|
||||
if PIL.Image.MAX_IMAGE_PIXELS == 0:
|
||||
PIL.Image.MAX_IMAGE_PIXELS = None # type: ignore
|
||||
|
||||
pikepdf_enable_mmap()
|
||||
executor = setup_executor(plugin_manager)
|
||||
@@ -340,12 +343,17 @@ def setup_pipeline(
|
||||
|
||||
|
||||
def do_get_pdfinfo(pdf_path: Path, executor: Executor, options) -> PdfInfo:
|
||||
# Handle pages field - it might be a string that needs conversion
|
||||
# Handle pages field - it might be a string that needs conversion.
|
||||
# A string indicates the ``end`` alias was used and resolution was
|
||||
# deferred; we resolve it now using the document's actual page count.
|
||||
check_pages = options.pages
|
||||
if isinstance(check_pages, str):
|
||||
from ocrmypdf._options import _pages_from_ranges
|
||||
|
||||
check_pages = _pages_from_ranges(check_pages)
|
||||
with Pdf.open(pdf_path) as pdf:
|
||||
total_pages = len(pdf.pages)
|
||||
check_pages = _pages_from_ranges(check_pages, total_pages=total_pages)
|
||||
options.pages = check_pages
|
||||
|
||||
return get_pdfinfo(
|
||||
pdf_path,
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
# SPDX-FileCopyrightText: 2022 James R. Barlow
|
||||
# SPDX-License-Identifier: MPL-2.0
|
||||
__version__ = "17.4.0"
|
||||
__version__ = "17.5.0"
|
||||
|
||||
@@ -54,6 +54,27 @@ class GhostscriptOptions(BaseModel):
|
||||
pdfa_image_compression: Annotated[
|
||||
PdfaImageCompression, Field(description="PDF/A image compression method")
|
||||
] = PdfaImageCompression.AUTO
|
||||
jpeg_quality: Annotated[
|
||||
int | None,
|
||||
Field(
|
||||
ge=0,
|
||||
le=100,
|
||||
description=(
|
||||
"JPEG quality (0-100) for Ghostscript image recompression during "
|
||||
"PDF/A generation; None uses Ghostscript's default."
|
||||
),
|
||||
),
|
||||
] = None
|
||||
jpeg_maxdpi: Annotated[
|
||||
int | None,
|
||||
Field(
|
||||
ge=1,
|
||||
description=(
|
||||
"Maximum DPI for Ghostscript image downsampling during PDF/A "
|
||||
"generation."
|
||||
),
|
||||
),
|
||||
] = None
|
||||
|
||||
@classmethod
|
||||
def add_arguments_to_parser(cls, parser, namespace: str = 'ghostscript'):
|
||||
@@ -86,6 +107,35 @@ class GhostscriptOptions(BaseModel):
|
||||
"skipped. Not supported for --output-type=pdf ; that setting "
|
||||
"preserves the original compression of all images.",
|
||||
)
|
||||
gs.add_argument(
|
||||
'--ghostscript-jpeg-quality',
|
||||
type=int,
|
||||
metavar='Q',
|
||||
default=None,
|
||||
dest=f'{namespace}_jpeg_quality',
|
||||
help=(
|
||||
"Advanced: Set Ghostscript's -dJPEGQ for images that Ghostscript "
|
||||
"transcodes to JPEG during PDF/A generation. 0 is maximum "
|
||||
"compression; 100 is best quality. If omitted, Ghostscript's "
|
||||
"default is used. This only affects images Ghostscript chooses "
|
||||
"to recompress; for general JPEG quality tuning prefer "
|
||||
"--jpeg-quality, which is applied by the OCRmyPDF optimizer."
|
||||
),
|
||||
)
|
||||
gs.add_argument(
|
||||
'--ghostscript-jpeg-maxdpi',
|
||||
type=int,
|
||||
metavar='DPI',
|
||||
default=None,
|
||||
dest=f'{namespace}_jpeg_maxdpi',
|
||||
help=(
|
||||
"Advanced: Force Ghostscript to downsample color, grayscale, "
|
||||
"and monochrome images in PDF/A output to the given maximum DPI. "
|
||||
"Reducing JPEG quality usually gives better results than "
|
||||
"downsampling at the same file size, and can degrade quality "
|
||||
"of high-resolution monochrome masks."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@hookimpl
|
||||
@@ -131,10 +181,10 @@ def check_options(options):
|
||||
)
|
||||
if gs_version >= Version('10.6.0'):
|
||||
log.warning(
|
||||
"Ghostscript 10.6.x contains JPEG encoding errors that may corrupt "
|
||||
"images. OCRmyPDF will attempt to mitigate, but this version is "
|
||||
"strongly not recommended. Please upgrade to a newer version. "
|
||||
"As of 2025-12, 10.6.0 is the latest version of Ghostscript."
|
||||
"Ghostscript %s contains JPEG encoding errors that may corrupt "
|
||||
"images. OCRmyPDF will attempt to mitigate, but versions 10.6.0+ "
|
||||
"are strongly not recommended until this is fixed upstream.",
|
||||
gs_version,
|
||||
)
|
||||
if options.output_type == 'pdfa':
|
||||
options.output_type = 'pdfa-2'
|
||||
@@ -352,6 +402,8 @@ def generate_pdfa(
|
||||
output_file=output_file,
|
||||
compression=context.options.ghostscript.pdfa_image_compression,
|
||||
color_conversion_strategy=context.options.ghostscript.color_conversion_strategy,
|
||||
jpeg_quality=context.options.ghostscript.jpeg_quality,
|
||||
jpeg_maxdpi=context.options.ghostscript.jpeg_maxdpi,
|
||||
pdf_version=pdf_version,
|
||||
pdfa_part=pdfa_part,
|
||||
progressbar_class=progressbar_class,
|
||||
|
||||
@@ -48,27 +48,18 @@ def _open_pdf_document(input_file: Path):
|
||||
return pdfium.PdfDocument(input_file)
|
||||
|
||||
|
||||
def _calculate_mediabox_crop(page) -> tuple[float, float, float, float]:
|
||||
"""Calculate crop values to expand rendering from CropBox to MediaBox.
|
||||
def _expand_cropbox_to_mediabox(page) -> None:
|
||||
"""Set the page's CropBox to its MediaBox so PDFium renders the full page.
|
||||
|
||||
By default pypdfium2 renders to the CropBox. To render the full MediaBox,
|
||||
we need negative crop values to expand the rendering area.
|
||||
|
||||
Returns:
|
||||
Tuple of (left, bottom, right, top) crop values. Negative values
|
||||
expand the rendering area beyond the CropBox to the MediaBox.
|
||||
PDFium renders to the CropBox by default. Negative ``crop`` values to
|
||||
``render()`` are not supported and only pad the output canvas without
|
||||
expanding the rendered area — content outside the CropBox is clipped.
|
||||
The supported approach is to widen the CropBox in memory before rendering.
|
||||
The document is never saved back to disk, so this mutation is local.
|
||||
See https://github.com/ocrmypdf/OCRmyPDF/issues/1685.
|
||||
"""
|
||||
mediabox = page.get_mediabox() # (left, bottom, right, top)
|
||||
cropbox = page.get_cropbox() # (left, bottom, right, top), defaults to mediabox
|
||||
|
||||
# Calculate how much to expand from cropbox to mediabox
|
||||
# Negative values = expand, positive = shrink
|
||||
return (
|
||||
mediabox[0] - cropbox[0], # Expand left
|
||||
mediabox[1] - cropbox[1], # Expand bottom
|
||||
cropbox[2] - mediabox[2], # Expand right
|
||||
cropbox[3] - mediabox[3], # Expand top
|
||||
)
|
||||
page.set_cropbox(*mediabox)
|
||||
|
||||
|
||||
def _render_page_to_bitmap(
|
||||
@@ -107,14 +98,13 @@ def _render_page_to_bitmap(
|
||||
# Render in grayscale for mono and gray devices (better input for 1-bit conversion)
|
||||
grayscale = raster_device.lower() in ('pngmono', 'pnggray', 'jpeggray')
|
||||
|
||||
# Calculate crop to render the appropriate box
|
||||
# Default (use_cropbox=False) renders MediaBox for consistency with Ghostscript
|
||||
crop = (0, 0, 0, 0) if use_cropbox else _calculate_mediabox_crop(page)
|
||||
if not use_cropbox:
|
||||
_expand_cropbox_to_mediabox(page)
|
||||
|
||||
bitmap = page.render(
|
||||
scale=scale,
|
||||
rotation=0, # We already set rotation on the page
|
||||
crop=crop,
|
||||
may_draw_forms=True,
|
||||
draw_annots=True,
|
||||
grayscale=grayscale,
|
||||
|
||||
+2
-1
@@ -387,7 +387,8 @@ Online documentation is located at:
|
||||
type=str,
|
||||
help=(
|
||||
"Limit OCR to the specified pages (ranges or comma separated), "
|
||||
"skipping others"
|
||||
"skipping others. The token 'end' is an alias for the last page, "
|
||||
"so e.g. '3-end' OCRs from page 3 to the last page."
|
||||
),
|
||||
)
|
||||
advanced.add_argument(
|
||||
|
||||
@@ -10,6 +10,7 @@ OCR text layers.
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import unicodedata
|
||||
from dataclasses import dataclass
|
||||
from math import atan, cos, degrees, radians, sin, sqrt
|
||||
from pathlib import Path
|
||||
@@ -24,6 +25,21 @@ from ocrmypdf.models.ocr_element import OcrClass, OcrElement
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _is_rtl_text(text: str) -> bool:
|
||||
"""Check if text is right-to-left based on Unicode bidi properties.
|
||||
|
||||
Looks for the first character with a strong directional type
|
||||
(R, AL, or L) to determine the text's base direction.
|
||||
"""
|
||||
for char in text:
|
||||
bidi = unicodedata.bidirectional(char)
|
||||
if bidi in ('R', 'AL'):
|
||||
return True
|
||||
if bidi == 'L':
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
def transform_point(matrix: Matrix, x: float, y: float) -> tuple[float, float]:
|
||||
"""Transform a point (x, y) by a matrix.
|
||||
|
||||
@@ -66,6 +82,17 @@ def transform_box(
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class WordRenderData:
|
||||
"""Rendering parameters for a single word on a line."""
|
||||
|
||||
text: str
|
||||
x_baseline: float
|
||||
font_family: str
|
||||
word_tz: float
|
||||
is_rtl: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
class DebugRenderOptions:
|
||||
"""Options for debug visualization during rendering.
|
||||
@@ -426,8 +453,7 @@ class Fpdf2PdfRenderer:
|
||||
):
|
||||
return
|
||||
|
||||
# Collect word rendering data: (text, x_baseline, font_family, word_tz)
|
||||
word_render_data: list[tuple[str, float, str, float]] = []
|
||||
word_render_data: list[WordRenderData] = []
|
||||
for word in words:
|
||||
if word is None or not word.text or word.bbox is None:
|
||||
continue
|
||||
@@ -459,13 +485,35 @@ class Fpdf2PdfRenderer:
|
||||
)
|
||||
font_family = self._register_font(pdf, font_manager)
|
||||
pdf.set_font(font_family, size=font_size)
|
||||
natural_width = pdf.get_string_width(word.text)
|
||||
|
||||
# For RTL words with invisible text, we use encode_text()
|
||||
# (which maps characters 1:1 in logical order) combined with
|
||||
# a -1 x-scale text matrix. This avoids an fpdf2 issue where
|
||||
# shaped RTL ligature glyphs (e.g. lam-alef) get multi-char
|
||||
# CMap entries whose character order is reversed by the bidi
|
||||
# algorithm during text extraction.
|
||||
# Since the text is invisible, glyph mirroring is harmless.
|
||||
# Compute Tz using unshaped widths to match encode_text().
|
||||
word_is_rtl = self.invisible_text and _is_rtl_text(word.text)
|
||||
if word_is_rtl:
|
||||
saved_shaping = pdf.text_shaping
|
||||
pdf.text_shaping = None
|
||||
natural_width = pdf.get_string_width(word.text)
|
||||
pdf.text_shaping = saved_shaping
|
||||
else:
|
||||
natural_width = pdf.get_string_width(word.text)
|
||||
if natural_width > 0 and word_width_pt > 0:
|
||||
word_tz = (word_width_pt / natural_width) * 100
|
||||
else:
|
||||
word_tz = 100.0
|
||||
|
||||
word_render_data.append((word.text, box_llx, font_family, word_tz))
|
||||
word_render_data.append(WordRenderData(
|
||||
text=word.text,
|
||||
x_baseline=box_llx,
|
||||
font_family=font_family,
|
||||
word_tz=word_tz,
|
||||
is_rtl=word_is_rtl,
|
||||
))
|
||||
|
||||
if not word_render_data:
|
||||
return
|
||||
@@ -564,7 +612,7 @@ class Fpdf2PdfRenderer:
|
||||
def _emit_line_bt_block(
|
||||
self,
|
||||
pdf: FPDF,
|
||||
word_render_data: list[tuple[str, float, str, float]],
|
||||
word_render_data: list[WordRenderData],
|
||||
baseline_matrix: Matrix,
|
||||
font_size: float,
|
||||
total_rotation_deg: float,
|
||||
@@ -580,8 +628,7 @@ class Fpdf2PdfRenderer:
|
||||
|
||||
Args:
|
||||
pdf: FPDF instance
|
||||
word_render_data: List of (text, x_baseline, font_family, word_tz)
|
||||
tuples, one per word on this line
|
||||
word_render_data: List of WordRenderData, one per word on this line
|
||||
baseline_matrix: Transform from baseline coords to page coords
|
||||
font_size: Font size in points
|
||||
total_rotation_deg: Total rotation angle (textangle + slope)
|
||||
@@ -626,7 +673,7 @@ class Fpdf2PdfRenderer:
|
||||
ops.append(f'{tr} Tr')
|
||||
|
||||
# Initial text position
|
||||
first_x_baseline = word_render_data[0][1]
|
||||
first_x_baseline = word_render_data[0].x_baseline
|
||||
if has_rotation:
|
||||
# In the cm-transformed space, origin is at the baseline start
|
||||
ops.append(f'{first_x_baseline:.2f} 0 Td')
|
||||
@@ -641,14 +688,12 @@ class Fpdf2PdfRenderer:
|
||||
prev_font_family: str | None = None
|
||||
prev_x_baseline = first_x_baseline
|
||||
|
||||
for i, (text, x_baseline, font_family, word_tz) in enumerate(
|
||||
word_render_data
|
||||
):
|
||||
for i, word in enumerate(word_render_data):
|
||||
is_last = i == len(word_render_data) - 1
|
||||
|
||||
# Set font if changed
|
||||
if font_family != prev_font_family:
|
||||
pdf.set_font(font_family, size=font_size)
|
||||
if word.font_family != prev_font_family:
|
||||
pdf.set_font(word.font_family, size=font_size)
|
||||
# Register font resource on this page
|
||||
pdf._resource_catalog.add(
|
||||
PDFResourceType.FONT, pdf.current_font.i, pdf.page
|
||||
@@ -656,13 +701,13 @@ class Fpdf2PdfRenderer:
|
||||
ops.append(
|
||||
f'/F{pdf.current_font.i} {pdf.font_size_pt:.2f} Tf'
|
||||
)
|
||||
prev_font_family = font_family
|
||||
prev_font_family = word.font_family
|
||||
|
||||
# Relative positioning (for words after the first)
|
||||
if i > 0:
|
||||
if has_rotation:
|
||||
# In rotated space, advance is purely along x-axis
|
||||
dx_baseline = x_baseline - prev_x_baseline
|
||||
dx_baseline = word.x_baseline - prev_x_baseline
|
||||
ops.append(f'{dx_baseline:.2f} 0 Td')
|
||||
else:
|
||||
# Non-rotated: compute delta in PDF coordinates
|
||||
@@ -670,7 +715,7 @@ class Fpdf2PdfRenderer:
|
||||
baseline_matrix, prev_x_baseline, 0
|
||||
)
|
||||
px_curr, py_curr_f = transform_point(
|
||||
baseline_matrix, x_baseline, 0
|
||||
baseline_matrix, word.x_baseline, 0
|
||||
)
|
||||
dx_pdf = px_curr - px_prev
|
||||
# Flip y delta for PDF coordinates (y-up)
|
||||
@@ -679,31 +724,31 @@ class Fpdf2PdfRenderer:
|
||||
|
||||
# Determine text to render
|
||||
if not is_last:
|
||||
next_text, next_x_baseline, _, _ = word_render_data[i + 1]
|
||||
advance = next_x_baseline - x_baseline
|
||||
next_word = word_render_data[i + 1]
|
||||
advance = next_word.x_baseline - word.x_baseline
|
||||
|
||||
# Add trailing space for text extraction unless both are CJK
|
||||
if (
|
||||
advance > 0
|
||||
and not (
|
||||
self._is_cjk_only(text)
|
||||
and self._is_cjk_only(next_text)
|
||||
self._is_cjk_only(word.text)
|
||||
and self._is_cjk_only(next_word.text)
|
||||
)
|
||||
):
|
||||
text_to_render = text + ' '
|
||||
text_to_render = word.text + ' '
|
||||
else:
|
||||
text_to_render = text
|
||||
text_to_render = word.text
|
||||
else:
|
||||
text_to_render = text
|
||||
text_to_render = word.text
|
||||
|
||||
# Use word_tz (fits word into its hOCR bbox) — Td handles
|
||||
# inter-word gaps, so Tz should not stretch to fill them.
|
||||
render_tz = word_tz
|
||||
ops.append(f'{word.word_tz:.2f} Tz')
|
||||
ops.append(
|
||||
self._encode_shaped_text(pdf, text_to_render, word.is_rtl)
|
||||
)
|
||||
|
||||
ops.append(f'{render_tz:.2f} Tz')
|
||||
ops.append(self._encode_shaped_text(pdf, text_to_render))
|
||||
|
||||
prev_x_baseline = x_baseline
|
||||
prev_x_baseline = word.x_baseline
|
||||
|
||||
# End text object
|
||||
ops.append('ET')
|
||||
@@ -717,15 +762,35 @@ class Fpdf2PdfRenderer:
|
||||
# don't think Tz is still set from our raw operators
|
||||
pdf.font_stretching = 100
|
||||
|
||||
def _encode_shaped_text(self, pdf: FPDF, text: str) -> str:
|
||||
def _encode_shaped_text(
|
||||
self, pdf: FPDF, text: str, is_rtl: bool = False
|
||||
) -> str:
|
||||
"""Encode text using HarfBuzz text shaping for complex script support.
|
||||
|
||||
Unlike font.encode_text() which maps unicode characters one-by-one to
|
||||
glyph IDs, this uses HarfBuzz to handle BiDi reordering, Arabic joining
|
||||
forms, Devanagari conjuncts, and other complex script shaping. Falls
|
||||
back to encode_text() when text shaping is not enabled.
|
||||
|
||||
For RTL words with invisible text, we use encode_text() instead of
|
||||
shape_text(). fpdf2's shape_text() produces RTL ligature glyphs
|
||||
(e.g. lam-alef) with multi-character CMap entries whose character
|
||||
order gets reversed by the bidi algorithm during text extraction,
|
||||
producing garbled output (e.g. "سالح" instead of "سلاح").
|
||||
encode_text() maps characters 1:1 in logical order, giving correct
|
||||
extraction. Since the text is invisible (Tr=3), the lack of proper
|
||||
joining forms and ligature shaping is harmless.
|
||||
"""
|
||||
font = pdf.current_font
|
||||
if is_rtl:
|
||||
# Reverse the text so that after bidi reversal by the text
|
||||
# extractor, the characters end up in correct logical order.
|
||||
# The text cursor advances left-to-right from the word's left
|
||||
# edge (set by Td), so characters are positioned left-to-right
|
||||
# in the PDF. The extractor sees RTL characters in L-to-R
|
||||
# positions and applies bidi reversal, which reverses them.
|
||||
# By pre-reversing, the double reversal yields the original.
|
||||
return font.encode_text(text[::-1])
|
||||
if pdf.text_shaping and pdf.text_shaping.get("use_shaping_engine"):
|
||||
shaped = font.shape_text(text, pdf.font_size_pt, pdf.text_shaping)
|
||||
if shaped:
|
||||
|
||||
@@ -260,10 +260,19 @@ def _find_image_xrefs_container(
|
||||
exclude_xrefs: MutableSet[Xref],
|
||||
pageno_for_xref: dict[Xref, int],
|
||||
depth: int = 0,
|
||||
visited_forms: MutableSet[Xref] | None = None,
|
||||
):
|
||||
"""Find all image XRefs or Form XObject and add to the include/exclude sets."""
|
||||
# Form XObjects are not added to include/exclude_xrefs, so the dedup
|
||||
# check below doesn't catch Form-XObject cycles or DAGs. Track them in
|
||||
# a shared set so each Form is only descended into once per document
|
||||
# (issue #1321).
|
||||
if visited_forms is None:
|
||||
visited_forms = set()
|
||||
if depth > 10:
|
||||
log.warning("Recursion depth exceeded in _find_image_xrefs_page")
|
||||
# With visited_forms memoization, this is a soft DAG-height guard
|
||||
# rather than a cycle defense, so a debug log is sufficient.
|
||||
log.debug("Recursion depth exceeded in _find_image_xrefs_page")
|
||||
return
|
||||
try:
|
||||
xobjs = container.Resources.XObject
|
||||
@@ -276,7 +285,9 @@ def _find_image_xrefs_container(
|
||||
if xref in include_xrefs or xref in exclude_xrefs:
|
||||
continue # Already processed
|
||||
if Name.Subtype in image and image.Subtype == Name.Form:
|
||||
# Recurse into Form XObjects
|
||||
if xref in visited_forms:
|
||||
continue
|
||||
visited_forms.add(xref)
|
||||
log.debug(f"Recursing into Form XObject {_imname} in page {pageno}")
|
||||
_find_image_xrefs_container(
|
||||
pdf,
|
||||
@@ -286,6 +297,7 @@ def _find_image_xrefs_container(
|
||||
exclude_xrefs,
|
||||
pageno_for_xref,
|
||||
depth + 1,
|
||||
visited_forms,
|
||||
)
|
||||
continue
|
||||
if Name.SMask in image:
|
||||
|
||||
@@ -1,345 +1,31 @@
|
||||
# SPDX-FileCopyrightText: 2022 James R. Barlow
|
||||
# SPDX-License-Identifier: MPL-2.0
|
||||
"""Wrappers to manage subprocess calls."""
|
||||
"""Wrappers to manage subprocess calls.
|
||||
|
||||
This package is split into three private submodules by concern:
|
||||
|
||||
- :mod:`ocrmypdf.subprocess._run` - low-level execution wrappers (``run``,
|
||||
``run_polling_stderr``) that add OCRmyPDF-aware logging and Windows PATH
|
||||
resolution. Useful as drop-in replacements for :func:`subprocess.run`.
|
||||
- :mod:`ocrmypdf.subprocess._version` - version probing (``get_version``).
|
||||
- :mod:`ocrmypdf.subprocess._check` - startup validation
|
||||
(``check_external_program``) with platform-aware error messages.
|
||||
|
||||
The names below are the stable public API. Importing from the private
|
||||
submodules directly is not supported for external code.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from contextlib import suppress
|
||||
from pathlib import Path
|
||||
from subprocess import PIPE, STDOUT, CalledProcessError, CompletedProcess, Popen
|
||||
from subprocess import run as subprocess_run
|
||||
from ocrmypdf.subprocess._check import check_external_program
|
||||
from ocrmypdf.subprocess._run import Args, Environ, run, run_polling_stderr
|
||||
from ocrmypdf.subprocess._version import get_version
|
||||
|
||||
from packaging.version import Version
|
||||
|
||||
from ocrmypdf.exceptions import MissingDependencyError
|
||||
|
||||
# pylint: disable=logging-format-interpolation
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
Args = Sequence[Path | str]
|
||||
Environ = Mapping[str, str] | os._Environ # pylint: disable=protected-access
|
||||
|
||||
|
||||
def run(
|
||||
args: Args,
|
||||
*,
|
||||
env: Environ | None = None,
|
||||
logs_errors_to_stdout: bool = False,
|
||||
check: bool = False,
|
||||
**kwargs,
|
||||
) -> CompletedProcess:
|
||||
"""Wrapper around :py:func:`subprocess.run`.
|
||||
|
||||
The main purpose of this wrapper is to log subprocess output in an orderly
|
||||
fashion that identifies the responsible subprocess. An additional
|
||||
task is that this function goes to greater lengths to find possible Windows
|
||||
locations of our dependencies when they are not on the system PATH.
|
||||
|
||||
Arguments should be identical to ``subprocess.run``, except for following:
|
||||
|
||||
Args:
|
||||
args: Positional arguments to pass to ``subprocess.run``.
|
||||
env: A set of environment variables. If None, the OS environment is used.
|
||||
logs_errors_to_stdout: If True, indicates that the process writes its error
|
||||
messages to stdout rather than stderr, so stdout should be logged
|
||||
if there is an error. If False, stderr is logged. Could be used with
|
||||
stderr=STDOUT, stdout=PIPE for example.
|
||||
check: If True, raise an exception if the process exits with a non-zero
|
||||
status code. If False, the return value will indicate success or failure.
|
||||
kwargs: Additional arguments to pass to ``subprocess.run``.
|
||||
"""
|
||||
args, env, process_log, _text = _fix_process_args(args, env, kwargs)
|
||||
|
||||
stderr = None
|
||||
stderr_name = 'stderr' if not logs_errors_to_stdout else 'stdout'
|
||||
try:
|
||||
proc = subprocess_run(args, env=env, check=check, **kwargs)
|
||||
except CalledProcessError as e:
|
||||
stderr = getattr(e, stderr_name, None)
|
||||
raise
|
||||
else:
|
||||
stderr = getattr(proc, stderr_name, None)
|
||||
finally:
|
||||
if process_log.isEnabledFor(logging.DEBUG) and stderr:
|
||||
with suppress(AttributeError, UnicodeDecodeError):
|
||||
stderr = stderr.decode('utf-8', 'replace')
|
||||
if logs_errors_to_stdout:
|
||||
process_log.debug("stdout/stderr = %s", stderr)
|
||||
else:
|
||||
process_log.debug("stderr = %s", stderr)
|
||||
return proc
|
||||
|
||||
|
||||
def run_polling_stderr(
|
||||
args: Args,
|
||||
*,
|
||||
callback: Callable[[str], None],
|
||||
check: bool = False,
|
||||
env: Environ | None = None,
|
||||
**kwargs,
|
||||
) -> CompletedProcess:
|
||||
"""Run a process like ``ocrmypdf.subprocess.run``, and poll stderr.
|
||||
|
||||
Every line of produced by stderr will be forwarded to the callback function.
|
||||
The intended use is monitoring progress of subprocesses that output their
|
||||
own progress indicators. In addition, each line will be logged if debug
|
||||
logging is enabled.
|
||||
|
||||
Requires stderr to be opened in text mode for ease of handling errors. In
|
||||
addition the expected encoding= and errors= arguments should be set. Note
|
||||
that if stdout is already set up, it need not be binary.
|
||||
"""
|
||||
args, env, process_log, text = _fix_process_args(args, env, kwargs)
|
||||
assert text, "Must use text=True"
|
||||
|
||||
with Popen(args, env=env, **kwargs) as proc:
|
||||
lines = []
|
||||
while proc.poll() is None:
|
||||
if proc.stderr is None:
|
||||
continue
|
||||
for msg in iter(proc.stderr.readline, ''):
|
||||
if process_log.isEnabledFor(logging.DEBUG):
|
||||
process_log.debug(msg.strip())
|
||||
callback(msg)
|
||||
lines.append(msg)
|
||||
stderr = ''.join(lines)
|
||||
|
||||
if check and proc.returncode != 0:
|
||||
raise CalledProcessError(proc.returncode, args, output=None, stderr=stderr)
|
||||
return CompletedProcess(args, proc.returncode, None, stderr=stderr)
|
||||
|
||||
|
||||
def _fix_process_args(
|
||||
args: Args, env: Environ | None, kwargs
|
||||
) -> tuple[Args, Environ, logging.Logger, bool]:
|
||||
if not env:
|
||||
env = os.environ
|
||||
|
||||
# Search in spoof path if necessary
|
||||
program = str(args[0])
|
||||
|
||||
if sys.platform == 'win32':
|
||||
# pylint: disable=import-outside-toplevel
|
||||
from ocrmypdf.subprocess._windows import fix_windows_args
|
||||
|
||||
args = fix_windows_args(program, args, env)
|
||||
|
||||
log.debug("Running: %s", args)
|
||||
process_log = log.getChild(os.path.basename(program))
|
||||
text = bool(kwargs.get('text', False))
|
||||
|
||||
return args, env, process_log, text
|
||||
|
||||
|
||||
def get_version(
|
||||
program: str,
|
||||
*,
|
||||
version_arg: str = '--version',
|
||||
regex=r'(\d+(\.\d+)*)',
|
||||
env: Environ | None = None,
|
||||
) -> str:
|
||||
"""Get the version of the specified program.
|
||||
|
||||
Arguments:
|
||||
program: The program to version check.
|
||||
version_arg: The argument needed to ask for its version, e.g. ``--version``.
|
||||
regex: A regular expression to parse the program's output and obtain the
|
||||
version.
|
||||
env: Custom ``os.environ`` in which to run program.
|
||||
"""
|
||||
args_prog = [program, version_arg]
|
||||
try:
|
||||
proc = run(
|
||||
args_prog,
|
||||
close_fds=True,
|
||||
text=True,
|
||||
stdout=PIPE,
|
||||
stderr=STDOUT,
|
||||
check=True,
|
||||
env=env,
|
||||
)
|
||||
output: str = proc.stdout
|
||||
except FileNotFoundError as e:
|
||||
raise MissingDependencyError(
|
||||
f"Could not find program '{program}' on the PATH"
|
||||
) from e
|
||||
except CalledProcessError as e:
|
||||
if e.returncode != 0:
|
||||
log.exception(e)
|
||||
raise MissingDependencyError(
|
||||
f"Ran program '{program}' but it exited with an error:\n{e.output}"
|
||||
) from e
|
||||
raise MissingDependencyError(
|
||||
f"Could not find program '{program}' on the PATH"
|
||||
) from e
|
||||
|
||||
match = re.match(regex, output.strip())
|
||||
if not match:
|
||||
raise MissingDependencyError(
|
||||
f"The program '{program}' did not report its version. "
|
||||
f"Message was:\n{output}"
|
||||
)
|
||||
version = match.group(1)
|
||||
|
||||
return version
|
||||
|
||||
|
||||
MISSING_PROGRAM = '''
|
||||
The program '{program}' could not be executed or was not found on your
|
||||
system PATH.
|
||||
'''
|
||||
|
||||
MISSING_OPTIONAL_PROGRAM = '''
|
||||
The program '{program}' could not be executed or was not found on your
|
||||
system PATH. This program is required when you use the
|
||||
{required_for} arguments. You could try omitting these arguments, or install
|
||||
the package.
|
||||
'''
|
||||
|
||||
MISSING_RECOMMEND_PROGRAM = '''
|
||||
The program '{program}' could not be executed or was not found on your
|
||||
system PATH. This program is recommended when using the {required_for} arguments,
|
||||
but not required, so we will proceed. For best results, install the program.
|
||||
'''
|
||||
|
||||
OLD_VERSION = '''
|
||||
OCRmyPDF requires '{program}' {need_version} or higher. Your system appears
|
||||
to have {found_version}. Please update this program.
|
||||
'''
|
||||
|
||||
OLD_VERSION_REQUIRED_FOR = '''
|
||||
OCRmyPDF requires '{program}' {need_version} or higher when run with the
|
||||
{required_for} arguments. {program} {found_version} is installed.
|
||||
|
||||
If you omit these arguments, OCRmyPDF may be able to
|
||||
proceed. For best results, update the program.
|
||||
'''
|
||||
|
||||
OSX_INSTALL_ADVICE = '''
|
||||
If you have homebrew installed, try these command to install the missing
|
||||
package:
|
||||
brew install {package}
|
||||
'''
|
||||
|
||||
LINUX_INSTALL_ADVICE = '''
|
||||
On systems with the aptitude package manager (Debian, Ubuntu), try these
|
||||
commands:
|
||||
sudo apt update
|
||||
sudo apt install {package}
|
||||
|
||||
On RPM-based systems (Red Hat, Fedora), try this command:
|
||||
sudo dnf install {package}
|
||||
'''
|
||||
|
||||
WINDOWS_INSTALL_ADVICE = '''
|
||||
If not already installed, install the Chocolatey package manager. Then use
|
||||
a command prompt to install the missing package:
|
||||
choco install {package}
|
||||
'''
|
||||
|
||||
|
||||
def _get_platform() -> str:
|
||||
if sys.platform.startswith('freebsd'):
|
||||
return 'freebsd'
|
||||
elif sys.platform.startswith('linux'):
|
||||
return 'linux'
|
||||
elif sys.platform.startswith('win'):
|
||||
return 'windows'
|
||||
return sys.platform
|
||||
|
||||
|
||||
def _error_trailer(program: str, package: str | Mapping[str, str], **kwargs) -> None:
|
||||
del kwargs
|
||||
if isinstance(package, Mapping):
|
||||
package = package.get(_get_platform(), program)
|
||||
|
||||
if _get_platform() == 'darwin':
|
||||
log.info(OSX_INSTALL_ADVICE.format(**locals()))
|
||||
elif _get_platform() == 'linux':
|
||||
log.info(LINUX_INSTALL_ADVICE.format(**locals()))
|
||||
elif _get_platform() == 'windows':
|
||||
log.info(WINDOWS_INSTALL_ADVICE.format(**locals()))
|
||||
|
||||
|
||||
def _error_missing_program(
|
||||
program: str, package: str, required_for: str | None, recommended: bool
|
||||
) -> None:
|
||||
# pylint: disable=unused-argument
|
||||
if recommended:
|
||||
log.warning(MISSING_RECOMMEND_PROGRAM.format(**locals()))
|
||||
elif required_for:
|
||||
log.error(MISSING_OPTIONAL_PROGRAM.format(**locals()))
|
||||
else:
|
||||
log.error(MISSING_PROGRAM.format(**locals()))
|
||||
_error_trailer(**locals())
|
||||
|
||||
|
||||
def _error_old_version(
|
||||
program: str,
|
||||
package: str,
|
||||
need_version: str,
|
||||
found_version: str,
|
||||
required_for: str | None,
|
||||
) -> None:
|
||||
# pylint: disable=unused-argument
|
||||
if required_for:
|
||||
log.error(OLD_VERSION_REQUIRED_FOR.format(**locals()))
|
||||
else:
|
||||
log.error(OLD_VERSION.format(**locals()))
|
||||
_error_trailer(**locals())
|
||||
|
||||
|
||||
def check_external_program(
|
||||
*,
|
||||
program: str,
|
||||
package: str,
|
||||
version_checker: Callable[[], Version],
|
||||
need_version: str | Version,
|
||||
required_for: str | None = None,
|
||||
recommended: bool = False,
|
||||
version_parser: type[Version] = Version,
|
||||
) -> None:
|
||||
"""Check for required version of external program and raise exception if not.
|
||||
|
||||
Args:
|
||||
program: The name of the program to test.
|
||||
package: The name of a software package that typically supplies this program.
|
||||
Usually the same as program.
|
||||
version_checker: A callable without arguments that retrieves the installed
|
||||
version of program.
|
||||
need_version: The minimum required version.
|
||||
required_for: The name of an argument of feature that requires this program.
|
||||
recommended: If this external program is recommended, instead of raising
|
||||
an exception, log a warning and allow execution to continue.
|
||||
version_parser: A class that should be used to parse and compare version
|
||||
numbers. Used when version numbers do not follow standard conventions.
|
||||
"""
|
||||
if not isinstance(need_version, Version):
|
||||
need_version = version_parser(need_version)
|
||||
try:
|
||||
found_version = version_checker()
|
||||
except (CalledProcessError, FileNotFoundError) as e:
|
||||
_error_missing_program(program, package, required_for, recommended)
|
||||
if not recommended:
|
||||
raise MissingDependencyError(program) from e
|
||||
return
|
||||
except MissingDependencyError:
|
||||
_error_missing_program(program, package, required_for, recommended)
|
||||
if not recommended:
|
||||
raise
|
||||
return
|
||||
|
||||
if found_version and found_version < need_version:
|
||||
_error_old_version(
|
||||
program, package, str(need_version), str(found_version), required_for
|
||||
)
|
||||
if not recommended:
|
||||
raise MissingDependencyError(program)
|
||||
|
||||
log.debug('Found %s %s', program, found_version)
|
||||
__all__ = [
|
||||
'Args',
|
||||
'Environ',
|
||||
'check_external_program',
|
||||
'get_version',
|
||||
'run',
|
||||
'run_polling_stderr',
|
||||
]
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
# SPDX-FileCopyrightText: 2022 James R. Barlow
|
||||
# SPDX-License-Identifier: MPL-2.0
|
||||
"""Validate that required external programs are installed and new enough."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sys
|
||||
from collections.abc import Callable, Mapping
|
||||
from subprocess import CalledProcessError
|
||||
|
||||
from packaging.version import Version
|
||||
|
||||
from ocrmypdf.exceptions import MissingDependencyError
|
||||
|
||||
log = logging.getLogger('ocrmypdf.subprocess')
|
||||
|
||||
|
||||
MISSING_PROGRAM = '''
|
||||
The program '{program}' could not be executed or was not found on your
|
||||
system PATH.
|
||||
'''
|
||||
|
||||
MISSING_OPTIONAL_PROGRAM = '''
|
||||
The program '{program}' could not be executed or was not found on your
|
||||
system PATH. This program is required when you use the
|
||||
{required_for} arguments. You could try omitting these arguments, or install
|
||||
the package.
|
||||
'''
|
||||
|
||||
MISSING_RECOMMEND_PROGRAM = '''
|
||||
The program '{program}' could not be executed or was not found on your
|
||||
system PATH. This program is recommended when using the {required_for} arguments,
|
||||
but not required, so we will proceed. For best results, install the program.
|
||||
'''
|
||||
|
||||
OLD_VERSION = '''
|
||||
OCRmyPDF requires '{program}' {need_version} or higher. Your system appears
|
||||
to have {found_version}. Please update this program.
|
||||
'''
|
||||
|
||||
OLD_VERSION_REQUIRED_FOR = '''
|
||||
OCRmyPDF requires '{program}' {need_version} or higher when run with the
|
||||
{required_for} arguments. {program} {found_version} is installed.
|
||||
|
||||
If you omit these arguments, OCRmyPDF may be able to
|
||||
proceed. For best results, update the program.
|
||||
'''
|
||||
|
||||
OSX_INSTALL_ADVICE = '''
|
||||
If you have homebrew installed, try these command to install the missing
|
||||
package:
|
||||
brew install {package}
|
||||
'''
|
||||
|
||||
LINUX_INSTALL_ADVICE = '''
|
||||
On systems with the aptitude package manager (Debian, Ubuntu), try these
|
||||
commands:
|
||||
sudo apt update
|
||||
sudo apt install {package}
|
||||
|
||||
On RPM-based systems (Red Hat, Fedora), try this command:
|
||||
sudo dnf install {package}
|
||||
'''
|
||||
|
||||
WINDOWS_INSTALL_ADVICE = '''
|
||||
If not already installed, install the Chocolatey package manager. Then use
|
||||
a command prompt to install the missing package:
|
||||
choco install {package}
|
||||
'''
|
||||
|
||||
|
||||
def _get_platform() -> str:
|
||||
if sys.platform.startswith('freebsd'):
|
||||
return 'freebsd'
|
||||
elif sys.platform.startswith('linux'):
|
||||
return 'linux'
|
||||
elif sys.platform.startswith('win'):
|
||||
return 'windows'
|
||||
return sys.platform
|
||||
|
||||
|
||||
def _error_trailer(program: str, package: str | Mapping[str, str], **kwargs) -> None:
|
||||
del kwargs
|
||||
if isinstance(package, Mapping):
|
||||
package = package.get(_get_platform(), program)
|
||||
|
||||
if _get_platform() == 'darwin':
|
||||
log.info(OSX_INSTALL_ADVICE.format(**locals()))
|
||||
elif _get_platform() == 'linux':
|
||||
log.info(LINUX_INSTALL_ADVICE.format(**locals()))
|
||||
elif _get_platform() == 'windows':
|
||||
log.info(WINDOWS_INSTALL_ADVICE.format(**locals()))
|
||||
|
||||
|
||||
def _error_missing_program(
|
||||
program: str, package: str, required_for: str | None, recommended: bool
|
||||
) -> None:
|
||||
# pylint: disable=unused-argument
|
||||
if recommended:
|
||||
log.warning(MISSING_RECOMMEND_PROGRAM.format(**locals()))
|
||||
elif required_for:
|
||||
log.error(MISSING_OPTIONAL_PROGRAM.format(**locals()))
|
||||
else:
|
||||
log.error(MISSING_PROGRAM.format(**locals()))
|
||||
_error_trailer(**locals())
|
||||
|
||||
|
||||
def _error_old_version(
|
||||
program: str,
|
||||
package: str,
|
||||
need_version: str,
|
||||
found_version: str,
|
||||
required_for: str | None,
|
||||
) -> None:
|
||||
# pylint: disable=unused-argument
|
||||
if required_for:
|
||||
log.error(OLD_VERSION_REQUIRED_FOR.format(**locals()))
|
||||
else:
|
||||
log.error(OLD_VERSION.format(**locals()))
|
||||
_error_trailer(**locals())
|
||||
|
||||
|
||||
def check_external_program(
|
||||
*,
|
||||
program: str,
|
||||
package: str,
|
||||
version_checker: Callable[[], Version],
|
||||
need_version: str | Version,
|
||||
required_for: str | None = None,
|
||||
recommended: bool = False,
|
||||
version_parser: type[Version] = Version,
|
||||
) -> None:
|
||||
"""Check for required version of external program and raise exception if not.
|
||||
|
||||
Args:
|
||||
program: The name of the program to test.
|
||||
package: The name of a software package that typically supplies this program.
|
||||
Usually the same as program.
|
||||
version_checker: A callable without arguments that retrieves the installed
|
||||
version of program.
|
||||
need_version: The minimum required version.
|
||||
required_for: The name of an argument of feature that requires this program.
|
||||
recommended: If this external program is recommended, instead of raising
|
||||
an exception, log a warning and allow execution to continue.
|
||||
version_parser: A class that should be used to parse and compare version
|
||||
numbers. Used when version numbers do not follow standard conventions.
|
||||
"""
|
||||
if not isinstance(need_version, Version):
|
||||
need_version = version_parser(need_version)
|
||||
try:
|
||||
found_version = version_checker()
|
||||
except (CalledProcessError, FileNotFoundError) as e:
|
||||
_error_missing_program(program, package, required_for, recommended)
|
||||
if not recommended:
|
||||
raise MissingDependencyError(program) from e
|
||||
return
|
||||
except MissingDependencyError:
|
||||
_error_missing_program(program, package, required_for, recommended)
|
||||
if not recommended:
|
||||
raise
|
||||
return
|
||||
|
||||
if found_version and found_version < need_version:
|
||||
_error_old_version(
|
||||
program, package, str(need_version), str(found_version), required_for
|
||||
)
|
||||
if not recommended:
|
||||
raise MissingDependencyError(program)
|
||||
|
||||
log.debug('Found %s %s', program, found_version)
|
||||
@@ -0,0 +1,137 @@
|
||||
# SPDX-FileCopyrightText: 2022 James R. Barlow
|
||||
# SPDX-License-Identifier: MPL-2.0
|
||||
"""Low-level wrappers around :py:mod:`subprocess`.
|
||||
|
||||
These functions exist to give OCRmyPDF child processes uniform logging
|
||||
behavior and to route through any platform-specific PATH fix-ups before
|
||||
invocation. They are intended as drop-in replacements for
|
||||
:py:func:`subprocess.run` in contexts where that routing is desirable
|
||||
(for example, plugin-provided tools).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from contextlib import suppress
|
||||
from pathlib import Path
|
||||
from subprocess import CalledProcessError, CompletedProcess, Popen
|
||||
from subprocess import run as subprocess_run
|
||||
|
||||
log = logging.getLogger('ocrmypdf.subprocess')
|
||||
|
||||
Args = Sequence[Path | str]
|
||||
Environ = Mapping[str, str] | os._Environ # pylint: disable=protected-access
|
||||
|
||||
|
||||
def run(
|
||||
args: Args,
|
||||
*,
|
||||
env: Environ | None = None,
|
||||
logs_errors_to_stdout: bool = False,
|
||||
check: bool = False,
|
||||
**kwargs,
|
||||
) -> CompletedProcess:
|
||||
"""Wrapper around :py:func:`subprocess.run`.
|
||||
|
||||
The main purpose of this wrapper is to log subprocess output in an orderly
|
||||
fashion that identifies the responsible subprocess. An additional
|
||||
task is that this function goes to greater lengths to find possible Windows
|
||||
locations of our dependencies when they are not on the system PATH.
|
||||
|
||||
Arguments should be identical to ``subprocess.run``, except for following:
|
||||
|
||||
Args:
|
||||
args: Positional arguments to pass to ``subprocess.run``.
|
||||
env: A set of environment variables. If None, the OS environment is used.
|
||||
logs_errors_to_stdout: If True, indicates that the process writes its error
|
||||
messages to stdout rather than stderr, so stdout should be logged
|
||||
if there is an error. If False, stderr is logged. Could be used with
|
||||
stderr=STDOUT, stdout=PIPE for example.
|
||||
check: If True, raise an exception if the process exits with a non-zero
|
||||
status code. If False, the return value will indicate success or failure.
|
||||
kwargs: Additional arguments to pass to ``subprocess.run``.
|
||||
"""
|
||||
args, env, process_log, _text = _fix_process_args(args, env, kwargs)
|
||||
|
||||
stderr = None
|
||||
stderr_name = 'stderr' if not logs_errors_to_stdout else 'stdout'
|
||||
try:
|
||||
proc = subprocess_run(args, env=env, check=check, **kwargs)
|
||||
except CalledProcessError as e:
|
||||
stderr = getattr(e, stderr_name, None)
|
||||
raise
|
||||
else:
|
||||
stderr = getattr(proc, stderr_name, None)
|
||||
finally:
|
||||
if process_log.isEnabledFor(logging.DEBUG) and stderr:
|
||||
with suppress(AttributeError, UnicodeDecodeError):
|
||||
stderr = stderr.decode('utf-8', 'replace')
|
||||
if logs_errors_to_stdout:
|
||||
process_log.debug("stdout/stderr = %s", stderr)
|
||||
else:
|
||||
process_log.debug("stderr = %s", stderr)
|
||||
return proc
|
||||
|
||||
|
||||
def run_polling_stderr(
|
||||
args: Args,
|
||||
*,
|
||||
callback: Callable[[str], None],
|
||||
check: bool = False,
|
||||
env: Environ | None = None,
|
||||
**kwargs,
|
||||
) -> CompletedProcess:
|
||||
"""Run a process like ``ocrmypdf.subprocess.run``, and poll stderr.
|
||||
|
||||
Every line of produced by stderr will be forwarded to the callback function.
|
||||
The intended use is monitoring progress of subprocesses that output their
|
||||
own progress indicators. In addition, each line will be logged if debug
|
||||
logging is enabled.
|
||||
|
||||
Requires stderr to be opened in text mode for ease of handling errors. In
|
||||
addition the expected encoding= and errors= arguments should be set. Note
|
||||
that if stdout is already set up, it need not be binary.
|
||||
"""
|
||||
args, env, process_log, text = _fix_process_args(args, env, kwargs)
|
||||
assert text, "Must use text=True"
|
||||
|
||||
with Popen(args, env=env, **kwargs) as proc:
|
||||
lines = []
|
||||
while proc.poll() is None:
|
||||
if proc.stderr is None:
|
||||
continue
|
||||
for msg in iter(proc.stderr.readline, ''):
|
||||
if process_log.isEnabledFor(logging.DEBUG):
|
||||
process_log.debug(msg.strip())
|
||||
callback(msg)
|
||||
lines.append(msg)
|
||||
stderr = ''.join(lines)
|
||||
|
||||
if check and proc.returncode != 0:
|
||||
raise CalledProcessError(proc.returncode, args, output=None, stderr=stderr)
|
||||
return CompletedProcess(args, proc.returncode, None, stderr=stderr)
|
||||
|
||||
|
||||
def _fix_process_args(
|
||||
args: Args, env: Environ | None, kwargs
|
||||
) -> tuple[Args, Environ, logging.Logger, bool]:
|
||||
if not env:
|
||||
env = os.environ
|
||||
|
||||
# Search in spoof path if necessary
|
||||
program = str(args[0])
|
||||
|
||||
if sys.platform == 'win32':
|
||||
# pylint: disable=import-outside-toplevel
|
||||
from ocrmypdf.subprocess._windows import fix_windows_args
|
||||
|
||||
args = fix_windows_args(program, args, env)
|
||||
|
||||
log.debug("Running: %s", args)
|
||||
process_log = log.getChild(os.path.basename(program))
|
||||
text = bool(kwargs.get('text', False))
|
||||
|
||||
return args, env, process_log, text
|
||||
@@ -0,0 +1,73 @@
|
||||
# SPDX-FileCopyrightText: 2022 James R. Barlow
|
||||
# SPDX-License-Identifier: MPL-2.0
|
||||
"""Extract version strings from external programs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from subprocess import PIPE, STDOUT, CalledProcessError
|
||||
|
||||
from ocrmypdf.exceptions import MissingDependencyError
|
||||
from ocrmypdf.subprocess._run import Environ
|
||||
|
||||
log = logging.getLogger('ocrmypdf.subprocess')
|
||||
|
||||
|
||||
def get_version(
|
||||
program: str,
|
||||
*,
|
||||
version_arg: str = '--version',
|
||||
regex=r'(\d+(\.\d+)*)',
|
||||
env: Environ | None = None,
|
||||
) -> str:
|
||||
"""Get the version of the specified program.
|
||||
|
||||
Arguments:
|
||||
program: The program to version check.
|
||||
version_arg: The argument needed to ask for its version, e.g. ``--version``.
|
||||
regex: A regular expression to parse the program's output and obtain the
|
||||
version.
|
||||
env: Custom ``os.environ`` in which to run program.
|
||||
"""
|
||||
# Late import of the public ``run`` so that tests patching
|
||||
# ``ocrmypdf.subprocess.run`` affect this function. Binding ``run`` at
|
||||
# module load time would capture the real implementation and bypass the
|
||||
# patch.
|
||||
from ocrmypdf import subprocess as _sp
|
||||
|
||||
args_prog = [program, version_arg]
|
||||
try:
|
||||
proc = _sp.run(
|
||||
args_prog,
|
||||
close_fds=True,
|
||||
text=True,
|
||||
stdout=PIPE,
|
||||
stderr=STDOUT,
|
||||
check=True,
|
||||
env=env,
|
||||
)
|
||||
output: str = proc.stdout
|
||||
except FileNotFoundError as e:
|
||||
raise MissingDependencyError(
|
||||
f"Could not find program '{program}' on the PATH"
|
||||
) from e
|
||||
except CalledProcessError as e:
|
||||
if e.returncode != 0:
|
||||
log.exception(e)
|
||||
raise MissingDependencyError(
|
||||
f"Ran program '{program}' but it exited with an error:\n{e.output}"
|
||||
) from e
|
||||
raise MissingDependencyError(
|
||||
f"Could not find program '{program}' on the PATH"
|
||||
) from e
|
||||
|
||||
match = re.match(regex, output.strip())
|
||||
if not match:
|
||||
raise MissingDependencyError(
|
||||
f"The program '{program}' did not report its version. "
|
||||
f"Message was:\n{output}"
|
||||
)
|
||||
version = match.group(1)
|
||||
|
||||
return version
|
||||
+3
-6
@@ -19,14 +19,11 @@ from ocrmypdf.exceptions import ExitCode
|
||||
|
||||
|
||||
class Gs106WarningFilter(logging.Filter):
|
||||
"""Filter out expected Ghostscript 10.6.x warning from test logs."""
|
||||
"""Filter out expected Ghostscript 10.6+ JPEG encoding warning from test logs."""
|
||||
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
# Allow all records except the expected Ghostscript 10.6.x warning
|
||||
return (
|
||||
"Ghostscript 10.6.x contains JPEG encoding errors"
|
||||
not in record.getMessage()
|
||||
)
|
||||
# Allow all records except the expected Ghostscript 10.6+ warning
|
||||
return "contains JPEG encoding errors" not in record.getMessage()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
|
||||
@@ -5,6 +5,8 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
@@ -37,6 +39,26 @@ def resources():
|
||||
return Path(__file__).parent / "resources"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def pdftotext():
|
||||
"""Return a function to extract text from PDF using pdftotext.
|
||||
|
||||
Skips the test if pdftotext is not available.
|
||||
"""
|
||||
pdftotext_path = shutil.which('pdftotext')
|
||||
if pdftotext_path is None:
|
||||
pytest.skip("pdftotext not available")
|
||||
|
||||
def extract_text(pdf_path: Path) -> str:
|
||||
return subprocess.check_output(
|
||||
['pdftotext', '-enc', 'UTF-8', str(pdf_path), '-'],
|
||||
text=True,
|
||||
encoding='utf-8',
|
||||
)
|
||||
|
||||
return extract_text
|
||||
|
||||
|
||||
class TestFpdf2RendererImports:
|
||||
"""Test that all fpdf2 renderer modules can be imported."""
|
||||
|
||||
@@ -443,14 +465,16 @@ class TestWordSegmentation:
|
||||
assert "World" in words_found
|
||||
assert "Test" in words_found
|
||||
|
||||
def test_cjk_no_spurious_spaces(self, multi_font_manager, tmp_path):
|
||||
def test_cjk_no_spurious_spaces(self, multi_font_manager, tmp_path, pdftotext):
|
||||
"""Test that CJK text does not get spurious spaces inserted.
|
||||
|
||||
CJK scripts don't use spaces between characters/words, so we should
|
||||
not insert spaces between adjacent CJK words.
|
||||
"""
|
||||
from pdfminer.high_level import extract_text
|
||||
|
||||
Uses pdftotext (poppler) instead of pdfminer.six because the latter
|
||||
cannot decode the custom Encoding CMap that fpdf2 >= 2.8.7 emits for
|
||||
subsetted CFF-based CID fonts (e.g. NotoSansCJK).
|
||||
"""
|
||||
from ocrmypdf.models.ocr_element import BoundingBox, OcrElement
|
||||
|
||||
# Create a page with CJK words (Chinese characters)
|
||||
@@ -487,15 +511,15 @@ class TestWordSegmentation:
|
||||
output_path = tmp_path / "test_cjk_segmentation.pdf"
|
||||
renderer.render(output_path)
|
||||
|
||||
# Extract text using pdfminer.six
|
||||
extracted_text = extract_text(str(output_path))
|
||||
extracted_text = pdftotext(output_path)
|
||||
|
||||
# CJK text should be present
|
||||
assert "你好" in extracted_text
|
||||
assert "世界" in extracted_text
|
||||
|
||||
# There should NOT be spaces between CJK characters
|
||||
# (but pdfminer may add some whitespace, so we check the raw chars)
|
||||
# (a space between the two words is acceptable, since they are
|
||||
# separated horizontally on the rendered page)
|
||||
extracted_chars = extracted_text.replace(" ", "").replace("\n", "")
|
||||
assert "你好世界" in extracted_chars or (
|
||||
"你好" in extracted_chars and "世界" in extracted_chars
|
||||
|
||||
+129
-4
@@ -137,6 +137,129 @@ def test_rasterize_low_dpi_one_axis(francais, outdir):
|
||||
assert im.info['dpi'] == forced_dpi
|
||||
|
||||
|
||||
def test_generate_pdfa_default_jpeg_quality(outdir):
|
||||
"""When jpeg_quality is None, Ghostscript receives -dJPEGQ=95 (default)."""
|
||||
with (
|
||||
patch('ocrmypdf._exec.ghostscript.version', return_value=Version('10.05.1')),
|
||||
patch('ocrmypdf._exec.ghostscript.run_polling_stderr') as run_mock,
|
||||
):
|
||||
run_mock.return_value = subprocess.CompletedProcess(
|
||||
['gs'], returncode=0, stdout='', stderr=''
|
||||
)
|
||||
ghostscript.generate_pdfa(
|
||||
pdf_pages=[outdir / 'input.pdf'],
|
||||
output_file=outdir / 'out.pdf',
|
||||
compression='auto',
|
||||
color_conversion_strategy='LeaveColorUnchanged',
|
||||
)
|
||||
|
||||
args = run_mock.call_args.args[0]
|
||||
assert '-dJPEGQ=95' in args
|
||||
# No downsample switches when jpeg_maxdpi is not set
|
||||
assert not any(a.startswith('-dDownsampleColorImages') for a in args)
|
||||
assert not any(a.startswith('-dColorImageResolution') for a in args)
|
||||
|
||||
|
||||
def test_generate_pdfa_uses_user_jpeg_quality(outdir):
|
||||
with (
|
||||
patch('ocrmypdf._exec.ghostscript.version', return_value=Version('10.05.1')),
|
||||
patch('ocrmypdf._exec.ghostscript.run_polling_stderr') as run_mock,
|
||||
):
|
||||
run_mock.return_value = subprocess.CompletedProcess(
|
||||
['gs'], returncode=0, stdout='', stderr=''
|
||||
)
|
||||
ghostscript.generate_pdfa(
|
||||
pdf_pages=[outdir / 'input.pdf'],
|
||||
output_file=outdir / 'out.pdf',
|
||||
compression='jpeg',
|
||||
color_conversion_strategy='RGB',
|
||||
jpeg_quality=72,
|
||||
)
|
||||
|
||||
args = run_mock.call_args.args[0]
|
||||
assert '-dJPEGQ=72' in args
|
||||
assert '-dJPEGQ=95' not in args
|
||||
|
||||
|
||||
def test_generate_pdfa_jpeg_quality_zero_is_max_compression(outdir):
|
||||
"""Explicit jpeg_quality=0 must reach Ghostscript as -dJPEGQ=0.
|
||||
|
||||
Ghostscript accepts 0 as a valid quality value (maximum compression);
|
||||
it must not be silently replaced by the default 95.
|
||||
"""
|
||||
with (
|
||||
patch('ocrmypdf._exec.ghostscript.version', return_value=Version('10.05.1')),
|
||||
patch('ocrmypdf._exec.ghostscript.run_polling_stderr') as run_mock,
|
||||
):
|
||||
run_mock.return_value = subprocess.CompletedProcess(
|
||||
['gs'], returncode=0, stdout='', stderr=''
|
||||
)
|
||||
ghostscript.generate_pdfa(
|
||||
pdf_pages=[outdir / 'input.pdf'],
|
||||
output_file=outdir / 'out.pdf',
|
||||
compression='jpeg',
|
||||
color_conversion_strategy='RGB',
|
||||
jpeg_quality=0,
|
||||
)
|
||||
|
||||
args = run_mock.call_args.args[0]
|
||||
assert '-dJPEGQ=0' in args
|
||||
assert '-dJPEGQ=95' not in args
|
||||
|
||||
|
||||
def test_generate_pdfa_honors_jpeg_maxdpi(outdir):
|
||||
with (
|
||||
patch('ocrmypdf._exec.ghostscript.version', return_value=Version('10.05.1')),
|
||||
patch('ocrmypdf._exec.ghostscript.run_polling_stderr') as run_mock,
|
||||
):
|
||||
run_mock.return_value = subprocess.CompletedProcess(
|
||||
['gs'], returncode=0, stdout='', stderr=''
|
||||
)
|
||||
ghostscript.generate_pdfa(
|
||||
pdf_pages=[outdir / 'input.pdf'],
|
||||
output_file=outdir / 'out.pdf',
|
||||
compression='auto',
|
||||
color_conversion_strategy='LeaveColorUnchanged',
|
||||
jpeg_maxdpi=300,
|
||||
)
|
||||
|
||||
args = run_mock.call_args.args[0]
|
||||
assert '-dJPEGQ=95' in args
|
||||
assert '-dDownsampleColorImages=true' in args
|
||||
assert '-dColorImageDownsampleThreshold=1.0' in args
|
||||
assert '-dDownsampleGrayImages=true' in args
|
||||
assert '-dGrayImageDownsampleThreshold=1.0' in args
|
||||
assert '-dDownsampleMonoImages=true' in args
|
||||
assert '-dMonoImageDownsampleThreshold=1.0' in args
|
||||
assert '-dColorImageResolution=300' in args
|
||||
assert '-dGrayImageResolution=300' in args
|
||||
assert '-dMonoImageResolution=300' in args
|
||||
|
||||
|
||||
def test_ghostscript_jpeg_options_via_cli(resources, outpdf):
|
||||
"""End-to-end: CLI flags reach the ghostscript plugin namespace."""
|
||||
with patch(
|
||||
'ocrmypdf._exec.ghostscript.generate_pdfa',
|
||||
wraps=ghostscript.generate_pdfa,
|
||||
) as gen_mock:
|
||||
run_ocrmypdf_api(
|
||||
resources / 'francais.pdf',
|
||||
outpdf,
|
||||
'--output-type',
|
||||
'pdfa',
|
||||
'--ghostscript-jpeg-quality',
|
||||
'60',
|
||||
'--ghostscript-jpeg-maxdpi',
|
||||
'150',
|
||||
'--plugin',
|
||||
'tests/plugins/tesseract_noop.py',
|
||||
)
|
||||
assert gen_mock.called
|
||||
call_kwargs = gen_mock.call_args.kwargs
|
||||
assert call_kwargs['jpeg_quality'] == 60
|
||||
assert call_kwargs['jpeg_maxdpi'] == 150
|
||||
|
||||
|
||||
def test_gs_render_failure(resources, outpdf, caplog):
|
||||
exitcode = run_ocrmypdf_api(
|
||||
resources / 'blank.pdf',
|
||||
@@ -176,9 +299,9 @@ def test_ghostscript_pdfa_failure(resources, outpdf, caplog):
|
||||
'--plugin',
|
||||
'tests/plugins/gs_pdfa_failure.py',
|
||||
)
|
||||
assert (
|
||||
exitcode == ExitCode.pdfa_conversion_failed
|
||||
), "Unexpected return when PDF/A fails"
|
||||
assert exitcode == ExitCode.pdfa_conversion_failed, (
|
||||
"Unexpected return when PDF/A fails"
|
||||
)
|
||||
|
||||
|
||||
def test_ghostscript_feature_elision(resources, outpdf):
|
||||
@@ -439,7 +562,9 @@ class TestGs106JpegCorruptionRepair:
|
||||
repaired_bytes_list.append(obj.read_raw_bytes())
|
||||
|
||||
assert len(repaired_bytes_list) == len(original_bytes_list)
|
||||
for orig, repaired_bytes in zip(original_bytes_list, repaired_bytes_list, strict=False):
|
||||
for orig, repaired_bytes in zip(
|
||||
original_bytes_list, repaired_bytes_list, strict=False
|
||||
):
|
||||
assert orig == repaired_bytes, "Repaired bytes should match original"
|
||||
|
||||
# Check that error/warning was logged
|
||||
|
||||
@@ -215,6 +215,46 @@ def test_find_formx(resources):
|
||||
assert pagenos[xref] == 0
|
||||
|
||||
|
||||
def test_find_formx_circular_reference(resources, tmp_path, caplog):
|
||||
"""Regression for issue #1321.
|
||||
|
||||
Some PDFs (notably PowerPoint exports) contain Form XObjects that
|
||||
reference themselves or each other in a cycle. The recursion guard in
|
||||
_find_image_xrefs_container only deduplicates *image* xrefs, so a Form
|
||||
XObject cycle would re-enter every branch until the depth limit fired,
|
||||
producing thousands of "Recursion depth exceeded" warnings (and minutes
|
||||
of wall-clock time on real-world inputs).
|
||||
"""
|
||||
import logging
|
||||
|
||||
src = resources / 'formxobject.pdf'
|
||||
out = tmp_path / 'circular_form.pdf'
|
||||
with pikepdf.open(src) as pdf:
|
||||
# /Form1 lives at xref 10. Replace its Resources.XObject with three
|
||||
# entries that all point back to /Form1 itself, creating a fan-out
|
||||
# cycle of branching factor 3.
|
||||
form = pdf.pages[0].obj.Resources.XObject.Form1
|
||||
form.Resources.XObject = Dictionary(
|
||||
{'/Fm0': form, '/Fm1': form, '/Fm2': form}
|
||||
)
|
||||
pdf.save(out)
|
||||
|
||||
caplog.set_level(logging.WARNING, logger='ocrmypdf.optimize')
|
||||
with pikepdf.open(out) as pdf:
|
||||
opt._find_image_xrefs(pdf)
|
||||
|
||||
n_warnings = sum(
|
||||
1
|
||||
for r in caplog.records
|
||||
if 'Recursion depth exceeded' in r.getMessage()
|
||||
)
|
||||
# Without the fix this is in the tens of thousands.
|
||||
assert n_warnings == 0, (
|
||||
f"Form XObject cycle should be detected without depth-limit warnings; "
|
||||
f"got {n_warnings}"
|
||||
)
|
||||
|
||||
|
||||
def test_extract_image_filter_with_pdf_image():
|
||||
image = Dictionary()
|
||||
image.Subtype = Name.Image
|
||||
|
||||
@@ -42,6 +42,35 @@ def test_pages(pages, result):
|
||||
assert _pages_from_ranges(pages) == result
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'pages, total_pages, result',
|
||||
[
|
||||
['end', 10, {9}],
|
||||
['END', 10, {9}],
|
||||
['1-end', 3, {0, 1, 2}],
|
||||
['3-end', 5, {2, 3, 4}],
|
||||
['end-end', 7, {6}],
|
||||
['1,end', 4, {0, 3}],
|
||||
['2-4,end', 10, {1, 2, 3, 9}],
|
||||
['end,end,end', 5, {4}],
|
||||
['end-1', 5, BadArgsError], # empty range when end > 1
|
||||
],
|
||||
)
|
||||
def test_pages_end_alias(pages, total_pages, result):
|
||||
if isinstance(result, type):
|
||||
with pytest.raises(result):
|
||||
_pages_from_ranges(pages, total_pages=total_pages)
|
||||
else:
|
||||
assert _pages_from_ranges(pages, total_pages=total_pages) == result
|
||||
|
||||
|
||||
def test_end_alias_requires_total_pages():
|
||||
with pytest.raises(BadArgsError, match="total page count"):
|
||||
_pages_from_ranges('1-end')
|
||||
with pytest.raises(BadArgsError, match="total page count"):
|
||||
_pages_from_ranges('end')
|
||||
|
||||
|
||||
def test_nonmonotonic_warning(caplog):
|
||||
pages = _pages_from_ranges('1, 3, 2')
|
||||
assert pages == {0, 1, 2}
|
||||
@@ -61,3 +90,33 @@ def test_limited_pages(multipage, outpdf):
|
||||
assert not pi.pages[0].has_text
|
||||
assert pi.pages[4].has_text
|
||||
assert pi.pages[5].has_text
|
||||
|
||||
|
||||
def test_limited_pages_end_alias(multipage, outpdf):
|
||||
# multipage has 6 pages; 5-end == pages 5..6
|
||||
ocrmypdf.ocr(
|
||||
multipage,
|
||||
outpdf,
|
||||
pages='5-end',
|
||||
optimize=0,
|
||||
output_type='pdf',
|
||||
plugins=['tests/plugins/tesseract_cache.py'],
|
||||
)
|
||||
pi = PdfInfo(outpdf)
|
||||
assert not pi.pages[0].has_text
|
||||
assert pi.pages[4].has_text
|
||||
assert pi.pages[5].has_text
|
||||
|
||||
|
||||
def test_pages_end_alone(multipage, outpdf):
|
||||
ocrmypdf.ocr(
|
||||
multipage,
|
||||
outpdf,
|
||||
pages='end',
|
||||
optimize=0,
|
||||
output_type='pdf',
|
||||
plugins=['tests/plugins/tesseract_cache.py'],
|
||||
)
|
||||
pi = PdfInfo(outpdf)
|
||||
assert not pi.pages[0].has_text
|
||||
assert pi.pages[5].has_text
|
||||
|
||||
@@ -5,9 +5,11 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from io import StringIO
|
||||
from pathlib import Path
|
||||
|
||||
import pikepdf
|
||||
import pytest
|
||||
from pdfminer.converter import TextConverter
|
||||
from pdfminer.layout import LAParams
|
||||
@@ -597,3 +599,312 @@ class TestFpdf2PdfRendererLineTypes:
|
||||
check_pdf(str(output_pdf))
|
||||
extracted_text = text_from_pdf(output_pdf)
|
||||
assert "Caption" in extracted_text
|
||||
|
||||
|
||||
def create_rtl_page(
|
||||
words: list[tuple[str, tuple[float, float, float, float]]],
|
||||
language: str = "ara",
|
||||
width: float = 1000,
|
||||
height: float = 500,
|
||||
) -> OcrElement:
|
||||
"""Create an OcrElement page with a single RTL paragraph/line.
|
||||
|
||||
Args:
|
||||
words: List of (text, (left, top, right, bottom)) tuples.
|
||||
language: Language code for the paragraph.
|
||||
width: Page width in pixels.
|
||||
height: Page height in pixels.
|
||||
|
||||
Returns:
|
||||
OcrElement page.
|
||||
"""
|
||||
word_elements = [
|
||||
OcrElement(
|
||||
ocr_class=OcrClass.WORD,
|
||||
text=text,
|
||||
bbox=BoundingBox(
|
||||
left=bbox[0], top=bbox[1], right=bbox[2], bottom=bbox[3]
|
||||
),
|
||||
)
|
||||
for text, bbox in words
|
||||
]
|
||||
line = OcrElement(
|
||||
ocr_class=OcrClass.LINE,
|
||||
bbox=BoundingBox(left=50, top=100, right=950, bottom=200),
|
||||
baseline=Baseline(slope=0.0, intercept=0),
|
||||
direction="rtl",
|
||||
children=word_elements,
|
||||
)
|
||||
paragraph = OcrElement(
|
||||
ocr_class=OcrClass.PARAGRAPH,
|
||||
bbox=BoundingBox(left=50, top=100, right=950, bottom=200),
|
||||
direction="rtl",
|
||||
language=language,
|
||||
children=[line],
|
||||
)
|
||||
return OcrElement(
|
||||
ocr_class=OcrClass.PAGE,
|
||||
bbox=BoundingBox(left=0, top=0, right=width, bottom=height),
|
||||
children=[paragraph],
|
||||
)
|
||||
|
||||
|
||||
def _tounicode_map(pdf_path: Path) -> dict[int, str]:
|
||||
"""Extract all ToUnicode CMap entries from the first page's OCR overlay.
|
||||
|
||||
Returns a dict mapping subset glyph index -> unicode string.
|
||||
"""
|
||||
pdf = pikepdf.open(pdf_path)
|
||||
page = pdf.pages[0]
|
||||
resources = page.get('/Resources', {})
|
||||
|
||||
# Collect fonts from the page and from any Form XObjects (OCR overlay)
|
||||
fonts: dict[str, pikepdf.Object] = {}
|
||||
if '/Font' in resources:
|
||||
for name, obj in resources['/Font'].items():
|
||||
fonts[str(name)] = obj
|
||||
for xobj in resources.get('/XObject', {}).values():
|
||||
if xobj.get('/Subtype') == '/Form':
|
||||
for name, obj in xobj.get('/Resources', {}).get('/Font', {}).items():
|
||||
fonts[str(name)] = obj
|
||||
|
||||
result: dict[int, str] = {}
|
||||
for fobj in fonts.values():
|
||||
tounicode = fobj.get('/ToUnicode')
|
||||
if tounicode is None:
|
||||
continue
|
||||
cmap = bytes(tounicode.read_bytes()).decode('latin-1', errors='replace')
|
||||
for m in re.finditer(r'<([0-9A-Fa-f]+)>\s*<([0-9A-Fa-f]+)>', cmap):
|
||||
src_int = int(m.group(1), 16)
|
||||
dst_hex = m.group(2)
|
||||
chars = ''.join(
|
||||
chr(int(dst_hex[i : i + 4], 16))
|
||||
for i in range(0, len(dst_hex), 4)
|
||||
if int(dst_hex[i : i + 4], 16) > 0
|
||||
)
|
||||
if src_int > 0 and chars:
|
||||
result[src_int] = chars
|
||||
return result
|
||||
|
||||
|
||||
def _decode_tounicode_stream(
|
||||
pdf_path: Path,
|
||||
) -> tuple[dict[int, str], list[int]]:
|
||||
"""Extract ToUnicode CMap and Tj glyph stream from a test PDF.
|
||||
|
||||
Searches the page content stream and any Form XObjects for fonts
|
||||
and Tj operations.
|
||||
|
||||
Returns:
|
||||
(cmap, glyph_ids) where *cmap* maps subset index -> Unicode string
|
||||
and *glyph_ids* is the flat list of 2-byte glyph indices found in
|
||||
the first Tj string.
|
||||
"""
|
||||
pdf = pikepdf.open(pdf_path)
|
||||
page = pdf.pages[0]
|
||||
resources = page.get('/Resources', {})
|
||||
|
||||
# Collect fonts from page and from Form XObjects
|
||||
cmap: dict[int, str] = {}
|
||||
for font_dict in [resources.get('/Font', {})]:
|
||||
for fobj in font_dict.values():
|
||||
tounicode = fobj.get('/ToUnicode')
|
||||
if tounicode is None:
|
||||
continue
|
||||
raw = bytes(tounicode.read_bytes()).decode('latin-1', errors='replace')
|
||||
for m in re.finditer(r'<([0-9A-Fa-f]+)>\s*<([0-9A-Fa-f]+)>', raw):
|
||||
src = int(m.group(1), 16)
|
||||
dst_hex = m.group(2)
|
||||
chars = ''.join(
|
||||
chr(int(dst_hex[i : i + 4], 16))
|
||||
for i in range(0, len(dst_hex), 4)
|
||||
if int(dst_hex[i : i + 4], 16) > 0
|
||||
)
|
||||
if src > 0 and chars:
|
||||
cmap[src] = chars
|
||||
for xobj in resources.get('/XObject', {}).values():
|
||||
if xobj.get('/Subtype') != '/Form':
|
||||
continue
|
||||
for fobj in xobj.get('/Resources', {}).get('/Font', {}).values():
|
||||
tounicode = fobj.get('/ToUnicode')
|
||||
if tounicode is None:
|
||||
continue
|
||||
raw = bytes(tounicode.read_bytes()).decode('latin-1', errors='replace')
|
||||
for m in re.finditer(r'<([0-9A-Fa-f]+)>\s*<([0-9A-Fa-f]+)>', raw):
|
||||
src = int(m.group(1), 16)
|
||||
dst_hex = m.group(2)
|
||||
chars = ''.join(
|
||||
chr(int(dst_hex[i : i + 4], 16))
|
||||
for i in range(0, len(dst_hex), 4)
|
||||
if int(dst_hex[i : i + 4], 16) > 0
|
||||
)
|
||||
if src > 0 and chars:
|
||||
cmap[src] = chars
|
||||
|
||||
# Find first Tj glyph IDs from page content or XObject streams
|
||||
glyph_ids: list[int] = []
|
||||
streams: list[bytes] = []
|
||||
contents = page.get('/Contents')
|
||||
if contents:
|
||||
streams.append(bytes(contents.read_bytes()))
|
||||
for xobj in resources.get('/XObject', {}).values():
|
||||
if xobj.get('/Subtype') == '/Form':
|
||||
streams.append(bytes(xobj.read_bytes()))
|
||||
for data in streams:
|
||||
if glyph_ids:
|
||||
break
|
||||
tj = re.search(rb'\(([^\)]+)\)\s*Tj', data)
|
||||
if tj:
|
||||
raw_bytes = tj.group(1)
|
||||
for j in range(0, len(raw_bytes) - 1, 2):
|
||||
glyph_ids.append((raw_bytes[j] << 8) | raw_bytes[j + 1])
|
||||
return cmap, glyph_ids
|
||||
|
||||
|
||||
class TestRtlTextExtraction:
|
||||
"""Verify that RTL text is extracted in correct logical order.
|
||||
|
||||
The fpdf2 renderer must produce PDF text layers where text extractors
|
||||
(pdftotext, pdfminer) return characters in correct logical (reading)
|
||||
order for Arabic, Hebrew, and Farsi scripts.
|
||||
|
||||
These tests exercise invisible_text=True (the production path) to
|
||||
catch issues like the lam-alef ligature CMap ordering bug (issue #1655).
|
||||
"""
|
||||
|
||||
def test_arabic_lam_alef_extraction_order(self, tmp_path, multi_font_manager):
|
||||
"""Arabic words with lam-alef ligature extract in correct order.
|
||||
|
||||
The lam-alef (لا) ligature was the primary trigger for issue #1655:
|
||||
fpdf2's shape_text() produced a multi-char CMap entry whose
|
||||
character order was reversed by the bidi algorithm during
|
||||
extraction, giving "سالم" instead of "سلام".
|
||||
"""
|
||||
# سلام contains lam-alef: sin(س) lam(ل) alef(ا) meem(م)
|
||||
page = create_rtl_page(
|
||||
[("سلام", (600, 100, 900, 200))],
|
||||
language="fas",
|
||||
)
|
||||
output_pdf = tmp_path / "rtl_lam_alef.pdf"
|
||||
renderer = Fpdf2PdfRenderer(
|
||||
page=page,
|
||||
dpi=72.0,
|
||||
multi_font_manager=multi_font_manager,
|
||||
invisible_text=True,
|
||||
)
|
||||
renderer.render(output_pdf)
|
||||
|
||||
cmap, glyph_ids = _decode_tounicode_stream(output_pdf)
|
||||
# Decode the glyph stream via the CMap
|
||||
decoded = ''.join(cmap.get(g, '') for g in glyph_ids)
|
||||
# The stream is pre-reversed for RTL, so reversing it back
|
||||
# must yield the original logical text
|
||||
logical = decoded[::-1]
|
||||
assert logical == 'سلام', (
|
||||
f"Expected logical text 'سلام', got {logical!r} "
|
||||
f"(stream: {decoded!r}, glyph_ids: {glyph_ids})"
|
||||
)
|
||||
|
||||
def test_arabic_multiple_words_extraction(self, tmp_path, multi_font_manager):
|
||||
"""Multiple Arabic words produce correct Unicode mappings."""
|
||||
page = create_rtl_page(
|
||||
[
|
||||
("مرحبا", (600, 100, 900, 200)),
|
||||
("بالعالم", (100, 100, 500, 200)),
|
||||
],
|
||||
language="ara",
|
||||
)
|
||||
output_pdf = tmp_path / "rtl_arabic_words.pdf"
|
||||
renderer = Fpdf2PdfRenderer(
|
||||
page=page,
|
||||
dpi=72.0,
|
||||
multi_font_manager=multi_font_manager,
|
||||
invisible_text=True,
|
||||
)
|
||||
renderer.render(output_pdf)
|
||||
|
||||
cmap, _ = _decode_tounicode_stream(output_pdf)
|
||||
# Every CMap value should contain valid Arabic characters
|
||||
arabic_chars = {c for chars in cmap.values() for c in chars}
|
||||
expected = set('مرحبابالعالم')
|
||||
assert expected.issubset(arabic_chars | {' '}), (
|
||||
f"CMap missing Arabic characters; got {arabic_chars}"
|
||||
)
|
||||
|
||||
def test_hebrew_extraction_order(self, tmp_path, multi_font_manager):
|
||||
"""Hebrew text produces correct stream order for extraction."""
|
||||
page = create_rtl_page(
|
||||
[("שלום", (600, 100, 900, 200))],
|
||||
language="heb",
|
||||
)
|
||||
output_pdf = tmp_path / "rtl_hebrew.pdf"
|
||||
renderer = Fpdf2PdfRenderer(
|
||||
page=page,
|
||||
dpi=72.0,
|
||||
multi_font_manager=multi_font_manager,
|
||||
invisible_text=True,
|
||||
)
|
||||
renderer.render(output_pdf)
|
||||
|
||||
cmap, glyph_ids = _decode_tounicode_stream(output_pdf)
|
||||
decoded = ''.join(cmap.get(g, '') for g in glyph_ids)
|
||||
logical = decoded[::-1]
|
||||
assert logical == 'שלום', (
|
||||
f"Expected logical text 'שלום', got {logical!r} "
|
||||
f"(stream: {decoded!r})"
|
||||
)
|
||||
|
||||
def test_rtl_tounicode_one_to_one(self, tmp_path, multi_font_manager):
|
||||
"""RTL invisible text produces 1:1 glyph-to-Unicode CMap entries.
|
||||
|
||||
When using encode_text() for RTL words, each glyph maps to exactly
|
||||
one Unicode character. Multi-char ligature CMap entries (produced by
|
||||
shape_text()) are the root cause of the extraction order bug, so
|
||||
their absence confirms the fix.
|
||||
"""
|
||||
page = create_rtl_page(
|
||||
[("سلام", (600, 100, 900, 200))],
|
||||
language="ara",
|
||||
)
|
||||
output_pdf = tmp_path / "rtl_tounicode.pdf"
|
||||
renderer = Fpdf2PdfRenderer(
|
||||
page=page,
|
||||
dpi=72.0,
|
||||
multi_font_manager=multi_font_manager,
|
||||
invisible_text=True,
|
||||
)
|
||||
renderer.render(output_pdf)
|
||||
|
||||
cmap, _ = _decode_tounicode_stream(output_pdf)
|
||||
# Every CMap entry should map to exactly one Unicode character
|
||||
for glyph_id, chars in cmap.items():
|
||||
assert len(chars) == 1, (
|
||||
f"Glyph {glyph_id} maps to {len(chars)} chars {chars!r}; "
|
||||
f"expected 1:1 mapping for RTL invisible text"
|
||||
)
|
||||
|
||||
def test_visible_rtl_still_uses_shaping(self, tmp_path, multi_font_manager):
|
||||
"""Visible RTL text (debug mode) still uses text shaping.
|
||||
|
||||
The encode_text() bypass is only for invisible text. When
|
||||
invisible_text=False, shaping must remain active for correct
|
||||
glyph rendering (joining forms, ligatures).
|
||||
"""
|
||||
page = create_rtl_page(
|
||||
[("سلام", (600, 100, 900, 200))],
|
||||
language="ara",
|
||||
)
|
||||
output_pdf = tmp_path / "rtl_visible.pdf"
|
||||
renderer = Fpdf2PdfRenderer(
|
||||
page=page,
|
||||
dpi=72.0,
|
||||
multi_font_manager=multi_font_manager,
|
||||
invisible_text=False,
|
||||
)
|
||||
renderer.render(output_pdf)
|
||||
|
||||
# Shaped text may have multi-char CMap entries (ligatures);
|
||||
# just verify the PDF is valid and non-empty
|
||||
check_pdf(str(output_pdf))
|
||||
text = text_from_pdf(output_pdf)
|
||||
assert len(text.strip()) > 0, "Visible RTL should produce extractable text"
|
||||
|
||||
@@ -128,7 +128,6 @@ def test_timeout(caplog):
|
||||
(b'Error in boxClipToRectangle', ''),
|
||||
(b'an unexpected error', 'an unexpected error'),
|
||||
(b'a dire warning', 'a dire warning'),
|
||||
(b'read_params_file something', 'read_params_file'),
|
||||
(b'an innocent message', 'innocent'),
|
||||
(b'\x7f\x7f\x80innocent unicode failure', 'innocent'),
|
||||
],
|
||||
@@ -148,6 +147,13 @@ def test_tesseract_log_output_raises(caplog):
|
||||
assert 'not found' in caplog.text
|
||||
|
||||
|
||||
def test_tesseract_log_output_raises_on_missing_config(caplog):
|
||||
with pytest.raises(tesseract.TesseractConfigError) as excinfo:
|
||||
tesseract.tesseract_log_output(b"read_params_file: Can't open hocr")
|
||||
assert 'hocr' in excinfo.value.args[0]
|
||||
assert 'read_params_file' in caplog.text
|
||||
|
||||
|
||||
def test_blocked_language(resources, no_outpdf):
|
||||
infile = resources / 'masks.pdf'
|
||||
for bad_lang in ['osd', 'equ']:
|
||||
|
||||
@@ -102,6 +102,45 @@ def test_pillow_options():
|
||||
with pytest.raises(ValueError, match="max_image_mpixels must be non-negative"):
|
||||
make_ocr_opts(max_image_mpixels=-1)
|
||||
|
||||
# Default is None, meaning "do not override host-set PIL.Image.MAX_IMAGE_PIXELS"
|
||||
opts = make_ocr_opts()
|
||||
assert opts.max_image_mpixels is None
|
||||
|
||||
|
||||
def test_pillow_max_image_pixels_not_overridden_when_unset():
|
||||
"""Issue #1665: respect host-set PIL.Image.MAX_IMAGE_PIXELS.
|
||||
|
||||
API callers (e.g. Paperless-NGX) that set PIL.Image.MAX_IMAGE_PIXELS
|
||||
before invoking ocrmypdf should not have their setting clobbered when
|
||||
max_image_mpixels is not explicitly passed.
|
||||
"""
|
||||
import PIL.Image
|
||||
|
||||
from ocrmypdf._pipelines._common import setup_pipeline
|
||||
|
||||
parser = get_parser()
|
||||
pm = setup_plugin_infrastructure(plugins=[])
|
||||
pm.add_options(parser=parser)
|
||||
|
||||
saved = PIL.Image.MAX_IMAGE_PIXELS
|
||||
try:
|
||||
PIL.Image.MAX_IMAGE_PIXELS = None # host disables the limit
|
||||
opts = make_ocr_opts()
|
||||
assert opts.max_image_mpixels is None
|
||||
setup_pipeline(opts, pm)
|
||||
assert PIL.Image.MAX_IMAGE_PIXELS is None
|
||||
|
||||
PIL.Image.MAX_IMAGE_PIXELS = 1_000_000_000 # host sets a high limit
|
||||
setup_pipeline(opts, pm)
|
||||
assert PIL.Image.MAX_IMAGE_PIXELS == 1_000_000_000
|
||||
|
||||
# When explicitly passed, it still takes effect.
|
||||
opts = make_ocr_opts(max_image_mpixels=100)
|
||||
setup_pipeline(opts, pm)
|
||||
assert PIL.Image.MAX_IMAGE_PIXELS == 100_000_000
|
||||
finally:
|
||||
PIL.Image.MAX_IMAGE_PIXELS = saved
|
||||
|
||||
|
||||
def test_output_tty():
|
||||
with patch('sys.stdout.isatty', return_value=True), pytest.raises(BadArgsError):
|
||||
|
||||
Reference in New Issue
Block a user