menekis/OCRmyPDF
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into feature/drop-3.7

Browse Source
This commit is contained in:
James R. Barlow
2022-09-15 23:00:27 -07:00
parent 5156fe7662 2c99c89e45
commit 6dbaebdc0c
21 changed files with 200 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+7 -11
View File
@@ -2,16 +2,13 @@
# SPDX-License-Identifier: CC-BY-SA-4.0
# dotfiles
.*
!.coveragerc
!.dockerignore
!.git_archival.txt
!.gitattributes
!.gitignore
!.pre-commit-config.yaml
!.readthedocs.yaml
!.github/
!.docker/
.coverage
.venv*/
.tox/
.vscode/
.ipynb_checkpoints/
.mypy_cache/
.pytest_cache/
# Dev scratch
*.ipynb
@@ -45,4 +42,3 @@ docs/_build/
docs/_static/
docs/_templates/
docs/Makefile
ocrmypdf/lib/_*.py
.reuse/dep5
+131
View File
@@ -0,0 +1,131 @@
Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
Upstream-Name: OCRmyPDF
Upstream-Contact: James R. Barlow <james@purplerock.ca>
Source: https://github.com/ocrmypdf/OCRmyPDF
Files:
.git_archival.txt
docs/images/logo-social.png
docs/images/logo-square-256.svg
docs/images/logo-square.png
docs/images/logo-square.svg
docs/images/logo.svg
setup.cfg
Copyright: (C) 2022 James R. Barlow
License: MPL-2.0
Files:
docs/images/macos-workflow.png
Copyright: (C) 2022 James R. Barlow
License: CC-BY-SA-4.0
Files:
tests/resources/acroform.pdf
tests/resources/aspect.pdf
tests/resources/blank.pdf
tests/resources/cmyk.pdf
tests/resources/crom.png
tests/resources/enormous.pdf
tests/resources/formxobject.pdf
tests/resources/francais.pdf
tests/resources/hugemono.pdf
tests/resources/invalid.pdf
tests/resources/kcs.pdf
tests/resources/livecycle.pdf
tests/resources/missing_docinfo.pdf
tests/resources/negzero.pdf
tests/resources/no_contents.pdf
tests/resources/toc.pdf
tests/resources/trivial.pdf
tests/resources/truetype_font_nomapping.pdf
tests/resources/type3_font_nomapping.pdf
Copyright: (C) 2022 James R. Barlow
License: CC-BY-SA-4.0
Files:
tests/resources/graph.pdf
tests/resources/graph_ocred.pdf
Copyright: (C) 2012 SmokeyJoe
License: GFDL-1.2-or-later or CC-BY-SA-3.0
Files: tests/resources/c02-22.pdf
tests/resources/congress.jpg
tests/resources/multipage.pdf
Copyright: Public domain
License: public-domain
Copyright on these files has expired.
Files: docs/images/bitmap_vs_svg.svg
Copyright: (C) 2006 Yug
License: CC-BY-SA-2.5
Files: tests/cache/*
Copyright: (C) 2022 James R. Barlow
License: CC-BY-SA-4.0
Files: tests/resources/linn.png
tests/resources/linn.pdf
tests/resources/linn.txt
tests/resources/ccitt.pdf
tests/resources/cardinal.pdf
tests/resources/jbig2.pdf
tests/resources/skew.pdf
tests/resources/rotated_skew.pdf
tests/resources/poster.pdf
Copyright: (C) 1985 Forat Electronics
License: GFDL-1.2-or-later or CC-BY-SA-3.0
Files: tests/resources/lichtenstein.pdf
Copyright: (C) 2001 Andreas Tille
(C) 2007 Alessio Damato
License: GFDL-1.2-or-later or CC-BY-SA-3.0
Files: tests/resources/masks.pdf
Copyright: held by the contributors to the German Wikipedia article "Linux"
see: https://de.wikipedia.org/w/index.php?title=Linux&action=history
(masks.pdf generated from Wikipedia article as of 2016-08-24)
License: CC-BY-SA-3.0
Files: tests/resources/epson.pdf
Copyright: held by the contributors to the Wikipedia article "Optical character recognition"
see: https://en.wikipedia.org/w/index.php?title=Optical_character_recognition&action=history
(epson.pdf generated from Wikipedia article as of 2016-09-14)
License: CC-BY-SA-3.0
Files: tests/resources/typewriter.png tests/resources/2400dpi.pdf
Copyright: (C) 2005 Ellywa
License: GFDL-1.2-or-later or CC-BY-SA-1.0 or CC-BY-SA-2.0 or CC-BY-SA-2.5 or CC-BY-SA-3.0
Comment:
Obtained from: https://commons.wikimedia.org/wiki/File:Triumph.typewriter_text_Linzensoep.gif
Files: tests/resources/overlay.pdf
Copyright: (C) 2017 Max Anderson
License: MIT
Files:
tests/resources/baiona*.png
tests/resources/baiona*.jpg
tests/resources/link.pdf
tests/resources/palette.pdf
Copyright: (C) 2014 Euskaldunaa
License: CC-BY-SA-4.0
Files: tests/resources/vector.pdf
Copyright: (C) 2018 Catscratch
License: MIT
Files: src/ocrmypdf/data/sRGB.icc
Copyright: Kai-Uwe Behrmann <www.behrmann.name>
Marti Maria <www.littlecms.com>
Photogamut <www.photogamut.org>
Graeme Gill <www.argyllcms.com>
ColorSolutions <www.basICColor.com>
License: Zlib
Files: tests/resources/3small.pdf
Copyright: (C) 2014 Euskaldunaa
(C) 2017 James R. Barlow
(C) 2005 Ellywa
License: CC-BY-SA-4.0 and (GFDL-1.2-or-later or CC-BY-SA-1.0 or CC-BY-SA-2.0 or CC-BY-SA-2.5 or CC-BY-SA-3.0)
Comment: concatenation of baiona_gray.png, crom.png and typewriter.png/2400dpi.pdf
docs/jbig2.rst
+2 -1
View File
@@ -37,7 +37,8 @@ For all other Linux, you must build a JBIG2 encoder from source:
.. _jbig2-lossy:
Dependencies include libtoolize and libleptonica, which on Ubuntu systems
are packaged as libtool and libleptonica-dev.
are packaged as libtool and libleptonica-dev. On Fedora (35) they are packaged
as libtool and leptonica-devel.
Lossy mode JBIG2
================
src/ocrmypdf/_exec/ghostscript.py
+2 -1
View File
@@ -188,7 +188,8 @@ def generate_pdfa(
strategy = 'LeaveColorUnchanged'
gs_version = version()
if gs_version == '9.56.0':
# 9.56.0 introduced a new rendering mode that breaks our OCR
# 9.56.0 breaks our OCR, should be fixed in 9.56.1
# https://bugs.ghostscript.com/show_bug.cgi?id=705187
compression_args.append('-dNEWPDF=false')
# nb no need to specify ProcessColorModel when ColorConversionStrategy
src/ocrmypdf/_pipeline.py
+2 -1
View File
@@ -1,4 +1,5 @@
# SPDX-FileCopyrightText: 2022 James R. Barlow
# SPDX-FileCopyrightText: 2018-2022 James R. Barlow
# SPDX-FileCopyrightText: 2019 Martin Wind
# SPDX-License-Identifier: MPL-2.0
"""OCRmyPDF page processing pipeline functions."""
src/ocrmypdf/_sync.py
+2 -1
View File
@@ -1,4 +1,5 @@
# SPDX-FileCopyrightText: 2022 James R. Barlow
# SPDX-FileCopyrightText: 2019-2022 James R. Barlow
# SPDX-FileCopyrightText: 2019 Martin Wind
# SPDX-License-Identifier: MPL-2.0
"""Implements the concurrent and page synchronous parts of the pipeline."""
tests/conftest.py
+6 -1
View File
@@ -44,7 +44,7 @@ TESTS_ROOT = Path(__file__).parent.resolve()
PROJECT_ROOT = TESTS_ROOT
@pytest.fixture
@pytest.fixture(scope="session")
def resources() -> Path:
return Path(TESTS_ROOT) / 'resources'
@@ -78,6 +78,11 @@ def no_outpdf(tmp_path) -> Path:
return tmp_path / 'no_output.pdf'
@pytest.fixture(scope="session")
def multipage(resources):
return resources / 'multipage.pdf'
def check_ocrmypdf(input_file: Path, output_file: Path, *args) -> Path:
"""Run ocrmypdf and confirm that a valid plausible PDF was created."""
api_args = [str(input_file), str(output_file)] + [
tests/plugins/tesseract_cache.py
+9 -8
View File
@@ -140,18 +140,19 @@ def cached_run(options, run_args, **run_kwargs):
tessfile = args.outputbase + '.' + configfile
shutil.copy(tessfile, str(cache_folder / configfile) + '.bin')
manifest = {}
manifest['tesseract_version'] = TesseractOcrEngine.version().replace('\n', ' ')
manifest['system'] = platform.system()
manifest['python'] = platform.python_version()
manifest['argv_slug'] = cache_folder.name
manifest['sourcefile'] = str(Path(source_file).relative_to(TESTS_ROOT))
def clean_sys_argv():
for arg in run_args[1:]:
yield re.sub(r'.*/ocrmypdf[.]io[.][^/]+[/](.*)', r'$TMPDIR/\1', arg)
manifest['args'] = list(clean_sys_argv())
manifest = {
'tesseract_version': TesseractOcrEngine.version().replace('\n', ' '),
'system': platform.system(),
'python': platform.python_version(),
'argv_slug': cache_folder.name,
'sourcefile': str(Path(source_file).relative_to(TESTS_ROOT)),
'args': list(clean_sys_argv()),
}
with (Path(CACHE_ROOT) / 'manifest.jsonl').open('a') as f:
json.dump(manifest, f)
f.write('\n')
tests/resources/README.rst
+4 -6
View File
@@ -23,7 +23,7 @@ the copyright holder(s) and license(s) applicable to these resources.
- `US Congressional Records`_
- difficult OCR image (color background)
* - graph.pdf
- `Wikimedia: Pandas text analysis.png`_
- `Wikimedia:Simple_line_graph_of_ACE_2012_results_by_candidate_sj01.png`_
- image with slanted text
* - lichtenstein.pdf
- `Wikimedia: JPEG2000 Lichtenstein`_
@@ -111,14 +111,12 @@ These test resources are assemblies or derivatives from other previously mention
- baiona_alpha.png (from baiona.png, RGB+A version)
- cardinal.pdf (four cardinal directions, baked-in rotated copies of linn.png)
- ccitt.pdf (linn.png, converted to CCITT encoding)
- encrypted_algo4.pdf (congress.jpg, encrypted with algorithm 4 - not supported by PyPDF2)
- graph_ocred.pdf (from graph.pdf)
- jbig2.pdf (congress.jpg, converted to JBIG2 encoding)
- jbig2.pdf (from linn.png)
- multipage.pdf (from several other files)
- palette.pdf (congress.jpg, converted to a 256-color palette)
- palette.pdf (from baiona_colormapped.png)
- poster.pdf (from linn.png)
- rotated_skew.pdf (a /Rotate'd and skewed document from linn.png)
- skew-encrypted.pdf (skew.pdf with encryption - access supported by PyPDF2, password is "password")
- skew.pdf (from linn.png, skew simulated by adjusting the transformation matrix)
- toc.pdf (from formxobject.pdf, trivial.pdf)
@@ -129,7 +127,7 @@ These test resources are assemblies or derivatives from other previously mention
.. _`US Congressional Records`: http://www.baxleystamps.com/litho/meiji/courts_1871.jpg
.. _`Wikimedia: Pandas text analysis.png`: https://en.wikipedia.org/wiki/File:Pandas_text_analysis.png
.. _`Wikimedia: Simple_line_graph_of_ACE_2012_results_by_candidate_sj01.png`: https://en.wikipedia.org/wiki/File:Simple_line_graph_of_ACE_2012_results_by_candidate_sj01.png
.. _`Wikimedia: JPEG2000 Lichtenstein`: https://en.wikipedia.org/wiki/JPEG_2000#/media/File:Jpeg2000_2-level_wavelet_transform-lichtenstein.png
tests/resources/encrypted_algo4.pdf
BIN
View File
Binary file not shown.
tests/resources/graph.pdf
BIN
View File
Binary file not shown.
tests/resources/graph_ocred.pdf
BIN
View File
Binary file not shown.
tests/resources/jbig2.pdf
BIN
View File
Binary file not shown.
tests/resources/multipage.pdf
Regular → Executable
BIN
View File
Binary file not shown.
tests/resources/palette.pdf
BIN
View File
Binary file not shown.
tests/resources/skew-encrypted.pdf
BIN
View File
Binary file not shown.
tests/test_concurrency.py
+2 -2
View File
@@ -14,9 +14,9 @@ from .conftest import run_ocrmypdf_api
@pytest.mark.skipif(True, reason="--use-threads is currently default")
@pytest.mark.skipif(os.name == 'nt', reason="Windows doesn't have SIGKILL")
def test_simulate_oom_killer(resources, no_outpdf):
def test_simulate_oom_killer(multipage, no_outpdf):
exitcode = run_ocrmypdf_api(
resources / 'multipage.pdf',
multipage,
no_outpdf,
'--force-ocr',
'--plugin',
tests/test_main.py
+20 -14
View File
@@ -7,11 +7,10 @@ import os
import shutil
from math import isclose
from pathlib import Path
from subprocess import PIPE, run
from subprocess import run
from unittest.mock import patch
import pikepdf
import PIL
import pytest
from PIL import Image
@@ -153,9 +152,9 @@ def test_skip_big(resources, outpdf):
@pytest.mark.parametrize('renderer', RENDERERS)
@pytest.mark.parametrize('output_type', ['pdf', 'pdfa'])
def test_maximum_options(renderer, output_type, resources, outpdf):
def test_maximum_options(renderer, output_type, multipage, outpdf):
check_ocrmypdf(
resources / 'multipage.pdf',
multipage,
outpdf,
'-d',
'-ci' if have_unpaper() else None,
@@ -292,12 +291,6 @@ def test_input_file_not_a_pdf(caplog, no_outpdf):
assert input_file in caplog.text
def test_encrypted(resources, caplog, no_outpdf):
result = run_ocrmypdf_api(resources / 'skew-encrypted.pdf', no_outpdf)
assert result == ExitCode.encrypted_pdf
assert 'encryption must be removed' in caplog.text
@pytest.mark.parametrize('renderer', RENDERERS)
def test_pagesegmode(renderer, resources, outpdf):
check_ocrmypdf(
@@ -398,14 +391,27 @@ def test_tesseract_image_too_big(renderer, resources, outpdf):
)
def test_algo4(resources, outpdf):
p = run_ocrmypdf(
resources / 'encrypted_algo4.pdf',
@pytest.mark.parametrize('encryption_level', [2, 3, 4, 6])
def test_encrypted(resources, outpdf, encryption_level, caplog):
encryption = pikepdf.models.encryption.Encryption(
owner='ocrmypdf',
user='ocrmypdf',
R=encryption_level,
aes=(encryption_level >= 4),
metadata=(encryption_level == 6),
)
with pikepdf.open(resources / 'jbig2.pdf') as pdf:
pdf.save(outpdf, encryption=encryption)
exitcode = run_ocrmypdf_api(
outpdf,
outpdf,
'--plugin',
'tests/plugins/tesseract_noop.py',
)
assert p.returncode == ExitCode.encrypted_pdf
assert exitcode == ExitCode.encrypted_pdf
assert 'encryption must be removed' in caplog.text
def test_jbig2_passthrough(resources, outpdf):
tests/test_optimize.py
+8 -3
View File
@@ -27,10 +27,15 @@ needs_jbig2enc = pytest.mark.skipif(
)
@pytest.fixture(scope="session")
def palette(resources):
return resources / 'palette.pdf'
@needs_pngquant
@pytest.mark.parametrize('pdf', ['multipage.pdf', 'palette.pdf'])
def test_basic(resources, pdf, outpdf):
infile = resources / pdf
@pytest.mark.parametrize('pdf', ['multipage', 'palette'])
def test_basic(multipage, palette, pdf, outpdf):
infile = multipage if pdf == 'multipage' else palette
opt.main(infile, outpdf, level=3)
assert 0.98 * Path(outpdf).stat().st_size <= Path(infile).stat().st_size
tests/test_page_numbers.py
+2 -3
View File
@@ -48,10 +48,9 @@ def test_nonmonotonic_warning(caplog):
assert 'out of order' in caplog.text
def test_limited_pages(resources, outpdf):
multi = resources / 'multipage.pdf'
def test_limited_pages(multipage, outpdf):
ocrmypdf.ocr(
multi,
multipage,
outpdf,
pages='5-6',
optimize=0,
tests/test_rotation.py
+3 -2
View File
@@ -276,7 +276,8 @@ def test_rasterize_rotates(resources, tmp_path):
rotation=90,
filter_vector=False,
)
assert Image.open(img).size == (123, 151), "Image not rotated"
with Image.open(img) as im:
assert im.size == (83, 200), "Image not rotated"
img = tmp_path / 'img180.png'
pm.hook.rasterize_pdf_page(
@@ -289,7 +290,7 @@ def test_rasterize_rotates(resources, tmp_path):
rotation=180,
filter_vector=False,
)
assert Image.open(img).size == (151, 123), "Image not rotated"
assert Image.open(img).size == (200, 83), "Image not rotated"
def test_simulated_scan(outdir):