diff --git a/.gitignore b/.gitignore index 8d4afa31..19426d81 100644 --- a/.gitignore +++ b/.gitignore @@ -2,16 +2,13 @@ # SPDX-License-Identifier: CC-BY-SA-4.0 # dotfiles -.* -!.coveragerc -!.dockerignore -!.git_archival.txt -!.gitattributes -!.gitignore -!.pre-commit-config.yaml -!.readthedocs.yaml -!.github/ -!.docker/ +.coverage +.venv*/ +.tox/ +.vscode/ +.ipynb_checkpoints/ +.mypy_cache/ +.pytest_cache/ # Dev scratch *.ipynb @@ -45,4 +42,3 @@ docs/_build/ docs/_static/ docs/_templates/ docs/Makefile -ocrmypdf/lib/_*.py diff --git a/.reuse/dep5 b/.reuse/dep5 new file mode 100644 index 00000000..f025677c --- /dev/null +++ b/.reuse/dep5 @@ -0,0 +1,131 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: OCRmyPDF +Upstream-Contact: James R. Barlow +Source: https://github.com/ocrmypdf/OCRmyPDF + + +Files: + .git_archival.txt + docs/images/logo-social.png + docs/images/logo-square-256.svg + docs/images/logo-square.png + docs/images/logo-square.svg + docs/images/logo.svg + setup.cfg +Copyright: (C) 2022 James R. Barlow +License: MPL-2.0 + +Files: + docs/images/macos-workflow.png +Copyright: (C) 2022 James R. Barlow +License: CC-BY-SA-4.0 + +Files: + tests/resources/acroform.pdf + tests/resources/aspect.pdf + tests/resources/blank.pdf + tests/resources/cmyk.pdf + tests/resources/crom.png + tests/resources/enormous.pdf + tests/resources/formxobject.pdf + tests/resources/francais.pdf + tests/resources/hugemono.pdf + tests/resources/invalid.pdf + tests/resources/kcs.pdf + tests/resources/livecycle.pdf + tests/resources/missing_docinfo.pdf + tests/resources/negzero.pdf + tests/resources/no_contents.pdf + tests/resources/toc.pdf + tests/resources/trivial.pdf + tests/resources/truetype_font_nomapping.pdf + tests/resources/type3_font_nomapping.pdf +Copyright: (C) 2022 James R. Barlow +License: CC-BY-SA-4.0 + +Files: + tests/resources/graph.pdf + tests/resources/graph_ocred.pdf +Copyright: (C) 2012 SmokeyJoe +License: GFDL-1.2-or-later or CC-BY-SA-3.0 + +Files: tests/resources/c02-22.pdf + tests/resources/congress.jpg + tests/resources/multipage.pdf +Copyright: Public domain +License: public-domain + Copyright on these files has expired. + +Files: docs/images/bitmap_vs_svg.svg +Copyright: (C) 2006 Yug +License: CC-BY-SA-2.5 + +Files: tests/cache/* +Copyright: (C) 2022 James R. Barlow +License: CC-BY-SA-4.0 + +Files: tests/resources/linn.png + tests/resources/linn.pdf + tests/resources/linn.txt + tests/resources/ccitt.pdf + tests/resources/cardinal.pdf + tests/resources/jbig2.pdf + tests/resources/skew.pdf + tests/resources/rotated_skew.pdf + tests/resources/poster.pdf +Copyright: (C) 1985 Forat Electronics +License: GFDL-1.2-or-later or CC-BY-SA-3.0 + +Files: tests/resources/lichtenstein.pdf +Copyright: (C) 2001 Andreas Tille + (C) 2007 Alessio Damato +License: GFDL-1.2-or-later or CC-BY-SA-3.0 + +Files: tests/resources/masks.pdf +Copyright: held by the contributors to the German Wikipedia article "Linux" + see: https://de.wikipedia.org/w/index.php?title=Linux&action=history + (masks.pdf generated from Wikipedia article as of 2016-08-24) +License: CC-BY-SA-3.0 + +Files: tests/resources/epson.pdf +Copyright: held by the contributors to the Wikipedia article "Optical character recognition" + see: https://en.wikipedia.org/w/index.php?title=Optical_character_recognition&action=history + (epson.pdf generated from Wikipedia article as of 2016-09-14) +License: CC-BY-SA-3.0 + +Files: tests/resources/typewriter.png tests/resources/2400dpi.pdf +Copyright: (C) 2005 Ellywa +License: GFDL-1.2-or-later or CC-BY-SA-1.0 or CC-BY-SA-2.0 or CC-BY-SA-2.5 or CC-BY-SA-3.0 +Comment: + Obtained from: https://commons.wikimedia.org/wiki/File:Triumph.typewriter_text_Linzensoep.gif + +Files: tests/resources/overlay.pdf +Copyright: (C) 2017 Max Anderson +License: MIT + +Files: + tests/resources/baiona*.png + tests/resources/baiona*.jpg + tests/resources/link.pdf + tests/resources/palette.pdf +Copyright: (C) 2014 Euskaldunaa +License: CC-BY-SA-4.0 + +Files: tests/resources/vector.pdf +Copyright: (C) 2018 Catscratch +License: MIT + +Files: src/ocrmypdf/data/sRGB.icc +Copyright: Kai-Uwe Behrmann + Marti Maria + Photogamut + Graeme Gill + ColorSolutions +License: Zlib + +Files: tests/resources/3small.pdf +Copyright: (C) 2014 Euskaldunaa + (C) 2017 James R. Barlow + (C) 2005 Ellywa +License: CC-BY-SA-4.0 and (GFDL-1.2-or-later or CC-BY-SA-1.0 or CC-BY-SA-2.0 or CC-BY-SA-2.5 or CC-BY-SA-3.0) +Comment: concatenation of baiona_gray.png, crom.png and typewriter.png/2400dpi.pdf diff --git a/docs/jbig2.rst b/docs/jbig2.rst index b435d794..95ae027c 100644 --- a/docs/jbig2.rst +++ b/docs/jbig2.rst @@ -37,7 +37,8 @@ For all other Linux, you must build a JBIG2 encoder from source: .. _jbig2-lossy: Dependencies include libtoolize and libleptonica, which on Ubuntu systems -are packaged as libtool and libleptonica-dev. +are packaged as libtool and libleptonica-dev. On Fedora (35) they are packaged +as libtool and leptonica-devel. Lossy mode JBIG2 ================ diff --git a/src/ocrmypdf/_exec/ghostscript.py b/src/ocrmypdf/_exec/ghostscript.py index d67044e7..b4b0cd29 100644 --- a/src/ocrmypdf/_exec/ghostscript.py +++ b/src/ocrmypdf/_exec/ghostscript.py @@ -188,7 +188,8 @@ def generate_pdfa( strategy = 'LeaveColorUnchanged' gs_version = version() if gs_version == '9.56.0': - # 9.56.0 introduced a new rendering mode that breaks our OCR + # 9.56.0 breaks our OCR, should be fixed in 9.56.1 + # https://bugs.ghostscript.com/show_bug.cgi?id=705187 compression_args.append('-dNEWPDF=false') # nb no need to specify ProcessColorModel when ColorConversionStrategy diff --git a/src/ocrmypdf/_pipeline.py b/src/ocrmypdf/_pipeline.py index 0cefc19c..ead6d14c 100644 --- a/src/ocrmypdf/_pipeline.py +++ b/src/ocrmypdf/_pipeline.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: 2022 James R. Barlow +# SPDX-FileCopyrightText: 2018-2022 James R. Barlow +# SPDX-FileCopyrightText: 2019 Martin Wind # SPDX-License-Identifier: MPL-2.0 """OCRmyPDF page processing pipeline functions.""" diff --git a/src/ocrmypdf/_sync.py b/src/ocrmypdf/_sync.py index 1c971484..8521d1c6 100644 --- a/src/ocrmypdf/_sync.py +++ b/src/ocrmypdf/_sync.py @@ -1,4 +1,5 @@ -# SPDX-FileCopyrightText: 2022 James R. Barlow +# SPDX-FileCopyrightText: 2019-2022 James R. Barlow +# SPDX-FileCopyrightText: 2019 Martin Wind # SPDX-License-Identifier: MPL-2.0 """Implements the concurrent and page synchronous parts of the pipeline.""" diff --git a/tests/conftest.py b/tests/conftest.py index d9dfebea..d3b86061 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -44,7 +44,7 @@ TESTS_ROOT = Path(__file__).parent.resolve() PROJECT_ROOT = TESTS_ROOT -@pytest.fixture +@pytest.fixture(scope="session") def resources() -> Path: return Path(TESTS_ROOT) / 'resources' @@ -78,6 +78,11 @@ def no_outpdf(tmp_path) -> Path: return tmp_path / 'no_output.pdf' +@pytest.fixture(scope="session") +def multipage(resources): + return resources / 'multipage.pdf' + + def check_ocrmypdf(input_file: Path, output_file: Path, *args) -> Path: """Run ocrmypdf and confirm that a valid plausible PDF was created.""" api_args = [str(input_file), str(output_file)] + [ diff --git a/tests/plugins/tesseract_cache.py b/tests/plugins/tesseract_cache.py index bc2750f9..eec8f4c9 100644 --- a/tests/plugins/tesseract_cache.py +++ b/tests/plugins/tesseract_cache.py @@ -140,18 +140,19 @@ def cached_run(options, run_args, **run_kwargs): tessfile = args.outputbase + '.' + configfile shutil.copy(tessfile, str(cache_folder / configfile) + '.bin') - manifest = {} - manifest['tesseract_version'] = TesseractOcrEngine.version().replace('\n', ' ') - manifest['system'] = platform.system() - manifest['python'] = platform.python_version() - manifest['argv_slug'] = cache_folder.name - manifest['sourcefile'] = str(Path(source_file).relative_to(TESTS_ROOT)) - def clean_sys_argv(): for arg in run_args[1:]: yield re.sub(r'.*/ocrmypdf[.]io[.][^/]+[/](.*)', r'$TMPDIR/\1', arg) - manifest['args'] = list(clean_sys_argv()) + manifest = { + 'tesseract_version': TesseractOcrEngine.version().replace('\n', ' '), + 'system': platform.system(), + 'python': platform.python_version(), + 'argv_slug': cache_folder.name, + 'sourcefile': str(Path(source_file).relative_to(TESTS_ROOT)), + 'args': list(clean_sys_argv()), + } + with (Path(CACHE_ROOT) / 'manifest.jsonl').open('a') as f: json.dump(manifest, f) f.write('\n') diff --git a/tests/resources/README.rst b/tests/resources/README.rst index 91a6b8e1..c83990ee 100644 --- a/tests/resources/README.rst +++ b/tests/resources/README.rst @@ -23,7 +23,7 @@ the copyright holder(s) and license(s) applicable to these resources. - `US Congressional Records`_ - difficult OCR image (color background) * - graph.pdf - - `Wikimedia: Pandas text analysis.png`_ + - `Wikimedia:Simple_line_graph_of_ACE_2012_results_by_candidate_sj01.png`_ - image with slanted text * - lichtenstein.pdf - `Wikimedia: JPEG2000 Lichtenstein`_ @@ -111,14 +111,12 @@ These test resources are assemblies or derivatives from other previously mention - baiona_alpha.png (from baiona.png, RGB+A version) - cardinal.pdf (four cardinal directions, baked-in rotated copies of linn.png) - ccitt.pdf (linn.png, converted to CCITT encoding) -- encrypted_algo4.pdf (congress.jpg, encrypted with algorithm 4 - not supported by PyPDF2) - graph_ocred.pdf (from graph.pdf) -- jbig2.pdf (congress.jpg, converted to JBIG2 encoding) +- jbig2.pdf (from linn.png) - multipage.pdf (from several other files) -- palette.pdf (congress.jpg, converted to a 256-color palette) +- palette.pdf (from baiona_colormapped.png) - poster.pdf (from linn.png) - rotated_skew.pdf (a /Rotate'd and skewed document from linn.png) -- skew-encrypted.pdf (skew.pdf with encryption - access supported by PyPDF2, password is "password") - skew.pdf (from linn.png, skew simulated by adjusting the transformation matrix) - toc.pdf (from formxobject.pdf, trivial.pdf) @@ -129,7 +127,7 @@ These test resources are assemblies or derivatives from other previously mention .. _`US Congressional Records`: http://www.baxleystamps.com/litho/meiji/courts_1871.jpg -.. _`Wikimedia: Pandas text analysis.png`: https://en.wikipedia.org/wiki/File:Pandas_text_analysis.png +.. _`Wikimedia: Simple_line_graph_of_ACE_2012_results_by_candidate_sj01.png`: https://en.wikipedia.org/wiki/File:Simple_line_graph_of_ACE_2012_results_by_candidate_sj01.png .. _`Wikimedia: JPEG2000 Lichtenstein`: https://en.wikipedia.org/wiki/JPEG_2000#/media/File:Jpeg2000_2-level_wavelet_transform-lichtenstein.png diff --git a/tests/resources/encrypted_algo4.pdf b/tests/resources/encrypted_algo4.pdf deleted file mode 100644 index bc21948a..00000000 Binary files a/tests/resources/encrypted_algo4.pdf and /dev/null differ diff --git a/tests/resources/graph.pdf b/tests/resources/graph.pdf index 9b48e608..a30ae549 100644 Binary files a/tests/resources/graph.pdf and b/tests/resources/graph.pdf differ diff --git a/tests/resources/graph_ocred.pdf b/tests/resources/graph_ocred.pdf index 46ac8fc2..f58194f0 100644 Binary files a/tests/resources/graph_ocred.pdf and b/tests/resources/graph_ocred.pdf differ diff --git a/tests/resources/jbig2.pdf b/tests/resources/jbig2.pdf index 3ae244f0..d1a62d4a 100644 Binary files a/tests/resources/jbig2.pdf and b/tests/resources/jbig2.pdf differ diff --git a/tests/resources/multipage.pdf b/tests/resources/multipage.pdf old mode 100644 new mode 100755 index 80351bed..b3240567 Binary files a/tests/resources/multipage.pdf and b/tests/resources/multipage.pdf differ diff --git a/tests/resources/palette.pdf b/tests/resources/palette.pdf index 0efe066d..bc28d5c7 100644 Binary files a/tests/resources/palette.pdf and b/tests/resources/palette.pdf differ diff --git a/tests/resources/skew-encrypted.pdf b/tests/resources/skew-encrypted.pdf deleted file mode 100644 index 2a8fb769..00000000 Binary files a/tests/resources/skew-encrypted.pdf and /dev/null differ diff --git a/tests/test_concurrency.py b/tests/test_concurrency.py index e5244ddb..4e600bc3 100644 --- a/tests/test_concurrency.py +++ b/tests/test_concurrency.py @@ -14,9 +14,9 @@ from .conftest import run_ocrmypdf_api @pytest.mark.skipif(True, reason="--use-threads is currently default") @pytest.mark.skipif(os.name == 'nt', reason="Windows doesn't have SIGKILL") -def test_simulate_oom_killer(resources, no_outpdf): +def test_simulate_oom_killer(multipage, no_outpdf): exitcode = run_ocrmypdf_api( - resources / 'multipage.pdf', + multipage, no_outpdf, '--force-ocr', '--plugin', diff --git a/tests/test_main.py b/tests/test_main.py index d4e3135e..198ed3f1 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -7,11 +7,10 @@ import os import shutil from math import isclose from pathlib import Path -from subprocess import PIPE, run +from subprocess import run from unittest.mock import patch import pikepdf -import PIL import pytest from PIL import Image @@ -153,9 +152,9 @@ def test_skip_big(resources, outpdf): @pytest.mark.parametrize('renderer', RENDERERS) @pytest.mark.parametrize('output_type', ['pdf', 'pdfa']) -def test_maximum_options(renderer, output_type, resources, outpdf): +def test_maximum_options(renderer, output_type, multipage, outpdf): check_ocrmypdf( - resources / 'multipage.pdf', + multipage, outpdf, '-d', '-ci' if have_unpaper() else None, @@ -292,12 +291,6 @@ def test_input_file_not_a_pdf(caplog, no_outpdf): assert input_file in caplog.text -def test_encrypted(resources, caplog, no_outpdf): - result = run_ocrmypdf_api(resources / 'skew-encrypted.pdf', no_outpdf) - assert result == ExitCode.encrypted_pdf - assert 'encryption must be removed' in caplog.text - - @pytest.mark.parametrize('renderer', RENDERERS) def test_pagesegmode(renderer, resources, outpdf): check_ocrmypdf( @@ -398,14 +391,27 @@ def test_tesseract_image_too_big(renderer, resources, outpdf): ) -def test_algo4(resources, outpdf): - p = run_ocrmypdf( - resources / 'encrypted_algo4.pdf', +@pytest.mark.parametrize('encryption_level', [2, 3, 4, 6]) +def test_encrypted(resources, outpdf, encryption_level, caplog): + encryption = pikepdf.models.encryption.Encryption( + owner='ocrmypdf', + user='ocrmypdf', + R=encryption_level, + aes=(encryption_level >= 4), + metadata=(encryption_level == 6), + ) + + with pikepdf.open(resources / 'jbig2.pdf') as pdf: + pdf.save(outpdf, encryption=encryption) + + exitcode = run_ocrmypdf_api( + outpdf, outpdf, '--plugin', 'tests/plugins/tesseract_noop.py', ) - assert p.returncode == ExitCode.encrypted_pdf + assert exitcode == ExitCode.encrypted_pdf + assert 'encryption must be removed' in caplog.text def test_jbig2_passthrough(resources, outpdf): diff --git a/tests/test_optimize.py b/tests/test_optimize.py index 9c63a41d..5717269e 100644 --- a/tests/test_optimize.py +++ b/tests/test_optimize.py @@ -27,10 +27,15 @@ needs_jbig2enc = pytest.mark.skipif( ) +@pytest.fixture(scope="session") +def palette(resources): + return resources / 'palette.pdf' + + @needs_pngquant -@pytest.mark.parametrize('pdf', ['multipage.pdf', 'palette.pdf']) -def test_basic(resources, pdf, outpdf): - infile = resources / pdf +@pytest.mark.parametrize('pdf', ['multipage', 'palette']) +def test_basic(multipage, palette, pdf, outpdf): + infile = multipage if pdf == 'multipage' else palette opt.main(infile, outpdf, level=3) assert 0.98 * Path(outpdf).stat().st_size <= Path(infile).stat().st_size diff --git a/tests/test_page_numbers.py b/tests/test_page_numbers.py index a04dbb53..aac80c0d 100644 --- a/tests/test_page_numbers.py +++ b/tests/test_page_numbers.py @@ -48,10 +48,9 @@ def test_nonmonotonic_warning(caplog): assert 'out of order' in caplog.text -def test_limited_pages(resources, outpdf): - multi = resources / 'multipage.pdf' +def test_limited_pages(multipage, outpdf): ocrmypdf.ocr( - multi, + multipage, outpdf, pages='5-6', optimize=0, diff --git a/tests/test_rotation.py b/tests/test_rotation.py index 1fad82e5..130c6922 100644 --- a/tests/test_rotation.py +++ b/tests/test_rotation.py @@ -276,7 +276,8 @@ def test_rasterize_rotates(resources, tmp_path): rotation=90, filter_vector=False, ) - assert Image.open(img).size == (123, 151), "Image not rotated" + with Image.open(img) as im: + assert im.size == (83, 200), "Image not rotated" img = tmp_path / 'img180.png' pm.hook.rasterize_pdf_page( @@ -289,7 +290,7 @@ def test_rasterize_rotates(resources, tmp_path): rotation=180, filter_vector=False, ) - assert Image.open(img).size == (151, 123), "Image not rotated" + assert Image.open(img).size == (200, 83), "Image not rotated" def test_simulated_scan(outdir):