From ad9a3b530266051fc95e0d55aefed691386ae2ce Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Wed, 13 Nov 2019 01:45:06 -0800 Subject: [PATCH 01/35] Update version of pdfminer.six supported --- requirements/dev.txt | 2 -- requirements/main.txt | 2 +- setup.py | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 4faf987d..ab2e5ff6 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,4 +1,2 @@ -check-manifest >= 0.35 twine >= 1.8.1 coverage >= 4.5 -GitPython == 2.1.3 diff --git a/requirements/main.txt b/requirements/main.txt index c353ed51..2b6f1fca 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -3,7 +3,7 @@ # installation cffi == 1.13.2 img2pdf == 0.3.3 -pdfminer.six == 20191020 +pdfminer.six == 20191110 pikepdf == 1.7.0 Pillow >= 6.2.0 reportlab == 3.5.32 diff --git a/setup.py b/setup.py index 1a1d026f..7e0302eb 100644 --- a/setup.py +++ b/setup.py @@ -96,7 +96,7 @@ setup( 'chardet >= 3.0.4, < 4', # unlisted requirement of pdfminer.six 20181108 'cffi >= 1.9.1', # must be a setup and install requirement 'img2pdf >= 0.3.0, < 0.4', # pure Python, so track HEAD closely - 'pdfminer.six >= 20181108, <= 20191020', + 'pdfminer.six >= 20181108, <= 20191110', 'pikepdf >= 1.7.0, < 2', 'Pillow >= 6.2.0', 'reportlab >= 3.3.0', # oldest released version with sane image handling From b7f63bc93d524cef99f9c445bdded50b091218ac Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Sun, 17 Nov 2019 15:40:09 -0800 Subject: [PATCH 02/35] Make devnull check compatible with Windows --- src/ocrmypdf/_sync.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/ocrmypdf/_sync.py b/src/ocrmypdf/_sync.py index c02543b7..4c25b300 100644 --- a/src/ocrmypdf/_sync.py +++ b/src/ocrmypdf/_sync.py @@ -304,6 +304,13 @@ class NeverRaise(Exception): pass # pylint: disable=unnecessary-pass +def samefile(f1, f2): + if os.name == 'nt': + return f1 == f2 + else: + return os.path.samefile(f1, f2) + + def run_pipeline(options, api=False): log = make_logger(options, __name__) @@ -339,7 +346,7 @@ def run_pipeline(options, api=False): if options.output_file == '-': log.info("Output sent to stdout") - elif os.path.samefile(options.output_file, os.devnull): + elif samefile(options.output_file, os.devnull): pass # Say nothing when sending to dev null else: if options.output_type.startswith('pdfa'): From 84cc49b14b1cb4c395615dbaef78a38c7ab78dde Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Mon, 25 Nov 2019 14:20:59 -0800 Subject: [PATCH 03/35] black: don't reformat _leptonica.py --- .pre-commit-config.yaml | 7 ++++--- pyproject.toml | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b268b628..2ccebb79 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,7 @@ repos: -- repo: https://github.com/ambv/black + - repo: https://github.com/psf/black rev: stable hooks: - - id: black - language_version: python3.7 + - id: black + language_version: python3.7 + exclude: ^src/ocrmypdf/lib/_leptonica.py diff --git a/pyproject.toml b/pyproject.toml index 4d61be1a..a28f55c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,5 +28,6 @@ exclude = ''' | docs | misc | \.egg-info + | src/ocrmypdf/lib/_leptonica.py )/ ''' From 17c419dfcb6dd50bbe0310233ae47b60cab401d5 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Fri, 29 Nov 2019 04:00:40 -0800 Subject: [PATCH 04/35] compile_leptonica: move to correct location --- src/ocrmypdf/lib/compile_leptonica.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/ocrmypdf/lib/compile_leptonica.py b/src/ocrmypdf/lib/compile_leptonica.py index c76dd324..5cfbb6f2 100644 --- a/src/ocrmypdf/lib/compile_leptonica.py +++ b/src/ocrmypdf/lib/compile_leptonica.py @@ -491,3 +491,8 @@ ffibuilder.set_source("ocrmypdf.lib._leptonica", None) if __name__ == '__main__': ffibuilder.compile(verbose=True) + if Path('ocrmypdf/lib/_leptonica.py').exists() and Path('src/ocrmypdf').exists(): + output = Path('ocrmypdf/lib/_leptonica.py') + output.rename('src/ocrmypdf/lib/_leptonica.py') + Path('ocrmypdf/lib').rmdir() + Path('ocrmypdf').rmdir() From 72d3ee3a87bbcd4c2bf9ef66d608d40d42513cdb Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Sun, 17 Nov 2019 15:56:45 -0800 Subject: [PATCH 05/35] Refactor symlink usage to support Windows --- src/ocrmypdf/_pipeline.py | 6 +++--- src/ocrmypdf/_validation.py | 4 ++-- src/ocrmypdf/exec/tesseract.py | 4 ++-- src/ocrmypdf/helpers.py | 12 +++++++++--- src/ocrmypdf/optimize.py | 6 +++--- tests/test_main.py | 1 + 6 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/ocrmypdf/_pipeline.py b/src/ocrmypdf/_pipeline.py index 56c17e35..445d0faa 100644 --- a/src/ocrmypdf/_pipeline.py +++ b/src/ocrmypdf/_pipeline.py @@ -37,7 +37,7 @@ from .exceptions import ( UnsupportedImageFormatError, ) from .exec import ghostscript, tesseract -from .helpers import re_symlink +from .helpers import safe_symlink from .hocrtransform import HocrTransform from .optimize import optimize from .pdfa import generate_pdfa_ps @@ -132,7 +132,7 @@ def triage(input_file, output_file, options, log): "input file is a PDF, not an image." ) # Origin file is a pdf create a symlink with pdf extension - re_symlink(input_file, output_file) + safe_symlink(input_file, output_file) return output_file except EnvironmentError as e: log.error(e) @@ -701,7 +701,7 @@ def convert_to_pdfa(input_pdf, input_ps_stub, context): if modified: pdf_file.save(fix_docinfo_file) else: - os.symlink(input_pdf, fix_docinfo_file) + safe_symlink(input_pdf, fix_docinfo_file) ghostscript.generate_pdfa( pdf_version=input_pdfinfo.min_version, diff --git a/src/ocrmypdf/_validation.py b/src/ocrmypdf/_validation.py index b9df4b12..b02e4532 100644 --- a/src/ocrmypdf/_validation.py +++ b/src/ocrmypdf/_validation.py @@ -42,7 +42,7 @@ from .exec import ( tesseract, unpaper, ) -from .helpers import is_file_writable, is_iterable_notstr, monotonic, re_symlink +from .helpers import is_file_writable, is_iterable_notstr, monotonic, safe_symlink # ------------- # External dependencies @@ -374,7 +374,7 @@ def create_input_file(options, work_folder): else: try: target = os.path.join(work_folder, 'origin') - re_symlink(options.input_file, target) + safe_symlink(options.input_file, target) return target except FileNotFoundError: raise InputFileError(f"File not found - {options.input_file}") diff --git a/src/ocrmypdf/exec/tesseract.py b/src/ocrmypdf/exec/tesseract.py index afa0e4a6..cb837dfd 100644 --- a/src/ocrmypdf/exec/tesseract.py +++ b/src/ocrmypdf/exec/tesseract.py @@ -30,7 +30,7 @@ from ..exceptions import ( SubprocessOutputError, TesseractConfigError, ) -from ..helpers import page_number +from ..helpers import page_number, safe_symlink from . import get_version OrientationConfidence = namedtuple('OrientationConfidence', ('angle', 'confidence')) @@ -324,7 +324,7 @@ def use_skip_page(text_only, skip_pdf, output_pdf, output_text): # Substitute a "skipped page" with suppress(FileNotFoundError): os.remove(output_pdf) # In case it was partially created - os.symlink(skip_pdf, output_pdf) + safe_symlink(skip_pdf, output_pdf) return # Or normally, just write a 0 byte file to the output to indicate a skip diff --git a/src/ocrmypdf/helpers.py b/src/ocrmypdf/helpers.py index 80eff55f..b719d4e7 100644 --- a/src/ocrmypdf/helpers.py +++ b/src/ocrmypdf/helpers.py @@ -18,6 +18,7 @@ import logging import multiprocessing import os +import shutil import warnings from collections.abc import Iterable from contextlib import suppress @@ -27,14 +28,14 @@ from pathlib import Path log = logging.getLogger(__name__) -def re_symlink(input_file, soft_link_name, *args, **kwargs): +def safe_symlink(input_file, soft_link_name, *args, **kwargs): """ Helper function: relinks soft symbolic link if necessary """ if len(args) == 1 and isinstance(args[0], logging.Logger): - log.warning("Deprecated: re_symlink(,log)") + log.warning("Deprecated: safe_symlink(,log)") if 'log' in kwargs: - log.warning('Deprecated: re_symlink(...log=)') + log.warning('Deprecated: safe_symlink(...log=)') input_file = os.fspath(input_file) soft_link_name = os.fspath(soft_link_name) @@ -60,6 +61,11 @@ def re_symlink(input_file, soft_link_name, *args, **kwargs): if not os.path.exists(input_file): raise FileNotFoundError(f"trying to create a broken symlink to {input_file}") + if os.name == 'nt': + # Don't actually use symlinks on Windows due to permission issues + shutil.copyfile(input_file, soft_link_name) + return + log.debug("os.symlink(%s, %s)", input_file, soft_link_name) # Create symbolic link using absolute path diff --git a/src/ocrmypdf/optimize.py b/src/ocrmypdf/optimize.py index bb1269db..72022741 100644 --- a/src/ocrmypdf/optimize.py +++ b/src/ocrmypdf/optimize.py @@ -31,7 +31,7 @@ from . import leptonica from ._jobcontext import PDFContext from .exceptions import OutputFileAccessError from .exec import jbig2enc, pngquant -from .helpers import re_symlink +from .helpers import safe_symlink DEFAULT_JPEG_QUALITY = 75 DEFAULT_PNG_QUALITY = 70 @@ -492,7 +492,7 @@ def optimize(input_file, output_file, context, save_settings): log = context.log options = context.options if options.optimize == 0: - re_symlink(input_file, output_file) + safe_symlink(input_file, output_file) return if options.jpeg_quality == 0: @@ -538,7 +538,7 @@ def optimize(input_file, output_file, context, save_settings): pike.remove_unreferenced_resources() pike.save(output_file, **save_settings) else: - re_symlink(target_file, output_file) + safe_symlink(target_file, output_file) def main(infile, outfile, level, jobs=1): diff --git a/tests/test_main.py b/tests/test_main.py index 79625d12..c55186b0 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -946,6 +946,7 @@ def test_output_is_dir(spoof_tesseract_noop, resources, outdir): assert 'is not a writable file' in err +@pytest.mark.skipif(os.name == 'nt', reason="symlink needs admin permissions") def test_output_is_symlink(spoof_tesseract_noop, resources, outdir): sym = Path(outdir / 'this_is_a_symlink') sym.symlink_to(outdir / 'out.pdf') From d5bb9929f390fc8d2607e3ce1588ad6a969496c8 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Sun, 17 Nov 2019 15:41:48 -0800 Subject: [PATCH 06/35] leptonica: Use Windows name for DLL Thanks to @dibu28 --- src/ocrmypdf/leptonica.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/ocrmypdf/leptonica.py b/src/ocrmypdf/leptonica.py index 1cdec312..ef2a2b82 100644 --- a/src/ocrmypdf/leptonica.py +++ b/src/ocrmypdf/leptonica.py @@ -39,7 +39,11 @@ from .lib._leptonica import ffi logger = logging.getLogger(__name__) -lept = ffi.dlopen(find_library('lept')) +if os.name == 'nt': + libname = 'liblept-5' +else: + libname = 'lept' +lept = ffi.dlopen(find_library(libname)) lept.setMsgSeverity(lept.L_SEVERITY_WARNING) From 9baccee8c5c7fdbbef0a7b48b81bc5ab285ea94c Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Wed, 20 Nov 2019 00:29:48 -0800 Subject: [PATCH 07/35] leptonica: Handle API change for pixFindPageForeground --- src/ocrmypdf/leptonica.py | 13 +++++------- src/ocrmypdf/lib/compile_leptonica.py | 30 +++++++++++++++++++-------- tests/test_lept.py | 4 ++++ 3 files changed, 30 insertions(+), 17 deletions(-) diff --git a/src/ocrmypdf/leptonica.py b/src/ocrmypdf/leptonica.py index ef2a2b82..831a0695 100644 --- a/src/ocrmypdf/leptonica.py +++ b/src/ocrmypdf/leptonica.py @@ -506,17 +506,14 @@ class Pix(LeptonicaObject): display=0, pdfdir=ffi.NULL, ): + if get_leptonica_version() < 'leptonica-1.76': + # Leptonica 1.76 changed the API for pixFindPageForeground; we don't + # support the old version + raise LeptonicaError("Not available in this version of Leptonica") with _LeptonicaErrorTrap(): cropbox = Box( lept.pixFindPageForeground( - self._cdata, - threshold, - mindist, - erasedist, - pagenum, - showmorph, - display, - pdfdir, + self._cdata, threshold, mindist, erasedist, showmorph, ffi.NULL ) ) diff --git a/src/ocrmypdf/lib/compile_leptonica.py b/src/ocrmypdf/lib/compile_leptonica.py index 5cfbb6f2..e6d3c9f3 100644 --- a/src/ocrmypdf/lib/compile_leptonica.py +++ b/src/ocrmypdf/lib/compile_leptonica.py @@ -74,6 +74,17 @@ struct Pixa }; typedef struct Pixa PIXA; +/*! Array of compressed pix */ +struct PixaComp +{ + l_int32 n; /*!< number of PixComp in ptr array */ + l_int32 nalloc; /*!< number of PixComp ptrs allocated */ + l_int32 offset; /*!< indexing offset into ptr array */ + struct PixComp **pixc; /*!< the array of ptrs to PixComp */ + struct Boxa *boxa; /*!< array of boxes */ +}; +typedef struct PixaComp PIXAC; + struct Box { l_int32 x; @@ -294,14 +305,12 @@ pixCleanBackgroundToWhite(PIX *pixs, l_int32 whiteval); BOX * -pixFindPageForeground(PIX *pixs, - l_int32 threshold, - l_int32 mindist, - l_int32 erasedist, - l_int32 pagenum, - l_int32 showmorph, - l_int32 display, - const char *pdfdir); +pixFindPageForeground ( PIX *pixs, + l_int32 threshold, + l_int32 mindist, + l_int32 erasedist, + l_int32 showmorph, + PIXAC *pixac ); PIX * pixClipRectangle(PIX *pixs, @@ -414,7 +423,10 @@ pixExtractBarcodes(PIX *pixs, l_int32 debugflag); BOXA * -pixLocateBarcodes ( PIX *pixs, l_int32 thresh, PIX **ppixb, PIX **ppixm ); +pixLocateBarcodes ( PIX *pixs, + l_int32 thresh, + PIX **ppixb, + PIX **ppixm ); SARRAY * pixReadBarcodes(PIXA *pixa, diff --git a/tests/test_lept.py b/tests/test_lept.py index 2504c600..b74f417b 100644 --- a/tests/test_lept.py +++ b/tests/test_lept.py @@ -63,6 +63,10 @@ def test_pix_otsu(crom_pix): assert im1bpp.mode == '1' +@pytest.mark.skipif( + lept.get_leptonica_version() < 'leptonica-1.76', + reason="needs new leptonica for API change", +) def test_crop(resources): pix = lept.Pix.open(resources / 'linn.png') foreground = pix.crop_to_foreground() From fe7c69ce95639dfb150916671c8b2780cfe1badb Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Thu, 28 Nov 2019 14:41:37 -0800 Subject: [PATCH 08/35] leptonica: don't open files by name; use memory buffers Avoids encoding issues and makes error trap unnecessary in some cases. --- src/ocrmypdf/leptonica.py | 26 ++++++++++++++++++-------- src/ocrmypdf/lib/_leptonica.py | 10 +++++----- src/ocrmypdf/lib/compile_leptonica.py | 8 ++++++++ tests/test_lept.py | 16 +++------------- 4 files changed, 34 insertions(+), 26 deletions(-) diff --git a/src/ocrmypdf/leptonica.py b/src/ocrmypdf/leptonica.py index 831a0695..ca0c546e 100644 --- a/src/ocrmypdf/leptonica.py +++ b/src/ocrmypdf/leptonica.py @@ -296,9 +296,11 @@ class Pix(LeptonicaObject): Leptonica can load TIFF, PNM (PBM, PGM, PPM), PNG, and JPEG. If loading fails then the object will wrap a C null pointer. """ - filename = fspath(path) - with _LeptonicaErrorTrap(): - return cls(lept.pixRead(os.fsencode(filename))) + with open(path, 'rb') as py_file: + data = py_file.read() + buffer = ffi.from_buffer(data) + with _LeptonicaErrorTrap(): + return cls(lept.pixReadMem(buffer, len(buffer))) def write_implied_format(self, path, jpeg_quality=0, jpeg_progressive=0): """Write pix to the filename, with the extension indicating format. @@ -306,11 +308,19 @@ class Pix(LeptonicaObject): jpeg_quality -- quality (iff JPEG; 1 - 100, 0 for default) jpeg_progressive -- (iff JPEG; 0 for baseline seq., 1 for progressive) """ - filename = fspath(path) - with _LeptonicaErrorTrap(): - lept.pixWriteImpliedFormat( - os.fsencode(filename), self._cdata, jpeg_quality, jpeg_progressive - ) + lept_format = lept.getImpliedFileFormat(os.fsencode(path)) + with open(path, 'wb') as py_file: + data = ffi.new('l_uint8 **pdata') + size = ffi.new('size_t *psize') + with _LeptonicaErrorTrap(): + if lept_format == lept.L_JPEG_ENCODE: + lept.pixWriteMemJpeg( + data, size, self._cdata, jpeg_quality, jpeg_progressive + ) + else: + lept.pixWriteMem(data, size, self._cdata, lept_format) + buffer = ffi.buffer(data[0], size[0]) + py_file.write(buffer) @classmethod def frompil(self, pillow_image): diff --git a/src/ocrmypdf/lib/_leptonica.py b/src/ocrmypdf/lib/_leptonica.py index 17a2f757..549098c1 100644 --- a/src/ocrmypdf/lib/_leptonica.py +++ b/src/ocrmypdf/lib/_leptonica.py @@ -3,9 +3,9 @@ import _cffi_backend ffi = _cffi_backend.FFI('ocrmypdf.lib._leptonica', _version = 0x2601, - _types = b'\x00\x00\x01\x0D\x00\x01\x33\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x34\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x37\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x01\x3F\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x38\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x04\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x09\x03\x00\x00\x1A\x11\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x01\x11\x00\x00\x01\x03\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x0D\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x01\x3C\x03\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x0D\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x0D\x01\x00\x00\x05\x03\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x0D\x01\x00\x00\x60\x11\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x0D\x01\x00\x00\x0D\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x13\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x10\x11\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x01\x4E\x03\x00\x00\x1C\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x01\x50\x03\x00\x00\x1C\x01\x00\x00\x00\x0F\x00\x00\x13\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x01\x3A\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x01\x3A\x03\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x01\x3A\x0D\x00\x00\x13\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x9C\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x45\x0D\x00\x00\x10\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x10\x11\x00\x00\x00\x0F\x00\x00\x45\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x01\x3E\x0D\x00\x00\x45\x11\x00\x00\x00\x0F\x00\x01\x3E\x0D\x00\x00\x00\x0F\x00\x00\x60\x0D\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x04\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x32\x03\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x60\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xC3\x11\x00\x00\xC3\x11\x00\x00\xC3\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\xC3\x11\x00\x00\xC3\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x60\x11\x00\x00\x60\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x0D\x01\x00\x00\x07\x01\x00\x00\x60\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x01\x35\x03\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xC3\x11\x00\x00\xC3\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x0D\x01\x00\x00\x1A\x11\x00\x00\x1A\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x01\x4F\x03\x00\x00\x8E\x03\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x10\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\xEC\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x10\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x01\x4D\x03\x00\x01\x04\x11\x00\x00\x09\x11\x00\x00\x0D\x01\x00\x00\x00\x0F\x00\x01\x53\x0D\x00\x00\x23\x11\x00\x00\x00\x0F\x00\x01\x53\x0D\x00\x00\x04\x03\x00\x00\x00\x0F\x00\x01\x53\x0D\x00\x00\xEC\x11\x00\x00\x00\x0F\x00\x01\x53\x0D\x00\x00\x1A\x11\x00\x00\x00\x0F\x00\x01\x53\x0D\x00\x00\x13\x03\x00\x00\x00\x0F\x00\x01\x53\x0D\x00\x00\x9C\x11\x00\x00\x00\x0F\x00\x01\x53\x0D\x00\x00\x45\x03\x00\x00\x00\x0F\x00\x01\x53\x0D\x00\x01\x53\x03\x00\x00\x00\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x01\x36\x03\x00\x00\x02\x09\x00\x00\x03\x09\x00\x00\x05\x09\x00\x00\x04\x09\x00\x01\x3B\x03\x00\x00\x06\x09\x00\x00\x07\x09\x00\x01\x3E\x03\x00\x01\x3F\x03\x00\x00\x02\x01\x00\x00\x0E\x01\x00\x00\x00\x0B\x00\x00\x01\x0B\x00\x00\x02\x0B\x00\x00\x03\x0B\x00\x00\x04\x0B\x00\x00\x05\x0B\x00\x00\x06\x0B\x00\x00\x60\x03\x00\x00\x0B\x01\x00\x00\x05\x01\x00\x00\x03\x01\x00\x01\x39\x03\x00\x01\x4E\x03\x00\x00\x04\x01\x00\x01\x50\x03\x00\x00\x08\x01\x00\x00\x0C\x01\x00\x00\x06\x01\x00\x00\x00\x01', - _globals = (b'\xFF\xFF\xFF\x0BL_BF_ANY',1,b'\xFF\xFF\xFF\x0BL_BF_CODABAR',9,b'\xFF\xFF\xFF\x0BL_BF_CODE128',2,b'\xFF\xFF\xFF\x0BL_BF_CODE2OF5',5,b'\xFF\xFF\xFF\x0BL_BF_CODE39',7,b'\xFF\xFF\xFF\x0BL_BF_CODE93',8,b'\xFF\xFF\xFF\x0BL_BF_CODEI2OF5',6,b'\xFF\xFF\xFF\x0BL_BF_EAN13',4,b'\xFF\xFF\xFF\x0BL_BF_EAN8',3,b'\xFF\xFF\xFF\x0BL_BF_UNKNOWN',0,b'\xFF\xFF\xFF\x0BL_BF_UPCA',10,b'\xFF\xFF\xFF\x0BL_CLONE',2,b'\xFF\xFF\xFF\x0BL_COPY',1,b'\xFF\xFF\xFF\x0BL_COPY_CLONE',3,b'\xFF\xFF\xFF\x0BL_DEFAULT_ENCODE',0,b'\xFF\xFF\xFF\x0BL_FLATE_ENCODE',3,b'\xFF\xFF\xFF\x0BL_G4_ENCODE',2,b'\xFF\xFF\xFF\x0BL_INSERT',0,b'\xFF\xFF\xFF\x0BL_JP2K_ENCODE',4,b'\xFF\xFF\xFF\x0BL_JPEG_ENCODE',1,b'\xFF\xFF\xFF\x0BL_NOCOPY',0,b'\xFF\xFF\xFF\x0BL_SEVERITY_ALL',1,b'\xFF\xFF\xFF\x0BL_SEVERITY_DEBUG',2,b'\xFF\xFF\xFF\x0BL_SEVERITY_ERROR',5,b'\xFF\xFF\xFF\x0BL_SEVERITY_EXTERNAL',0,b'\xFF\xFF\xFF\x0BL_SEVERITY_INFO',3,b'\xFF\xFF\xFF\x0BL_SEVERITY_NONE',6,b'\xFF\xFF\xFF\x0BL_SEVERITY_WARNING',4,b'\xFF\xFF\xFF\x0BL_USE_WIDTHS',1,b'\xFF\xFF\xFF\x0BL_USE_WINDOWS',2,b'\xFF\xFF\xFF\x0BREMOVE_CMAP_BASED_ON_SRC',4,b'\xFF\xFF\xFF\x0BREMOVE_CMAP_TO_BINARY',0,b'\xFF\xFF\xFF\x0BREMOVE_CMAP_TO_FULL_COLOR',2,b'\xFF\xFF\xFF\x0BREMOVE_CMAP_TO_GRAYSCALE',1,b'\xFF\xFF\xFF\x0BREMOVE_CMAP_WITH_ALPHA',3,b'\xFF\xFF\xFF\x0BSEL_DONT_CARE',0,b'\xFF\xFF\xFF\x0BSEL_HIT',1,b'\xFF\xFF\xFF\x0BSEL_MISS',2,b'\x00\x00\x00\x23boxClone',0,b'\x00\x01\x1B\x23boxDestroy',0,b'\x00\x01\x1E\x23boxaDestroy',0,b'\x00\x00\x03\x23boxaGetBox',0,b'\x00\x00\xB6\x23getLeptonicaVersion',0,b'\x00\x01\x21\x23l_CIDataDestroy',0,b'\x00\x01\x06\x23l_generateCIDataForPdf',0,b'\x00\x01\x30\x23lept_free',0,b'\x00\x00\xB8\x23makePixelSumTab8',0,b'\x00\x00\x29\x23pixAnd',0,b'\x00\x00\x36\x23pixBackgroundNorm',0,b'\x00\x00\x2E\x23pixCleanBackgroundToWhite',0,b'\x00\x00\x20\x23pixClipRectangle',0,b'\x00\x00\xEE\x23pixColorFraction',0,b'\x00\x00\x7D\x23pixColorMagnitude',0,b'\x00\x00\x1D\x23pixConvertRGBToLuminance',0,b'\x00\x00\x74\x23pixConvertTo8',0,b'\x00\x00\xC0\x23pixCorrelationBinary',0,b'\x00\x00\xDA\x23pixCountPixels',0,b'\x00\x00\x90\x23pixDeserializeFromMemory',0,b'\x00\x00\x74\x23pixDeskew',0,b'\x00\x01\x24\x23pixDestroy',0,b'\x00\x00\x42\x23pixDilate',0,b'\x00\x00\x1D\x23pixEndianByteSwapNew',0,b'\x00\x00\xC5\x23pixEqual',0,b'\x00\x00\x42\x23pixErode',0,b'\x00\x00\x94\x23pixExtractBarcodes',0,b'\x00\x00\x08\x23pixFindPageForeground',0,b'\x00\x00\xD5\x23pixFindSkew',0,b'\x00\x00\x47\x23pixGammaTRC',0,b'\x00\x00\xE7\x23pixGenerateCIData',0,b'\x00\x00\xCA\x23pixGetAverageMaskedRGB',0,b'\x00\x00\x4E\x23pixGlobalNormRGB',0,b'\x00\x00\x42\x23pixHMT',0,b'\x00\x00\x25\x23pixInvert',0,b'\x00\x00\x17\x23pixLocateBarcodes',0,b'\x00\x00\x78\x23pixMaskOverColorPixels',0,b'\x00\x00\x56\x23pixMaskedThreshOnBackgroundNorm',0,b'\x00\x00\xDF\x23pixNumSignificantGrayColors',0,b'\x00\x00\xF7\x23pixOtsuAdaptiveThreshold',0,b'\x00\x00\x62\x23pixOtsuThreshOnBackgroundNorm',0,b'\x00\x00\x98\x23pixProcessBarcodes',0,b'\x00\x00\x89\x23pixRead',0,b'\x00\x00\x9F\x23pixReadBarcodes',0,b'\x00\x00\x8C\x23pixReadMem',0,b'\x00\x00\x74\x23pixRemoveColormap',0,b'\x00\x00\x78\x23pixRemoveColormapGeneral',0,b'\x00\x00\xBA\x23pixRenderBoxa',0,b'\x00\x00\x25\x23pixRotate180',0,b'\x00\x00\x74\x23pixRotateOrth',0,b'\x00\x00\x6F\x23pixScale',0,b'\x00\x01\x01\x23pixSerializeToMemory',0,b'\x00\x00\x29\x23pixSubtract',0,b'\x00\x01\x0C\x23pixWriteImpliedFormat',0,b'\x00\x01\x15\x23pixWriteMemPng',0,b'\x00\x01\x27\x23pixaDestroy',0,b'\x00\x00\x12\x23pixaGetBox',0,b'\x00\x00\x84\x23pixaGetPix',0,b'\x00\x01\x2A\x23sarrayDestroy',0,b'\x00\x00\xAC\x23selCreateBrick',0,b'\x00\x00\xA6\x23selCreateFromString',0,b'\x00\x01\x2D\x23selDestroy',0,b'\x00\x00\xB3\x23selPrintToString',0,b'\x00\x01\x12\x23setMsgSeverity',0), - _struct_unions = ((b'\x00\x00\x01\x33\x00\x00\x00\x02Box',b'\x00\x00\x05\x11x',b'\x00\x00\x05\x11y',b'\x00\x00\x05\x11w',b'\x00\x00\x05\x11h',b'\x00\x01\x50\x11refcount'),(b'\x00\x00\x01\x34\x00\x00\x00\x02Boxa',b'\x00\x00\x05\x11n',b'\x00\x00\x05\x11nalloc',b'\x00\x01\x50\x11refcount',b'\x00\x00\x23\x11box'),(b'\x00\x00\x01\x36\x00\x00\x00\x02L_Compressed_Data',b'\x00\x00\x05\x11type',b'\x00\x01\x4D\x11datacomp',b'\x00\x00\x8E\x11nbytescomp',b'\x00\x01\x3E\x11data85',b'\x00\x00\x8E\x11nbytes85',b'\x00\x01\x3E\x11cmapdata85',b'\x00\x01\x3E\x11cmapdatahex',b'\x00\x00\x05\x11ncolors',b'\x00\x00\x05\x11w',b'\x00\x00\x05\x11h',b'\x00\x00\x05\x11bps',b'\x00\x00\x05\x11spp',b'\x00\x00\x05\x11minisblack',b'\x00\x00\x05\x11predictor',b'\x00\x00\x8E\x11nbytes',b'\x00\x00\x05\x11res'),(b'\x00\x00\x01\x37\x00\x00\x00\x02Pix',b'\x00\x01\x50\x11w',b'\x00\x01\x50\x11h',b'\x00\x01\x50\x11d',b'\x00\x01\x50\x11spp',b'\x00\x01\x50\x11wpl',b'\x00\x01\x50\x11refcount',b'\x00\x00\x05\x11xres',b'\x00\x00\x05\x11yres',b'\x00\x00\x05\x11informat',b'\x00\x00\x05\x11special',b'\x00\x01\x3E\x11text',b'\x00\x01\x4C\x11colormap',b'\x00\x01\x4F\x11data'),(b'\x00\x00\x01\x39\x00\x00\x00\x02PixColormap',b'\x00\x01\x31\x11array',b'\x00\x00\x05\x11depth',b'\x00\x00\x05\x11nalloc',b'\x00\x00\x05\x11n'),(b'\x00\x00\x01\x38\x00\x00\x00\x02Pixa',b'\x00\x00\x05\x11n',b'\x00\x00\x05\x11nalloc',b'\x00\x01\x50\x11refcount',b'\x00\x00\x1A\x11pix',b'\x00\x00\x04\x11boxa'),(b'\x00\x00\x01\x3B\x00\x00\x00\x02Sarray',b'\x00\x00\x05\x11nalloc',b'\x00\x00\x05\x11n',b'\x00\x00\x05\x11refcount',b'\x00\x01\x3D\x11array'),(b'\x00\x00\x01\x3C\x00\x00\x00\x02Sel',b'\x00\x00\x05\x11sy',b'\x00\x00\x05\x11sx',b'\x00\x00\x05\x11cy',b'\x00\x00\x05\x11cx',b'\x00\x01\x48\x11data',b'\x00\x01\x3E\x11name')), - _enums = (b'\x00\x00\x01\x41\x00\x00\x00\x16$1\x00L_DEFAULT_ENCODE,L_JPEG_ENCODE,L_G4_ENCODE,L_FLATE_ENCODE,L_JP2K_ENCODE',b'\x00\x00\x01\x42\x00\x00\x00\x16$2\x00REMOVE_CMAP_TO_BINARY,REMOVE_CMAP_TO_GRAYSCALE,REMOVE_CMAP_TO_FULL_COLOR,REMOVE_CMAP_WITH_ALPHA,REMOVE_CMAP_BASED_ON_SRC',b'\x00\x00\x01\x43\x00\x00\x00\x16$3\x00L_NOCOPY,L_INSERT,L_COPY,L_CLONE,L_COPY_CLONE',b'\x00\x00\x01\x44\x00\x00\x00\x16$4\x00L_USE_WIDTHS,L_USE_WINDOWS',b'\x00\x00\x01\x45\x00\x00\x00\x16$5\x00L_BF_UNKNOWN,L_BF_ANY,L_BF_CODE128,L_BF_EAN8,L_BF_EAN13,L_BF_CODE2OF5,L_BF_CODEI2OF5,L_BF_CODE39,L_BF_CODE93,L_BF_CODABAR,L_BF_UPCA',b'\x00\x00\x01\x46\x00\x00\x00\x16$6\x00L_SEVERITY_EXTERNAL,L_SEVERITY_ALL,L_SEVERITY_DEBUG,L_SEVERITY_INFO,L_SEVERITY_WARNING,L_SEVERITY_ERROR,L_SEVERITY_NONE',b'\x00\x00\x01\x47\x00\x00\x00\x16$7\x00SEL_DONT_CARE,SEL_HIT,SEL_MISS'), - _typenames = (b'\x00\x00\x01\x33BOX',b'\x00\x00\x01\x34BOXA',b'\x00\x00\x01\x36L_COMP_DATA',b'\x00\x00\x01\x37PIX',b'\x00\x00\x01\x38PIXA',b'\x00\x00\x01\x39PIXCMAP',b'\x00\x00\x01\x3BSARRAY',b'\x00\x00\x01\x3CSEL',b'\x00\x00\x00\x32l_float32',b'\x00\x00\x01\x40l_float64',b'\x00\x00\x01\x4Al_int16',b'\x00\x00\x00\x05l_int32',b'\x00\x00\x01\x49l_int64',b'\x00\x00\x01\x4Bl_int8',b'\x00\x00\x00\x05l_ok',b'\x00\x00\x01\x52l_uint16',b'\x00\x00\x01\x50l_uint32',b'\x00\x00\x01\x51l_uint64',b'\x00\x00\x01\x4El_uint8'), + _types = b'\x00\x00\x01\x0D\x00\x01\x50\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x51\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x55\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x01\x57\x03\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x01\x56\x03\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x04\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x09\x03\x00\x00\x18\x11\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x01\x52\x03\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x01\x11\x00\x00\x01\x03\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x0D\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x01\x5B\x03\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x0D\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x0D\x01\x00\x00\x05\x03\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x0D\x01\x00\x00\x62\x11\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x0D\x01\x00\x00\x0D\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x01\x5E\x03\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x01\x70\x03\x00\x00\x1C\x01\x00\x00\x00\x0F\x00\x00\x09\x0D\x00\x01\x72\x03\x00\x00\x1C\x01\x00\x00\x00\x0F\x00\x00\x11\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x01\x59\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x01\x59\x03\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x01\x59\x0D\x00\x00\x11\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x9E\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x47\x0D\x00\x00\x8C\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x8C\x11\x00\x00\x00\x0F\x00\x00\x47\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x01\x5D\x0D\x00\x00\x47\x11\x00\x00\x00\x0F\x00\x01\x5D\x0D\x00\x00\x00\x0F\x00\x00\x62\x0D\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x1C\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x1C\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x04\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x34\x03\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x62\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xD0\x11\x00\x00\xD0\x11\x00\x00\xD0\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\xD0\x11\x00\x00\xD0\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x62\x11\x00\x00\x62\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x0D\x01\x00\x00\x07\x01\x00\x00\x62\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x01\x53\x03\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\xD0\x11\x00\x00\xD0\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x0D\x01\x00\x00\x18\x11\x00\x00\x18\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x09\x11\x00\x01\x71\x03\x00\x00\x90\x03\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x8C\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x8C\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\xF9\x11\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x8C\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x01\x6F\x03\x00\x01\x11\x11\x00\x00\x09\x11\x00\x00\x0D\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x01\x26\x11\x00\x01\x11\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x05\x0D\x00\x01\x26\x11\x00\x01\x11\x11\x00\x00\x09\x11\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x01\x75\x0D\x00\x00\x25\x11\x00\x00\x00\x0F\x00\x01\x75\x0D\x00\x00\x04\x03\x00\x00\x00\x0F\x00\x01\x75\x0D\x00\x00\xF9\x11\x00\x00\x00\x0F\x00\x01\x75\x0D\x00\x00\x18\x11\x00\x00\x00\x0F\x00\x01\x75\x0D\x00\x00\x11\x03\x00\x00\x00\x0F\x00\x01\x75\x0D\x00\x00\x9E\x11\x00\x00\x00\x0F\x00\x01\x75\x0D\x00\x00\x47\x03\x00\x00\x00\x0F\x00\x01\x75\x0D\x00\x01\x75\x03\x00\x00\x00\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x0A\x09\x00\x01\x54\x03\x00\x00\x02\x09\x00\x00\x03\x09\x00\x00\x06\x09\x00\x00\x07\x09\x00\x00\x04\x09\x00\x01\x5A\x03\x00\x00\x08\x09\x00\x00\x09\x09\x00\x01\x5D\x03\x00\x01\x5E\x03\x00\x00\x02\x01\x00\x00\x0E\x01\x00\x00\x00\x0B\x00\x00\x01\x0B\x00\x00\x02\x0B\x00\x00\x03\x0B\x00\x00\x04\x0B\x00\x00\x05\x0B\x00\x00\x06\x0B\x00\x00\x62\x03\x00\x00\x0B\x01\x00\x00\x05\x01\x00\x00\x03\x01\x00\x01\x58\x03\x00\x01\x6D\x03\x00\x01\x6E\x03\x00\x00\x05\x09\x00\x01\x70\x03\x00\x00\x04\x01\x00\x01\x72\x03\x00\x00\x08\x01\x00\x00\x0C\x01\x00\x00\x06\x01\x00\x00\x00\x01', + _globals = (b'\xFF\xFF\xFF\x0BL_BF_ANY',1,b'\xFF\xFF\xFF\x0BL_BF_CODABAR',9,b'\xFF\xFF\xFF\x0BL_BF_CODE128',2,b'\xFF\xFF\xFF\x0BL_BF_CODE2OF5',5,b'\xFF\xFF\xFF\x0BL_BF_CODE39',7,b'\xFF\xFF\xFF\x0BL_BF_CODE93',8,b'\xFF\xFF\xFF\x0BL_BF_CODEI2OF5',6,b'\xFF\xFF\xFF\x0BL_BF_EAN13',4,b'\xFF\xFF\xFF\x0BL_BF_EAN8',3,b'\xFF\xFF\xFF\x0BL_BF_UNKNOWN',0,b'\xFF\xFF\xFF\x0BL_BF_UPCA',10,b'\xFF\xFF\xFF\x0BL_CLONE',2,b'\xFF\xFF\xFF\x0BL_COPY',1,b'\xFF\xFF\xFF\x0BL_COPY_CLONE',3,b'\xFF\xFF\xFF\x0BL_DEFAULT_ENCODE',0,b'\xFF\xFF\xFF\x0BL_FLATE_ENCODE',3,b'\xFF\xFF\xFF\x0BL_G4_ENCODE',2,b'\xFF\xFF\xFF\x0BL_INSERT',0,b'\xFF\xFF\xFF\x0BL_JP2K_ENCODE',4,b'\xFF\xFF\xFF\x0BL_JPEG_ENCODE',1,b'\xFF\xFF\xFF\x0BL_NOCOPY',0,b'\xFF\xFF\xFF\x0BL_SEVERITY_ALL',1,b'\xFF\xFF\xFF\x0BL_SEVERITY_DEBUG',2,b'\xFF\xFF\xFF\x0BL_SEVERITY_ERROR',5,b'\xFF\xFF\xFF\x0BL_SEVERITY_EXTERNAL',0,b'\xFF\xFF\xFF\x0BL_SEVERITY_INFO',3,b'\xFF\xFF\xFF\x0BL_SEVERITY_NONE',6,b'\xFF\xFF\xFF\x0BL_SEVERITY_WARNING',4,b'\xFF\xFF\xFF\x0BL_USE_WIDTHS',1,b'\xFF\xFF\xFF\x0BL_USE_WINDOWS',2,b'\xFF\xFF\xFF\x0BREMOVE_CMAP_BASED_ON_SRC',4,b'\xFF\xFF\xFF\x0BREMOVE_CMAP_TO_BINARY',0,b'\xFF\xFF\xFF\x0BREMOVE_CMAP_TO_FULL_COLOR',2,b'\xFF\xFF\xFF\x0BREMOVE_CMAP_TO_GRAYSCALE',1,b'\xFF\xFF\xFF\x0BREMOVE_CMAP_WITH_ALPHA',3,b'\xFF\xFF\xFF\x0BSEL_DONT_CARE',0,b'\xFF\xFF\xFF\x0BSEL_HIT',1,b'\xFF\xFF\xFF\x0BSEL_MISS',2,b'\x00\x00\x00\x23boxClone',0,b'\x00\x01\x38\x23boxDestroy',0,b'\x00\x01\x3B\x23boxaDestroy',0,b'\x00\x00\x03\x23boxaGetBox',0,b'\x00\x01\x13\x23getImpliedFileFormat',0,b'\x00\x00\xB8\x23getLeptonicaVersion',0,b'\x00\x01\x3E\x23l_CIDataDestroy',0,b'\x00\x01\x16\x23l_generateCIDataForPdf',0,b'\x00\x01\x4D\x23lept_free',0,b'\x00\x00\xBA\x23makePixelSumTab8',0,b'\x00\x00\x2B\x23pixAnd',0,b'\x00\x00\x38\x23pixBackgroundNorm',0,b'\x00\x00\x30\x23pixCleanBackgroundToWhite',0,b'\x00\x00\x22\x23pixClipRectangle',0,b'\x00\x00\xFB\x23pixColorFraction',0,b'\x00\x00\x7F\x23pixColorMagnitude',0,b'\x00\x00\x1F\x23pixConvertRGBToLuminance',0,b'\x00\x00\x76\x23pixConvertTo8',0,b'\x00\x00\xCD\x23pixCorrelationBinary',0,b'\x00\x00\xE7\x23pixCountPixels',0,b'\x00\x00\x92\x23pixDeserializeFromMemory',0,b'\x00\x00\x76\x23pixDeskew',0,b'\x00\x01\x41\x23pixDestroy',0,b'\x00\x00\x44\x23pixDilate',0,b'\x00\x00\x1F\x23pixEndianByteSwapNew',0,b'\x00\x00\xD2\x23pixEqual',0,b'\x00\x00\x44\x23pixErode',0,b'\x00\x00\x96\x23pixExtractBarcodes',0,b'\x00\x00\x08\x23pixFindPageForeground',0,b'\x00\x00\xE2\x23pixFindSkew',0,b'\x00\x00\x49\x23pixGammaTRC',0,b'\x00\x00\xF4\x23pixGenerateCIData',0,b'\x00\x00\xD7\x23pixGetAverageMaskedRGB',0,b'\x00\x00\x50\x23pixGlobalNormRGB',0,b'\x00\x00\x44\x23pixHMT',0,b'\x00\x00\x27\x23pixInvert',0,b'\x00\x00\x15\x23pixLocateBarcodes',0,b'\x00\x00\x7A\x23pixMaskOverColorPixels',0,b'\x00\x00\x58\x23pixMaskedThreshOnBackgroundNorm',0,b'\x00\x00\xEC\x23pixNumSignificantGrayColors',0,b'\x00\x01\x04\x23pixOtsuAdaptiveThreshold',0,b'\x00\x00\x64\x23pixOtsuThreshOnBackgroundNorm',0,b'\x00\x00\x9A\x23pixProcessBarcodes',0,b'\x00\x00\x8B\x23pixRead',0,b'\x00\x00\xA1\x23pixReadBarcodes',0,b'\x00\x00\x8E\x23pixReadMem',0,b'\x00\x00\x1B\x23pixReadStream',0,b'\x00\x00\x76\x23pixRemoveColormap',0,b'\x00\x00\x7A\x23pixRemoveColormapGeneral',0,b'\x00\x00\xC7\x23pixRenderBoxa',0,b'\x00\x00\x27\x23pixRotate180',0,b'\x00\x00\x76\x23pixRotateOrth',0,b'\x00\x00\x71\x23pixScale',0,b'\x00\x01\x0E\x23pixSerializeToMemory',0,b'\x00\x00\x2B\x23pixSubtract',0,b'\x00\x01\x1C\x23pixWriteImpliedFormat',0,b'\x00\x01\x2B\x23pixWriteMem',0,b'\x00\x01\x31\x23pixWriteMemJpeg',0,b'\x00\x01\x25\x23pixWriteMemPng',0,b'\x00\x00\xBC\x23pixWriteStream',0,b'\x00\x00\xC1\x23pixWriteStreamJpeg',0,b'\x00\x01\x44\x23pixaDestroy',0,b'\x00\x00\x10\x23pixaGetBox',0,b'\x00\x00\x86\x23pixaGetPix',0,b'\x00\x01\x47\x23sarrayDestroy',0,b'\x00\x00\xAE\x23selCreateBrick',0,b'\x00\x00\xA8\x23selCreateFromString',0,b'\x00\x01\x4A\x23selDestroy',0,b'\x00\x00\xB5\x23selPrintToString',0,b'\x00\x01\x22\x23setMsgSeverity',0), + _struct_unions = ((b'\x00\x00\x01\x50\x00\x00\x00\x02Box',b'\x00\x00\x05\x11x',b'\x00\x00\x05\x11y',b'\x00\x00\x05\x11w',b'\x00\x00\x05\x11h',b'\x00\x01\x72\x11refcount'),(b'\x00\x00\x01\x51\x00\x00\x00\x02Boxa',b'\x00\x00\x05\x11n',b'\x00\x00\x05\x11nalloc',b'\x00\x01\x72\x11refcount',b'\x00\x00\x25\x11box'),(b'\x00\x00\x01\x54\x00\x00\x00\x02L_Compressed_Data',b'\x00\x00\x05\x11type',b'\x00\x01\x6F\x11datacomp',b'\x00\x00\x90\x11nbytescomp',b'\x00\x01\x5D\x11data85',b'\x00\x00\x90\x11nbytes85',b'\x00\x01\x5D\x11cmapdata85',b'\x00\x01\x5D\x11cmapdatahex',b'\x00\x00\x05\x11ncolors',b'\x00\x00\x05\x11w',b'\x00\x00\x05\x11h',b'\x00\x00\x05\x11bps',b'\x00\x00\x05\x11spp',b'\x00\x00\x05\x11minisblack',b'\x00\x00\x05\x11predictor',b'\x00\x00\x90\x11nbytes',b'\x00\x00\x05\x11res'),(b'\x00\x00\x01\x55\x00\x00\x00\x02Pix',b'\x00\x01\x72\x11w',b'\x00\x01\x72\x11h',b'\x00\x01\x72\x11d',b'\x00\x01\x72\x11spp',b'\x00\x01\x72\x11wpl',b'\x00\x01\x72\x11refcount',b'\x00\x00\x05\x11xres',b'\x00\x00\x05\x11yres',b'\x00\x00\x05\x11informat',b'\x00\x00\x05\x11special',b'\x00\x01\x5D\x11text',b'\x00\x01\x6B\x11colormap',b'\x00\x01\x71\x11data'),(b'\x00\x00\x01\x58\x00\x00\x00\x02PixColormap',b'\x00\x01\x4E\x11array',b'\x00\x00\x05\x11depth',b'\x00\x00\x05\x11nalloc',b'\x00\x00\x05\x11n'),(b'\x00\x00\x01\x6E\x00\x00\x00\x10PixComp',),(b'\x00\x00\x01\x56\x00\x00\x00\x02Pixa',b'\x00\x00\x05\x11n',b'\x00\x00\x05\x11nalloc',b'\x00\x01\x72\x11refcount',b'\x00\x00\x18\x11pix',b'\x00\x00\x04\x11boxa'),(b'\x00\x00\x01\x57\x00\x00\x00\x02PixaComp',b'\x00\x00\x05\x11n',b'\x00\x00\x05\x11nalloc',b'\x00\x00\x05\x11offset',b'\x00\x01\x6C\x11pixc',b'\x00\x00\x04\x11boxa'),(b'\x00\x00\x01\x5A\x00\x00\x00\x02Sarray',b'\x00\x00\x05\x11nalloc',b'\x00\x00\x05\x11n',b'\x00\x00\x05\x11refcount',b'\x00\x01\x5C\x11array'),(b'\x00\x00\x01\x5B\x00\x00\x00\x02Sel',b'\x00\x00\x05\x11sy',b'\x00\x00\x05\x11sx',b'\x00\x00\x05\x11cy',b'\x00\x00\x05\x11cx',b'\x00\x01\x67\x11data',b'\x00\x01\x5D\x11name'),(b'\x00\x00\x01\x52\x00\x00\x00\x10_IO_FILE',)), + _enums = (b'\x00\x00\x01\x60\x00\x00\x00\x16$1\x00L_DEFAULT_ENCODE,L_JPEG_ENCODE,L_G4_ENCODE,L_FLATE_ENCODE,L_JP2K_ENCODE',b'\x00\x00\x01\x61\x00\x00\x00\x16$2\x00REMOVE_CMAP_TO_BINARY,REMOVE_CMAP_TO_GRAYSCALE,REMOVE_CMAP_TO_FULL_COLOR,REMOVE_CMAP_WITH_ALPHA,REMOVE_CMAP_BASED_ON_SRC',b'\x00\x00\x01\x62\x00\x00\x00\x16$3\x00L_NOCOPY,L_INSERT,L_COPY,L_CLONE,L_COPY_CLONE',b'\x00\x00\x01\x63\x00\x00\x00\x16$4\x00L_USE_WIDTHS,L_USE_WINDOWS',b'\x00\x00\x01\x64\x00\x00\x00\x16$5\x00L_BF_UNKNOWN,L_BF_ANY,L_BF_CODE128,L_BF_EAN8,L_BF_EAN13,L_BF_CODE2OF5,L_BF_CODEI2OF5,L_BF_CODE39,L_BF_CODE93,L_BF_CODABAR,L_BF_UPCA',b'\x00\x00\x01\x65\x00\x00\x00\x16$6\x00L_SEVERITY_EXTERNAL,L_SEVERITY_ALL,L_SEVERITY_DEBUG,L_SEVERITY_INFO,L_SEVERITY_WARNING,L_SEVERITY_ERROR,L_SEVERITY_NONE',b'\x00\x00\x01\x66\x00\x00\x00\x16$7\x00SEL_DONT_CARE,SEL_HIT,SEL_MISS'), + _typenames = (b'\x00\x00\x01\x50BOX',b'\x00\x00\x01\x51BOXA',b'\x00\x00\x01\x52FILE',b'\x00\x00\x01\x54L_COMP_DATA',b'\x00\x00\x01\x55PIX',b'\x00\x00\x01\x56PIXA',b'\x00\x00\x01\x57PIXAC',b'\x00\x00\x01\x58PIXCMAP',b'\x00\x00\x01\x5ASARRAY',b'\x00\x00\x01\x5BSEL',b'\x00\x00\x00\x34l_float32',b'\x00\x00\x01\x5Fl_float64',b'\x00\x00\x01\x69l_int16',b'\x00\x00\x00\x05l_int32',b'\x00\x00\x01\x68l_int64',b'\x00\x00\x01\x6Al_int8',b'\x00\x00\x00\x05l_ok',b'\x00\x00\x01\x74l_uint16',b'\x00\x00\x01\x72l_uint32',b'\x00\x00\x01\x73l_uint64',b'\x00\x00\x01\x70l_uint8'), ) diff --git a/src/ocrmypdf/lib/compile_leptonica.py b/src/ocrmypdf/lib/compile_leptonica.py index e6d3c9f3..4d50943d 100644 --- a/src/ocrmypdf/lib/compile_leptonica.py +++ b/src/ocrmypdf/lib/compile_leptonica.py @@ -16,6 +16,8 @@ # You should have received a copy of the GNU General Public License # along with OCRmyPDF. If not, see . +from pathlib import Path + from cffi import FFI ffibuilder = FFI() @@ -221,9 +223,15 @@ ffibuilder.cdef( """ PIX * pixRead ( const char *filename ); PIX * pixReadMem ( const l_uint8 *data, size_t size ); +PIX * pixReadStream ( FILE *fp, l_int32 hint ); PIX * pixScale ( PIX *pixs, l_float32 scalex, l_float32 scaley ); l_int32 pixFindSkew ( PIX *pixs, l_float32 *pangle, l_float32 *pconf ); l_int32 pixWriteImpliedFormat ( const char *filename, PIX *pix, l_int32 quality, l_int32 progressive ); +l_int32 getImpliedFileFormat ( const char *filename ); +l_ok pixWriteStream ( FILE *fp, PIX *pix, l_int32 format ); +l_ok pixWriteStreamJpeg ( FILE *fp, PIX *pixs, l_int32 quality, l_int32 progressive ); +l_ok pixWriteMem ( l_uint8 **pdata, size_t *psize, PIX *pix, l_int32 format ); +l_ok pixWriteMemJpeg ( l_uint8 **pdata, size_t *psize, PIX *pix, l_int32 quality, l_int32 progressive ); l_int32 pixWriteMemPng(l_uint8 **pdata, size_t *psize, diff --git a/tests/test_lept.py b/tests/test_lept.py index b74f417b..116060c3 100644 --- a/tests/test_lept.py +++ b/tests/test_lept.py @@ -94,16 +94,6 @@ def test_leptonica_compile(tmp_path): ffibuilder.compile(tmpdir=fspath(tmp_path), target=fspath(tmp_path / 'lepttest.*')) -def test_with_stderr(capsys): - # pytest redirects stderr too; we must disable this for the test to be valid - with capsys.disabled(): - with pytest.raises(FileNotFoundError): - lept.Pix.open("does_not_exist1") - - -def test_without_stderr(capsys): - # pytest redirects stderr too; we must disable this for the test to be valid - with capsys.disabled(): - with patch('sys.stderr', new=None): - with pytest.raises(FileNotFoundError): - lept.Pix.open("does_not_exist2") +def test_file_not_found(): + with pytest.raises(FileNotFoundError): + lept.Pix.open("does_not_exist1") From 17d20309c7f06baec8a53b3cefd8fda961bfa804 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Mon, 25 Nov 2019 14:17:32 -0800 Subject: [PATCH 09/35] leptonica: fix missing Leptonica error message for Windows Since it has the unintuitive fix of adding Tesseract to PATH. --- src/ocrmypdf/leptonica.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/ocrmypdf/leptonica.py b/src/ocrmypdf/leptonica.py index ca0c546e..6fbf2d99 100644 --- a/src/ocrmypdf/leptonica.py +++ b/src/ocrmypdf/leptonica.py @@ -34,6 +34,7 @@ from os import fspath from tempfile import TemporaryFile from .lib._leptonica import ffi +from .exceptions import MissingDependencyError # pylint: disable=protected-access @@ -43,7 +44,12 @@ if os.name == 'nt': libname = 'liblept-5' else: libname = 'lept' -lept = ffi.dlopen(find_library(libname)) +_libpath = find_library(libname) +if not _libpath and os.name == 'nt': + raise MissingDependencyError( + "Please ensure that 'tesseract' is on your PATH environment variable. " + ) +lept = ffi.dlopen(_libpath) lept.setMsgSeverity(lept.L_SEVERITY_WARNING) From e63503d64bbd4505a802509c7ee0eb13e7fbb866 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Wed, 27 Nov 2019 01:12:21 -0800 Subject: [PATCH 10/35] Fix difference in Windows error message breaking test_no_languages --- src/ocrmypdf/exec/tesseract.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/ocrmypdf/exec/tesseract.py b/src/ocrmypdf/exec/tesseract.py index cb837dfd..4b0560d8 100644 --- a/src/ocrmypdf/exec/tesseract.py +++ b/src/ocrmypdf/exec/tesseract.py @@ -127,9 +127,10 @@ def languages(tesseract_env=None): except CalledProcessError as e: raise MissingDependencyError(lang_error(e.output)) from e + for line in output.splitlines(): + if line.startswith('Error'): + raise MissingDependencyError(lang_error(output)) header, *rest = output.splitlines() - if not header.startswith('List of available languages'): - raise MissingDependencyError(lang_error(output)) return set(lang.strip() for lang in rest) From 3f92867ae678da3270722bc24e4ec9b3b26383c4 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Tue, 19 Nov 2019 18:01:10 -0800 Subject: [PATCH 11/35] Fix TypeError "environment can only contain strings" Apparently Windows Python doesn't coerce pathlib.Path to str. --- src/ocrmypdf/api.py | 2 +- tests/conftest.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ocrmypdf/api.py b/src/ocrmypdf/api.py index b735f398..a05edbc1 100644 --- a/src/ocrmypdf/api.py +++ b/src/ocrmypdf/api.py @@ -158,7 +158,7 @@ def create_options(*, input_file, output_file, **kwargs): # If we are running a Tesseract spoof, ensure it knows what the input file is if os.environ.get('PYTEST_CURRENT_TEST') and options.tesseract_env: - options.tesseract_env['_OCRMYPDF_TEST_INFILE'] = input_file + options.tesseract_env['_OCRMYPDF_TEST_INFILE'] = os.fspath(input_file) return options diff --git a/tests/conftest.py b/tests/conftest.py index b443ac1f..e875315a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -183,7 +183,7 @@ def check_ocrmypdf(input_file, output_file, *args, env=None): api.check_options(options) if env: options.tesseract_env = env - options.tesseract_env['_OCRMYPDF_TEST_INFILE'] = input_file + options.tesseract_env['_OCRMYPDF_TEST_INFILE'] = os.fspath(input_file) result = api.run_pipeline(options, api=True) assert result == 0 From 37f6f72df3aad5e24fb1db1792371ef87ed038a5 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Tue, 19 Nov 2019 18:07:33 -0800 Subject: [PATCH 12/35] tests: a few Windows fixes --- tests/test_main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index c55186b0..a2ba24cb 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -586,7 +586,7 @@ def test_overlay(spoof_tesseract_noop, resources, outpdf): def test_destination_not_writable(spoof_tesseract_noop, resources, outdir): - if os.getuid() == 0 or os.geteuid() == 0: + if os.name != 'nt' and (os.getuid() == 0 or os.geteuid() == 0): pytest.xfail(reason="root can write to anything") protected_file = outdir / 'protected.pdf' protected_file.touch() @@ -872,7 +872,7 @@ def test_sidecar_pagecount(spoof_tesseract_cache, resources, outpdf): pdfinfo = PdfInfo(resources / 'multipage.pdf') num_pages = len(pdfinfo) - with open(sidecar, 'r') as f: + with open(sidecar, 'r', encoding='utf-8') as f: ocr_text = f.read() # There should a formfeed between each pair of pages, so the count of @@ -888,7 +888,7 @@ def test_sidecar_nonempty(spoof_tesseract_cache, resources, outpdf): resources / 'ccitt.pdf', outpdf, '--sidecar', sidecar, env=spoof_tesseract_cache ) - with open(sidecar, 'r') as f: + with open(sidecar, 'r', encoding='utf-8') as f: ocr_text = f.read() assert 'the' in ocr_text From 4ab0a8ff35a5a96728ac6b6cca0e711b6c640d05 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Wed, 27 Nov 2019 02:26:13 -0800 Subject: [PATCH 13/35] Fix test_single_page_inline_image - remove temp file --- tests/test_pdfinfo.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/test_pdfinfo.py b/tests/test_pdfinfo.py index 81669a62..a775e950 100644 --- a/tests/test_pdfinfo.py +++ b/tests/test_pdfinfo.py @@ -91,21 +91,21 @@ def test_single_page_image(outdir): def test_single_page_inline_image(outdir): filename = outdir / 'image-mono-inline.pdf' pdf = Canvas(str(filename), pagesize=(8 * 72, 6 * 72)) - with NamedTemporaryFile() as im_tmp: - im = Image.new('1', (8, 8), 0) - for n in range(8): - im.putpixel((n, n), 1) - im.save(im_tmp.name, format='PNG') - # Draw image in a 72x72 pt or 1"x1" area - pdf.drawInlineImage(im_tmp.name, 0, 0, width=72, height=72) - pdf.showPage() - pdf.save() - pdf = pdfinfo.PdfInfo(filename) - print(pdf) - pdfimage = pdf[0].images[0] + im = Image.new('1', (8, 8), 0) + for n in range(8): + im.putpixel((n, n), 1) + + # Draw image in a 72x72 pt or 1"x1" area + pdf.drawInlineImage(im, 0, 0, width=72, height=72) + pdf.showPage() + pdf.save() + + info = pdfinfo.PdfInfo(filename) + print(info) + pdfimage = info[0].images[0] assert isclose(pdfimage.xres, 8) - assert pdfimage.color == Colorspace.rgb # reportlab produces color image + assert pdfimage.color == Colorspace.gray assert pdfimage.width == 8 From a3726e4ce3ba092b8f9981814fa690aa89ca6670 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Wed, 27 Nov 2019 02:34:53 -0800 Subject: [PATCH 14/35] Fix test_metadata: use mmap in a Windows and POSIX compatible way --- tests/test_metadata.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_metadata.py b/tests/test_metadata.py index e0650a34..6d33db12 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -21,6 +21,7 @@ from datetime import timezone import logging import mmap from os import fspath +import os from pathlib import Path from shutil import copyfile, move from unittest.mock import MagicMock, patch @@ -330,10 +331,8 @@ def test_prevent_gs_invalid_xml(resources, outdir): str(outdir / 'layers.rendered.pdf'), str(outdir / 'pdfa.ps'), context ) - with open(outdir / 'pdfa.pdf', 'rb') as f: - with mmap.mmap( - f.fileno(), 0, flags=mmap.MAP_PRIVATE, prot=mmap.PROT_READ - ) as mm: + with open(outdir / 'pdfa.pdf', 'r+b') as f: + with mmap.mmap(f.fileno(), 0) as mm: # Since the XML may be invalid, we scan instead of actually feeding it # to a parser. XMP_MAGIC = b'W5M0MpCehiHzreSzNTczkc9d' From fde550f9a708d62bde595b4a056cac5d51f3f1a6 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Thu, 28 Nov 2019 14:03:18 -0800 Subject: [PATCH 15/35] test: Replace many instances of run_ocrmypdf in subprocess with inline --- tests/conftest.py | 16 ++++++++++ tests/test_main.py | 69 +++++++++++++++++++++--------------------- tests/test_stdio.py | 1 + tests/test_userunit.py | 8 +++-- 4 files changed, 57 insertions(+), 37 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index e875315a..1a63a5d2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -193,6 +193,22 @@ def check_ocrmypdf(input_file, output_file, *args, env=None): return output_file +@pytest.helpers.register +def run_ocrmypdf_api(input_file, output_file, *args, env=None): + "Run ocrmypdf and let caller deal with results" + + options = cli.parser.parse_args( + [str(input_file), str(output_file)] + + [str(arg) for arg in args if arg is not None] + ) + api.check_options(options) + if env: + options.tesseract_env = env + options.tesseract_env['_OCRMYPDF_TEST_INFILE'] = os.fspath(input_file) + + return api.run_pipeline(options, api=False) + + @pytest.helpers.register def run_ocrmypdf(input_file, output_file, *args, env=None, universal_newlines=True): "Run ocrmypdf and let caller deal with results" diff --git a/tests/test_main.py b/tests/test_main.py index a2ba24cb..77ea154e 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -39,6 +39,7 @@ from ocrmypdf.pdfinfo import Colorspace, Encoding, PdfInfo check_ocrmypdf = pytest.helpers.check_ocrmypdf run_ocrmypdf = pytest.helpers.run_ocrmypdf +run_ocrmypdf_api = pytest.helpers.run_ocrmypdf_api spoof = pytest.helpers.spoof @@ -197,8 +198,8 @@ def test_oversample(spoof_tesseract_cache, renderer, resources, outpdf): def test_repeat_ocr(resources, no_outpdf): - p, _, _ = run_ocrmypdf(resources / 'graph_ocred.pdf', no_outpdf) - assert p.returncode != 0 + result = run_ocrmypdf_api(resources / 'graph_ocred.pdf', no_outpdf) + assert result == ExitCode.already_done_ocr def test_force_ocr(spoof_tesseract_cache, resources, outpdf): @@ -300,34 +301,34 @@ def test_maximum_options( ) -def test_tesseract_missing_tessdata(resources, no_outpdf): +def test_tesseract_missing_tessdata(resources, no_outpdf, tmpdir): env = os.environ.copy() - env['TESSDATA_PREFIX'] = '/tmp' + env['TESSDATA_PREFIX'] = tmpdir - p, _, err = run_ocrmypdf( - resources / 'graph_ocred.pdf', no_outpdf, '-v', '1', '--skip-text', env=env + returncode = run_ocrmypdf_api( + resources / 'graph.pdf', no_outpdf, '-v', '1', '--skip-text', env=env ) - assert p.returncode == ExitCode.missing_dependency, err + assert returncode == ExitCode.missing_dependency def test_invalid_input_pdf(resources, no_outpdf): - p, out, err = run_ocrmypdf(resources / 'invalid.pdf', no_outpdf) - assert p.returncode == ExitCode.input_file, err + result = run_ocrmypdf_api(resources / 'invalid.pdf', no_outpdf) + assert result == ExitCode.input_file def test_blank_input_pdf(resources, outpdf): - p, out, err = run_ocrmypdf(resources / 'blank.pdf', outpdf) - assert p.returncode == ExitCode.ok + result = run_ocrmypdf_api(resources / 'blank.pdf', outpdf) + assert result == ExitCode.ok def test_force_ocr_on_pdf_with_no_images(spoof_tesseract_crash, resources, no_outpdf): # As a correctness test, make sure that --force-ocr on a PDF with no # content still triggers tesseract. If tesseract crashes, then it was # called. - p, _, err = run_ocrmypdf( + result = run_ocrmypdf_api( resources / 'blank.pdf', no_outpdf, '--force-ocr', env=spoof_tesseract_crash ) - assert p.returncode == ExitCode.child_process_error, err + assert result == ExitCode.child_process_error assert not os.path.exists(no_outpdf) @@ -340,7 +341,7 @@ def test_german(spoof_tesseract_cache, resources, outdir): # properly. It is fine that we are testing -l deu on a French file because # we are exercising the functionality not going for accuracy. sidecar = outdir / 'francais.txt' - p, out, err = run_ocrmypdf( + result = run_ocrmypdf_api( resources / 'francais.pdf', outdir / 'francais.pdf', '-l', @@ -351,16 +352,16 @@ def test_german(spoof_tesseract_cache, resources, outdir): ) if 'deu' not in tesseract.languages(): pytest.xfail(reason="tesseract-deu language pack not installed") - assert p.returncode == ExitCode.ok, "Requires tesseract deu language pack" + assert result == ExitCode.ok, "Requires tesseract deu language pack" def test_klingon(resources, outpdf): - p, out, err = run_ocrmypdf(resources / 'francais.pdf', outpdf, '-l', 'klz') + p, _, _ = run_ocrmypdf(resources / 'francais.pdf', outpdf, '-l', 'klz') assert p.returncode == ExitCode.missing_dependency def test_missing_docinfo(spoof_tesseract_noop, resources, outpdf): - p, out, err = run_ocrmypdf( + result = run_ocrmypdf_api( resources / 'missing_docinfo.pdf', outpdf, '-l', @@ -368,7 +369,7 @@ def test_missing_docinfo(spoof_tesseract_noop, resources, outpdf): '--skip-text', env=spoof_tesseract_noop, ) - assert p.returncode == ExitCode.ok, err + assert result == ExitCode.ok def test_uppercase_extension(spoof_tesseract_noop, resources, outdir): @@ -379,24 +380,24 @@ def test_uppercase_extension(spoof_tesseract_noop, resources, outdir): ) -def test_input_file_not_found(no_outpdf): +def test_input_file_not_found(caplog, no_outpdf): input_file = "does not exist.pdf" - p, out, err = run_ocrmypdf(input_file, no_outpdf) - assert p.returncode == ExitCode.input_file - assert input_file in out or input_file in err + result = run_ocrmypdf_api(input_file, no_outpdf) + assert result == ExitCode.input_file + assert input_file in caplog.text -def test_input_file_not_a_pdf(no_outpdf): +def test_input_file_not_a_pdf(caplog, no_outpdf): input_file = __file__ # Try to OCR this file - p, out, err = run_ocrmypdf(input_file, no_outpdf) - assert p.returncode == ExitCode.input_file - assert input_file in out or input_file in err + result = run_ocrmypdf_api(input_file, no_outpdf) + assert result == ExitCode.input_file + assert input_file in caplog.text -def test_encrypted(resources, no_outpdf): - p, out, err = run_ocrmypdf(resources / 'skew-encrypted.pdf', no_outpdf) - assert p.returncode == ExitCode.encrypted_pdf - assert out.find('encrypted') +def test_encrypted(resources, caplog, no_outpdf): + result = run_ocrmypdf_api(resources / 'skew-encrypted.pdf', no_outpdf) + assert result == ExitCode.encrypted_pdf + assert 'encryption must be removed' in caplog.text @pytest.mark.parametrize('renderer', RENDERERS) @@ -415,8 +416,8 @@ def test_pagesegmode(renderer, spoof_tesseract_cache, resources, outpdf): @pytest.mark.parametrize('renderer', RENDERERS) -def test_tesseract_crash(renderer, spoof_tesseract_crash, resources, no_outpdf): - p, out, err = run_ocrmypdf( +def test_tesseract_crash(renderer, spoof_tesseract_crash, resources, no_outpdf, caplog): + result = run_ocrmypdf_api( resources / 'ccitt.pdf', no_outpdf, '-v', @@ -425,9 +426,9 @@ def test_tesseract_crash(renderer, spoof_tesseract_crash, resources, no_outpdf): renderer, env=spoof_tesseract_crash, ) - assert p.returncode == ExitCode.child_process_error + assert result == ExitCode.child_process_error assert not os.path.exists(no_outpdf) - assert "ERROR" in err + assert "SubprocessOutputError" in caplog.text def test_tesseract_crash_autorotate(spoof_tesseract_crash, resources, no_outpdf): diff --git a/tests/test_stdio.py b/tests/test_stdio.py index 76c99d56..a0f07ddb 100644 --- a/tests/test_stdio.py +++ b/tests/test_stdio.py @@ -29,6 +29,7 @@ from ocrmypdf.exec import qpdf # pylint: disable=no-member,redefined-outer-name run_ocrmypdf = pytest.helpers.run_ocrmypdf +run_ocrmypdf_api = pytest.helpers.run_ocrmypdf spoof = pytest.helpers.spoof diff --git a/tests/test_userunit.py b/tests/test_userunit.py index 81282431..83ad01d4 100644 --- a/tests/test_userunit.py +++ b/tests/test_userunit.py @@ -24,6 +24,7 @@ from ocrmypdf.pdfinfo import PdfInfo check_ocrmypdf = pytest.helpers.check_ocrmypdf run_ocrmypdf = pytest.helpers.run_ocrmypdf +run_ocrmypdf_api = pytest.helpers.run_ocrmypdf_api spoof = pytest.helpers.spoof @@ -32,9 +33,10 @@ def poster(resources): return resources / 'poster.pdf' -def test_userunit_ghostscript_fails(poster, no_outpdf): - p, out, err = run_ocrmypdf(poster, no_outpdf, '--output-type=pdfa') - assert p.returncode == ExitCode.input_file +def test_userunit_ghostscript_fails(poster, no_outpdf, caplog): + result = run_ocrmypdf_api(poster, no_outpdf, '--output-type=pdfa') + assert result == ExitCode.input_file + assert 'not supported by Ghostscript' in caplog.text def test_userunit_qpdf_passes(spoof_tesseract_cache, poster, outpdf): From 0cd424ffcbecf3b85e30c6476f1145e68b278549 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Thu, 28 Nov 2019 14:44:32 -0800 Subject: [PATCH 16/35] Enforce str-only environment for Windows since it's more strict --- tests/conftest.py | 2 ++ tests/test_main.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 1a63a5d2..9f796d3a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -205,6 +205,8 @@ def run_ocrmypdf_api(input_file, output_file, *args, env=None): if env: options.tesseract_env = env options.tesseract_env['_OCRMYPDF_TEST_INFILE'] = os.fspath(input_file) + if options.tesseract_env: + assert all(isinstance(v, (str, bytes)) for v in options.tesseract_env.values()) return api.run_pipeline(options, api=False) diff --git a/tests/test_main.py b/tests/test_main.py index 77ea154e..17946965 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -303,7 +303,7 @@ def test_maximum_options( def test_tesseract_missing_tessdata(resources, no_outpdf, tmpdir): env = os.environ.copy() - env['TESSDATA_PREFIX'] = tmpdir + env['TESSDATA_PREFIX'] = os.fspath(tmpdir) returncode = run_ocrmypdf_api( resources / 'graph.pdf', no_outpdf, '-v', '1', '--skip-text', env=env From 8a1dddc3eeec9e7ca3bdbe9921ba72346d40bf48 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Thu, 28 Nov 2019 14:45:03 -0800 Subject: [PATCH 17/35] Don't worry about closed streams on Windows --- tests/test_stdio.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_stdio.py b/tests/test_stdio.py index a0f07ddb..53c0398f 100644 --- a/tests/test_stdio.py +++ b/tests/test_stdio.py @@ -77,6 +77,7 @@ def test_stdout(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf): @pytest.mark.skipif( sys.version_info[0:3] >= (3, 6, 4), reason="issue fixed in Python 3.6.4" ) +@pytest.mark.skipif(os.name == 'nt', reason="POSIX problem") def test_closed_streams(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf): input_file = str(resources / 'francais.pdf') output_file = str(outpdf) From ca9669742d632ff1c773e45921d1c98401727441 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Thu, 28 Nov 2019 16:19:58 -0800 Subject: [PATCH 18/35] Move gs tests to test_ghostscript --- tests/test_ghostscript.py | 64 +++++++++++++++++++++++++++++++++++++++ tests/test_main.py | 57 ---------------------------------- 2 files changed, 64 insertions(+), 57 deletions(-) diff --git a/tests/test_ghostscript.py b/tests/test_ghostscript.py index 8756e8a8..c659fd68 100644 --- a/tests/test_ghostscript.py +++ b/tests/test_ghostscript.py @@ -18,12 +18,47 @@ import logging from decimal import Decimal + import pikepdf import pytest from PIL import Image +from ocrmypdf.exceptions import ExitCode from ocrmypdf.exec.ghostscript import rasterize_pdf +check_ocrmypdf = pytest.helpers.check_ocrmypdf +run_ocrmypdf = pytest.helpers.run_ocrmypdf +run_ocrmypdf_api = pytest.helpers.run_ocrmypdf_api +spoof = pytest.helpers.spoof + + +@pytest.fixture(scope='session') +def spoof_no_tess_gs_render_fail(tmp_path_factory): + return spoof( + tmp_path_factory, tesseract='tesseract_noop.py', gs='gs_render_failure.py' + ) + + +@pytest.fixture(scope='session') +def spoof_no_tess_gs_raster_fail(tmp_path_factory): + return spoof( + tmp_path_factory, tesseract='tesseract_noop.py', gs='gs_raster_failure.py' + ) + + +@pytest.fixture(scope='session') +def spoof_no_tess_no_pdfa(tmp_path_factory): + return spoof( + tmp_path_factory, tesseract='tesseract_noop.py', gs='gs_pdfa_failure.py' + ) + + +@pytest.fixture(scope='session') +def spoof_no_tess_pdfa_warning(tmp_path_factory): + return spoof( + tmp_path_factory, tesseract='tesseract_noop.py', gs='gs_feature_elision.py' + ) + @pytest.fixture def linn(resources): @@ -79,3 +114,32 @@ def test_rasterize_rotated(linn, outdir, caplog): with Image.open(outdir / 'out.png') as im: assert im.size == (target_size[1], target_size[0]) assert im.info['dpi'] == (target_dpi[1], target_dpi[0]) + + +def test_gs_render_failure(spoof_no_tess_gs_render_fail, resources, outpdf): + p, out, err = run_ocrmypdf( + resources / 'blank.pdf', outpdf, env=spoof_no_tess_gs_render_fail + ) + print(err) + assert p.returncode == ExitCode.child_process_error + + +def test_gs_raster_failure(spoof_no_tess_gs_raster_fail, resources, outpdf): + p, out, err = run_ocrmypdf( + resources / 'ccitt.pdf', outpdf, env=spoof_no_tess_gs_raster_fail + ) + print(err) + assert p.returncode == ExitCode.child_process_error + + +def test_ghostscript_pdfa_failure(spoof_no_tess_no_pdfa, resources, outpdf): + p, out, err = run_ocrmypdf( + resources / 'ccitt.pdf', outpdf, env=spoof_no_tess_no_pdfa + ) + assert ( + p.returncode == ExitCode.pdfa_conversion_failed + ), "Unexpected return when PDF/A fails" + + +def test_ghostscript_feature_elision(spoof_no_tess_pdfa_warning, resources, outpdf): + check_ocrmypdf(resources / 'ccitt.pdf', outpdf, env=spoof_no_tess_pdfa_warning) diff --git a/tests/test_main.py b/tests/test_main.py index 17946965..73f91e5e 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -56,34 +56,6 @@ def spoof_tesseract_big_image_error(tmp_path_factory): return spoof(tmp_path_factory, tesseract='tesseract_big_image_error.py') -@pytest.fixture(scope='session') -def spoof_no_tess_no_pdfa(tmp_path_factory): - return spoof( - tmp_path_factory, tesseract='tesseract_noop.py', gs='gs_pdfa_failure.py' - ) - - -@pytest.fixture(scope='session') -def spoof_no_tess_pdfa_warning(tmp_path_factory): - return spoof( - tmp_path_factory, tesseract='tesseract_noop.py', gs='gs_feature_elision.py' - ) - - -@pytest.fixture(scope='session') -def spoof_no_tess_gs_render_fail(tmp_path_factory): - return spoof( - tmp_path_factory, tesseract='tesseract_noop.py', gs='gs_render_failure.py' - ) - - -@pytest.fixture(scope='session') -def spoof_no_tess_gs_raster_fail(tmp_path_factory): - return spoof( - tmp_path_factory, tesseract='tesseract_noop.py', gs='gs_raster_failure.py' - ) - - def test_quick(spoof_tesseract_cache, resources, outpdf): check_ocrmypdf(resources / 'ccitt.pdf', outpdf, env=spoof_tesseract_cache) @@ -557,19 +529,6 @@ def test_linearized_pdf_and_indirect_object(spoof_tesseract_noop, resources, out check_ocrmypdf(resources / 'epson.pdf', outpdf, env=spoof_tesseract_noop) -def test_ghostscript_pdfa_failure(spoof_no_tess_no_pdfa, resources, outpdf): - p, out, err = run_ocrmypdf( - resources / 'ccitt.pdf', outpdf, env=spoof_no_tess_no_pdfa - ) - assert ( - p.returncode == ExitCode.pdfa_conversion_failed - ), "Unexpected return when PDF/A fails" - - -def test_ghostscript_feature_elision(spoof_no_tess_pdfa_warning, resources, outpdf): - check_ocrmypdf(resources / 'ccitt.pdf', outpdf, env=spoof_no_tess_pdfa_warning) - - def test_very_high_dpi(spoof_tesseract_cache, resources, outpdf): "Checks for a Decimal quantize error with high DPI, etc" check_ocrmypdf(resources / '2400dpi.pdf', outpdf, env=spoof_tesseract_cache) @@ -718,22 +677,6 @@ def test_skip_big_with_no_images(spoof_tesseract_noop, resources, outpdf): ) -def test_gs_render_failure(spoof_no_tess_gs_render_fail, resources, outpdf): - p, out, err = run_ocrmypdf( - resources / 'blank.pdf', outpdf, env=spoof_no_tess_gs_render_fail - ) - print(err) - assert p.returncode == ExitCode.child_process_error - - -def test_gs_raster_failure(spoof_no_tess_gs_raster_fail, resources, outpdf): - p, out, err = run_ocrmypdf( - resources / 'ccitt.pdf', outpdf, env=spoof_no_tess_gs_raster_fail - ) - print(err) - assert p.returncode == ExitCode.child_process_error - - @pytest.mark.skipif( '8.0.0' <= qpdf.version() <= '8.0.1', reason="qpdf regression on pages with no contents", From 43ab7c88d7604d7c9b6f59d5eb15a21699879c42 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Thu, 28 Nov 2019 16:52:56 -0800 Subject: [PATCH 19/35] Remove os_environ() context manager --- src/ocrmypdf/_graft.py | 2 +- tests/conftest.py | 25 ------------------------- tests/test_graft.py | 7 ++----- 3 files changed, 3 insertions(+), 31 deletions(-) diff --git a/src/ocrmypdf/_graft.py b/src/ocrmypdf/_graft.py index 6a7cf23a..a535d492 100644 --- a/src/ocrmypdf/_graft.py +++ b/src/ocrmypdf/_graft.py @@ -21,7 +21,7 @@ from pathlib import Path import pikepdf -MAX_REPLACE_PAGES = int(os.environ.get('_OCRMYPDF_MAX_REPLACE_PAGES', 100)) +MAX_REPLACE_PAGES = 100 def _update_page_resources(*, page, font, font_key, procset): diff --git a/tests/conftest.py b/tests/conftest.py index 9f796d3a..3a2850a4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,7 +18,6 @@ import os import platform import sys -from contextlib import contextmanager from pathlib import Path from subprocess import PIPE, run from ocrmypdf import api, cli @@ -107,30 +106,6 @@ def spoof(tmp_path_factory, **kwargs): return env -@pytest.helpers.register -@contextmanager -def os_environ(new_env): - old_env = os.environ.copy() - if new_env is None: - new_env = {} - - for k, v in new_env.items(): - if k != 'PYTEST_CURRENT_TEST': - os.environ[k] = v - yield - new_keys = set(os.environ.copy()) - set(old_env) - for k in new_keys: - if k != 'PYTEST_CURRENT_TEST': - del os.environ[k] - for k in old_env: - if k != 'PYTEST_CURRENT_TEST': - os.environ[k] = old_env[k] - - for k, v in os.environ.copy().items(): - if k != 'PYTEST_CURRENT_TEST': - assert v == old_env[k] - - @pytest.fixture(scope='session') def spoof_tesseract_noop(tmp_path_factory): return spoof(tmp_path_factory, tesseract='tesseract_noop.py') diff --git a/tests/test_graft.py b/tests/test_graft.py index 2fd3d480..52aa336c 100644 --- a/tests/test_graft.py +++ b/tests/test_graft.py @@ -16,14 +16,13 @@ # along with OCRmyPDF. If not, see . import os +from unittest.mock import patch import pytest import ocrmypdf import pikepdf -os_environ = pytest.helpers.os_environ - def test_no_glyphless_graft(resources, outdir): pdf = pikepdf.open(resources / 'francais.pdf') @@ -33,9 +32,7 @@ def test_no_glyphless_graft(resources, outdir): pdf.pages.extend(pdf_cmyk.pages) pdf.save(outdir / 'test.pdf') - env = os.environ.copy() - env['_OCRMYPDF_MAX_REPLACE_PAGES'] = '2' - with os_environ(env): + with patch('ocrmypdf._graft.MAX_REPLACE_PAGES', 2): ocrmypdf.ocr( outdir / 'test.pdf', outdir / 'out.pdf', deskew=True, tesseract_timeout=0 ) From d249aef57d60fd38b07613c483aca5b669bbb7fa Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Wed, 13 Nov 2019 03:33:40 -0800 Subject: [PATCH 20/35] ghostscript: don't use NamedTemporaryFile Temporary files are more awkward for Windows. --- src/ocrmypdf/exec/ghostscript.py | 192 ++++++++++++++++--------------- 1 file changed, 99 insertions(+), 93 deletions(-) diff --git a/src/ocrmypdf/exec/ghostscript.py b/src/ocrmypdf/exec/ghostscript.py index 44fed271..65d15794 100644 --- a/src/ocrmypdf/exec/ghostscript.py +++ b/src/ocrmypdf/exec/ghostscript.py @@ -20,11 +20,12 @@ import logging import re import warnings +from contextlib import suppress from functools import lru_cache +from io import BytesIO from os import fspath -from shutil import copy -from subprocess import PIPE, STDOUT, run -from tempfile import NamedTemporaryFile +from pathlib import Path +from subprocess import PIPE, run from PIL import Image @@ -141,54 +142,57 @@ def rasterize_pdf( if not log: log = gslog - with NamedTemporaryFile(delete=True) as tmp: - args_gs = ( - [ - 'gs', - '-dQUIET', - '-dSAFER', - '-dBATCH', - '-dNOPAUSE', - f'-sDEVICE={raster_device}', - f'-dFirstPage={pageno}', - f'-dLastPage={pageno}', - f'-r{res[0]:f}x{res[1]:f}', - ] - + (['-dFILTERVECTOR'] if filter_vector else []) - + [ - '-o', - tmp.name, - '-dAutoRotatePages=/None', # Probably has no effect on raster - '-f', - fspath(input_file), - ] - ) + args_gs = ( + [ + 'gs', + '-dQUIET', + '-dSAFER', + '-dBATCH', + '-dNOPAUSE', + f'-sDEVICE={raster_device}', + f'-dFirstPage={pageno}', + f'-dLastPage={pageno}', + f'-r{res[0]:f}x{res[1]:f}', + ] + + (['-dFILTERVECTOR'] if filter_vector else []) + + [ + '-o', + '%stdout', + '-sstdout=%stderr', + '-dAutoRotatePages=/None', # Probably has no effect on raster + '-f', + fspath(input_file), + ] + ) - log.debug(args_gs) - p = run(args_gs, stdout=PIPE, stderr=STDOUT, universal_newlines=True) - if _gs_error_reported(p.stdout): - log.error(p.stdout) - elif p.stdout: - log.debug(p.stdout) + log.debug(args_gs) + with Path(output_file).open("wb") as output: + p = run(args_gs, stdout=PIPE, stderr=PIPE, check=False) + stderr = p.stderr.decode('utf-8', errors='replace') + if _gs_error_reported(stderr): + log.error(stderr) + elif stderr: + log.debug(stderr) - if p.returncode != 0: - raise SubprocessOutputError('Ghostscript rasterizing failed') + if p.returncode != 0: + with suppress(OSError): + Path(output_file).unlink() # no unfinished files + raise SubprocessOutputError('Ghostscript rasterizing failed') - tmp.seek(0) - with Image.open(tmp) as im: - if rotation is not None: - log.debug("Rotating output by %i", rotation) - # rotation is a clockwise angle and Image.ROTATE_* is - # counterclockwise so this cancels out the rotation - if rotation == 90: - im = im.transpose(Image.ROTATE_90) - elif rotation == 180: - im = im.transpose(Image.ROTATE_180) - elif rotation == 270: - im = im.transpose(Image.ROTATE_270) - if rotation % 180 == 90: - page_dpi = page_dpi[1], page_dpi[0] - im.save(fspath(output_file), dpi=page_dpi) + with Image.open(BytesIO(p.stdout)) as im: + if rotation is not None: + log.debug("Rotating output by %i", rotation) + # rotation is a clockwise angle and Image.ROTATE_* is + # counterclockwise so this cancels out the rotation + if rotation == 90: + im = im.transpose(Image.ROTATE_90) + elif rotation == 180: + im = im.transpose(Image.ROTATE_180) + elif rotation == 270: + im = im.transpose(Image.ROTATE_270) + if rotation % 180 == 90: + page_dpi = page_dpi[1], page_dpi[0] + im.save(fspath(output_file), dpi=page_dpi) def generate_pdfa( @@ -256,50 +260,52 @@ def generate_pdfa( # https://bugs.ghostscript.com/show_bug.cgi?id=699216 compression_args.append('-dPassThroughJPEGImages=false') - with NamedTemporaryFile(delete=True) as gs_pdf: - # nb no need to specify ProcessColorModel when ColorConversionStrategy - # is set; see: - # https://bugs.ghostscript.com/show_bug.cgi?id=699392 - args_gs = ( - [ - "gs", - "-dQUIET", - "-dBATCH", - "-dNOPAUSE", - "-dSAFER", - "-dCompatibilityLevel=" + str(pdf_version), - "-sDEVICE=pdfwrite", - "-dAutoRotatePages=/None", - "-sColorConversionStrategy=" + strategy, - ] - + compression_args - + [ - "-dJPEGQ=95", - "-dPDFA=" + pdfa_part, - "-dPDFACompatibilityPolicy=1", - "-sOutputFile=" + gs_pdf.name, - ] + # nb no need to specify ProcessColorModel when ColorConversionStrategy + # is set; see: + # https://bugs.ghostscript.com/show_bug.cgi?id=699392 + args_gs = ( + [ + "gs", + "-dQUIET", + "-dBATCH", + "-dNOPAUSE", + "-dSAFER", + "-dCompatibilityLevel=" + str(pdf_version), + "-sDEVICE=pdfwrite", + "-dAutoRotatePages=/None", + "-sColorConversionStrategy=" + strategy, + ] + + compression_args + + [ + "-dJPEGQ=95", + "-dPDFA=" + pdfa_part, + "-dPDFACompatibilityPolicy=1", + "-sOutputFile=%stdout", + "-sstdout=%stderr", + ] + ) + args_gs.extend(fspath(s) for s in pdf_pages) # Stringify Path objs + log.debug(args_gs) + with Path(output_file).open('wb') as output: + p = run(args_gs, stdout=output, stderr=PIPE, check=False) + + stderr = p.stderr.decode('utf-8', errors='replace') + if _gs_error_reported(stderr): + log.error(stderr) + elif 'overprint mode not set' in stderr: + # Unless someone is going to print PDF/A documents on a + # magical sRGB printer I can't see the removal of overprinting + # being a problem.... + log.debug( + "Ghostscript had to remove PDF 'overprinting' from the " + "input file to complete PDF/A conversion. " ) - args_gs.extend(fspath(s) for s in pdf_pages) # Stringify Path objs - log.debug(args_gs) - p = run(args_gs, stdout=PIPE, stderr=STDOUT, universal_newlines=True) + else: + log.debug(stderr) - if _gs_error_reported(p.stdout): - log.error(p.stdout) - elif 'overprint mode not set' in p.stdout: - # Unless someone is going to print PDF/A documents on a - # magical sRGB printer I can't see the removal of overprinting - # being a problem.... - log.debug( - "Ghostscript had to remove PDF 'overprinting' from the " - "input file to complete PDF/A conversion. " - ) - else: - log.debug(p.stdout) - - if p.returncode == 0: - # Ghostscript does not change return code when it fails to create - # PDF/A - check PDF/A status elsewhere - copy(gs_pdf.name, fspath(output_file)) - else: - raise SubprocessOutputError('Ghostscript PDF/A rendering failed') + if p.returncode != 0: + # Ghostscript does not change return code when it fails to create + # PDF/A - check PDF/A status elsewhere + with suppress(OSError): + Path(output_file).unlink() + raise SubprocessOutputError('Ghostscript PDF/A rendering failed') From bf99587aa1024255907eff74cb7bdbf532da91f7 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Sat, 16 Nov 2019 16:29:53 -0800 Subject: [PATCH 21/35] ghostscript: use correct executable name on Windows --- src/ocrmypdf/exec/ghostscript.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/ocrmypdf/exec/ghostscript.py b/src/ocrmypdf/exec/ghostscript.py index 65d15794..3613b43f 100644 --- a/src/ocrmypdf/exec/ghostscript.py +++ b/src/ocrmypdf/exec/ghostscript.py @@ -19,6 +19,7 @@ import logging import re +import os import warnings from contextlib import suppress from functools import lru_cache @@ -29,15 +30,24 @@ from subprocess import PIPE, run from PIL import Image -from ..exceptions import SubprocessOutputError +from ..exceptions import SubprocessOutputError, MissingDependencyError from . import get_version gslog = logging.getLogger() +GS = 'gs' +if os.name == 'nt': + GS = 'gswin64c' + try: + get_version(GS) + except MissingDependencyError: + GS = 'gswin32c' + get_version(GS) + @lru_cache(maxsize=1) def version(): - return get_version('gs') + return get_version(GS) def jpeg_passthrough_available(): @@ -84,7 +94,7 @@ def extract_text(input_file, pageno=1): args_gs = ( [ - 'gs', + GS, '-dQUIET', '-dSAFER', '-dBATCH', @@ -144,7 +154,7 @@ def rasterize_pdf( args_gs = ( [ - 'gs', + GS, '-dQUIET', '-dSAFER', '-dBATCH', @@ -265,7 +275,7 @@ def generate_pdfa( # https://bugs.ghostscript.com/show_bug.cgi?id=699392 args_gs = ( [ - "gs", + GS, "-dQUIET", "-dBATCH", "-dNOPAUSE", From c5fa72bd4ec7d46dad864263ab1e2883dc1a5282 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Sat, 16 Nov 2019 16:30:28 -0800 Subject: [PATCH 22/35] ghostscript: use run(check=True) for more consistent error handling --- src/ocrmypdf/exec/ghostscript.py | 74 +++++++++++++++++--------------- 1 file changed, 39 insertions(+), 35 deletions(-) diff --git a/src/ocrmypdf/exec/ghostscript.py b/src/ocrmypdf/exec/ghostscript.py index 3613b43f..4ce625fd 100644 --- a/src/ocrmypdf/exec/ghostscript.py +++ b/src/ocrmypdf/exec/ghostscript.py @@ -26,7 +26,7 @@ from functools import lru_cache from io import BytesIO from os import fspath from pathlib import Path -from subprocess import PIPE, run +from subprocess import PIPE, run, CalledProcessError from PIL import Image @@ -103,14 +103,15 @@ def extract_text(input_file, pageno=1): '-dTextFormat=0', ] + pages - + ['-o', '-', fspath(input_file)] + + ['-o', '-', fspath(input_file), "-sstdout=%stderr"] ) - p = run(args_gs, stdout=PIPE, stderr=PIPE) - if p.returncode != 0: + try: + p = run(args_gs, stdout=PIPE, stderr=PIPE, check=True) + except CalledProcessError as e: raise SubprocessOutputError( - 'Ghostscript text extraction failed\n%s\n%s\n%s' - % (input_file, p.stdout.decode(), p.stderr.decode()) + 'Ghostscript text extraction failed\n%s\n%s' + % (input_file, e.stderr.decode(errors='replace')) ) return p.stdout @@ -167,7 +168,7 @@ def rasterize_pdf( + (['-dFILTERVECTOR'] if filter_vector else []) + [ '-o', - '%stdout', + '-', '-sstdout=%stderr', '-dAutoRotatePages=/None', # Probably has no effect on raster '-f', @@ -176,18 +177,19 @@ def rasterize_pdf( ) log.debug(args_gs) - with Path(output_file).open("wb") as output: - p = run(args_gs, stdout=PIPE, stderr=PIPE, check=False) - stderr = p.stderr.decode('utf-8', errors='replace') - if _gs_error_reported(stderr): - log.error(stderr) - elif stderr: - log.debug(stderr) - - if p.returncode != 0: + try: + p = run(args_gs, stdout=PIPE, stderr=PIPE, check=True) + except CalledProcessError as e: with suppress(OSError): Path(output_file).unlink() # no unfinished files + log.error(e.stderr.decode(errors='replace')) raise SubprocessOutputError('Ghostscript rasterizing failed') + else: + stderr = p.stderr.decode(errors='replace') + if _gs_error_reported(stderr): + log.error(stderr) + elif stderr: + log.debug(stderr) with Image.open(BytesIO(p.stdout)) as im: if rotation is not None: @@ -290,32 +292,34 @@ def generate_pdfa( "-dJPEGQ=95", "-dPDFA=" + pdfa_part, "-dPDFACompatibilityPolicy=1", - "-sOutputFile=%stdout", + "-o", + "-", "-sstdout=%stderr", ] ) args_gs.extend(fspath(s) for s in pdf_pages) # Stringify Path objs log.debug(args_gs) - with Path(output_file).open('wb') as output: - p = run(args_gs, stdout=output, stderr=PIPE, check=False) - - stderr = p.stderr.decode('utf-8', errors='replace') - if _gs_error_reported(stderr): - log.error(stderr) - elif 'overprint mode not set' in stderr: - # Unless someone is going to print PDF/A documents on a - # magical sRGB printer I can't see the removal of overprinting - # being a problem.... - log.debug( - "Ghostscript had to remove PDF 'overprinting' from the " - "input file to complete PDF/A conversion. " - ) - else: - log.debug(stderr) - - if p.returncode != 0: + try: + with Path(output_file).open('wb') as output: + p = run(args_gs, stdout=output, stderr=PIPE, check=True) + except CalledProcessError as e: # Ghostscript does not change return code when it fails to create # PDF/A - check PDF/A status elsewhere with suppress(OSError): Path(output_file).unlink() + log.error(e.stderr.decode(errors='replace')) raise SubprocessOutputError('Ghostscript PDF/A rendering failed') + else: + stderr = p.stderr.decode('utf-8', errors='replace') + if _gs_error_reported(stderr): + log.error(stderr) + elif 'overprint mode not set' in stderr: + # Unless someone is going to print PDF/A documents on a + # magical sRGB printer I can't see the removal of overprinting + # being a problem.... + log.debug( + "Ghostscript had to remove PDF 'overprinting' from the " + "input file to complete PDF/A conversion. " + ) + else: + log.debug(stderr) From e51e21c6b6422fdf96287d296f1f1bae5efe7a96 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Mon, 25 Nov 2019 12:54:55 -0800 Subject: [PATCH 23/35] ghostscript: Refactor checking for executable name on Windows --- src/ocrmypdf/exec/ghostscript.py | 12 +++++----- tests/spoof/gs.py | 40 +++++++++++++++++++++++++++++++ tests/spoof/gs_feature_elision.py | 5 +--- tests/spoof/gs_pdfa_failure.py | 6 +---- tests/spoof/gs_raster_failure.py | 5 +--- tests/spoof/gs_render_failure.py | 5 +--- 6 files changed, 50 insertions(+), 23 deletions(-) create mode 100644 tests/spoof/gs.py diff --git a/src/ocrmypdf/exec/ghostscript.py b/src/ocrmypdf/exec/ghostscript.py index 4ce625fd..434859ad 100644 --- a/src/ocrmypdf/exec/ghostscript.py +++ b/src/ocrmypdf/exec/ghostscript.py @@ -27,6 +27,7 @@ from io import BytesIO from os import fspath from pathlib import Path from subprocess import PIPE, run, CalledProcessError +from shutil import which from PIL import Image @@ -37,12 +38,11 @@ gslog = logging.getLogger() GS = 'gs' if os.name == 'nt': - GS = 'gswin64c' - try: - get_version(GS) - except MissingDependencyError: - GS = 'gswin32c' - get_version(GS) + GS = which('gswin64c') + if not GS: + GS = which('gswin32c') + if not GS: + raise MissingDependencyError("Ghostscript (gswin64c or gswin32c)") @lru_cache(maxsize=1) diff --git a/tests/spoof/gs.py b/tests/spoof/gs.py new file mode 100644 index 00000000..978f346b --- /dev/null +++ b/tests/spoof/gs.py @@ -0,0 +1,40 @@ +# © 2019 James R. Barlow: github.com/jbarlow83 +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +"""Find Ghostscript executable""" + + +import os +import shutil + + +def real_ghostscript(argv): + if os.name != 'nt': + gs = shutil.which('gs') + gs_args = [gs] + argv[1:] + os.execv(gs_args[0], gs_args) + else: + gs = shutil.which('gswin64c') + if not gs: + gs = shutil.which('gswin32c') + os.execv(gs, argv[1:]) + + return # Not reachable diff --git a/tests/spoof/gs_feature_elision.py b/tests/spoof/gs_feature_elision.py index ad65a619..0ae46b46 100755 --- a/tests/spoof/gs_feature_elision.py +++ b/tests/spoof/gs_feature_elision.py @@ -30,10 +30,7 @@ from subprocess import check_call PDF/A creation.""" -def real_ghostscript(argv): - gs_args = ['gs'] + argv[1:] - os.execvp("gs", gs_args) - return # Not reachable +from gs import real_ghostscript elision_warning = """GPL Ghostscript 9.20: Setting Overprint Mode to 1 diff --git a/tests/spoof/gs_pdfa_failure.py b/tests/spoof/gs_pdfa_failure.py index 730fa5b5..b8559192 100755 --- a/tests/spoof/gs_pdfa_failure.py +++ b/tests/spoof/gs_pdfa_failure.py @@ -27,11 +27,7 @@ import sys """Replicate Ghostscript PDF/A conversion failure by suppressing some arguments""" - -def real_ghostscript(argv): - gs_args = ['gs'] + argv[1:] - os.execvp("gs", gs_args) - return # Not reachable +from gs import real_ghostscript def main(): diff --git a/tests/spoof/gs_raster_failure.py b/tests/spoof/gs_raster_failure.py index b404cca8..f7269b3f 100755 --- a/tests/spoof/gs_raster_failure.py +++ b/tests/spoof/gs_raster_failure.py @@ -27,10 +27,7 @@ import sys """Replicate Ghostscript raster failure while allowing rendering""" -def real_ghostscript(argv): - gs_args = ['gs'] + argv[1:] - os.execvp("gs", gs_args) - return # Not reachable +from gs import real_ghostscript def main(): diff --git a/tests/spoof/gs_render_failure.py b/tests/spoof/gs_render_failure.py index 3027a684..5bb6ce7c 100755 --- a/tests/spoof/gs_render_failure.py +++ b/tests/spoof/gs_render_failure.py @@ -26,10 +26,7 @@ import os import sys -def real_ghostscript(argv): - gs_args = ['gs'] + argv[1:] - os.execvp("gs", gs_args) - return # Not reachable +from gs import real_ghostscript def main(): From 06a1f987d499f855d1576c3c67a4e49483c3e40d Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Thu, 28 Nov 2019 16:40:04 -0800 Subject: [PATCH 24/35] Use _OCRMYPDF_TEST_PATH for testing and .py stubs to simulate symlinks --- src/ocrmypdf/api.py | 1 + src/ocrmypdf/exec/__init__.py | 23 ++++++++++++++++++- src/ocrmypdf/exec/ghostscript.py | 5 +++-- src/ocrmypdf/exec/jbig2enc.py | 4 ++-- src/ocrmypdf/exec/qpdf.py | 4 ++-- src/ocrmypdf/exec/tesseract.py | 4 ++-- src/ocrmypdf/exec/unpaper.py | 5 ++--- tests/conftest.py | 37 ++++++++++++++++++++++++++----- tests/spoof/gs_feature_elision.py | 1 - tests/spoof/gs_pdfa_failure.py | 1 - tests/spoof/gs_raster_failure.py | 1 - tests/spoof/gs_render_failure.py | 1 - tests/spoof/tesseract_cache.py | 2 -- tests/test_main.py | 10 ++++----- 14 files changed, 70 insertions(+), 29 deletions(-) diff --git a/src/ocrmypdf/api.py b/src/ocrmypdf/api.py index a05edbc1..af47c969 100644 --- a/src/ocrmypdf/api.py +++ b/src/ocrmypdf/api.py @@ -18,6 +18,7 @@ import logging import os import sys +import warnings from enum import IntEnum from pathlib import Path diff --git a/src/ocrmypdf/exec/__init__.py b/src/ocrmypdf/exec/__init__.py index c1a18c9a..1f5656e7 100644 --- a/src/ocrmypdf/exec/__init__.py +++ b/src/ocrmypdf/exec/__init__.py @@ -21,14 +21,35 @@ import logging import os import re import sys +import shutil from collections.abc import Mapping -from subprocess import PIPE, STDOUT, CalledProcessError, run +from subprocess import PIPE, STDOUT, CalledProcessError, run as subprocess_run from ..exceptions import ExitCode, MissingDependencyError log = logging.Logger(__name__) +def _get_program(args, env=None): + program = args[0] + test_path = env.get('_OCRMYPDF_TEST_PATH', '') + if test_path: + program = shutil.which(program, path=test_path) + return program + + +def run(args, *, env=None, **kwargs): + if not env: + env = os.environ + program = _get_program(args, env) + if os.name == 'nt' and program.lower().endswith('.py'): + args = [sys.executable, program] + args[1:] + else: + args = [program] + args[1:] + log.debug(args) + return subprocess_run(args, env=env, **kwargs) + + def get_version(program, *, version_arg='--version', regex=r'(\d+(\.\d+)*)', env=None): "Get the version of the specified program" args_prog = [program, version_arg] diff --git a/src/ocrmypdf/exec/ghostscript.py b/src/ocrmypdf/exec/ghostscript.py index 434859ad..1d13122c 100644 --- a/src/ocrmypdf/exec/ghostscript.py +++ b/src/ocrmypdf/exec/ghostscript.py @@ -26,13 +26,13 @@ from functools import lru_cache from io import BytesIO from os import fspath from pathlib import Path -from subprocess import PIPE, run, CalledProcessError +from subprocess import PIPE, CalledProcessError from shutil import which from PIL import Image from ..exceptions import SubprocessOutputError, MissingDependencyError -from . import get_version +from . import get_version, run gslog = logging.getLogger() @@ -43,6 +43,7 @@ if os.name == 'nt': GS = which('gswin32c') if not GS: raise MissingDependencyError("Ghostscript (gswin64c or gswin32c)") + GS = Path(GS).stem @lru_cache(maxsize=1) diff --git a/src/ocrmypdf/exec/jbig2enc.py b/src/ocrmypdf/exec/jbig2enc.py index dff450c8..5218edbd 100644 --- a/src/ocrmypdf/exec/jbig2enc.py +++ b/src/ocrmypdf/exec/jbig2enc.py @@ -18,10 +18,10 @@ """Interface to jbig2 executable""" from functools import lru_cache -from subprocess import PIPE, run +from subprocess import PIPE from ..exceptions import MissingDependencyError -from . import get_version +from . import get_version, run @lru_cache(maxsize=1) diff --git a/src/ocrmypdf/exec/qpdf.py b/src/ocrmypdf/exec/qpdf.py index e96848c0..9be8692b 100644 --- a/src/ocrmypdf/exec/qpdf.py +++ b/src/ocrmypdf/exec/qpdf.py @@ -19,9 +19,9 @@ from functools import lru_cache from os import fspath -from subprocess import PIPE, STDOUT, CalledProcessError, run +from subprocess import PIPE, STDOUT, CalledProcessError -from . import get_version +from . import get_version, run @lru_cache(maxsize=1) diff --git a/src/ocrmypdf/exec/tesseract.py b/src/ocrmypdf/exec/tesseract.py index 4b0560d8..34bd8983 100644 --- a/src/ocrmypdf/exec/tesseract.py +++ b/src/ocrmypdf/exec/tesseract.py @@ -23,7 +23,7 @@ from collections import namedtuple from contextlib import suppress import logging from os import fspath -from subprocess import PIPE, STDOUT, CalledProcessError, TimeoutExpired, run +from subprocess import PIPE, STDOUT, CalledProcessError, TimeoutExpired from ..exceptions import ( MissingDependencyError, @@ -31,7 +31,7 @@ from ..exceptions import ( TesseractConfigError, ) from ..helpers import page_number, safe_symlink -from . import get_version +from . import get_version, run OrientationConfidence = namedtuple('OrientationConfidence', ('angle', 'confidence')) diff --git a/src/ocrmypdf/exec/unpaper.py b/src/ocrmypdf/exec/unpaper.py index 4515a33f..1143c0e9 100644 --- a/src/ocrmypdf/exec/unpaper.py +++ b/src/ocrmypdf/exec/unpaper.py @@ -22,7 +22,6 @@ import os import shlex -import subprocess from functools import lru_cache from subprocess import PIPE, STDOUT, CalledProcessError from tempfile import TemporaryDirectory @@ -30,7 +29,7 @@ from tempfile import TemporaryDirectory from PIL import Image from ..exceptions import MissingDependencyError, SubprocessOutputError -from . import get_version +from . import get_version, run as external_run @lru_cache(maxsize=1) @@ -77,7 +76,7 @@ def run(input_file, output_file, dpi, log, mode_args): # their unpaper arguments (whether intentionally or otherwise) args_unpaper.extend([input_pnm, output_pnm]) try: - proc = subprocess.run( + proc = external_run( args_unpaper, check=True, close_fds=True, diff --git a/tests/conftest.py b/tests/conftest.py index 3a2850a4..08b45fb2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -80,6 +80,19 @@ PROJECT_ROOT = os.path.dirname(TESTS_ROOT) OCRMYPDF = [sys.executable, '-m', 'ocrmypdf'] +PY_FILE_TEMPLATE = """ +import os +import subprocess +import sys + +args = [sys.executable, {spoofer}, *sys.argv[1:]] +p = subprocess.run(args, check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) +sys.stdout.buffer.write(p.stdout) +sys.stderr.buffer.write(p.stderr) +sys.exit(p.returncode) +""" + + @pytest.helpers.register def spoof(tmp_path_factory, **kwargs): """Modify PATH to override subprocess executables @@ -97,12 +110,24 @@ def spoof(tmp_path_factory, **kwargs): for replace_program, with_spoof in kwargs.items(): spoofer = Path(SPOOF_PATH) / with_spoof - spoofer.chmod(0o755) - (tmpdir / replace_program).symlink_to(spoofer) - - env['_OCRMYPDF_SAVE_PATH'] = env['PATH'] - env['PATH'] = str(tmpdir) + ":" + env['PATH'] + if os.name != 'nt': + spoofer.chmod(0o755) + (tmpdir / replace_program).symlink_to(spoofer) + else: + py_file = PY_FILE_TEMPLATE.format( + python=sys.executable, spoofer=repr(os.fspath(spoofer.absolute())) + ) + if replace_program == 'gs': + programs = ['gswin64c', 'gswin32c'] + else: + programs = [replace_program] + for prog in programs: + (tmpdir / f'{prog}.py').write_text(py_file, encoding='utf-8') + env['_OCRMYPDF_TEST_PATH'] = str(tmpdir) + os.pathsep + env['PATH'] + if os.name == 'nt': + if '.py' not in env['PATHEXT'].lower(): + raise EnvironmentError("PATHEXT is not configured to support .py") return env @@ -178,7 +203,7 @@ def run_ocrmypdf_api(input_file, output_file, *args, env=None): ) api.check_options(options) if env: - options.tesseract_env = env + options.tesseract_env = env.copy() options.tesseract_env['_OCRMYPDF_TEST_INFILE'] = os.fspath(input_file) if options.tesseract_env: assert all(isinstance(v, (str, bytes)) for v in options.tesseract_env.values()) diff --git a/tests/spoof/gs_feature_elision.py b/tests/spoof/gs_feature_elision.py index 0ae46b46..f9856311 100755 --- a/tests/spoof/gs_feature_elision.py +++ b/tests/spoof/gs_feature_elision.py @@ -38,7 +38,6 @@ not permitted in PDF/A-2, overprint mode not set""" def main(): - os.environ['PATH'] = os.environ['_OCRMYPDF_SAVE_PATH'] if '--version' in sys.argv: print('9.20') print('SPOOFED: ' + os.path.basename(__file__)) diff --git a/tests/spoof/gs_pdfa_failure.py b/tests/spoof/gs_pdfa_failure.py index b8559192..6dd90e29 100755 --- a/tests/spoof/gs_pdfa_failure.py +++ b/tests/spoof/gs_pdfa_failure.py @@ -31,7 +31,6 @@ from gs import real_ghostscript def main(): - os.environ['PATH'] = os.environ['_OCRMYPDF_SAVE_PATH'] if '--version' in sys.argv: print('9.20') print('SPOOFED: ' + os.path.basename(__file__)) diff --git a/tests/spoof/gs_raster_failure.py b/tests/spoof/gs_raster_failure.py index f7269b3f..c07b881b 100755 --- a/tests/spoof/gs_raster_failure.py +++ b/tests/spoof/gs_raster_failure.py @@ -31,7 +31,6 @@ from gs import real_ghostscript def main(): - os.environ['PATH'] = os.environ['_OCRMYPDF_SAVE_PATH'] if '--version' in sys.argv: print('9.20') print('SPOOFED: ' + os.path.basename(__file__)) diff --git a/tests/spoof/gs_render_failure.py b/tests/spoof/gs_render_failure.py index 5bb6ce7c..a43833a8 100755 --- a/tests/spoof/gs_render_failure.py +++ b/tests/spoof/gs_render_failure.py @@ -30,7 +30,6 @@ from gs import real_ghostscript def main(): - os.environ['PATH'] = os.environ['_OCRMYPDF_SAVE_PATH'] if '--version' in sys.argv: print('9.20') print('SPOOFED: ' + os.path.basename(__file__)) diff --git a/tests/spoof/tesseract_cache.py b/tests/spoof/tesseract_cache.py index 528e5a1e..83c09535 100755 --- a/tests/spoof/tesseract_cache.py +++ b/tests/spoof/tesseract_cache.py @@ -59,8 +59,6 @@ import subprocess import sys from pathlib import Path -if '_OCRMYPDF_SAVE_PATH' in os.environ: - os.environ['PATH'] = os.environ['_OCRMYPDF_SAVE_PATH'] __version__ = subprocess.check_output( ['tesseract', '--version'], stderr=subprocess.STDOUT diff --git a/tests/test_main.py b/tests/test_main.py index 73f91e5e..c18a23f1 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -297,10 +297,10 @@ def test_force_ocr_on_pdf_with_no_images(spoof_tesseract_crash, resources, no_ou # As a correctness test, make sure that --force-ocr on a PDF with no # content still triggers tesseract. If tesseract crashes, then it was # called. - result = run_ocrmypdf_api( + p, _, _ = run_ocrmypdf( resources / 'blank.pdf', no_outpdf, '--force-ocr', env=spoof_tesseract_crash ) - assert result == ExitCode.child_process_error + assert p.returncode == ExitCode.child_process_error assert not os.path.exists(no_outpdf) @@ -389,7 +389,7 @@ def test_pagesegmode(renderer, spoof_tesseract_cache, resources, outpdf): @pytest.mark.parametrize('renderer', RENDERERS) def test_tesseract_crash(renderer, spoof_tesseract_crash, resources, no_outpdf, caplog): - result = run_ocrmypdf_api( + p, _, err = run_ocrmypdf( resources / 'ccitt.pdf', no_outpdf, '-v', @@ -398,9 +398,9 @@ def test_tesseract_crash(renderer, spoof_tesseract_crash, resources, no_outpdf, renderer, env=spoof_tesseract_crash, ) - assert result == ExitCode.child_process_error + assert p.returncode == ExitCode.child_process_error assert not os.path.exists(no_outpdf) - assert "SubprocessOutputError" in caplog.text + assert "SubprocessOutputError" in err def test_tesseract_crash_autorotate(spoof_tesseract_crash, resources, no_outpdf): From 66d04dd6e32c7930baec14ebf3fbb008992c1c1a Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Wed, 4 Dec 2019 13:33:00 -0800 Subject: [PATCH 25/35] Don't expect filenames to be replicated on NT --- tests/test_main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_main.py b/tests/test_main.py index c18a23f1..fa7473c4 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -363,7 +363,8 @@ def test_input_file_not_a_pdf(caplog, no_outpdf): input_file = __file__ # Try to OCR this file result = run_ocrmypdf_api(input_file, no_outpdf) assert result == ExitCode.input_file - assert input_file in caplog.text + if os.name != 'nt': # name will be mangled with \\'s on nt + assert input_file in caplog.text def test_encrypted(resources, caplog, no_outpdf): From cff37bf6814d3ea44c96f1bd4f43cee3e77e1113 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Wed, 4 Dec 2019 13:40:48 -0800 Subject: [PATCH 26/35] Make test_german more Windows-friendly --- tests/test_main.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index fa7473c4..e703f07f 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -313,18 +313,20 @@ def test_german(spoof_tesseract_cache, resources, outdir): # properly. It is fine that we are testing -l deu on a French file because # we are exercising the functionality not going for accuracy. sidecar = outdir / 'francais.txt' - result = run_ocrmypdf_api( - resources / 'francais.pdf', - outdir / 'francais.pdf', - '-l', - 'deu', # more commonly installed - '--sidecar', - sidecar, - env=spoof_tesseract_cache, - ) - if 'deu' not in tesseract.languages(): - pytest.xfail(reason="tesseract-deu language pack not installed") - assert result == ExitCode.ok, "Requires tesseract deu language pack" + try: + check_ocrmypdf( + resources / 'francais.pdf', + outdir / 'francais.pdf', + '-l', + 'deu', # more commonly installed + '--sidecar', + sidecar, + env=spoof_tesseract_cache, + ) + except MissingDependencyError: + if 'deu' not in tesseract.languages(): + pytest.xfail(reason="tesseract-deu language pack not installed") + raise def test_klingon(resources, outpdf): From d0301813cc30f4282f2d360de04426e544da7ec5 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Wed, 4 Dec 2019 14:58:46 -0800 Subject: [PATCH 27/35] ghosttext: mention page number differences --- src/ocrmypdf/pdfinfo/ghosttext.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ocrmypdf/pdfinfo/ghosttext.py b/src/ocrmypdf/pdfinfo/ghosttext.py index 43156154..9626fad7 100644 --- a/src/ocrmypdf/pdfinfo/ghosttext.py +++ b/src/ocrmypdf/pdfinfo/ghosttext.py @@ -96,6 +96,7 @@ def extract_text_xml(infile, pdf, pageno=None, log=gslog): page_count_difference = len(pdf.pages) - len(page_xml) if page_count_difference != 0: log.error("The number of pages in the input file is inconsistent.") + log.error(f"Expected {len(pdf.pages)}, txtwrite says {len(page_xml)}") if page_count_difference > 0: page_xml.extend([None] * page_count_difference) return page_xml From 9db01c7ff5cdea61f5b5d807fd9d5493754ad8b9 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Wed, 4 Dec 2019 15:00:12 -0800 Subject: [PATCH 28/35] Remove test_bad_utf8 Due to difficulties of getting this to work on Python 3.8, Windows, and high probability that this behavior is now gone from Tesseract 4.0+. Originally added in 2017. --- src/ocrmypdf/exec/tesseract.py | 7 +------ tests/test_stdio.py | 18 +----------------- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/src/ocrmypdf/exec/tesseract.py b/src/ocrmypdf/exec/tesseract.py index 34bd8983..a4a42b7d 100644 --- a/src/ocrmypdf/exec/tesseract.py +++ b/src/ocrmypdf/exec/tesseract.py @@ -194,12 +194,7 @@ def tesseract_log_output(mainlog, stdout, input_file): try: text = stdout.decode() except UnicodeDecodeError: - log.error( - "command line output was not utf-8. " - + "This usually means Tesseract's language packs do not match " - "the installed version of Tesseract." - ) - text = stdout.decode('utf-8', 'backslashreplace') + text = stdout.decode('utf-8', 'ignore') lines = text.splitlines() for line in lines: diff --git a/tests/test_stdio.py b/tests/test_stdio.py index 53c0398f..0f2609af 100644 --- a/tests/test_stdio.py +++ b/tests/test_stdio.py @@ -18,7 +18,7 @@ import os import sys from pathlib import Path -from subprocess import DEVNULL, PIPE, run, Popen +from subprocess import DEVNULL, PIPE, run, Popen, CalledProcessError import pytest @@ -115,22 +115,6 @@ def test_bad_locale(): assert 'configured to use ASCII as encoding' in err, "should whine" -@pytest.mark.parametrize('renderer', ['hocr', 'sandwich']) -def test_bad_utf8(spoof_tess_bad_utf8, renderer, resources, no_outpdf): - p, out, err = run_ocrmypdf( - resources / 'ccitt.pdf', - no_outpdf, - '--pdf-renderer', - renderer, - env=spoof_tess_bad_utf8, - ) - - assert out == '', "stdout not clean" - assert p.returncode != 0 - assert 'not utf-8' in err, "should whine about utf-8" - assert '\\x96' in err, 'should repeat backslash encoded output' - - def test_dev_null(spoof_tesseract_noop, resources): p, out, err = run_ocrmypdf( resources / 'trivial.pdf', os.devnull, '--force-ocr', env=spoof_tesseract_noop From cb3cfaa055e2f6b2fca99657daf4f60ec4b1dbd7 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Wed, 4 Dec 2019 15:56:18 -0800 Subject: [PATCH 29/35] Add Windows install advice --- src/ocrmypdf/_validation.py | 2 +- src/ocrmypdf/exec/__init__.py | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/ocrmypdf/_validation.py b/src/ocrmypdf/_validation.py index b02e4532..af518b2b 100644 --- a/src/ocrmypdf/_validation.py +++ b/src/ocrmypdf/_validation.py @@ -446,7 +446,7 @@ def report_output_file_size(options, input_file, output_file): def check_dependency_versions(options): check_external_program( program='tesseract', - package={'darwin': 'tesseract', 'linux': 'tesseract-ocr'}, + package={'linux': 'tesseract-ocr'}, version_checker=tesseract.version, need_version='4.0.0', # using backport for Travis CI ) diff --git a/src/ocrmypdf/exec/__init__.py b/src/ocrmypdf/exec/__init__.py index 1f5656e7..62ea5c47 100644 --- a/src/ocrmypdf/exec/__init__.py +++ b/src/ocrmypdf/exec/__init__.py @@ -132,23 +132,33 @@ On RPM-based systems (Red Hat, Fedora), search for instructions on installing the RPM for {program}. ''' +windows_install_advice = ''' +If not already installed, install the Chocolatey package manager. Then use +a command prompt to install the missing package: + choco install {package} +''' + def _get_platform(): if sys.platform.startswith('freebsd'): return 'freebsd' elif sys.platform.startswith('linux'): return 'linux' + elif sys.platform.startswith('win'): + return 'windows' return sys.platform def _error_trailer(program, package, **kwargs): if isinstance(package, Mapping): - package = package[_get_platform()] + package = package.get(_get_platform(), program) if _get_platform() == 'darwin': log.info(osx_install_advice.format(**locals())) elif _get_platform() == 'linux': log.info(linux_install_advice.format(**locals())) + elif _get_platform() == 'windows': + log.info(windows_install_advice.format(**locals())) def _error_missing_program(program, package, required_for, recommended): From d4abe88452e07f919406246c9d1e4b1926c8472b Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Tue, 19 Nov 2019 12:52:48 -0800 Subject: [PATCH 30/35] docs: sketch Windows install procedure --- docs/installation.rst | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index 46bee8d3..70a015b9 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -431,8 +431,24 @@ See `OCRmyPDF Docker Image `__ for more information. Installing on Windows ===================== -Direct installation on Windows is not currently possible, but it works well in -Windows Subsystem for Linux: +You must install the following for Windows using their installers: + +* Python 3.7 (64-bit recommended) +* Tesseract 4.0 or later +* Ghostscript 9.50 or later +* QPDF 9.0.2 or later + +You can install all except Tesseract with the Chocolatey package manager: + +* ``choco install python3`` +* ``choco install ghostscript`` +* ``choco install qpdf`` + +Modify your ``PATH`` environment variable so that Tesseract, Ghostscript and QPDF +executables on the ``PATH``. + +Installing on Windows Subsystem for Linux +========================================= #. Install Ubuntu 18.04 for Windows Subsystem for Linux, if not already installed. #. Follow the procedure to install :ref:`OCRmyPDF on Ubuntu 18.04 `. @@ -451,15 +467,6 @@ Then confirm that the expected version from PyPI (|latest|) is installed: You can then run OCRmyPDF in the Windows command prompt or Powershell, prefixing ``wsl``, and call it from Windows programs or batch files. -Why no native Windows? -^^^^^^^^^^^^^^^^^^^^^^ - -It would probably not be too difficult to port on Windows. The main -reason this has been avoided is the difficulty of packaging and -installing the various non-Python dependencies: Tesseract, QPDF, -Ghostscript, Leptonica. Pull requests to add or improve Windows support -would be quite welcome. - Docker ^^^^^^ From b8b7ecfe7f7d05d30037f4f2d9ac97d24c952c94 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Wed, 4 Dec 2019 21:10:27 -0800 Subject: [PATCH 31/35] Fix DecompressionBomb related errors due to Windows process differences --- src/ocrmypdf/_sync.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/ocrmypdf/_sync.py b/src/ocrmypdf/_sync.py index 4c25b300..27262135 100644 --- a/src/ocrmypdf/_sync.py +++ b/src/ocrmypdf/_sync.py @@ -26,6 +26,7 @@ from collections import namedtuple from tempfile import mkdtemp from tqdm import tqdm +import PIL from ._graft import OcrGrafter from ._jobcontext import PDFContext, cleanup_working_files, make_logger @@ -176,7 +177,7 @@ def post_process(pdf_file, context): return optimize_pdf(pdf_out, context) -def worker_init(queue): +def worker_init(queue, max_pixels): """Initialize a process pool worker""" # Ignore SIGINT (our parent process will kill us gracefully) @@ -188,9 +189,15 @@ def worker_init(queue): root.handlers = [] root.addHandler(h) + # In Windows, child process will not inherit our change to this value in + # the parent process, so ensure workers get it set + PIL.Image.MAX_IMAGE_PIXELS = max_pixels -def worker_thread_init(_queue): - pass + +def worker_thread_init(_queue, max_pixels): + # This is probably not needed since threads should all see the same memory, + # but done for consistency. + PIL.Image.MAX_IMAGE_PIXELS = max_pixels def log_listener(queue): @@ -261,7 +268,9 @@ def exec_concurrent(context): unit_scale=0.5, disable=not context.options.progress_bar, ) as pbar, Pool( - processes=max_workers, initializer=initializer, initargs=(log_queue,) + processes=max_workers, + initializer=initializer, + initargs=(log_queue, PIL.Image.MAX_IMAGE_PIXELS), ) as pool: results = pool.imap_unordered(exec_page_sync, context.get_page_contexts()) while True: From 5607429d9a5e6e2bbfdee50b001681def84db5ed Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Wed, 4 Dec 2019 21:31:01 -0800 Subject: [PATCH 32/35] tests: error message from tesseract change --- tests/test_main.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_main.py b/tests/test_main.py index e703f07f..8df33562 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -612,7 +612,10 @@ THIS FILE IS INVALID '--tesseract-config', cfg_file, ) - assert "parameter not found" in err.lower(), "No error message" + assert ( + "parameter not found" in err.lower() + or "error occurred while parsing" in err.lower() + ), "No error message" assert p.returncode == ExitCode.invalid_config From 51abd791363116b0fd4150e247a4e2bd99e067df Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Wed, 4 Dec 2019 21:35:28 -0800 Subject: [PATCH 33/35] Tesseract no longer posts an error message if config file not found --- tests/test_main.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index 8df33562..a5af5b10 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -576,23 +576,6 @@ language_model_penalty_non_freq_dict_word 0 ) -@pytest.mark.slow # This test sometimes times out in CI -@pytest.mark.parametrize('renderer', RENDERERS) -def test_tesseract_config_notfound(renderer, resources, outdir): - cfg_file = outdir / 'nofile.cfg' - - p, out, err = run_ocrmypdf( - resources / 'ccitt.pdf', - outdir / 'out.pdf', - '--pdf-renderer', - renderer, - '--tesseract-config', - cfg_file, - ) - assert "Can't open" in err, "No error message about missing config file" - assert p.returncode == ExitCode.ok, err - - @pytest.mark.slow # This test sometimes times out in CI @pytest.mark.parametrize('renderer', RENDERERS) def test_tesseract_config_invalid(renderer, resources, outdir): From f6510e2b1512a2c760256c42fe57b5e0b8e68612 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Fri, 6 Dec 2019 15:00:12 -0800 Subject: [PATCH 34/35] Document function of symlink shim --- tests/conftest.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 08b45fb2..1b1ece82 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,6 +15,7 @@ # You should have received a copy of the GNU General Public License # along with OCRmyPDF. If not, see . +import ast import os import platform import sys @@ -80,7 +81,9 @@ PROJECT_ROOT = os.path.dirname(TESTS_ROOT) OCRMYPDF = [sys.executable, '-m', 'ocrmypdf'] -PY_FILE_TEMPLATE = """ +WINDOWS_SHIM_TEMPLATE = """ +# This is a shim for Windows that has the same effect as a symlink to the target .py +# file import os import subprocess import sys @@ -92,6 +95,8 @@ sys.stderr.buffer.write(p.stderr) sys.exit(p.returncode) """ +assert ast.parse(WINDOWS_SHIM_TEMPLATE.format(spoofer=repr(r"C:\\Temp\\file.py"))) + @pytest.helpers.register def spoof(tmp_path_factory, **kwargs): @@ -114,8 +119,8 @@ def spoof(tmp_path_factory, **kwargs): spoofer.chmod(0o755) (tmpdir / replace_program).symlink_to(spoofer) else: - py_file = PY_FILE_TEMPLATE.format( - python=sys.executable, spoofer=repr(os.fspath(spoofer.absolute())) + py_file = WINDOWS_SHIM_TEMPLATE.format( + spoofer=repr(os.fspath(spoofer.absolute())) ) if replace_program == 'gs': programs = ['gswin64c', 'gswin32c'] From 66bda3420a2fbd13fe6ae2aa5089fee556e3deea Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Fri, 6 Dec 2019 15:03:20 -0800 Subject: [PATCH 35/35] docs: cause about using Windows in production --- docs/installation.rst | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index 70a015b9..414239f1 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -431,6 +431,13 @@ See `OCRmyPDF Docker Image `__ for more information. Installing on Windows ===================== +.. warning:: + + Native Windows support is new. Consider it "beta" software. Some + functionality is missing or may be more difficult to enable. If you need a + production-ready solution, use Windows Subsystem for Linux or a Docker + image. + You must install the following for Windows using their installers: * Python 3.7 (64-bit recommended) @@ -438,12 +445,17 @@ You must install the following for Windows using their installers: * Ghostscript 9.50 or later * QPDF 9.0.2 or later -You can install all except Tesseract with the Chocolatey package manager: +You can install these with the Chocolatey package manager: * ``choco install python3`` +* ``choco install tesseract`` * ``choco install ghostscript`` * ``choco install qpdf`` +Also consider adding: + +* ``choco install pngquant`` + Modify your ``PATH`` environment variable so that Tesseract, Ghostscript and QPDF executables on the ``PATH``.