From b49f5a7d7716b1effba05c424b841c72b16c3da2 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Fri, 4 Dec 2015 01:35:07 -0800 Subject: [PATCH 1/6] Support optionally using leptonica to deskew unpaper doesn't seem to be good at deskewing. It fails on test case with a lot of italics. I think it also struggles on pages with a lot of whitespace. Leptonica continues to shine here. However, this is only a first crack at Leptonica. The leptonica module should be redone to use cffi (more extensible). Also considering the possibility of making all Lept calls in a forked process to insulate the calling process from C code crashes and the messy redirect of stdout/stderr to read Leptonica's errors. I don't think the redirect is a huge problem as long as multiprocesses rather than multithreads are used. The ruffus child process that is handling a page is single threaded and will not be affected by the redirection. It just feels dirty. The main reason to consider a child process is crash isolation. --- ocrmypdf/main.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ocrmypdf/main.py b/ocrmypdf/main.py index c8ade0ae..3790bd92 100755 --- a/ocrmypdf/main.py +++ b/ocrmypdf/main.py @@ -185,6 +185,9 @@ advanced.add_argument( '--tesseract-timeout', default=180.0, type=float, metavar='SECONDS', help='give up on OCR after the timeout, but copy the preprocessed page ' 'into the final output') +advanced.add_argument( + '--deskewer', choices=['leptonica', 'unpaper'], default='leptonica', + help='choose deskew provider') debugging = parser.add_argument_group( "Debugging", @@ -493,7 +496,11 @@ def preprocess_deskew( pageinfo = get_pageinfo(input_file, pdfinfo, pdfinfo_lock) dpi = int(pageinfo['xres']) - unpaper.deskew(input_file, output_file, dpi, log) + if options.deskewer == 'unpaper': + unpaper.deskew(input_file, output_file, dpi, log) + else: + from . import leptonica + leptonica.deskew(input_file, output_file, dpi) @transform( From f3b588764ee0779a45be4ab653f4dcb6e444e15f Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Wed, 20 Jan 2016 15:02:48 -0800 Subject: [PATCH 2/6] Suppress tesseract argument printout --- ocrmypdf/tesseract.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ocrmypdf/tesseract.py b/ocrmypdf/tesseract.py index e1084555..105a2cbd 100644 --- a/ocrmypdf/tesseract.py +++ b/ocrmypdf/tesseract.py @@ -94,7 +94,6 @@ def generate_hocr(input_file, output_hocr, language: list, tessconfig: list, badxml, 'hocr' ] + tessconfig) - print(args_tesseract) p = Popen(args_tesseract, close_fds=True, stdout=PIPE, stderr=PIPE, universal_newlines=True) try: From 350ad5210e075f2b9496931c26c2fdd495db8514 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Wed, 20 Jan 2016 15:03:07 -0800 Subject: [PATCH 3/6] Leptonica: convert to CFFI --- ocrmypdf/leptonica.py | 104 +++++++++--------------------------------- requirements.txt | 1 + setup.py | 3 +- 3 files changed, 24 insertions(+), 84 deletions(-) diff --git a/ocrmypdf/leptonica.py b/ocrmypdf/leptonica.py index a03a1af5..4e2eaecb 100644 --- a/ocrmypdf/leptonica.py +++ b/ocrmypdf/leptonica.py @@ -10,11 +10,14 @@ from __future__ import print_function, absolute_import, division import argparse -import ctypes as C import sys import os import logging from tempfile import TemporaryFile +from ctypes.util import find_library +from ._leptonica import ffi + +lept = ffi.dlopen(find_library('lept')) logger = logging.getLogger(__name__) @@ -25,67 +28,6 @@ def stderr(*objs): print("leptonica.py:", *objs, file=sys.stderr) -from ctypes.util import find_library -lept_lib = find_library('lept') -if not lept_lib: - stderr("Could not find the Leptonica library") - sys.exit(3) -try: - lept = C.cdll.LoadLibrary(lept_lib) -except Exception: - stderr("Could not load the Leptonica library from %s", lept_lib) - sys.exit(3) - - -class _PIXCOLORMAP(C.Structure): - """struct PixColormap from Leptonica src/pix.h - """ - - _fields_ = [ - ("array", C.c_void_p), - ("depth", C.c_int32), - ("nalloc", C.c_int32), - ("n", C.c_int32) - ] - - -class _PIX(C.Structure): - """struct Pix from Leptonica src/pix.h - """ - - _fields_ = [ - ("w", C.c_uint32), - ("h", C.c_uint32), - ("d", C.c_uint32), - ("wpl", C.c_uint32), - ("refcount", C.c_uint32), - ("xres", C.c_int32), - ("yres", C.c_int32), - ("informat", C.c_int32), - ("text", C.POINTER(C.c_char)), - ("colormap", C.POINTER(_PIXCOLORMAP)), - ("data", C.POINTER(C.c_uint32)) - ] - - -PIX = C.POINTER(_PIX) - -lept.pixRead.argtypes = [C.c_char_p] -lept.pixRead.restype = PIX -lept.pixScale.argtypes = [PIX, C.c_float, C.c_float] -lept.pixScale.restype = PIX -lept.pixDeskew.argtypes = [PIX, C.c_int32] -lept.pixDeskew.restype = PIX -lept.pixFindSkew.argtypes = [PIX, C.POINTER(C.c_float), C.POINTER(C.c_float)] -lept.pixFindSkew.restype = C.c_int32 -lept.pixWriteImpliedFormat.argtypes = [C.c_char_p, PIX, C.c_int32, C.c_int32] -lept.pixWriteImpliedFormat.restype = C.c_int32 -lept.pixDestroy.argtypes = [C.POINTER(PIX)] -lept.pixDestroy.restype = None -lept.getLeptonicaVersion.argtypes = [] -lept.getLeptonicaVersion.restype = C.c_char_p - - class LeptonicaErrorTrap(object): """Context manager to trap errors reported by Leptonica. @@ -140,6 +82,12 @@ class LeptonicaIOError(LeptonicaError): pass +def _pix_destroy(pix): + ptr_to_pix = ffi.new('PIX **', pix) + lept.pixDestroy(ptr_to_pix) + print('pix destroy ' + repr(pix)) + + def pixRead(filename): """Load an image file into a PIX object. @@ -148,7 +96,8 @@ def pixRead(filename): """ with LeptonicaErrorTrap(): - return lept.pixRead(filename.encode(sys.getfilesystemencoding())) + pix = lept.pixRead(filename.encode(sys.getfilesystemencoding())) + return ffi.gc(pix, _pix_destroy) def pixScale(pix, scalex, scaley): @@ -168,7 +117,8 @@ def pixDeskew(pix, reduction_factor=0): """ with LeptonicaErrorTrap(): - return lept.pixDeskew(pix, reduction_factor) + deskewed = lept.pixDeskew(pix, reduction_factor) + return ffi.gc(deskewed, _pix_destroy) def pixFindSkew(pix): @@ -178,11 +128,11 @@ def pixFindSkew(pix): """ with LeptonicaErrorTrap(): - angle = C.c_float(0.0) - confidence = C.c_float(0.0) - result = lept.pixFindSkew(pix, C.byref(angle), C.byref(confidence)) + angle = ffi.new('float *', 0.0) + confidence = ffi.new('float *', 0.0) + result = lept.pixFindSkew(pix, angle, confidence) if result == 0: - return (angle.value, confidence.value) + return (angle[0], confidence[0]) else: return (None, None) @@ -212,17 +162,6 @@ def pixWriteImpliedFormat(filename, pix, jpeg_quality=0, jpeg_progressive=0): move(filename, filename[:-4]) # Remove .pnm suffix -def pixDestroy(pix): - """Destroy the pix object. - - Function signature is pixDestroy(struct Pix **), hence C.byref() to pass - the address of the pointer. - - """ - with LeptonicaErrorTrap(): - lept.pixDestroy(C.byref(pix)) - - def getLeptonicaVersion(): """Get Leptonica version string. @@ -231,12 +170,13 @@ def getLeptonicaVersion(): a pointless effort to reclaim 100 bytes of memory. """ - return lept.getLeptonicaVersion().decode() + return ffi.string(lept.getLeptonicaVersion()).decode() def deskew(infile, outfile, dpi): try: pix_source = pixRead(infile) + print(repr(pix_source)) except LeptonicaIOError: raise LeptonicaIOError("Failed to open file: %s" % infile) @@ -245,13 +185,12 @@ def deskew(infile, outfile, dpi): else: reduction_factor = 0 # Use default pix_deskewed = pixDeskew(pix_source, reduction_factor) + print(repr(pix_deskewed)) try: pixWriteImpliedFormat(outfile, pix_deskewed) except LeptonicaIOError: raise LeptonicaIOError("Failed to open destination file: %s" % outfile) - pixDestroy(pix_source) - pixDestroy(pix_deskewed) if __name__ == '__main__': @@ -325,7 +264,6 @@ def test_skew_angle(): rotated_im.save(tmpfile) pix = pixRead(tmpfile.name) angle, confidence = pixFindSkew(pix) - pixDestroy(pix) print('{0} {1} {2}'.format(rotate_angle, angle, confidence), file=sys.stderr) diff --git a/requirements.txt b/requirements.txt index f36dafe9..1a5c0436 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,5 @@ ruffus>=2.6.3 Pillow>=2.4.0 reportlab>=3.1.44 PyPDF2>=1.25.1 +cffi>=1.5.0 git+https://github.com/jbarlow83/img2pdf.git@e9bcce0afc3720752ca53a991db93f911a1df709#egg=img2pdf-0.1.5.dev diff --git a/setup.py b/setup.py index 187e8ddb..c141ee43 100644 --- a/setup.py +++ b/setup.py @@ -214,7 +214,8 @@ setup( 'Pillow', 'reportlab', 'PyPDF2', - 'img2pdf' + 'img2pdf', + 'cffi' ], tests_require=tests_require, entry_points={ From 411981efbcba27175dbc928f5ce528e959782ce0 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Sat, 30 Jan 2016 15:06:25 -0800 Subject: [PATCH 4/6] Experiment with CFFI instead of ctypes --- ocrmypdf/lept.py | 58 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 ocrmypdf/lept.py diff --git a/ocrmypdf/lept.py b/ocrmypdf/lept.py new file mode 100644 index 00000000..ed8e6295 --- /dev/null +++ b/ocrmypdf/lept.py @@ -0,0 +1,58 @@ +from cffi import FFI + +ffi = FFI() +ffi.set_source("_leptonica", None) +ffi.cdef(""" +typedef signed char l_int8; +typedef unsigned char l_uint8; +typedef short l_int16; +typedef unsigned short l_uint16; +typedef int l_int32; +typedef unsigned int l_uint32; +typedef float l_float32; +typedef double l_float64; +typedef long long l_int64; +typedef unsigned long long l_uint64; + +struct Pix +{ + l_uint32 w; /* width in pixels */ + l_uint32 h; /* height in pixels */ + l_uint32 d; /* depth in bits (bpp) */ + l_uint32 spp; /* number of samples per pixel */ + l_uint32 wpl; /* 32-bit words/line */ + l_uint32 refcount; /* reference count (1 if no clones) */ + l_int32 xres; /* image res (ppi) in x direction */ + /* (use 0 if unknown) */ + l_int32 yres; /* image res (ppi) in y direction */ + /* (use 0 if unknown) */ + l_int32 informat; /* input file format, IFF_* */ + l_int32 special; /* special instructions for I/O, etc */ + char *text; /* text string associated with pix */ + struct PixColormap *colormap; /* colormap (may be null) */ + l_uint32 *data; /* the image data */ +}; +typedef struct Pix PIX; + +struct PixColormap +{ + void *array; /* colormap table (array of RGBA_QUAD) */ + l_int32 depth; /* of pix (1, 2, 4 or 8 bpp) */ + l_int32 nalloc; /* number of color entries allocated */ + l_int32 n; /* number of color entries used */ +}; +typedef struct PixColormap PIXCMAP; +""") + +ffi.cdef(""" +PIX * pixRead ( const char *filename ); +PIX * pixScale ( PIX *pixs, l_float32 scalex, l_float32 scaley ); +l_int32 pixFindSkew ( PIX *pixs, l_float32 *pangle, l_float32 *pconf ); +l_int32 pixWriteImpliedFormat ( const char *filename, PIX *pix, l_int32 quality, l_int32 progressive ); +void pixDestroy ( PIX **ppix ); +PIX * pixDeskew ( PIX *pixs, l_int32 redsearch ); +char * getLeptonicaVersion ( ); +""") + +if __name__ == '__main__': + ffi.compile() From 66a095d7de3d440379b69d428bd3d0d39c701891 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Sat, 30 Jan 2016 15:19:40 -0800 Subject: [PATCH 5/6] Improve organization of CFFI setup --- .gitignore | 5 +++-- ocrmypdf/leptonica.py | 2 +- ocrmypdf/{lept.py => lib/compile_leptonica.py} | 2 +- setup.py | 6 +++++- 4 files changed, 10 insertions(+), 5 deletions(-) rename ocrmypdf/{lept.py => lib/compile_leptonica.py} (98%) diff --git a/.gitignore b/.gitignore index 1e47924e..01577f03 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,7 @@ venv-3.5/ */test/output bin/ include/ -lib/ +ocrmypdf/lib/_*.py pip-selfcheck.json pyvenv.cfg htmlcov/ @@ -23,4 +23,5 @@ htmlcov/ .ipynb_checkpoints/ tests/cache/ tests/resources/private -ocrmypdf/version.py \ No newline at end of file +ocrmypdf/version.py +.eggs/ \ No newline at end of file diff --git a/ocrmypdf/leptonica.py b/ocrmypdf/leptonica.py index 4e2eaecb..7b3fb67a 100644 --- a/ocrmypdf/leptonica.py +++ b/ocrmypdf/leptonica.py @@ -15,7 +15,7 @@ import os import logging from tempfile import TemporaryFile from ctypes.util import find_library -from ._leptonica import ffi +from .lib._leptonica import ffi lept = ffi.dlopen(find_library('lept')) diff --git a/ocrmypdf/lept.py b/ocrmypdf/lib/compile_leptonica.py similarity index 98% rename from ocrmypdf/lept.py rename to ocrmypdf/lib/compile_leptonica.py index ed8e6295..427ffbb8 100644 --- a/ocrmypdf/lept.py +++ b/ocrmypdf/lib/compile_leptonica.py @@ -1,7 +1,7 @@ from cffi import FFI ffi = FFI() -ffi.set_source("_leptonica", None) +ffi.set_source("ocrmypdf.lib._leptonica", None) ffi.cdef(""" typedef signed char l_int8; typedef unsigned char l_uint8; diff --git a/setup.py b/setup.py index c141ee43..2e21cc1d 100644 --- a/setup.py +++ b/setup.py @@ -206,9 +206,13 @@ setup( "Topic :: Text Processing :: Linguistic", ], setup_requires=[ - 'setuptools_scm' + 'setuptools_scm', + 'cffi>=1.0.0' ], use_scm_version={'version_scheme': 'post-release'}, + cffi_modules=[ + 'ocrmypdf/lib/compile_leptonica.py:ffi' + ], install_requires=[ 'ruffus', 'Pillow', From ec3d92ad8e71971f606a45d476324447970ef4c4 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Sat, 30 Jan 2016 15:28:24 -0800 Subject: [PATCH 6/6] Reorg gitignore --- .gitignore | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/.gitignore b/.gitignore index 01577f03..2e738aa9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,27 +1,31 @@ -tmp/ -log/ +# Development environment *.pyc -tests/output/ -.ruffus_history.sqlite *.sublime-* -/*.pdf -build/ -dist/ -*.egg-info/ -venv/ venv-3.4/ venv-3.5/ -*/test/output -bin/ -include/ -ocrmypdf/lib/_*.py -pip-selfcheck.json +venv/ pyvenv.cfg -htmlcov/ -.coverage + +# Package building +*.egg-info/ .cache/ +.eggs/ +build/ +dist/ + +# Automatically generated files +ocrmypdf/lib/_*.py +ocrmypdf/version.py + +# Code coverage +.coverage +htmlcov/ + +# Testing +log/ +/*.pdf .ipynb_checkpoints/ tests/cache/ +tests/output/ tests/resources/private -ocrmypdf/version.py -.eggs/ \ No newline at end of file +tmp/