menekis/OCRmyPDF
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge v7.3.0 development

Browse Source
This commit is contained in:
James R. Barlow
2018-11-11 01:38:42 -08:00
parent 817d520e63 700abbb8a5
commit 9593aa4fb9
29 changed files with 1311 additions and 303 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ocrmypdf/__main__.py
+37 -7
View File
@@ -244,19 +244,38 @@ preprocessing.add_argument(
'--oversample', metavar='DPI', type=numeric(int, 0, 5000), default=0,
help="Oversample images to at least the specified DPI, to improve OCR "
"results slightly")
preprocessing.add_argument(
'--remove-vectors', action='store_true',
help="EXPERIMENTAL. Mask out any vector objects in the PDF so that they "
"will not be included in OCR. This can eliminate false characters.")
preprocessing.add_argument(
'--mask-barcodes', action='store_true',
help="EXPERIMENTAL. Mask out any barcodes that appear in the PDF so they are not "
"considered during OCR. Barcodes can introduce false characters into "
"OCR.")
preprocessing.add_argument(
'--threshold', action='store_true',
help="EXPERIMENTAL. Threshold image to 1bpp before sending it to Tesseract for OCR. Can "
"improve OCR quality compared to Tesseract's thresholder.")
ocrsettings = parser.add_argument_group(
"OCR options",
"Control how OCR is applied")
ocrsettings.add_argument(
'-f', '--force-ocr', action='store_true',
help="Rasterize any fonts or vector objects on each page, apply OCR, and "
help="Rasterize any text or vector objects on each page, apply OCR, and "
"save the rastered output (this rewrites the PDF)")
ocrsettings.add_argument(
'-s', '--skip-text', action='store_true',
help="Skip OCR on any pages that already contain text, but include the "
"page in final output; useful for PDFs that contain a mix of "
"images, text pages, and/or previously OCRed pages")
ocrsettings.add_argument(
'--redo-ocr', action='store_true',
help="Attempt to detect and remove the hidden OCR layer from files that "
"were previously OCRed with OCRmyPDF or another program. Apply OCR "
"to text found in raster images. Existing visible text objects will "
"not be changed. If there is no existing OCR, OCR will be added.")
ocrsettings.add_argument(
'--skip-big', type=numeric(float, 0, 5000), metavar='MPixels',
help="Skip OCR on pages larger than the specified amount of megapixels, "
@@ -461,12 +480,18 @@ def check_options_output(options, log):
)
lossless_reconstruction = False
if options.pdf_renderer in ('hocr', 'sandwich'):
if not any((options.deskew, options.clean_final, options.force_ocr,
options.remove_background)):
lossless_reconstruction = True
if not any((options.deskew, options.clean_final, options.force_ocr,
options.remove_background)):
lossless_reconstruction = True
options.lossless_reconstruction = lossless_reconstruction
if not options.lossless_reconstruction and options.redo_ocr:
raise argparse.ArgumentError(
None,
"--redo-ocr is not currently compatible with --deskew, "
"--clean-final, and --remove-background"
)
def check_options_sidecar(options, log):
if options.sidecar == '\0':
@@ -511,10 +536,15 @@ def check_options_preprocessing(options, log):
def check_options_ocr_behavior(options, log):
if options.force_ocr and options.skip_text:
exclusive_options = sum(
[(1 if opt else 0)
for opt in (options.force_ocr, options.skip_text, options.redo_ocr)
]
)
if exclusive_options >= 2:
raise argparse.ArgumentError(
None,
"Error: --force-ocr and --skip-text are mutually exclusive.")
"Error: choose only one of --force-ocr, --skip-text, --redo-ocr.")
def check_options_optimizing(options, log):
src/ocrmypdf/_pipeline.py
+66 -12
View File
@@ -162,6 +162,10 @@ def repair_and_parse_pdf(
options = context.get_options()
copyfile(input_file, output_file)
detailed_page_analysis = False
if options.redo_ocr:
detailed_page_analysis = True
try:
pdfinfo = PdfInfo(output_file, log=log)
except pikepdf.PasswordError as e:
@@ -193,6 +197,25 @@ def repair_and_parse_pdf(
"high page count files. Python 3.6 or newer is recommended."
)
if pdfinfo.has_acroform:
if options.redo_ocr:
log.error(
"This PDF has a user fillable form. --redo-ocr is not "
"currently possible on such files."
)
raise PriorOcrFoundError()
else:
log.warning(
"This PDF has a fillable form. Chances are it is a pure digital "
"document that does not need OCR."
)
if not options.force_ocr:
log.info(
"Use the option --force-ocr to produce an image of the "
"form and all filled form fields. The output PDF will be "
"'flattened' and will no longer be fillable."
)
context.set_pdfinfo(pdfinfo)
log.debug(pdfinfo)
@@ -243,7 +266,7 @@ def is_ocr_required(pageinfo, log, options):
if pageinfo.has_text:
msg = "{0:4d}: page already has text! – {1}"
if not options.force_ocr and not options.skip_text:
if not options.force_ocr and not (options.skip_text or options.redo_ocr):
log.error(msg.format(page,
"aborting (use --force-ocr to force OCR)"))
raise PriorOcrFoundError()
@@ -251,6 +274,18 @@ def is_ocr_required(pageinfo, log, options):
log.info(msg.format(page,
"rasterizing text and running OCR anyway"))
ocr_required = True
elif options.redo_ocr:
if pageinfo.has_corrupt_text:
log.warning(msg.format(
page,
"some text on this page cannot be mapped to characters: "
"consider using --force-ocr instead")
)
raise PriorOcrFoundError() # Wrong error but will do for now
else:
log.info(msg.format(page,
"redoing OCR"))
ocr_required = True
elif options.skip_text:
log.info(msg.format(page,
"skipping all processing on this page"))
@@ -493,7 +528,8 @@ def rasterize_with_ghostscript(
ghostscript.rasterize_pdf(
input_file, output_file, xres=canvas_dpi, yres=canvas_dpi,
raster_device=device, log=log, page_dpi=(page_dpi, page_dpi),
pageno=page_number(input_file), rotation=correction)
pageno=page_number(input_file), rotation=correction,
filter_vector=options.remove_vectors)
def preprocess_remove_background(
@@ -559,12 +595,13 @@ def select_ocr_image(
user."""
image = infiles[0]
if context.get_options().force_ocr:
options = context.get_options()
pageinfo = get_pageinfo(image, context)
if options.force_ocr:
re_symlink(image, output_file, log)
return
pageinfo = get_pageinfo(image, context)
with Image.open(image) as im:
from PIL import ImageColor
from PIL import ImageDraw
@@ -575,22 +612,39 @@ def select_ocr_image(
xres, yres = im.info['dpi']
log.debug('resolution %r %r', xres, yres)
for textarea in pageinfo.get_textareas():
mask = None # Exclude both visible and invisible text from OCR
if options.redo_ocr:
mask = True # Mask visible text, but not invisible text
for textarea in pageinfo.get_textareas(visible=mask, corrupt=None):
# Calculate resolution based on the image size and page dimensions
# without regard whatever resolution is in pageinfo (may differ or
# be None)
bbox = textarea
pixcoords = [Decimal(bbox[0]) / Decimal(72) * xres,
Decimal(bbox[1]) / Decimal(72) * yres,
Decimal(bbox[2]) / Decimal(72) * xres,
Decimal(bbox[3]) / Decimal(72) * yres]
pixcoords = [int(c) for c in pixcoords]
xscale, yscale = float(xres) / 72.0, float(yres) / 72.0
pixcoords = [bbox[0] * xscale,
im.height - bbox[3] * yscale,
bbox[2] * xscale,
im.height - bbox[1] * yscale]
pixcoords = [int(round(c)) for c in pixcoords]
log.debug('blanking %r', pixcoords)
draw.rectangle(pixcoords, fill=white)
#draw.rectangle(pixcoords, outline=pink)
del draw
if options.mask_barcodes or options.threshold:
pix = leptonica.Pix.frompil(im)
if options.threshold:
pix = pix.masked_threshold_on_background_norm()
if options.mask_barcodes:
barcodes = pix.locate_barcodes()
for barcode in barcodes:
decoded, rect = barcode
log.info('masking barcode %s %r', decoded, rect)
draw.rectangle(rect, fill=white)
im = pix.topil()
# Pillow requires integer DPI
dpi = round(xres), round(yres)
im.save(output_file, dpi=dpi)
src/ocrmypdf/_unicodefun.py
+1 -1
View File
@@ -38,7 +38,7 @@ import sys
import codecs
def verify_python3_env():
def verify_python3_env(): # pragma: no cover
"""Ensures that the environment is good for unicode on Python 3."""
# PEP 538 changes in Python 3.7 should make this wrangling unnecessary
src/ocrmypdf/_weave.py
+52 -2
View File
@@ -43,8 +43,54 @@ def _update_page_resources(*, page, font, font_key, procset):
resources['/ProcSet'] = procset
def strip_invisible_text(pdf, page, log):
stream = []
in_text_obj = False
render_mode = 0
text_objects = []
page.page_contents_coalesce()
for operands, operator in pikepdf.parse_content_stream(page, ''):
if not in_text_obj:
if operator == pikepdf.Operator('BT'):
in_text_obj = True
render_mode = 0
text_objects.append((operands, operator))
else:
stream.append((operands, operator))
else:
if operator == pikepdf.Operator('Tr'):
render_mode = operands[0]
text_objects.append((operands, operator))
if operator == pikepdf.Operator('ET'):
in_text_obj = False
if render_mode != 3:
stream.extend(text_objects)
text_objects.clear()
def convert(op):
try:
return op.unparse()
except AttributeError:
return str(op).encode('ascii')
lines = []
for operands, operator in stream:
if operator == pikepdf.Operator('INLINE IMAGE'):
iim = operands[0]
line = iim.unparse()
else:
line = b' '.join(convert(op) for op in operands) + b' ' + operator.unparse()
lines.append(line)
content_stream = b'\n'.join(lines)
page.Contents = pikepdf.Stream(pdf, content_stream)
def _weave_layers_graft(
*, pdf_base, page_num, text, font, font_key, procset, rotation, log):
*, pdf_base, page_num, text, font, font_key, procset, rotation,
strip_old_text, log):
"""Insert the text layer from text page 0 on to pdf_base at page_num"""
log.debug("Grafting")
@@ -109,6 +155,9 @@ def _weave_layers_graft(
new_text_layer = pikepdf.Stream(pdf_base, pdf_text_contents)
if strip_old_text:
strip_invisible_text(pdf_base, base_page, log)
base_page.page_contents_add(new_text_layer, prepend=True)
_update_page_resources(
@@ -336,10 +385,11 @@ def weave_layers(
if text and font:
# Graft the text layer onto this page, whether new or old
strip_old = context.get_options().redo_ocr
_weave_layers_graft(
pdf_base=pdf_base, page_num=page_num, text=text, font=font,
font_key=font_key, rotation=text_misaligned, procset=procset,
log=log
strip_old_text=strip_old, log=log
)
# Correct the rotation if applicable
src/ocrmypdf/exec/ghostscript.py
+4 -1
View File
@@ -101,7 +101,7 @@ def extract_text(input_file, pageno=1):
def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
pageno=1, page_dpi=None, rotation=None):
pageno=1, page_dpi=None, rotation=None, filter_vector=False):
"""Rasterize one page of a PDF at resolution (xres, yres) in canvas units.
The image is sized to match the integer pixels dimensions implied by
@@ -116,6 +116,8 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
:param log:
:param pageno: page number to rasterize (beginning at page 1)
:param page_dpi: resolution tuple (x, y) overriding output image DPI
:param rotation: 0, 90, 180, 270: clockwise angle to rotate page
:param filter_vector: if True, remove vector graphics objects
:return:
"""
res = xres, yres
@@ -134,6 +136,7 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
'-dFirstPage=%i' % pageno,
'-dLastPage=%i' % pageno,
'-r{0}x{1}'.format(str(int_res[0]), str(int_res[1])),
] + (['-dFILTERVECTOR'] if filter_vector else []) + [
'-o', tmp.name,
'-dAutoRotatePages=/None', # Probably has no effect on raster
'-f',
src/ocrmypdf/exec/qpdf.py
+2 -1
View File
@@ -19,6 +19,7 @@ from subprocess import CalledProcessError, STDOUT, PIPE, run
from functools import lru_cache
from . import get_version
from ..helpers import fspath
@lru_cache(maxsize=1)
@@ -30,7 +31,7 @@ def check(input_file, log=None):
args_qpdf = [
'qpdf',
'--check',
input_file
fspath(input_file)
]
if log is None:
src/ocrmypdf/exec/tesseract.py
+23 -12
View File
@@ -21,11 +21,11 @@ import shutil
from functools import lru_cache
from collections import namedtuple
from textwrap import dedent
from subprocess import CalledProcessError, TimeoutExpired, check_output, STDOUT
from subprocess import CalledProcessError, TimeoutExpired, check_output, STDOUT, run, PIPE
from contextlib import suppress
from ..exceptions import MissingDependencyError, TesseractConfigError
from ..helpers import page_number
from ..helpers import page_number, fspath
from . import get_version
OrientationConfidence = namedtuple(
@@ -92,22 +92,33 @@ def psm():
@lru_cache(maxsize=1)
def languages():
def lang_error(output):
msg = dedent("""Tesseract failed to report available languages.
Output from Tesseract:
-----------
""")
msg += output
print(msg, file=sys.stderr)
args_tess = [
'tesseract',
'--list-langs'
]
try:
langs = check_output(
args_tess, universal_newlines=True, stderr=STDOUT)
proc = run(
args_tess, universal_newlines=True, stdout=PIPE, stderr=STDOUT,
check=True
)
output = proc.stdout
except CalledProcessError as e:
msg = dedent("""Tesseract failed to report available languages.
Output from Tesseract:
-----------
""")
msg += e.output
print(msg, file=sys.stderr)
lang_error(e.output)
raise MissingDependencyError from e
return set(lang.strip() for lang in langs.splitlines()[1:])
header, *rest = output.splitlines()
if not header.startswith('List of available languages'):
lang_error(output)
raise MissingDependencyError
return set(lang.strip() for lang in rest)
def tess_base_args(langs, engine_mode):
@@ -124,7 +135,7 @@ def tess_base_args(langs, engine_mode):
def get_orientation(input_file, engine_mode, timeout: float, log):
args_tesseract = tess_base_args(['osd'], engine_mode) + [
psm(), '0',
input_file,
fspath(input_file),
'stdout'
]
src/ocrmypdf/leptonica.py
+285 -91
View File
@@ -20,14 +20,16 @@
#
# Python FFI wrapper for Leptonica library
import argparse
import sys
import os
import logging
import warnings
from tempfile import TemporaryFile
from collections.abc import Sequence
from ctypes.util import find_library
from functools import lru_cache
from io import BytesIO
from tempfile import TemporaryFile
import argparse
import logging
import os
import sys
import warnings
from .lib._leptonica import ffi
from .helpers import fspath
@@ -38,6 +40,7 @@ lept = ffi.dlopen(find_library('lept'))
logger = logging.getLogger(__name__)
lept.setMsgSeverity(lept.L_SEVERITY_WARNING)
def stderr(*objs):
"""Shorthand print to stderr."""
@@ -103,6 +106,8 @@ class _LeptonicaErrorTrap:
raise FileNotFoundError()
if 'pixWrite: stream not opened' in leptonica_output:
raise LeptonicaIOError()
if 'index not valid' in leptonica_output:
raise IndexError()
raise LeptonicaError(leptonica_output)
return False
@@ -116,7 +121,43 @@ class LeptonicaIOError(LeptonicaError):
pass
class Pix:
class LeptonicaObject:
"""General wrapper for Leptonica objects
When Leptonica returns an object, we bundled it in a wrapper class, which
manages its memory. The wrapper class assumes that it will be calling some
sort of lept.thingDestroy() function when the instance is deleted. Most
Leptonica objects are reference counted, and destroy decrements the
refcount.
Most of the time, when Leptonica returns something, we wrap and it the job
is done. When wrapping objects that came from a Leptonica container, like
a PIXA returning PIX, the subclass must clone the object before passing it
here, to maintain the reference count.
CFFI ensures that the destroy function is called at garbage collection time
so we do not need to mess with __del__.
"""
cdata_destroy = lambda cdata: None
LEPTONICA_TYPENAME = ''
def __init__(self, cdata):
if not cdata:
raise ValueError('Tried to wrap a NULL ' + self.LEPTONICA_TYPENAME)
self._cdata = ffi.gc(cdata, self._destroy)
@classmethod
def _destroy(cls, cdata):
"""Destroy some cdata"""
# Leptonica API uses double-pointers for its destroy APIs to prevent
# dangling pointers. This means we need to put our single pointer,
# cdata, in a temporary CDATA**.
pp = ffi.new('{} **'.format(cls.LEPTONICA_TYPENAME), cdata)
cls.cdata_destroy(pp)
class Pix(LeptonicaObject):
"""
Wrapper around leptonica's PIX object.
@@ -135,18 +176,17 @@ class Pix:
modified objects. This allows convenient chaining:
>>> Pix.open('filename.jpg').scale((0.5, 0.5)).deskew().show()
"""
def __init__(self, pix):
self._pix = ffi.gc(pix, Pix._pix_destroy)
LEPTONICA_TYPENAME = "PIX"
cdata_destroy = lept.pixDestroy
def __repr__(self):
if self._pix:
if self._cdata:
s = "<leptonica.Pix image size={0}x{1} depth={2}{4} at 0x{3:x}>"
return s.format(self._pix.w, self._pix.h, self._pix.d,
int(ffi.cast('intptr_t', self._pix)),
'(colormapped)' if self._pix.colormap else '')
return s.format(self._cdata.w, self._cdata.h, self._cdata.d,
int(ffi.cast('intptr_t', self._cdata)),
'(colormapped)' if self._cdata.colormap else '')
else:
return "<leptonica.Pix image NULL>"
@@ -159,7 +199,7 @@ class Pix:
data = ffi.new('l_uint8 **')
size = ffi.new('size_t *')
err = lept.pixWriteMemPng(data, size, self._pix, 0)
err = lept.pixWriteMemPng(data, size, self._cdata, 0)
if err != 0:
raise LeptonicaIOError("pixWriteMemPng")
@@ -170,7 +210,7 @@ class Pix:
data = ffi.new('l_uint32 **')
size = ffi.new('size_t *')
err = lept.pixSerializeToMemory(self._pix, data, size)
err = lept.pixSerializeToMemory(self._cdata, data, size)
if err != 0:
raise LeptonicaIOError("pixSerializeToMemory")
@@ -187,32 +227,38 @@ class Pix:
cdata_bytes = ffi.new('char[]', state['data'])
cdata_uint32 = ffi.cast('l_uint32 *', cdata_bytes)
pix = lept.pixDeserializeFromMemory(
cdata_uint32, len(state['data']))
pix = lept.pixDeserializeFromMemory(cdata_uint32, len(state['data']))
Pix.__init__(self, pix)
def __eq__(self, other):
return self.__getstate__() == other.__getstate__()
if not isinstance(other, Pix):
return NotImplemented
same = ffi.new('l_int32 *', 0)
with _LeptonicaErrorTrap():
err = lept.pixEqual(self._cdata, other._cdata, same)
if err:
raise TypeError()
return bool(same[0])
@property
def width(self):
return self._pix.w
return self._cdata.w
@property
def height(self):
return self._pix.h
return self._cdata.h
@property
def depth(self):
return self._pix.d
return self._cdata.d
@property
def size(self):
return (self._pix.w, self._pix.h)
return (self._cdata.w, self._cdata.h)
@property
def info(self):
return {'dpi': (self._pix.xres, self._pix.yres)}
return {'dpi': (self._cdata.xres, self._cdata.yres)}
@property
def mode(self):
@@ -221,7 +267,7 @@ class Pix:
return '1'
elif self.depth >= 16:
return 'RGB'
elif not self._pix.colormap:
elif not self._cdata.colormap:
return 'L'
else:
return 'P'
@@ -253,10 +299,21 @@ class Pix:
with _LeptonicaErrorTrap():
lept.pixWriteImpliedFormat(
os.fsencode(filename),
self._pix, jpeg_quality, jpeg_progressive)
self._cdata, jpeg_quality, jpeg_progressive)
@classmethod
def frompil(self, pillow_image):
"""Create a copy of a PIL.Image from this Pix"""
bio = BytesIO()
pillow_image.save(bio, format='png', compress_level=1)
py_buffer = bio.getbuffer()
c_buffer = ffi.from_buffer(py_buffer)
with _LeptonicaErrorTrap():
pix = Pix(lept.pixReadMem(c_buffer, len(c_buffer)))
return pix
def topil(self):
"Returns a PIL.Image version of this Pix"
"""Returns a PIL.Image version of this Pix"""
from PIL import Image
# Leptonica manages data in words, so it implicitly does an endian
@@ -269,17 +326,17 @@ class Pix:
raw_mode = 'ABGR'
elif self.mode == '1':
raw_mode = '1;I'
pix = Pix(lept.pixEndianByteSwapNew(pix._pix))
pix = Pix(lept.pixEndianByteSwapNew(pix._cdata))
else:
raw_mode = self.mode
pix = Pix(lept.pixEndianByteSwapNew(pix._pix))
pix = Pix(lept.pixEndianByteSwapNew(pix._cdata))
else:
raw_mode = self.mode # no endian swap needed
size = (pix._pix.w, pix._pix.h)
bytecount = pix._pix.wpl * 4 * pix._pix.h
buf = ffi.buffer(pix._pix.data, bytecount)
stride = pix._pix.wpl * 4
size = (pix._cdata.w, pix._cdata.h)
bytecount = pix._cdata.wpl * 4 * pix._cdata.h
buf = ffi.buffer(pix._cdata.data, bytecount)
stride = pix._cdata.wpl * 4
im = Image.frombytes(self.mode, size, buf, 'raw', raw_mode, stride)
@@ -298,21 +355,21 @@ class Pix:
for skew angle
"""
with _LeptonicaErrorTrap():
return Pix(lept.pixDeskew(self._pix, reduction_factor))
return Pix(lept.pixDeskew(self._cdata, reduction_factor))
def scale(self, scale_xy):
"Returns the pix object rescaled according to the proportions given."
with _LeptonicaErrorTrap():
return Pix(lept.pixScale(self._pix, scale_xy[0], scale_xy[1]))
return Pix(lept.pixScale(self._cdata, scale_xy[0], scale_xy[1]))
def rotate180(self):
with _LeptonicaErrorTrap():
return Pix(lept.pixRotate180(ffi.NULL, self._pix))
return Pix(lept.pixRotate180(ffi.NULL, self._cdata))
def rotate_orth(self, quads):
"Orthographic rotation, quads: 0-3, number of clockwise rotations"
with _LeptonicaErrorTrap():
return Pix(lept.pixRotateOrth(self._pix, quads))
return Pix(lept.pixRotateOrth(self._cdata, quads))
def find_skew(self):
"""Returns a tuple (deskew angle in degrees, confidence value).
@@ -322,7 +379,7 @@ class Pix:
with _LeptonicaErrorTrap():
angle = ffi.new('float *', 0.0)
confidence = ffi.new('float *', 0.0)
result = lept.pixFindSkew(self._pix, angle, confidence)
result = lept.pixFindSkew(self._cdata, angle, confidence)
if result == 0:
return (angle[0], confidence[0])
else:
@@ -330,7 +387,7 @@ class Pix:
def convert_rgb_to_luminance(self):
with _LeptonicaErrorTrap():
gray_pix = lept.pixConvertRGBToLuminance(self._pix)
gray_pix = lept.pixConvertRGBToLuminance(self._cdata)
if gray_pix:
return Pix(gray_pix)
return None
@@ -344,7 +401,7 @@ class Pix:
"""
with _LeptonicaErrorTrap():
return Pix(lept.pixRemoveColormapGeneral(
self._pix, removal_type, lept.L_COPY))
self._cdata, removal_type, lept.L_COPY))
def otsu_adaptive_threshold(
self, tile_size=(300, 300), kernel_size=(4, 4), scorefract=0.1):
@@ -354,7 +411,7 @@ class Pix:
p_pix = ffi.new('PIX **')
result = lept.pixOtsuAdaptiveThreshold(
self._pix,
self._cdata,
sx, sy,
smoothx, smoothy,
scorefract,
@@ -371,22 +428,41 @@ class Pix:
with _LeptonicaErrorTrap():
sx, sy = tile_size
smoothx, smoothy = kernel_size
if mask is None:
mask = ffi.NULL
mask = ffi.NULL
if isinstance(mask, Pix):
mask = mask._pix
mask = mask._cdata
thresh_pix = lept.pixOtsuThreshOnBackgroundNorm(
self._pix,
self._cdata,
mask,
sx, sy,
thresh, mincount, bgval,
smoothx, smoothy,
scorefract,
ffi.NULL
)
if thresh_pix == ffi.NULL:
return None
)
return Pix(thresh_pix)
def masked_threshold_on_background_norm(
self, mask=None, tile_size=(10, 15), thresh=100, mincount=50,
kernel_size=(2, 2), scorefract=0.1):
with _LeptonicaErrorTrap():
sx, sy = tile_size
smoothx, smoothy = kernel_size
mask = ffi.NULL
if isinstance(mask, Pix):
mask = mask._cdata
pix = Pix(lept.pixConvertTo8(self._cdata, 0))
thresh_pix = lept.pixMaskedThreshOnBackgroundNorm(
pix._cdata,
mask,
sx, sy,
thresh, mincount,
smoothx, smoothy,
scorefract,
ffi.NULL
)
return Pix(thresh_pix)
def crop_to_foreground(
@@ -394,7 +470,7 @@ class Pix:
showmorph=0, display=0, pdfdir=ffi.NULL):
with _LeptonicaErrorTrap():
cropbox = Box(lept.pixFindPageForeground(
self._pix,
self._cdata,
threshold,
mindist,
erasedist,
@@ -403,11 +479,9 @@ class Pix:
display,
pdfdir))
print(repr(cropbox))
cropped_pix = lept.pixClipRectangle(
self._pix,
cropbox._box,
self._cdata,
cropbox._cdata,
ffi.NULL)
return Pix(cropped_pix)
@@ -416,7 +490,7 @@ class Pix:
self, mask=None, grayscale=None, gamma=1.0, black=0, white=255):
with _LeptonicaErrorTrap():
return Pix(lept.pixCleanBackgroundToWhite(
self._pix,
self._cdata,
mask or ffi.NULL,
grayscale or ffi.NULL,
gamma,
@@ -427,7 +501,7 @@ class Pix:
with _LeptonicaErrorTrap():
return Pix(lept.pixGammaTRC(
ffi.NULL,
self._pix,
self._cdata,
gamma,
minval,
maxval
@@ -440,7 +514,7 @@ class Pix:
target_pix = self.remove_colormap(lept.REMOVE_CMAP_BASED_ON_SRC)
with _LeptonicaErrorTrap():
return Pix(lept.pixBackgroundNorm(
target_pix._pix,
target_pix._cdata,
mask or ffi.NULL,
grayscale or ffi.NULL,
tile_size[0],
@@ -467,7 +541,7 @@ class Pix:
raise LeptonicaError("Leptonica version is too old")
correlation = ffi.new('float *', 0.0)
result = lept.pixCorrelationBinary(pix1._pix, pix2._pix,
result = lept.pixCorrelationBinary(pix1._cdata, pix2._cdata,
correlation)
if result != 0:
raise LeptonicaError("Correlation failed")
@@ -476,25 +550,68 @@ class Pix:
def generate_pdf_ci_data(self, type_, quality):
"Convert to PDF data, with transcoding"
p_compdata = ffi.new('L_COMP_DATA **')
result = lept.pixGenerateCIData(self._pix, type_, quality, 0,
result = lept.pixGenerateCIData(self._cdata, type_, quality, 0,
p_compdata)
if result != 0:
raise LeptonicaError("Generate PDF data failed")
return CompressedData(p_compdata[0])
def invert(self):
return Pix(lept.pixInvert(ffi.NULL, self._pix))
return Pix(lept.pixInvert(ffi.NULL, self._cdata))
@staticmethod
def _pix_destroy(pix):
p_pix = ffi.new('PIX **', pix)
lept.pixDestroy(p_pix)
# print('pix destroy ' + repr(pix))
def locate_barcodes(self):
with _LeptonicaErrorTrap():
pix = Pix(lept.pixConvertTo8(self._cdata, 0))
pixa_candidates = PixArray(lept.pixExtractBarcodes(pix._cdata, 0))
sarray = StringArray(lept.pixReadBarcodes(pixa_candidates._cdata,
lept.L_BF_ANY,
lept.L_USE_WIDTHS,
ffi.NULL,
0))
for n, s in enumerate(sarray):
decoded = s.decode()
if s.strip() == '':
continue
box = pixa_candidates.get_box(n)
left, top = box.x, box.y
right, bottom = box.x + box.w, box.y + box.h
yield (decoded, (left, top, right, bottom))
def despeckle(self, size):
if size == 2:
speckle2 = """
oooo
oC o
o o
oooo
"""
sel1 = Sel.from_selstr(speckle2, 'speckle2')
sel2 = Sel.create_brick(2, 2, 0, 0, lept.SEL_HIT)
elif size == 3:
speckle3 = """
ooooo
oC o
o o
o o
ooooo
"""
sel1 = Sel.from_selstr(speckle3, 'speckle3')
sel2 = Sel.create_brick(3, 3, 0, 0, lept.SEL_HIT)
else:
raise ValueError(size)
pixhmt = Pix(lept.pixHMT(ffi.NULL, self._cdata, sel1._cdata))
pixdilated = Pix(lept.pixDilate(ffi.NULL, pixhmt._cdata, sel2._cdata))
pixsub = Pix(lept.pixSubtract(ffi.NULL, self._cdata, pixdilated._cdata))
return pixsub
class CompressedData:
def __init__(self, compdata):
self._compdata = ffi.gc(compdata, CompressedData._destroy)
class CompressedData(LeptonicaObject):
"""Wrapper for L_COMP_DATA - abstract compressed image data"""
LEPTONICA_TYPENAME = 'L_COMP_DATA'
cdata_destroy = lept.l_CIDataDestroy
@classmethod
def open(cls, path, jpeg_quality=75):
@@ -509,64 +626,141 @@ class CompressedData:
return CompressedData(p_compdata[0])
def __len__(self):
return self._compdata.nbytescomp
return self._cdata.nbytescomp
def read(self):
buf = ffi.buffer(self._compdata.datacomp, self._compdata.nbytescomp)
buf = ffi.buffer(self._cdata.datacomp, self._cdata.nbytescomp)
return bytes(buf)
def __getattr__(self, name):
if hasattr(self._compdata, name):
return getattr(self._compdata, name)
if hasattr(self._cdata, name):
return getattr(self._cdata, name)
raise AttributeError(name)
def get_palette_pdf_string(self):
"Returns palette pre-formatted for use in PDF"
buflen = len('< ') + len(' rrggbb') * self._compdata.ncolors + len('>')
buf = ffi.buffer(self._compdata.cmapdatahex, buflen)
buflen = len('< ') + len(' rrggbb') * self._cdata.ncolors + len('>')
buf = ffi.buffer(self._cdata.cmapdatahex, buflen)
return bytes(buf)
@staticmethod
def _destroy(compdata):
pp = ffi.new('L_COMP_DATA **', compdata)
lept.l_CIDataDestroy(pp)
class PixArray(LeptonicaObject, Sequence):
"""Wrapper around PIXA (array of PIX)"""
LEPTONICA_TYPENAME = 'PIXA'
cdata_destroy = lept.pixaDestroy
def __len__(self):
return self._cdata[0].n
def __getitem__(self, n):
with _LeptonicaErrorTrap():
return Pix(lept.pixaGetPix(self._cdata, n, lept.L_CLONE))
def get_box(self, n):
with _LeptonicaErrorTrap():
return Box(lept.pixaGetBox(self._cdata, n, lept.L_CLONE))
class Box:
"""Wrapper around Leptonica's BOX objects.
class Box(LeptonicaObject):
"""Wrapper around Leptonica's BOX objects (a pixel rectangle)
See class Pix for notes about reference counting.
Uses x, y, w, h coordinates.
"""
def __init__(self, box):
self._box = ffi.gc(box, Box._box_destroy)
LEPTONICA_TYPENAME = 'BOX'
cdata_destroy = lept.boxDestroy
def __repr__(self):
if self._box:
if self._cdata:
return '<leptonica.Box x={0} y={1} w={2} h={3}>'.format(
self.x, self.y, self.w, self.h)
return '<leptonica.Box NULL>'
@property
def x(self):
return self._box.x
return self._cdata.x
@property
def y(self):
return self._box.y
return self._cdata.y
@property
def w(self):
return self._box.w
return self._cdata.w
@property
def h(self):
return self._box.h
return self._cdata.h
@staticmethod
def _box_destroy(box):
p_box = ffi.new('BOX **', box)
lept.boxDestroy(p_box)
class BoxArray(LeptonicaObject, Sequence):
"""Wrapper around Leptonica's BOXA (Array of BOX) objects."""
LEPTONICA_TYPENAME = 'BOXA'
cdata_destroy = lept.boxaDestroy
def __repr__(self):
if not self._cdata:
return '<BoxArray>'
boxes = (repr(box) for box in self)
return '<BoxArray [' + ', '.join(boxes) + ']>'
def __len__(self):
return self._cdata.n
def __getitem__(self, n):
if not isinstance(n, int):
raise TypeError('list indices must be integers')
if 0 <= n < len(self):
return Box(lept.boxaGetBox(self._cdata, n, lept.L_CLONE))
raise IndexError(n)
class StringArray(LeptonicaObject, Sequence):
"""Leptonica SARRAY/string array"""
LEPTONICA_TYPENAME = 'SARRAY'
cdata_destroy = lept.sarrayDestroy
def __len__(self):
return self._cdata.n
def __getitem__(self, n):
if 0 <= n < len(self):
return ffi.string(self._cdata.array[n])
raise IndexError(n)
class Sel(LeptonicaObject):
"""Leptonica 'sel'/selection element for hit-miss transform"""
LEPTONICA_TYPENAME = 'SEL'
cdata_destroy = lept.selDestroy
@classmethod
def from_selstr(cls, selstr, name):
lines = [line.strip() for line in selstr.split('\n') if line.strip()]
h = len(lines)
w = len(lines[0])
lengths = set(len(line) for line in lines)
if len(lengths) != 1:
raise ValueError("All lines in selstr must be same length")
repacked = ''.join(line.strip() for line in lines)
buf_selstr = ffi.from_buffer(repacked.encode('ascii'))
buf_name = ffi.from_buffer(name.encode('ascii'))
sel = lept.selCreateFromString(buf_selstr, h, w, buf_name)
return cls(sel)
@classmethod
def create_brick(cls, h, w, cy, cx, type_):
sel = lept.selCreateBrick(h, w, cy, cx, type_)
return cls(sel)
def __repr__(self):
selstr = ffi.gc(lept.selPrintToString(self._cdata), lept.lept_free)
return '<Sel \n' + ffi.string(selstr).decode('ascii') + '\n>'
@lru_cache(maxsize=1)
src/ocrmypdf/lib/_leptonica.py
+5 -5
View File
File diff suppressed because one or more lines are too long
src/ocrmypdf/lib/compile_leptonica.py
+188 -9
View File
@@ -19,7 +19,6 @@
from cffi import FFI
ffibuilder = FFI()
ffibuilder.set_source("ocrmypdf.lib._leptonica", None)
ffibuilder.cdef("""
typedef signed char l_int8;
typedef unsigned char l_uint8;
@@ -32,6 +31,8 @@ typedef double l_float64;
typedef long long l_int64;
typedef unsigned long long l_uint64;
typedef int l_ok; /*!< return type 0 if OK, 1 on error */
struct Pix
{
l_uint32 w; /* width in pixels */
@@ -61,6 +62,17 @@ struct PixColormap
};
typedef struct PixColormap PIXCMAP;
/*! Array of pix */
struct Pixa
{
l_int32 n; /*!< number of Pix in ptr array */
l_int32 nalloc; /*!< number of Pix ptrs allocated */
l_uint32 refcount; /*!< reference count (1 if no clones) */
struct Pix **pix; /*!< the array of ptrs to pix */
struct Boxa *boxa; /*!< array of boxes */
};
typedef struct Pixa PIXA;
struct Box
{
l_int32 x;
@@ -72,6 +84,25 @@ struct Box
};
typedef struct Box BOX;
/*! Array of Box */
struct Boxa
{
l_int32 n; /*!< number of box in ptr array */
l_int32 nalloc; /*!< number of box ptrs allocated */
l_uint32 refcount; /*!< reference count (1 if no clones) */
struct Box **box; /*!< box ptr array */
};
typedef struct Boxa BOXA;
/*! String array: an array of C strings */
struct Sarray
{
l_int32 nalloc; /*!< size of allocated ptr array */
l_int32 n; /*!< number of strings allocated */
l_int32 refcount; /*!< reference count (1 if no clones) */
char **array; /*!< string array */
};
typedef struct Sarray SARRAY;
/*! Pdf formatted encoding types */
enum {
@@ -104,6 +135,18 @@ struct L_Compressed_Data
};
typedef struct L_Compressed_Data L_COMP_DATA;
/*! Selection */
struct Sel
{
l_int32 sy; /*!< sel height */
l_int32 sx; /*!< sel width */
l_int32 cy; /*!< y location of sel origin */
l_int32 cx; /*!< x location of sel origin */
l_int32 **data; /*!< {0,1,2}; data[i][j] in [row][col] order */
char *name; /*!< used to find sel by name */
};
typedef struct Sel SEL;
enum {
REMOVE_CMAP_TO_BINARY = 0, /*!< remove colormap for conv to 1 bpp */
REMOVE_CMAP_TO_GRAYSCALE = 1, /*!< remove colormap for conv to 8 bpp */
@@ -122,10 +165,48 @@ enum {
/*!< the array with clones (e.g., pix) */
};
/*! Flags for method of extracting barcode widths */
enum {
L_USE_WIDTHS = 1, /*!< use histogram of barcode widths */
L_USE_WINDOWS = 2 /*!< find best window for decoding transitions */
};
/*! Flags for barcode formats */
enum {
L_BF_UNKNOWN = 0, /*!< unknown format */
L_BF_ANY = 1, /*!< try decoding with all known formats */
L_BF_CODE128 = 2, /*!< decode with Code128 format */
L_BF_EAN8 = 3, /*!< decode with EAN8 format */
L_BF_EAN13 = 4, /*!< decode with EAN13 format */
L_BF_CODE2OF5 = 5, /*!< decode with Code 2 of 5 format */
L_BF_CODEI2OF5 = 6, /*!< decode with Interleaved 2 of 5 format */
L_BF_CODE39 = 7, /*!< decode with Code39 format */
L_BF_CODE93 = 8, /*!< decode with Code93 format */
L_BF_CODABAR = 9, /*!< decode with Code93 format */
L_BF_UPCA = 10 /*!< decode with UPC A format */
};
enum {
L_SEVERITY_EXTERNAL = 0, /* Get the severity from the environment */
L_SEVERITY_ALL = 1, /* Lowest severity: print all messages */
L_SEVERITY_DEBUG = 2, /* Print debugging and higher messages */
L_SEVERITY_INFO = 3, /* Print informational and higher messages */
L_SEVERITY_WARNING = 4, /* Print warning and higher messages */
L_SEVERITY_ERROR = 5, /* Print error and higher messages */
L_SEVERITY_NONE = 6 /* Highest severity: print no messages */
};
enum {
SEL_DONT_CARE = 0,
SEL_HIT = 1,
SEL_MISS = 2
};
""")
ffibuilder.cdef("""
PIX * pixRead ( const char *filename );
PIX * pixReadMem ( const l_uint8 *data, size_t size );
PIX * pixScale ( PIX *pixs, l_float32 scalex, l_float32 scaley );
l_int32 pixFindSkew ( PIX *pixs, l_float32 *pangle, l_float32 *pconf );
l_int32 pixWriteImpliedFormat ( const char *filename, PIX *pix, l_int32 quality, l_int32 progressive );
@@ -137,6 +218,11 @@ pixWriteMemPng(l_uint8 **pdata,
void pixDestroy ( PIX **ppix );
l_ok
pixEqual(PIX *pix1,
PIX *pix2,
l_int32 *psame);
PIX *
pixEndianByteSwapNew(PIX *pixs);
@@ -157,6 +243,8 @@ l_int32 pixSerializeToMemory ( PIX *pixs, l_uint32 **pdata, size_t *pnbytes );
PIX * pixConvertRGBToLuminance(PIX *pixs);
PIX * pixConvertTo8(PIX *pixs, l_int32 cmapflag);
PIX * pixRemoveColormap(PIX *pixs, l_int32 type);
l_int32
@@ -182,6 +270,18 @@ pixOtsuThreshOnBackgroundNorm(PIX *pixs,
l_float32 scorefract,
l_int32 *pthresh);
PIX *
pixMaskedThreshOnBackgroundNorm(PIX *pixs,
PIX *pixim,
l_int32 sx,
l_int32 sy,
l_int32 thresh,
l_int32 mincount,
l_int32 smoothx,
l_int32 smoothy,
l_float32 scorefract,
l_int32 *pthresh);
PIX *
pixCleanBackgroundToWhite(PIX *pixs,
PIX *pixim,
@@ -265,13 +365,13 @@ pixGetAverageMaskedRGB(PIX *pixs,
l_float32 *pgval,
l_float32 *pbval);
PIX *
PIX *
pixGlobalNormRGB(PIX * pixd,
PIX * pixs,
l_int32 rval,
l_int32 gval,
l_int32 bval,
l_int32 mapval);
l_int32 mapval);
PIX *
pixInvert(PIX * pixd,
@@ -289,22 +389,101 @@ pixGenerateCIData(PIX *pixs,
l_int32 ascii85,
L_COMP_DATA **pcid);
l_int32
l_generateCIDataForPdf(const char *fname,
PIX *pix,
l_int32 quality,
SARRAY *
pixProcessBarcodes(PIX *pixs,
l_int32 format,
l_int32 method,
SARRAY **psaw,
l_int32 debugflag);
PIX *
pixaGetPix(PIXA *pixa,
l_int32 index,
l_int32 accesstype);
BOX*
pixaGetBox (PIXA * pixa,
l_int32 index,
l_int32 accesstype );
PIXA *
pixExtractBarcodes(PIX *pixs,
l_int32 debugflag);
BOXA *
pixLocateBarcodes ( PIX *pixs, l_int32 thresh, PIX **ppixb, PIX **ppixm );
SARRAY *
pixReadBarcodes(PIXA *pixa,
l_int32 format,
l_int32 method,
SARRAY **psaw,
l_int32 debugflag);
l_int32
l_generateCIDataForPdf(const char *fname,
PIX *pix,
l_int32 quality,
L_COMP_DATA **pcid);
void
BOX *
boxClone ( BOX *box );
BOX *
boxaGetBox ( BOXA *boxa, l_int32 index, l_int32 accessflag );
SEL *
selCreateFromString ( const char *text, l_int32 h, l_int32 w, const char *name );
SEL *
selCreateBrick ( l_int32 h, l_int32 w, l_int32 cy, l_int32 cx, l_int32 type );
char *
selPrintToString(SEL *sel);
PIX *
pixDilate ( PIX *pixd, PIX *pixs, SEL *sel );
PIX *
pixErode ( PIX *pixd, PIX *pixs, SEL *sel );
PIX *
pixHMT ( PIX *pixd, PIX *pixs, SEL *sel );
PIX *
pixSubtract ( PIX *pixd, PIX *pixs1, PIX *pixs2 );
void
boxDestroy(BOX **pbox);
void
boxaDestroy ( BOXA **pboxa );
void
pixaDestroy(PIXA **ppixa);
l_ok
pixRenderBoxa ( PIX *pix, BOXA *boxa, l_int32 width, l_int32 op );
void
l_CIDataDestroy(L_COMP_DATA **pcid);
void
lept_free(void *ptr);
sarrayDestroy(SARRAY **psa);
void
lept_free(void *ptr);
void selDestroy ( SEL **psel );
l_int32
setMsgSeverity(l_int32 newsev);
""")
ffibuilder.set_source("ocrmypdf.lib._leptonica", None)
if __name__ == '__main__':
ffibuilder.compile(verbose=True)
src/ocrmypdf/pdfinfo.py → src/ocrmypdf/pdfinfo/__init__.py
+99 -113
View File
@@ -23,14 +23,16 @@ from math import hypot, isclose
from pathlib import Path
from unittest.mock import Mock
import re
import xml.etree.ElementTree as ET
from .exec import ghostscript
from .helpers import fspath
from pikepdf import PdfMatrix
import pikepdf
from . import ghosttext
from .layout import get_page_analysis, get_text_boxes
from ..helpers import fspath
Colorspace = Enum('Colorspace',
'gray rgb cmyk lab icc index sep devn pattern jpeg2000')
@@ -38,17 +40,6 @@ Encoding = Enum('Encoding',
'ccitt jpeg jpeg2000 jbig2 asciihex ascii85 lzw flate ' + \
'runlength')
# Forgive me for I have sinned
# I am using regular expressions to parse XML. However the XML in this case,
# generated by Ghostscript, is self-consistent enough to be parseable.
regex_remove_char_tags = re.compile(br"""
<char\b
(?: [^>] # anything single character but >
| \">\" # special case: trap ">"
)*
/> # terminate with '/>'
""", re.VERBOSE)
FRIENDLY_COLORSPACE = {
'/DeviceGray': Colorspace.gray,
'/CalGray': Colorspace.gray,
@@ -104,7 +95,15 @@ InlineSettings = namedtuple('InlineSettings',
['iimage', 'shorthand', 'stack_depth'])
ContentsInfo = namedtuple('ContentsInfo',
['xobject_settings', 'inline_images', 'found_text', 'found_vector'])
['xobject_settings', 'inline_images', 'found_vector'])
TextboxInfo = namedtuple('TextboxInfo',
['bbox', 'is_visible', 'is_corrupt'])
class VectorInfo:
def __init__(self):
pass
def _normalize_stack(graphobjs):
@@ -142,10 +141,10 @@ def _interpret_contents(contentstream, initial_shorthand=UNIT_SQUARE):
ctm = PdfMatrix(initial_shorthand)
xobject_settings = []
inline_images = []
found_text, found_vector = False, False
text_operators = set("""Tj " ' TJ""".split())
vector_operators = set('S s f F f* B B* b b*'.split())
operator_whitelist = """q Q Do cm TJ Tj " ' BI ID EI S s f F f* B B* b b*"""
found_vector = False
vector_ops = set('S s f F f* B B* b b*'.split())
image_ops = set('BI ID EI q Q Do cm'.split())
operator_whitelist = ' '.join(vector_ops | image_ops)
for n, graphobj in enumerate(_normalize_stack(
pikepdf.parse_content_stream(contentstream, operator_whitelist))):
@@ -176,15 +175,12 @@ def _interpret_contents(contentstream, initial_shorthand=UNIT_SQUARE):
iimage=iimage, shorthand=ctm.shorthand,
stack_depth=len(stack))
inline_images.append(inline)
elif operator in text_operators:
found_text = True
elif operator in vector_operators:
elif operator in vector_ops:
found_vector = True
return ContentsInfo(
xobject_settings=xobject_settings,
inline_images=inline_images,
found_text=found_text,
found_vector=found_vector)
@@ -252,11 +248,6 @@ def _get_dpi(ctm_shorthand, image_size):
return dpi_w, dpi_h
class VectorInfo:
def __init__(self):
pass
class ImageInfo:
DPI_PREC = Decimal('1.000')
@@ -394,7 +385,6 @@ def _find_regular_images(container, contentsinfo):
that contains images.
Generates images with their DPI at time of drawing.
"""
for pdfimage, xobj in _image_xobjects(container):
@@ -444,11 +434,11 @@ def _find_form_xobject_images(pdf, container, contentsinfo):
# but in practice both Form XObjects and multiple drawing of the
# same object are both very rare.
ctm_shorthand = settings.shorthand
yield from _find_images(
yield from _process_content_streams(
pdf=pdf, container=form_xobject, shorthand=ctm_shorthand)
def _find_images(*, pdf, container, shorthand=None):
def _process_content_streams(*, pdf, container, shorthand=None):
"""Find all individual instances of images drawn in the container
Usually the container is a page, but it may also be a Form XObject.
@@ -496,43 +486,6 @@ def _find_images(*, pdf, container, shorthand=None):
yield from _find_form_xobject_images(pdf, container, contentsinfo)
def _page_get_textblocks(infile, pageno, xmltext):
"""Smarter text detection"""
root = xmltext
if not hasattr(xmltext, 'findall'):
return []
def blocks():
for span in root.findall('.//span'):
bbox_str = span.attrib['bbox']
font_size = span.attrib['size']
pts = [int(pt) for pt in bbox_str.split()]
pts[1] = pts[1] - int(float(font_size) + 0.5)
bbox = tuple(pts)
yield bbox
def joined_blocks():
prev = None
for bbox in blocks():
if prev is None:
prev = bbox
if bbox[1] == prev[1] and bbox[3] == prev[3]:
gap = prev[2] - bbox[0]
height = bbox[3] - bbox[1]
if gap < height:
# Join boxes
prev = (prev[0], prev[1], bbox[2], bbox[3])
continue
# yield previously joined bboxes and start anew
yield prev
prev = bbox
if prev is not None:
yield prev
return [block for block in joined_blocks()]
def _page_has_text(text_blocks, page_width, page_height):
"""Smarter text detection that ignores text in margins"""
@@ -540,18 +493,19 @@ def _page_has_text(text_blocks, page_width, page_height):
margin_ratio = 0.125
interior_bbox = (
margin_ratio * pw, margin_ratio * ph,
(1 - margin_ratio) * pw, (1 - margin_ratio) * ph
margin_ratio * pw, # left
(1 - margin_ratio) * ph, # top
(1 - margin_ratio) * pw, # right
margin_ratio * ph # bottom (first quadrant: bottom < top)
)
def rects_intersect(a, b):
"""
Where (a,b) are 4-tuple rects (left-0, top-1, right-2, bottom-3)
https://stackoverflow.com/questions/306316/determine-if-two-rectangles-overlap-each-other
Negative signs to account for our coordinates being in the fourth quadrant
and the formula assuming the first
Formula assumes all boxes are in first quadrant
"""
return a[0] < b[2] and a[2] > b[0] and -a[1] > -b[3] and -a[3] < -b[1]
return a[0] < b[2] and a[2] > b[0] and a[1] > b[3] and a[3] < b[1]
has_text = False
for bbox in text_blocks:
@@ -561,22 +515,43 @@ def _page_has_text(text_blocks, page_width, page_height):
return has_text
def simplify_textboxes(miner):
"""Extract only limited content from text boxes
We do this to save memory and ensure that our objects are pickleable.
"""
for box in get_text_boxes(miner):
first_line = box._objs[0]
first_char = first_line._objs[0]
visible = (first_char.rendermode != 3)
corrupt = (first_char.get_text() == '\ufffd')
yield TextboxInfo(box.bbox, visible, corrupt)
def _pdf_get_pageinfo(pdf, pageno: int, infile, xmltext):
pageinfo = {}
pageinfo['pageno'] = pageno
pageinfo['images'] = []
page = pdf.pages[pageno]
pageinfo['textinfo'] = _page_get_textblocks(
fspath(infile), pageno, xmltext=xmltext)
mediabox = [Decimal(d) for d in page.MediaBox.as_list()]
width_pt = mediabox[2] - mediabox[0]
height_pt = mediabox[3] - mediabox[1]
if xmltext:
bboxes = ghosttext.page_get_textblocks(
fspath(infile), pageno, xmltext=xmltext, height=height_pt)
pageinfo['bboxes'] = bboxes
else:
pscript5_mode = str(pdf.metadata.get('/Creator')).startswith('PScript5')
miner = get_page_analysis(infile, pageno, pscript5_mode)
pageinfo['textboxes'] = list(simplify_textboxes(miner))
bboxes = (box.bbox for box in pageinfo['textboxes'])
pageinfo['has_text'] = _page_has_text(
pageinfo['textinfo'], width_pt, height_pt)
bboxes, width_pt, height_pt
)
userunit = page.get('/UserUnit', Decimal(1.0))
if not isinstance(userunit, Decimal):
@@ -591,16 +566,16 @@ def _pdf_get_pageinfo(pdf, pageno: int, infile, xmltext):
pageinfo['rotate'] = 0
userunit_shorthand = (userunit, 0, 0, userunit, 0, 0)
pageinfo['images'] = [im for im in
_find_images(pdf=pdf, container=page,
shorthand=userunit_shorthand)]
contentsinfo = [ci for ci in
_process_content_streams(pdf=pdf, container=page,
shorthand=userunit_shorthand)]
pageinfo['has_vector'] = False
if any(isinstance(im, VectorInfo) for im in pageinfo['images']):
if any(isinstance(ci, VectorInfo) for ci in contentsinfo):
pageinfo['has_vector'] = True
pageinfo['images'] = [im for im in pageinfo['images']
if not isinstance(im, VectorInfo)]
pageinfo['images'] = [im for im in contentsinfo
if isinstance(im, ImageInfo)]
if pageinfo['images']:
xres = Decimal(max(image.xres for image in pageinfo['images']))
yres = Decimal(max(image.yres for image in pageinfo['images']))
@@ -613,37 +588,20 @@ def _pdf_get_pageinfo(pdf, pageno: int, infile, xmltext):
return pageinfo
def _pdf_get_all_pageinfo(infile, log=None):
def _pdf_get_all_pageinfo(infile, detailed_page_analysis, log=None):
if not log:
log = Mock()
pdf = pikepdf.open(infile)
existing_text = ghostscript.extract_text(infile, pageno=None)
existing_text = regex_remove_char_tags.sub(b' ', existing_text)
try:
root = ET.fromstringlist([
b'<document>\n', existing_text, b'</document>\n'
])
page_xml = root.findall('page')
except ET.ParseError as e:
log.error(
"An error occurred while attempting to retrieve existing text in "
"the input file. Will attempt to continue assuming that there is "
"no existing text in the file. The error was:")
log.error(e)
page_xml = [None] * len(pdf.pages)
page_count_difference = len(pdf.pages) - len(page_xml)
if page_count_difference != 0:
log.error("The number of pages in the input file is inconsistent.")
if page_count_difference > 0:
page_xml.extend([None] * page_count_difference)
if not detailed_page_analysis:
pages_xml = None
else:
pages_xml = ghosttext.extract_text_xml(infile, pdf, pageno=None, log=log)
pages = []
for n in range(len(pdf.pages)):
page = PageInfo(pdf, n, infile, page_xml[n])
page_xml = pages_xml[n] if pages_xml else None
page = PageInfo(pdf, n, infile, page_xml)
pages.append(page)
return pages, pdf
@@ -663,6 +621,10 @@ class PageInfo:
def has_text(self):
return self._pageinfo['has_text']
@property
def has_corrupt_text(self):
return any(tbox.is_corrupt for tbox in self._pageinfo['textboxes'])
@property
def has_vector(self):
return self._pageinfo['has_vector']
@@ -698,8 +660,26 @@ class PageInfo:
def images(self):
return self._pageinfo['images']
def get_textareas(self):
yield from self._pageinfo['textinfo']
def get_textareas(self, visible=None, corrupt=None):
def predicate(obj, want_visible, want_corrupt):
result = True
if want_visible is not None:
if obj.is_visible != want_visible:
result = False
if want_corrupt is not None:
if obj.is_corrupt != want_corrupt:
result = False
return result
if 'textboxes' not in self._pageinfo:
if visible is not None and corrupt is not None:
raise NotImplementedError(
'Ghostscript textboxes cannot be classified')
return self._pageinfo['bboxes']
return (obj.bbox for obj in self._pageinfo['textboxes']
if predicate(obj, visible, corrupt))
@property
def xres(self):
@@ -733,10 +713,12 @@ class PageInfo:
class PdfInfo:
"""Get summary information about a PDF"""
def __init__(self, infile, log=None):
def __init__(self, infile, detailed_page_analysis=False, log=None):
self._infile = infile
self._pages, pdf = _pdf_get_all_pageinfo(infile, log=log)
self._pages, pdf = _pdf_get_all_pageinfo(
infile, detailed_page_analysis, log=log)
self._needs_rendering = pdf.root.get('/NeedsRendering', False)
self._has_acroform = '/AcroForm' in pdf.root
@property
def pages(self):
@@ -751,6 +733,10 @@ class PdfInfo:
def has_userunit(self):
return any(page.userunit != 1.0 for page in self.pages)
@property
def has_acroform(self):
return self._has_acroform
@property
def filename(self):
if not isinstance(self._infile, (str, Path)):
src/ocrmypdf/pdfinfo/ghosttext.py
+96
View File
@@ -0,0 +1,96 @@
# © 2018 James R. Barlow: github.com/jbarlow83
#
# This file is part of OCRmyPDF.
#
# OCRmyPDF is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OCRmyPDF is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with OCRmyPDF. If not, see <http://www.gnu.org/licenses/>.
import re
import xml.etree.ElementTree as ET
from ..exec import ghostscript
# Forgive me for I have sinned
# I am using regular expressions to parse XML. However the XML in this case,
# generated by Ghostscript, is self-consistent enough to be parseable.
regex_remove_char_tags = re.compile(br"""
<char\b
(?: [^>] # anything single character but >
| \">\" # special case: trap ">"
)*
/> # terminate with '/>'
""", re.VERBOSE)
def page_get_textblocks(infile, pageno, xmltext, height):
"""Get text boxes out of Ghostscript txtwrite xml"""
root = xmltext
if not hasattr(xmltext, 'findall'):
return []
def blocks():
for span in root.findall('.//span'):
bbox_str = span.attrib['bbox']
font_size = span.attrib['size']
pts = [int(pt) for pt in bbox_str.split()]
pts[1] = pts[1] - int(float(font_size) + 0.5)
bbox_topdown = tuple(pts)
bb = bbox_topdown
bbox_bottomup = (bb[0], height - bb[3], bb[2], height - bb[1])
yield bbox_bottomup
def joined_blocks():
prev = None
for bbox in blocks():
if prev is None:
prev = bbox
if bbox[1] == prev[1] and bbox[3] == prev[3]:
gap = prev[2] - bbox[0]
height = abs(bbox[3] - bbox[1])
if gap < height:
# Join boxes
prev = (prev[0], prev[1], bbox[2], bbox[3])
continue
# yield previously joined bboxes and start anew
yield prev
prev = bbox
if prev is not None:
yield prev
return [block for block in joined_blocks()]
def extract_text_xml(infile, pdf, pageno=None, log=None):
existing_text = ghostscript.extract_text(infile, pageno=None)
existing_text = regex_remove_char_tags.sub(b' ', existing_text)
try:
root = ET.fromstringlist([
b'<document>\n', existing_text, b'</document>\n'
])
page_xml = root.findall('page')
except ET.ParseError as e:
log.error(
"An error occurred while attempting to retrieve existing text in "
"the input file. Will attempt to continue assuming that there is "
"no existing text in the file. The error was:")
log.error(e)
page_xml = [None] * len(pdf.pages)
page_count_difference = len(pdf.pages) - len(page_xml)
if page_count_difference != 0:
log.error("The number of pages in the input file is inconsistent.")
if page_count_difference > 0:
page_xml.extend([None] * page_count_difference)
return page_xml
src/ocrmypdf/pdfinfo/layout.py
+220
View File
@@ -0,0 +1,220 @@
# © 2018 James R. Barlow: github.com/jbarlow83
#
# This file is part of OCRmyPDF.
#
# OCRmyPDF is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OCRmyPDF is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with OCRmyPDF. If not, see <http://www.gnu.org/licenses/>.
import re
from math import copysign
from pathlib import Path
from unittest.mock import patch
import pdfminer.encodingdb
import pdfminer.pdfdevice
import pdfminer.pdfinterp
from pdfminer.converter import PDFLayoutAnalyzer
from pdfminer.glyphlist import glyphname2unicode
from pdfminer.layout import (LAParams, LTChar, LTContainer, LTLayoutContainer,
LTPage, LTTextBox, LTTextLine)
from pdfminer.pdfdocument import PDFTextExtractionNotAllowed
from pdfminer.pdffont import (PDFCIDFont, PDFFont, PDFType3Font,
PDFUnicodeNotDefined)
from pdfminer.pdfpage import PDFPage
from pdfminer.utils import bbox2str, fsplit, matrix2str
from ..exceptions import EncryptedPdfError
STRIP_NAME = re.compile(r'[0-9]+')
#
# Unconditional pdfminer patches
#
def name2unicode(name):
"""Fix pdfminer's regex in name2unicode function
Font cids that are mapped to names of the form /g123 seem to be, by convention
characters with no corresponding Unicode entry. These can be subsetted fonts
or symbolic fonts. There seems to be no way to map /g123 fonts to Unicode,
barring a ToUnicode data structure.
"""
if name in glyphname2unicode:
return glyphname2unicode[name]
if name.startswith('g'):
raise KeyError(name)
m = STRIP_NAME.search(name)
if not m:
raise KeyError(name)
return chr(int(m.group(0)))
pdfminer.encodingdb.name2unicode = name2unicode
original_PDFFont_init = PDFFont.__init__
def PDFFont__init__(self, descriptor, widths, default_width=None):
original_PDFFont_init(self, descriptor, widths, default_width)
# PDF spec says descent should be negative
# A font with a positive descent implies it floats entirely above the
# baseline, i.e. it's not really a baseline anymore. I have fonts that
# claim a positive descent, but treating descent as positive always seems
# to misposition text.
if self.descent > 0:
self.descent = -self.descent
PDFFont.__init__ = PDFFont__init__
#
# pdfminer patches when creator is PScript5.dll
#
def PDFType3Font__PScript5_get_height(self):
h = self.bbox[3]-self.bbox[1]
if h == 0:
h = self.ascent - self.descent
return h * copysign(1.0, self.vscale)
def PDFType3Font__PScript5_get_descent(self):
return self.descent * copysign(1.0, self.vscale)
def PDFType3Font__PScript5_get_ascent(self):
return self.ascent * copysign(1.0, self.vscale)
class LTStateAwareChar(LTChar):
"""A subclass of LTChar that tracks text render mode at time of drawing"""
__slots__ = (
'rendermode', '_text', 'matrix', 'fontname', 'adv', 'upright', 'size',
'width', 'height', 'bbox', 'x0', 'x1', 'y0', 'y1'
)
def __init__(self, matrix, font, fontsize, scaling, rise, text, textwidth,
textdisp, textstate, *args):
super().__init__(matrix, font, fontsize, scaling, rise, text, textwidth,
textdisp, *args)
self.rendermode = textstate.render
def is_compatible(self, obj):
"""Check if characters can be combined into a textline
We consider characters compatible if:
- the Unicode mapping is known, and both have the same render mode
- the Unicode mapping is unknown but both are part of the same font
"""
both_unicode_mapped = (isinstance(self._text, str) and
isinstance(obj._text, str))
try:
if both_unicode_mapped:
return self.rendermode == obj.rendermode
font0, _ = self._text
font1, _ = obj._text
return font0 == font1 and self.rendermode == obj.rendermode
except (ValueError, AttributeError):
return False
def get_text(self):
if isinstance(self._text, tuple):
return '�'
return self._text
def __repr__(self):
return ('<%s %s matrix=%s rendermode=%r font=%r adv=%s text=%r>' %
(self.__class__.__name__, bbox2str(self.bbox),
matrix2str(self.matrix), self.rendermode, self.fontname, self.adv,
self.get_text()))
class TextPositionTracker(PDFLayoutAnalyzer):
"""A page layout analyzer that pays attention to text visibility"""
def __init__(self, rsrcmgr, pageno=1, laparams=None):
super().__init__(rsrcmgr, pageno, laparams)
self.textstate = None
self.result = None
def begin_page(self, page, ctm):
super().begin_page(page, ctm)
self.cur_item = LTPage(self.pageno, page.mediabox)
def end_page(self, page):
assert not self._stack, str(len(self._stack))
assert isinstance(self.cur_item, LTPage), str(type(self.cur_item))
if self.laparams is not None:
self.cur_item.analyze(self.laparams)
self.pageno += 1
self.receive_layout(self.cur_item)
def render_string(self, textstate, seq, *args):
self.textstate = textstate.copy()
super().render_string(self.textstate, seq, *args)
def render_char(self, matrix, font, fontsize, scaling, rise, cid, *args):
try:
text = font.to_unichr(cid)
assert isinstance(text, str), str(type(text))
except PDFUnicodeNotDefined:
text = self.handle_undefined_char(font, cid)
textwidth = font.char_width(cid)
textdisp = font.char_disp(cid)
item = LTStateAwareChar(
matrix, font, fontsize, scaling, rise, text,
textwidth, textdisp, self.textstate, *args)
self.cur_item.add(item)
return item.adv
def handle_undefined_char(self, font, cid):
#log.info('undefined: %r, %r', font, cid)
return (font.fontname, cid)
def receive_layout(self, ltpage):
self.result = ltpage
def get_result(self):
return self.result
def get_page_analysis(infile, pageno, pscript5_mode):
rman = pdfminer.pdfinterp.PDFResourceManager(caching=True)
dev = TextPositionTracker(rman, laparams=LAParams())
interp = pdfminer.pdfinterp.PDFPageInterpreter(rman, dev)
if pscript5_mode:
patcher = patch.multiple(
'pdfminer.pdffont.PDFType3Font',
spec=True,
get_ascent=PDFType3Font__PScript5_get_ascent,
get_descent=PDFType3Font__PScript5_get_descent,
get_height=PDFType3Font__PScript5_get_height
)
patcher.start()
with Path(infile).open('rb') as f:
page = PDFPage.get_pages(f, pagenos=[pageno], maxpages=0)
try:
interp.process_page(next(page))
except PDFTextExtractionNotAllowed as e:
raise EncryptedPdfError()
finally:
if pscript5_mode:
patcher.stop()
return dev.get_result()
def get_text_boxes(obj):
for child in obj:
if isinstance(child, (LTTextBox)):
yield child
else:
try:
yield from get_text_boxes(child)
except TypeError:
continue