menekis/OCRmyPDF
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'feature/modernhocr'

Browse Source
This commit is contained in:
James R. Barlow
2023-12-03 15:17:02 -08:00
parent 2affa83efe 445617a1a5
commit 5b2f2e6290
122 changed files with 4373 additions and 2368 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ocrmypdf/_concurrent.py
+6 -2
View File
@@ -8,7 +8,7 @@ from __future__ import annotations
import threading
from abc import ABC, abstractmethod
from collections.abc import Iterable
from typing import Callable, TypeVar
from typing import Any, Callable, TypeVar
from ocrmypdf._progressbar import NullProgressBar, ProgressBar
@@ -19,6 +19,10 @@ def _task_noop(*_args, **_kwargs):
return
def _task_finished_noop(_result: Any, pbar: ProgressBar):
pbar.update()
class Executor(ABC):
"""Abstract concurrent executor."""
@@ -66,7 +70,7 @@ class Executor(ABC):
if not worker_initializer:
worker_initializer = _task_noop
if not task_finished:
task_finished = _task_noop
task_finished = _task_finished_noop
if not task:
task = _task_noop
src/ocrmypdf/_exec/unpaper.py
+1 -21
View File
@@ -14,6 +14,7 @@ from contextlib import contextmanager
from decimal import Decimal
from pathlib import Path
from subprocess import PIPE, STDOUT
from tempfile import TemporaryDirectory
from typing import Union
from packaging.version import Version
@@ -26,27 +27,6 @@ from ocrmypdf.subprocess import get_version, run
# https://github.com/Flameeyes/unpaper/blob/main/doc/basic-concepts.md
if sys.version_info >= (3, 10):
from tempfile import TemporaryDirectory
else:
from tempfile import TemporaryDirectory as _TemporaryDirectory
class TemporaryDirectory(_TemporaryDirectory):
"""Shim to consume ignore_cleanup_errors kwarg on Python 3.9 and older.
The argument is consumed without action. If users are getting errors related
to temporary file cleanup, they should upgrade to Python 3.10 which properly
cleans up temporary directories on Windows.
See: https://github.com/python/cpython/pull/24793
"""
def __init__(self, ignore_cleanup_errors=False, **kwargs):
super().__init__(**kwargs)
del _TemporaryDirectory
UNPAPER_IMAGE_PIXEL_LIMIT = 256 * 1024 * 1024
DecFloat = Union[Decimal, float]
src/ocrmypdf/_graft.py
+1 -1
View File
@@ -313,7 +313,7 @@ class OcrGrafter:
strip_invisible_text(self.pdf_base, base_page)
base_page.contents_add(
new_text_layer, prepend=self.render_mode == RenderMode.UNDERNEATH
new_text_layer, prepend=self.render_mode == RenderMode.ON_TOP
)
_update_resources(obj=base_page.obj, font=font, font_key=font_key)
src/ocrmypdf/_pipeline.py
+5 -5
View File
@@ -743,13 +743,13 @@ def render_hocr_page(hocr: Path, page_context: PageContext) -> Path:
dpi = get_page_square_dpi(page_context, calculate_image_dpi(page_context))
debug_mode = options.pdf_renderer == 'hocrdebug'
hocrtransform = HocrTransform(hocr_filename=hocr, dpi=dpi.to_scalar()) # square
hocrtransform.to_pdf(
HocrTransform(
hocr_filename=hocr,
dpi=dpi.to_scalar(), # square
debug=debug_mode,
).to_pdf(
out_filename=output_file,
image_filename=None,
show_bounding_boxes=False if not debug_mode else True,
invisible_text=True if not debug_mode else False,
interword_spaces=True,
)
return output_file
src/ocrmypdf/_validation.py
-10
View File
@@ -28,7 +28,6 @@ from ocrmypdf.exceptions import (
OutputFileAccessError,
)
from ocrmypdf.helpers import is_file_writable, monotonic, safe_symlink
from ocrmypdf.hocrtransform import HOCR_OK_LANGS
from ocrmypdf.subprocess import check_external_program
# -------------
@@ -83,15 +82,6 @@ def check_options_languages(
def check_options_output(options: Namespace) -> None:
is_latin = set(options.languages).issubset(HOCR_OK_LANGS)
if options.pdf_renderer.startswith('hocr') and not is_latin:
log.warning(
"The 'hocr' PDF renderer is known to cause problems with one "
"or more of the languages in your document. Use "
"`--pdf-renderer auto` (the default) to avoid this issue."
)
if options.output_type == 'none' and options.output_file not in (os.devnull, '-'):
raise BadArgsError(
"Since you specified `--output-type none`, the output file "
src/ocrmypdf/builtin_plugins/concurrency.py
-2
View File
@@ -30,8 +30,6 @@ Queue = Union[multiprocessing.Queue, queue.Queue]
UserInit = Callable[[], None]
WorkerInit = Callable[[Queue, UserInit, int], None]
RichTqdmProgressAdapter = RichProgressBar # Deprecated shim; remove in OCRmyPDF 16
def log_listener(q: Queue):
"""Listen to the worker processes and forward the messages to logging.
src/ocrmypdf/builtin_plugins/tesseract_ocr.py
+2 -2
View File
@@ -146,7 +146,7 @@ def check_options(options):
# Decide on what renderer to use
if options.pdf_renderer == 'auto':
options.pdf_renderer = 'sandwich'
options.pdf_renderer = 'hocr'
if not tesseract.has_thresholding() and options.tesseract_thresholding != 0:
log.warning(
@@ -216,7 +216,7 @@ class TesseractOcrEngine(OcrEngine):
@staticmethod
def creator_tag(options):
tag = '-PDF' if options.pdf_renderer == 'sandwich' else ''
tag = '-PDF' if options.pdf_renderer == 'sandwich' else 'hOCR'
return f"Tesseract OCR{tag} {TesseractOcrEngine.version()}"
def __str__(self):
src/ocrmypdf/data/pdf.ttf
BIN
View File
Binary file not shown.
src/ocrmypdf/hocrtransform.py
-461
View File
@@ -1,461 +0,0 @@
#!/usr/bin/env python3
# SPDX-FileCopyrightText: 2010 Jonathan Brinley
# SPDX-FileCopyrightText: 2013-2014 Julien Pfefferkorn
# SPDX-FileCopyrightText: 2022 James R. Barlow
# SPDX-License-Identifier: MIT
"""Transform .hocr and page image to text PDF."""
from __future__ import annotations
import argparse
import os
import re
import warnings
from math import atan, cos, sin
from pathlib import Path
from typing import Any, NamedTuple
from xml.etree import ElementTree
with warnings.catch_warnings():
# reportlab uses deprecated load_module
# shim can be removed when we require reportlab >= 3.7
warnings.filterwarnings(
'ignore', category=DeprecationWarning, message=r".*load_module.*"
)
from reportlab.lib.colors import black, cyan, magenta, red
from reportlab.lib.units import inch
from reportlab.pdfgen.canvas import Canvas
# According to Wikipedia these languages are supported in the ISO-8859-1 character
# set, meaning reportlab can generate them and they are compatible with hocr,
# assuming Tesseract has the necessary languages installed. Note that there may
# not be language packs for them.
HOCR_OK_LANGS = frozenset(
[
# Languages fully covered by Latin-1:
'afr', # Afrikaans
'alb', # Albanian
'ast', # Leonese
'baq', # Basque
'bre', # Breton
'cos', # Corsican
'eng', # English
'eus', # Basque
'fao', # Faoese
'gla', # Scottish Gaelic
'glg', # Galician
'glv', # Manx
'ice', # Icelandic
'ind', # Indonesian
'isl', # Icelandic
'ita', # Italian
'ltz', # Luxembourgish
'mal', # Malay Rumi
'mga', # Irish
'nor', # Norwegian
'oci', # Occitan
'por', # Portugeuse
'roh', # Romansh
'sco', # Scots
'sma', # Sami
'spa', # Spanish
'sqi', # Albanian
'swa', # Swahili
'swe', # Swedish
'tgl', # Tagalog
'wln', # Walloon
# Languages supported by Latin-1 except for a few rare characters that OCR
# is probably not trained to recognize anyway:
'cat', # Catalan
'cym', # Welsh
'dan', # Danish
'deu', # German
'dut', # Dutch
'est', # Estonian
'fin', # Finnish
'fra', # French
'hun', # Hungarian
'kur', # Kurdish
'nld', # Dutch
'wel', # Welsh
]
)
Element = ElementTree.Element
class Rect(NamedTuple):
"""A rectangle for managing PDF coordinates."""
x1: Any
y1: Any
x2: Any
y2: Any
class HocrTransformError(Exception):
"""Error while applying hOCR transform."""
class HocrTransform:
"""A class for converting documents from the hOCR format.
For details of the hOCR format, see:
http://kba.cloud/hocr-spec/.
"""
box_pattern = re.compile(r'bbox((\s+\d+){4})')
baseline_pattern = re.compile(
r'''
baseline \s+
([\-\+]?\d*\.?\d*) \s+ # +/- decimal float
([\-\+]?\d+) # +/- int''',
re.VERBOSE,
)
ligatures = str.maketrans(
{'ff': 'ff', 'ffi': 'f‌f‌i', 'ffl': 'f‌f‌l', 'fi': 'fi', 'fl': 'fl'}
)
def __init__(self, *, hocr_filename: str | Path, dpi: float):
"""Initialize the HocrTransform object."""
self.dpi = dpi
self.hocr = ElementTree.parse(os.fspath(hocr_filename))
# if the hOCR file has a namespace, ElementTree requires its use to
# find elements
matches = re.match(r'({.*})html', self.hocr.getroot().tag)
self.xmlns = ''
if matches:
self.xmlns = matches.group(1)
# get dimension in pt (not pixel!!!!) of the OCRed image
self.width, self.height = None, None
for div in self.hocr.findall(self._child_xpath('div', 'ocr_page')):
coords = self.element_coordinates(div)
pt_coords = self.pt_from_pixel(coords)
self.width = pt_coords.x2 - pt_coords.x1
self.height = pt_coords.y2 - pt_coords.y1
# there shouldn't be more than one, and if there is, we don't want
# it
break
if self.width is None or self.height is None:
raise HocrTransformError("hocr file is missing page dimensions")
def __str__(self): # pragma: no cover
"""Return the textual content of the HTML body."""
if self.hocr is None:
return ''
body = self.hocr.find(self._child_xpath('body'))
if body:
return self._get_element_text(body)
else:
return ''
def _get_element_text(self, element: Element):
"""Return the textual content of the element and its children."""
text = ''
if element.text is not None:
text += element.text
for child in element:
text += self._get_element_text(child)
if element.tail is not None:
text += element.tail
return text
@classmethod
def element_coordinates(cls, element: Element) -> Rect:
"""Get coordinates of the bounding box around an element."""
out = Rect._make(0 for _ in range(4))
if 'title' in element.attrib:
matches = cls.box_pattern.search(element.attrib['title'])
if matches:
coords = matches.group(1).split()
out = Rect._make(int(coords[n]) for n in range(4))
return out
@classmethod
def baseline(cls, element: Element) -> tuple[float, float]:
"""Get baseline's slope and intercept."""
if 'title' in element.attrib:
matches = cls.baseline_pattern.search(element.attrib['title'])
if matches:
return float(matches.group(1)), int(matches.group(2))
return (0.0, 0.0)
def pt_from_pixel(self, pxl) -> Rect:
"""Returns the quantity in PDF units (pt) given quantity in pixels."""
return Rect._make((c / self.dpi * inch) for c in pxl)
def _child_xpath(self, html_tag: str, html_class: str | None = None) -> str:
xpath = f".//{self.xmlns}{html_tag}"
if html_class:
xpath += f"[@class='{html_class}']"
return xpath
@classmethod
def replace_unsupported_chars(cls, s: str) -> str:
"""Replaces characters with those available in the Helvetica typeface."""
return s.translate(cls.ligatures)
def to_pdf(
self,
*,
out_filename: Path,
image_filename: Path | None = None,
show_bounding_boxes: bool = False,
fontname: str = "Helvetica",
invisible_text: bool = False,
interword_spaces: bool = False,
) -> None:
"""Creates a PDF file with an image superimposed on top of the text.
Text is positioned according to the bounding box of the lines in
the hOCR file.
The image need not be identical to the image used to create the hOCR
file.
It can have a lower resolution, different color mode, etc.
Arguments:
out_filename: Path of PDF to write.
image_filename: Image to use for this file. If omitted, the OCR text
is shown.
show_bounding_boxes: Show bounding boxes around various text regions,
for debugging.
fontname: Name of font to use.
invisible_text: If True, text is rendered invisible so that is
selectable but never drawn. If False, text is visible and may
be seen if the image is skipped or deleted in Acrobat.
interword_spaces: If True, insert spaces between words rather than
drawing each word without spaces. Generally this improves text
extraction.
"""
# create the PDF file
# page size in points (1/72 in.)
pdf = Canvas(
os.fspath(out_filename),
pagesize=(self.width, self.height),
pageCompression=1,
)
# draw bounding box for each paragraph
# light blue for bounding box of paragraph
pdf.setStrokeColor(cyan)
# light blue for bounding box of paragraph
pdf.setFillColor(cyan)
pdf.setLineWidth(0) # no line for bounding box
for elem in self.hocr.iterfind(self._child_xpath('p', 'ocr_par')):
elemtxt = self._get_element_text(elem).rstrip()
if len(elemtxt) == 0:
continue
pxl_coords = self.element_coordinates(elem)
pt = self.pt_from_pixel(pxl_coords) # pylint: disable=invalid-name
# draw the bbox border
if show_bounding_boxes: # pragma: no cover
pdf.rect(
pt.x1, self.height - pt.y2, pt.x2 - pt.x1, pt.y2 - pt.y1, fill=1
)
found_lines = False
for line in (
element
for element in self.hocr.iterfind(self._child_xpath('span'))
if 'class' in element.attrib
and element.attrib['class'] in {'ocr_header', 'ocr_line', 'ocr_textfloat'}
):
found_lines = True
self._do_line(
pdf,
line,
"ocrx_word",
fontname,
invisible_text,
interword_spaces,
show_bounding_boxes,
)
if not found_lines:
# Tesseract did not report any lines (just words)
root = self.hocr.find(self._child_xpath('div', 'ocr_page'))
self._do_line(
pdf,
root,
"ocrx_word",
fontname,
invisible_text,
interword_spaces,
show_bounding_boxes,
)
# put the image on the page, scaled to fill the page
if image_filename is not None:
pdf.drawImage(
os.fspath(image_filename), 0, 0, width=self.width, height=self.height
)
# finish up the page and save it
pdf.showPage()
pdf.save()
@classmethod
def polyval(cls, poly, x): # pragma: no cover
"""Calculate the value of a polynomial at a point."""
return x * poly[0] + poly[1]
def _do_line(
self,
pdf: Canvas,
line: Element | None,
elemclass: str,
fontname: str,
invisible_text: bool,
interword_spaces: bool,
show_bounding_boxes: bool,
):
if line is None:
return
pxl_line_coords = self.element_coordinates(line)
line_box = self.pt_from_pixel(pxl_line_coords)
line_height = line_box.y2 - line_box.y1
slope, pxl_intercept = self.baseline(line)
if abs(slope) < 0.005:
slope = 0.0
angle = atan(slope)
cos_a, sin_a = cos(angle), sin(angle)
text = pdf.beginText()
intercept = pxl_intercept / self.dpi * inch
# Don't allow the font to break out of the bounding box. Division by
# cos_a accounts for extra clearance between the glyph's vertical axis
# on a sloped baseline and the edge of the bounding box.
fontsize = (line_height - abs(intercept)) / cos_a
text.setFont(fontname, fontsize)
if invisible_text:
text.setTextRenderMode(3) # Invisible (indicates OCR text)
# Intercept is normally negative, so this places it above the bottom
# of the line box
baseline_y2 = self.height - (line_box.y2 + intercept)
if show_bounding_boxes: # pragma: no cover
# draw the baseline in magenta, dashed
pdf.setDash()
pdf.setStrokeColor(magenta)
pdf.setLineWidth(0.5)
# negate slope because it is defined as a rise/run in pixel
# coordinates and page coordinates have the y axis flipped
pdf.line(
line_box.x1,
baseline_y2,
line_box.x2,
self.polyval((-slope, baseline_y2), line_box.x2 - line_box.x1),
)
# light green for bounding box of word/line
pdf.setDash(6, 3)
pdf.setStrokeColor(red)
text.setTextTransform(cos_a, -sin_a, sin_a, cos_a, line_box.x1, baseline_y2)
pdf.setFillColor(black) # text in black
elements = line.findall(self._child_xpath('span', elemclass))
for elem in elements:
elemtxt = self._get_element_text(elem).strip()
elemtxt = self.replace_unsupported_chars(elemtxt)
if elemtxt == '':
continue
pxl_coords = self.element_coordinates(elem)
box = self.pt_from_pixel(pxl_coords)
if interword_spaces:
# if `--interword-spaces` is true, append a space
# to the end of each text element to allow simpler PDF viewers
# such as PDF.js to better recognize words in search and copy
# and paste. Do not remove space from last word in line, even
# though it would look better, because it will interfere with
# naive text extraction. \n does not work either.
elemtxt += ' '
box = Rect._make(
(
box.x1,
line_box.y1,
box.x2 + pdf.stringWidth(' ', fontname, line_height),
line_box.y2,
)
)
box_width = box.x2 - box.x1
font_width = pdf.stringWidth(elemtxt, fontname, fontsize)
# draw the bbox border
if show_bounding_boxes: # pragma: no cover
pdf.rect(
box.x1, self.height - line_box.y2, box_width, line_height, fill=0
)
# Adjust relative position of cursor
# This is equivalent to:
# text.setTextOrigin(pt.x1, self.height - line_box.y2)
# but the former generates a full text reposition matrix (Tm) in the
# content stream while this issues a "offset" (Td) command.
# .moveCursor() is relative to start of the text line, where the
# "text line" means whatever reportlab defines it as. Do not use
# use .getCursor(), since moveCursor() rather unintuitively plans
# its moves relative to .getStartOfLine().
# For skewed lines, in the text transform we set up a rotated
# coordinate system, so we don't have to account for the
# incremental offset. Surprisingly most PDF viewers can handle this.
cursor = text.getStartOfLine()
dx = box.x1 - cursor[0]
dy = baseline_y2 - cursor[1]
text.moveCursor(dx, dy)
# If reportlab tells us this word is 0 units wide, our best seems
# to be to suppress this text
if font_width > 0:
text.setHorizScale(100 * box_width / font_width)
text.textOut(elemtxt)
pdf.drawText(text)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Convert hocr file to PDF')
parser.add_argument(
'-b',
'--boundingboxes',
action="store_true",
default=False,
help='Show bounding boxes borders',
)
parser.add_argument(
'-r',
'--resolution',
type=int,
default=300,
help='Resolution of the image that was OCRed',
)
parser.add_argument(
'-i',
'--image',
default=None,
help='Path to the image to be placed above the text',
)
parser.add_argument(
'--interword-spaces',
action='store_true',
default=False,
help='Add spaces between words',
)
parser.add_argument('hocrfile', help='Path to the hocr file to be parsed')
parser.add_argument('outputfile', help='Path to the PDF file to be generated')
args = parser.parse_args()
hocr = HocrTransform(hocr_filename=args.hocrfile, dpi=args.resolution)
hocr.to_pdf(
out_filename=args.outputfile,
image_filename=args.image,
show_bounding_boxes=args.boundingboxes,
interword_spaces=args.interword_spaces,
)
src/ocrmypdf/hocrtransform/__init__.py
Executable
+18
View File
@@ -0,0 +1,18 @@
# SPDX-FileCopyrightText: 2023 James R. Barlow
# SPDX-License-Identifier: MIT
"""Transform .hocr and page image to text PDF."""
from __future__ import annotations
from ocrmypdf.hocrtransform._hocr import (
DebugRenderOptions,
HocrTransform,
HocrTransformError,
)
__all__ = (
'HocrTransform',
'HocrTransformError',
'DebugRenderOptions',
)
src/ocrmypdf/hocrtransform/__main__.py
+40
View File
@@ -0,0 +1,40 @@
# SPDX-FileCopyrightText: 2023 James R. Barlow
# SPDX-License-Identifier: MIT
"""Simple CLI for testing HOCR."""
import argparse
from ocrmypdf.hocrtransform import HocrTransform
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Convert hocr file to PDF')
parser.add_argument(
'-b',
'--boundingboxes',
action="store_true",
default=False,
help='Show bounding boxes borders',
)
parser.add_argument(
'-r',
'--resolution',
type=int,
default=300,
help='Resolution of the image that was OCRed',
)
parser.add_argument(
'-i',
'--image',
default=None,
help='Path to the image to be placed above the text',
)
parser.add_argument('hocrfile', help='Path to the hocr file to be parsed')
parser.add_argument('outputfile', help='Path to the PDF file to be generated')
args = parser.parse_args()
hocr = HocrTransform(hocr_filename=args.hocrfile, dpi=args.resolution)
hocr.to_pdf(
out_filename=args.outputfile,
image_filename=args.image,
)
src/ocrmypdf/hocrtransform/_font.py
+112
View File
@@ -0,0 +1,112 @@
# SPDX-FileCopyrightText: 2023 James R. Barlow
# SPDX-License-Identifier: MPL-2.0
from __future__ import annotations
import logging
import unicodedata
import zlib
from importlib.resources import files as package_files
from pikepdf import (
Dictionary,
Name,
Pdf,
)
from pikepdf.canvas import Font
log = logging.getLogger(__name__)
class GlyphlessFont(Font):
CID_TO_GID_DATA = zlib.compress(b"\x00\x01" * 65536)
GLYPHLESS_FONT_NAME = 'pdf.ttf'
GLYPHLESS_FONT = (package_files('ocrmypdf.data') / GLYPHLESS_FONT_NAME).read_bytes()
CHAR_ASPECT = 2
def __init__(self):
pass
def text_width(self, text: str, fontsize: float) -> int:
"""Estimate the width of a text string when rendered with the given font."""
# NFKC: split ligatures, combine diacritics
return len(unicodedata.normalize("NFKC", text)) * (fontsize / self.CHAR_ASPECT)
def register(self, pdf: Pdf):
"""Register the glyphless font.
Create several data structures in the Pdf to describe the font. While it create
the data, a reference should be set in at least one page's /Resources dictionary
to retain the font in the output PDF and ensure it is usable on that page.
"""
PLACEHOLDER = Name.Placeholder
basefont = pdf.make_indirect(
Dictionary(
BaseFont=Name.GlyphLessFont,
DescendantFonts=[PLACEHOLDER],
Encoding=Name("/Identity-H"),
Subtype=Name.Type0,
ToUnicode=PLACEHOLDER,
Type=Name.Font,
)
)
cid_font_type2 = pdf.make_indirect(
Dictionary(
BaseFont=Name.GlyphLessFont,
CIDToGIDMap=PLACEHOLDER,
CIDSystemInfo=Dictionary(
Ordering="Identity",
Registry="Adobe",
Supplement=0,
),
FontDescriptor=PLACEHOLDER,
Subtype=Name.CIDFontType2,
Type=Name.Font,
DW=1000 // self.CHAR_ASPECT,
)
)
basefont.DescendantFonts = [cid_font_type2]
cid_font_type2.CIDToGIDMap = pdf.make_stream(
self.CID_TO_GID_DATA, Filter=Name.FlateDecode
)
basefont.ToUnicode = pdf.make_stream(
b"/CIDInit /ProcSet findresource begin\n"
b"12 dict begin\n"
b"begincmap\n"
b"/CIDSystemInfo\n"
b"<<\n"
b" /Registry (Adobe)\n"
b" /Ordering (UCS)\n"
b" /Supplement 0\n"
b">> def\n"
b"/CMapName /Adobe-Identify-UCS def\n"
b"/CMapType 2 def\n"
b"1 begincodespacerange\n"
b"<0000> <FFFF>\n"
b"endcodespacerange\n"
b"1 beginbfrange\n"
b"<0000> <FFFF> <0000>\n"
b"endbfrange\n"
b"endcmap\n"
b"CMapName currentdict /CMap defineresource pop\n"
b"end\n"
b"end\n"
)
font_descriptor = pdf.make_indirect(
Dictionary(
Ascent=1000,
CapHeight=1000,
Descent=-1,
Flags=5, # Fixed pitch and symbolic
FontBBox=[0, 0, 1000 // self.CHAR_ASPECT, 1000],
FontFile2=PLACEHOLDER,
FontName=Name.GlyphLessFont,
ItalicAngle=0,
StemV=80,
Type=Name.FontDescriptor,
)
)
font_descriptor.FontFile2 = pdf.make_stream(self.GLYPHLESS_FONT)
cid_font_type2.FontDescriptor = font_descriptor
return basefont
src/ocrmypdf/hocrtransform/_hocr.py
+473
View File
@@ -0,0 +1,473 @@
# SPDX-FileCopyrightText: 2010 Jonathan Brinley
# SPDX-FileCopyrightText: 2013-2014 Julien Pfefferkorn
# SPDX-FileCopyrightText: 2023 James R. Barlow
# SPDX-License-Identifier: MIT
"""hOCR transform implementation."""
from __future__ import annotations
import logging
import os
import re
import unicodedata
from dataclasses import dataclass
from itertools import pairwise
from math import atan, cos, pi
from pathlib import Path
from xml.etree import ElementTree
from pikepdf import Matrix, Name, Rectangle
from pikepdf.canvas import (
BLACK,
BLUE,
CYAN,
DARKGREEN,
GREEN,
MAGENTA,
RED,
Canvas,
Font,
Text,
TextDirection,
)
from ocrmypdf.hocrtransform._font import GlyphlessFont
log = logging.getLogger(__name__)
INCH = 72.0
Element = ElementTree.Element
@dataclass
class DebugRenderOptions:
"""A class for managing rendering options."""
render_paragraph_bbox: bool
render_baseline: bool
render_triangle: bool
render_line_bbox: bool
render_word_bbox: bool
render_space_bbox: bool
class HocrTransformError(Exception):
"""Error while applying hOCR transform."""
class HocrTransform:
"""A class for converting documents from the hOCR format.
For details of the hOCR format, see:
http://kba.cloud/hocr-spec/.
"""
box_pattern = re.compile(r'bbox (\d+) (\d+) (\d+) (\d+)')
baseline_pattern = re.compile(
r'''
baseline \s+
([\-\+]?\d*\.?\d*) \s+ # +/- decimal float
([\-\+]?\d+) # +/- int''',
re.VERBOSE,
)
def __init__(
self,
*,
hocr_filename: str | Path,
dpi: float,
debug: bool = False,
fontname: Name = Name("/f-0-0"),
font: Font = GlyphlessFont(),
):
"""Initialize the HocrTransform object."""
self.dpi = dpi
self.hocr = ElementTree.parse(os.fspath(hocr_filename))
self._fontname = fontname
self._font = font
# if the hOCR file has a namespace, ElementTree requires its use to
# find elements
matches = re.match(r'({.*})html', self.hocr.getroot().tag)
self.xmlns = ''
if matches:
self.xmlns = matches.group(1)
for div in self.hocr.findall(self._child_xpath('div', 'ocr_page')):
coords = self.element_coordinates(div)
if not coords:
raise HocrTransformError("hocr file is missing page dimensions")
self.width = (coords.urx - coords.llx) / (self.dpi / INCH)
self.height = (coords.ury - coords.lly) / (self.dpi / INCH)
# Stop after first div that has page coordinates
break
self.render_options = DebugRenderOptions(
render_baseline=debug,
render_triangle=debug,
render_line_bbox=debug,
render_word_bbox=debug,
render_paragraph_bbox=debug,
render_space_bbox=debug,
)
def _get_element_text(self, element: Element):
"""Return the textual content of the element and its children."""
text = ''
if element.text is not None:
text += element.text
for child in element:
text += self._get_element_text(child)
if element.tail is not None:
text += element.tail
return text
@classmethod
def element_coordinates(cls, element: Element) -> Rectangle | None:
"""Get coordinates of the bounding box around an element."""
matches = cls.box_pattern.search(element.attrib.get('title', ''))
if not matches:
return None
return Rectangle(
float(matches.group(1)), # llx = left
float(matches.group(2)), # lly = top
float(matches.group(3)), # urx = right
float(matches.group(4)), # ury = bottom
)
@classmethod
def baseline(cls, element: Element) -> tuple[float, float]:
"""Get baseline's slope and intercept."""
matches = cls.baseline_pattern.search(element.attrib.get('title', ''))
if not matches:
return (0.0, 0.0)
return float(matches.group(1)), int(matches.group(2))
def _child_xpath(self, html_tag: str, html_class: str | None = None) -> str:
xpath = f".//{self.xmlns}{html_tag}"
if html_class:
xpath += f"[@class='{html_class}']"
return xpath
@classmethod
def normalize_text(cls, s: str) -> str:
"""Normalize the given text using the NFKC normalization form."""
return unicodedata.normalize("NFKC", s)
def to_pdf(
self,
*,
out_filename: Path,
image_filename: Path | None = None,
invisible_text: bool = True,
) -> None:
"""Creates a PDF file with an image superimposed on top of the text.
Text is positioned according to the bounding box of the lines in
the hOCR file.
The image need not be identical to the image used to create the hOCR
file.
It can have a lower resolution, different color mode, etc.
Arguments:
out_filename: Path of PDF to write.
image_filename: Image to use for this file. If omitted, the OCR text
is shown.
invisible_text: If True, text is rendered invisible so that is
selectable but never drawn. If False, text is visible and may
be seen if the image is skipped or deleted in Acrobat.
"""
# create the PDF file
# page size in points (1/72 in.)
canvas = Canvas(page_size=(self.width, self.height))
canvas.add_font(self._fontname, self._font)
page_matrix = (
Matrix()
.translated(0, self.height)
.scaled(1, -1)
.scaled(INCH / self.dpi, INCH / self.dpi)
)
log.debug(page_matrix)
with canvas.do.save_state(cm=page_matrix):
self._debug_draw_paragraph_boxes(canvas)
found_lines = False
for par in self.hocr.iterfind(self._child_xpath('p', 'ocr_par')):
for line in (
element
for element in par.iterfind(self._child_xpath('span'))
if 'class' in element.attrib
and element.attrib['class']
in {'ocr_header', 'ocr_line', 'ocr_textfloat'}
):
found_lines = True
direction = self._get_text_direction(par)
inject_word_breaks = self._get_inject_word_breaks(par)
self._do_line(
canvas,
line,
"ocrx_word",
invisible_text,
direction,
inject_word_breaks,
)
if not found_lines:
# Tesseract did not report any lines (just words)
root = self.hocr.find(self._child_xpath('div', 'ocr_page'))
self._do_line(
canvas,
root,
"ocrx_word",
invisible_text,
TextDirection.LTR,
True,
)
# put the image on the page, scaled to fill the page
if image_filename is not None:
canvas.do.draw_image(
image_filename, 0, 0, width=self.width, height=self.height
)
# finish up the page and save it
canvas.to_pdf().save(out_filename)
def _get_text_direction(self, par):
"""Get the text direction of the paragraph.
Arabic, Hebrew, Persian, are right-to-left languages.
"""
return (
TextDirection.RTL
if par.attrib.get('dir', 'ltr') == 'rtl'
else TextDirection.LTR
)
def _get_inject_word_breaks(self, par):
"""Determine whether word breaks should be injected.
In Chinese, Japanese, and Korean, word breaks are not injected, because
words are usually one or two characters and separators are usually explicit.
In all other languages, we inject word breaks to help word segmentation.
"""
lang = par.attrib.get('lang', '')
log.debug(lang)
if lang in {'chi_sim', 'chi_tra', 'jpn', 'kor'}:
return False
return True
@classmethod
def polyval(cls, poly, x): # pragma: no cover
"""Calculate the value of a polynomial at a point."""
return x * poly[0] + poly[1]
def _do_line(
self,
canvas: Canvas,
line: Element | None,
elemclass: str,
invisible_text: bool,
text_direction: TextDirection,
inject_word_breaks: bool,
):
"""Render the text for a given line.
The canvas's coordinate system must be configured so that hOCR pixel
coordinates are mapped to PDF coordinates.
"""
if line is None:
return
line_box = self.element_coordinates(line)
if not line_box:
return
assert line_box.ury > line_box.lly # lly is top, ury is bottom
self._debug_draw_line_bbox(canvas, line_box)
# Baseline is a polynomial (usually straight line) that describes the
# text baseline relative to the bottom left corner of the line bounding
# box.
bottom_left_corner = line_box.llx, line_box.ury
slope, intercept = self.baseline(line)
if abs(slope) < 0.005:
slope = 0.0
angle = atan(slope)
# Setup a new coordinate system on the line box's intercept and rotated by
# its slope.
line_matrix = (
Matrix()
.translated(*bottom_left_corner)
.translated(0, intercept)
.rotated(angle / pi * 180)
)
log.debug(line_matrix)
with canvas.do.save_state(cm=line_matrix):
text = Text(direction=text_direction)
# Don't allow the font to break out of the bounding box. Division by
# cos_a accounts for extra clearance between the glyph's vertical axis
# on a sloped baseline and the edge of the bounding box.
line_box_height = abs(line_box.height) / cos(angle)
fontsize = line_box_height + intercept
text.font(self._fontname, fontsize)
if invisible_text or True:
text.render_mode(3) # Invisible (indicates OCR text)
self._debug_draw_baseline(
canvas, line_matrix.inverse().transform(line_box), 0
)
canvas.do.fill_color(BLACK) # text in black
elements = line.findall(self._child_xpath('span', elemclass))
for elem, next_elem in pairwise(elements + [None]):
self._do_line_word(
canvas,
line_matrix,
text,
fontsize,
elem,
next_elem,
text_direction,
inject_word_breaks,
)
canvas.do.draw_text(text)
def _do_line_word(
self,
canvas: Canvas,
line_matrix: Matrix,
text: Text,
fontsize: float,
elem: Element,
next_elem: Element | None,
text_direction: TextDirection,
inject_word_breaks: bool,
):
"""Render the text for a single word."""
if elem is None:
return
elemtxt = self.normalize_text(self._get_element_text(elem).strip())
if elemtxt == '':
return
hocr_box = self.element_coordinates(elem)
if hocr_box is None:
return
box = line_matrix.inverse().transform(hocr_box)
font_width = self._font.text_width(elemtxt, fontsize)
# Debug sketches
self._debug_draw_word_triangle(canvas, box)
self._debug_draw_word_bbox(canvas, box)
# If this word is 0 units wide, our best bet seems to be to suppress this text
if font_width > 0:
text.text_transform(Matrix(1, 0, 0, 1, box.llx, 0))
text.horiz_scale(100 * box.width / font_width)
text.show(elemtxt)
# Get coordinates of the next word (if there is one)
hocr_next_box = (
self.element_coordinates(next_elem) if next_elem is not None else None
)
if hocr_next_box is None:
return
# Render a space this word and the next word. The explicit space helps
# PDF viewers identify the word break, and horizontally scaling it to
# occupy the space the between the words helps the PDF viewer
# avoid combiningthewordstogether.
if not inject_word_breaks:
return
next_box = line_matrix.inverse().transform(hocr_next_box)
if text_direction == TextDirection.LTR:
space_box = Rectangle(box.urx, box.lly, next_box.llx, next_box.ury)
elif text_direction == TextDirection.RTL:
space_box = Rectangle(next_box.urx, box.lly, box.llx, next_box.ury)
self._debug_draw_space_bbox(canvas, space_box)
text.text_transform(Matrix(1, 0, 0, 1, space_box.llx, 0))
space_width = self._font.text_width(' ', fontsize)
if space_width > 0:
text.horiz_scale(100 * space_box.width / space_width)
text.show(' ')
def _debug_draw_paragraph_boxes(self, canvas: Canvas, color=CYAN):
"""Draw boxes around paragraphs in the document."""
if not self.render_options.render_paragraph_bbox: # pragma: no cover
return
with canvas.do.save_state():
# draw box around paragraph
canvas.do.stroke_color(color).line_width(0.1)
for elem in self.hocr.iterfind(self._child_xpath('p', 'ocr_par')):
elemtxt = self._get_element_text(elem).strip()
if len(elemtxt) == 0:
continue
ocr_par = self.element_coordinates(elem)
if ocr_par is None:
continue
canvas.do.rect(
ocr_par.llx, ocr_par.lly, ocr_par.width, ocr_par.height, fill=0
)
def _debug_draw_line_bbox(self, canvas: Canvas, line_box: Rectangle, color=BLUE):
"""Render the bounding box of a text line."""
if not self.render_options.render_line_bbox: # pragma: no cover
return
with canvas.do.save_state():
canvas.do.stroke_color(color).line_width(0.15).rect(
line_box.llx, line_box.lly, line_box.width, line_box.height, fill=0
)
def _debug_draw_word_triangle(
self, canvas: Canvas, box: Rectangle, color=RED, line_width=0.1
):
"""Render a triangle that conveys word height and drawing direction."""
if not self.render_options.render_triangle: # pragma: no cover
return
with canvas.do.save_state():
canvas.do.stroke_color(color).line_width(line_width).line(
box.llx, box.lly, box.urx, box.lly
).line(box.urx, box.lly, box.llx, box.ury).line(
box.llx, box.lly, box.llx, box.ury
)
def _debug_draw_word_bbox(
self, canvas: Canvas, box: Rectangle, color=GREEN, line_width=0.1
):
"""Render a box depicting the word."""
if not self.render_options.render_word_bbox: # pragma: no cover
return
with canvas.do.save_state():
canvas.do.stroke_color(color).line_width(line_width).rect(
box.llx, box.lly, box.width, box.height, fill=0
)
def _debug_draw_space_bbox(
self, canvas: Canvas, box: Rectangle, color=DARKGREEN, line_width=0.1
):
"""Render a box depicting the space between two words."""
if not self.render_options.render_space_bbox: # pragma: no cover
return
with canvas.do.save_state():
canvas.do.fill_color(color).line_width(line_width).rect(
box.llx, box.lly, box.width, box.height, fill=1
)
def _debug_draw_baseline(
self,
canvas: Canvas,
line_box: Rectangle,
baseline_lly,
color=MAGENTA,
line_width=0.25,
):
"""Render the text baseline."""
if not self.render_options.render_baseline:
return
with canvas.do.save_state():
canvas.do.stroke_color(color).line_width(line_width).line(
line_box.llx,
baseline_lly,
line_box.urx,
baseline_lly,
)
src/ocrmypdf/imageops.py
+6 -14
View File
@@ -7,17 +7,9 @@ from __future__ import annotations
import logging
from math import floor, sqrt
from typing import Optional
from PIL import Image
# While from __future__ import annotations, we use singledispatch here, which
# does not support annotations. Disable check about using old-style typing
# until Python 3.10, OR when drop singledispatch in ocrmypdf 15.
# ruff: noqa: UP006
# ruff: noqa: UP007
log = logging.getLogger(__name__)
@@ -38,9 +30,9 @@ def _calculate_downsample(
image_size: tuple[int, int],
bytes_per_pixel: int,
*,
max_size: Optional[tuple[int, int]] = None,
max_pixels: Optional[int] = None,
max_bytes: Optional[int] = None,
max_size: tuple[int, int] | None = None,
max_pixels: int | None = None,
max_bytes: int | None = None,
) -> tuple[int, int]:
"""Calculate image size required to downsample an image to fit limits.
@@ -98,9 +90,9 @@ def _calculate_downsample(
def calculate_downsample(
image: Image.Image,
*,
max_size: Optional[tuple[int, int]] = None,
max_pixels: Optional[int] = None,
max_bytes: Optional[int] = None,
max_size: tuple[int, int] | None = None,
max_pixels: int | None = None,
max_bytes: int | None = None,
) -> tuple[int, int]:
"""Calculate image size required to downsample an image to fit limits.
src/ocrmypdf/optimize.py
+2 -8
View File
@@ -71,15 +71,9 @@ def jpg_name(root: Path, xref: Xref) -> Path:
def extract_image_filter(
image: Stream, xref: Xref, *args
image: Stream, xref: Xref
) -> tuple[PdfImage, tuple[Name, Object]] | None:
"""Determine if an image is extractable."""
if isinstance(image, Pdf):
# Support deprecated old function signature
# TODO Remove for v16 and drop *args from current function signature
image, xref = args[0], args[1]
warn("extract_image_filter: pdf, root parameters ignored", DeprecationWarning)
if image.Subtype != Name.Image:
return None
if image.Length < 100:
@@ -376,7 +370,7 @@ def _produce_jbig2_images(
options.jbig2_threshold,
)
def jbig2_single_args(root, groups: dict[int, list[XrefExt]]):
def jbig2_single_args(root: Path, groups: dict[int, list[XrefExt]]):
for group, xref_exts in groups.items():
prefix = f'group{group:08d}'
# Second loop is to ensure multiple images per page are unpacked