pdfinfo: further layout improvements

Rather than grouping visible/invisible in a custom analysis step,
use pdfminer's analysis and iterate.
Make iteration predicate and return more generic.

src/ocrmypdf/pdfinfo/__init__.py

@@ -28,7 +28,7 @@ import xml.etree.ElementTree as ET
 from pikepdf import PdfMatrix
 import pikepdf
 
-from .layout import get_textblocks, bboxes
+from .layout import get_textblocks, filter_textboxes, textbox_predicate
 from ..exec import ghostscript
 from ..helpers import fspath
 
@@ -567,18 +567,19 @@ def _page_has_text(text_blocks, page_width, page_height):
 
     margin_ratio = 0.125
     interior_bbox = (
-        margin_ratio * pw, margin_ratio * ph,
-        (1 - margin_ratio) * pw, (1 - margin_ratio) * ph
+        margin_ratio * pw,  # left
+        (1 - margin_ratio) * ph, # top
+        (1 - margin_ratio) * pw, # right
+        margin_ratio * ph  # bottom  (first quadrant: bottom < top)
     )
 
     def rects_intersect(a, b):
         """
         Where (a,b) are 4-tuple rects (left-0, top-1, right-2, bottom-3)
         https://stackoverflow.com/questions/306316/determine-if-two-rectangles-overlap-each-other
-        Negative signs to account for our coordinates being in the fourth quadrant
-        and the formula assuming the first
+        Formula assumes all boxes are in first quadrant
         """
-        return a[0] < b[2] and a[2] > b[0] and -a[1] > -b[3] and -a[3] < -b[1]
+        return a[0] < b[2] and a[2] > b[0] and a[1] > b[3] and a[3] < b[1]
 
     has_text = False
     for bbox in text_blocks:
@@ -605,8 +606,12 @@ def _pdf_get_pageinfo(pdf, pageno: int, infile, xmltext):
     width_pt = mediabox[2] - mediabox[0]
     height_pt = mediabox[3] - mediabox[1]
 
+    bboxes = (textbox.bbox for textbox in filter_textboxes(
+            pageinfo['objects'], lambda obj: True)
+    )
     pageinfo['has_text'] = _page_has_text(
-        bboxes(pageinfo['objects']), width_pt, height_pt)
+        bboxes, width_pt, height_pt
+    )
 
     userunit = page.get('/UserUnit', Decimal(1.0))
     if not isinstance(userunit, Decimal):
@@ -629,7 +634,7 @@ def _pdf_get_pageinfo(pdf, pageno: int, infile, xmltext):
     if any(isinstance(ci, VectorInfo) for ci in contentsinfo):
         pageinfo['has_vector'] = True
 
-    textinfos = [ti for ti in contentsinfo if isinstance(ti, TextInfo)]
+    textinfos = (ti for ti in contentsinfo if isinstance(ti, TextInfo))
     all_invisible = not any(ti.visible for ti in textinfos)
     pageinfo['only_ocr_text'] = all_invisible
 
@@ -736,11 +741,11 @@ class PageInfo:
     def images(self):
         return self._pageinfo['images']
 
-    def get_textareas(self, visible=True, invisible=True):
-        if visible:
-            yield from bboxes(self._pageinfo['objects'][0])
-        if invisible:
-            yield from bboxes(self._pageinfo['objects'][1])
+    def get_textareas(self, visible=None, corrupt=None):
+        return (obj.bbox for obj in filter_textboxes(
+                self._pageinfo['objects'],
+                textbox_predicate(visible=visible, corrupt=corrupt)
+            ))
 
     @property
     def xres(self):

src/ocrmypdf/pdfinfo/layout.py

@@ -96,43 +96,6 @@ class LTStateAwareChar(LTChar):
                  self.get_text()))
 
 
-class LTStateAwarePage(LTPage):
-    """A page container that exploits character type information"""
-
-    def __init__(self, pageid, bbox, rotate=0):
-        LTPage.__init__(self, pageid, bbox, rotate)
-
-    def analyze(self, laparams):
-        """Analysis taking rendering mode into account
-
-        Looks at visible and invisible characters separately.
-        Depends on some superclass implementation details, largely because only
-        LTPage has the "group into textboxes" code, so we have to manipulate
-        our _objs to create multiple collections.
-        """
-
-        objs = self._objs[:]
-
-        # Split into invisible text objects and all others
-        (invisible_textobjs, other_objs) = fsplit(
-                lambda obj: getattr(obj, 'rendermode', 0) == 3, self)
-
-        # Analyze all invisible text objects and group them into text lines and
-        # text boxes
-        self._objs = invisible_textobjs
-        LTPage.analyze(self, laparams)
-        invisible_analyzed = self._objs[:]
-
-        # Analyze all other objects
-        self._objs = other_objs
-        LTPage.analyze(self, laparams)
-        other_analyzed = self._objs[:]
-
-        self._objs = invisible_analyzed + other_analyzed
-        self.visible = other_analyzed
-        self.invisible = invisible_analyzed
-
-
 class TextPositionTracker(PDFLayoutAnalyzer):
     """A page layout analyzer that pays attention to text visibility"""
 
@@ -143,7 +106,7 @@ class TextPositionTracker(PDFLayoutAnalyzer):
 
     def begin_page(self, page, ctm):
         super().begin_page(page, ctm)
-        self.cur_item = LTStateAwarePage(self.pageno, page.mediabox)
+        self.cur_item = LTPage(self.pageno, page.mediabox)
 
     def end_page(self, page):
         assert not self._stack, str(len(self._stack))
@@ -176,7 +139,7 @@ class TextPositionTracker(PDFLayoutAnalyzer):
         return (font, cid)
 
     def receive_layout(self, ltpage):
-        self.result = (ltpage.visible, ltpage.invisible)
+        self.result = ltpage
 
     def get_result(self):
         return self.result
@@ -194,12 +157,33 @@ def get_textblocks(infile, pageno):
     return dev.get_result()
 
 
-def bboxes(hierarchical_textinfo):
-    for obj in hierarchical_textinfo:
-        if isinstance(hierarchical_textinfo, (LTTextBox)):
-            yield hierarchical_textinfo.bbox
+def textbox_predicate(*, visible, corrupt):
+    def real_predicate(textbox, want_visible=visible, want_corrupt=corrupt):
+        textline = textbox._objs[0]
+        first_char = textline._objs[0]
+
+        result = True
+
+        is_visible = (first_char.rendermode != 3)
+        if want_visible is not None:
+            if is_visible != want_visible:
+                result = False
+        is_corrupt = (first_char.get_text() == '\ufffd')
+        if want_corrupt is not None:
+            if is_corrupt != want_corrupt:
+                result = False
+
+        return result
+    return real_predicate
+
+
+def filter_textboxes(obj, predicate):
+    for child in obj:
+        if isinstance(child, (LTTextBox)):
+            if predicate(child):
+                yield child
         else:
             try:
-                yield from bboxes(obj)
+                yield from filter_textboxes(child, predicate)
             except TypeError:
                 continue