feat: discard stale embedded text search index when rewriting PDF

Adobe Acrobat can embed a proprietary full-text search index in the
document catalog at /Root/PieceInfo/SearchIndex. Only Acrobat reads it;
other viewers ignore it and search the text on the fly. Any change to a
PDF invalidates the index, so once OCRmyPDF rewrites the document the
retained index is stale and returns incorrect search results in Acrobat.

OcrGrafter.finalize() now discards it before saving (covering both the
OCR and hOCR pipelines), preserving any other PieceInfo owner data and
dropping an empty PieceInfo. Modern viewers rebuild a search index on
demand, so there is no loss of search capability.

docs/releasenotes/version17.md

@@ -5,6 +5,14 @@
 
 ## v17.6.0
 
+- OCRmyPDF now discards an embedded Adobe full-text search index
+  (``/Root/PieceInfo/SearchIndex``) from its output. This proprietary index,
+  produced by Acrobat's "Embed Index" feature, is read only by Adobe Acrobat;
+  other viewers ignore it and search the text on the fly. Because any change to
+  a PDF invalidates the index, retaining it after OCRmyPDF rewrites the document
+  would leave a stale index that returns incorrect search results in Acrobat.
+  Modern viewers rebuild a search index on demand, so there is no loss of
+  search capability.
 - Fixed a regression in OCR quality for PDFs that paint a 1-bit image mask
   (stencil) with a gray or colored fill color. Previously such pages were
   rasterized as 1-bit black-and-white before OCR, so Ghostscript dithered

src/ocrmypdf/_graft.py

@@ -211,6 +211,36 @@ def strip_invisible_text(pdf: Pdf, page: Page):
     page.Contents = Stream(pdf, content_stream)
 
 
+def discard_text_search_index(pdf: Pdf) -> bool:
+    """Discard an embedded Adobe full-text search index from the catalog.
+
+    Adobe Acrobat can embed a full-text search index in the document catalog at
+    ``/Root/PieceInfo/SearchIndex``. It is built from the page text, and only
+    Acrobat reads it; other viewers ignore it and search the text on the fly.
+    Any change to the PDF invalidates the index, so once OCRmyPDF rewrites the
+    document (editing the text layer, rasterizing, optimizing) a retained index
+    would be stale and return incorrect search results in Acrobat. We cannot
+    update this vendor-private data, so we discard it; modern viewers rebuild a
+    search index on demand. Returns True if the catalog was modified.
+    """
+    try:
+        pieceinfo = pdf.Root.get(Name.PieceInfo)
+        if not isinstance(pieceinfo, Dictionary) or Name.SearchIndex not in pieceinfo:
+            return False
+        del pieceinfo[Name.SearchIndex]
+        log.debug(
+            "Discarded embedded text search index "
+            "(/Root/PieceInfo/SearchIndex) because the PDF was rewritten; "
+            "it would otherwise be stale."
+        )
+        # Drop an empty PieceInfo rather than leave a husk behind.
+        if len(pieceinfo) == 0:
+            del pdf.Root.PieceInfo
+        return True
+    except (KeyError, TypeError, AttributeError):
+        return False
+
+
 class OcrGrafter:
     """Manages grafting text-only PDFs onto regular PDFs."""
 
@@ -319,9 +349,9 @@ class OcrGrafter:
 
     def finalize(self):
         # Can have hocr OR parsed pages OR neither (no OCR), but not both
-        assert not (
+        assert not (self.fpdf2_hocr_pages and self.fpdf2_parsed_pages), (
-            self.fpdf2_hocr_pages and self.fpdf2_parsed_pages
+            "Can't have both hocr and ocrtree pages"
-        ), "Can't have both hocr and ocrtree pages"
+        )
 
         if self.fpdf2_hocr_pages:
             # Render all pages with fpdf2, then graft
@@ -331,6 +361,7 @@ class OcrGrafter:
         if self.fpdf2_parsed_pages:
             self._render_and_graft_fpdf2_pages()
 
+        discard_text_search_index(self.pdf_base)
         self.pdf_base.save(self.output_file)
         self.pdf_base.close()
         return self.output_file

tests/test_search_index.py

@@ -0,0 +1,101 @@
+# SPDX-FileCopyrightText: 2024 James R. Barlow
+# SPDX-License-Identifier: MPL-2.0
+
+from __future__ import annotations
+
+import logging
+
+import pikepdf
+import pytest
+from pikepdf import Dictionary, Name, String
+
+from ocrmypdf._graft import discard_text_search_index
+
+from .conftest import check_ocrmypdf
+
+# pylint: disable=redefined-outer-name
+
+
+def _add_search_index(pdf: pikepdf.Pdf, *, other_owner: bool = False) -> None:
+    """Attach an Adobe-style embedded search index to the document catalog."""
+    pieceinfo = Dictionary(
+        SearchIndex=Dictionary(
+            LastModified=String("D:20240101000000Z"),
+            Private=Dictionary(IndexFile=String("dummy.pdx")),
+        )
+    )
+    if other_owner:
+        pieceinfo[Name.SomeOtherApp] = Dictionary(
+            LastModified=String("D:20240101000000Z")
+        )
+    pdf.Root.PieceInfo = pdf.make_indirect(pieceinfo)
+
+
+def test_discard_text_search_index_removes_only_search_index(resources):
+    with pikepdf.open(resources / 'francais.pdf') as pdf:
+        # No PieceInfo at all -> nothing to do
+        assert not discard_text_search_index(pdf)
+
+        _add_search_index(pdf, other_owner=True)
+        assert discard_text_search_index(pdf), "Expected file to be modified"
+
+        # SearchIndex gone, but the other application's private data is preserved
+        assert Name.SearchIndex not in pdf.Root.PieceInfo
+        assert Name.SomeOtherApp in pdf.Root.PieceInfo
+
+        # Idempotent: a second call finds nothing to remove
+        assert not discard_text_search_index(pdf)
+
+
+def test_discard_text_search_index_drops_empty_pieceinfo(resources):
+    with pikepdf.open(resources / 'francais.pdf') as pdf:
+        _add_search_index(pdf, other_owner=False)
+        assert discard_text_search_index(pdf)
+        # PieceInfo held only the SearchIndex, so the whole husk is removed
+        assert Name.PieceInfo not in pdf.Root
+
+
+def test_discard_text_search_index_tolerates_malformed_pieceinfo(resources):
+    with pikepdf.open(resources / 'francais.pdf') as pdf:
+        pdf.Root.PieceInfo = String("not a dictionary")
+        assert not discard_text_search_index(pdf)
+
+
+@pytest.fixture
+def pdf_with_search_index(resources, outdir):
+    out = outdir / 'with_search_index.pdf'
+    with pikepdf.open(resources / 'graph.pdf') as pdf:
+        _add_search_index(pdf, other_owner=False)
+        assert Name.SearchIndex in pdf.Root.PieceInfo
+        pdf.save(out)
+    return out
+
+
+def test_search_index_discarded_end_to_end(pdf_with_search_index, outpdf, caplog):
+    caplog.set_level(logging.DEBUG)
+    check_ocrmypdf(
+        pdf_with_search_index,
+        outpdf,
+        '--output-type',
+        'pdf',
+        '--plugin',
+        'tests/plugins/tesseract_noop.py',
+    )
+    with pikepdf.open(outpdf) as pdf:
+        assert Name.PieceInfo not in pdf.Root
+    assert 'search index' in caplog.text.lower()
+
+
+def test_search_index_discarded_with_ocr_engine_none(pdf_with_search_index, outpdf):
+    # Even in pure image-processing mode, OCRmyPDF rewrites the PDF, which
+    # invalidates the embedded index, so it must still be discarded.
+    check_ocrmypdf(
+        pdf_with_search_index,
+        outpdf,
+        '--ocr-engine',
+        'none',
+        '--output-type',
+        'pdf',
+    )
+    with pikepdf.open(outpdf) as pdf:
+        assert Name.PieceInfo not in pdf.Root

uv.lock

@@ -1434,7 +1434,7 @@ wheels = [
 
 [[package]]
 name = "ocrmypdf"
-version = "17.4.2"
+version = "17.5.0"
 source = { editable = "." }
 dependencies = [
     { name = "deprecation" },