Merge branch 'feature/discard-obsolete-pdf-features'

docs/releasenotes/version17.md

@@ -5,6 +5,20 @@
 
 ## v17.6.0
 
+- OCRmyPDF now discards an embedded Adobe full-text search index
+  (``/Root/PieceInfo/SearchIndex``) from its output. This proprietary index,
+  produced by Acrobat's "Embed Index" feature, is read only by Adobe Acrobat;
+  other viewers ignore it and search the text on the fly. Because any change to
+  a PDF invalidates the index, retaining it after OCRmyPDF rewrites the document
+  would leave a stale index that returns incorrect search results in Acrobat.
+  Modern viewers rebuild a search index on demand, so there is no loss of
+  search capability.
+- OCRmyPDF now discards embedded per-page thumbnail images (the optional
+  ``/Thumb`` image XObject on a page) from its output. OCRmyPDF alters page
+  appearance (deskew, clean, rasterize, re-render) and plugins may edit pages
+  arbitrarily, so a retained thumbnail would be stale and no longer match its
+  page. Embedded thumbnails are a navigation aid that modern viewers generate
+  on demand, so there is no loss of functionality.
 - Fixed a regression in OCR quality for PDFs that paint a 1-bit image mask
   (stencil) with a gray or colored fill color. Previously such pages were
   rasterized as 1-bit black-and-white before OCR, so Ghostscript dithered

src/ocrmypdf/_graft.py

@@ -211,6 +211,62 @@ def strip_invisible_text(pdf: Pdf, page: Page):
     page.Contents = Stream(pdf, content_stream)
 
 
+def discard_text_search_index(pdf: Pdf) -> bool:
+    """Discard an embedded Adobe full-text search index from the catalog.
+
+    Adobe Acrobat can embed a full-text search index in the document catalog at
+    ``/Root/PieceInfo/SearchIndex``. It is built from the page text, and only
+    Acrobat reads it; other viewers ignore it and search the text on the fly.
+    Any change to the PDF invalidates the index, so once OCRmyPDF rewrites the
+    document (editing the text layer, rasterizing, optimizing) a retained index
+    would be stale and return incorrect search results in Acrobat. We cannot
+    update this vendor-private data, so we discard it; modern viewers rebuild a
+    search index on demand. Returns True if the catalog was modified.
+    """
+    try:
+        pieceinfo = pdf.Root.get(Name.PieceInfo)
+        if not isinstance(pieceinfo, Dictionary) or Name.SearchIndex not in pieceinfo:
+            return False
+        del pieceinfo[Name.SearchIndex]
+        log.debug(
+            "Discarded embedded text search index "
+            "(/Root/PieceInfo/SearchIndex) because the PDF was rewritten; "
+            "it would otherwise be stale."
+        )
+        # Drop an empty PieceInfo rather than leave a husk behind.
+        if len(pieceinfo) == 0:
+            del pdf.Root.PieceInfo
+        return True
+    except (KeyError, TypeError, AttributeError):
+        return False
+
+
+def discard_page_thumbnails(pdf: Pdf) -> int:
+    """Discard embedded per-page thumbnail images.
+
+    A page object may carry an optional ``/Thumb`` image XObject — a miniature
+    rendering of the page (ISO 32000-2, 12.3.4). It is only a navigation aid and
+    modern viewers generate page thumbnails on demand. OCRmyPDF alters page
+    appearance (deskew, clean, rasterize, re-render) and plugins may edit pages
+    arbitrarily, so any retained thumbnail would be stale and misrepresent its
+    page. We discard them; viewers rebuild thumbnails as needed. Returns the
+    number of thumbnails removed.
+    """
+    removed = 0
+    for page in pdf.pages:
+        pageobj = page.obj
+        if Name.Thumb in pageobj:
+            del pageobj[Name.Thumb]
+            removed += 1
+    if removed:
+        log.debug(
+            "Discarded %d embedded page thumbnail(s) (/Thumb) because the PDF "
+            "was rewritten; they would otherwise be stale.",
+            removed,
+        )
+    return removed
+
+
 class OcrGrafter:
     """Manages grafting text-only PDFs onto regular PDFs."""
 
@@ -319,9 +375,9 @@ class OcrGrafter:
 
     def finalize(self):
         # Can have hocr OR parsed pages OR neither (no OCR), but not both
-        assert not (
-            self.fpdf2_hocr_pages and self.fpdf2_parsed_pages
-        ), "Can't have both hocr and ocrtree pages"
+        assert not (self.fpdf2_hocr_pages and self.fpdf2_parsed_pages), (
+            "Can't have both hocr and ocrtree pages"
+        )
 
         if self.fpdf2_hocr_pages:
             # Render all pages with fpdf2, then graft
@@ -331,6 +387,8 @@ class OcrGrafter:
         if self.fpdf2_parsed_pages:
             self._render_and_graft_fpdf2_pages()
 
+        discard_text_search_index(self.pdf_base)
+        discard_page_thumbnails(self.pdf_base)
         self.pdf_base.save(self.output_file)
         self.pdf_base.close()
         return self.output_file

tests/test_search_index.py

@@ -0,0 +1,101 @@
+# SPDX-FileCopyrightText: 2024 James R. Barlow
+# SPDX-License-Identifier: MPL-2.0
+
+from __future__ import annotations
+
+import logging
+
+import pikepdf
+import pytest
+from pikepdf import Dictionary, Name, String
+
+from ocrmypdf._graft import discard_text_search_index
+
+from .conftest import check_ocrmypdf
+
+# pylint: disable=redefined-outer-name
+
+
+def _add_search_index(pdf: pikepdf.Pdf, *, other_owner: bool = False) -> None:
+    """Attach an Adobe-style embedded search index to the document catalog."""
+    pieceinfo = Dictionary(
+        SearchIndex=Dictionary(
+            LastModified=String("D:20240101000000Z"),
+            Private=Dictionary(IndexFile=String("dummy.pdx")),
+        )
+    )
+    if other_owner:
+        pieceinfo[Name.SomeOtherApp] = Dictionary(
+            LastModified=String("D:20240101000000Z")
+        )
+    pdf.Root.PieceInfo = pdf.make_indirect(pieceinfo)
+
+
+def test_discard_text_search_index_removes_only_search_index(resources):
+    with pikepdf.open(resources / 'francais.pdf') as pdf:
+        # No PieceInfo at all -> nothing to do
+        assert not discard_text_search_index(pdf)
+
+        _add_search_index(pdf, other_owner=True)
+        assert discard_text_search_index(pdf), "Expected file to be modified"
+
+        # SearchIndex gone, but the other application's private data is preserved
+        assert Name.SearchIndex not in pdf.Root.PieceInfo
+        assert Name.SomeOtherApp in pdf.Root.PieceInfo
+
+        # Idempotent: a second call finds nothing to remove
+        assert not discard_text_search_index(pdf)
+
+
+def test_discard_text_search_index_drops_empty_pieceinfo(resources):
+    with pikepdf.open(resources / 'francais.pdf') as pdf:
+        _add_search_index(pdf, other_owner=False)
+        assert discard_text_search_index(pdf)
+        # PieceInfo held only the SearchIndex, so the whole husk is removed
+        assert Name.PieceInfo not in pdf.Root
+
+
+def test_discard_text_search_index_tolerates_malformed_pieceinfo(resources):
+    with pikepdf.open(resources / 'francais.pdf') as pdf:
+        pdf.Root.PieceInfo = String("not a dictionary")
+        assert not discard_text_search_index(pdf)
+
+
+@pytest.fixture
+def pdf_with_search_index(resources, outdir):
+    out = outdir / 'with_search_index.pdf'
+    with pikepdf.open(resources / 'graph.pdf') as pdf:
+        _add_search_index(pdf, other_owner=False)
+        assert Name.SearchIndex in pdf.Root.PieceInfo
+        pdf.save(out)
+    return out
+
+
+def test_search_index_discarded_end_to_end(pdf_with_search_index, outpdf, caplog):
+    caplog.set_level(logging.DEBUG)
+    check_ocrmypdf(
+        pdf_with_search_index,
+        outpdf,
+        '--output-type',
+        'pdf',
+        '--plugin',
+        'tests/plugins/tesseract_noop.py',
+    )
+    with pikepdf.open(outpdf) as pdf:
+        assert Name.PieceInfo not in pdf.Root
+    assert 'search index' in caplog.text.lower()
+
+
+def test_search_index_discarded_with_ocr_engine_none(pdf_with_search_index, outpdf):
+    # Even in pure image-processing mode, OCRmyPDF rewrites the PDF, which
+    # invalidates the embedded index, so it must still be discarded.
+    check_ocrmypdf(
+        pdf_with_search_index,
+        outpdf,
+        '--ocr-engine',
+        'none',
+        '--output-type',
+        'pdf',
+    )
+    with pikepdf.open(outpdf) as pdf:
+        assert Name.PieceInfo not in pdf.Root

tests/test_thumbnails.py

@@ -0,0 +1,93 @@
+# SPDX-FileCopyrightText: 2024 James R. Barlow
+# SPDX-License-Identifier: MPL-2.0
+
+from __future__ import annotations
+
+import logging
+
+import pikepdf
+import pytest
+from pikepdf import Name
+
+from ocrmypdf._graft import discard_page_thumbnails
+
+from .conftest import check_ocrmypdf
+
+# pylint: disable=redefined-outer-name
+
+
+def _add_thumbnail(pdf: pikepdf.Pdf, pageindex: int = 0) -> None:
+    """Attach a minimal /Thumb image XObject to a page."""
+    width, height = 4, 4
+    thumb = pikepdf.Stream(pdf, b'\x00' * (width * height))
+    thumb.Type = Name.XObject
+    thumb.Subtype = Name.Image
+    thumb.Width = width
+    thumb.Height = height
+    thumb.ColorSpace = Name.DeviceGray
+    thumb.BitsPerComponent = 8
+    pdf.pages[pageindex].obj.Thumb = pdf.make_indirect(thumb)
+
+
+def test_discard_page_thumbnails_removes_thumbnails(resources):
+    with pikepdf.open(resources / 'francais.pdf') as pdf:
+        # No thumbnails -> nothing to do
+        assert discard_page_thumbnails(pdf) == 0
+
+        _add_thumbnail(pdf, 0)
+        assert Name.Thumb in pdf.pages[0].obj
+
+        assert discard_page_thumbnails(pdf) == 1
+        assert Name.Thumb not in pdf.pages[0].obj
+
+        # Idempotent: a second call finds nothing to remove
+        assert discard_page_thumbnails(pdf) == 0
+
+
+def test_discard_page_thumbnails_counts_each_page(resources):
+    with pikepdf.open(resources / 'multipage.pdf') as pdf:
+        assert len(pdf.pages) >= 2
+        _add_thumbnail(pdf, 0)
+        _add_thumbnail(pdf, 1)
+        assert discard_page_thumbnails(pdf) == 2
+        assert all(Name.Thumb not in page.obj for page in pdf.pages)
+
+
+@pytest.fixture
+def pdf_with_thumbnail(resources, outdir):
+    out = outdir / 'with_thumbnail.pdf'
+    with pikepdf.open(resources / 'graph.pdf') as pdf:
+        _add_thumbnail(pdf, 0)
+        assert Name.Thumb in pdf.pages[0].obj
+        pdf.save(out)
+    return out
+
+
+def test_thumbnail_discarded_end_to_end(pdf_with_thumbnail, outpdf, caplog):
+    caplog.set_level(logging.DEBUG)
+    check_ocrmypdf(
+        pdf_with_thumbnail,
+        outpdf,
+        '--output-type',
+        'pdf',
+        '--plugin',
+        'tests/plugins/tesseract_noop.py',
+    )
+    with pikepdf.open(outpdf) as pdf:
+        assert all(Name.Thumb not in page.obj for page in pdf.pages)
+    assert 'thumbnail' in caplog.text.lower()
+
+
+def test_thumbnail_discarded_with_ocr_engine_none(pdf_with_thumbnail, outpdf):
+    # Even in pure image-processing mode, OCRmyPDF rewrites the PDF, which can
+    # alter page appearance, so the stale thumbnail must still be discarded.
+    check_ocrmypdf(
+        pdf_with_thumbnail,
+        outpdf,
+        '--ocr-engine',
+        'none',
+        '--output-type',
+        'pdf',
+    )
+    with pikepdf.open(outpdf) as pdf:
+        assert all(Name.Thumb not in page.obj for page in pdf.pages)

uv.lock

@@ -1434,7 +1434,7 @@ wheels = [
 
 [[package]]
 name = "ocrmypdf"
-version = "17.4.2"
+version = "17.5.0"
 source = { editable = "." }
 dependencies = [
     { name = "deprecation" },