Merge branch 'main' into dev

2025-12-24 00:44:18 -08:00
parent e9bfce34f1 94d7735862
commit 0d6e0c4560
4 changed files with 302 additions and 17 deletions
@@ -33,7 +33,7 @@ jobs:
      PYTHON: ${{ matrix.python }}

    steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
        with:
          fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags

@@ -109,7 +109,7 @@ jobs:
      PYTHON: ${{ matrix.python }}

    steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
        with:
          fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags

@@ -171,7 +171,7 @@ jobs:
      PYTHON: ${{ matrix.python }}

    steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
        with:
          fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags

@@ -210,7 +210,7 @@ jobs:
    name: Build sdist and wheels
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
        with:
          fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags

@@ -223,7 +223,7 @@ jobs:
        run: |
          uv build --sdist --wheel

-      - uses: actions/upload-artifact@v5
+      - uses: actions/upload-artifact@v6
        with:
          name: artifact
          path: |
@@ -239,7 +239,7 @@ jobs:
      id-token: write # mandatory for PyPI publishing
    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v')
    steps:
-      - uses: actions/download-artifact@v6
+      - uses: actions/download-artifact@v7
        with:
          name: artifact
          path: dist
@@ -257,13 +257,13 @@ jobs:
      contents: write
      id-token: write
    steps:
-      - uses: actions/download-artifact@v6
+      - uses: actions/download-artifact@v7
        with:
          name: artifact
          path: dist

      - name: Sign the dists with Sigstore
-        uses: sigstore/gh-action-sigstore-python@v3.1.0
+        uses: sigstore/gh-action-sigstore-python@v3.2.0
        with:
          inputs: |
            ./dist/*.tar.gz
@@ -308,7 +308,7 @@ jobs:
      - name: Set image name
        run: echo "DOCKER_IMAGE_NAME=ocrmypdf" >> $GITHUB_ENV

-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
        with:
          fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags

@@ -356,7 +356,7 @@ jobs:
      - name: Set image name
        run: echo "DOCKER_IMAGE_NAME=ocrmypdf-alpine" >> $GITHUB_ENV

-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
        with:
          fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags

@@ -25,15 +25,15 @@ about a forthcoming release that has not been tagged yet. A release is only
 official when it's tagged and posted to PyPI.
 :::

-## v16.13.0
+## v17.0.0 (unreleased)

 **Breaking changes**

- **Plugin interface migration**: Plugin hooks now receive `OCROptions` objects instead of 
-  `argparse.Namespace` objects. Most plugins will continue working due to duck-typing 
-  compatibility, but plugin developers should update their type hints from `Namespace` 
+- **Plugin interface migration**: Plugin hooks now receive `OCROptions` objects instead of
+  `argparse.Namespace` objects. Most plugins will continue working due to duck-typing
+  compatibility, but plugin developers should update their type hints from `Namespace`
  to `OCROptions`.
- Built-in plugins no longer modify options in-place, improving immutability and 
+- Built-in plugins no longer modify options in-place, improving immutability and
  code clarity.

 **API improvements**
@@ -51,10 +51,21 @@ official when it's tagged and posted to PyPI.
 - Remove any in-place option modifications - compute values at point of use instead
 - Most existing plugins will continue working without changes due to duck-typing

+## v16.13.0
+
+- Added detection and repair for Ghostscript 10.6 JPEG corruption. When GS 10.6
+  truncates JPEG data by 1-15 bytes, OCRmyPDF now restores the original image
+  bytes from the input PDF. A warning is issued when GS 10.6+ is detected.
+  {issue}`1603`
+- We continue to force re-optimization of JPEGs, since this catches some issues with corruption for situations where Ghostscript modifies an image. It is likely there are still cases where we cannot mitigate all corruption issues. {issue}`1585`
+- Fixed handling of PDF page boxes (ArtBox, BleedBox) which were not being
+  processed correctly in some cases. {issue}`1181,1360`
+- Documentation: clarified podman usage instructions.
+
 ## v16.12.0

- Disable Ghostscript's subset fonts feature, which was found to corrupt text in
-  certain PDFs. Thanks @mnaegler for identifying this issue. {issue}`1592`
+- Disable Ghostscript's subset fonts feature, which was found to corrupt text in certain
+  PDFs. Thanks @mnaegler for identifying this issue. {issue}`1592`
 - Users of Ghostscript 10.6.0+ reported that Ghostscript seems to generate corrupted
  JPEGs. We force re-optimization of these JPEGs to mitigate the corruption until
  Ghostscript fixes the issue. {issue}`1585`
@@ -5,9 +5,11 @@
 from __future__ import annotations

 import logging
+from pathlib import Path
 from typing import Annotated

 from packaging.version import Version
+from pikepdf import Name, Pdf, Stream
 from pydantic import BaseModel, Field

 from ocrmypdf import hookimpl
@@ -103,6 +105,15 @@ def check_options(options):
            "newer version, or use --output-type pdf to avoid Ghostscript, or "
            "use --force-ocr to discard existing text."
        )
+    if gs_version >= Version('10.6.0') and options.output_type.startswith('pdfa'):
+        log.warning(
+            "Ghostscript 10.6.x contains JPEG encoding errors that may corrupt "
+            "images. OCRmyPDF will attempt to mitigate, but this version is "
+            "strongly not recommended. Please upgrade to a newer version. "
+            "As of 2025-12, 10.6.0 is the latest version of Ghostscript."
+        )
+    if options.output_type == 'pdfa':
+        options.output_type = 'pdfa-2'

    if (
        options.ghostscript.color_conversion_strategy
@@ -156,6 +167,144 @@ def rasterize_pdf_page(
    return output_file


+def _collect_dctdecode_images(pdf: Pdf) -> dict[tuple, list[tuple[Stream, bytes]]]:
+    """Collect all DCTDecode (JPEG) images from a PDF.
+
+    Returns a dict mapping image signatures to a list of (stream, raw_bytes) tuples.
+    The signature is (Width, Height, Filter, BitsPerComponent, ColorSpace).
+    """
+    images: dict[tuple, list[tuple[Stream, bytes]]] = {}
+
+    def get_colorspace_key(obj):
+        """Get a hashable key for the colorspace."""
+        cs = obj.get(Name.ColorSpace)
+        if cs is None:
+            return None
+        if isinstance(cs, Name):
+            return str(cs)
+        # For array colorspaces like [/ICCBased ...], use the first element
+        try:
+            return str(cs[0]) if len(cs) > 0 else str(cs)
+        except (TypeError, KeyError):
+            return str(cs)
+
+    def process_xobject_dict(xobjects, depth=0):
+        """Process an XObject dictionary for DCTDecode images."""
+        if xobjects is None:
+            return
+        if depth > 10:
+            log.warning("Recursion depth exceeded in _collect_dctdecode_images")
+            return
+        for key in xobjects.keys():
+            obj = xobjects[key]
+            if obj is None:
+                continue
+            # Check if it's an image with DCTDecode
+            if obj.get(Name.Subtype) == Name.Image:
+                filt = obj.get(Name.Filter)
+                if filt == Name.DCTDecode:
+                    sig = (
+                        int(obj.get(Name.Width, 0)),
+                        int(obj.get(Name.Height, 0)),
+                        str(filt),
+                        int(obj.get(Name.BitsPerComponent, 0)),
+                        get_colorspace_key(obj),
+                    )
+                    raw_bytes = obj.read_raw_bytes()
+                    if sig not in images:
+                        images[sig] = []
+                    images[sig].append((obj, raw_bytes))
+            # Recurse into Form XObjects
+            elif obj.get(Name.Subtype) == Name.Form:
+                if Name.Resources in obj:
+                    res = obj[Name.Resources]
+                    if Name.XObject in res:
+                        process_xobject_dict(res[Name.XObject], depth=depth + 1)
+
+    for page in pdf.pages:
+        if Name.Resources not in page:
+            continue
+        resources = page[Name.Resources]
+        if Name.XObject not in resources:
+            continue
+        process_xobject_dict(resources[Name.XObject])
+
+    return images
+
+
+def _repair_gs106_jpeg_corruption(
+    input_pdf_path: Path,
+    output_pdf_path: Path,
+) -> bool:
+    """Repair JPEG corruption caused by Ghostscript 10.6.
+
+    Ghostscript 10.6 has a bug that truncates JPEG data by 1-15 bytes.
+    This function detects and repairs such corruption by copying the
+    original JPEG bytes from the input PDF.
+
+    Returns True if any repairs were made.
+    """
+    repaired_count = 0
+    first_error_logged = False
+
+    with (
+        Pdf.open(input_pdf_path) as input_pdf,
+        Pdf.open(output_pdf_path, allow_overwriting_input=True) as output_pdf,
+    ):
+        # Collect all DCTDecode images from both PDFs
+        input_images = _collect_dctdecode_images(input_pdf)
+        output_images = _collect_dctdecode_images(output_pdf)
+
+        # For each output image, try to find a corresponding input image
+        for sig, output_list in output_images.items():
+            if sig not in input_images:
+                continue
+            input_list = input_images[sig]
+
+            for output_stream, output_bytes in output_list:
+                # Try to find a matching input image
+                for _input_stream, input_bytes in input_list:
+                    input_len = len(input_bytes)
+                    output_len = len(output_bytes)
+
+                    # Check if output is 1-15 bytes shorter
+                    diff = input_len - output_len
+                    if not (1 <= diff <= 15):
+                        continue
+
+                    # Check if the bytes are identical up to the truncation point
+                    if output_bytes != input_bytes[:output_len]:
+                        continue
+
+                    # This is a corrupt image - repair it
+                    if not first_error_logged:
+                        log.error(
+                            "Ghostscript 10.6 JPEG corruption detected. "
+                            "Repairing damaged images from original PDF."
+                        )
+                        first_error_logged = True
+                    log.warning(
+                        f"Replacing corrupt JPEG image "
+                        f"({sig[0]}x{sig[1]}, {diff} bytes truncated)"
+                    )
+
+                    # Write the original bytes back to the output stream
+                    output_stream.write(
+                        input_bytes,
+                        filter=Name.DCTDecode,
+                    )
+                    repaired_count += 1
+                    break  # Move to next output image
+
+        if repaired_count > 0:
+            output_pdf.save(output_pdf_path)
+            log.info(
+                f"Repaired {repaired_count} JPEG image(s) corrupted by Ghostscript"
+            )
+
+    return repaired_count > 0
+
+
@hookimpl
 def generate_pdfa(
    pdf_pages,
@@ -183,4 +332,11 @@ def generate_pdfa(
        progressbar_class=progressbar_class,
        stop_on_error=stop_on_soft_error,
    )
+
+    # Repair JPEG corruption caused by Ghostscript 10.6.x
+    gs_version = ghostscript.version()
+    if gs_version >= Version('10.6.0') and len(pdf_pages) == 1:
+        input_pdf = Path(pdf_pages[0])
+        _repair_gs106_jpeg_corruption(input_pdf, Path(output_file))
+
    return output_file
@@ -17,6 +17,7 @@ from PIL import Image, UnidentifiedImageError

 from ocrmypdf._exec import ghostscript
 from ocrmypdf._exec.ghostscript import DuplicateFilter, rasterize_pdf
+from ocrmypdf.builtin_plugins.ghostscript import _repair_gs106_jpeg_corruption
 from ocrmypdf.exceptions import ColorConversionNeededError, ExitCode, InputFileError
 from ocrmypdf.helpers import Resolution

@@ -287,3 +288,120 @@ def test_recoverable_image_error_with_stop(pdf_with_invalid_image, outdir, caplo
            stop_on_error=True,
        )
    # out2.png will not be created; if it were it would be blank.
+
+
+class TestGs106JpegCorruptionRepair:
+    """Test the Ghostscript 10.6 JPEG corruption repair function."""
+
+    @pytest.fixture
+    def create_damaged_pdf(self, resources, outdir):
+        """Create a damaged PDF by truncating JPEG data by 2 bytes."""
+
+        def _create_damaged(source_pdf_name='francais.pdf', truncate_bytes=2):
+            source_path = resources / source_pdf_name
+            damaged_path = outdir / 'damaged.pdf'
+
+            with pikepdf.open(source_path) as pdf:
+                # Find and truncate DCTDecode images
+                Name = pikepdf.Name
+                damaged_count = 0
+                for page in pdf.pages:
+                    if Name.Resources not in page:
+                        continue
+                    resources_dict = page[Name.Resources]
+                    if Name.XObject not in resources_dict:
+                        continue
+                    for key in resources_dict[Name.XObject].keys():
+                        obj = resources_dict[Name.XObject][key]
+                        if obj.get(Name.Subtype) != Name.Image:
+                            continue
+                        if obj.get(Name.Filter) != Name.DCTDecode:
+                            continue
+                        # Truncate the JPEG data
+                        original_bytes = obj.read_raw_bytes()
+                        truncated_bytes = original_bytes[:-truncate_bytes]
+                        obj.write(truncated_bytes, filter=Name.DCTDecode)
+                        damaged_count += 1
+
+                pdf.save(damaged_path)
+                return source_path, damaged_path, damaged_count
+
+        return _create_damaged
+
+    def test_repair_truncated_jpeg(self, create_damaged_pdf, caplog):
+        """Test that truncated JPEG images are repaired."""
+        caplog.set_level(logging.DEBUG)
+        source_path, damaged_path, damaged_count = create_damaged_pdf()
+
+        assert damaged_count > 0, "Test PDF should have DCTDecode images"
+
+        # Get original image bytes for comparison
+        with pikepdf.open(source_path) as pdf:
+            Name = pikepdf.Name
+            original_bytes_list = []
+            for page in pdf.pages:
+                if Name.Resources not in page:
+                    continue
+                resources_dict = page[Name.Resources]
+                if Name.XObject not in resources_dict:
+                    continue
+                for key in resources_dict[Name.XObject].keys():
+                    obj = resources_dict[Name.XObject][key]
+                    if obj.get(Name.Subtype) != Name.Image:
+                        continue
+                    if obj.get(Name.Filter) != Name.DCTDecode:
+                        continue
+                    original_bytes_list.append(obj.read_raw_bytes())
+
+        # Run the repair function
+        repaired = _repair_gs106_jpeg_corruption(source_path, damaged_path)
+        assert repaired is True, "Repair should have been performed"
+
+        # Verify the repaired PDF has correct image bytes
+        with pikepdf.open(damaged_path) as pdf:
+            Name = pikepdf.Name
+            repaired_bytes_list = []
+            for page in pdf.pages:
+                if Name.Resources not in page:
+                    continue
+                resources_dict = page[Name.Resources]
+                if Name.XObject not in resources_dict:
+                    continue
+                for key in resources_dict[Name.XObject].keys():
+                    obj = resources_dict[Name.XObject][key]
+                    if obj.get(Name.Subtype) != Name.Image:
+                        continue
+                    if obj.get(Name.Filter) != Name.DCTDecode:
+                        continue
+                    repaired_bytes_list.append(obj.read_raw_bytes())
+
+        assert len(repaired_bytes_list) == len(original_bytes_list)
+        for orig, repaired_bytes in zip(original_bytes_list, repaired_bytes_list):
+            assert orig == repaired_bytes, "Repaired bytes should match original"
+
+        # Check that error/warning was logged
+        assert "JPEG corruption detected" in caplog.text
+
+    def test_no_repair_when_not_truncated(self, resources, outdir, caplog):
+        """Test that no repair is done when images are not truncated."""
+        caplog.set_level(logging.DEBUG)
+        source_path = resources / 'francais.pdf'
+
+        # Copy source to output (no damage)
+        output_path = outdir / 'undamaged.pdf'
+        with pikepdf.open(source_path) as pdf:
+            pdf.save(output_path)
+
+        # Run the repair function - should not repair anything
+        repaired = _repair_gs106_jpeg_corruption(source_path, output_path)
+        assert repaired is False, "No repair should have been performed"
+        assert "JPEG corruption detected" not in caplog.text
+
+    def test_no_repair_when_truncation_too_large(self, create_damaged_pdf, caplog):
+        """Test that images truncated by more than 15 bytes are not repaired."""
+        caplog.set_level(logging.DEBUG)
+        source_path, damaged_path, _ = create_damaged_pdf(truncate_bytes=20)
+
+        repaired = _repair_gs106_jpeg_corruption(source_path, damaged_path)
+        assert repaired is False, "Should not repair truncation > 15 bytes"
+        assert "JPEG corruption detected" not in caplog.text