Fix Ghostscript 10.6 JPEG corruption by repairing truncated images
Ghostscript 10.6 has a bug that truncates JPEG data by 1-15 bytes. This adds detection and repair by comparing output images to input images and restoring the original bytes when truncation is detected. - Add warning when GS 10.6+ is used with PDF/A output - Add _repair_gs106_jpeg_corruption() to fix damaged JPEGs after Ghostscript processing - Add unit tests for the repair function
This commit is contained in:
James R. Barlow
@@ -5,8 +5,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from packaging.version import Version
|
||||
from pikepdf import Name, Pdf, Stream
|
||||
|
||||
from ocrmypdf import hookimpl
|
||||
from ocrmypdf._exec import ghostscript
|
||||
@@ -74,6 +76,13 @@ def check_options(options):
|
||||
"use --force-ocr to discard existing text."
|
||||
)
|
||||
|
||||
if gs_version >= Version('10.6.0') and options.output_type.startswith('pdfa'):
|
||||
log.warning(
|
||||
"Ghostscript 10.6.x contains JPEG encoding errors that may corrupt "
|
||||
"images. OCRmyPDF will attempt to mitigate, but this version is "
|
||||
"strongly not recommended. Please upgrade to a newer version. "
|
||||
"As of 2025-12, 10.6.0 is the latest version of Ghostscript."
|
||||
)
|
||||
if options.output_type == 'pdfa':
|
||||
options.output_type = 'pdfa-2'
|
||||
if options.color_conversion_strategy not in ghostscript.COLOR_CONVERSION_STRATEGIES:
|
||||
@@ -116,6 +125,144 @@ def rasterize_pdf_page(
|
||||
return output_file
|
||||
|
||||
|
||||
def _collect_dctdecode_images(pdf: Pdf) -> dict[tuple, list[tuple[Stream, bytes]]]:
|
||||
"""Collect all DCTDecode (JPEG) images from a PDF.
|
||||
|
||||
Returns a dict mapping image signatures to a list of (stream, raw_bytes) tuples.
|
||||
The signature is (Width, Height, Filter, BitsPerComponent, ColorSpace).
|
||||
"""
|
||||
images: dict[tuple, list[tuple[Stream, bytes]]] = {}
|
||||
|
||||
def get_colorspace_key(obj):
|
||||
"""Get a hashable key for the colorspace."""
|
||||
cs = obj.get(Name.ColorSpace)
|
||||
if cs is None:
|
||||
return None
|
||||
if isinstance(cs, Name):
|
||||
return str(cs)
|
||||
# For array colorspaces like [/ICCBased ...], use the first element
|
||||
try:
|
||||
return str(cs[0]) if len(cs) > 0 else str(cs)
|
||||
except (TypeError, KeyError):
|
||||
return str(cs)
|
||||
|
||||
def process_xobject_dict(xobjects, depth=0):
|
||||
"""Process an XObject dictionary for DCTDecode images."""
|
||||
if xobjects is None:
|
||||
return
|
||||
if depth > 10:
|
||||
log.warning("Recursion depth exceeded in _collect_dctdecode_images")
|
||||
return
|
||||
for key in xobjects.keys():
|
||||
obj = xobjects[key]
|
||||
if obj is None:
|
||||
continue
|
||||
# Check if it's an image with DCTDecode
|
||||
if obj.get(Name.Subtype) == Name.Image:
|
||||
filt = obj.get(Name.Filter)
|
||||
if filt == Name.DCTDecode:
|
||||
sig = (
|
||||
int(obj.get(Name.Width, 0)),
|
||||
int(obj.get(Name.Height, 0)),
|
||||
str(filt),
|
||||
int(obj.get(Name.BitsPerComponent, 0)),
|
||||
get_colorspace_key(obj),
|
||||
)
|
||||
raw_bytes = obj.read_raw_bytes()
|
||||
if sig not in images:
|
||||
images[sig] = []
|
||||
images[sig].append((obj, raw_bytes))
|
||||
# Recurse into Form XObjects
|
||||
elif obj.get(Name.Subtype) == Name.Form:
|
||||
if Name.Resources in obj:
|
||||
res = obj[Name.Resources]
|
||||
if Name.XObject in res:
|
||||
process_xobject_dict(res[Name.XObject], depth=depth + 1)
|
||||
|
||||
for page in pdf.pages:
|
||||
if Name.Resources not in page:
|
||||
continue
|
||||
resources = page[Name.Resources]
|
||||
if Name.XObject not in resources:
|
||||
continue
|
||||
process_xobject_dict(resources[Name.XObject])
|
||||
|
||||
return images
|
||||
|
||||
|
||||
def _repair_gs106_jpeg_corruption(
|
||||
input_pdf_path: Path,
|
||||
output_pdf_path: Path,
|
||||
) -> bool:
|
||||
"""Repair JPEG corruption caused by Ghostscript 10.6.
|
||||
|
||||
Ghostscript 10.6 has a bug that truncates JPEG data by 1-15 bytes.
|
||||
This function detects and repairs such corruption by copying the
|
||||
original JPEG bytes from the input PDF.
|
||||
|
||||
Returns True if any repairs were made.
|
||||
"""
|
||||
repaired_count = 0
|
||||
first_error_logged = False
|
||||
|
||||
with (
|
||||
Pdf.open(input_pdf_path) as input_pdf,
|
||||
Pdf.open(output_pdf_path, allow_overwriting_input=True) as output_pdf,
|
||||
):
|
||||
# Collect all DCTDecode images from both PDFs
|
||||
input_images = _collect_dctdecode_images(input_pdf)
|
||||
output_images = _collect_dctdecode_images(output_pdf)
|
||||
|
||||
# For each output image, try to find a corresponding input image
|
||||
for sig, output_list in output_images.items():
|
||||
if sig not in input_images:
|
||||
continue
|
||||
input_list = input_images[sig]
|
||||
|
||||
for output_stream, output_bytes in output_list:
|
||||
# Try to find a matching input image
|
||||
for _input_stream, input_bytes in input_list:
|
||||
input_len = len(input_bytes)
|
||||
output_len = len(output_bytes)
|
||||
|
||||
# Check if output is 1-15 bytes shorter
|
||||
diff = input_len - output_len
|
||||
if not (1 <= diff <= 15):
|
||||
continue
|
||||
|
||||
# Check if the bytes are identical up to the truncation point
|
||||
if output_bytes != input_bytes[:output_len]:
|
||||
continue
|
||||
|
||||
# This is a corrupt image - repair it
|
||||
if not first_error_logged:
|
||||
log.error(
|
||||
"Ghostscript 10.6 JPEG corruption detected. "
|
||||
"Repairing damaged images from original PDF."
|
||||
)
|
||||
first_error_logged = True
|
||||
log.warning(
|
||||
f"Replacing corrupt JPEG image "
|
||||
f"({sig[0]}x{sig[1]}, {diff} bytes truncated)"
|
||||
)
|
||||
|
||||
# Write the original bytes back to the output stream
|
||||
output_stream.write(
|
||||
input_bytes,
|
||||
filter=Name.DCTDecode,
|
||||
)
|
||||
repaired_count += 1
|
||||
break # Move to next output image
|
||||
|
||||
if repaired_count > 0:
|
||||
output_pdf.save(output_pdf_path)
|
||||
log.info(
|
||||
f"Repaired {repaired_count} JPEG image(s) corrupted by Ghostscript"
|
||||
)
|
||||
|
||||
return repaired_count > 0
|
||||
|
||||
|
||||
@hookimpl
|
||||
def generate_pdfa(
|
||||
pdf_pages,
|
||||
@@ -138,4 +285,11 @@ def generate_pdfa(
|
||||
progressbar_class=progressbar_class,
|
||||
stop_on_error=stop_on_soft_error,
|
||||
)
|
||||
|
||||
# Repair JPEG corruption caused by Ghostscript 10.6.x
|
||||
gs_version = ghostscript.version()
|
||||
if gs_version >= Version('10.6.0') and len(pdf_pages) == 1:
|
||||
input_pdf = Path(pdf_pages[0])
|
||||
_repair_gs106_jpeg_corruption(input_pdf, Path(output_file))
|
||||
|
||||
return output_file
|
||||
|
||||
@@ -17,6 +17,7 @@ from PIL import Image, UnidentifiedImageError
|
||||
|
||||
from ocrmypdf._exec import ghostscript
|
||||
from ocrmypdf._exec.ghostscript import DuplicateFilter, rasterize_pdf
|
||||
from ocrmypdf.builtin_plugins.ghostscript import _repair_gs106_jpeg_corruption
|
||||
from ocrmypdf.exceptions import ColorConversionNeededError, ExitCode, InputFileError
|
||||
from ocrmypdf.helpers import Resolution
|
||||
|
||||
@@ -287,3 +288,120 @@ def test_recoverable_image_error_with_stop(pdf_with_invalid_image, outdir, caplo
|
||||
stop_on_error=True,
|
||||
)
|
||||
# out2.png will not be created; if it were it would be blank.
|
||||
|
||||
|
||||
class TestGs106JpegCorruptionRepair:
|
||||
"""Test the Ghostscript 10.6 JPEG corruption repair function."""
|
||||
|
||||
@pytest.fixture
|
||||
def create_damaged_pdf(self, resources, outdir):
|
||||
"""Create a damaged PDF by truncating JPEG data by 2 bytes."""
|
||||
|
||||
def _create_damaged(source_pdf_name='francais.pdf', truncate_bytes=2):
|
||||
source_path = resources / source_pdf_name
|
||||
damaged_path = outdir / 'damaged.pdf'
|
||||
|
||||
with pikepdf.open(source_path) as pdf:
|
||||
# Find and truncate DCTDecode images
|
||||
Name = pikepdf.Name
|
||||
damaged_count = 0
|
||||
for page in pdf.pages:
|
||||
if Name.Resources not in page:
|
||||
continue
|
||||
resources_dict = page[Name.Resources]
|
||||
if Name.XObject not in resources_dict:
|
||||
continue
|
||||
for key in resources_dict[Name.XObject].keys():
|
||||
obj = resources_dict[Name.XObject][key]
|
||||
if obj.get(Name.Subtype) != Name.Image:
|
||||
continue
|
||||
if obj.get(Name.Filter) != Name.DCTDecode:
|
||||
continue
|
||||
# Truncate the JPEG data
|
||||
original_bytes = obj.read_raw_bytes()
|
||||
truncated_bytes = original_bytes[:-truncate_bytes]
|
||||
obj.write(truncated_bytes, filter=Name.DCTDecode)
|
||||
damaged_count += 1
|
||||
|
||||
pdf.save(damaged_path)
|
||||
return source_path, damaged_path, damaged_count
|
||||
|
||||
return _create_damaged
|
||||
|
||||
def test_repair_truncated_jpeg(self, create_damaged_pdf, caplog):
|
||||
"""Test that truncated JPEG images are repaired."""
|
||||
caplog.set_level(logging.DEBUG)
|
||||
source_path, damaged_path, damaged_count = create_damaged_pdf()
|
||||
|
||||
assert damaged_count > 0, "Test PDF should have DCTDecode images"
|
||||
|
||||
# Get original image bytes for comparison
|
||||
with pikepdf.open(source_path) as pdf:
|
||||
Name = pikepdf.Name
|
||||
original_bytes_list = []
|
||||
for page in pdf.pages:
|
||||
if Name.Resources not in page:
|
||||
continue
|
||||
resources_dict = page[Name.Resources]
|
||||
if Name.XObject not in resources_dict:
|
||||
continue
|
||||
for key in resources_dict[Name.XObject].keys():
|
||||
obj = resources_dict[Name.XObject][key]
|
||||
if obj.get(Name.Subtype) != Name.Image:
|
||||
continue
|
||||
if obj.get(Name.Filter) != Name.DCTDecode:
|
||||
continue
|
||||
original_bytes_list.append(obj.read_raw_bytes())
|
||||
|
||||
# Run the repair function
|
||||
repaired = _repair_gs106_jpeg_corruption(source_path, damaged_path)
|
||||
assert repaired is True, "Repair should have been performed"
|
||||
|
||||
# Verify the repaired PDF has correct image bytes
|
||||
with pikepdf.open(damaged_path) as pdf:
|
||||
Name = pikepdf.Name
|
||||
repaired_bytes_list = []
|
||||
for page in pdf.pages:
|
||||
if Name.Resources not in page:
|
||||
continue
|
||||
resources_dict = page[Name.Resources]
|
||||
if Name.XObject not in resources_dict:
|
||||
continue
|
||||
for key in resources_dict[Name.XObject].keys():
|
||||
obj = resources_dict[Name.XObject][key]
|
||||
if obj.get(Name.Subtype) != Name.Image:
|
||||
continue
|
||||
if obj.get(Name.Filter) != Name.DCTDecode:
|
||||
continue
|
||||
repaired_bytes_list.append(obj.read_raw_bytes())
|
||||
|
||||
assert len(repaired_bytes_list) == len(original_bytes_list)
|
||||
for orig, repaired_bytes in zip(original_bytes_list, repaired_bytes_list):
|
||||
assert orig == repaired_bytes, "Repaired bytes should match original"
|
||||
|
||||
# Check that error/warning was logged
|
||||
assert "JPEG corruption detected" in caplog.text
|
||||
|
||||
def test_no_repair_when_not_truncated(self, resources, outdir, caplog):
|
||||
"""Test that no repair is done when images are not truncated."""
|
||||
caplog.set_level(logging.DEBUG)
|
||||
source_path = resources / 'francais.pdf'
|
||||
|
||||
# Copy source to output (no damage)
|
||||
output_path = outdir / 'undamaged.pdf'
|
||||
with pikepdf.open(source_path) as pdf:
|
||||
pdf.save(output_path)
|
||||
|
||||
# Run the repair function - should not repair anything
|
||||
repaired = _repair_gs106_jpeg_corruption(source_path, output_path)
|
||||
assert repaired is False, "No repair should have been performed"
|
||||
assert "JPEG corruption detected" not in caplog.text
|
||||
|
||||
def test_no_repair_when_truncation_too_large(self, create_damaged_pdf, caplog):
|
||||
"""Test that images truncated by more than 15 bytes are not repaired."""
|
||||
caplog.set_level(logging.DEBUG)
|
||||
source_path, damaged_path, _ = create_damaged_pdf(truncate_bytes=20)
|
||||
|
||||
repaired = _repair_gs106_jpeg_corruption(source_path, damaged_path)
|
||||
assert repaired is False, "Should not repair truncation > 15 bytes"
|
||||
assert "JPEG corruption detected" not in caplog.text
|
||||
|
||||
Reference in New Issue
Block a user