From 0f3c7765aa90a5522cf176ccfec5a47e4779dc34 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 11:31:33 +0000 Subject: [PATCH 01/14] Bump actions/checkout from 5 to 6 Bumps [actions/checkout](https://github.com/actions/checkout) from 5 to 6. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index eef64de4..1f84f090 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,7 +33,7 @@ jobs: PYTHON: ${{ matrix.python }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags @@ -109,7 +109,7 @@ jobs: PYTHON: ${{ matrix.python }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags @@ -171,7 +171,7 @@ jobs: PYTHON: ${{ matrix.python }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags @@ -210,7 +210,7 @@ jobs: name: Build sdist and wheels runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags @@ -308,7 +308,7 @@ jobs: - name: Set image name run: echo "DOCKER_IMAGE_NAME=ocrmypdf" >> $GITHUB_ENV - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags @@ -356,7 +356,7 @@ jobs: - name: Set image name run: echo "DOCKER_IMAGE_NAME=ocrmypdf-alpine" >> $GITHUB_ENV - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags From f673da9ab975a21070d77afdc927de00c6c08c46 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Dec 2025 10:02:38 +0000 Subject: [PATCH 02/14] Bump sigstore/gh-action-sigstore-python from 3.1.0 to 3.2.0 Bumps [sigstore/gh-action-sigstore-python](https://github.com/sigstore/gh-action-sigstore-python) from 3.1.0 to 3.2.0. - [Release notes](https://github.com/sigstore/gh-action-sigstore-python/releases) - [Changelog](https://github.com/sigstore/gh-action-sigstore-python/blob/main/CHANGELOG.md) - [Commits](https://github.com/sigstore/gh-action-sigstore-python/compare/v3.1.0...v3.2.0) --- updated-dependencies: - dependency-name: sigstore/gh-action-sigstore-python dependency-version: 3.2.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index eef64de4..298f9f78 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -263,7 +263,7 @@ jobs: path: dist - name: Sign the dists with Sigstore - uses: sigstore/gh-action-sigstore-python@v3.1.0 + uses: sigstore/gh-action-sigstore-python@v3.2.0 with: inputs: | ./dist/*.tar.gz From c6b21d4dea7eb4beac8913121abc620681ee6dd5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Dec 2025 10:02:24 +0000 Subject: [PATCH 03/14] Bump actions/upload-artifact from 5 to 6 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 5 to 6. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index eef64de4..6b6467eb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -223,7 +223,7 @@ jobs: run: | uv build --sdist --wheel - - uses: actions/upload-artifact@v5 + - uses: actions/upload-artifact@v6 with: name: artifact path: | From cdf956ffc4012994162bcdcde19b03c78b43c9ca Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Dec 2025 10:02:30 +0000 Subject: [PATCH 04/14] Bump actions/download-artifact from 6 to 7 Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 6 to 7. - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v6...v7) --- updated-dependencies: - dependency-name: actions/download-artifact dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index eef64de4..383d9ed0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -239,7 +239,7 @@ jobs: id-token: write # mandatory for PyPI publishing if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v') steps: - - uses: actions/download-artifact@v6 + - uses: actions/download-artifact@v7 with: name: artifact path: dist @@ -257,7 +257,7 @@ jobs: contents: write id-token: write steps: - - uses: actions/download-artifact@v6 + - uses: actions/download-artifact@v7 with: name: artifact path: dist From eace567f7bfc82d70d7b4771cb1ea7874761bb48 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Tue, 11 Feb 2025 00:40:01 -0800 Subject: [PATCH 05/14] Test and fix page box issues --- src/ocrmypdf/_pipeline.py | 14 +++-- tests/test_page_boxes.py | 122 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 4 deletions(-) create mode 100644 tests/test_page_boxes.py diff --git a/src/ocrmypdf/_pipeline.py b/src/ocrmypdf/_pipeline.py index 90524d58..3eab75f9 100644 --- a/src/ocrmypdf/_pipeline.py +++ b/src/ocrmypdf/_pipeline.py @@ -549,7 +549,9 @@ def rasterize( device = colorspaces[device_idx] - log.debug(f"Rasterize with {device}, rotation {correction}") + log.debug( + f"Rasterize with {device}, rotation {correction}, mediabox {pageinfo.mediabox}" + ) canvas_dpi, page_dpi = calculate_raster_dpi(page_context) @@ -840,7 +842,7 @@ def fix_pagepdf_boxes( The single page PDF is created with a normal MediaBox with its lower left corner at (0, 0). infile is the single page PDF. page_context.mediabox has the original - file's mediabox, which may have a different origin. We needto adjust the other + file's mediabox, which may have a different origin. We need to adjust the other boxes in the single page PDF to match the effect they had on the original page. When correcting page rotation, we create a single page PDF that is correctly @@ -856,16 +858,20 @@ def fix_pagepdf_boxes( for page in pdf.pages: # page.BleedBox = page_context.pageinfo.bleedbox # page.ArtBox = page_context.pageinfo.artbox + log.debug( + f"initial mediabox={page.MediaBox} and pageinfo mediabox={page_context.pageinfo.mediabox}" + ) mediabox = page_context.pageinfo.mediabox - offset = mediabox[0], mediabox[1] + offset = -mediabox[0], -mediabox[1] cropbox = _offset_rect(page_context.pageinfo.cropbox, offset) trimbox = _offset_rect(page_context.pageinfo.trimbox, offset) - if swap_axis: cropbox = cropbox[1], cropbox[0], cropbox[3], cropbox[2] trimbox = trimbox[1], trimbox[0], trimbox[3], trimbox[2] + mediabox = mediabox[1], mediabox[0], mediabox[3], mediabox[2] page.CropBox = cropbox page.TrimBox = trimbox + log.debug(f"cropbox={cropbox}, trimbox={trimbox}, mediabox={mediabox}") pdf.save(out_file) return out_file diff --git a/tests/test_page_boxes.py b/tests/test_page_boxes.py new file mode 100644 index 00000000..cc3386b1 --- /dev/null +++ b/tests/test_page_boxes.py @@ -0,0 +1,122 @@ +# SPDX-FileCopyrightText: 2025 James R. Barlow +# SPDX-License-Identifier: MPL-2.0 + +from __future__ import annotations + +import pikepdf +import pytest + +from .conftest import check_ocrmypdf + +page_rect = [0, 0, 612, 792] +inset_rect = [200, 200, 612, 792] +wh_rect = [0, 0, 412, 592] + +neg_rect = [-100, -100, 512, 692] + +mediabox_testdata = [ + ('hocr', 'pdfa', 'ccitt.pdf', None, inset_rect, wh_rect), + ('sandwich', 'pdfa', 'ccitt.pdf', None, inset_rect, wh_rect), + ('hocr', 'pdf', 'ccitt.pdf', None, inset_rect, inset_rect), + ('sandwich', 'pdf', 'ccitt.pdf', None, inset_rect, inset_rect), + ( + 'hocr', + 'pdfa', + 'ccitt.pdf', + '--force-ocr', + inset_rect, + wh_rect, + ), + ( + 'hocr', + 'pdf', + 'ccitt.pdf', + '--force-ocr', + inset_rect, + wh_rect, + ), + ('hocr', 'pdfa', 'ccitt.pdf', '--force-ocr', neg_rect, page_rect), + ('hocr', 'pdf', 'ccitt.pdf', '--force-ocr', neg_rect, page_rect), +] + + +@pytest.mark.parametrize( + 'renderer, output_type, in_pdf, mode, crop_to, crop_expected', mediabox_testdata +) +def test_media_box( + resources, outdir, renderer, output_type, in_pdf, mode, crop_to, crop_expected +): + with pikepdf.open(resources / in_pdf) as pdf: + page = pdf.pages[0] + page.MediaBox = crop_to + pdf.save(outdir / 'cropped.pdf') + args = [ + '--jobs', + '1', + '--pdf-renderer', + renderer, + '--output-type', + output_type, + ] + if mode: + args.append(mode) + + check_ocrmypdf(outdir / 'cropped.pdf', outdir / 'processed.pdf', *args) + + with pikepdf.open(outdir / 'processed.pdf') as pdf: + page = pdf.pages[0] + assert page.MediaBox == crop_expected + + +cropbox_testdata = [ + ('hocr', 'pdfa', 'ccitt.pdf', None, inset_rect, inset_rect), + ('sandwich', 'pdfa', 'ccitt.pdf', None, inset_rect, inset_rect), + ('hocr', 'pdf', 'ccitt.pdf', None, inset_rect, inset_rect), + ('sandwich', 'pdf', 'ccitt.pdf', None, inset_rect, inset_rect), + ( + 'hocr', + 'pdfa', + 'ccitt.pdf', + '--force-ocr', + inset_rect, + inset_rect, + ), + ( + 'hocr', + 'pdf', + 'ccitt.pdf', + '--force-ocr', + inset_rect, + inset_rect, + ), +] + + +@pytest.mark.parametrize( + 'renderer, output_type, in_pdf, mode, crop_to, crop_expected', cropbox_testdata +) +def test_crop_box( + resources, outdir, renderer, output_type, in_pdf, mode, crop_to, crop_expected +): + with pikepdf.open(resources / in_pdf) as pdf: + page = pdf.pages[0] + page.CropBox = crop_to + pdf.save(outdir / 'cropped.pdf') + pdf.save('cropped.pdf') + args = [ + '--jobs', + '1', + '--pdf-renderer', + renderer, + '--output-type', + output_type, + ] + if mode: + args.append(mode) + + check_ocrmypdf(outdir / 'cropped.pdf', outdir / 'processed.pdf', *args) + + with pikepdf.open(outdir / 'processed.pdf') as pdf: + page = pdf.pages[0] + pdf.save('processed.pdf') + assert page.CropBox == crop_expected From 4c1ef0b47195fd8364b17040a53055a80954d05a Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Wed, 12 Feb 2025 12:49:26 -0800 Subject: [PATCH 06/14] Also process art and bleed boxes --- src/ocrmypdf/_pipeline.py | 41 +++++++++++++++++------ src/ocrmypdf/pdfinfo/info.py | 14 ++++++-- tests/test_page_boxes.py | 4 +-- tests/test_rotation.py | 65 +++++++++++++++++++++++++++++++----- 4 files changed, 101 insertions(+), 23 deletions(-) diff --git a/src/ocrmypdf/_pipeline.py b/src/ocrmypdf/_pipeline.py index 3eab75f9..89eb8382 100644 --- a/src/ocrmypdf/_pipeline.py +++ b/src/ocrmypdf/_pipeline.py @@ -39,6 +39,7 @@ from ocrmypdf.hocrtransform import DebugRenderOptions, HocrTransform from ocrmypdf.hocrtransform._font import Courier from ocrmypdf.pdfa import generate_pdfa_ps from ocrmypdf.pdfinfo import Colorspace, Encoding, PageInfo, PdfInfo +from ocrmypdf.pdfinfo.info import FloatRect from ocrmypdf.pluginspec import OrientationConfidence try: @@ -832,6 +833,23 @@ def _offset_rect(rect: tuple[float, float, float, float], offset: tuple[float, f ) +def _adjust_pagebox( + page: pikepdf.Page, + media_box: FloatRect, + name: pikepdf.Name, + target_box: FloatRect, + offset: tuple[float, float], + swap_axis: bool, +): + if media_box == target_box: + return + box = _offset_rect(target_box, offset) + if swap_axis: + box = box[1], box[0], box[3], box[2] + page[name] = box + log.debug(f"{str(name)} = {target_box}") + + def fix_pagepdf_boxes( infile: Path | BinaryIO, out_file: Path, @@ -856,22 +874,25 @@ def fix_pagepdf_boxes( """ with pikepdf.open(infile) as pdf: for page in pdf.pages: - # page.BleedBox = page_context.pageinfo.bleedbox - # page.ArtBox = page_context.pageinfo.artbox log.debug( - f"initial mediabox={page.MediaBox} and pageinfo mediabox={page_context.pageinfo.mediabox}" + f"initial mediabox={page.MediaBox} and pageinfo " + f"mediabox={page_context.pageinfo.mediabox}" ) mediabox = page_context.pageinfo.mediabox offset = -mediabox[0], -mediabox[1] - cropbox = _offset_rect(page_context.pageinfo.cropbox, offset) - trimbox = _offset_rect(page_context.pageinfo.trimbox, offset) if swap_axis: - cropbox = cropbox[1], cropbox[0], cropbox[3], cropbox[2] - trimbox = trimbox[1], trimbox[0], trimbox[3], trimbox[2] mediabox = mediabox[1], mediabox[0], mediabox[3], mediabox[2] - page.CropBox = cropbox - page.TrimBox = trimbox - log.debug(f"cropbox={cropbox}, trimbox={trimbox}, mediabox={mediabox}") + boxes = ['CropBox', 'TrimBox', 'ArtBox', 'BleedBox'] + for box_name in boxes: + _adjust_pagebox( + page, + mediabox, + pikepdf.Name(f"/{box_name}"), + getattr(page_context.pageinfo, box_name.lower()), + offset, + swap_axis, + ) + pdf.save(out_file) return out_file diff --git a/src/ocrmypdf/pdfinfo/info.py b/src/ocrmypdf/pdfinfo/info.py index 88d20c7f..31a77a22 100644 --- a/src/ocrmypdf/pdfinfo/info.py +++ b/src/ocrmypdf/pdfinfo/info.py @@ -901,8 +901,8 @@ class PageInfo: width_pt = mediabox[2] - mediabox[0] height_pt = mediabox[3] - mediabox[1] - # self._artbox = [float(d) for d in page.artbox.as_list()] - # self._bleedbox = [float(d) for d in page.bleedbox.as_list()] + self._artbox = [float(d) for d in page.artbox.as_list()] + self._bleedbox = [float(d) for d in page.bleedbox.as_list()] self._cropbox = [float(d) for d in page.cropbox.as_list()] self._mediabox = [float(d) for d in page.mediabox.as_list()] self._trimbox = [float(d) for d in page.trimbox.as_list()] @@ -1039,6 +1039,16 @@ class PageInfo: """Return trimbox of page in PDF coordinates.""" return self._trimbox + @property + def artbox(self) -> FloatRect: + """Return artbox of page in PDF coordinates.""" + return self._artbox + + @property + def bleedbox(self) -> FloatRect: + """Return bleedbox of page in PDF coordinates.""" + return self._bleedbox + @property def images(self) -> list[ImageInfo]: """Return images.""" diff --git a/tests/test_page_boxes.py b/tests/test_page_boxes.py index cc3386b1..9251a04d 100644 --- a/tests/test_page_boxes.py +++ b/tests/test_page_boxes.py @@ -102,7 +102,6 @@ def test_crop_box( page = pdf.pages[0] page.CropBox = crop_to pdf.save(outdir / 'cropped.pdf') - pdf.save('cropped.pdf') args = [ '--jobs', '1', @@ -110,6 +109,8 @@ def test_crop_box( renderer, '--output-type', output_type, + '--optimize', + '0', ] if mode: args.append(mode) @@ -118,5 +119,4 @@ def test_crop_box( with pikepdf.open(outdir / 'processed.pdf') as pdf: page = pdf.pages[0] - pdf.save('processed.pdf') assert page.CropBox == crop_expected diff --git a/tests/test_rotation.py b/tests/test_rotation.py index 23e0697b..bbd9afa4 100644 --- a/tests/test_rotation.py +++ b/tests/test_rotation.py @@ -51,20 +51,27 @@ def compare_images_monochrome( with Image.open(reference_png) as reference_im, Image.open(test_png) as test_im: assert reference_im.mode == test_im.mode == '1' - difference = ImageChops.logical_xor(reference_im, test_im) - assert difference.mode == '1' + # Pillow uses black is 0 for '1'. Invert so that foreground is 1, then + # compare + inv_ref_im = ImageChops.invert(reference_im) + inv_test_im = ImageChops.invert(test_im) + foreground_match = ImageChops.logical_and(inv_ref_im, inv_test_im) + foreground_total = ImageChops.logical_or(inv_ref_im, inv_test_im) + assert foreground_match.mode == '1' - histogram = difference.histogram() + histogram = foreground_match.histogram() + histogram_total = foreground_total.histogram() assert ( len(histogram) == 256 ), "Expected Pillow to convert to grayscale for histogram" # All entries other than first and last will be 0 - count_same = histogram[0] - count_different = histogram[-1] - total = count_same + count_different - - return count_same / (total) + # count_same = histogram[0] + # count_different = histogram[-1] + # total = count_same + count_different + # print(f"{count_same / (total)}") + # return count_same / (total) + return histogram[-1] / (histogram_total[-1] + 1) def test_monochrome_comparison(resources, outdir): @@ -211,7 +218,7 @@ def test_rotate_deskew_ocr_timeout(resources, outdir): assert cmp > 0.95 -def make_rotate_test(imagefile, outdir, prefix, image_angle, page_angle): +def make_rotate_test(imagefile, outdir, prefix, image_angle, page_angle, cropbox=None): memimg = BytesIO() with Image.open(fspath(imagefile)) as im: if image_angle != 0: @@ -230,6 +237,8 @@ def make_rotate_test(imagefile, outdir, prefix, image_angle, page_angle): with pikepdf.open(mempdf) as pdf: pdf.pages[0].Rotate = page_angle target = outdir / f'{prefix}_{image_angle}_{page_angle}.pdf' + if cropbox: + pdf.pages[0].CropBox = cropbox pdf.save(target) return target @@ -284,6 +293,44 @@ def test_page_rotate_tag(page_rotate_angle, resources, outdir, caplog): assert 'is a' in test_text, test_text +@pytest.mark.parametrize('page_rotate_angle', (0, 90, 180, 270)) +@pytest.mark.parametrize('renderer', ['sandwich', 'hocr']) +@pytest.mark.parametrize('output_type', ['pdf', 'pdfa']) +def test_rotate_and_crop( + resources, outdir, page_rotate_angle, renderer, output_type, caplog +): + cropbox = (100, 200, 1000, 800) + reference = make_rotate_test( + resources / 'typewriter.png', outdir, 'ref', 0, 0, cropbox + ) + test = make_rotate_test( + resources / 'typewriter.png', + outdir, + 'test', + -page_rotate_angle, + page_rotate_angle, + cropbox, + ) + out = test.with_suffix('.out.pdf') + + exitcode = run_ocrmypdf_api( + test, + out, + '-O0', + '--rotate-pages', + '--rotate-pages-threshold', + '0', + '--pdf-renderer', + renderer, + '--output-type', + output_type, + '--no-progress-bar', + ) + assert exitcode == 0, caplog.text + + assert compare_images_monochrome(outdir, reference, 1, out, 1) > 0.9 + + def test_rasterize_rotates(resources, tmp_path): pm = get_plugin_manager([]) From 742a4bac17972bafd5db5861cea271f5e24791fe Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Sun, 21 Dec 2025 14:42:14 -0800 Subject: [PATCH 07/14] Make rotation test more robust --- tests/test_rotation.py | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/tests/test_rotation.py b/tests/test_rotation.py index bbd9afa4..fa769125 100644 --- a/tests/test_rotation.py +++ b/tests/test_rotation.py @@ -51,27 +51,20 @@ def compare_images_monochrome( with Image.open(reference_png) as reference_im, Image.open(test_png) as test_im: assert reference_im.mode == test_im.mode == '1' - # Pillow uses black is 0 for '1'. Invert so that foreground is 1, then - # compare - inv_ref_im = ImageChops.invert(reference_im) - inv_test_im = ImageChops.invert(test_im) - foreground_match = ImageChops.logical_and(inv_ref_im, inv_test_im) - foreground_total = ImageChops.logical_or(inv_ref_im, inv_test_im) - assert foreground_match.mode == '1' + assert reference_im.size == test_im.size, "Images must be the same size" - histogram = foreground_match.histogram() - histogram_total = foreground_total.histogram() - assert ( - len(histogram) == 256 - ), "Expected Pillow to convert to grayscale for histogram" + # XOR the images: matching pixels become 0, different pixels become 1 + difference = ImageChops.logical_xor(reference_im, test_im) - # All entries other than first and last will be 0 - # count_same = histogram[0] - # count_different = histogram[-1] - # total = count_same + count_different - # print(f"{count_same / (total)}") - # return count_same / (total) - return histogram[-1] / (histogram_total[-1] + 1) + # Count matching pixels directly using getcolors() + # For a binary image, getcolors returns [(count, 0), (count, 1)] or subset + colors = difference.getcolors() + color_counts = {color: count for count, color in colors} + count_same = color_counts.get(0, 0) # 0 = matching pixels (XOR result is 0) + count_different = color_counts.get(255, 0) # 255 = different pixels + total = count_same + count_different + + return count_same / total def test_monochrome_comparison(resources, outdir): From e613db6a825a35df4b934386c6a6865ba6180f09 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Tue, 23 Dec 2025 14:56:24 -0800 Subject: [PATCH 08/14] Fix Ghostscript 10.6 JPEG corruption by repairing truncated images Ghostscript 10.6 has a bug that truncates JPEG data by 1-15 bytes. This adds detection and repair by comparing output images to input images and restoring the original bytes when truncation is detected. - Add warning when GS 10.6+ is used with PDF/A output - Add _repair_gs106_jpeg_corruption() to fix damaged JPEGs after Ghostscript processing - Add unit tests for the repair function --- src/ocrmypdf/builtin_plugins/ghostscript.py | 154 ++++++++++++++++++++ tests/test_ghostscript.py | 118 +++++++++++++++ 2 files changed, 272 insertions(+) diff --git a/src/ocrmypdf/builtin_plugins/ghostscript.py b/src/ocrmypdf/builtin_plugins/ghostscript.py index 8d2f3ec6..0934e850 100644 --- a/src/ocrmypdf/builtin_plugins/ghostscript.py +++ b/src/ocrmypdf/builtin_plugins/ghostscript.py @@ -5,8 +5,10 @@ from __future__ import annotations import logging +from pathlib import Path from packaging.version import Version +from pikepdf import Name, Pdf, Stream from ocrmypdf import hookimpl from ocrmypdf._exec import ghostscript @@ -74,6 +76,13 @@ def check_options(options): "use --force-ocr to discard existing text." ) + if gs_version >= Version('10.6.0') and options.output_type.startswith('pdfa'): + log.warning( + "Ghostscript 10.6.x contains JPEG encoding errors that may corrupt " + "images. OCRmyPDF will attempt to mitigate, but this version is " + "strongly not recommended. Please upgrade to a newer version. " + "As of 2025-12, 10.6.0 is the latest version of Ghostscript." + ) if options.output_type == 'pdfa': options.output_type = 'pdfa-2' if options.color_conversion_strategy not in ghostscript.COLOR_CONVERSION_STRATEGIES: @@ -116,6 +125,144 @@ def rasterize_pdf_page( return output_file +def _collect_dctdecode_images(pdf: Pdf) -> dict[tuple, list[tuple[Stream, bytes]]]: + """Collect all DCTDecode (JPEG) images from a PDF. + + Returns a dict mapping image signatures to a list of (stream, raw_bytes) tuples. + The signature is (Width, Height, Filter, BitsPerComponent, ColorSpace). + """ + images: dict[tuple, list[tuple[Stream, bytes]]] = {} + + def get_colorspace_key(obj): + """Get a hashable key for the colorspace.""" + cs = obj.get(Name.ColorSpace) + if cs is None: + return None + if isinstance(cs, Name): + return str(cs) + # For array colorspaces like [/ICCBased ...], use the first element + try: + return str(cs[0]) if len(cs) > 0 else str(cs) + except (TypeError, KeyError): + return str(cs) + + def process_xobject_dict(xobjects, depth=0): + """Process an XObject dictionary for DCTDecode images.""" + if xobjects is None: + return + if depth > 10: + log.warning("Recursion depth exceeded in _collect_dctdecode_images") + return + for key in xobjects.keys(): + obj = xobjects[key] + if obj is None: + continue + # Check if it's an image with DCTDecode + if obj.get(Name.Subtype) == Name.Image: + filt = obj.get(Name.Filter) + if filt == Name.DCTDecode: + sig = ( + int(obj.get(Name.Width, 0)), + int(obj.get(Name.Height, 0)), + str(filt), + int(obj.get(Name.BitsPerComponent, 0)), + get_colorspace_key(obj), + ) + raw_bytes = obj.read_raw_bytes() + if sig not in images: + images[sig] = [] + images[sig].append((obj, raw_bytes)) + # Recurse into Form XObjects + elif obj.get(Name.Subtype) == Name.Form: + if Name.Resources in obj: + res = obj[Name.Resources] + if Name.XObject in res: + process_xobject_dict(res[Name.XObject], depth=depth + 1) + + for page in pdf.pages: + if Name.Resources not in page: + continue + resources = page[Name.Resources] + if Name.XObject not in resources: + continue + process_xobject_dict(resources[Name.XObject]) + + return images + + +def _repair_gs106_jpeg_corruption( + input_pdf_path: Path, + output_pdf_path: Path, +) -> bool: + """Repair JPEG corruption caused by Ghostscript 10.6. + + Ghostscript 10.6 has a bug that truncates JPEG data by 1-15 bytes. + This function detects and repairs such corruption by copying the + original JPEG bytes from the input PDF. + + Returns True if any repairs were made. + """ + repaired_count = 0 + first_error_logged = False + + with ( + Pdf.open(input_pdf_path) as input_pdf, + Pdf.open(output_pdf_path, allow_overwriting_input=True) as output_pdf, + ): + # Collect all DCTDecode images from both PDFs + input_images = _collect_dctdecode_images(input_pdf) + output_images = _collect_dctdecode_images(output_pdf) + + # For each output image, try to find a corresponding input image + for sig, output_list in output_images.items(): + if sig not in input_images: + continue + input_list = input_images[sig] + + for output_stream, output_bytes in output_list: + # Try to find a matching input image + for _input_stream, input_bytes in input_list: + input_len = len(input_bytes) + output_len = len(output_bytes) + + # Check if output is 1-15 bytes shorter + diff = input_len - output_len + if not (1 <= diff <= 15): + continue + + # Check if the bytes are identical up to the truncation point + if output_bytes != input_bytes[:output_len]: + continue + + # This is a corrupt image - repair it + if not first_error_logged: + log.error( + "Ghostscript 10.6 JPEG corruption detected. " + "Repairing damaged images from original PDF." + ) + first_error_logged = True + log.warning( + f"Replacing corrupt JPEG image " + f"({sig[0]}x{sig[1]}, {diff} bytes truncated)" + ) + + # Write the original bytes back to the output stream + output_stream.write( + input_bytes, + filter=Name.DCTDecode, + ) + repaired_count += 1 + break # Move to next output image + + if repaired_count > 0: + output_pdf.save(output_pdf_path) + log.info( + f"Repaired {repaired_count} JPEG image(s) corrupted by Ghostscript" + ) + + return repaired_count > 0 + + @hookimpl def generate_pdfa( pdf_pages, @@ -138,4 +285,11 @@ def generate_pdfa( progressbar_class=progressbar_class, stop_on_error=stop_on_soft_error, ) + + # Repair JPEG corruption caused by Ghostscript 10.6.x + gs_version = ghostscript.version() + if gs_version >= Version('10.6.0') and len(pdf_pages) == 1: + input_pdf = Path(pdf_pages[0]) + _repair_gs106_jpeg_corruption(input_pdf, Path(output_file)) + return output_file diff --git a/tests/test_ghostscript.py b/tests/test_ghostscript.py index 01462004..79250d3b 100644 --- a/tests/test_ghostscript.py +++ b/tests/test_ghostscript.py @@ -17,6 +17,7 @@ from PIL import Image, UnidentifiedImageError from ocrmypdf._exec import ghostscript from ocrmypdf._exec.ghostscript import DuplicateFilter, rasterize_pdf +from ocrmypdf.builtin_plugins.ghostscript import _repair_gs106_jpeg_corruption from ocrmypdf.exceptions import ColorConversionNeededError, ExitCode, InputFileError from ocrmypdf.helpers import Resolution @@ -287,3 +288,120 @@ def test_recoverable_image_error_with_stop(pdf_with_invalid_image, outdir, caplo stop_on_error=True, ) # out2.png will not be created; if it were it would be blank. + + +class TestGs106JpegCorruptionRepair: + """Test the Ghostscript 10.6 JPEG corruption repair function.""" + + @pytest.fixture + def create_damaged_pdf(self, resources, outdir): + """Create a damaged PDF by truncating JPEG data by 2 bytes.""" + + def _create_damaged(source_pdf_name='francais.pdf', truncate_bytes=2): + source_path = resources / source_pdf_name + damaged_path = outdir / 'damaged.pdf' + + with pikepdf.open(source_path) as pdf: + # Find and truncate DCTDecode images + Name = pikepdf.Name + damaged_count = 0 + for page in pdf.pages: + if Name.Resources not in page: + continue + resources_dict = page[Name.Resources] + if Name.XObject not in resources_dict: + continue + for key in resources_dict[Name.XObject].keys(): + obj = resources_dict[Name.XObject][key] + if obj.get(Name.Subtype) != Name.Image: + continue + if obj.get(Name.Filter) != Name.DCTDecode: + continue + # Truncate the JPEG data + original_bytes = obj.read_raw_bytes() + truncated_bytes = original_bytes[:-truncate_bytes] + obj.write(truncated_bytes, filter=Name.DCTDecode) + damaged_count += 1 + + pdf.save(damaged_path) + return source_path, damaged_path, damaged_count + + return _create_damaged + + def test_repair_truncated_jpeg(self, create_damaged_pdf, caplog): + """Test that truncated JPEG images are repaired.""" + caplog.set_level(logging.DEBUG) + source_path, damaged_path, damaged_count = create_damaged_pdf() + + assert damaged_count > 0, "Test PDF should have DCTDecode images" + + # Get original image bytes for comparison + with pikepdf.open(source_path) as pdf: + Name = pikepdf.Name + original_bytes_list = [] + for page in pdf.pages: + if Name.Resources not in page: + continue + resources_dict = page[Name.Resources] + if Name.XObject not in resources_dict: + continue + for key in resources_dict[Name.XObject].keys(): + obj = resources_dict[Name.XObject][key] + if obj.get(Name.Subtype) != Name.Image: + continue + if obj.get(Name.Filter) != Name.DCTDecode: + continue + original_bytes_list.append(obj.read_raw_bytes()) + + # Run the repair function + repaired = _repair_gs106_jpeg_corruption(source_path, damaged_path) + assert repaired is True, "Repair should have been performed" + + # Verify the repaired PDF has correct image bytes + with pikepdf.open(damaged_path) as pdf: + Name = pikepdf.Name + repaired_bytes_list = [] + for page in pdf.pages: + if Name.Resources not in page: + continue + resources_dict = page[Name.Resources] + if Name.XObject not in resources_dict: + continue + for key in resources_dict[Name.XObject].keys(): + obj = resources_dict[Name.XObject][key] + if obj.get(Name.Subtype) != Name.Image: + continue + if obj.get(Name.Filter) != Name.DCTDecode: + continue + repaired_bytes_list.append(obj.read_raw_bytes()) + + assert len(repaired_bytes_list) == len(original_bytes_list) + for orig, repaired_bytes in zip(original_bytes_list, repaired_bytes_list): + assert orig == repaired_bytes, "Repaired bytes should match original" + + # Check that error/warning was logged + assert "JPEG corruption detected" in caplog.text + + def test_no_repair_when_not_truncated(self, resources, outdir, caplog): + """Test that no repair is done when images are not truncated.""" + caplog.set_level(logging.DEBUG) + source_path = resources / 'francais.pdf' + + # Copy source to output (no damage) + output_path = outdir / 'undamaged.pdf' + with pikepdf.open(source_path) as pdf: + pdf.save(output_path) + + # Run the repair function - should not repair anything + repaired = _repair_gs106_jpeg_corruption(source_path, output_path) + assert repaired is False, "No repair should have been performed" + assert "JPEG corruption detected" not in caplog.text + + def test_no_repair_when_truncation_too_large(self, create_damaged_pdf, caplog): + """Test that images truncated by more than 15 bytes are not repaired.""" + caplog.set_level(logging.DEBUG) + source_path, damaged_path, _ = create_damaged_pdf(truncate_bytes=20) + + repaired = _repair_gs106_jpeg_corruption(source_path, damaged_path) + assert repaired is False, "Should not repair truncation > 15 bytes" + assert "JPEG corruption detected" not in caplog.text From 01a3706281706ff6925b0d52228b7db54b8b43f8 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Tue, 23 Dec 2025 15:01:22 -0800 Subject: [PATCH 09/14] docs: Add release notes for v16.13.0 --- docs/release_notes.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/release_notes.md b/docs/release_notes.md index b097c99f..f88c53fb 100644 --- a/docs/release_notes.md +++ b/docs/release_notes.md @@ -25,6 +25,15 @@ about a forthcoming release that has not been tagged yet. A release is only official when it's tagged and posted to PyPI. ::: +## v16.13.0 + +- Added detection and repair for Ghostscript 10.6 JPEG corruption. When GS 10.6 + truncates JPEG data by 1-15 bytes, OCRmyPDF now restores the original image + bytes from the input PDF. A warning is issued when GS 10.6+ is detected. +- Fixed handling of PDF page boxes (ArtBox, BleedBox) which were not being + processed correctly in some cases. +- Documentation: clarified podman usage instructions. + ## v16.12.0 - Disable Ghostscript's subset fonts feature, which was found to corrupt text in From fc30cb8903c9c91ea64e0d0ee9302dc8ebb8c178 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Tue, 23 Dec 2025 15:03:51 -0800 Subject: [PATCH 10/14] Revert "Work around Ghostscript 10.6.0 JPEG encoding issue by forcing optimization." This reverts commit f4c6c8121ba8178ff3a1cb8f70037bbc3a31391b. The issue is now resolved by correcting the encoidng issue directly. --- src/ocrmypdf/optimize.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/ocrmypdf/optimize.py b/src/ocrmypdf/optimize.py index 41a1a385..a7c6cded 100644 --- a/src/ocrmypdf/optimize.py +++ b/src/ocrmypdf/optimize.py @@ -17,7 +17,6 @@ from typing import Any, NamedTuple, NewType from zlib import compress import img2pdf -from packaging.version import Version from pikepdf import ( Dictionary, Name, @@ -33,7 +32,7 @@ from pikepdf.models.image import HifiPrintImageNotTranscodableError from PIL import Image from ocrmypdf._concurrent import Executor, SerialExecutor -from ocrmypdf._exec import ghostscript, jbig2enc, pngquant +from ocrmypdf._exec import jbig2enc, pngquant from ocrmypdf._jobcontext import PdfContext from ocrmypdf._progressbar import ProgressBar from ocrmypdf.exceptions import OutputFileAccessError @@ -191,16 +190,6 @@ def extract_image_jbig2( return None -def _should_optimize_jpeg(options, filtdp): - if options.optimize >= 2: - return True - if options.optimize < 2 and ghostscript.version() >= Version('10.6.0'): - # Ghostscript 10.6.0+ introduced some sort of JPEG encoding issue. - # To resolve this, re-optimize the JPEG anyway. - return True - return False - - def extract_image_generic( *, pdf: Pdf, root: Path, image: Stream, xref: Xref, options ) -> XrefExt | None: @@ -214,7 +203,15 @@ def extract_image_generic( if pim.bits_per_component == 1: return None - if filtdp[0] == Name.DCTDecode and _should_optimize_jpeg(options, filtdp): + if filtdp[0] == Name.DCTDecode and options.optimize >= 2: + # This is a simple heuristic derived from some training data, that has + # about a 70% chance of guessing whether the JPEG is high quality, + # and possibly recompressible, or not. The number itself doesn't mean + # anything. + # bytes_per_pixel = int(raw_jpeg.Length) / (w * h) + # jpeg_quality_estimate = 117.0 * (bytes_per_pixel ** 0.213) + # if jpeg_quality_estimate < 65: + # return None try: imgname = root / f'{xref:08d}' with imgname.open('wb') as f: From 6ada11ddae2526fa858d8022c19395d2319efb8e Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Tue, 23 Dec 2025 15:05:49 -0800 Subject: [PATCH 11/14] docs: Update release notes --- docs/release_notes.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/release_notes.md b/docs/release_notes.md index f88c53fb..42efd29e 100644 --- a/docs/release_notes.md +++ b/docs/release_notes.md @@ -30,6 +30,8 @@ official when it's tagged and posted to PyPI. - Added detection and repair for Ghostscript 10.6 JPEG corruption. When GS 10.6 truncates JPEG data by 1-15 bytes, OCRmyPDF now restores the original image bytes from the input PDF. A warning is issued when GS 10.6+ is detected. + {issue}`1603` +- Reverted the force re-optimization of JPEGs, since this stop-gap was not sufficient to resolve the issue. {issue}`1585` - Fixed handling of PDF page boxes (ArtBox, BleedBox) which were not being processed correctly in some cases. - Documentation: clarified podman usage instructions. From 195344d307541705ee397cb0460138eb793cda9d Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Tue, 23 Dec 2025 15:41:27 -0800 Subject: [PATCH 12/14] Reinstate "Work around Ghostscript 10.6.0 JPEG encoding issue by forcing optimization."" This reverts commit fc30cb8903c9c91ea64e0d0ee9302dc8ebb8c178. It turns out that both fixes were necessary. --- src/ocrmypdf/optimize.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/ocrmypdf/optimize.py b/src/ocrmypdf/optimize.py index a7c6cded..41a1a385 100644 --- a/src/ocrmypdf/optimize.py +++ b/src/ocrmypdf/optimize.py @@ -17,6 +17,7 @@ from typing import Any, NamedTuple, NewType from zlib import compress import img2pdf +from packaging.version import Version from pikepdf import ( Dictionary, Name, @@ -32,7 +33,7 @@ from pikepdf.models.image import HifiPrintImageNotTranscodableError from PIL import Image from ocrmypdf._concurrent import Executor, SerialExecutor -from ocrmypdf._exec import jbig2enc, pngquant +from ocrmypdf._exec import ghostscript, jbig2enc, pngquant from ocrmypdf._jobcontext import PdfContext from ocrmypdf._progressbar import ProgressBar from ocrmypdf.exceptions import OutputFileAccessError @@ -190,6 +191,16 @@ def extract_image_jbig2( return None +def _should_optimize_jpeg(options, filtdp): + if options.optimize >= 2: + return True + if options.optimize < 2 and ghostscript.version() >= Version('10.6.0'): + # Ghostscript 10.6.0+ introduced some sort of JPEG encoding issue. + # To resolve this, re-optimize the JPEG anyway. + return True + return False + + def extract_image_generic( *, pdf: Pdf, root: Path, image: Stream, xref: Xref, options ) -> XrefExt | None: @@ -203,15 +214,7 @@ def extract_image_generic( if pim.bits_per_component == 1: return None - if filtdp[0] == Name.DCTDecode and options.optimize >= 2: - # This is a simple heuristic derived from some training data, that has - # about a 70% chance of guessing whether the JPEG is high quality, - # and possibly recompressible, or not. The number itself doesn't mean - # anything. - # bytes_per_pixel = int(raw_jpeg.Length) / (w * h) - # jpeg_quality_estimate = 117.0 * (bytes_per_pixel ** 0.213) - # if jpeg_quality_estimate < 65: - # return None + if filtdp[0] == Name.DCTDecode and _should_optimize_jpeg(options, filtdp): try: imgname = root / f'{xref:08d}' with imgname.open('wb') as f: From c540967429bcf97303593f793e3f9791111ac7be Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Tue, 23 Dec 2025 15:44:44 -0800 Subject: [PATCH 13/14] docs: Update release notes --- docs/release_notes.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/release_notes.md b/docs/release_notes.md index 42efd29e..f21cef3d 100644 --- a/docs/release_notes.md +++ b/docs/release_notes.md @@ -31,15 +31,15 @@ official when it's tagged and posted to PyPI. truncates JPEG data by 1-15 bytes, OCRmyPDF now restores the original image bytes from the input PDF. A warning is issued when GS 10.6+ is detected. {issue}`1603` -- Reverted the force re-optimization of JPEGs, since this stop-gap was not sufficient to resolve the issue. {issue}`1585` +- We continue to force re-optimization of JPEGs, since this catches some issues with corruption for situations where Ghostscript modifies an image. It is likely there are still cases where we cannot mitigate all corruption issues. {issue}`1585` - Fixed handling of PDF page boxes (ArtBox, BleedBox) which were not being processed correctly in some cases. - Documentation: clarified podman usage instructions. ## v16.12.0 -- Disable Ghostscript's subset fonts feature, which was found to corrupt text in - certain PDFs. Thanks @mnaegler for identifying this issue. {issue}`1592` +- Disable Ghostscript's subset fonts feature, which was found to corrupt text in certain + PDFs. Thanks @mnaegler for identifying this issue. {issue}`1592` - Users of Ghostscript 10.6.0+ reported that Ghostscript seems to generate corrupted JPEGs. We force re-optimization of these JPEGs to mitigate the corruption until Ghostscript fixes the issue. {issue}`1585` From 94d77358621aea67f5c97c068fa6054990f0eda2 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Wed, 24 Dec 2025 00:14:24 -0800 Subject: [PATCH 14/14] docs: missing issue ref --- docs/release_notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/release_notes.md b/docs/release_notes.md index f21cef3d..5539af6f 100644 --- a/docs/release_notes.md +++ b/docs/release_notes.md @@ -33,7 +33,7 @@ official when it's tagged and posted to PyPI. {issue}`1603` - We continue to force re-optimization of JPEGs, since this catches some issues with corruption for situations where Ghostscript modifies an image. It is likely there are still cases where we cannot mitigate all corruption issues. {issue}`1585` - Fixed handling of PDF page boxes (ArtBox, BleedBox) which were not being - processed correctly in some cases. + processed correctly in some cases. {issue}`1181,1360` - Documentation: clarified podman usage instructions. ## v16.12.0