From 11f53fe9a96daf12938495521914fa2215a9bc20 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Tue, 31 Oct 2023 00:12:15 -0700 Subject: [PATCH 1/4] First cut at propagating page boxes This would fix the immediate issue, but does not address an offset mediabox. --- docs/api.rst | 4 +++- src/ocrmypdf/_pipeline.py | 42 ++++++++++++++++++++++++++++++---- src/ocrmypdf/_pipelines/ocr.py | 3 +++ src/ocrmypdf/pdfinfo/info.py | 15 ++++++++++++ 4 files changed, 59 insertions(+), 5 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 4bb4e6db..abdbfda7 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -112,7 +112,9 @@ OCRmyPDF is strict about not writing to standard output so that users can safely use it in a pipeline and produce a valid output file. A caller application will have to ensure it does not write to standard output either, if it wants to be compatible with this -behavior and support piping to a file. +behavior and support piping to a file. Another benefit of running +OCRmyPDF in a child process, as recommended above, is that it will +not interfere with the parent process's standard output. Exceptions ---------- diff --git a/src/ocrmypdf/_pipeline.py b/src/ocrmypdf/_pipeline.py index cd67eafa..bc58bd07 100644 --- a/src/ocrmypdf/_pipeline.py +++ b/src/ocrmypdf/_pipeline.py @@ -12,6 +12,7 @@ import re import sys from collections.abc import Iterable, Iterator, Sequence from contextlib import suppress +from io import BytesIO from pathlib import Path from shutil import copyfileobj, copystat from typing import Any, BinaryIO, TypeVar, cast @@ -713,19 +714,30 @@ def create_pdf_page_from_image( pageinfo = page_context.pageinfo pagesize = 72.0 * float(pageinfo.width_inches), 72.0 * float(pageinfo.height_inches) effective_rotation = (pageinfo.rotation - orientation_correction) % 360 - if effective_rotation % 180 == 90: + swap_axis = effective_rotation % 180 == 90 + if swap_axis: pagesize = pagesize[1], pagesize[0] + bio = BytesIO() + # This create a single page PDF - with open(image, 'rb') as imfile, open(output_file, 'wb') as pdf: + with open(image, 'rb') as imfile: log.debug('convert') layout_fun = img2pdf.get_layout_fun(pagesize) img2pdf.convert( - imfile, layout_fun=layout_fun, outputstream=pdf, **IMG2PDF_KWARGS + imfile, + layout_fun=layout_fun, + outputstream=bio, + engine=img2pdf.Engine.pikepdf, + rotation=img2pdf.Rotation.ifvalid, ) log.debug('convert done') + # img2pdf does not generate boxes correctly, so we fix them + bio.seek(0) + fix_pagepdf_boxes(bio, output_file, page_context, swap_axis=swap_axis) + output_file = page_context.plugin_manager.hook.filter_pdf_page( page=page_context, image_filename=image, output_pdf=output_file ) @@ -780,7 +792,29 @@ def ocr_engine_textonly_pdf( output_text=output_text, options=options, ) - return (output_pdf, output_text) + return output_pdf, output_text + + +def fix_pagepdf_boxes( + infile: Path | BinaryIO, + out_file: Path, + page_context: PageContext, + swap_axis: bool = False, +) -> Path: + """Fix the bounding boxes in a single page PDF.""" + with pikepdf.open(infile) as pdf: + for page in pdf.pages: + # page.BleedBox = page_context.pageinfo.bleedbox + # page.ArtBox = page_context.pageinfo.artbox + cropbox = page_context.pageinfo.cropbox + trimbox = page_context.pageinfo.trimbox + if swap_axis: + cropbox = cropbox[1], cropbox[0], cropbox[3], cropbox[2] + trimbox = trimbox[1], trimbox[0], trimbox[3], trimbox[2] + page.CropBox = cropbox + page.TrimBox = trimbox + pdf.save(out_file) + return pdf def generate_postscript_stub(context: PdfContext) -> Path: diff --git a/src/ocrmypdf/_pipelines/ocr.py b/src/ocrmypdf/_pipelines/ocr.py index 374445ed..6d143038 100644 --- a/src/ocrmypdf/_pipelines/ocr.py +++ b/src/ocrmypdf/_pipelines/ocr.py @@ -22,6 +22,7 @@ from ocrmypdf._graft import OcrGrafter from ocrmypdf._jobcontext import PageContext, PdfContext from ocrmypdf._pipeline import ( copy_final, + fix_pagepdf_boxes, get_pdfinfo, is_ocr_required, merge_sidecars, @@ -81,6 +82,8 @@ def _exec_page_sync(page_context: PageContext) -> PageResult: ) ocr_out, text_out = _image_to_ocr_text(page_context, ocr_image_out) + # fix_pagepdf_boxes(ocr_out, page_context) + return PageResult( pageno=page_context.pageno, pdf_page_from_image=pdf_page_from_image_out, diff --git a/src/ocrmypdf/pdfinfo/info.py b/src/ocrmypdf/pdfinfo/info.py index e6b6f19d..b3c45212 100644 --- a/src/ocrmypdf/pdfinfo/info.py +++ b/src/ocrmypdf/pdfinfo/info.py @@ -854,6 +854,11 @@ class PageInfo: width_pt = mediabox[2] - mediabox[0] height_pt = mediabox[3] - mediabox[1] + # self._artbox = [float(d) for d in page.artbox.as_list()] + # self._bleedbox = [float(d) for d in page.bleedbox.as_list()] + self._cropbox = [float(d) for d in page.cropbox.as_list()] + self._trimbox = [float(d) for d in page.trimbox.as_list()] + check_this_page = pageno in check_pages if check_this_page and detailed_analysis: @@ -970,6 +975,16 @@ class PageInfo: else: raise ValueError("rotation must be a cardinal angle") + @property + def cropbox(self) -> FloatRect: + """Return trimbox of page in PDF coordinates.""" + return self._cropbox + + @property + def trimbox(self) -> FloatRect: + """Return trimbox of page in PDF coordinates.""" + return self._trimbox + @property def images(self) -> list[ImageInfo]: """Return images.""" From 3b9d4b7f0aa20f704448a6222590f4f2cb85f689 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Tue, 31 Oct 2023 00:33:10 -0700 Subject: [PATCH 2/4] Attempt to deal with oddball mediaboxes --- src/ocrmypdf/_pipeline.py | 37 +++++++++++++++++++++++++++++----- src/ocrmypdf/_pipelines/ocr.py | 4 ---- src/ocrmypdf/helpers.py | 2 ++ src/ocrmypdf/pdfinfo/info.py | 8 +++++++- 4 files changed, 41 insertions(+), 10 deletions(-) diff --git a/src/ocrmypdf/_pipeline.py b/src/ocrmypdf/_pipeline.py index bc58bd07..879782e2 100644 --- a/src/ocrmypdf/_pipeline.py +++ b/src/ocrmypdf/_pipeline.py @@ -718,9 +718,8 @@ def create_pdf_page_from_image( if swap_axis: pagesize = pagesize[1], pagesize[0] + # Create a new single page PDF to hold bio = BytesIO() - - # This create a single page PDF with open(image, 'rb') as imfile: log.debug('convert') @@ -795,19 +794,47 @@ def ocr_engine_textonly_pdf( return output_pdf, output_text +def _offset_rect(rect: tuple[float, float, float, float], offset: tuple[float, float]): + """Offset a rectangle by a given amount.""" + return ( + rect[0] + offset[0], + rect[1] + offset[1], + rect[2] + offset[0], + rect[3] + offset[1], + ) + + def fix_pagepdf_boxes( infile: Path | BinaryIO, out_file: Path, page_context: PageContext, swap_axis: bool = False, ) -> Path: - """Fix the bounding boxes in a single page PDF.""" + """Fix the bounding boxes in a single page PDF. + + The single page PDF is created with a normal MediaBox with its lower left corner + at (0, 0). infile is the single page PDF. page_context.mediabox has the original + file's mediabox, which may have a different origin. We needto adjust the other + boxes in the single page PDF to match the effect they had on the original page. + + When correcting page rotation, we create a single page PDF that is correctly + rotated instead of an incorrectly rotated and then setting page.Rotate on it. + If rotation is either 90 or 270 degrees, then this function can be called + with swap_axis to swap the X and Y coordinates of all the boxes. + + We are not concerned with solving degenerate cases where the boxes overlap or + or express invalid rectangles. We merely pass the boxes, producing a + transformation equivalent to the change made by constructing a new page image. + """ with pikepdf.open(infile) as pdf: for page in pdf.pages: # page.BleedBox = page_context.pageinfo.bleedbox # page.ArtBox = page_context.pageinfo.artbox - cropbox = page_context.pageinfo.cropbox - trimbox = page_context.pageinfo.trimbox + mediabox = page_context.pageinfo.mediabox + offset = mediabox[0], mediabox[1] + cropbox = _offset_rect(page_context.pageinfo.cropbox, offset) + trimbox = _offset_rect(page_context.pageinfo.trimbox, offset) + if swap_axis: cropbox = cropbox[1], cropbox[0], cropbox[3], cropbox[2] trimbox = trimbox[1], trimbox[0], trimbox[3], trimbox[2] diff --git a/src/ocrmypdf/_pipelines/ocr.py b/src/ocrmypdf/_pipelines/ocr.py index 6d143038..52014870 100644 --- a/src/ocrmypdf/_pipelines/ocr.py +++ b/src/ocrmypdf/_pipelines/ocr.py @@ -22,7 +22,6 @@ from ocrmypdf._graft import OcrGrafter from ocrmypdf._jobcontext import PageContext, PdfContext from ocrmypdf._pipeline import ( copy_final, - fix_pagepdf_boxes, get_pdfinfo, is_ocr_required, merge_sidecars, @@ -81,9 +80,6 @@ def _exec_page_sync(page_context: PageContext) -> PageResult: page_context ) ocr_out, text_out = _image_to_ocr_text(page_context, ocr_image_out) - - # fix_pagepdf_boxes(ocr_out, page_context) - return PageResult( pageno=page_context.pageno, pdf_page_from_image=pdf_page_from_image_out, diff --git a/src/ocrmypdf/helpers.py b/src/ocrmypdf/helpers.py index 131ebcd7..c535da7c 100644 --- a/src/ocrmypdf/helpers.py +++ b/src/ocrmypdf/helpers.py @@ -26,6 +26,7 @@ from typing import ( import img2pdf import pikepdf +from deprecation import deprecated log = logging.getLogger(__name__) @@ -136,6 +137,7 @@ class Resolution(Generic[T]): return self._isclose(self.x, other.x) and self._isclose(self.y, other.y) +@deprecated(deprecated_in='15.4.0') class NeverRaise(Exception): """An exception that is never raised.""" diff --git a/src/ocrmypdf/pdfinfo/info.py b/src/ocrmypdf/pdfinfo/info.py index b3c45212..4357c13c 100644 --- a/src/ocrmypdf/pdfinfo/info.py +++ b/src/ocrmypdf/pdfinfo/info.py @@ -857,6 +857,7 @@ class PageInfo: # self._artbox = [float(d) for d in page.artbox.as_list()] # self._bleedbox = [float(d) for d in page.bleedbox.as_list()] self._cropbox = [float(d) for d in page.cropbox.as_list()] + self._mediabox = [float(d) for d in page.mediabox.as_list()] self._trimbox = [float(d) for d in page.trimbox.as_list()] check_this_page = pageno in check_pages @@ -977,9 +978,14 @@ class PageInfo: @property def cropbox(self) -> FloatRect: - """Return trimbox of page in PDF coordinates.""" + """Return cropbox of page in PDF coordinates.""" return self._cropbox + @property + def mediabox(self) -> FloatRect: + """Return mediabox of page in PDF coordinates.""" + return self._mediabox + @property def trimbox(self) -> FloatRect: """Return trimbox of page in PDF coordinates.""" From 26470fe16a257c665a260eb509db5edabd189288 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Mon, 12 Feb 2024 01:17:08 -0800 Subject: [PATCH 3/4] Suppress reportlab deprecation warning --- pyproject.toml | 5 ++++- tests/test_rotation.py | 6 +----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 36c2e3f2..19660401 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -131,7 +131,10 @@ norecursedirs = ["lib", ".pc", ".git", "venv", "output", "cache", "resources"] testpaths = ["tests"] addopts = "-n auto" markers = ["slow"] -filterwarnings = ["ignore:.*XMLParser.*:DeprecationWarning"] +filterwarnings = [ + "ignore:.*XMLParser.*:DeprecationWarning", + 'ignore:.*ast.NameConstant.*:DeprecationWarning', +] [tool.mypy] diff --git a/tests/test_rotation.py b/tests/test_rotation.py index cac7e103..47443c4c 100644 --- a/tests/test_rotation.py +++ b/tests/test_rotation.py @@ -4,10 +4,10 @@ from __future__ import annotations import operator -import warnings from io import BytesIO from math import cos, pi, sin from os import fspath +from subprocess import run import img2pdf import pikepdf @@ -22,10 +22,6 @@ from ocrmypdf.pdfinfo import PdfInfo from .conftest import check_ocrmypdf, run_ocrmypdf_api -warnings.filterwarnings( - "ignore", category=DeprecationWarning, module="reportlab.lib.rl_safe_eval" -) - # pylintx: disable=unused-variable RENDERERS = ['hocr', 'sandwich'] From 42ff7fc84289a885c86d4629ebcf8dd9a65cef2d Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Mon, 12 Feb 2024 01:32:26 -0800 Subject: [PATCH 4/4] Fix handling of pages that are restored to correct orientation with /Rotate Appears inversion of CTM was incorrect, introduced in commit 9898904 --- src/ocrmypdf/_graft.py | 5 ++- tests/test_optimize.py | 7 ++-- tests/test_rotation.py | 76 +++++++++++++++++++++++++++++------------- 3 files changed, 58 insertions(+), 30 deletions(-) diff --git a/src/ocrmypdf/_graft.py b/src/ocrmypdf/_graft.py index 481f029c..1848e884 100644 --- a/src/ocrmypdf/_graft.py +++ b/src/ocrmypdf/_graft.py @@ -289,9 +289,8 @@ class OcrGrafter: # Translate the text so it is centered at (0, 0), rotate it there, adjust # for a size different between initial and text PDF, then untranslate, and - # finally move the lower left corner to match the mediabox. All transforms - # must be premultiplied so they are applied in reverse order here. - ctm = corner @ untranslate @ scale @ rotate @ translate + # finally move the lower left corner to match the mediabox. + ctm = translate @ rotate @ scale @ untranslate @ corner log.debug("Grafting with ctm %r", ctm) base_resources = _ensure_dictionary(base_page.obj, Name.Resources) diff --git a/tests/test_optimize.py b/tests/test_optimize.py index 69cdfcc2..4ba136bf 100644 --- a/tests/test_optimize.py +++ b/tests/test_optimize.py @@ -176,9 +176,10 @@ def test_multiple_pngs(resources, outdir): ) mock.assert_called() - with pikepdf.open(outdir / 'in.pdf') as inpdf, pikepdf.open( - outdir / 'out.pdf' - ) as outpdf: + with ( + pikepdf.open(outdir / 'in.pdf') as inpdf, + pikepdf.open(outdir / 'out.pdf') as outpdf, + ): for n in range(len(inpdf.pages)): inim = next(iter(inpdf.pages[n].images.values())) outim = next(iter(outpdf.pages[n].images.values())) diff --git a/tests/test_rotation.py b/tests/test_rotation.py index 47443c4c..b5e4ffbd 100644 --- a/tests/test_rotation.py +++ b/tests/test_rotation.py @@ -211,34 +211,35 @@ def test_rotate_deskew_ocr_timeout(resources, outdir): assert cmp > 0.95 +def make_rotate_test(imagefile, outdir, prefix, image_angle, page_angle): + memimg = BytesIO() + with Image.open(fspath(imagefile)) as im: + if image_angle != 0: + ccw_angle = -image_angle % 360 + im = im.transpose(getattr(Image.Transpose, f'ROTATE_{ccw_angle}')) + im.save(memimg, format='PNG') + memimg.seek(0) + mempdf = BytesIO() + img2pdf.convert( + memimg.read(), + layout_fun=img2pdf.get_fixed_dpi_layout_fun((200, 200)), + outputstream=mempdf, + **IMG2PDF_KWARGS, + ) + mempdf.seek(0) + with pikepdf.open(mempdf) as pdf: + pdf.pages[0].Rotate = page_angle + target = outdir / f'{prefix}_{image_angle}_{page_angle}.pdf' + pdf.save(target) + return target + + @pytest.mark.slow @pytest.mark.parametrize('page_angle', (0, 90, 180, 270)) @pytest.mark.parametrize('image_angle', (0, 90, 180, 270)) def test_rotate_page_level(image_angle, page_angle, resources, outdir, caplog): - def make_rotate_test(prefix, image_angle, page_angle): - memimg = BytesIO() - with Image.open(fspath(resources / 'typewriter.png')) as im: - if image_angle != 0: - ccw_angle = -image_angle % 360 - im = im.transpose(getattr(Image.Transpose, f'ROTATE_{ccw_angle}')) - im.save(memimg, format='PNG') - memimg.seek(0) - mempdf = BytesIO() - img2pdf.convert( - memimg.read(), - layout_fun=img2pdf.get_fixed_dpi_layout_fun((200, 200)), - outputstream=mempdf, - **IMG2PDF_KWARGS, - ) - mempdf.seek(0) - with pikepdf.open(mempdf) as pdf: - pdf.pages[0].Rotate = page_angle - target = outdir / f'{prefix}_{image_angle}_{page_angle}.pdf' - pdf.save(target) - return target - - reference = make_rotate_test('ref', 0, 0) - test = make_rotate_test('test', image_angle, page_angle) + reference = make_rotate_test(resources / 'typewriter.png', outdir, 'ref', 0, 0) + test = make_rotate_test(resources, outdir, 'test', image_angle, page_angle) out = test.with_suffix('.out.pdf') exitcode = run_ocrmypdf_api( @@ -254,6 +255,33 @@ def test_rotate_page_level(image_angle, page_angle, resources, outdir, caplog): assert compare_images_monochrome(outdir, reference, 1, out, 1) > 0.2 +@pytest.mark.slow +@pytest.mark.parametrize('page_rotate_angle', (0, 90, 180, 270)) +def test_page_rotate_tag(page_rotate_angle, resources, outdir, caplog): + # Check that pages that have an image that is misrotated but restored to + # correct rotation with a /Rotate will be processed correct and yield text. + test = make_rotate_test( + resources / 'crom.png', outdir, 'test', -page_rotate_angle, page_rotate_angle + ) + out = test.with_suffix('.out.pdf') + exitcode = run_ocrmypdf_api( + test, + out, + '-O0', + ) + assert exitcode == 0, caplog.text + + def pdftotext(filename): + return ( + run(['pdftotext', '-enc', 'UTF-8', filename, '-'], capture_output=True) + .stdout.strip() + .decode('utf-8') + ) + + test_text = pdftotext(out) + assert 'is a' in test_text, test_text + + def test_rasterize_rotates(resources, tmp_path): pm = get_plugin_manager([])