From a9ad805347e48df3cd6ed53be46bc3519b46a429 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Sun, 31 Jan 2021 00:08:20 -0800 Subject: [PATCH 1/7] optimize: Remove shim for unsupported pikepdf version --- src/ocrmypdf/optimize.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/ocrmypdf/optimize.py b/src/ocrmypdf/optimize.py index 93658076..3fa92355 100644 --- a/src/ocrmypdf/optimize.py +++ b/src/ocrmypdf/optimize.py @@ -150,11 +150,6 @@ def extract_image_generic( if pim.bits_per_component == 1: return None - try: - pim.indexed # pikepdf 1.6.3 can't handle [/Indexed [/Array...]] - except NotImplementedError: - return None - if filtdp[0] == Name.DCTDecode and options.optimize >= 2: # This is a simple heuristic derived from some training data, that has # about a 70% chance of guessing whether the JPEG is high quality, From 2898879be7bdf4b526c3924626ea673aa6768a56 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Sun, 14 Feb 2021 01:23:01 -0800 Subject: [PATCH 2/7] docker-compose: fix typo --- misc/docker-compose.example.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/docker-compose.example.yml b/misc/docker-compose.example.yml index 0db102b9..9668b2b9 100644 --- a/misc/docker-compose.example.yml +++ b/misc/docker-compose.example.yml @@ -9,7 +9,7 @@ services: - "/media/scan:/input" - "/mnt/scan:/output" environment: - - OCR_OUTPUT_DIRECTORY_YEAR_MONT=0 + - OCR_OUTPUT_DIRECTORY_YEAR_MONTH=0 user: ":" entrypoint: python3 command: watcher.py From 2a52c6dec2ea58a7ba0a108b5dd60b14691c8d7b Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Sun, 14 Feb 2021 01:35:33 -0800 Subject: [PATCH 3/7] optimize: skip images with unusually small dimensions They're unlikely to be handled well by our recompressors. It seems that JBIG2 cannot handle very small widths. Fixes #732 --- src/ocrmypdf/optimize.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ocrmypdf/optimize.py b/src/ocrmypdf/optimize.py index 3fa92355..05057d5c 100644 --- a/src/ocrmypdf/optimize.py +++ b/src/ocrmypdf/optimize.py @@ -76,6 +76,9 @@ def extract_image_filter( if image.Length < 100: log.debug(f"Skipping small image, xref {xref}") return None + if image.Width < 8 or image.Height < 8: # Issue 732 + log.debug(f"Skipping oddly sized image, xref {xref}") + return None pim = PdfImage(image) From 82de78b6b0ec35b038a9e9db560bd3e99f70d650 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Sun, 14 Feb 2021 01:51:26 -0800 Subject: [PATCH 4/7] v11.6.1 release notes --- docs/release_notes.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/release_notes.rst b/docs/release_notes.rst index 19c4f60a..35467958 100644 --- a/docs/release_notes.rst +++ b/docs/release_notes.rst @@ -12,6 +12,14 @@ may be unreliable. Use the API to depend on precise behavior. The public API may be useful in scripts that launch OCRmyPDF processes or that wish to use some of its features for working with PDFs. +v11.6.1 +======= + +- Fixed an issue with attempting optimize unusually narrow-width images by excluding + these images from optimization (#732). +- Remove an obsolete compatibility shim for a version of pikepdf that is no longer + supported. + v11.6.0 ======= From 8770fff96885dd16b2bc95e5815592bb18d7ef94 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Mon, 15 Feb 2021 01:05:08 -0800 Subject: [PATCH 5/7] tests: remove unreliable/incomplete test --- tests/test_rotation.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tests/test_rotation.py b/tests/test_rotation.py index bb699124..8ee08344 100644 --- a/tests/test_rotation.py +++ b/tests/test_rotation.py @@ -249,16 +249,6 @@ def test_rotate_page_level(image_angle, page_angle, resources, outdir): assert check_monochrome_correlation(outdir, reference, 1, out, 1) > 0.2 -def test_tesseract_orientation(resources, tmp_path): - pix = leptonica.Pix.open(resources / 'crom.png') - pix_rotated = pix.rotate_orth(2) # 180 degrees clockwise - pix_rotated.write_implied_format(tmp_path / '000001.png') - - tesseract.get_orientation( # Test results of this are unreliable - tmp_path / '000001.png', engine_mode='3', timeout=10 - ) - - def test_rasterize_rotates(resources, tmp_path): pm = get_plugin_manager([]) From 064f935699e9763045f89f8128d18e6ba9b26635 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Mon, 15 Feb 2021 01:47:09 -0800 Subject: [PATCH 6/7] Fix page rotation regression Page size fixes in commit b26749 did accounted for a "kept" rotation, but not a corrected rotation. Fixes #730. --- src/ocrmypdf/_pipeline.py | 8 +- src/ocrmypdf/_sync.py | 2 +- tests/plugins/tesseract_debug_rotate.py | 112 ++++++++++++++++++++++++ tests/test_rotation.py | 51 ++++++++++- 4 files changed, 169 insertions(+), 4 deletions(-) create mode 100644 tests/plugins/tesseract_debug_rotate.py diff --git a/src/ocrmypdf/_pipeline.py b/src/ocrmypdf/_pipeline.py index 6de1f2e9..28ddf6e2 100644 --- a/src/ocrmypdf/_pipeline.py +++ b/src/ocrmypdf/_pipeline.py @@ -584,7 +584,9 @@ def create_visible_page_jpg(image: Path, page_context: PageContext) -> Path: return output_file -def create_pdf_page_from_image(image: Path, page_context: PageContext): +def create_pdf_page_from_image( + image: Path, page_context: PageContext, orientation_correction +): # We rasterize a square DPI version of each page because most image # processing tools don't support rectangular DPI. Use the square DPI as it # accurately describes the image. It would be possible to resample the image @@ -595,7 +597,8 @@ def create_pdf_page_from_image(image: Path, page_context: PageContext): pageinfo = page_context.pageinfo pagesize = 72.0 * float(pageinfo.width_inches), 72.0 * float(pageinfo.height_inches) - if pageinfo.rotation % 180 == 90: + effective_rotation = (pageinfo.rotation - orientation_correction) % 360 + if effective_rotation % 180 == 90: pagesize = pagesize[1], pagesize[0] # This create a single page PDF @@ -607,6 +610,7 @@ def create_pdf_page_from_image(image: Path, page_context: PageContext): imfile, with_pdfrw=False, layout_fun=layout_fun, outputstream=pdf ) log.debug('convert done') + return output_file diff --git a/src/ocrmypdf/_sync.py b/src/ocrmypdf/_sync.py index af32ad8c..43e5008a 100644 --- a/src/ocrmypdf/_sync.py +++ b/src/ocrmypdf/_sync.py @@ -204,7 +204,7 @@ def exec_page_sync(page_context: PageContext): if filtered_image: visible_image_out = filtered_image pdf_page_from_image_out = create_pdf_page_from_image( - visible_image_out, page_context + visible_image_out, page_context, orientation_correction ) if options.pdf_renderer.startswith('hocr'): diff --git a/tests/plugins/tesseract_debug_rotate.py b/tests/plugins/tesseract_debug_rotate.py new file mode 100644 index 00000000..30c613bd --- /dev/null +++ b/tests/plugins/tesseract_debug_rotate.py @@ -0,0 +1,112 @@ +# © 2020 James R. Barlow: github.com/jbarlow83 +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +"""Tesseract no-op/fixed rotate plugin + +To quickly run tests where getting OCR output is not necessary and we want to test +the rotation pipeline. + +In 'hocr' mode, create a .hocr file that specifies no text found. + +In 'pdf' mode, convert the image to PDF using another program. + +In orientation check mode, report 0, 90, 180, 270... based on page number. +""" + +import pikepdf +from PIL import Image + +from ocrmypdf import OcrEngine, OrientationConfidence, hookimpl +from ocrmypdf.helpers import page_number + +HOCR_TEMPLATE = ''' + + + + + + + + + +
+
+

+ + +

+
+
+ +''' + + +class FixedRotateNoopOcrEngine(OcrEngine): + @staticmethod + def version(): + return '4.0.0' + + @staticmethod + def creator_tag(options): + tag = '-PDF' if options.pdf_renderer == 'sandwich' else '' + return f"NO-OP {tag} {FixedRotateNoopOcrEngine.version()}" + + def __str__(self): + return f"NO-OP {FixedRotateNoopOcrEngine.version()}" + + @staticmethod + def languages(options): + return {'eng'} + + @staticmethod + def get_orientation(input_file, options): + page = page_number(input_file) + + angle = ((page - 1) * 90) % 360 + + return OrientationConfidence(angle=angle, confidence=99.9) + + @staticmethod + def generate_hocr(input_file, output_hocr, output_text, options): + with Image.open(input_file) as im, open( + output_hocr, 'w', encoding='utf-8' + ) as f: + w, h = im.size + f.write(HOCR_TEMPLATE.format(str(w), str(h))) + with open(output_text, 'w') as f: + f.write('') + + @staticmethod + def generate_pdf(input_file, output_pdf, output_text, options): + with Image.open(input_file) as im: + dpi = im.info['dpi'] + pagesize = im.size[0] / dpi[0], im.size[1] / dpi[1] + ptsize = pagesize[0] * 72, pagesize[1] * 72 + pdf = pikepdf.new() + pdf.add_blank_page(page_size=ptsize) + pdf.save(output_pdf, static_id=True) + output_text.write_text('') + + +@hookimpl +def get_ocr_engine(): + return FixedRotateNoopOcrEngine() diff --git a/tests/test_rotation.py b/tests/test_rotation.py index 8ee08344..b826a09e 100644 --- a/tests/test_rotation.py +++ b/tests/test_rotation.py @@ -6,15 +6,17 @@ from io import BytesIO +from math import cos, pi, sin from os import fspath import img2pdf import pikepdf import pytest from PIL import Image +from reportlab.pdfgen.canvas import Canvas from ocrmypdf import leptonica -from ocrmypdf._exec import ghostscript, tesseract +from ocrmypdf._exec import ghostscript from ocrmypdf._plugin_manager import get_plugin_manager from ocrmypdf.helpers import Resolution from ocrmypdf.pdfinfo import PdfInfo @@ -277,3 +279,50 @@ def test_rasterize_rotates(resources, tmp_path): filter_vector=False, ) assert Image.open(img).size == (151, 123), "Image not rotated" + + +def test_simulated_scan(outdir): + canvas = Canvas( + fspath(outdir / 'fakescan.pdf'), + pagesize=(209.8, 297.6), + ) + + page_vars = [(2, 36, 250), (91, 170, 240), (179, 190, 36), (271, 36, 36)] + + for n, page_var in enumerate(page_vars): + text = canvas.beginText() + text.setFont('Helvetica', 20) + + angle, x, y = page_var + cos_a, sin_a = cos(angle / 180.0 * pi), sin(angle / 180.0 * pi) + + text.setTextTransform(cos_a, -sin_a, sin_a, cos_a, x, y) + text.textOut(f'Page {n + 1}') + canvas.drawText(text) + canvas.showPage() + canvas.save() + + check_ocrmypdf( + outdir / 'fakescan.pdf', + outdir / 'out.pdf', + '--force-ocr', + '--deskew', + '--rotate-pages', + '--plugin', + 'tests/plugins/tesseract_debug_rotate.py', + ) + + with pikepdf.open(outdir / 'out.pdf') as pdf: + assert ( + pdf.pages[1].MediaBox[2] > pdf.pages[1].MediaBox[3] + ), "Wrong orientation: not landscape" + assert ( + pdf.pages[3].MediaBox[2] > pdf.pages[3].MediaBox[3] + ), "Wrong orientation: Not landscape" + + assert ( + pdf.pages[0].MediaBox[2] < pdf.pages[0].MediaBox[3] + ), "Wrong orientation: Not portrait" + assert ( + pdf.pages[2].MediaBox[2] < pdf.pages[2].MediaBox[3] + ), "Wrong orientation: Not portrait" From 3692868004c2c4255603b3f75b4b1d872ffd1bec Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Mon, 15 Feb 2021 01:48:14 -0800 Subject: [PATCH 7/7] v11.6.2 release notes --- docs/release_notes.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/release_notes.rst b/docs/release_notes.rst index 35467958..15357e00 100644 --- a/docs/release_notes.rst +++ b/docs/release_notes.rst @@ -12,6 +12,12 @@ may be unreliable. Use the API to depend on precise behavior. The public API may be useful in scripts that launch OCRmyPDF processes or that wish to use some of its features for working with PDFs. +v11.6.2 +======= + +- Fixed a regression where the wrong page orientation would be produced when using + arguments such as ``--deskew --rotate-pages`` (#730). + v11.6.1 =======