Reactivate pngquant on windows

Acrobat insists that PDF/A-1b should not have object streams.
Other programs like veraPDF disagree with this restriction, but
we can accommodate Acrobat so we will.

Also add more tests around this.

azure-pipelines.yml

@@ -32,7 +32,7 @@ stages:
               choco install --yes --no-progress --pre tesseract
               choco install --yes --no-progress python3
               choco install --yes --no-progress ghostscript
-              # choco install --yes --no-progress pngquant
+              choco install --yes --no-progress pngquant
             displayName: "Install system packages"
           - pwsh: |
               refreshenv

docs/release_notes.rst

@@ -12,6 +12,30 @@ may be unreliable. Use the API to depend on precise behavior.
 The public API may be useful in scripts that launch OCRmyPDF processes or that
 wish to use some of its features for working with PDFs.
 
+v11.7.0
+=======
+
+-  We now support using ``--sidecar`` in conjunction with ``--pages``; these arguments
+   used to be mutually exclusive. (#735)
+-  Fixed a possible issue with PDF/A-1b generation. Acrobat complained that our PDFs use
+   object streams. More robust PDF/A validators like veraPDF don't consider this a
+   problem, but we'll honor Acrobat's objection from here on. This may increase file
+   size of PDF/A-1b files. PDF/A-2b files will not be affected.
+
+v11.6.2
+=======
+
+-  Fixed a regression where the wrong page orientation would be produced when using
+   arguments such as ``--deskew --rotate-pages`` (#730).
+
+v11.6.1
+=======
+
+-  Fixed an issue with attempting optimize unusually narrow-width images by excluding
+   these images from optimization (#732).
+-  Remove an obsolete compatibility shim for a version of pikepdf that is no longer
+   supported.
+
 v11.6.0
 =======
 

misc/docker-compose.example.yml

@@ -9,7 +9,7 @@ services:
       - "/media/scan:/input"
       - "/mnt/scan:/output"
     environment:
-      - OCR_OUTPUT_DIRECTORY_YEAR_MONT=0
+      - OCR_OUTPUT_DIRECTORY_YEAR_MONTH=0
     user: "<SET TO YOUR USER ID>:<SET TO YOUR GROUP ID>"
     entrypoint: python3
     command: watcher.py

src/ocrmypdf/_pipeline.py

@@ -584,7 +584,9 @@ def create_visible_page_jpg(image: Path, page_context: PageContext) -> Path:
     return output_file
 
 
-def create_pdf_page_from_image(image: Path, page_context: PageContext):
+def create_pdf_page_from_image(
+    image: Path, page_context: PageContext, orientation_correction
+):
     # We rasterize a square DPI version of each page because most image
     # processing tools don't support rectangular DPI. Use the square DPI as it
     # accurately describes the image. It would be possible to resample the image
@@ -595,7 +597,8 @@ def create_pdf_page_from_image(image: Path, page_context: PageContext):
 
     pageinfo = page_context.pageinfo
     pagesize = 72.0 * float(pageinfo.width_inches), 72.0 * float(pageinfo.height_inches)
-    if pageinfo.rotation % 180 == 90:
+    effective_rotation = (pageinfo.rotation - orientation_correction) % 360
+    if effective_rotation % 180 == 90:
         pagesize = pagesize[1], pagesize[0]
 
     # This create a single page PDF
@@ -607,6 +610,7 @@ def create_pdf_page_from_image(image: Path, page_context: PageContext):
             imfile, with_pdfrw=False, layout_fun=layout_fun, outputstream=pdf
         )
         log.debug('convert done')
+
     return output_file
 
 
@@ -735,6 +739,24 @@ def should_linearize(working_file: Path, context: PdfContext):
     return False
 
 
+def get_pdf_save_settings(output_type: str):
+    if output_type == 'pdfa-1':
+        # Trigger recompression to ensure object streams are removed, because
+        # Acrobat complains about them in PDF/A-1b validation.
+        return dict(
+            preserve_pdfa=True,
+            compress_streams=True,
+            stream_decode_level=pikepdf.StreamDecodeLevel.generalized,
+            object_stream_mode=pikepdf.ObjectStreamMode.disable,
+        )
+    else:
+        return dict(
+            preserve_pdfa=True,
+            compress_streams=True,
+            object_stream_mode=(pikepdf.ObjectStreamMode.generate),
+        )
+
+
 def metadata_fixup(working_file: Path, context: PdfContext):
     output_file = context.get_path('metafix.pdf')
     options = context.options
@@ -779,9 +801,7 @@ def metadata_fixup(working_file: Path, context: PdfContext):
 
         pdf.save(
             output_file,
-            compress_streams=True,
-            preserve_pdfa=True,
-            object_stream_mode=pikepdf.ObjectStreamMode.generate,
+            **get_pdf_save_settings(options.output_type),
             linearize=(  # Don't linearize if optimize() will be linearizing too
                 should_linearize(working_file, context)
                 if options.optimize == 0
@@ -795,20 +815,34 @@ def metadata_fixup(working_file: Path, context: PdfContext):
 def optimize_pdf(input_file: Path, context: PdfContext):
     output_file = context.get_path('optimize.pdf')
     save_settings = dict(
-        compress_streams=True,
-        preserve_pdfa=True,
-        object_stream_mode=pikepdf.ObjectStreamMode.generate,
         linearize=should_linearize(input_file, context),
+        **get_pdf_save_settings(context.options.output_type),
     )
     optimize(input_file, output_file, context, save_settings)
     return output_file
 
 
+def enumerate_compress_ranges(iterable):
+    skipped_from = None
+    for index, txt_file in enumerate(iterable):
+        index += 1
+        if txt_file:
+            if skipped_from is not None:
+                yield (skipped_from, index - 1), None
+                skipped_from = None
+            yield (index, index), txt_file
+        else:
+            if skipped_from is None:
+                skipped_from = index
+    if skipped_from is not None:
+        yield (skipped_from, index), None
+
+
 def merge_sidecars(txt_files: Iterable[Optional[Path]], context: PdfContext):
     output_file = context.get_path('sidecar.txt')
     with open(output_file, 'w', encoding="utf-8") as stream:
-        for page_num, txt_file in enumerate(txt_files):
-            if page_num != 0:
+        for (frm, to), txt_file in enumerate_compress_ranges(txt_files):
+            if frm != 1:
                 stream.write('\f')  # Form feed between pages
             if txt_file:
                 with open(txt_file, 'r', encoding="utf-8") as in_:
@@ -821,7 +855,11 @@ def merge_sidecars(txt_files: Iterable[Optional[Path]], context: PdfContext):
                     else:
                         stream.write(txt)
             else:
-                stream.write(f'[OCR skipped on page {(page_num + 1)}]')
+                if frm != to:
+                    pages = f'{frm}-{to}'
+                else:
+                    pages = f'{frm}'
+                stream.write(f'[OCR skipped on page(s) {pages}]')
     return output_file
 
 

src/ocrmypdf/_sync.py

@@ -204,7 +204,7 @@ def exec_page_sync(page_context: PageContext):
         if filtered_image:
             visible_image_out = filtered_image
         pdf_page_from_image_out = create_pdf_page_from_image(
-            visible_image_out, page_context
+            visible_image_out, page_context, orientation_correction
         )
 
     if options.pdf_renderer.startswith('hocr'):

src/ocrmypdf/_validation.py

@@ -184,8 +184,6 @@ def check_options_ocr_behavior(options):
     )
     if exclusive_options >= 2:
         raise BadArgsError("Choose only one of --force-ocr, --skip-text, --redo-ocr.")
-    if options.pages and options.sidecar:
-        raise BadArgsError("--pages and --sidecar are mutually exclusive")
     if options.pages:
         options.pages = _pages_from_ranges(options.pages)
 

src/ocrmypdf/helpers.py

@@ -181,45 +181,42 @@ def check_pdf(input_file: Path) -> bool:
     Checks for proper formatting and proper linearization. Uses pikepdf (which in
     turn, uses QPDF) to perform the checks.
     """
-    pdf = None
     try:
         pdf = pikepdf.open(input_file)
     except pikepdf.PdfError as e:
         log.error(e)
         return False
     else:
-        messages = pdf.check()
-        for msg in messages:
-            if 'error' in msg.lower():
-                log.error(msg)
+        with pdf:
+            messages = pdf.check()
+            for msg in messages:
+                if 'error' in msg.lower():
+                    log.error(msg)
+                else:
+                    log.warning(msg)
+
+            sio = StringIO()
+            linearize_msgs = ''
+            try:
+                # If linearization is missing entirely, we do not complain. We do
+                # complain if linearization is present but incorrect.
+                pdf.check_linearization(sio)
+            except RuntimeError:
+                pass
+            except (  # Workaround for a problematic pikepdf version
+                getattr(pikepdf, 'ForeignObjectError')
+                if pikepdf.__version__ == '2.1.0'
+                else NeverRaise
+            ):
+                pass
             else:
-                log.warning(msg)
+                linearize_msgs = sio.getvalue()
+                if linearize_msgs:
+                    log.warning(linearize_msgs)
 
-        sio = StringIO()
-        linearize_msgs = ''
-        try:
-            # If linearization is missing entirely, we do not complain. We do
-            # complain if linearization is present but incorrect.
-            pdf.check_linearization(sio)
-        except RuntimeError:
-            pass
-        except (
-            getattr(pikepdf, 'ForeignObjectError')
-            if pikepdf.__version__ == '2.1.0'  # This version may throw wrong exception
-            else NeverRaise
-        ):
-            pass
-        else:
-            linearize_msgs = sio.getvalue()
-            if linearize_msgs:
-                log.warning(linearize_msgs)
-
-        if not messages and not linearize_msgs:
-            return True
-        return False
-    finally:
-        if pdf:
-            pdf.close()
+            if not messages and not linearize_msgs:
+                return True
+            return False
 
 
 def clamp(n, smallest, largest):  # mypy doesn't understand types for this

src/ocrmypdf/optimize.py

@@ -76,6 +76,9 @@ def extract_image_filter(
     if image.Length < 100:
         log.debug(f"Skipping small image, xref {xref}")
         return None
+    if image.Width < 8 or image.Height < 8:  # Issue 732
+        log.debug(f"Skipping oddly sized image, xref {xref}")
+        return None
 
     pim = PdfImage(image)
 
@@ -150,11 +153,6 @@ def extract_image_generic(
     if pim.bits_per_component == 1:
         return None
 
-    try:
-        pim.indexed  # pikepdf 1.6.3 can't handle [/Indexed [/Array...]]
-    except NotImplementedError:
-        return None
-
     if filtdp[0] == Name.DCTDecode and options.optimize >= 2:
         # This is a simple heuristic derived from some training data, that has
         # about a 70% chance of guessing whether the JPEG is high quality,

tests/plugins/tesseract_debug_rotate.py

@@ -0,0 +1,112 @@
+# © 2020 James R. Barlow: github.com/jbarlow83
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+"""Tesseract no-op/fixed rotate plugin
+
+To quickly run tests where getting OCR output is not necessary and we want to test
+the rotation pipeline.
+
+In 'hocr' mode, create a .hocr file that specifies no text found.
+
+In 'pdf' mode, convert the image to PDF using another program.
+
+In orientation check mode, report 0, 90, 180, 270... based on page number.
+"""
+
+import pikepdf
+from PIL import Image
+
+from ocrmypdf import OcrEngine, OrientationConfidence, hookimpl
+from ocrmypdf.helpers import page_number
+
+HOCR_TEMPLATE = '''<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+  <title></title>
+  <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+  <meta name='ocr-system' content='tesseract 4.0.0' />
+  <meta name='ocr-capabilities' content='ocr_page ocr_carea ocr_par ocr_line ocrx_word'/>
+ </head>
+ <body>
+  <div class='ocr_page' id='page_1' title='image "x.tif"; bbox 0 0 {0} {1}; ppageno 0'>
+   <div class='ocr_carea' id='block_1_1' title="bbox 0 1 {0} {1}">
+    <p class='ocr_par' dir='ltr' id='par_1' title="bbox 0 1 {0} {1}">
+     <span class='ocr_line' id='line_1' title="bbox 0 1 {0} {1}"><span class='ocrx_word' id='word_1' title="bbox 0 1 {0} {1}"> </span>
+     </span>
+    </p>
+   </div>
+  </div>
+ </body>
+</html>'''
+
+
+class FixedRotateNoopOcrEngine(OcrEngine):
+    @staticmethod
+    def version():
+        return '4.0.0'
+
+    @staticmethod
+    def creator_tag(options):
+        tag = '-PDF' if options.pdf_renderer == 'sandwich' else ''
+        return f"NO-OP {tag} {FixedRotateNoopOcrEngine.version()}"
+
+    def __str__(self):
+        return f"NO-OP {FixedRotateNoopOcrEngine.version()}"
+
+    @staticmethod
+    def languages(options):
+        return {'eng'}
+
+    @staticmethod
+    def get_orientation(input_file, options):
+        page = page_number(input_file)
+
+        angle = ((page - 1) * 90) % 360
+
+        return OrientationConfidence(angle=angle, confidence=99.9)
+
+    @staticmethod
+    def generate_hocr(input_file, output_hocr, output_text, options):
+        with Image.open(input_file) as im, open(
+            output_hocr, 'w', encoding='utf-8'
+        ) as f:
+            w, h = im.size
+            f.write(HOCR_TEMPLATE.format(str(w), str(h)))
+        with open(output_text, 'w') as f:
+            f.write('')
+
+    @staticmethod
+    def generate_pdf(input_file, output_pdf, output_text, options):
+        with Image.open(input_file) as im:
+            dpi = im.info['dpi']
+            pagesize = im.size[0] / dpi[0], im.size[1] / dpi[1]
+        ptsize = pagesize[0] * 72, pagesize[1] * 72
+        pdf = pikepdf.new()
+        pdf.add_blank_page(page_size=ptsize)
+        pdf.save(output_pdf, static_id=True)
+        output_text.write_text('')
+
+
+@hookimpl
+def get_ocr_engine():
+    return FixedRotateNoopOcrEngine()

tests/test_pdfa.py

@@ -0,0 +1,34 @@
+# © 2021 James R. Barlow: github.com/jbarlow83
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import pikepdf
+import pytest
+
+check_ocrmypdf = pytest.helpers.check_ocrmypdf
+
+
+@pytest.mark.parametrize('optimize', (0, 3))
+@pytest.mark.parametrize('pdfa_level', (1, 2, 3))
+def test_pdfa(resources, outpdf, optimize, pdfa_level):
+    check_ocrmypdf(
+        resources / 'francais.pdf',
+        outpdf,
+        '--plugin',
+        'tests/plugins/tesseract_noop.py',
+        f'--output-type=pdfa-{pdfa_level}',
+        f'--optimize={optimize}',
+    )
+    if pdfa_level in (2, 3):
+        # PDF/A-2 allows ObjStm
+        assert b'/ObjStm' in outpdf.read_bytes()
+    elif pdfa_level == 1:
+        # PDF/A-1 might allow ObjStm, but Acrobat does not approve it, so
+        # we don't use it
+        assert b'/ObjStm' not in outpdf.read_bytes()
+
+    with pikepdf.open(outpdf) as pdf:
+        with pdf.open_metadata() as m:
+            assert m.pdfa_status == f'{pdfa_level}B'

tests/test_pipeline.py

@@ -61,3 +61,90 @@ def test_dpi_needed(image, text, vector, result, rgb_image, outdir):
 
     assert _pipeline.get_canvas_square_dpi(pi[0], mock) == result
     assert _pipeline.get_page_square_dpi(pi[0], mock) == result
+
+
+@pytest.mark.parametrize(
+    # Name for nicer -v output
+    'name,input,output',
+    (
+        (
+            'empty_input',
+            # Input:
+            (),
+            # Output:
+            (),
+        ),
+        (
+            'no_values',
+            # Input:
+            ('', '', '', '', ''),
+            # Output:
+            (
+                ((1, 5), None),
+            ),
+        ),
+        (
+            'no_empty_values',
+            # Input:
+            ('v', 'w', 'x', 'y', 'z'),
+            # Output:
+            (
+                ((1, 1), 'v'),
+                ((2, 2), 'w'),
+                ((3, 3), 'x'),
+                ((4, 4), 'y'),
+                ((5, 5), 'z'),
+            ),
+        ),
+        (
+            'skip_head',
+            # Input:
+            ('', '', 'x', 'y', 'z'),
+            # Output:
+            (
+                ((1, 2), None),
+                ((3, 3), 'x'),
+                ((4, 4), 'y'),
+                ((5, 5), 'z'),
+            ),
+        ),
+        (
+            'skip_tail',
+            # Input:
+            ('x', 'y', 'z', '', ''),
+            # Output:
+            (
+                ((1, 1), 'x'),
+                ((2, 2), 'y'),
+                ((3, 3), 'z'),
+                ((4, 5), None),
+            ),
+        ),
+        (
+            'range_in_middle',
+            # Input:
+            ('x', '', '', '', 'y'),
+            # Output:
+            (
+                ((1, 1), 'x'),
+                ((2, 4), None),
+                ((5, 5), 'y'),
+            ),
+        ),
+        (
+            'range_in_middle_2',
+            # Input:
+            ('x', '', '', 'y', '', '', '', 'z'),
+            # Output:
+            (
+                ((1, 1), 'x'),
+                ((2, 3), None),
+                ((4, 4), 'y'),
+                ((5, 7), None),
+                ((8, 8), 'z'),
+            ),
+        ),
+    ),
+)
+def test_enumerate_compress_ranges(name, input, output):
+    assert output == tuple(_pipeline.enumerate_compress_ranges(input))

tests/test_rotation.py

@@ -6,15 +6,17 @@
 
 
 from io import BytesIO
+from math import cos, pi, sin
 from os import fspath
 
 import img2pdf
 import pikepdf
 import pytest
 from PIL import Image
+from reportlab.pdfgen.canvas import Canvas
 
 from ocrmypdf import leptonica
-from ocrmypdf._exec import ghostscript, tesseract
+from ocrmypdf._exec import ghostscript
 from ocrmypdf._plugin_manager import get_plugin_manager
 from ocrmypdf.helpers import Resolution
 from ocrmypdf.pdfinfo import PdfInfo
@@ -249,16 +251,6 @@ def test_rotate_page_level(image_angle, page_angle, resources, outdir):
     assert check_monochrome_correlation(outdir, reference, 1, out, 1) > 0.2
 
 
-def test_tesseract_orientation(resources, tmp_path):
-    pix = leptonica.Pix.open(resources / 'crom.png')
-    pix_rotated = pix.rotate_orth(2)  # 180 degrees clockwise
-    pix_rotated.write_implied_format(tmp_path / '000001.png')
-
-    tesseract.get_orientation(  # Test results of this are unreliable
-        tmp_path / '000001.png', engine_mode='3', timeout=10
-    )
-
-
 def test_rasterize_rotates(resources, tmp_path):
     pm = get_plugin_manager([])
 
@@ -287,3 +279,50 @@ def test_rasterize_rotates(resources, tmp_path):
         filter_vector=False,
     )
     assert Image.open(img).size == (151, 123), "Image not rotated"
+
+
+def test_simulated_scan(outdir):
+    canvas = Canvas(
+        fspath(outdir / 'fakescan.pdf'),
+        pagesize=(209.8, 297.6),
+    )
+
+    page_vars = [(2, 36, 250), (91, 170, 240), (179, 190, 36), (271, 36, 36)]
+
+    for n, page_var in enumerate(page_vars):
+        text = canvas.beginText()
+        text.setFont('Helvetica', 20)
+
+        angle, x, y = page_var
+        cos_a, sin_a = cos(angle / 180.0 * pi), sin(angle / 180.0 * pi)
+
+        text.setTextTransform(cos_a, -sin_a, sin_a, cos_a, x, y)
+        text.textOut(f'Page {n + 1}')
+        canvas.drawText(text)
+        canvas.showPage()
+    canvas.save()
+
+    check_ocrmypdf(
+        outdir / 'fakescan.pdf',
+        outdir / 'out.pdf',
+        '--force-ocr',
+        '--deskew',
+        '--rotate-pages',
+        '--plugin',
+        'tests/plugins/tesseract_debug_rotate.py',
+    )
+
+    with pikepdf.open(outdir / 'out.pdf') as pdf:
+        assert (
+            pdf.pages[1].MediaBox[2] > pdf.pages[1].MediaBox[3]
+        ), "Wrong orientation: not landscape"
+        assert (
+            pdf.pages[3].MediaBox[2] > pdf.pages[3].MediaBox[3]
+        ), "Wrong orientation: Not landscape"
+
+        assert (
+            pdf.pages[0].MediaBox[2] < pdf.pages[0].MediaBox[3]
+        ), "Wrong orientation: Not portrait"
+        assert (
+            pdf.pages[2].MediaBox[2] < pdf.pages[2].MediaBox[3]
+        ), "Wrong orientation: Not portrait"

tests/test_validation.py

@@ -90,8 +90,6 @@ def test_mutex_options():
         vd.check_options_ocr_behavior(make_opts(redo_ocr=True, skip_text=True))
     with pytest.raises(BadArgsError):
         vd.check_options_ocr_behavior(make_opts(redo_ocr=True, force_ocr=True))
-    with pytest.raises(BadArgsError):
-        vd.check_options_ocr_behavior(make_opts(pages='1-3', sidecar='file.txt'))
 
 
 def test_optimizing(caplog):