Merge branch 'master' into scriptsender

README.md

@@ -59,15 +59,23 @@ I searched the web for a free command line tool to OCR PDF files: I found many,
 
 Linux, Windows, macOS and FreeBSD are supported. Docker images are also available.
 
-| Operating system              | Install command               |
+Users of Debian 9 or later or Ubuntu 16.10 or later may simply
-| ----------------------------- | ------------------------------|
+
-| Debian, Ubuntu                | ``apt install ocrmypdf``      |
+```bash
-| Windows Subsystem for Linux   | ``apt install ocrmypdf``      |
+apt-get install ocrmypdf
-| Fedora                        | ``dnf install ocrmypdf``      |
+```
-| macOS                         | ``brew install ocrmypdf``     |
+
-| LinuxBrew                     | ``brew install ocrmypdf``     |
+and users of Fedora 29 or later may simply
-| FreeBSD                       | ``pkg install py37-ocrmypdf`` |
+
-| Conda                         | ``conda install ocrmypdf``    |
+```bash
+dnf install ocrmypdf
+```
+
+and Homebrew users (macOS, Linux, Windows Subsystem for Linux) may simply
+
+```bash
+brew install ocrmypdf
+```
 
 For everyone else, [see our documentation](https://ocrmypdf.readthedocs.io/en/latest/installation.html) for installation steps.
 

azure-pipelines.yml

@@ -32,7 +32,7 @@ stages:
               choco install --yes --no-progress --pre tesseract
               choco install --yes --no-progress python3
               choco install --yes --no-progress ghostscript
-              choco install --yes --no-progress pngquant
+              # choco install --yes --no-progress pngquant
             displayName: "Install system packages"
           - pwsh: |
               refreshenv

docs/installation.rst

@@ -12,21 +12,19 @@ system/platform. This version may be out of date, however.
 
 These platforms have one-liner installs:
 
-+-------------------------------+-------------------------------+
++-----------------------------+-------------------------------+
-| Debian, Ubuntu                | ``apt install ocrmypdf``      |
+| Debian, Ubuntu              | ``apt install ocrmypdf``      |
-+-------------------------------+-------------------------------+
++-----------------------------+-------------------------------+
-| Windows Subsystem for Linux   | ``apt install ocrmypdf``      |
+| Windows Subsystem for Linux | ``apt install ocrmypdf``      |
-+-------------------------------+-------------------------------+
++-----------------------------+-------------------------------+
-| Fedora                        | ``dnf install ocrmypdf``      |
+| Fedora                      | ``dnf install ocrmypdf``      |
-+-------------------------------+-------------------------------+
++-----------------------------+-------------------------------+
-| macOS                         | ``brew install ocrmypdf``     |
+| macOS                       | ``brew install ocrmypdf``     |
-+-------------------------------+-------------------------------+
++-----------------------------+-------------------------------+
-| LinuxBrew                     | ``brew install ocrmypdf``     |
+| LinuxBrew                   | ``brew install ocrmypdf``     |
-+-------------------------------+-------------------------------+
++-----------------------------+-------------------------------+
-| FreeBSD                       | ``pkg install py37-ocrmypdf`` |
+| FreeBSD                     | ``pkg install py37-ocrmypdf`` |
-+-------------------------------+-------------------------------+
++-----------------------------+-------------------------------+
-| Conda (WSL, macOS, Linux)     | ``conda install ocrmypdf``    |
-+-------------------------------+-------------------------------+
 
 More detailed procedures are outlined below. If you want to do a manual
 install, or install a more recent version than your platform provides, read on.
@@ -56,9 +54,6 @@ Debian and Ubuntu 18.04 or newer
 .. |ubu-2004| image:: https://repology.org/badge/version-for-repo/ubuntu_20_04/ocrmypdf.svg
     :alt: Ubuntu 20.04 LTS
 
-.. |ubu-2010| image:: https://repology.org/badge/version-for-repo/ubuntu_20_10/ocrmypdf.svg
-    :alt: Ubuntu 20.10
-
 +-----------------------------------------------+
 | **OCRmyPDF versions in Debian & Ubuntu**      |
 +-----------------------------------------------+
@@ -66,7 +61,7 @@ Debian and Ubuntu 18.04 or newer
 +-----------------------------------------------+
 | |deb-stable| |deb-testing| |deb-unstable|     |
 +-----------------------------------------------+
-| |ubu-1804| |ubu-2004| |ubu-2010|              |
+| |ubu-1804| |ubu-2004|                         |
 +-----------------------------------------------+
 
 Users of Debian 9 ("stretch") or later, or Ubuntu 18.04 or later, including users
@@ -95,15 +90,15 @@ For full details on version availability for your platform, check the
    automatically detect it (specifically the ``jbig2`` binary) on the
    ``PATH``. To add JBIG2 encoding, see :ref:`jbig2`.
 
-Fedora
+Fedora 29 or newer
-------
+------------------
+
+.. |fedora-31| image:: https://repology.org/badge/version-for-repo/fedora_31/ocrmypdf.svg
+    :alt: Fedora 31
 
 .. |fedora-32| image:: https://repology.org/badge/version-for-repo/fedora_32/ocrmypdf.svg
     :alt: Fedora 32
 
-.. |fedora-33| image:: https://repology.org/badge/version-for-repo/fedora_33/ocrmypdf.svg
-    :alt: Fedora 33
-
 .. |fedora-rawhide| image:: https://repology.org/badge/version-for-repo/fedora_rawhide/ocrmypdf.svg
     :alt: Fedore Rawhide
 
@@ -112,7 +107,7 @@ Fedora
 +-----------------------------------------------+
 | |latest|                                      |
 +-----------------------------------------------+
-| |fedora-32| |fedora-33| |fedora-rawhide|      |
+| |fedora-31| |fedora-32| |fedora-rawhide|      |
 +-----------------------------------------------+
 
 Users of Fedora 29 or later may simply
@@ -360,8 +355,7 @@ To install OCRmyPDF for Alpine Linux:
 Mageia 7
 --------
 
-There is no OS-level packaging available for Mageia, so you must install the
+Install the following dependencies:
-dependencies:
 
 .. code-block:: bash
 

docs/release_notes.rst

@@ -12,24 +12,6 @@ may be unreliable. Use the API to depend on precise behavior.
 The public API may be useful in scripts that launch OCRmyPDF processes or that
 wish to use some of its features for working with PDFs.
 
-v11.7.1
-=======
-
--  Some exceptions while attempting image optimization were only logged at the debug
-   level, causing them to be suppressed. These errors are now logged appropriately.
--  Improved the error message related to ``--unpaper-args``.
--  Updated documentation to mention the new conda distribution.
-
-v11.7.0
-=======
-
--  We now support using ``--sidecar`` in conjunction with ``--pages``; these arguments
-   used to be mutually exclusive. (#735)
--  Fixed a possible issue with PDF/A-1b generation. Acrobat complained that our PDFs use
-   object streams. More robust PDF/A validators like veraPDF don't consider this a
-   problem, but we'll honor Acrobat's objection from here on. This may increase file
-   size of PDF/A-1b files. PDF/A-2b files will not be affected.
-
 v11.6.2
 =======
 

pyproject.toml

@@ -10,7 +10,7 @@ build-backend = "setuptools.build_meta"
 
 [tool.black]
 line-length = 88
-target-version = ["py36", "py37", "py38"]
+target-version = ["py36", "py37", "py38", "py39"]
 skip-string-normalization = true
 include = '\.pyi?$'
 exclude = '''

src/ocrmypdf/_pipeline.py

@@ -739,24 +739,6 @@ def should_linearize(working_file: Path, context: PdfContext):
     return False
 
 
-def get_pdf_save_settings(output_type: str):
-    if output_type == 'pdfa-1':
-        # Trigger recompression to ensure object streams are removed, because
-        # Acrobat complains about them in PDF/A-1b validation.
-        return dict(
-            preserve_pdfa=True,
-            compress_streams=True,
-            stream_decode_level=pikepdf.StreamDecodeLevel.generalized,
-            object_stream_mode=pikepdf.ObjectStreamMode.disable,
-        )
-    else:
-        return dict(
-            preserve_pdfa=True,
-            compress_streams=True,
-            object_stream_mode=(pikepdf.ObjectStreamMode.generate),
-        )
-
-
 def metadata_fixup(working_file: Path, context: PdfContext):
     output_file = context.get_path('metafix.pdf')
     options = context.options
@@ -801,7 +783,9 @@ def metadata_fixup(working_file: Path, context: PdfContext):
 
         pdf.save(
             output_file,
-            **get_pdf_save_settings(options.output_type),
+            compress_streams=True,
+            preserve_pdfa=True,
+            object_stream_mode=pikepdf.ObjectStreamMode.generate,
             linearize=(  # Don't linearize if optimize() will be linearizing too
                 should_linearize(working_file, context)
                 if options.optimize == 0
@@ -815,34 +799,20 @@ def metadata_fixup(working_file: Path, context: PdfContext):
 def optimize_pdf(input_file: Path, context: PdfContext):
     output_file = context.get_path('optimize.pdf')
     save_settings = dict(
+        compress_streams=True,
+        preserve_pdfa=True,
+        object_stream_mode=pikepdf.ObjectStreamMode.generate,
         linearize=should_linearize(input_file, context),
-        **get_pdf_save_settings(context.options.output_type),
     )
     optimize(input_file, output_file, context, save_settings)
     return output_file
 
 
-def enumerate_compress_ranges(iterable):
-    skipped_from = None
-    for index, txt_file in enumerate(iterable):
-        index += 1
-        if txt_file:
-            if skipped_from is not None:
-                yield (skipped_from, index - 1), None
-                skipped_from = None
-            yield (index, index), txt_file
-        else:
-            if skipped_from is None:
-                skipped_from = index
-    if skipped_from is not None:
-        yield (skipped_from, index), None
-
-
 def merge_sidecars(txt_files: Iterable[Optional[Path]], context: PdfContext):
     output_file = context.get_path('sidecar.txt')
     with open(output_file, 'w', encoding="utf-8") as stream:
-        for (frm, to), txt_file in enumerate_compress_ranges(txt_files):
+        for page_num, txt_file in enumerate(txt_files):
-            if frm != 1:
+            if page_num != 0:
                 stream.write('\f')  # Form feed between pages
             if txt_file:
                 with open(txt_file, 'r', encoding="utf-8") as in_:
@@ -855,11 +825,7 @@ def merge_sidecars(txt_files: Iterable[Optional[Path]], context: PdfContext):
                     else:
                         stream.write(txt)
             else:
-                if frm != to:
+                stream.write(f'[OCR skipped on page {(page_num + 1)}]')
-                    pages = f'{frm}-{to}'
-                else:
-                    pages = f'{frm}'
-                stream.write(f'[OCR skipped on page(s) {pages}]')
     return output_file
 
 

src/ocrmypdf/_validation.py

@@ -133,7 +133,7 @@ def check_options_preprocessing(options):
                     options.unpaper_args
                 )
         except Exception as e:
-            raise BadArgsError("--unpaper-args: " + str(e)) from e
+            raise BadArgsError(str(e))
 
 
 def _pages_from_ranges(ranges: str) -> Set[int]:
@@ -184,6 +184,8 @@ def check_options_ocr_behavior(options):
     )
     if exclusive_options >= 2:
         raise BadArgsError("Choose only one of --force-ocr, --skip-text, --redo-ocr.")
+    if options.pages and options.sidecar:
+        raise BadArgsError("--pages and --sidecar are mutually exclusive")
     if options.pages:
         options.pages = _pages_from_ranges(options.pages)
 

src/ocrmypdf/helpers.py

@@ -181,42 +181,45 @@ def check_pdf(input_file: Path) -> bool:
     Checks for proper formatting and proper linearization. Uses pikepdf (which in
     turn, uses QPDF) to perform the checks.
     """
+    pdf = None
     try:
         pdf = pikepdf.open(input_file)
     except pikepdf.PdfError as e:
         log.error(e)
         return False
     else:
-        with pdf:
+        messages = pdf.check()
-            messages = pdf.check()
+        for msg in messages:
-            for msg in messages:
+            if 'error' in msg.lower():
-                if 'error' in msg.lower():
+                log.error(msg)
-                    log.error(msg)
-                else:
-                    log.warning(msg)
-
-            sio = StringIO()
-            linearize_msgs = ''
-            try:
-                # If linearization is missing entirely, we do not complain. We do
-                # complain if linearization is present but incorrect.
-                pdf.check_linearization(sio)
-            except RuntimeError:
-                pass
-            except (  # Workaround for a problematic pikepdf version
-                getattr(pikepdf, 'ForeignObjectError')
-                if pikepdf.__version__ == '2.1.0'
-                else NeverRaise
-            ):
-                pass
             else:
-                linearize_msgs = sio.getvalue()
+                log.warning(msg)
-                if linearize_msgs:
-                    log.warning(linearize_msgs)
 
-            if not messages and not linearize_msgs:
+        sio = StringIO()
-                return True
+        linearize_msgs = ''
-            return False
+        try:
+            # If linearization is missing entirely, we do not complain. We do
+            # complain if linearization is present but incorrect.
+            pdf.check_linearization(sio)
+        except RuntimeError:
+            pass
+        except (
+            getattr(pikepdf, 'ForeignObjectError')
+            if pikepdf.__version__ == '2.1.0'  # This version may throw wrong exception
+            else NeverRaise
+        ):
+            pass
+        else:
+            linearize_msgs = sio.getvalue()
+            if linearize_msgs:
+                log.warning(linearize_msgs)
+
+        if not messages and not linearize_msgs:
+            return True
+        return False
+    finally:
+        if pdf:
+            pdf.close()
 
 
 def clamp(n, smallest, largest):  # mypy doesn't understand types for this

src/ocrmypdf/optimize.py

@@ -258,8 +258,8 @@ def extract_images(
             result = extract_fn(
                 pike=pike, root=root, image=image, xref=xref, options=options
             )
-        except Exception:  # pylint: disable=broad-except
+        except Exception as e:  # pylint: disable=broad-except
-            log.exception(f"While extracting image xref {xref}, an error occurred")
+            log.debug("Image xref %s, error %s", xref, repr(e))
             errors += 1
         else:
             if result:

tests/test_pdfa.py

@@ -1,34 +0,0 @@
-# © 2021 James R. Barlow: github.com/jbarlow83
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import pikepdf
-import pytest
-
-check_ocrmypdf = pytest.helpers.check_ocrmypdf
-
-
-@pytest.mark.parametrize('optimize', (0, 3))
-@pytest.mark.parametrize('pdfa_level', (1, 2, 3))
-def test_pdfa(resources, outpdf, optimize, pdfa_level):
-    check_ocrmypdf(
-        resources / 'francais.pdf',
-        outpdf,
-        '--plugin',
-        'tests/plugins/tesseract_noop.py',
-        f'--output-type=pdfa-{pdfa_level}',
-        f'--optimize={optimize}',
-    )
-    if pdfa_level in (2, 3):
-        # PDF/A-2 allows ObjStm
-        assert b'/ObjStm' in outpdf.read_bytes()
-    elif pdfa_level == 1:
-        # PDF/A-1 might allow ObjStm, but Acrobat does not approve it, so
-        # we don't use it
-        assert b'/ObjStm' not in outpdf.read_bytes()
-
-    with pikepdf.open(outpdf) as pdf:
-        with pdf.open_metadata() as m:
-            assert m.pdfa_status == f'{pdfa_level}B'

tests/test_pipeline.py

@@ -61,90 +61,3 @@ def test_dpi_needed(image, text, vector, result, rgb_image, outdir):
 
     assert _pipeline.get_canvas_square_dpi(pi[0], mock) == result
     assert _pipeline.get_page_square_dpi(pi[0], mock) == result
-
-
-@pytest.mark.parametrize(
-    # Name for nicer -v output
-    'name,input,output',
-    (
-        (
-            'empty_input',
-            # Input:
-            (),
-            # Output:
-            (),
-        ),
-        (
-            'no_values',
-            # Input:
-            ('', '', '', '', ''),
-            # Output:
-            (
-                ((1, 5), None),
-            ),
-        ),
-        (
-            'no_empty_values',
-            # Input:
-            ('v', 'w', 'x', 'y', 'z'),
-            # Output:
-            (
-                ((1, 1), 'v'),
-                ((2, 2), 'w'),
-                ((3, 3), 'x'),
-                ((4, 4), 'y'),
-                ((5, 5), 'z'),
-            ),
-        ),
-        (
-            'skip_head',
-            # Input:
-            ('', '', 'x', 'y', 'z'),
-            # Output:
-            (
-                ((1, 2), None),
-                ((3, 3), 'x'),
-                ((4, 4), 'y'),
-                ((5, 5), 'z'),
-            ),
-        ),
-        (
-            'skip_tail',
-            # Input:
-            ('x', 'y', 'z', '', ''),
-            # Output:
-            (
-                ((1, 1), 'x'),
-                ((2, 2), 'y'),
-                ((3, 3), 'z'),
-                ((4, 5), None),
-            ),
-        ),
-        (
-            'range_in_middle',
-            # Input:
-            ('x', '', '', '', 'y'),
-            # Output:
-            (
-                ((1, 1), 'x'),
-                ((2, 4), None),
-                ((5, 5), 'y'),
-            ),
-        ),
-        (
-            'range_in_middle_2',
-            # Input:
-            ('x', '', '', 'y', '', '', '', 'z'),
-            # Output:
-            (
-                ((1, 1), 'x'),
-                ((2, 3), None),
-                ((4, 4), 'y'),
-                ((5, 7), None),
-                ((8, 8), 'z'),
-            ),
-        ),
-    ),
-)
-def test_enumerate_compress_ranges(name, input, output):
-    assert output == tuple(_pipeline.enumerate_compress_ranges(input))

tests/test_validation.py

@@ -90,6 +90,8 @@ def test_mutex_options():
         vd.check_options_ocr_behavior(make_opts(redo_ocr=True, skip_text=True))
     with pytest.raises(BadArgsError):
         vd.check_options_ocr_behavior(make_opts(redo_ocr=True, force_ocr=True))
+    with pytest.raises(BadArgsError):
+        vd.check_options_ocr_behavior(make_opts(pages='1-3', sidecar='file.txt'))
 
 
 def test_optimizing(caplog):