v11.7.3 release notes

See https://github.com/python-pillow/Pillow/blob/master/CHANGES.rst

.coveragerc

@@ -1,24 +0,0 @@
-[paths]
-source =
-	src
-	*/site-packages
-
-[run]
-branch = true
-parallel = true
-concurrency =
-	thread
-	multiprocessing
-source =
-	src/ocrmypdf
-
-[report]
-exclude_lines =
-	pragma: no cover
-	def __repr__
-	raise AssertionError
-	raise NotImplementedError
-	if 0:
-	if False:
-	if __name__ == .__main__.:
-	if TYPE_CHECKING:

.github/ISSUE_TEMPLATE/1-general-issues.md

@@ -24,5 +24,9 @@ If applicable, add screenshots to help explain your problem.
  - Python version:
  - OCRmyPDF version:
 
+**Installation**
+How did you install OCRmyPDF? Did you install it from your operating system's
+package manager, or using pip?
+
 **Additional context**
 Add any other context about the problem here.

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.1.0
+    rev: v3.4.0
     hooks:
       - id: check-case-conflict
       - id: check-merge-conflict
@@ -12,11 +12,11 @@ repos:
     hooks:
       - id: seed-isort-config
   - repo: https://github.com/pre-commit/mirrors-isort
-    rev: v5.0.5 # pick the isort version you'd like to use from https://github.com/pre-commit/mirrors-isort/releases
+    rev: v5.7.0 # pick the isort version you'd like to use from https://github.com/pre-commit/mirrors-isort/releases
     hooks:
       - id: isort
   - repo: https://github.com/psf/black
-    rev: 19.10b0
+    rev: 20.8b1
     hooks:
       - id: black
         language_version: python

README.md

@@ -59,23 +59,15 @@ I searched the web for a free command line tool to OCR PDF files: I found many,
 
 Linux, Windows, macOS and FreeBSD are supported. Docker images are also available.
 
-Users of Debian 9 or later or Ubuntu 16.10 or later may simply
+| Operating system              | Install command               |
-
+| ----------------------------- | ------------------------------|
-```bash
+| Debian, Ubuntu                | ``apt install ocrmypdf``      |
-apt-get install ocrmypdf
+| Windows Subsystem for Linux   | ``apt install ocrmypdf``      |
-```
+| Fedora                        | ``dnf install ocrmypdf``      |
-
+| macOS                         | ``brew install ocrmypdf``     |
-and users of Fedora 29 or later may simply
+| LinuxBrew                     | ``brew install ocrmypdf``     |
-
+| FreeBSD                       | ``pkg install py37-ocrmypdf`` |
-```bash
+| Conda                         | ``conda install ocrmypdf``    |
-dnf install ocrmypdf
-```
-
-and Homebrew users (macOS, Linux, Windows Subsystem for Linux) may simply
-
-```bash
-brew install ocrmypdf
-```
 
 For everyone else, [see our documentation](https://ocrmypdf.readthedocs.io/en/latest/installation.html) for installation steps.
 

docs/api.rst

@@ -56,6 +56,11 @@ Programs that call ``ocrmypdf.ocr()`` should also install a SIGBUS signal
 handler (except on Windows), to raise an exception if access to a memory
 mapped file fails. OCRmyPDF may use memory mapping.
 
+``ocrmypdf.ocr()`` will take a threading lock to prevent multiple runs of itself
+in the same Python interpreter process. This is not thread-safe, because of how
+OCRmyPDF's plugins and Python's library import system work. If you need to parallelize
+OCRmyPDF, use processes.
+
 .. warning::
 
     On Windows and macOS, the script that calls ``ocrmypdf.ocr()`` must be

docs/apiref.rst

@@ -5,6 +5,15 @@ API Reference
 This page summarizes the rest of the public API. Generally speaking this
 should mainly of interest to plugin developers.
 
+ocrmypdf
+========
+
+.. autoclass:: ocrmypdf.PageContext
+    :members:
+
+.. autoclass:: ocrmypdf.PdfContext
+    :members:
+
 ocrmypdf.exceptions
 ===================
 
@@ -17,6 +26,9 @@ ocrmypdf.helpers
 
 .. automodule:: ocrmypdf.helpers
     :members:
+    :noindex: deprecated
+
+    .. autodecorator:: deprecated
 
 ocrmypdf.hocrtransform
 ======================

docs/installation.rst

@@ -12,19 +12,21 @@ system/platform. This version may be out of date, however.
 
 These platforms have one-liner installs:
 
-+-----------------------------+-------------------------------+
++-------------------------------+-------------------------------+
-| Debian, Ubuntu              | ``apt install ocrmypdf``      |
+| Debian, Ubuntu                | ``apt install ocrmypdf``      |
-+-----------------------------+-------------------------------+
++-------------------------------+-------------------------------+
-| Windows Subsystem for Linux | ``apt install ocrmypdf``      |
+| Windows Subsystem for Linux   | ``apt install ocrmypdf``      |
-+-----------------------------+-------------------------------+
++-------------------------------+-------------------------------+
-| Fedora                      | ``dnf install ocrmypdf``      |
+| Fedora                        | ``dnf install ocrmypdf``      |
-+-----------------------------+-------------------------------+
++-------------------------------+-------------------------------+
-| macOS                       | ``brew install ocrmypdf``     |
+| macOS                         | ``brew install ocrmypdf``     |
-+-----------------------------+-------------------------------+
++-------------------------------+-------------------------------+
-| LinuxBrew                   | ``brew install ocrmypdf``     |
+| LinuxBrew                     | ``brew install ocrmypdf``     |
-+-----------------------------+-------------------------------+
++-------------------------------+-------------------------------+
-| FreeBSD                     | ``pkg install py37-ocrmypdf`` |
+| FreeBSD                       | ``pkg install py37-ocrmypdf`` |
-+-----------------------------+-------------------------------+
++-------------------------------+-------------------------------+
+| Conda (WSL, macOS, Linux)     | ``conda install ocrmypdf``    |
++-------------------------------+-------------------------------+
 
 More detailed procedures are outlined below. If you want to do a manual
 install, or install a more recent version than your platform provides, read on.
@@ -54,6 +56,9 @@ Debian and Ubuntu 18.04 or newer
 .. |ubu-2004| image:: https://repology.org/badge/version-for-repo/ubuntu_20_04/ocrmypdf.svg
     :alt: Ubuntu 20.04 LTS
 
+.. |ubu-2010| image:: https://repology.org/badge/version-for-repo/ubuntu_20_10/ocrmypdf.svg
+    :alt: Ubuntu 20.10
+
 +-----------------------------------------------+
 | **OCRmyPDF versions in Debian & Ubuntu**      |
 +-----------------------------------------------+
@@ -61,7 +66,7 @@ Debian and Ubuntu 18.04 or newer
 +-----------------------------------------------+
 | |deb-stable| |deb-testing| |deb-unstable|     |
 +-----------------------------------------------+
-| |ubu-1804| |ubu-2004|                         |
+| |ubu-1804| |ubu-2004| |ubu-2010|              |
 +-----------------------------------------------+
 
 Users of Debian 9 ("stretch") or later, or Ubuntu 18.04 or later, including users
@@ -90,15 +95,15 @@ For full details on version availability for your platform, check the
    automatically detect it (specifically the ``jbig2`` binary) on the
    ``PATH``. To add JBIG2 encoding, see :ref:`jbig2`.
 
-Fedora 29 or newer
+Fedora
-------------------
+------
-
-.. |fedora-31| image:: https://repology.org/badge/version-for-repo/fedora_31/ocrmypdf.svg
-    :alt: Fedora 31
 
 .. |fedora-32| image:: https://repology.org/badge/version-for-repo/fedora_32/ocrmypdf.svg
     :alt: Fedora 32
 
+.. |fedora-33| image:: https://repology.org/badge/version-for-repo/fedora_33/ocrmypdf.svg
+    :alt: Fedora 33
+
 .. |fedora-rawhide| image:: https://repology.org/badge/version-for-repo/fedora_rawhide/ocrmypdf.svg
     :alt: Fedore Rawhide
 
@@ -107,7 +112,7 @@ Fedora 29 or newer
 +-----------------------------------------------+
 | |latest|                                      |
 +-----------------------------------------------+
-| |fedora-31| |fedora-32| |fedora-rawhide|      |
+| |fedora-32| |fedora-33| |fedora-rawhide|      |
 +-----------------------------------------------+
 
 Users of Fedora 29 or later may simply
@@ -355,7 +360,8 @@ To install OCRmyPDF for Alpine Linux:
 Mageia 7
 --------
 
-Install the following dependencies:
+There is no OS-level packaging available for Mageia, so you must install the
+dependencies:
 
 .. code-block:: bash
 
@@ -520,7 +526,7 @@ DLLs or other Windows patches, and may require a reboot.
 You may then use ``pip`` to install ocrmypdf. (This can performed by a user or
 Administrator.):
 
-* ``pip install ocrmypdf
+* ``pip install ocrmypdf``
 
 Chocolatey automatically selects appropriate versions of these applications. If you
 are installing them manually, please install 64-bit versions of all applications for
@@ -533,6 +539,12 @@ override the versions OCRmyPDF selects, you can modify the ``PATH`` environment
 variable. `Follow these directions <https://www.computerhope.com/issues/ch000549.htm#dospath>`_
 to change the PATH.
 
+.. warning::
+
+    As of early 2021, users have reported problems with the Microsoft Store version of
+    Python affected most third party Python packages including OCRmyPDF. Please use
+    Python downloaded from Python.org or Chocolatey as recommended here.
+
 Windows Subsystem for Linux
 ---------------------------
 
@@ -637,7 +649,7 @@ Installing with Python pip
 OCRmyPDF is delivered by PyPI because it is a convenient way to install
 the latest version. However, PyPI and ``pip`` cannot address the fact
 that ``ocrmypdf`` depends on certain non-Python system libraries and
-programs being instsalled.
+programs being installed.
 
 For best results, first install `your platform's
 version <https://repology.org/metapackage/ocrmypdf/versions>`__ of

docs/plugins.rst

@@ -65,6 +65,27 @@ similar to ``pytest`` packages such as ``pytest-cov`` (the package) and
     ``ocrmypdf-`` (for the package name on PyPI) and ``ocrmypdf_`` (for the
     module), just like pytest plugins.
 
+Setuptools plugins
+==================
+
+You can also create a plugin that OCRmyPDF will always automatically load if both are
+installed in the same virtual environment, using a setuptools entrypoint.
+
+Your package's ``setup.py`` would need to contain the following, for a plugin
+named ``ocrmypdf-exampleplugin``:
+
+.. code-block:: python
+
+    # sample ./setup.py file
+    from setuptools import setup
+
+    setup(
+        name="ocrmypdf-exampleplugin",
+        packages=["exampleplugin"],
+        # the following makes a plugin available to pytest
+        entry_points={"ocrmypdf": ["exampleplugin = exampleplugin.pluginmodule"]},
+    )
+
 Plugin requirements
 ===================
 

docs/release_notes.rst

@@ -12,6 +12,100 @@ may be unreliable. Use the API to depend on precise behavior.
 The public API may be useful in scripts that launch OCRmyPDF processes or that
 wish to use some of its features for working with PDFs.
 
+v11.7.3
+=======
+
+-  Exclude CCITT Group 3 images from being optimized. Some libraries
+   OCRmyPDF uses do not seem to handle this obscure compression format properly.
+   You may get errors or possible corrupted output images without this fix.
+
+v11.7.2
+=======
+
+-  Updated pinned versions in main.txt, primarily to upgrade Pillow to 8.1.2, due
+   to recently disclosed security vulnerabilities in that software.
+-  The ``--sidecar`` parameter now causes an exception if set to the same file as
+   the input or output PDF.
+
+v11.7.1
+=======
+
+-  Some exceptions while attempting image optimization were only logged at the debug
+   level, causing them to be suppressed. These errors are now logged appropriately.
+-  Improved the error message related to ``--unpaper-args``.
+-  Updated documentation to mention the new conda distribution.
+
+v11.7.0
+=======
+
+-  We now support using ``--sidecar`` in conjunction with ``--pages``; these arguments
+   used to be mutually exclusive. (#735)
+-  Fixed a possible issue with PDF/A-1b generation. Acrobat complained that our PDFs use
+   object streams. More robust PDF/A validators like veraPDF don't consider this a
+   problem, but we'll honor Acrobat's objection from here on. This may increase file
+   size of PDF/A-1b files. PDF/A-2b files will not be affected.
+
+v11.6.2
+=======
+
+-  Fixed a regression where the wrong page orientation would be produced when using
+   arguments such as ``--deskew --rotate-pages`` (#730).
+
+v11.6.1
+=======
+
+-  Fixed an issue with attempting optimize unusually narrow-width images by excluding
+   these images from optimization (#732).
+-  Remove an obsolete compatibility shim for a version of pikepdf that is no longer
+   supported.
+
+v11.6.0
+=======
+
+-  OCRmyPDF will now automatically register plugins from the same virtual environment
+   with an appropriate setuptools entrypoint.
+-  Refactor the plugin manager to remove unnecessary complications and make plugin
+   registration more automatic.
+-  ``PageContext`` and ``PdfContext`` are now formally part of the API, as they
+   should have been, since they were part of ``ocrmypdf.pluginspec``.
+
+v11.5.0
+=======
+
+-  Fixed an issue where the output page size might differ by a fractional amount
+   due to rounding, when ``--force-ocr`` was used and the page contained objects
+   with multiple resolutions.
+-  When determining the resolution at which to rasterize a page, we now consider
+   printed text on the page as requiring a higher resolution. This fixes issues
+   with certain pages being rendered with unacceptably low resolution text, but
+   may increase output file sizes in some workflows where low resolution text
+   is acceptable.
+-  Added a workaround to fix an exception that occurs when trying to
+   ``import ocrmypdf.leptonica`` on Apple ARM silicon (or potentially, other
+   platforms that do not permit write+executable memory).
+
+v11.4.5
+=======
+
+-  Fixed an issue where files may not be closed when the API is used.
+-  Improved ``setup.cfg`` with better settings for test coverage.
+
+v11.4.4
+=======
+
+-  Fixed ``AttributeError: 'NoneType' object has no attribute 'userunit'``, issue #700,
+   related to OCRmyPDF not properly forwarded an error message from pdfminer.six.
+-  Adjusted typing of some arguments.
+-  ``ocrmypdf.ocr`` now takes a ``threading.Lock`` for reasons outlined in the
+   documentation.
+
+v11.4.3
+=======
+
+-  Removed a redundant debug message.
+-  Test suite now asserts that most patched functions are called when they should be.
+-  Test suite now skips a test that fails on two particular versions of piekpdf.
+
 v11.4.2
 =======
 

misc/docker-compose.example.yml

@@ -9,7 +9,7 @@ services:
       - "/media/scan:/input"
       - "/mnt/scan:/output"
     environment:
-      - OCR_OUTPUT_DIRECTORY_YEAR_MONT=0
+      - OCR_OUTPUT_DIRECTORY_YEAR_MONTH=0
     user: "<SET TO YOUR USER ID>:<SET TO YOUR GROUP ID>"
     entrypoint: python3
     command: watcher.py

requirements/main.txt

@@ -1,12 +1,12 @@
 # requirements.txt can be used to replicate the developer's build environment
 # setup.py lists a separate set of requirements that are looser to simplify
 # installation
-cffi == 1.14.3
+cffi == 1.14.5
-coloredlogs == 14.0  # technically optional
+coloredlogs == 15.0  # technically optional
 img2pdf == 0.4.0
 pdfminer.six == 20201018
-pikepdf == 2.0.0
+pikepdf == 2.9.0
 pluggy == 0.13.1
-Pillow == 8.0.1
+Pillow == 8.1.2
-reportlab == 3.5.55
+reportlab == 3.5.65
-tqdm == 4.51.0
+tqdm == 4.59.0

requirements/test.txt

@@ -1,7 +1,7 @@
-pytest >= 5.0.0
+pytest >= 6.0.0
 pytest-helpers-namespace >= 2019.1.8
-pytest-xdist >= 1.31.0
+pytest-xdist >= 2.2.0
-pytest-cov >= 2.10.0
+pytest-cov >= 2.11.1
 python-xmp-toolkit == 2.0.1  # requires apt-get install libexempi3
                      # or brew install exempi
 #PyMuPDF == 1.13.4    # optional

requirements/watcher.txt

@@ -1 +1 @@
-watchdog == 0.10.2
+watchdog == 1.0.2

setup.cfg

@@ -15,6 +15,8 @@ filterwarnings =
 	ignore:.*XMLParser.*:DeprecationWarning
 markers =
 	slow
+addopts =
+	-n auto
 
 [isort]
 multi_line_output=3
@@ -27,3 +29,30 @@ known_third_party = PIL,_cffi_backend,cffi,flask,img2pdf,pdfminer,pikepdf,pkg_re
 
 [metadata]
 license_file = LICENSE
+
+[coverage:paths]
+source =
+	src/
+
+[coverage:run]
+branch = true
+parallel = true
+concurrency = multiprocessing
+source =
+	src/ocrmypdf
+
+[coverage:report]
+# Regexes for lines to exclude from consideration
+exclude_lines =
+    # Have to re-enable the standard pragma
+    pragma: no cover
+
+    # Don't complain if tests don't hit defensive assertion code:
+    raise AssertionError
+    raise NotImplementedError
+
+    # Don't complain if non-runnable code isn't run:
+    if 0:
+	if False:
+    if __name__ == .__main__.:
+	if TYPE_CHECKING:

src/ocrmypdf/__init__.py

@@ -8,6 +8,7 @@
 from pluggy import HookimplMarker as _HookimplMarker
 
 from ocrmypdf import helpers, hocrtransform, leptonica, pdfa, pdfinfo
+from ocrmypdf._jobcontext import PageContext, PdfContext
 from ocrmypdf._version import PROGRAM_NAME, __version__
 from ocrmypdf.api import Verbosity, configure_logging, ocr
 from ocrmypdf.exceptions import (

src/ocrmypdf/_concurrent.py

@@ -109,15 +109,11 @@ def exec_progress_pool(
         )
         try:
             results = pool.imap_unordered(task, task_arguments)
-            while True:
+            for result in results:
-                try:
+                if task_finished:
-                    result = results.next()
+                    task_finished(result, pbar)
-                    if task_finished:
+                else:
-                        task_finished(result, pbar)
+                    pbar.update()
-                    else:
-                        pbar.update()
-                except StopIteration:
-                    break
         except KeyboardInterrupt:
             # Terminate pool so we exit instantly
             pool.terminate()

src/ocrmypdf/_exec/ghostscript.py

@@ -15,7 +15,7 @@ from os import fspath
 from pathlib import Path
 from shutil import which
 from subprocess import PIPE, CalledProcessError
-from typing import Optional, cast
+from typing import Optional
 
 from PIL import Image
 
@@ -56,16 +56,14 @@ def version():
 def jpeg_passthrough_available() -> bool:
     """Returns True if the installed version of Ghostscript supports JPEG passthru
 
-    Prior to 9.23, Ghostscript decode and re-encoded JPEGs internally. In 9.23
+    Prior to 9.23, Ghostscript decoded and re-encoded JPEGs internally. In 9.23
     it gained the ability to keep JPEGs unmodified. However, the 9.23
     implementation was buggy and would deletes the last two bytes of images in
     some cases, as reported here.
     https://bugs.ghostscript.com/show_bug.cgi?id=699216
 
     The issue was fixed for 9.24, hence that is the first version we consider
-    the feature available. (However, we don't use 9.24 at all, so the first
+    the feature available. (Ghostscript 9.24 has its own problems is blacklisted.)
-    version that allows JPEG passthrough is 9.25.
-
     """
     return version() >= '9.24'
 
@@ -254,6 +252,8 @@ def generate_pdfa(
         raise SubprocessOutputError('Ghostscript PDF/A rendering failed') from e
     else:
         stderr = p.stderr
+        # If there is an error we log the whole stderr, except for filtering
+        # duplicates.
         if _gs_error_reported(stderr):
             last_part = None
             repcount = 0
@@ -266,11 +266,3 @@ def generate_pdfa(
                 else:
                     repcount += 1
                 last_part = part
-        elif 'overprint mode not set' in stderr:
-            # Unless someone is going to print PDF/A documents on a
-            # magical sRGB printer I can't see the removal of overprinting
-            # being a problem....
-            log.debug(
-                "Ghostscript had to remove PDF 'overprinting' from the "
-                "input file to complete PDF/A conversion. "
-            )

src/ocrmypdf/_exec/pngquant.py

@@ -9,10 +9,8 @@
 
 from contextlib import contextmanager
 from io import BytesIO
-from os import fspath
 from pathlib import Path
 from subprocess import PIPE
-from tempfile import NamedTemporaryFile
 
 from PIL import Image
 

src/ocrmypdf/_exec/tesseract.py

@@ -14,7 +14,7 @@ from collections import namedtuple
 from os import fspath
 from pathlib import Path
 from subprocess import PIPE, STDOUT, CalledProcessError, TimeoutExpired
-from typing import List
+from typing import List, Optional
 
 from PIL import Image
 
@@ -118,7 +118,7 @@ def get_languages():
     return set(lang.strip() for lang in rest)
 
 
-def tess_base_args(langs: List[str], engine_mode: int) -> List[str]:
+def tess_base_args(langs: List[str], engine_mode: Optional[int]) -> List[str]:
     args = ['tesseract']
     if langs:
         args.extend(['-l', '+'.join(langs)])
@@ -127,7 +127,7 @@ def tess_base_args(langs: List[str], engine_mode: int) -> List[str]:
     return args
 
 
-def get_orientation(input_file: Path, engine_mode: int, timeout: float):
+def get_orientation(input_file: Path, engine_mode: Optional[int], timeout: float):
     args_tesseract = tess_base_args(['osd'], engine_mode) + [
         '--psm',
         '0',

src/ocrmypdf/_jobcontext.py

@@ -10,15 +10,22 @@ import shutil
 import sys
 from argparse import Namespace
 from copy import copy
-from io import IOBase
 from pathlib import Path
 from typing import Iterator
 
+from pluggy import PluginManager
+
 from ocrmypdf.pdfinfo import PdfInfo
+from ocrmypdf.pdfinfo.info import PageInfo
 
 
 class PdfContext:
-    """Holds our context for a particular run of the pipeline"""
+    """Holds the context for a particular run of the pipeline."""
+
+    options: Namespace  #: The specified options for processing this PDF.
+    origin: Path  #: The filename of the original input file.
+    pdfinfo: PdfInfo  #: Detailed data for this PDF.
+    plugin_manager: PluginManager  #: PluginManager for processing the current PDF.
 
     def __init__(
         self,
@@ -35,21 +42,33 @@ class PdfContext:
         self.plugin_manager = plugin_manager
 
     def get_path(self, name: str) -> Path:
+        """Generate a ``Path`` for an intermediate file involved in processing.
+
+        The path will be in a temporary folder that is common for all processing
+        of this particular PDF.
+        """
         return self.work_folder / name
 
     def get_page_contexts(self) -> Iterator['PageContext']:
+        """Get all ``PageContext`` for this PDF."""
         npages = len(self.pdfinfo)
         for n in range(npages):
             yield PageContext(self, n)
 
 
 class PageContext:
-    """Holds our context for a page
+    """Holds our context for a page.
 
     Must be pickable, so stores only intrinsic/simple data elements or those
-    capable of their serializing themselves via __getstate__.
+    capable of their serializing themselves via ``__getstate__``.
     """
 
+    options: Namespace  #: The specified options for processing this PDF.
+    origin: Path  #: The filename of the original input file.
+    pageno: int  #: This page number (zero-based).
+    pageinfo: PageInfo  #: Information on this page.
+    plugin_manager: PluginManager  #: PluginManager for processing the current PDF.
+
     def __init__(self, pdf_context: PdfContext, pageno):
         self.work_folder = pdf_context.work_folder
         self.origin = pdf_context.origin
@@ -59,6 +78,11 @@ class PageContext:
         self.plugin_manager = pdf_context.plugin_manager
 
     def get_path(self, name: str) -> Path:
+        """Generate a ``Path`` for a file that is part of processing this page.
+
+        The path will be based in a common temporary folder and have a prefix based
+        on the page number.
+        """
         return self.work_folder / ("%06d_%s" % (self.pageno + 1, name))
 
     def __getstate__(self):

src/ocrmypdf/_pipeline.py

@@ -206,17 +206,21 @@ def validate_pdfinfo_options(context: PdfContext):
     context.plugin_manager.hook.validate(pdfinfo=pdfinfo, options=options)
 
 
+def _vector_page_dpi(pageinfo):
+    return VECTOR_PAGE_DPI if pageinfo.has_vector or pageinfo.has_text else 0.0
+
+
 def get_page_dpi(pageinfo, options):
     "Get the DPI when nonsquare DPI is tolerable"
     xres = max(
         pageinfo.dpi.x or VECTOR_PAGE_DPI,
         options.oversample or 0.0,
-        VECTOR_PAGE_DPI if pageinfo.has_vector else 0.0,
+        _vector_page_dpi(pageinfo),
     )
     yres = max(
         pageinfo.dpi.y or VECTOR_PAGE_DPI,
         options.oversample or 0,
-        VECTOR_PAGE_DPI if pageinfo.has_vector else 0.0,
+        _vector_page_dpi(pageinfo),
     )
     return Resolution(float(xres), float(yres))
 
@@ -230,7 +234,7 @@ def get_page_square_dpi(pageinfo, options) -> Resolution:
         max(
             (xres * userunit) or VECTOR_PAGE_DPI,
             (yres * userunit) or VECTOR_PAGE_DPI,
-            VECTOR_PAGE_DPI if pageinfo.has_vector else 0.0,
+            _vector_page_dpi(pageinfo),
             options.oversample or 0.0,
         )
     )
@@ -243,7 +247,7 @@ def get_canvas_square_dpi(pageinfo, options) -> Resolution:
         max(
             (pageinfo.dpi.x) or VECTOR_PAGE_DPI,
             (pageinfo.dpi.y) or VECTOR_PAGE_DPI,
-            VECTOR_PAGE_DPI if pageinfo.has_vector else 0.0,
+            _vector_page_dpi(pageinfo),
             options.oversample or 0.0,
         )
     )
@@ -580,7 +584,9 @@ def create_visible_page_jpg(image: Path, page_context: PageContext) -> Path:
     return output_file
 
 
-def create_pdf_page_from_image(image: Path, page_context: PageContext):
+def create_pdf_page_from_image(
+    image: Path, page_context: PageContext, orientation_correction
+):
     # We rasterize a square DPI version of each page because most image
     # processing tools don't support rectangular DPI. Use the square DPI as it
     # accurately describes the image. It would be possible to resample the image
@@ -588,16 +594,23 @@ def create_pdf_page_from_image(image: Path, page_context: PageContext):
     # except that the hocr renderer does not understand non-square DPI. The
     # sandwich renderer would be fine.
     output_file = page_context.get_path('visible.pdf')
-    dpi = get_page_square_dpi(page_context.pageinfo, page_context.options)
+
-    layout_fun = img2pdf.get_fixed_dpi_layout_fun(dpi)
+    pageinfo = page_context.pageinfo
+    pagesize = 72.0 * float(pageinfo.width_inches), 72.0 * float(pageinfo.height_inches)
+    effective_rotation = (pageinfo.rotation - orientation_correction) % 360
+    if effective_rotation % 180 == 90:
+        pagesize = pagesize[1], pagesize[0]
 
     # This create a single page PDF
     with open(image, 'rb') as imfile, open(output_file, 'wb') as pdf:
         log.debug('convert')
+
+        layout_fun = img2pdf.get_layout_fun(pagesize)
         img2pdf.convert(
             imfile, with_pdfrw=False, layout_fun=layout_fun, outputstream=pdf
         )
         log.debug('convert done')
+
     return output_file
 
 
@@ -726,6 +739,24 @@ def should_linearize(working_file: Path, context: PdfContext):
     return False
 
 
+def get_pdf_save_settings(output_type: str):
+    if output_type == 'pdfa-1':
+        # Trigger recompression to ensure object streams are removed, because
+        # Acrobat complains about them in PDF/A-1b validation.
+        return dict(
+            preserve_pdfa=True,
+            compress_streams=True,
+            stream_decode_level=pikepdf.StreamDecodeLevel.generalized,
+            object_stream_mode=pikepdf.ObjectStreamMode.disable,
+        )
+    else:
+        return dict(
+            preserve_pdfa=True,
+            compress_streams=True,
+            object_stream_mode=(pikepdf.ObjectStreamMode.generate),
+        )
+
+
 def metadata_fixup(working_file: Path, context: PdfContext):
     output_file = context.get_path('metafix.pdf')
     options = context.options
@@ -756,25 +787,21 @@ def metadata_fixup(working_file: Path, context: PdfContext):
             if 'xmp:CreateDate' not in meta:
                 meta['xmp:CreateDate'] = meta.get('xmp:ModifyDate', '')
 
-            # Ghostscript likes to set title to Untitled if omitted from input.
+            with original.open_metadata(
-            # Reverse this, because PDF/A TechNote 0003:Metadata in PDF/A-1
+                set_pikepdf_as_editor=False, update_docinfo=False, strict=False
-            # and the XMP Spec do not make this recommendation.
+            ) as meta_original:
-            if meta.get('dc:title') == 'Untitled':
+                if meta.get('dc:title') == 'Untitled':
-                with original.open_metadata(
+                    # Ghostscript likes to set title to Untitled if omitted from input.
-                    set_pikepdf_as_editor=False, update_docinfo=False
+                    # Reverse this, because PDF/A TechNote 0003:Metadata in PDF/A-1
-                ) as original_meta:
+                    # and the XMP Spec do not make this recommendation.
-                    if 'dc:title' not in original_meta:
+                    if 'dc:title' not in meta_original:
                         del meta['dc:title']
-
+                missing = set(meta_original.keys()) - set(meta.keys())
-            meta_original = original.open_metadata()
+                report_on_metadata(missing)
-            missing = set(meta_original.keys()) - set(meta.keys())
-            report_on_metadata(missing)
 
         pdf.save(
             output_file,
-            compress_streams=True,
+            **get_pdf_save_settings(options.output_type),
-            preserve_pdfa=True,
-            object_stream_mode=pikepdf.ObjectStreamMode.generate,
             linearize=(  # Don't linearize if optimize() will be linearizing too
                 should_linearize(working_file, context)
                 if options.optimize == 0
@@ -788,20 +815,34 @@ def metadata_fixup(working_file: Path, context: PdfContext):
 def optimize_pdf(input_file: Path, context: PdfContext):
     output_file = context.get_path('optimize.pdf')
     save_settings = dict(
-        compress_streams=True,
-        preserve_pdfa=True,
-        object_stream_mode=pikepdf.ObjectStreamMode.generate,
         linearize=should_linearize(input_file, context),
+        **get_pdf_save_settings(context.options.output_type),
     )
     optimize(input_file, output_file, context, save_settings)
     return output_file
 
 
+def enumerate_compress_ranges(iterable):
+    skipped_from = None
+    for index, txt_file in enumerate(iterable):
+        index += 1
+        if txt_file:
+            if skipped_from is not None:
+                yield (skipped_from, index - 1), None
+                skipped_from = None
+            yield (index, index), txt_file
+        else:
+            if skipped_from is None:
+                skipped_from = index
+    if skipped_from is not None:
+        yield (skipped_from, index), None
+
+
 def merge_sidecars(txt_files: Iterable[Optional[Path]], context: PdfContext):
     output_file = context.get_path('sidecar.txt')
     with open(output_file, 'w', encoding="utf-8") as stream:
-        for page_num, txt_file in enumerate(txt_files):
+        for (frm, to), txt_file in enumerate_compress_ranges(txt_files):
-            if page_num != 0:
+            if frm != 1:
                 stream.write('\f')  # Form feed between pages
             if txt_file:
                 with open(txt_file, 'r', encoding="utf-8") as in_:
@@ -814,7 +855,11 @@ def merge_sidecars(txt_files: Iterable[Optional[Path]], context: PdfContext):
                     else:
                         stream.write(txt)
             else:
-                stream.write(f'[OCR skipped on page {(page_num + 1)}]')
+                if frm != to:
+                    pages = f'{frm}-{to}'
+                else:
+                    pages = f'{frm}'
+                stream.write(f'[OCR skipped on page(s) {pages}]')
     return output_file
 
 

src/ocrmypdf/_plugin_manager.py

@@ -8,13 +8,14 @@
 import argparse
 import importlib
 import importlib.util
+import pkgutil
 import sys
-from functools import partial
 from pathlib import Path
-from typing import Callable, List, Tuple, Union
+from typing import List, Tuple, Union
 
 import pluggy
 
+import ocrmypdf.builtin_plugins
 from ocrmypdf import pluginspec
 from ocrmypdf.cli import get_parser, plugins_only_parser
 
@@ -30,62 +31,69 @@ class OcrmypdfPluginManager(pluggy.PluginManager):
     """
 
     def __init__(
-        self, *args, setup_func: Callable[[pluggy.PluginManager], None], **kwargs
+        self,
+        *args,
+        plugins: List[Union[str, Path]],
+        builtins: bool = True,
+        **kwargs,
     ):
-        self._init_args = args
+        self.__init_args = args
-        self._setup_func = setup_func
+        self.__init_kwargs = kwargs
-        self._init_kwargs = kwargs
+        self.__plugins = plugins
+        self.__builtins = builtins
         super().__init__(*args, **kwargs)
-        setup_func(self)
+        self.setup_plugins()
 
     def __getstate__(self):
         state = dict(
-            _init_args=self._init_args,
+            init_args=self.__init_args,
-            _setup_func=self._setup_func,
+            plugins=self.__plugins,
-            _init_kwargs=self._init_kwargs,
+            builtins=self.__builtins,
+            init_kwargs=self.__init_kwargs,
         )
         return state
 
     def __setstate__(self, state):
         self.__init__(
-            *state['_init_args'],
+            *state['init_args'],
-            setup_func=state['_setup_func'],
+            plugins=state['plugins'],
-            **state['_init_kwargs'],
+            builtins=state['builtins'],
+            **state['init_kwargs'],
         )
 
+    def setup_plugins(self):
+        self.add_hookspecs(pluginspec)
 
-def _setup_plugins(
+        # 1. Register builtins
-    pm: pluggy.PluginManager, plugins: List[Union[str, Path]], builtins: bool = True
+        if self.__builtins:
-):
+            for module in pkgutil.iter_modules(ocrmypdf.builtin_plugins.__path__):
-    pm.add_hookspecs(pluginspec)
+                name = f'ocrmypdf.builtin_plugins.{module.name}'
+                module = importlib.import_module(name)
+                self.register(module)
 
-    all_plugins: List[Union[str, Path]] = []
+        # 2. Register setuptools plugins
-    if builtins:
+        self.load_setuptools_entrypoints('ocrmypdf')
-        all_plugins.extend(
+
-            [
+        # 3. Register plugins specified on command line
-                'ocrmypdf.builtin_plugins.ghostscript',
+        for name in self.__plugins:
-                'ocrmypdf.builtin_plugins.tesseract_ocr',
+            if isinstance(name, Path) or name.endswith('.py'):
-            ]
+                # Import by filename
-        )
+                module_name = Path(name).stem
-    all_plugins.extend(plugins)
+                spec = importlib.util.spec_from_file_location(module_name, name)
-    for name in all_plugins:
+                module = importlib.util.module_from_spec(spec)
-        if isinstance(name, Path) or name.endswith('.py'):
+                sys.modules[module_name] = module
-            # Import by filename
+                spec.loader.exec_module(module)
-            module_name = Path(name).stem
+            else:
-            spec = importlib.util.spec_from_file_location(module_name, name)
+                # Import by dotted module name
-            module = importlib.util.module_from_spec(spec)
+                module = importlib.import_module(name)
-            sys.modules[module_name] = module
+            self.register(module)
-            spec.loader.exec_module(module)
-        else:
-            # Import by dotted module name
-            module = importlib.import_module(name)
-        pm.register(module)
 
 
 def get_plugin_manager(plugins: List[Union[str, Path]], builtins=True):
     pm = OcrmypdfPluginManager(
         project_name='ocrmypdf',
-        setup_func=partial(_setup_plugins, plugins=plugins, builtins=builtins),
+        plugins=plugins,
+        builtins=builtins,
     )
     return pm
 

src/ocrmypdf/_sync.py

@@ -204,7 +204,7 @@ def exec_page_sync(page_context: PageContext):
         if filtered_image:
             visible_image_out = filtered_image
         pdf_page_from_image_out = create_pdf_page_from_image(
-            visible_image_out, page_context
+            visible_image_out, page_context, orientation_correction
         )
 
     if options.pdf_renderer.startswith('hocr'):
@@ -302,7 +302,7 @@ def exec_concurrent(context: PdfContext):
     copy_final(pdf, options.output_file, context)
 
 
-def configure_debug_logging(log_filename, prefix: str = ''):
+def configure_debug_logging(log_filename: Path, prefix: str = ''):
     """
     Create a debug log file at a specified location.
 
@@ -338,7 +338,11 @@ def run_pipeline(options, *, plugin_manager, api=False):
         and not api
     ):
         # Debug log for command line interface only with verbose output
-        debug_log_handler = configure_debug_logging(Path(work_folder) / "debug.log")
+        # See https://github.com/pytest-dev/pytest/issues/5502 for why we skip this
+        # when pytest is running
+        debug_log_handler = configure_debug_logging(
+            Path(work_folder) / "debug.log"
+        )  # pragma: no cover
 
     pikepdf_enable_mmap()
 

src/ocrmypdf/_validation.py

@@ -112,6 +112,10 @@ def check_options_sidecar(options):
                 "--sidecar filename must be specified when output file is stdout."
             )
         options.sidecar = options.output_file + '.txt'
+    if options.sidecar == options.input_file or options.sidecar == options.output_file:
+        raise BadArgsError(
+            "--sidecar file must be different from the input and output files"
+        )
 
 
 def check_options_preprocessing(options):
@@ -133,7 +137,7 @@ def check_options_preprocessing(options):
                     options.unpaper_args
                 )
         except Exception as e:
-            raise BadArgsError(str(e))
+            raise BadArgsError("--unpaper-args: " + str(e)) from e
 
 
 def _pages_from_ranges(ranges: str) -> Set[int]:
@@ -184,8 +188,6 @@ def check_options_ocr_behavior(options):
     )
     if exclusive_options >= 2:
         raise BadArgsError("Choose only one of --force-ocr, --skip-text, --redo-ocr.")
-    if options.pages and options.sidecar:
-        raise BadArgsError("--pages and --sidecar are mutually exclusive")
     if options.pages:
         options.pages = _pages_from_ranges(options.pages)
 

src/ocrmypdf/api.py

@@ -8,6 +8,7 @@
 import logging
 import os
 import sys
+import threading
 from enum import IntEnum
 from io import IOBase
 from pathlib import Path
@@ -30,6 +31,8 @@ except ModuleNotFoundError:
 StrPath = Union[os.PathLike, AnyStr]
 PathOrIO = Union[BinaryIO, StrPath]
 
+_api_lock = threading.Lock()
+
 
 class Verbosity(IntEnum):
     """Verbosity level for configure_logging."""
@@ -306,12 +309,18 @@ def ocr(  # pylint: disable=unused-argument
 
     parser = get_parser()
     create_options_kwargs['parser'] = parser
-    plugin_manager = get_plugin_manager(plugins)
-    plugin_manager.hook.add_options(parser=parser)  # pylint: disable=no-member
 
-    if 'verbose' in kwargs:
+    with _api_lock:
-        warn("ocrmypdf.ocr(verbose=) is ignored. Use ocrmypdf.configure_logging().")
+        # We can't allow multiple ocrmypdf.ocr() threads to run in parallel, because
+        # they might install different plugins, and generally speaking we have areas
+        # of code that use global state.
 
-    options = create_options(**create_options_kwargs)
+        plugin_manager = get_plugin_manager(plugins)
-    check_options(options, plugin_manager)
+        plugin_manager.hook.add_options(parser=parser)  # pylint: disable=no-member
-    return run_pipeline(options=options, plugin_manager=plugin_manager, api=True)
+
+        if 'verbose' in kwargs:
+            warn("ocrmypdf.ocr(verbose=) is ignored. Use ocrmypdf.configure_logging().")
+
+        options = create_options(**create_options_kwargs)
+        check_options(options, plugin_manager)
+        return run_pipeline(options=options, plugin_manager=plugin_manager, api=True)

src/ocrmypdf/cli.py

@@ -167,7 +167,8 @@ Online documentation is located at:
         metavar='FILE',
         help="Generate sidecar text files that contain the same text recognized "
         "by Tesseract. This may be useful for building a OCR text database. "
-        "If FILE is omitted, the sidecar file be named {output_file}.txt "
+        "If FILE is omitted, the sidecar file be named {output_file}.txt; the next "
+        "argument must NOT be the name of the input PDF. "
         "If FILE is set to '-', the sidecar is written to stdout (a "
         "convenient way to preview OCR quality). The output file and sidecar "
         "may not both use stdout at the same time.",

src/ocrmypdf/helpers.py

@@ -17,7 +17,7 @@ from functools import wraps
 from io import StringIO
 from math import isclose
 from pathlib import Path
-from typing import Any, Sequence, TypeVar
+from typing import Any, Sequence
 
 import pikepdf
 
@@ -63,8 +63,13 @@ class NeverRaise(Exception):
 
 
 def safe_symlink(input_file: os.PathLike, soft_link_name: os.PathLike):
-    """
+    """Create a symbolic link at ``soft_link_name``, which references ``input_file``.
-    Helper function: relinks soft symbolic link if necessary
+
+    Think of this as copying ``input_file`` to ``soft_link_name`` with less overhead.
+
+    Use symlinks safely. Self-linking loops are prevented. On Windows, file copy is
+    used since symlinks may require administrator privileges. An existing link at the
+    destination is removed.
     """
     input_file = os.fspath(input_file)
     soft_link_name = os.fspath(soft_link_name)
@@ -72,8 +77,8 @@ def safe_symlink(input_file: os.PathLike, soft_link_name: os.PathLike):
     # Guard against soft linking to oneself
     if input_file == soft_link_name:
         log.warning(
-            "No symbolic link made. You are using "
+            "No symbolic link created. You are using  the original data directory "
-            "the original data directory as the working directory."
+            "as the working directory."
         )
         return
 
@@ -114,7 +119,7 @@ def is_iterable_notstr(thing: Any) -> bool:
 
 
 def monotonic(L: Sequence) -> bool:
-    """Does list increase monotonically?"""
+    """Does this sequence increase monotonically?"""
     return all(b > a for a, b in zip(L, L[1:]))
 
 
@@ -173,51 +178,49 @@ def is_file_writable(test_file: os.PathLike) -> bool:
 def check_pdf(input_file: Path) -> bool:
     """Check if a PDF complies with the PDF specification.
 
-    Checks for proper formatting and proper linearization.
+    Checks for proper formatting and proper linearization. Uses pikepdf (which in
+    turn, uses QPDF) to perform the checks.
     """
-    pdf = None
     try:
         pdf = pikepdf.open(input_file)
     except pikepdf.PdfError as e:
         log.error(e)
         return False
     else:
-        messages = pdf.check()
+        with pdf:
-        for msg in messages:
+            messages = pdf.check()
-            if 'error' in msg.lower():
+            for msg in messages:
-                log.error(msg)
+                if 'error' in msg.lower():
+                    log.error(msg)
+                else:
+                    log.warning(msg)
+
+            sio = StringIO()
+            linearize_msgs = ''
+            try:
+                # If linearization is missing entirely, we do not complain. We do
+                # complain if linearization is present but incorrect.
+                pdf.check_linearization(sio)
+            except RuntimeError:
+                pass
+            except (  # Workaround for a problematic pikepdf version
+                getattr(pikepdf, 'ForeignObjectError')
+                if pikepdf.__version__ == '2.1.0'
+                else NeverRaise
+            ):
+                pass
             else:
-                log.warning(msg)
+                linearize_msgs = sio.getvalue()
+                if linearize_msgs:
+                    log.warning(linearize_msgs)
 
-        sio = StringIO()
+            if not messages and not linearize_msgs:
-        linearize_msgs = ''
+                return True
-        try:
+            return False
-            # If linearization is missing entirely, we do not complain. We do
-            # complain if linearization is present but incorrect.
-            pdf.check_linearization(sio)
-        except RuntimeError:
-            pass
-        except (
-            getattr(pikepdf, 'ForeignObjectError')
-            if pikepdf.__version__ == '2.1.0'  # This version may throw wrong exception
-            else NeverRaise
-        ):
-            pass
-        else:
-            linearize_msgs = sio.getvalue()
-            if linearize_msgs:
-                log.warning(linearize_msgs)
-
-        if not messages and not linearize_msgs:
-            return True
-        return False
-    finally:
-        if pdf:
-            pdf.close()
 
 
 def clamp(n, smallest, largest):  # mypy doesn't understand types for this
-    """Clamps the value of n to between smallest and largest."""
+    """Clamps the value of ``n`` to between ``smallest`` and ``largest``."""
     return max(smallest, min(n, largest))
 
 
@@ -235,7 +238,7 @@ def pikepdf_enable_mmap():
 
 
 def deprecated(func):
-    """Warn that function is deprecated"""
+    """Warn that function is deprecated."""
 
     @wraps(func)
     def new_func(*args, **kwargs):

src/ocrmypdf/hocrtransform.py

@@ -31,11 +31,10 @@
 import argparse
 import os
 import re
-from collections import namedtuple
 from itertools import chain
 from math import atan, cos, sin
 from pathlib import Path
-from typing import Optional, Tuple, Union
+from typing import Any, NamedTuple, Optional, Tuple, Union
 from xml.etree import ElementTree
 
 from reportlab.lib.colors import black, cyan, magenta, red
@@ -44,7 +43,14 @@ from reportlab.pdfgen.canvas import Canvas
 
 Element = ElementTree.Element
 
-Rect = namedtuple('Rect', ['x1', 'y1', 'x2', 'y2'])
+
+class Rect(NamedTuple):  # pylint: disable=inherit-non-class
+    """A rectangle for managing PDF coordinates."""
+
+    x1: Any
+    y1: Any
+    x2: Any
+    y2: Any
 
 
 class HocrTransformError(Exception):

src/ocrmypdf/leptonica.py

@@ -170,20 +170,6 @@ tls = threading.local()
 tls.trap = None
 
 
-@ffi.callback("void(char *)")
-def _stderr_handler(cstr):
-    msg = ffi.string(cstr).decode(errors='replace')
-    if msg.startswith("Error"):
-        logger.error(msg)
-    elif msg.startswith("Warning"):
-        logger.warning(msg)
-    else:
-        logger.debug(msg)
-    if tls.trap is not None:
-        tls.trap.append(msg)
-    return
-
-
 class _LeptonicaErrorTrap_Queue:
     def __init__(self):
         self.queue = deque()
@@ -213,9 +199,25 @@ class _LeptonicaErrorTrap_Queue:
 
 
 try:
+
+    @ffi.callback("void(char *)")
+    def _stderr_handler(cstr):
+        msg = ffi.string(cstr).decode(errors='replace')
+        if msg.startswith("Error"):
+            logger.error(msg)
+        elif msg.startswith("Warning"):
+            logger.warning(msg)
+        else:
+            logger.debug(msg)
+        if tls.trap is not None:
+            tls.trap.append(msg)
+        return
+
     lept.leptSetStderrHandler(_stderr_handler)
-except ffi.error:
+except (ffi.error, MemoryError):
     # Pre-1.79 Leptonica does not have leptSetStderrHandler
+    # And some platforms, notably Apple ARM 64, do not allow the write+execute
+    # memory needed to set up the callback function.
     _LeptonicaErrorTrap = _LeptonicaErrorTrap_Redirect
 else:
     # 1.79 have this new symbol

src/ocrmypdf/optimize.py

@@ -10,11 +10,9 @@ import sys
 import tempfile
 from collections import defaultdict
 from functools import partial
-from io import BytesIO
 from os import fspath
 from pathlib import Path
 from typing import (
-    Any,
     Callable,
     Dict,
     Iterator,
@@ -49,7 +47,7 @@ DEFAULT_PNG_QUALITY = 70
 Xref = NewType('Xref', int)
 
 
-class XrefExt(NamedTuple):
+class XrefExt(NamedTuple):  # pylint: disable=inherit-non-class
     xref: Xref
     ext: str
 
@@ -78,6 +76,9 @@ def extract_image_filter(
     if image.Length < 100:
         log.debug(f"Skipping small image, xref {xref}")
         return None
+    if image.Width < 8 or image.Height < 8:  # Issue 732
+        log.debug(f"Skipping oddly sized image, xref {xref}")
+        return None
 
     pim = PdfImage(image)
 
@@ -94,6 +95,10 @@ def extract_image_filter(
         log.debug(f"Skipping JPEG2000 iamge, xref {xref}")
         return None  # Don't do JPEG2000
 
+    if filtdp[0] == Name.CCITTFaxDecode and filtdp[1].get('/K', 0) >= 0:
+        log.debug(f"Skipping CCITT Group 3 image, xref {xref}")
+        return None  # pikepdf doesn't support Group 3 yet
+
     if Name.Decode in image:
         log.debug(f"Skipping image with Decode table, xref {xref}")
         return None  # Don't mess with custom Decode tables
@@ -152,11 +157,6 @@ def extract_image_generic(
     if pim.bits_per_component == 1:
         return None
 
-    try:
-        pim.indexed  # pikepdf 1.6.3 can't handle [/Indexed [/Array...]]
-    except NotImplementedError:
-        return None
-
     if filtdp[0] == Name.DCTDecode and options.optimize >= 2:
         # This is a simple heuristic derived from some training data, that has
         # about a 70% chance of guessing whether the JPEG is high quality,
@@ -212,7 +212,10 @@ def extract_image_generic(
 
 
 def extract_images(
-    pike: Pdf, root: Path, options, extract_fn: Callable[..., Optional[XrefExt]],
+    pike: Pdf,
+    root: Path,
+    options,
+    extract_fn: Callable[..., Optional[XrefExt]],
 ) -> Iterator[Tuple[int, XrefExt]]:
     """Extract image using extract_fn
 
@@ -259,8 +262,8 @@ def extract_images(
             result = extract_fn(
                 pike=pike, root=root, image=image, xref=xref, options=options
             )
-        except Exception as e:  # pylint: disable=broad-except
+        except Exception:  # pylint: disable=broad-except
-            log.debug("Image xref %s, error %s", xref, repr(e))
+            log.exception(f"While extracting image xref {xref}, an error occurred")
             errors += 1
         else:
             if result:
@@ -498,7 +501,7 @@ def transcode_pngs(
 
 
 @deprecated
-def rewrite_png_as_g4(pike: Pdf, im_obj: Object, compdata) -> None:
+def rewrite_png_as_g4(pike: Pdf, im_obj: Object, compdata) -> None:  # pragma: no cover
     im_obj.BitsPerComponent = 1
     im_obj.Width = compdata.w
     im_obj.Height = compdata.h
@@ -519,7 +522,7 @@ def rewrite_png_as_g4(pike: Pdf, im_obj: Object, compdata) -> None:
 
 
 @deprecated
-def rewrite_png(pike: Pdf, im_obj: Object, compdata) -> None:
+def rewrite_png(pike: Pdf, im_obj: Object, compdata) -> None:  # pragma: no cover
     # When a PNG is inserted into a PDF, we more or less copy the IDAT section from
     # the PDF and transfer the rest of the PNG headers to PDF image metadata.
     # One thing we have to do is tell the PDF reader whether a predictor was used
@@ -564,10 +567,10 @@ def rewrite_png(pike: Pdf, im_obj: Object, compdata) -> None:
         # ncolors == 0 means we are using a colorspace without a palette
         if compdata.spp == 1:
             cs = Name.DeviceGray
-        elif compdata.spp == 3:
-            cs = Name.DeviceRGB
         elif compdata.spp == 4:
             cs = Name.DeviceCMYK
+        else:  # spp == 3
+            cs = Name.DeviceRGB
     im_obj.ColorSpace = cs
     im_obj.write(compdata.read(), filter=Name.FlateDecode, decode_parms=dparms)
 

src/ocrmypdf/pdfa.py

@@ -112,7 +112,7 @@ def generate_pdfa_ps(target_filename: Path, icc: str = 'sRGB'):
 
 
 def file_claims_pdfa(filename: Path):
-    """Determines if the file claims to be PDF/A compliant
+    """Determines if the file claims to be PDF/A compliant.
 
     This only checks if the XMP metadata contains a PDF/A marker. It does not
     do full PDF/A validation.

src/ocrmypdf/pdfinfo/info.py

@@ -15,14 +15,14 @@ from functools import partial
 from math import hypot, isclose
 from os import PathLike
 from pathlib import Path
-from typing import Any, Container, Dict, Iterator, List, Optional, Tuple, Union
+from typing import Container, Iterator, Optional, Tuple, Union
 from warnings import warn
 
 import pikepdf
 from pikepdf import Object, Pdf, PdfMatrix
 
 from ocrmypdf._concurrent import exec_progress_pool
-from ocrmypdf.exceptions import EncryptedPdfError
+from ocrmypdf.exceptions import EncryptedPdfError, InputFileError
 from ocrmypdf.helpers import Resolution, available_cpu_count, pikepdf_enable_mmap
 from ocrmypdf.pdfinfo.layout import get_page_analysis, get_text_boxes
 
@@ -598,6 +598,8 @@ def _pdf_pageinfo_concurrent(
 
     def update_pageinfo(result, pbar):
         page = result
+        if not page:
+            raise InputFileError("Could read a page in the PDF")
         pages[page.pageno] = page
         pbar.update()
 

src/ocrmypdf/pdfinfo/layout.py

@@ -21,7 +21,7 @@ from pdfminer.pdffont import PDFSimpleFont, PDFUnicodeNotDefined
 from pdfminer.pdfpage import PDFPage
 from pdfminer.utils import bbox2str, matrix2str
 
-from ocrmypdf.exceptions import EncryptedPdfError
+from ocrmypdf.exceptions import EncryptedPdfError, InputFileError
 
 STRIP_NAME = re.compile(r'[0-9]+')
 
@@ -236,8 +236,13 @@ def get_page_analysis(infile, pageno, pscript5_mode):
 
     try:
         with Path(infile).open('rb') as f:
-            page = PDFPage.get_pages(f, pagenos=[pageno], maxpages=0)
+            page_iter = PDFPage.get_pages(f, pagenos=[pageno], maxpages=0)
-            interp.process_page(next(page))
+            page = next(page_iter, None)
+            if page is None:
+                raise InputFileError(
+                    f"pdfminer could not process page {pageno} (counting from 0)."
+                )
+            interp.process_page(page)
     except PDFTextExtractionNotAllowed as e:
         raise EncryptedPdfError() from e
     finally:

src/ocrmypdf/subprocess/__init__.py

@@ -77,20 +77,19 @@ def run_polling_stderr(args, *, callback, check=False, env=None, **kwargs):
     args, env, process_log, text = _fix_process_args(args, env, kwargs)
     assert text, "Must use text=True"
 
-    proc = Popen(args, env=env, **kwargs)
+    with Popen(args, env=env, **kwargs) as proc:
+        lines = []
+        while proc.poll() is None:
+            for msg in iter(proc.stderr.readline, ''):
+                if process_log.isEnabledFor(logging.DEBUG):
+                    process_log.debug(msg.strip())
+                callback(msg)
+                lines.append(msg)
+        stderr = ''.join(lines)
 
-    lines = []
+        if check and proc.returncode != 0:
-    while proc.poll() is None:
+            raise CalledProcessError(proc.returncode, args, output=None, stderr=stderr)
-        for msg in iter(proc.stderr.readline, ''):
+        return CompletedProcess(args, proc.returncode, None, stderr=stderr)
-            if process_log.isEnabledFor(logging.DEBUG):
-                process_log.debug(msg.strip())
-            callback(msg)
-            lines.append(msg)
-    stderr = ''.join(lines)
-
-    if check and proc.returncode != 0:
-        raise CalledProcessError(proc.returncode, args, output=None, stderr=stderr)
-    return CompletedProcess(args, proc.returncode, None, stderr=stderr)
 
 
 def _fix_process_args(args, env, kwargs):

tests/conftest.py

@@ -137,16 +137,7 @@ def run_ocrmypdf(input_file, output_file, *args, text=True):
         + [str(input_file), str(output_file)]
     )
 
-    # Tell subprocess where to find coverage.py configuration
-    # This has no unless except when coverage is running
-    # Details: https://coverage.readthedocs.io/en/coverage-5.0/subprocess.html
-    coverage_rc = Path(__file__).parent.parent / '.coveragerc'
     env = os.environ.copy()
-    if coverage_rc.exists():
-        env['COVERAGE_PROCESS_START'] = os.fspath(coverage_rc)
-    elif not running_in_docker():
-        assert False, "could not find .coveragerc"
-
     p = run(
         p_args,
         stdout=PIPE,

tests/plugins/gs_feature_elision.py

@@ -23,21 +23,22 @@ from unittest.mock import patch
 
 from ocrmypdf import hookimpl
 from ocrmypdf.builtin_plugins import ghostscript
-from ocrmypdf.subprocess import run
+from ocrmypdf.subprocess import run_polling_stderr
 
 elision_warning = """GPL Ghostscript 9.20: Setting Overprint Mode to 1
 not permitted in PDF/A-2, overprint mode not set"""
 
 
 def run_append_stderr(*args, **kwargs):
-    proc = run(*args, **kwargs)
+    proc = run_polling_stderr(*args, **kwargs)
-    proc.stderr = b'\n'.join([proc.stderr, elision_warning.encode('utf-8')])
+    proc.stderr += '\n' + elision_warning + '\n'
     return proc
 
 
 @hookimpl
 def generate_pdfa(pdf_pages, pdfmark, output_file, compression, pdf_version, pdfa_part):
-    with patch('ocrmypdf._exec.ghostscript.run', new=run_append_stderr):
+    with patch('ocrmypdf._exec.ghostscript.run_polling_stderr') as mock:
+        mock.side_effect = run_append_stderr
         ghostscript.generate_pdfa(
             pdf_pages=pdf_pages,
             pdfmark=pdfmark,
@@ -47,4 +48,5 @@ def generate_pdfa(pdf_pages, pdfmark, output_file, compression, pdf_version, pdf
             pdfa_part=pdfa_part,
             progressbar_class=None,
         )
-        return output_file
+        mock.assert_called_once()
+    return output_file

tests/plugins/gs_pdfa_failure.py

@@ -39,7 +39,8 @@ def run_rig_args(args, **kwargs):
 
 @hookimpl
 def generate_pdfa(pdf_pages, pdfmark, output_file, compression, pdf_version, pdfa_part):
-    with patch('ocrmypdf._exec.ghostscript.run_polling_stderr', new=run_rig_args):
+    with patch('ocrmypdf._exec.ghostscript.run_polling_stderr') as mock:
+        mock.side_effect = run_rig_args
         ghostscript.generate_pdfa(
             pdf_pages=pdf_pages,
             pdfmark=pdfmark,
@@ -49,4 +50,5 @@ def generate_pdfa(pdf_pages, pdfmark, output_file, compression, pdf_version, pdf
             pdfa_part=pdfa_part,
             progressbar_class=None,
         )
+        mock.assert_called()
         return output_file

tests/plugins/gs_raster_failure.py

@@ -44,7 +44,8 @@ def rasterize_pdf_page(
     rotation=None,
     filter_vector=False,
 ) -> Path:
-    with patch('ocrmypdf._exec.ghostscript.run', new=raise_gs_fail):
+    with patch('ocrmypdf._exec.ghostscript.run') as mock:
+        mock.side_effect = raise_gs_fail
         ghostscript.rasterize_pdf_page(
             input_file=input_file,
             output_file=output_file,
@@ -55,4 +56,5 @@ def rasterize_pdf_page(
             rotation=rotation,
             filter_vector=filter_vector,
         )
+        mock.assert_called()
         return output_file

tests/plugins/gs_render_failure.py

@@ -34,7 +34,8 @@ def raise_gs_fail(*args, **kwargs):
 
 @hookimpl
 def generate_pdfa(pdf_pages, pdfmark, output_file, compression, pdf_version, pdfa_part):
-    with patch('ocrmypdf._exec.ghostscript.run_polling_stderr', new=raise_gs_fail):
+    with patch('ocrmypdf._exec.ghostscript.run_polling_stderr') as mock:
+        mock.side_effect = raise_gs_fail
         ghostscript.generate_pdfa(
             pdf_pages=pdf_pages,
             pdfmark=pdfmark,
@@ -44,4 +45,5 @@ def generate_pdfa(pdf_pages, pdfmark, output_file, compression, pdf_version, pdf
             pdfa_part=pdfa_part,
             progressbar_class=None,
         )
+        mock.assert_called()
         return output_file

tests/plugins/tesseract_badutf8.py

@@ -26,6 +26,7 @@ that is not UTF-8 compatible, so we are forced to check that we can convert it
 and present it to the user.
 """
 
+from contextlib import contextmanager
 from subprocess import CalledProcessError
 from unittest.mock import patch
 
@@ -42,17 +43,25 @@ def bad_utf8(*args, **kwargs):
     )
 
 
+@contextmanager
+def patch_tesseract_run():
+    with patch('ocrmypdf._exec.tesseract.run') as mock:
+        mock.side_effect = bad_utf8
+        yield
+        mock.assert_called()
+
+
 class BadUtf8OcrEngine(TesseractOcrEngine):
     @staticmethod
     def generate_hocr(input_file, output_hocr, output_text, options):
-        with patch('ocrmypdf._exec.tesseract.run', new=bad_utf8):
+        with patch_tesseract_run():
             TesseractOcrEngine.generate_hocr(
                 input_file, output_hocr, output_text, options
             )
 
     @staticmethod
     def generate_pdf(input_file, output_pdf, output_text, options):
-        with patch('ocrmypdf._exec.tesseract.run', new=bad_utf8):
+        with patch_tesseract_run():
             TesseractOcrEngine.generate_pdf(
                 input_file, output_pdf, output_text, options
             )

tests/plugins/tesseract_big_image_error.py

@@ -19,6 +19,7 @@
 # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+from contextlib import contextmanager
 from subprocess import CalledProcessError
 from unittest.mock import patch
 
@@ -35,22 +36,30 @@ def raise_size_exception(*args, **kwargs):
     )
 
 
+@contextmanager
+def patch_tesseract_run():
+    with patch('ocrmypdf._exec.tesseract.run') as mock:
+        mock.side_effect = raise_size_exception
+        yield
+        mock.assert_called()
+
+
 class BigImageErrorOcrEngine(TesseractOcrEngine):
     @staticmethod
     def get_orientation(input_file, options):
-        with patch('ocrmypdf._exec.tesseract.run', new=raise_size_exception):
+        with patch_tesseract_run():
             return TesseractOcrEngine.get_orientation(input_file, options)
 
     @staticmethod
     def generate_hocr(input_file, output_hocr, output_text, options):
-        with patch('ocrmypdf._exec.tesseract.run', new=raise_size_exception):
+        with patch_tesseract_run():
             TesseractOcrEngine.generate_hocr(
                 input_file, output_hocr, output_text, options
             )
 
     @staticmethod
     def generate_pdf(input_file, output_pdf, output_text, options):
-        with patch('ocrmypdf._exec.tesseract.run', new=raise_size_exception):
+        with patch_tesseract_run():
             TesseractOcrEngine.generate_pdf(
                 input_file, output_pdf, output_text, options
             )

tests/plugins/tesseract_crash.py

@@ -20,6 +20,7 @@
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 import signal
+from contextlib import contextmanager
 from subprocess import CalledProcessError
 from unittest.mock import patch
 
@@ -37,22 +38,30 @@ def raise_crash(*args, **kwargs):
     )
 
 
+@contextmanager
+def patch_tesseract_run():
+    with patch('ocrmypdf._exec.tesseract.run') as mock:
+        mock.side_effect = raise_crash
+        yield
+        mock.assert_called()
+
+
 class CrashOcrEngine(TesseractOcrEngine):
     @staticmethod
     def get_orientation(input_file, options):
-        with patch('ocrmypdf._exec.tesseract.run', new=raise_crash):
+        with patch_tesseract_run():
             return TesseractOcrEngine.get_orientation(input_file, options)
 
     @staticmethod
     def generate_hocr(input_file, output_hocr, output_text, options):
-        with patch('ocrmypdf._exec.tesseract.run', new=raise_crash):
+        with patch_tesseract_run():
             TesseractOcrEngine.generate_hocr(
                 input_file, output_hocr, output_text, options
             )
 
     @staticmethod
     def generate_pdf(input_file, output_pdf, output_text, options):
-        with patch('ocrmypdf._exec.tesseract.run', new=raise_crash):
+        with patch_tesseract_run():
             TesseractOcrEngine.generate_pdf(
                 input_file, output_pdf, output_text, options
             )

tests/plugins/tesseract_debug_rotate.py

@@ -0,0 +1,112 @@
+# © 2020 James R. Barlow: github.com/jbarlow83
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+"""Tesseract no-op/fixed rotate plugin
+
+To quickly run tests where getting OCR output is not necessary and we want to test
+the rotation pipeline.
+
+In 'hocr' mode, create a .hocr file that specifies no text found.
+
+In 'pdf' mode, convert the image to PDF using another program.
+
+In orientation check mode, report 0, 90, 180, 270... based on page number.
+"""
+
+import pikepdf
+from PIL import Image
+
+from ocrmypdf import OcrEngine, OrientationConfidence, hookimpl
+from ocrmypdf.helpers import page_number
+
+HOCR_TEMPLATE = '''<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+ <head>
+  <title></title>
+  <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+  <meta name='ocr-system' content='tesseract 4.0.0' />
+  <meta name='ocr-capabilities' content='ocr_page ocr_carea ocr_par ocr_line ocrx_word'/>
+ </head>
+ <body>
+  <div class='ocr_page' id='page_1' title='image "x.tif"; bbox 0 0 {0} {1}; ppageno 0'>
+   <div class='ocr_carea' id='block_1_1' title="bbox 0 1 {0} {1}">
+    <p class='ocr_par' dir='ltr' id='par_1' title="bbox 0 1 {0} {1}">
+     <span class='ocr_line' id='line_1' title="bbox 0 1 {0} {1}"><span class='ocrx_word' id='word_1' title="bbox 0 1 {0} {1}"> </span>
+     </span>
+    </p>
+   </div>
+  </div>
+ </body>
+</html>'''
+
+
+class FixedRotateNoopOcrEngine(OcrEngine):
+    @staticmethod
+    def version():
+        return '4.0.0'
+
+    @staticmethod
+    def creator_tag(options):
+        tag = '-PDF' if options.pdf_renderer == 'sandwich' else ''
+        return f"NO-OP {tag} {FixedRotateNoopOcrEngine.version()}"
+
+    def __str__(self):
+        return f"NO-OP {FixedRotateNoopOcrEngine.version()}"
+
+    @staticmethod
+    def languages(options):
+        return {'eng'}
+
+    @staticmethod
+    def get_orientation(input_file, options):
+        page = page_number(input_file)
+
+        angle = ((page - 1) * 90) % 360
+
+        return OrientationConfidence(angle=angle, confidence=99.9)
+
+    @staticmethod
+    def generate_hocr(input_file, output_hocr, output_text, options):
+        with Image.open(input_file) as im, open(
+            output_hocr, 'w', encoding='utf-8'
+        ) as f:
+            w, h = im.size
+            f.write(HOCR_TEMPLATE.format(str(w), str(h)))
+        with open(output_text, 'w') as f:
+            f.write('')
+
+    @staticmethod
+    def generate_pdf(input_file, output_pdf, output_text, options):
+        with Image.open(input_file) as im:
+            dpi = im.info['dpi']
+            pagesize = im.size[0] / dpi[0], im.size[1] / dpi[1]
+        ptsize = pagesize[0] * 72, pagesize[1] * 72
+        pdf = pikepdf.new()
+        pdf.add_blank_page(page_size=ptsize)
+        pdf.save(output_pdf, static_id=True)
+        output_text.write_text('')
+
+
+@hookimpl
+def get_ocr_engine():
+    return FixedRotateNoopOcrEngine()

tests/test_helpers.py

@@ -36,12 +36,17 @@ class TestSafeSymlink:
 
 
 def test_no_cpu_count(monkeypatch):
+    invoked = False
+
     def cpu_count_raises():
+        nonlocal invoked
+        invoked = True
         raise NotImplementedError()
 
     monkeypatch.setattr(multiprocessing, 'cpu_count', cpu_count_raises)
     with pytest.warns(expected_warning=UserWarning):
         assert helpers.available_cpu_count() == 1
+    assert invoked, "Patched function called during test"
 
 
 def test_deprecated():

tests/test_image_input.py

@@ -65,11 +65,12 @@ def test_cmyk_no_icc(caplog, resources, no_outpdf):
 def test_img2pdf_fails(resources, no_outpdf):
     with patch(
         'ocrmypdf._pipeline.img2pdf.convert', side_effect=img2pdf.ImageOpenError()
-    ):
+    ) as mock:
         rc = run_ocrmypdf_api(
             resources / 'baiona_gray.png', no_outpdf, '--image-dpi', '200'
         )
         assert rc == ocrmypdf.ExitCode.input_file
+        mock.assert_called()
 
 
 def test_jpeg_in_jpeg_out(resources, outpdf):

tests/test_logging.py

@@ -0,0 +1,21 @@
+# © 2021 James R. Barlow: github.com/jbarlow83
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import logging
+
+import pytest
+
+from ocrmypdf._sync import configure_debug_logging
+
+
+def test_debug_logging(tmp_path):
+    # Just exercise the debug logger but don't validate it
+    # See https://github.com/pytest-dev/pytest/issues/5502 for pytest logging quirks
+    prefix = 'test_debug_logging'
+    log = logging.getLogger(prefix)
+    handler = configure_debug_logging(tmp_path / 'test.log', prefix)
+    log.info("test message")
+    log.removeHandler(handler)

tests/test_metadata.py

@@ -306,6 +306,9 @@ def test_kodak_toc(resources, outpdf):
         assert isinstance(p.Root.Outlines.First, pikepdf.Dictionary)
 
 
+@pytest.mark.skipif(
+    pikepdf.__version__ in ('2.2.2', '2.2.3'), reason="Raises wrong warning"
+)
 def test_metadata_fixup_warning(resources, outdir, caplog):
     options = get_parser().parse_args(
         args=['--output-type', 'pdfa-2', 'graph.pdf', 'out.pdf']
@@ -318,7 +321,7 @@ def test_metadata_fixup_warning(resources, outdir, caplog):
     )
     metadata_fixup(working_file=outdir / 'graph.pdf', context=context)
     for record in caplog.records:
-        assert record.levelname != 'WARNING'
+        assert record.levelname != 'WARNING', "Unexpected warning"
 
     # Now add some metadata that will not be copyable
     graph = pikepdf.open(outdir / 'graph.pdf')

tests/test_optimize.py

@@ -21,7 +21,15 @@ from ocrmypdf.helpers import Resolution
 
 check_ocrmypdf = pytest.helpers.check_ocrmypdf  # pylint: disable=e1101
 
+needs_pngquant = pytest.mark.skipif(
+    not pngquant.available(), reason="pngquant not installed"
+)
+needs_jbig2enc = pytest.mark.skipif(
+    not jbig2enc.available(), reason="jbig2enc not installed"
+)
 
+
+@needs_pngquant
 @pytest.mark.parametrize('pdf', ['multipage.pdf', 'palette.pdf'])
 def test_basic(resources, pdf, outpdf):
     infile = resources / pdf
@@ -30,6 +38,7 @@ def test_basic(resources, pdf, outpdf):
     assert 0.98 * Path(outpdf).stat().st_size <= Path(infile).stat().st_size
 
 
+@needs_pngquant
 def test_mono_not_inverted(resources, outdir):
     infile = resources / '2400dpi.pdf'
     opt.main(infile, outdir / 'out.pdf', level=3)
@@ -45,7 +54,7 @@ def test_mono_not_inverted(resources, outdir):
         assert im.getpixel((0, 0)) == 255, "Expected white background"
 
 
-@pytest.mark.skipif(not pngquant.available(), reason='need pngquant')
+@needs_pngquant
 def test_jpg_png_params(resources, outpdf):
     check_ocrmypdf(
         resources / 'crom.png',
@@ -63,7 +72,7 @@ def test_jpg_png_params(resources, outpdf):
     )
 
 
-@pytest.mark.skipif(not jbig2enc.available(), reason='need jbig2enc')
+@needs_jbig2enc
 @pytest.mark.parametrize('lossy', [False, True])
 def test_jbig2_lossy(lossy, resources, outpdf):
     args = [
@@ -95,10 +104,8 @@ def test_jbig2_lossy(lossy, resources, outpdf):
         assert len(pim.decode_parms) == 0
 
 
-@pytest.mark.skipif(
+@needs_pngquant
-    not jbig2enc.available() or not pngquant.available(),
+@needs_jbig2enc
-    reason='need jbig2enc and pngquant',
-)
 def test_flate_to_jbig2(resources, outdir):
     # This test requires an image that pngquant is capable of converting to
     # to 1bpp - so use an existing 1bpp image, convert up, confirm it can
@@ -126,6 +133,7 @@ def test_flate_to_jbig2(resources, outdir):
     assert pim.filters[0] == '/JBIG2Decode'
 
 
+@needs_pngquant
 def test_multiple_pngs(resources, outdir):
     with Path.open(outdir / 'in.pdf', 'wb') as inpdf:
         img2pdf.convert(
@@ -141,7 +149,8 @@ def test_multiple_pngs(resources, outdir):
             draw.rectangle((0, 0, im.width, im.height), fill=128)
             im.save(output_file)
 
-    with patch('ocrmypdf.optimize.pngquant.quantize', new=mockquant):
+    with patch('ocrmypdf.optimize.pngquant.quantize') as mock:
+        mock.side_effect = mockquant
         check_ocrmypdf(
             outdir / 'in.pdf',
             outdir / 'out.pdf',
@@ -155,6 +164,7 @@ def test_multiple_pngs(resources, outdir):
             '--plugin',
             'tests/plugins/tesseract_noop.py',
         )
+        mock.assert_called()
 
     with pikepdf.open(outdir / 'in.pdf') as inpdf, pikepdf.open(
         outdir / 'out.pdf'
@@ -163,3 +173,28 @@ def test_multiple_pngs(resources, outdir):
             inim = next(iter(inpdf.pages[n].images.values()))
             outim = next(iter(outpdf.pages[n].images.values()))
             assert len(outim.read_raw_bytes()) < len(inim.read_raw_bytes()), n
+
+
+def test_optimize_off(resources, outpdf):
+    check_ocrmypdf(
+        resources / 'trivial.pdf',
+        outpdf,
+        '--optimize=0',
+        '--output-type',
+        'pdf',
+        '--plugin',
+        'tests/plugins/tesseract_noop.py',
+    )
+
+
+def test_group3(resources, outdir):
+    with pikepdf.open(resources / 'ccitt.pdf') as pdf:
+        im = pdf.pages[0].Resources.XObject['/Im1']
+        assert (
+            opt.extract_image_filter(pdf, outdir, im, im.objgen[0]) is not None
+        ), "Group 4 should be allowed"
+
+        im.DecodeParms['/K'] = 0
+        assert (
+            opt.extract_image_filter(pdf, outdir, im, im.objgen[0]) is None
+        ), "Group 3 should be disallowed"

tests/test_pdfa.py

@@ -0,0 +1,34 @@
+# © 2021 James R. Barlow: github.com/jbarlow83
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import pikepdf
+import pytest
+
+check_ocrmypdf = pytest.helpers.check_ocrmypdf
+
+
+@pytest.mark.parametrize('optimize', (0, 3))
+@pytest.mark.parametrize('pdfa_level', (1, 2, 3))
+def test_pdfa(resources, outpdf, optimize, pdfa_level):
+    check_ocrmypdf(
+        resources / 'francais.pdf',
+        outpdf,
+        '--plugin',
+        'tests/plugins/tesseract_noop.py',
+        f'--output-type=pdfa-{pdfa_level}',
+        f'--optimize={optimize}',
+    )
+    if pdfa_level in (2, 3):
+        # PDF/A-2 allows ObjStm
+        assert b'/ObjStm' in outpdf.read_bytes()
+    elif pdfa_level == 1:
+        # PDF/A-1 might allow ObjStm, but Acrobat does not approve it, so
+        # we don't use it
+        assert b'/ObjStm' not in outpdf.read_bytes()
+
+    with pikepdf.open(outpdf) as pdf:
+        with pdf.open_metadata() as m:
+            assert m.pdfa_status == f'{pdfa_level}B'

tests/test_pdfinfo.py

@@ -4,28 +4,31 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-
 import pickle
+from io import BytesIO
 from math import isclose
 
 import img2pdf
 import pikepdf
 import pytest
 from PIL import Image
+from reportlab.lib.units import inch
 from reportlab.pdfgen.canvas import Canvas
 
 from ocrmypdf import pdfinfo
+from ocrmypdf.exceptions import InputFileError
 from ocrmypdf.pdfinfo import Colorspace, Encoding
+from ocrmypdf.pdfinfo.layout import PDFPage
 
 # pylint: disable=protected-access
 
 
 def test_single_page_text(outdir):
     filename = outdir / 'text.pdf'
-    pdf = Canvas(str(filename), pagesize=(8 * 72, 6 * 72))
+    pdf = Canvas(str(filename), pagesize=(8 * inch, 6 * inch))
     text = pdf.beginText()
     text.setFont('Helvetica', 12)
-    text.setTextOrigin(1 * 72, 3 * 72)
+    text.setTextOrigin(1 * inch, 3 * inch)
     text.textLine(
         "Methink'st thou art a general offence and every" " man should beat thee."
     )
@@ -42,25 +45,32 @@ def test_single_page_text(outdir):
     assert len(page.images) == 0
 
 
-def test_single_page_image(outdir):
+@pytest.fixture(scope='session')
-    filename = outdir / 'image-mono.pdf'
+def eight_by_eight():
-
-    im_tmp = outdir / 'tmp.png'
     im = Image.new('1', (8, 8), 0)
     for n in range(8):
         im.putpixel((n, n), 1)
-    im.save(str(im_tmp), format='PNG')
+    return im
+
+
+def test_single_page_image(eight_by_eight, outpdf):
+    im = eight_by_eight
+    bio = BytesIO()
+    im.save(bio, format='PNG')
+    bio.seek(0)
 
     imgsize = ((img2pdf.ImgSize.dpi, 8), (img2pdf.ImgSize.dpi, 8))
     layout_fun = img2pdf.get_layout_fun(None, imgsize, None, None, None)
 
-    im_bytes = im_tmp.read_bytes()
+    with outpdf.open('wb') as f:
-    pdf_bytes = img2pdf.convert(
+        img2pdf.convert(
-        im_bytes, producer="img2pdf", with_pdfrw=False, layout_fun=layout_fun
+            bio,
-    )
+            producer="img2pdf",
-    filename.write_bytes(pdf_bytes)
+            with_pdfrw=False,
-
+            layout_fun=layout_fun,
-    info = pdfinfo.PdfInfo(filename)
+            outputstream=f,
+        )
+    info = pdfinfo.PdfInfo(outpdf)
 
     assert len(info) == 1
     page = info[0]
@@ -77,16 +87,12 @@ def test_single_page_image(outdir):
     assert isclose(pdfimage.dpi.y, 8)
 
 
-def test_single_page_inline_image(outdir):
+def test_single_page_inline_image(eight_by_eight, outdir):
     filename = outdir / 'image-mono-inline.pdf'
     pdf = Canvas(str(filename), pagesize=(8 * 72, 6 * 72))
 
-    im = Image.new('1', (8, 8), 0)
-    for n in range(8):
-        im.putpixel((n, n), 1)
-
     # Draw image in a 72x72 pt or 1"x1" area
-    pdf.drawInlineImage(im, 0, 0, width=72, height=72)
+    pdf.drawInlineImage(eight_by_eight, 0, 0, width=72, height=72)
     pdf.showPage()
     pdf.save()
 
@@ -179,3 +185,18 @@ def test_stack_abuse():
     with pytest.warns(None):
         with pytest.raises(RuntimeError):
             pdfinfo.info._interpret_contents(stream)
+
+
+def test_pages_issue700(monkeypatch, resources):
+    def get_no_pages(*args, **kwargs):
+        return iter([])
+
+    monkeypatch.setattr(PDFPage, 'get_pages', get_no_pages)
+
+    with pytest.raises(InputFileError, match="pdfminer"):
+        pdfinfo.PdfInfo(
+            resources / 'cardinal.pdf',
+            detailed_analysis=True,
+            progbar=False,
+            max_workers=1,
+        )

tests/test_pipeline.py

@@ -0,0 +1,150 @@
+# © 2021 James R. Barlow: github.com/jbarlow83
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from unittest.mock import Mock
+
+import pytest
+from PIL import Image
+from reportlab.lib.units import inch
+from reportlab.lib.utils import ImageReader
+from reportlab.pdfgen.canvas import Canvas
+
+from ocrmypdf import _pipeline, pdfinfo
+from ocrmypdf.helpers import Resolution
+
+
+@pytest.fixture(scope='session')
+def rgb_image():
+    im = Image.new('RGB', (8, 8))
+    im.putpixel((4, 4), (255, 0, 0))
+    im.putpixel((5, 5), (0, 255, 0))
+    im.putpixel((6, 6), (0, 0, 255))
+    return ImageReader(im)
+
+
+DUMMY_OVERSAMPLE_RESOLUTION = Resolution(42.0, 42.0)
+VECTOR_RESOLUTION = Resolution(_pipeline.VECTOR_PAGE_DPI, _pipeline.VECTOR_PAGE_DPI)
+
+
+@pytest.mark.parametrize(
+    'image, text, vector, result',
+    [
+        (False, False, False, VECTOR_RESOLUTION),
+        (False, True, False, VECTOR_RESOLUTION),
+        (True, False, False, DUMMY_OVERSAMPLE_RESOLUTION),
+        (True, True, False, VECTOR_RESOLUTION),
+        (False, False, True, VECTOR_RESOLUTION),
+        (False, True, True, VECTOR_RESOLUTION),
+        (True, False, True, VECTOR_RESOLUTION),
+        (True, True, True, VECTOR_RESOLUTION),
+    ],
+)
+def test_dpi_needed(image, text, vector, result, rgb_image, outdir):
+
+    c = Canvas(str(outdir / 'dpi.pdf'), pagesize=(5 * inch, 5 * inch))
+    if image:
+        c.drawImage(rgb_image, 1 * inch, 1 * inch, width=1 * inch, height=1 * inch)
+    if text:
+        c.drawString(1 * inch, 4 * inch, "Actual text")
+    if vector:
+        c.ellipse(3 * inch, 3 * inch, 4 * inch, 4 * inch)
+    c.showPage()
+    c.save()
+
+    mock = Mock()
+    mock.oversample = DUMMY_OVERSAMPLE_RESOLUTION[0]
+
+    pi = pdfinfo.PdfInfo(outdir / 'dpi.pdf')
+
+    assert _pipeline.get_canvas_square_dpi(pi[0], mock) == result
+    assert _pipeline.get_page_square_dpi(pi[0], mock) == result
+
+
+@pytest.mark.parametrize(
+    # Name for nicer -v output
+    'name,input,output',
+    (
+        (
+            'empty_input',
+            # Input:
+            (),
+            # Output:
+            (),
+        ),
+        (
+            'no_values',
+            # Input:
+            ('', '', '', '', ''),
+            # Output:
+            (
+                ((1, 5), None),
+            ),
+        ),
+        (
+            'no_empty_values',
+            # Input:
+            ('v', 'w', 'x', 'y', 'z'),
+            # Output:
+            (
+                ((1, 1), 'v'),
+                ((2, 2), 'w'),
+                ((3, 3), 'x'),
+                ((4, 4), 'y'),
+                ((5, 5), 'z'),
+            ),
+        ),
+        (
+            'skip_head',
+            # Input:
+            ('', '', 'x', 'y', 'z'),
+            # Output:
+            (
+                ((1, 2), None),
+                ((3, 3), 'x'),
+                ((4, 4), 'y'),
+                ((5, 5), 'z'),
+            ),
+        ),
+        (
+            'skip_tail',
+            # Input:
+            ('x', 'y', 'z', '', ''),
+            # Output:
+            (
+                ((1, 1), 'x'),
+                ((2, 2), 'y'),
+                ((3, 3), 'z'),
+                ((4, 5), None),
+            ),
+        ),
+        (
+            'range_in_middle',
+            # Input:
+            ('x', '', '', '', 'y'),
+            # Output:
+            (
+                ((1, 1), 'x'),
+                ((2, 4), None),
+                ((5, 5), 'y'),
+            ),
+        ),
+        (
+            'range_in_middle_2',
+            # Input:
+            ('x', '', '', 'y', '', '', '', 'z'),
+            # Output:
+            (
+                ((1, 1), 'x'),
+                ((2, 3), None),
+                ((4, 4), 'y'),
+                ((5, 7), None),
+                ((8, 8), 'z'),
+            ),
+        ),
+    ),
+)
+def test_enumerate_compress_ranges(name, input, output):
+    assert output == tuple(_pipeline.enumerate_compress_ranges(input))

tests/test_rotation.py

@@ -6,15 +6,17 @@
 
 
 from io import BytesIO
+from math import cos, pi, sin
 from os import fspath
 
 import img2pdf
 import pikepdf
 import pytest
 from PIL import Image
+from reportlab.pdfgen.canvas import Canvas
 
 from ocrmypdf import leptonica
-from ocrmypdf._exec import ghostscript, tesseract
+from ocrmypdf._exec import ghostscript
 from ocrmypdf._plugin_manager import get_plugin_manager
 from ocrmypdf.helpers import Resolution
 from ocrmypdf.pdfinfo import PdfInfo
@@ -249,16 +251,6 @@ def test_rotate_page_level(image_angle, page_angle, resources, outdir):
     assert check_monochrome_correlation(outdir, reference, 1, out, 1) > 0.2
 
 
-def test_tesseract_orientation(resources, tmp_path):
-    pix = leptonica.Pix.open(resources / 'crom.png')
-    pix_rotated = pix.rotate_orth(2)  # 180 degrees clockwise
-    pix_rotated.write_implied_format(tmp_path / '000001.png')
-
-    tesseract.get_orientation(  # Test results of this are unreliable
-        tmp_path / '000001.png', engine_mode='3', timeout=10
-    )
-
-
 def test_rasterize_rotates(resources, tmp_path):
     pm = get_plugin_manager([])
 
@@ -287,3 +279,50 @@ def test_rasterize_rotates(resources, tmp_path):
         filter_vector=False,
     )
     assert Image.open(img).size == (151, 123), "Image not rotated"
+
+
+def test_simulated_scan(outdir):
+    canvas = Canvas(
+        fspath(outdir / 'fakescan.pdf'),
+        pagesize=(209.8, 297.6),
+    )
+
+    page_vars = [(2, 36, 250), (91, 170, 240), (179, 190, 36), (271, 36, 36)]
+
+    for n, page_var in enumerate(page_vars):
+        text = canvas.beginText()
+        text.setFont('Helvetica', 20)
+
+        angle, x, y = page_var
+        cos_a, sin_a = cos(angle / 180.0 * pi), sin(angle / 180.0 * pi)
+
+        text.setTextTransform(cos_a, -sin_a, sin_a, cos_a, x, y)
+        text.textOut(f'Page {n + 1}')
+        canvas.drawText(text)
+        canvas.showPage()
+    canvas.save()
+
+    check_ocrmypdf(
+        outdir / 'fakescan.pdf',
+        outdir / 'out.pdf',
+        '--force-ocr',
+        '--deskew',
+        '--rotate-pages',
+        '--plugin',
+        'tests/plugins/tesseract_debug_rotate.py',
+    )
+
+    with pikepdf.open(outdir / 'out.pdf') as pdf:
+        assert (
+            pdf.pages[1].MediaBox[2] > pdf.pages[1].MediaBox[3]
+        ), "Wrong orientation: not landscape"
+        assert (
+            pdf.pages[3].MediaBox[2] > pdf.pages[3].MediaBox[3]
+        ), "Wrong orientation: Not landscape"
+
+        assert (
+            pdf.pages[0].MediaBox[2] < pdf.pages[0].MediaBox[3]
+        ), "Wrong orientation: Not portrait"
+        assert (
+            pdf.pages[2].MediaBox[2] < pdf.pages[2].MediaBox[3]
+        ), "Wrong orientation: Not portrait"

tests/test_tesseract.py

@@ -45,7 +45,8 @@ def test_skip_pages_does_not_replicate(resources, basename, outdir):
         assert len(page.images) == 1, "skipped page was replicated"
 
     for n, info_out_n in enumerate(info):
-        assert info_out_n.width_inches == info_in[n].width_inches
+        assert info_out_n.width_inches == info_in[n].width_inches, "output resized"
+        assert info_out_n.height_inches == info_in[n].height_inches, "output resized"
 
 
 def test_content_preservation(resources, outpdf):

tests/test_unpaper.py

@@ -28,11 +28,12 @@ def test_no_unpaper(resources, no_outpdf):
     output = fspath(no_outpdf)
 
     _parser, options, pm = get_parser_options_plugins(["--clean", input_, output])
-    with patch("ocrmypdf._exec.unpaper.version") as mock_unpaper_version:
+    with patch("ocrmypdf._exec.unpaper.version") as mock:
-        mock_unpaper_version.side_effect = FileNotFoundError("unpaper")
+        mock.side_effect = FileNotFoundError("unpaper")
 
         with pytest.raises(MissingDependencyError):
             check_options(options, pm)
+        mock.assert_called()
 
 
 def test_old_unpaper(resources, no_outpdf):
@@ -40,11 +41,12 @@ def test_old_unpaper(resources, no_outpdf):
     output = fspath(no_outpdf)
 
     _parser, options, pm = get_parser_options_plugins(["--clean", input_, output])
-    with patch("ocrmypdf._exec.unpaper.version") as mock_unpaper_version:
+    with patch("ocrmypdf._exec.unpaper.version") as mock:
-        mock_unpaper_version.return_value = '0.5'
+        mock.return_value = '0.5'
 
         with pytest.raises(MissingDependencyError):
             check_options(options, pm)
+        mock.assert_called()
 
 
 @pytest.mark.skipif(not have_unpaper(), reason="requires unpaper")

tests/test_validation.py

@@ -18,6 +18,8 @@ from ocrmypdf.cli import get_parser
 from ocrmypdf.exceptions import BadArgsError, MissingDependencyError
 from ocrmypdf.pdfinfo import PdfInfo
 
+run_ocrmypdf_api = pytest.helpers.run_ocrmypdf_api
+
 
 def make_opts_pm(input_file='a.pdf', output_file='b.pdf', language='eng', **kwargs):
     if language is not None:
@@ -90,8 +92,6 @@ def test_mutex_options():
         vd.check_options_ocr_behavior(make_opts(redo_ocr=True, skip_text=True))
     with pytest.raises(BadArgsError):
         vd.check_options_ocr_behavior(make_opts(redo_ocr=True, force_ocr=True))
-    with pytest.raises(BadArgsError):
-        vd.check_options_ocr_behavior(make_opts(pages='1-3', sidecar='file.txt'))
 
 
 def test_optimizing(caplog):
@@ -188,18 +188,20 @@ def test_language_warning(caplog):
     caplog.set_level(logging.DEBUG)
     with patch(
         'ocrmypdf._validation.locale.getlocale', return_value=('en_US', 'UTF-8')
-    ):
+    ) as mock:
         vd.check_options_languages(opts, {'eng'})
         assert opts.languages == {'eng'}
         assert '' in caplog.text
+        mock.assert_called_once()
 
     opts = make_opts(language=None)
     with patch(
         'ocrmypdf._validation.locale.getlocale', return_value=('fr_FR', 'UTF-8')
-    ):
+    ) as mock:
         vd.check_options_languages(opts, {'eng'})
         assert opts.languages == {'eng'}
         assert 'assuming --language' in caplog.text
+        mock.assert_called_once()
 
 
 def test_version_comparison():
@@ -265,7 +267,14 @@ def test_pagesegmode_warning(caplog):
 
 
 def test_two_languages():
-    with patch('ocrmypdf._exec.tesseract.has_textonly_pdf', return_value=True):
+    with patch('ocrmypdf._exec.tesseract.has_textonly_pdf', return_value=True) as mock:
         vd._check_options(
             *make_opts_pm(language='fakelang1+fakelang2'), {'fakelang1', 'fakelang2'}
         )
+        mock.assert_called()
+
+
+def test_sidecar_equals_output(resources, no_outpdf):
+    op = no_outpdf
+    with pytest.raises(BadArgsError, match=r'--sidecar'):
+        run_ocrmypdf_api(resources / 'trivial.pdf', op, '--sidecar', op)