Merge remote-tracking branch 'origin/master'

Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 1 to 3.
- [Release notes](https://github.com/codecov/codecov-action/releases)
- [Changelog](https://github.com/codecov/codecov-action/blob/master/CHANGELOG.md)
- [Commits](https://github.com/codecov/codecov-action/compare/v1...v3)

---
updated-dependencies:
- dependency-name: codecov/codecov-action
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.git_archival.txt

@@ -1 +1,4 @@
-ref-names: $Format:%D$
+node: $Format:%H$
+node-date: $Format:%cI$
+describe-name: $Format:%(describe:tags=true)$
+ref-names: $Format:%D$

.github/dependabot.yml

@@ -0,0 +1,11 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "github-actions" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"

.github/workflows/build.yml

@@ -41,7 +41,7 @@ jobs:
       PYTHON: ${{ matrix.python }}
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags
 
@@ -111,7 +111,7 @@ jobs:
           python -m pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
 
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v1
+        uses: codecov/codecov-action@v3
         with:
           files: ./coverage.xml
           env_vars: OS,PYTHON
@@ -129,7 +129,7 @@ jobs:
       PYTHON: ${{ matrix.python }}
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags
 
@@ -166,7 +166,7 @@ jobs:
           python -m pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
 
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v1
+        uses: codecov/codecov-action@v3
         with:
           files: ./coverage.xml
           env_vars: OS,PYTHON
@@ -184,7 +184,7 @@ jobs:
       PYTHON: ${{ matrix.python }}
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags
 
@@ -208,7 +208,7 @@ jobs:
           python -m pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
 
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v1
+        uses: codecov/codecov-action@v3
         with:
           files: ./coverage.xml
           env_vars: OS,PYTHON
@@ -217,7 +217,7 @@ jobs:
     name: Build sdist and wheels
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags
 
@@ -232,7 +232,7 @@ jobs:
           python setup.py sdist
           python setup.py bdist_wheel
 
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         with:
           path: |
             ./dist/*.whl
@@ -274,18 +274,18 @@ jobs:
       - name: Set image name
         run: echo "DOCKER_IMAGE_NAME=ocrmypdf" >> $GITHUB_ENV
 
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags
 
       - name: Login to Docker Hub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           username: jbarlow83
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v2
 
       - name: Set up Docker Buildx
         id: buildx

README.md

@@ -62,7 +62,8 @@ Linux, Windows, macOS and FreeBSD are supported. Docker images are also availabl
 | Debian, Ubuntu                | ``apt install ocrmypdf``      |
 | Windows Subsystem for Linux   | ``apt install ocrmypdf``      |
 | Fedora                        | ``dnf install ocrmypdf``      |
-| macOS                         | ``brew install ocrmypdf``     |
+| macOS (Homebrew)              | ``brew install ocrmypdf``     |
+| macOS (nix)                   | ``nix-env -i  ocrmypdf``      |
 | LinuxBrew                     | ``brew install ocrmypdf``     |
 | FreeBSD                       | ``pkg install py37-ocrmypdf`` |
 | Conda                         | ``conda install ocrmypdf``    |

docs/advanced.rst

@@ -223,7 +223,9 @@ The ``hocr`` renderer
 The ``hocr`` renderer works with older versions of Tesseract. The image
 layer is copied from the original PDF page if possible, avoiding
 potentially lossy transcoding or loss of other PDF information. If
-preprocessing is specified, then the image layer is a new PDF.
+preprocessing is specified, then the image layer is a new PDF. (You may
+need to disable PDF/A conversion nad optimization to eliminate all
+lossy transformations.)
 
 Unlike ``sandwich`` this renderer is implemented within OCRmyPDF; anyone
 looking to customize how OCR is presented should look here. A major

docs/batch.rst

@@ -36,18 +36,21 @@ Directory trees
 ===============
 
 This will walk through a directory tree and run OCR on all files in
-place, printing the output in a way that makes
+place, and printing each filename in between runs:
 
 .. code-block:: bash
 
-   find . -printf '%p' -name '*.pdf' -exec ocrmypdf '{}' '{}' \;
+   find . -printf '%p\n' -name '*.pdf' -exec ocrmypdf '{}' '{}' \;
 
-Alternatively, with a docker container (mounts a volume to the container
-where the PDFs are stored):
+Alternatively, with a Docker container and streaming the file through
+standard input and output:
 
 .. code-block:: bash
 
-   find . -printf '%p' -name '*.pdf' -exec docker run --rm -v <host dir>:<container dir> jbarlow83/ocrmypdf '<container dir>/{}' '<container dir>/{}' \;
+   find . -name '*.pdf' -print0 | xargs -0 | while read pdf; do
+       pdfout=$(mktemp)
+       docker run --rm -i jbarlow83/ocrmypdf - - <$pdf >$pdfout && cp $pdfout $pdf
+   done
 
 This only runs one ``ocrmypdf`` process at a time. This variation uses
 ``find`` to create a directory list and ``parallel`` to parallelize runs
@@ -124,7 +127,9 @@ Users may need to customize the script to meet their requirements.
 
     "OCR_INPUT_DIRECTORY", "Set input directory to monitor (recursive)"
     "OCR_OUTPUT_DIRECTORY", "Set output directory (should not be under input)"
+    "OCR_ARCHIVE_DIRECTORY", "Set archive directory for processed originals (should not be under input, requires ``OCR_ON_SUCCESS_ARCHIVE`` to be set)"
     "OCR_ON_SUCCESS_DELETE", "This will delete the input file if the exit code is 0 (OK)"
+    "OCR_ON_SUCCESS_ARCHIVE", "This will move the processed orignal file to ``OCR_ARCHIVE_DIRECTORY`` if the exit code is 0 (OK). Note that ``OCR_ON_SUCCESS_DELETE`` takes precedence over this option, i.e. if both options are set, the input file will be deleted."
     "OCR_OUTPUT_DIRECTORY_YEAR_MONTH", "This will place files in the output in ``{output}/{year}/{month}/{filename}``"
     "OCR_DESKEW", "Apply deskew to crooked input PDFs"
     "OCR_JSON_SETTINGS", "A JSON string specifying any other arguments for ``ocrmypdf.ocr``, e.g. ``'OCR_JSON_SETTINGS={""rotate_pages"": true}'``."
@@ -144,16 +149,18 @@ The watcher service is included in the OCRmyPDF Docker image. To run it:
     docker run \
         -v <path to files to convert>:/input \
         -v <path to store results>:/output \
+        -v <path to store processed originals>:/archive \
         -e OCR_OUTPUT_DIRECTORY_YEAR_MONTH=1 \
-        -e OCR_ON_SUCCESS_DELETE=1 \
+        -e OCR_ON_SUCCESS_ARCHIVE=1 \
         -e OCR_DESKEW=1 \
         -e PYTHONUNBUFFERED=1 \
         -it --entrypoint python3 \
         jbarlow83/ocrmypdf \
         watcher.py
 
-This service will watch for a file that matches ``/input/\*.pdf`` and will
-convert it to a OCRed PDF in ``/output/``. The parameters to this image are:
+This service will watch for a file that matches ``/input/\*.pdf``,
+convert it to a OCRed PDF in ``/output/``, and move the processed
+original to ``/archive``. The parameters to this image are:
 
 .. csv-table:: watcher.py parameters for Docker
     :header: "Parameter", "Description"
@@ -161,10 +168,11 @@ convert it to a OCRed PDF in ``/output/``. The parameters to this image are:
 
     "``-v <path to files to convert>:/input``", "Files placed in this location will be OCRed"
     "``-v <path to store results>:/output``", "This is where OCRed files will be stored"
-    "``-e OCR_OUTPUT_DIRECTORY_YEAR_MONTH=1``", "Define environment variable OCR_OUTPUT_DIRECTORY_YEAR_MONTH=1"
-    "``-e OCR_ON_SUCCESS_DELETE=1``", "Define environment variable"
-    "``-e OCR_DESKEW=1``", "Define environment variable"
-    "``-e PYTHONBUFFERED=1``", "This will force STDOUT to be unbuffered and allow you to see messages in docker logs"
+    "``-v <path to store processed originals>:/archive``", "Archive processed originals here"
+    "``-e OCR_OUTPUT_DIRECTORY_YEAR_MONTH=1``", "Define environment variable ``OCR_OUTPUT_DIRECTORY_YEAR_MONTH=1`` to place files in the output in ``{output}/{year}/{month}/{filename}``"
+    "``-e OCR_ON_SUCCESS_ARCHIVE=1``", "Define environment variable ``OCR_ON_SUCCESS_ARCHIVE`` to move processed originals"
+    "``-e OCR_DESKEW=1``", "Define environment variable ``OCR_DESKEW``  to apply deskew to crooked input PDFs"
+    "``-e PYTHONBUFFERED=1``", "This will force ``STDOUT`` to be unbuffered and allow you to see messages in docker logs"
 
 This service relies on polling to check for changes to the filesystem. It
 may not be suitable for some environments, such as filesystems shared on a

docs/cookbook.rst

@@ -200,7 +200,7 @@ might remove desirable content, especially from poor quality scans.
 
 .. warning::
 
-   ``--clean-final`` and ``-remove-background`` may leave undesirable
+   ``--clean-final`` and ``--remove-background`` may leave undesirable
    visual artifacts in some images where their algorithms have
    shortcomings. Files should be visually reviewed after using these
    options.
@@ -340,6 +340,9 @@ levels in the GCC compiler.
         - Enables lossless optimizations, such as transcoding images to more
           efficient formats. Also compress other uncompressed objects in the
           PDF and enables the more efficient "object streams" within the PDF.
+          (If ``--jbig2-lossy`` is issued, then lossy JBIG2 optimization is used.
+          The decision to use lossy JBIG2 is separate from standard optimization
+          settings.)
     *   - ``--optimize 2``
         - All of the above, and enables lossy optimizations and color quantization.
     *   - ``--optimize 3``
@@ -360,3 +363,8 @@ fo a PDF.
     ocrmypdf --optimize 3 in.pdf out.pdf  # Make it small
 
 Some users may consider enabling lossy JBIG2. See: :ref:`jbig2-lossy`.
+
+.. note::
+
+    Image processing and PDF/A conversion can also introduce lossy transformations
+    to your PDF images, even when ``--optimize 1`` is in use.

docs/jbig2.rst

@@ -32,7 +32,7 @@ For all other Linux, you must build a JBIG2 encoder from source:
 
 .. _jbig2-lossy:
 
-Dependencies include libtoolize and libleptonica, which on Ubuntu systems 
+Dependencies include libtoolize and libleptonica, which on Ubuntu systems
 are packaged as libtool and libleptonica-dev.
 
 Lossy mode JBIG2

docs/plugins.rst

@@ -162,6 +162,11 @@ Examples
   management system.
 
 
+Suppressing or overriding other plugins
+---------------------------------------
+
+.. autofunction:: ocrmypdf.pluginspec.initialize
+
 Custom command line arguments
 -----------------------------
 
@@ -172,7 +177,7 @@ Custom command line arguments
 Execution and progress reporting
 --------------------------------
 
-.. autoclass: ocrmypdf.pluginspec.Executor
+.. autoclass:: ocrmypdf.pluginspec.Executor
     :members:
 
 .. autofunction:: ocrmypdf.pluginspec.get_logging_console
@@ -216,3 +221,10 @@ PDF/A production
 ----------------
 
 .. autofunction:: ocrmypdf.pluginspec.generate_pdfa
+
+PDF optimization
+----------------
+
+.. autofunction:: ocrmypdf.pluginspec.optimize_pdf
+
+.. autofunction:: ocrmypdf.pluginspec.is_optimization_enabled

docs/release_notes.rst

@@ -24,10 +24,25 @@ tagged yet.
 
 .. |OCRmyPDF PyPI| image:: https://img.shields.io/pypi/v/ocrmypdf.svg
 
+v13.6.1
+=======
+
+-  Require setuptools-scm 7.0.5 to avoid possible issues with source distributions in
+   earlier versions of setuptools-scm.
+-  Suppress a spurious warning, improve tests, improve typing and other miscellany.
+
+v13.6.0
+=======
+
+-  Added a new ``initialize`` plugin hook, making it possible to suppress built-in
+   plugins more easily, among other possibilities.
+-  Fixed an issue where unpaper would exit with a "wrong stream" error, probably
+   related to images with an odd integer width. :issue:`887, 665`
+
 v13.5.0
 =======
 
--  Added a new ``optimize_pdf`` pluginhook, making it possible to create plugins that
+-  Added a new ``optimize_pdf`` plugin hook, making it possible to create plugins that
    replace or enhance OCRmyPDF's PDF optimizer.
 -  Removed all max version restrictions. Our new policy is to blacklist known-bad releases
    and only block known-bad versions of dependencies.

misc/watcher.py

@@ -23,6 +23,7 @@
 import json
 import logging
 import os
+import shutil
 import sys
 import time
 from datetime import datetime
@@ -44,8 +45,10 @@ def getenv_bool(name: str, default: str = 'False'):
 
 INPUT_DIRECTORY = os.getenv('OCR_INPUT_DIRECTORY', '/input')
 OUTPUT_DIRECTORY = os.getenv('OCR_OUTPUT_DIRECTORY', '/output')
+ARCHIVE_DIRECTORY = os.getenv('OCR_ARCHIVE_DIRECTORY', '/processed')
 OUTPUT_DIRECTORY_YEAR_MONTH = getenv_bool('OCR_OUTPUT_DIRECTORY_YEAR_MONTH')
 ON_SUCCESS_DELETE = getenv_bool('OCR_ON_SUCCESS_DELETE')
+ON_SUCCESS_ARCHIVE = getenv_bool('OCR_ON_SUCCESS_ARCHIVE')
 DESKEW = getenv_bool('OCR_DESKEW')
 OCR_JSON_SETTINGS = json.loads(os.getenv('OCR_JSON_SETTINGS', '{}'))
 POLL_NEW_FILE_SECONDS = int(os.getenv('OCR_POLL_NEW_FILE_SECONDS', '1'))
@@ -108,9 +111,13 @@ def execute_ocrmypdf(file_path):
         deskew=DESKEW,
         **OCR_JSON_SETTINGS,
     )
-    if exit_code == 0 and ON_SUCCESS_DELETE:
-        log.info(f'OCR is done. Deleting: {file_path}')
-        file_path.unlink()
+    if exit_code == 0:
+        if ON_SUCCESS_DELETE:
+            log.info(f'OCR is done. Deleting: {file_path}')
+            file_path.unlink()
+        elif ON_SUCCESS_ARCHIVE:
+            log.info(f'OCR is done. Archiving {file_path.name} to {ARCHIVE_DIRECTORY}')
+            shutil.move(file_path, f'{ARCHIVE_DIRECTORY}/{file_path.name}')
     else:
         log.info('OCR is done')
 
@@ -135,13 +142,16 @@ def main():
         f"Starting OCRmyPDF watcher with config:\n"
         f"Input Directory: {INPUT_DIRECTORY}\n"
         f"Output Directory: {OUTPUT_DIRECTORY}\n"
-        f"Output Directory Year & Month: {OUTPUT_DIRECTORY_YEAR_MONTH}"
+        f"Output Directory Year & Month: {OUTPUT_DIRECTORY_YEAR_MONTH}\n"
+        f"Archive Directory: {ARCHIVE_DIRECTORY}"
     )
     log.debug(
         f"INPUT_DIRECTORY: {INPUT_DIRECTORY}\n"
         f"OUTPUT_DIRECTORY: {OUTPUT_DIRECTORY}\n"
         f"OUTPUT_DIRECTORY_YEAR_MONTH: {OUTPUT_DIRECTORY_YEAR_MONTH}\n"
+        f"ARCHIVE_DIRECTORY: {ARCHIVE_DIRECTORY}\n"
         f"ON_SUCCESS_DELETE: {ON_SUCCESS_DELETE}\n"
+        f"ON_SUCCESS_ARCHIVE: {ON_SUCCESS_ARCHIVE}\n"
         f"DESKEW: {DESKEW}\n"
         f"ARGS: {OCR_JSON_SETTINGS}\n"
         f"POLL_NEW_FILE_SECONDS: {POLL_NEW_FILE_SECONDS}\n"

pyproject.toml

@@ -1,14 +1,12 @@
 [build-system]
 requires = [
-  "setuptools >= 30.3.0",
-  "wheel",
-  "setuptools_scm[toml] >= 3.4",
-  "setuptools_scm_git_archive"
+  "setuptools >= 52",
+  "setuptools_scm[toml] >= 7.0.5",
+  "wheel"
 ]
 build-backend = "setuptools.build_meta"
 
 [tool.setuptools_scm]
-version_scheme = "post-release"
 
 [tool.black]
 line-length = 88
@@ -96,7 +94,8 @@ module = [
   'pdfminer.*',
   'reportlab.*',
   'fitz',
-  'libxmp.utils'
+  'libxmp.utils',
+  'importlib_metadata'
 ]
 ignore_missing_imports = true
 

setup.cfg

@@ -86,6 +86,8 @@ test =
     pytest-cov>=2.11.1
     pytest-xdist>=2.2.0
     python-xmp-toolkit==2.0.1  # also requires apt-get install libexempi3
+    types-humanfriendly
+    types-Pillow
 watcher =
     watchdog>=1.0.2
 webservice =

setup.py

@@ -8,11 +8,4 @@
 
 from setuptools import setup
 
-# Minimal setup to support older setuptools/setuptools_scm
-setup(
-    setup_requires=[  # can be removed whenever we can drop pip 9 support
-        'setuptools_scm',  # so that version will work
-        'setuptools_scm_git_archive',  # enable version from github tarballs
-    ],
-    use_scm_version={'version_scheme': 'post-release'},
-)
+setup()

src/ocrmypdf/_exec/unpaper.py

@@ -72,13 +72,13 @@ def version() -> str:
     return get_version('unpaper')
 
 
-SUFFIXES = {'1': '.pbm', 'L': '.pgm', 'RGB': '.ppm'}
+SUPPORTED_MODES = {'1', 'L', 'RGB'}
 
 
-def _convert_image(im: Image.Image) -> Tuple[Image.Image, bool, str]:
+def _convert_image(im: Image.Image) -> Tuple[Image.Image, bool]:
     im_modified = False
 
-    if im.mode not in SUFFIXES:
+    if im.mode not in SUPPORTED_MODES:
         log.info("Converting image to other colorspace")
         try:
             if im.mode == 'P' and len(im.getcolors()) == 2:
@@ -91,13 +91,11 @@ def _convert_image(im: Image.Image) -> Tuple[Image.Image, bool, str]:
             ) from e
         else:
             im_modified = True
-    try:
-        suffix = SUFFIXES[im.mode]
-    except KeyError:
-        raise MissingDependencyError(
-            "Failed to convert image to a supported format."
-        ) from None
-    return im, im_modified, suffix
+        if im.mode not in SUPPORTED_MODES:
+            raise MissingDependencyError(
+                "Failed to convert image to a supported format."
+            ) from None
+    return im, im_modified
 
 
 @contextmanager
@@ -105,19 +103,21 @@ def _setup_unpaper_io(input_file: Path) -> Iterator[Tuple[Path, Path, Path]]:
     with Image.open(input_file) as im:
         if im.width * im.height >= UNPAPER_IMAGE_PIXEL_LIMIT:
             raise UnpaperImageTooLargeError(w=im.width, h=im.height)
-        im, im_modified, suffix = _convert_image(im)
+        im, im_modified = _convert_image(im)
 
         with TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
             tmppath = Path(tmpdir)
-            if im_modified or input_file.suffix != '.pnm':
-                input_pnm = tmppath / 'input.pnm'
-                im.save(input_pnm, format='PPM')
+            if im_modified or input_file.suffix != '.png':
+                input_png = tmppath / 'input.png'
+                im.save(input_png, format='PNG')
             else:
                 # No changes, PNG input, just use the file we already have
-                input_pnm = input_file
+                input_png = input_file
 
-            output_pnm = tmppath / f'output{suffix}'
-            yield input_pnm, output_pnm, tmppath
+            # unpaper can write .png too, but it seems to write them slowly
+            # adds a few seconds to test suite - so just use pnm
+            output_pnm = tmppath / 'output.pnm'
+            yield input_png, output_pnm, tmppath
 
 
 def run_unpaper(
@@ -125,7 +125,7 @@ def run_unpaper(
 ) -> None:
     args_unpaper = ['unpaper', '-v', '--dpi', str(round(dpi, 6))] + mode_args
 
-    with _setup_unpaper_io(input_file) as (input_pnm, output_pnm, tmpdir):
+    with _setup_unpaper_io(input_file) as (input_png, output_pnm, tmpdir):
         # To prevent any shenanigans from accepting arbitrary parameters in
         # --unpaper-args, we:
         # 1) run with cwd set to a tmpdir with only unpaper's files
@@ -133,7 +133,7 @@ def run_unpaper(
         # 3) append absolute paths for the input and output file
         # This should ensure that a user cannot clobber some other file with
         # their unpaper arguments (whether intentionally or otherwise)
-        args_unpaper.extend([os.fspath(input_pnm), os.fspath(output_pnm)])
+        args_unpaper.extend([os.fspath(input_png), os.fspath(output_pnm)])
         run(
             args_unpaper,
             close_fds=True,

src/ocrmypdf/_pipeline.py

@@ -41,7 +41,7 @@ from ocrmypdf.pdfinfo import Colorspace, Encoding, PdfInfo
 # Remove this workaround when we require Pillow >= 10
 try:
     BICUBIC = Image.Resampling.BICUBIC  # type: ignore
-except AttributeError:
+except AttributeError:  # pragma: no cover
     # Pillow 9 shim
     BICUBIC = Image.BICUBIC  # type: ignore
 
@@ -679,15 +679,14 @@ def get_docinfo(base_pdf: pikepdf.Pdf, context: PdfContext) -> Dict[str, str]:
         k: from_document_info(k)
         for k in ('/Title', '/Author', '/Keywords', '/Subject', '/CreationDate')
     }
-    if options is not None:
-        if options.title:
-            pdfmark['/Title'] = options.title
-        if options.author:
-            pdfmark['/Author'] = options.author
-        if options.keywords:
-            pdfmark['/Keywords'] = options.keywords
-        if options.subject:
-            pdfmark['/Subject'] = options.subject
+    if options.title:
+        pdfmark['/Title'] = options.title
+    if options.author:
+        pdfmark['/Author'] = options.author
+    if options.keywords:
+        pdfmark['/Keywords'] = options.keywords
+    if options.subject:
+        pdfmark['/Subject'] = options.subject
 
     creator_tag = context.plugin_manager.hook.get_ocr_engine().creator_tag(options)
 

src/ocrmypdf/_plugin_manager.py

@@ -116,6 +116,9 @@ def get_parser_options_plugins(
     plugin_manager = get_plugin_manager(pre_options.plugins)
 
     parser = get_parser()
+    plugin_manager.hook.initialize(  # pylint: disable=no-member
+        plugin_manager=plugin_manager
+    )
     plugin_manager.hook.add_options(parser=parser)  # pylint: disable=no-member
 
     options = parser.parse_args(args=args)

src/ocrmypdf/_sync.py

@@ -302,7 +302,7 @@ def exec_concurrent(context: PdfContext, executor: Executor) -> Sequence[str]:
     # Merge layers to one single pdf
     pdf = ocrgraft.finalize()
 
-    messages: List[str] = []
+    messages: Sequence[str] = []
     if options.output_type != 'none':
         # PDF/A and metadata
         log.info("Postprocessing...")

src/ocrmypdf/_version.py

@@ -10,9 +10,9 @@ OCRmyPDF uses setuptools_scm to derive version from git tags.
 """
 
 try:
-    from importlib_metadata import version as _package_version
-except ImportError:
     from importlib.metadata import version as _package_version
+except ImportError:
+    from importlib_metadata import version as _package_version  # type: ignore
 
 PROGRAM_NAME = 'ocrmypdf'
 

src/ocrmypdf/builtin_plugins/optimize.py

@@ -12,8 +12,7 @@ import logging
 from pathlib import Path
 from typing import Sequence, Tuple
 
-from ocrmypdf import PdfContext, hookimpl
-from ocrmypdf._concurrent import Executor
+from ocrmypdf import Executor, PdfContext, hookimpl
 from ocrmypdf._exec import jbig2enc, pngquant
 from ocrmypdf._pipeline import get_pdf_save_settings
 from ocrmypdf.cli import numeric

src/ocrmypdf/hocrtransform.py

@@ -33,14 +33,19 @@
 import argparse
 import os
 import re
+import warnings
 from math import atan, cos, sin
 from pathlib import Path
 from typing import Any, NamedTuple, Optional, Tuple, Union
 from xml.etree import ElementTree
 
-from reportlab.lib.colors import black, cyan, magenta, red
-from reportlab.lib.units import inch
-from reportlab.pdfgen.canvas import Canvas
+with warnings.catch_warnings():
+    warnings.filterwarnings(
+        'ignore', category=DeprecationWarning, message=r".*load_module.*"
+    )
+    from reportlab.lib.colors import black, cyan, magenta, red
+    from reportlab.lib.units import inch
+    from reportlab.pdfgen.canvas import Canvas
 
 # According to Wikipedia these languages are supported in the ISO-8859-1 character
 # set, meaning reportlab can generate them and they are compatible with hocr,

src/ocrmypdf/pdfa.py

@@ -14,9 +14,9 @@ from pathlib import Path
 from typing import Dict, Iterator, Union
 
 try:
-    from importlib_resources import files as package_files
-except ImportError:
     from importlib.resources import files as package_files
+except ImportError:
+    from importlib_resources import files as package_files  # type: ignore
 
 import pikepdf
 

src/ocrmypdf/pdfinfo/info.py

@@ -400,7 +400,7 @@ class ImageInfo:
                     self._comp = 4
                 else:
                     self._comp = 3
-            except UnsupportedImageTypeError as ex:
+            except (AttributeError, UnsupportedImageTypeError) as ex:
                 self._comp = None
                 logger.warning(
                     f"An image with a corrupt or unreadable ICC profile was found. "

src/ocrmypdf/pluginspec.py

@@ -22,8 +22,7 @@ from typing import (
 
 import pluggy
 
-from ocrmypdf import PdfContext
-from ocrmypdf._concurrent import Executor
+from ocrmypdf import Executor, PdfContext
 from ocrmypdf.helpers import Resolution
 
 if TYPE_CHECKING:
@@ -52,6 +51,33 @@ def get_logging_console() -> Handler:
     """
 
 
+@hookspec
+def initialize(plugin_manager: pluggy.PluginManager):
+    """Called when this plugin is first loaded into OCRmyPDF.
+
+    The primary intended use of this is for plugins to check compatibility with other
+    plugins and possibly block other blocks, a plugin that wishes to block ocrmypdf's
+    built-in optimize plugin could do:
+
+    .. code-block::
+
+        plugin_manager.set_blocked('ocrmypdf.builtin_plugins.optimize')
+
+    It would also be reasonable for an plugin implementation to check if it is unable
+    to proceed, for example, because a required dependency is missing. (If the plugin's
+    ability to proceed depends on options and arguments, use ``validate`` instead.)
+
+    Raises:
+        ocrmypdf.exceptions.ExitCodeException: If options are not acceptable
+            and the application should terminate gracefully with an informative
+            message and error code.
+
+    Note:
+        This hook will be called from the main process, and may modify global state
+        before child worker processes are forked.
+    """
+
+
 @hookspec
 def add_options(parser: ArgumentParser) -> None:
     """Allows the plugin to add its own command line and API arguments.
@@ -484,6 +510,9 @@ def optimize_pdf(
     If the implementation fails to produce a smaller file than the input file, it
     should return input_pdf instead.
 
+    A plugin that implements a new optimizer may need to suppress the built-in
+    optimizer by implementing an ``initialize`` hook.
+
     Arguments:
         input_pdf: The input PDF, which has OCR added.
         output_pdf: The requested filename of the output PDF which should be created
@@ -512,8 +541,8 @@ def optimize_pdf(
 def is_optimization_enabled(context: PdfContext) -> bool:
     """For a given PdfContext, OCRmyPDF asks the plugin if optimization is enabled.
 
-    It is assumed that an optimization plugin might be installed but could be
-    disabled by user settings.
+    An optimization plugin might be installed and active but could be disabled by
+    user settings.
 
     If this returns False, OCRmyPDF will take certain actions to finalize the PDF.
 

src/ocrmypdf/subprocess/__init__.py

@@ -11,12 +11,12 @@ import logging
 import os
 import re
 import sys
-from collections.abc import Mapping
 from contextlib import suppress
 from functools import lru_cache
+from pathlib import Path
 from subprocess import PIPE, STDOUT, CalledProcessError, CompletedProcess, Popen
 from subprocess import run as subprocess_run
-from typing import Callable, Optional, Type, Union
+from typing import Callable, Mapping, Optional, Sequence, Tuple, Type, Union
 
 from packaging.version import Version
 
@@ -26,11 +26,14 @@ from ocrmypdf.exceptions import MissingDependencyError
 
 log = logging.getLogger(__name__)
 
+Args = Sequence[Union[Path, str]]
+OsEnviron = os._Environ  # pylint: disable=protected-access
+
 
 def run(
-    args,
+    args: Args,
     *,
-    env=None,
+    env: Optional[OsEnviron] = None,
     logs_errors_to_stdout: bool = False,
     check: bool = False,
     **kwargs,
@@ -73,7 +76,12 @@ def run(
 
 
 def run_polling_stderr(
-    args, *, callback: Callable[[str], None], check: bool = False, env=None, **kwargs
+    args: Args,
+    *,
+    callback: Callable[[str], None],
+    check: bool = False,
+    env: Optional[OsEnviron] = None,
+    **kwargs,
 ) -> CompletedProcess:
     """Run a process like ``ocrmypdf.subprocess.run``, and poll stderr.
 
@@ -106,7 +114,9 @@ def run_polling_stderr(
         return CompletedProcess(args, proc.returncode, None, stderr=stderr)
 
 
-def _fix_process_args(args, env, kwargs):
+def _fix_process_args(
+    args: Args, env: Optional[OsEnviron], kwargs
+) -> Tuple[Args, OsEnviron, logging.Logger, bool]:
     assert 'universal_newlines' not in kwargs, "Use text= instead of universal_newlines"
 
     if not env:
@@ -123,14 +133,18 @@ def _fix_process_args(args, env, kwargs):
 
     log.debug("Running: %s", args)
     process_log = log.getChild(os.path.basename(program))
-    text = kwargs.get('text', False)
+    text = bool(kwargs.get('text', False))
 
     return args, env, process_log, text
 
 
 @lru_cache(maxsize=None)
 def get_version(
-    program: str, *, version_arg: str = '--version', regex=r'(\d+(\.\d+)*)', env=None
+    program: str,
+    *,
+    version_arg: str = '--version',
+    regex=r'(\d+(\.\d+)*)',
+    env: Optional[OsEnviron] = None,
 ) -> str:
     """Get the version of the specified program
 
@@ -229,7 +243,7 @@ a command prompt to install the missing package:
 '''
 
 
-def _get_platform():
+def _get_platform() -> str:
     if sys.platform.startswith('freebsd'):
         return 'freebsd'
     elif sys.platform.startswith('linux'):
@@ -239,7 +253,9 @@ def _get_platform():
     return sys.platform
 
 
-def _error_trailer(program, package, **kwargs):
+def _error_trailer(
+    program: str, package: Union[str, Mapping[str, str]], **kwargs
+) -> None:
     del kwargs
     if isinstance(package, Mapping):
         package = package.get(_get_platform(), program)
@@ -252,7 +268,9 @@ def _error_trailer(program, package, **kwargs):
         log.info(WINDOWS_INSTALL_ADVICE.format(**locals()))
 
 
-def _error_missing_program(program, package, required_for, recommended):
+def _error_missing_program(
+    program: str, package: str, required_for: Optional[str], recommended: bool
+) -> None:
     # pylint: disable=unused-argument
     if recommended:
         log.warning(MISSING_RECOMMEND_PROGRAM.format(**locals()))
@@ -263,7 +281,13 @@ def _error_missing_program(program, package, required_for, recommended):
     _error_trailer(**locals())
 
 
-def _error_old_version(program, package, need_version, found_version, required_for):
+def _error_old_version(
+    program: str,
+    package: str,
+    need_version: str,
+    found_version: str,
+    required_for: Optional[str],
+) -> None:
     # pylint: disable=unused-argument
     if required_for:
         log.error(OLD_VERSION_REQUIRED_FOR.format(**locals()))
@@ -272,7 +296,7 @@ def _error_old_version(program, package, need_version, found_version, required_f
     _error_trailer(**locals())
 
 
-def _remove_leading_v(s):
+def _remove_leading_v(s: str) -> str:
     if sys.version_info >= (3, 9):
         return s.removeprefix('v')
 
@@ -285,12 +309,12 @@ def check_external_program(
     *,
     program: str,
     package: str,
-    version_checker: Callable,
+    version_checker: Callable[[], str],
     need_version: str,
     required_for: Optional[str] = None,
-    recommended=False,
+    recommended: bool = False,
     version_parser: Type[Version] = Version,
-):
+) -> None:
     """Check for required version of external program and raise exception if not.
 
     Args:

tests/cache/francais/__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt/stderr.bin

@@ -0,0 +1 @@
+Tesseract Open Source OCR Engine v4.1.1 with Leptonica

tests/cache/francais/__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt/txt.bin

@@ -0,0 +1,13 @@
+Portez ce vieux whisky au juge
+blond qui fume sur son Ile
+interieure, a cöte de l'alcöve
+ovoide, oU les büches se
+consument dans l'ätre, ce qui
+lui permet de penser & la
+caenogenese de |'etre dont il
+est question dans la cause
+ambigu& entendue a MoY, dans
+un capharnaüm qui, pense-t-il,
+diminue ca et la la qualite de son
+ceuvre.
+

tests/cache/manifest.jsonl

@@ -82,3 +82,4 @@
 {"tesseract_version": "4.1.1", "system": "Linux", "python": "3.9.5", "argv_slug": "__-l__eng__--oem__1__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/trivial.pdf", "args": ["-l", "eng", "--oem", "1", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
 {"tesseract_version": "5.0.0", "system": "Linux", "python": "3.9.5", "argv_slug": "__-l__eng__thresholding_method=1__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/trivial.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "-c", "thresholding_method=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
 {"tesseract_version": "5.0.0", "system": "Linux", "python": "3.9.5", "argv_slug": "__-l__eng__thresholding_method=2__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/trivial.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "-c", "thresholding_method=2", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}
+{"tesseract_version": "4.1.1", "system": "Linux", "python": "3.10.4", "argv_slug": "__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/francais.pdf", "args": ["-l", "deu", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}

tests/test_pdfinfo.py

@@ -24,20 +24,24 @@ from ocrmypdf.pdfinfo.layout import PDFPage
 # pylint: disable=protected-access
 
 
-def test_single_page_text(outdir):
+@pytest.fixture
+def single_page_text(outdir):
     filename = outdir / 'text.pdf'
     pdf = Canvas(str(filename), pagesize=(8 * inch, 6 * inch))
     text = pdf.beginText()
     text.setFont('Helvetica', 12)
     text.setTextOrigin(1 * inch, 3 * inch)
     text.textLine(
-        "Methink'st thou art a general offence and every" " man should beat thee."
+        "Methink'st thou art a general offence and every man should beat thee."
     )
     pdf.drawText(text)
     pdf.showPage()
     pdf.save()
+    return filename
 
-    info = pdfinfo.PdfInfo(filename)
+
+def test_single_page_text(single_page_text):
+    info = pdfinfo.PdfInfo(single_page_text)
 
     assert len(info) == 1
     page = info[0]
@@ -54,7 +58,8 @@ def eight_by_eight():
     return im
 
 
-def test_single_page_image(eight_by_eight, outpdf):
+@pytest.fixture
+def eight_by_eight_regular_image(eight_by_eight, outpdf):
     im = eight_by_eight
     bio = BytesIO()
     im.save(bio, format='PNG')
@@ -71,7 +76,11 @@ def test_single_page_image(eight_by_eight, outpdf):
             outputstream=f,
             **IMG2PDF_KWARGS,
         )
-    info = pdfinfo.PdfInfo(outpdf)
+    return outpdf
+
+
+def test_single_page_image(eight_by_eight_regular_image):
+    info = pdfinfo.PdfInfo(eight_by_eight_regular_image)
 
     assert len(info) == 1
     page = info[0]
@@ -88,16 +97,18 @@ def test_single_page_image(eight_by_eight, outpdf):
     assert isclose(pdfimage.dpi.y, 8)
 
 
-def test_single_page_inline_image(eight_by_eight, outdir):
-    filename = outdir / 'image-mono-inline.pdf'
-    pdf = Canvas(str(filename), pagesize=(8 * 72, 6 * 72))
-
+@pytest.fixture
+def eight_by_eight_inline_image(eight_by_eight, outpdf):
+    pdf = Canvas(str(outpdf), pagesize=(8 * 72, 6 * 72))
     # Draw image in a 72x72 pt or 1"x1" area
     pdf.drawInlineImage(eight_by_eight, 0, 0, width=72, height=72)
     pdf.showPage()
     pdf.save()
+    return outpdf
 
-    info = pdfinfo.PdfInfo(filename)
+
+def test_single_page_inline_image(eight_by_eight_inline_image):
+    info = pdfinfo.PdfInfo(eight_by_eight_inline_image)
     print(info)
     pdfimage = info[0].images[0]
     assert isclose(pdfimage.dpi.x, 8)
@@ -177,7 +188,7 @@ def test_stack_abuse():
         pdfinfo.info._interpret_contents(stream)
 
     stream = pikepdf.Stream(p, b'q Q Q Q Q')
-    with pytest.warns(UserWarning, match="underflowed") as record:
+    with pytest.warns(UserWarning, match="underflowed"):
         pdfinfo.info._interpret_contents(stream)
 
     stream = pikepdf.Stream(p, b'q ' * 135)
@@ -201,7 +212,8 @@ def test_pages_issue700(monkeypatch, resources):
         )
 
 
-def test_image_scale0(resources, outpdf):
+@pytest.fixture
+def image_scale0(resources, outpdf):
     with pikepdf.open(resources / 'cmyk.pdf') as cmyk:
         xobj = cmyk.pages[0].as_form_xobject()
 
@@ -215,7 +227,12 @@ def test_image_scale0(resources, outpdf):
             p, b"q 0 0 0 0 0 0 cm %s Do Q" % bytes(objname)
         )
         p.save(outpdf)
+    return outpdf
 
-    pi = pdfinfo.PdfInfo(outpdf, detailed_analysis=True, progbar=False, max_workers=1)
+
+def test_image_scale0(image_scale0):
+    pi = pdfinfo.PdfInfo(
+        image_scale0, detailed_analysis=True, progbar=False, max_workers=1
+    )
     assert not pi.pages[0]._images[0].dpi.is_finite
     assert pi.pages[0].dpi == Resolution(0, 0)