unpaper: super syntax

Fixes #974 when used in conjunction with Python 3.10.

Reviewed other uses of TemporaryDirectory in ocrmypdf and decided it
was not worth fixing them since neither are exactly production code.

.docker/Dockerfile

@@ -1,7 +1,7 @@
 # OCRmyPDF
 #
 
-FROM ubuntu:21.04 as base
+FROM ubuntu:22.04 as base
 
 ENV LANG=C.UTF-8
 ENV TZ=UTC

.github/ISSUE_TEMPLATE/2-problem-with-a-specific-input-file.md

@@ -22,7 +22,7 @@ Run with verbosity or higher `-v1` to see more detailed logging. This informatio
 **Example file**
 If your issue is a problem that affects only certain files, and we will require an input file (PDF or image) that demonstrates your issue.
 
-Please provide an input file with no personal or confidential information. At your option you may [GPG-encrypt the file](https://github.com/jbarlow83/OCRmyPDF/wiki) for OCRmyPDF's author only.
+Please provide an input file with no personal or confidential information. At your option you may [GPG-encrypt the file](https://github.com/ocrmypdf/OCRmyPDF/wiki) for OCRmyPDF's author only.
 
 Links to files hosted elsewhere are perfectly acceptable. You could also look in ``tests/resources`` and see if any of those files reproduce your issue.
 

.github/ISSUE_TEMPLATE/3-feature_request.md

@@ -19,7 +19,7 @@ A clear and concise description of any alternative solutions or features you've
 **Example file**
 If your issue concerns how OCRmyPDF processes certain files, and please provide an example file that helps illustrate how OCRmyPDF's output could be improve.
 
-Please provide an input file with no personal or confidential information. At your option you may [GPG-encrypt the file](https://github.com/jbarlow83/OCRmyPDF/wiki) for OCRmyPDF's author only.
+Please provide an input file with no personal or confidential information. At your option you may [GPG-encrypt the file](https://github.com/ocrmypdf/OCRmyPDF/wiki) for OCRmyPDF's author only.
 
 Links to files hosted elsewhere are perfectly acceptable. You could also look in ``tests/resources`` and see if any of those files reproduce your issue.
 

.github/workflows/build.yml

@@ -21,17 +21,19 @@ jobs:
       matrix:
         include:
           - os: ubuntu-18.04
-            python: 3.7
+            python: "3.7"
           - os: ubuntu-20.04
-            python: 3.8
+            python: "3.8"
           - os: ubuntu-20.04
-            python: 3.9
+            python: "3.9"
+          - os: ubuntu-20.04
+            python: "3.10"
           - os: ubuntu-latest
-            python: 3.9
+            python: "3.9"
           - os: ubuntu-latest
-            python: "pypy-3.7"
+            python: "pypy-3.8"
           - os: ubuntu-latest
-            python: 3.9
+            python: "3.9"
             tesseract5: true
 
     env:
@@ -120,7 +122,7 @@ jobs:
     strategy:
       matrix:
         os: [macos-latest]
-        python: ["3.9"]
+        python: ["3.9", "3.10"]
 
     env:
       OS: ${{ matrix.os }}
@@ -175,7 +177,7 @@ jobs:
     strategy:
       matrix:
         os: [windows-latest]
-        python: ["3.9"]
+        python: ["3.9", "3.10"]
 
     env:
       OS: ${{ matrix.os }}
@@ -194,7 +196,7 @@ jobs:
       - name: Install system packages
         run: |
           choco install --yes --no-progress --pre tesseract
-          choco install --yes --no-progress --ignore-checksums ghostscript pngquant
+          choco install --yes --no-progress --ignore-checksums ghostscript
 
       - name: Install Python packages
         run: |

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.0.1
+    rev: v4.2.0
     hooks:
       - id: check-case-conflict
       - id: check-merge-conflict
@@ -8,26 +8,26 @@ repos:
       - id: check-yaml
       - id: debug-statements
   - repo: https://github.com/pycqa/isort
-    rev: 5.9.3
+    rev: 5.10.1
     hooks:
       - id: isort
         args: ["--profile", "black"]
   - repo: https://github.com/psf/black
-    rev: 21.9b0
+    rev: 22.3.0
     hooks:
       - id: black
         language_version: python
   - repo: https://github.com/asottile/setup-cfg-fmt
-    rev: v1.19.0
+    rev: v1.20.1
     hooks:
       - id: setup-cfg-fmt
   - repo: https://github.com/asottile/pyupgrade
-    rev: v2.29.0
+    rev: v2.32.1
     hooks:
       - id: pyupgrade
         args: ["--py37-plus"]
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.910-1
+    rev: v0.950
     hooks:
       - id: mypy
         additional_dependencies:

README.md

@@ -1,9 +1,7 @@
 <img src="docs/images/logo.svg" width="240" alt="OCRmyPDF">
 
-[![Build Status](https://github.com/jbarlow83/OCRmyPDF/actions/workflows/build.yml/badge.svg)](https://github.com/jbarlow83/OCRmyPDF/actions/workflows/build.yml) [![PyPI version][pypi]](https://pypi.org/project/ocrmypdf/) ![Homebrew version][homebrew] ![ReadTheDocs][docs] ![Python versions][pyversions]
+[![Build Status](https://github.com/ocrmypdf/OCRmyPDF/actions/workflows/build.yml/badge.svg)](https://github.com/ocrmypdf/OCRmyPDF/actions/workflows/build.yml) [![PyPI version][pypi]](https://pypi.org/project/ocrmypdf/) ![Homebrew version][homebrew] ![ReadTheDocs][docs] ![Python versions][pyversions]
 
-[azure]: https://dev.azure.com/jim0585/ocrmypdf/_apis/build/status/jbarlow83.OCRmyPDF?branchName=master
-[travis]: https://travis-ci.org/jbarlow83/OCRmyPDF.svg?branch=master "Travis build status"
 [pypi]: https://img.shields.io/pypi/v/ocrmypdf.svg "PyPI version"
 [homebrew]: https://img.shields.io/homebrew/v/ocrmypdf.svg "Homebrew version"
 [docs]: https://readthedocs.org/projects/ocrmypdf/badge/?version=latest "RTD"
@@ -106,7 +104,7 @@ ocrmypdf --help
 
 Our [documentation is served on Read the Docs](https://ocrmypdf.readthedocs.io/en/latest/index.html).
 
-Please report issues on our [GitHub issues](https://github.com/jbarlow83/OCRmyPDF/issues) page, and follow the issue template for quick response.
+Please report issues on our [GitHub issues](https://github.com/ocrmypdf/OCRmyPDF/issues) page, and follow the issue template for quick response.
 
 ## Requirements
 

debian/copyright

@@ -1,7 +1,7 @@
 Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
 Upstream-Name: OCRmyPDF
 Upstream-Contact: James R. Barlow <barlow.jim@gmail.com>
-Source: https://github.com/jbarlow83/OCRmyPDF
+Source: https://github.com/ocrmypdf/OCRmyPDF
 
 Files: *
 Copyright:

docs/advanced.rst

@@ -67,11 +67,11 @@ without modifying the PDF. This is to ensure that PDFs that were
 previously OCRed or were "born digital" rather than scanned are not
 processed.
 
-If ``--skip-text`` is issued, then no OCR will be performed on pages
+If ``--skip-text`` is issued, then no image processing or OCR will be
-that already have text. The page will be copied to the output. This may
+performed on pages that already have text. The page will be copied to
-be useful for documents that contain both "born digital" and scanned
+the output. This may be useful for documents that contain both "born
-content, or to use OCRmyPDF to normalize and convert to PDF/A regardless
+digital" and scanned content, or to use OCRmyPDF to normalize and
-of their contents.
+convert to PDF/A regardless of their contents.
 
 If ``--redo-ocr`` is issued, then a detailed text analysis is performed.
 Text is categorized as either visible or invisible. Invisible text (OCR)

docs/conf.py

@@ -42,7 +42,7 @@ extensions = [
 # Extension settings
 intersphinx_mapping = {'https://docs.python.org/': None}
 napoleon_use_rtype = False
-issues_github_path = "jbarlow83/OCRmyPDF"
+issues_github_path = "ocrmypdf/OCRmyPDF"
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']

docs/cookbook.rst

@@ -243,10 +243,11 @@ You can also optimize all images without performing any OCR:
 
     ocrmypdf --tesseract-timeout=0 --optimize 3 --skip-text input.pdf output.pdf
 
-Perform OCR only certain pages
+Process only certain pages
-------------------------------
+--------------------------
 
-You can ask OCRmyPDF to only apply OCR to certain pages.
+You can ask OCRmyPDF to only apply `image processing <#image-processing>`__
+and OCR to certain pages.
 
 .. code-block:: bash
 
@@ -260,10 +261,10 @@ overlap pages. OCRmyPDF does not currently account for document page numbers,
 such as an introduction section of a book that uses Roman numerals. It simply
 counts the number of virtual pieces of paper since the start.
 
-Regardless of the argument to ``--pages``, OCRmyPDF will optimize all pages in
+Regardless of the argument to ``--pages``, OCRmyPDF will optimize all pages/images
-the file and convert it to PDF/A, unless you disable those options. In this
+in the file and convert it to PDF/A, unless you disable those options. Both of these
-example, we want to OCR only the title and otherwise change the PDF as little
+steps are "whole file" operations. In this example, we want to OCR only the title
-as possible:
+and otherwise change the PDF as little as possible:
 
 .. code-block:: bash
 

docs/docker.rst

@@ -59,7 +59,7 @@ Using the Docker image on the command line
 ==========================================
 
 **Unlike typical Docker containers**, in this section the OCRmyPDF Docker
-container is emphemeral – it runs for one OCR job and terminates, just like a
+container is ephemeral – it runs for one OCR job and terminates, just like a
 command line program. We are using Docker to deliver an application (as opposed
 to the more conventional case, where a Docker container runs as a server).
 

docs/installation.rst

@@ -375,7 +375,7 @@ languages you can optionally install them all:
 .. note::
 
    Users who previously installed OCRmyPDF from the private tap should
-   switch to the mainline version (``brew untap jbarlow83/ocrmypdf``)
+   switch to the mainline version (``brew untap ocrmypdf/OCRmyPDF``)
    and install from there.
 
 Manual installation on macOS
@@ -696,7 +696,7 @@ environment:
 
 .. code-block:: bash
 
-    pip3 install git+https://github.com/jbarlow83/OCRmyPDF.git
+    pip3 install git+https://github.com/ocrmypdf/OCRmyPDF.git
 
 Or, to install in `development
 mode <https://pythonhosted.org/setuptools/setuptools.html#development-mode>`__,
@@ -704,14 +704,14 @@ allowing customization of OCRmyPDF, use the ``-e`` flag:
 
 .. code-block:: bash
 
-    pip3 install -e git+https://github.com/jbarlow83/OCRmyPDF.git
+    pip3 install -e git+https://github.com/ocrmypdf/OCRmyPDF.git
 
 You may find it easiest to install in a virtual environment, rather than
 system-wide:
 
 .. code-block:: bash
 
-    git clone -b master https://github.com/jbarlow83/OCRmyPDF.git
+    git clone -b master https://github.com/ocrmypdf/OCRmyPDF.git
     python3 -m venv
     source venv/bin/activate
     cd OCRmyPDF
@@ -738,7 +738,7 @@ To install all of the development and test requirements:
 
 .. code-block:: bash
 
-    git clone -b master https://github.com/jbarlow83/OCRmyPDF.git
+    git clone -b master https://github.com/ocrmypdf/OCRmyPDF.git
     python3 -m venv
     source venv/bin/activate
     cd OCRmyPDF

docs/jbig2.rst

@@ -32,6 +32,9 @@ For all other Linux, you must build a JBIG2 encoder from source:
 
 .. _jbig2-lossy:
 
+Dependencies include libtoolize and libleptonica, which on Ubuntu systems 
+are packaged as libtool and libleptonica-dev.
+
 Lossy mode JBIG2
 ================
 

docs/languages.rst

@@ -54,6 +54,33 @@ to what languages it should search for. Multiple languages can be
 requested using either ``-l eng+fra`` (English and French) or
 ``-l eng -l fra``.
 
+Gentoo users
+============
+
+On Gentoo the package ``app-text/tessdata_fast``, which ``app-text/tesseract`` depends on, handles Tesseract languages.
+It accepts USE flags to select what languages should be installed, these can be set in ``/etc/portage/package.use``.
+Alternatively one can globally set the `L10N use extension <https://wiki.gentoo.org/wiki/Localization/Guide#L10N>`__ in ``/etc/portage/make.conf``.
+This enables these languages for all packages (e.g. including aspell).
+
+.. code-block:: bash
+
+   # Display a list of all Tesseract language packs
+   equery uses app-text/tessdata_fast
+   
+   # Add English and German language support for Tesseract only
+   echo 'app-text/tessdata_fast l10n_de l10n_en' >> /etc/portage/package.use
+   
+   # Add global English and German language support (the `l10n_` from equery has to be omited)
+   echo L10N="de en" >> /etc/portage/make.conf
+   
+   # update system to reflect changed USE flags
+   emerge --update --deep --newuse @world
+
+You can then pass the ``-l LANG`` argument to OCRmyPDF to give a hint as
+to what languages it should search for. Multiple languages can be
+requested using either ``-l eng+fra`` (English and French) or
+``-l eng -l fra``.
+
 macOS users
 ===========
 

docs/maintainers.rst

@@ -2,7 +2,7 @@
 Maintainer notes
 ================
 
-This is for those who package OCRmyPDF for downstream use.  (Thank you 
+This is for those who package OCRmyPDF for downstream use.  (Thank you
 for your hard work.)
 
 Known ports/packagers
@@ -25,7 +25,7 @@ Non-Python dependencies
 
 Note that we have non-Python dependencies. In particular, OCRmyPDF requires
 Ghostscript and Tesseract OCR to be installed and needs to be able to locate their
-binaries on the system PATH. On Windows, OCRmyPDF will also check the registry 
+binaries on the system PATH. On Windows, OCRmyPDF will also check the registry
 for their locations.
 
 Tesseract OCR relies on SIMD for performance and only has proper support for this
@@ -38,13 +38,13 @@ OCRmyPDF uses setuptools-scm for versioning, which derives the version from
 Git as a single source of truth. This may be unsuitable for some distributions, e.g.
 to indicate that your distribution modifies OCRmyPDF in some way.
 
-You can patch the ``__version__`` variable in ``src/ocrmypdf/_version.py`` if 
+You can patch the ``__version__`` variable in ``src/ocrmypdf/_version.py`` if
 necessary.
 
 OCRmyPDF uses setuptools-scm-git-archive to ensure that tarballs downloaded from
 GitHub contain version information. Unfortunately, these tarballs are not always
-deterministic. See this 
+deterministic. See this
-`issue <https://github.com/jbarlow83/OCRmyPDF/issues/841#issuecomment-936562696>`_.
+`issue <https://github.com/ocrmypdf/OCRmyPDF/issues/841#issuecomment-936562696>`_.
 
 jbig2enc
 --------

docs/plugins.rst

@@ -152,6 +152,16 @@ hooks. As such, you cannot "chain" a series of plugin filters together in this
 way. Instead, a single hook implementation should be responsible for any such
 chaining operations.
 
+Examples
+========
+
+* OCRmyPDF's test suite contains several plugins that are used to simulate certain
+  test conditions.
+* `ocrmypdf-papermerge <https://github.com/papermerge/OCRmyPDF_papermerge>`_ is
+  a production plugin that integrates OCRmyPDF and the Papermerge document
+  management system.
+
+
 Custom command line arguments
 -----------------------------
 

docs/release_notes.rst

@@ -10,7 +10,79 @@ that is, output messages may be improved at any release level, so parsing them
 may be unreliable. Use the API to depend on precise behavior.
 
 The public API may be useful in scripts that launch OCRmyPDF processes or that
-wish to use some of its features for working with PDFs..
+wish to use some of its features for working with PDFs.
+
+The most recent release of OCRmyPDF is |OCRmyPDF PyPI|. Any newer versions
+referred to in these notes may exist the main branch but have not been
+tagged yet.
+
+.. |OCRmyPDF PyPI| image:: https://img.shields.io/pypi/v/ocrmypdf.svg
+
+v13.4.7
+=======
+
+-  Fixed PermissionError when cleaning up temporary files in rare cases. :issue:`974`
+-  Fixed PermissionError when calling ``os.nice`` on platforms that lack it. :issue:`973`
+-  Suppressed some warnings from libxmp during tests.
+
+v13.4.6
+=======
+
+-  Convert error on corrupt ICC profiles into a warning. Thanks to @oscherler.
+
+v13.4.5
+=======
+
+-  Remove upper bound on pdfminer.six version.
+-  Documentation.
+
+v13.4.4
+=======
+
+-  Updated pdfminer.six version.
+-  Docker image changed to Ubuntu 22.04 now that it is released and provides the
+   dependencies we need. This seems more consistent than our recent change to
+   Debian.
+
+v13.4.3
+=======
+
+-  Fix error on pytest.skip() with older versions of pytest.
+-  Documentation updates.
+
+v13.4.2
+=======
+
+-  Worked around a
+   `major regression in Ghostscript 9.56.0 <https://bugs.ghostscript.com/show_bug.cgi?id=705187>`__
+   where **all OCR text is stripped out of the PDF**. It simply removes all text,
+   even generated by software other than OCRmyPDF. Fortunately, we can ask
+   Ghostscript 9.56.0 to use its old behavior that worked correctly for our purposes.
+   Users must avoid the combination (Ghostscript 9.56.0, ocrmypdf <13.4.2) since
+   older versions of OCRmyPDF have no way of detecting that this particular
+   version of Ghostscript removes all OCR text.
+-  Marked pdfminer 20220319 as supported.
+-  Fixed some deprecation warnings from recent versions of Pillow and pytest.
+-  Test suite now covers Python 3.10 (Python 3.10 worked fine before, but was not
+   being tested).
+-  Docker image now uses debian:bookworm-slim as the base image to fix the Docker
+   image build.
+
+v13.4.1
+=======
+
+-  Temporarily make threads rather than processes the default executor worker, due
+   to a persistent deadlock issue when processes are used. Add a new command line
+   argument ``--no-use-threads`` to disable this.
+
+v13.4.0
+=======
+
+-  Fixed test failures when using pikepdf 5.0.0.
+-  Various improvements to the optimizer. In particular, we now recognize PDF images
+   that are encoded with both deflate (PNG) and DCT (JPEG), and also produce PDF
+   with images compressed with deflate and DCT, since this often yields file size
+   improvements compared to plain DCT.
 
 v13.3.0
 =======
@@ -21,8 +93,8 @@ v13.3.0
    C library used by unpaper so it cannot be rectified easily.
 -  We now use better default settings when calling img2pdf.
 -  We no longer try to optimize images that we failed to save in certain situations.
--  We now account for some differences in text output from Tesseract 5 that differs
+-  We now account for some differences in text output from Tesseract 5 compared to
-   from Tesseract 4.
+   Tesseract 4.
 -  Better handling of Ghostscript producing empty images when attempting to rasterize
    page images.
 
@@ -73,7 +145,7 @@ v13.0.0
    Tesseract 5.x has implemented improvements to thresholding, so this feature will be
    redundant anyway.
 -  ``--deskew`` was previous calculated by a Leptonica algorithm. We now use a feature
-   of Tesseract to term the appropriate the angle to deskew a page. The deskew angle
+   of Tesseract to find the appropriate the angle to deskew a page. The deskew angle
    according to Tesseract may differ from Leptonica's algorithm. At least in theory,
    Tesseract's deskew angle is informed by a more complex analysis than Leptonica,
    so this should improve results in general. We also use Pillow to perform the

misc/webservice.py

@@ -59,7 +59,7 @@ def do_ocrmypdf(file):
         return Response("--sidecar not supported", 501, mimetype='text/plain')
 
     ocrmypdf_args = ["ocrmypdf", *cmd_args, up_file, down_file]
-    proc = run(ocrmypdf_args, stdout=PIPE, stderr=PIPE, encoding="utf-8")
+    proc = run(ocrmypdf_args, capture_output=True, encoding="utf-8")
     if proc.returncode != 0:
         stderr = proc.stderr
         return Response(stderr, 400, mimetype='text/plain')

setup.cfg

@@ -3,7 +3,7 @@ name = ocrmypdf
 description = OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched
 long_description = file: README.md
 long_description_content_type = text/markdown
-url = https://github.com/jbarlow83/OCRmyPDF
+url = https://github.com/ocrmypdf/OCRmyPDF
 author = James R. Barlow
 author_email = james@purplerock.ca
 license = MPL-2.0
@@ -39,8 +39,8 @@ keywords =
     scanning
 project_urls =
     Documentation = https://ocrmypdf.readthedocs.io/
-    Source = https://github.com/jbarlow83/ocrmypdf
+    Source = https://github.com/ocrmypdf/OCRmyPDF
-    Tracker = https://github.com/jbarlow83/ocrmypdf/issues
+    Tracker = https://github.com/ocrmypdf/OCRmyPDF/issues
 
 [options]
 packages = find:
@@ -49,8 +49,8 @@ install_requires =
     coloredlogs>=14.0  # strictly optional
     img2pdf>=0.3.0,<0.5  # pure Python
     packaging>=20
-    pdfminer.six!=20200720,>=20191110,<=20211012
+    pdfminer.six!=20200720,>=20191110
-    pikepdf>=4.0.0
+    pikepdf!=5.0.0,>=4.0.0
     pluggy>=0.13.0,<2
     reportlab>=3.5.66
     tqdm>=4

src/ocrmypdf/__main__.py

@@ -10,6 +10,7 @@ import logging
 import os
 import signal
 import sys
+from contextlib import suppress
 from multiprocessing import set_start_method
 
 from ocrmypdf import __version__
@@ -34,7 +35,7 @@ def sigbus(*args):
 def run(args=None):
     _parser, options, plugin_manager = get_parser_options_plugins(args=args)
 
-    if hasattr(os, 'nice'):
+    with suppress(AttributeError, PermissionError):
         os.nice(5)
 
     verbosity = options.verbose

src/ocrmypdf/_exec/ghostscript.py

@@ -24,6 +24,13 @@ from ocrmypdf.exceptions import MissingDependencyError, SubprocessOutputError
 from ocrmypdf.helpers import Resolution
 from ocrmypdf.subprocess import get_version, run, run_polling_stderr
 
+# Remove this workaround when we require Pillow >= 10
+try:
+    Transpose = Image.Transpose  # type: ignore
+except AttributeError:
+    # Pillow 9 shim
+    Transpose = Image  # type: ignore
+
 log = logging.getLogger(__name__)
 
 missing_gs_error = """
@@ -132,11 +139,11 @@ def rasterize_pdf(
                 # rotation is a clockwise angle and Image.ROTATE_* is
                 # counterclockwise so this cancels out the rotation
                 if rotation == 90:
-                    im = im.transpose(Image.ROTATE_90)
+                    im = im.transpose(Transpose.ROTATE_90)
                 elif rotation == 180:
-                    im = im.transpose(Image.ROTATE_180)
+                    im = im.transpose(Transpose.ROTATE_180)
                 elif rotation == 270:
-                    im = im.transpose(Image.ROTATE_270)
+                    im = im.transpose(Transpose.ROTATE_270)
                 if rotation % 180 == 90:
                     page_dpi = page_dpi.flip_axis()
             im.save(fspath(output_file), dpi=page_dpi)
@@ -210,14 +217,18 @@ def generate_pdfa(
     # Older versions of Ghostscript expect a leading slash in
     # sColorConversionStrategy, newer ones should not have it. See Ghostscript
     # git commit fe1c025d.
-    strategy = ('/' + strategy) if version() < '9.19' else strategy
+    gs_version = version()
+    strategy = ('/' + strategy) if gs_version < '9.19' else strategy
 
-    if version() == '9.23':
+    if gs_version == '9.23':
         # 9.23: added JPEG passthrough as a new feature, but with a bug that
         # incorrectly formats some images. Fixed as of 9.24. So we disable this
         # feature for 9.23.
         # https://bugs.ghostscript.com/show_bug.cgi?id=699216
         compression_args.append('-dPassThroughJPEGImages=false')
+    elif gs_version == '9.56.0':
+        # 9.56.0 breaks our OCR...?
+        compression_args.append('-dNEWPDF=false')
 
     # nb no need to specify ProcessColorModel when ColorConversionStrategy
     # is set; see:

src/ocrmypdf/_exec/unpaper.py

@@ -13,18 +13,35 @@
 import logging
 import os
 import shlex
+import sys
 from contextlib import contextmanager
 from decimal import Decimal
 from pathlib import Path
 from subprocess import PIPE, STDOUT
-from tempfile import TemporaryDirectory
+from typing import Iterator, List, Optional, Tuple, Union
-from typing import List, Optional, Tuple, Union
 
 from PIL import Image
 
 from ocrmypdf.exceptions import MissingDependencyError, SubprocessOutputError
 from ocrmypdf.subprocess import get_version, run
 
+if sys.version_info >= (3, 10):
+    from tempfile import TemporaryDirectory
+else:
+    from tempfile import TemporaryDirectory as _TemporaryDirectory
+
+    # Consume the ignore_cleanup_errors kwarg in Python 3.9 and older, without acting
+    # on this keyword. Users who need this issue full resolved should upgrade to Python
+    # 3.10.
+    # See: https://github.com/python/cpython/pull/24793
+
+    class TemporaryDirectory(_TemporaryDirectory):
+        def __init__(self, ignore_cleanup_errors=False, **kwargs):
+            super().__init__(**kwargs)
+
+    del _TemporaryDirectory
+
+
 UNPAPER_IMAGE_PIXEL_LIMIT = 256 * 1024 * 1024
 
 DecFloat = Union[Decimal, float]
@@ -76,13 +93,13 @@ def _convert_image(im: Image.Image) -> Tuple[Image.Image, bool, str]:
 
 
 @contextmanager
-def _setup_unpaper_io(input_file: Path) -> Tuple[Path, Path, Path]:
+def _setup_unpaper_io(input_file: Path) -> Iterator[Tuple[Path, Path, Path]]:
     with Image.open(input_file) as im:
         if im.width * im.height >= UNPAPER_IMAGE_PIXEL_LIMIT:
             raise UnpaperImageTooLargeError(w=im.width, h=im.height)
         im, im_modified, suffix = _convert_image(im)
 
-        with TemporaryDirectory() as tmpdir:
+        with TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
             tmppath = Path(tmpdir)
             if im_modified or input_file.suffix != '.pnm':
                 input_pnm = tmppath / 'input.pnm'

src/ocrmypdf/_pipeline.py

@@ -18,7 +18,7 @@ from typing import Dict, Iterable, Optional
 import img2pdf
 import pikepdf
 from pikepdf.models.metadata import encode_pdf_date
-from PIL import Image, ImageDraw
+from PIL import Image, ImageColor, ImageDraw
 
 from ocrmypdf._concurrent import Executor
 from ocrmypdf._exec import unpaper
@@ -38,6 +38,13 @@ from ocrmypdf.optimize import optimize
 from ocrmypdf.pdfa import generate_pdfa_ps
 from ocrmypdf.pdfinfo import Colorspace, Encoding, PdfInfo
 
+# Remove this workaround when we require Pillow >= 10
+try:
+    BICUBIC = Image.Resampling.BICUBIC  # type: ignore
+except AttributeError:
+    # Pillow 9 shim
+    BICUBIC = Image.BICUBIC  # type: ignore
+
 log = logging.getLogger(__name__)
 
 VECTOR_PAGE_DPI = 400
@@ -484,7 +491,9 @@ def preprocess_deskew(input_file: Path, page_context: PageContext):
         # According to Pillow docs, .rotate() will automatically use Image.NEAREST
         # resampling if image is mode '1' or 'P'
         deskewed = im.rotate(
-            deskew_angle_degrees, resample=Image.BICUBIC, fillcolor='white'
+            deskew_angle_degrees,
+            resample=BICUBIC,
+            fillcolor=ImageColor.getcolor('white', mode=im.mode),
         )
         deskewed.save(output_file, dpi=dpi)
 

src/ocrmypdf/_validation.py

@@ -41,7 +41,7 @@ log = logging.getLogger(__name__)
 
 
 def check_platform():
-    if os.name == 'nt' and sys.maxsize <= 2 ** 32:  # pragma: no cover
+    if os.name == 'nt' and sys.maxsize <= 2**32:  # pragma: no cover
         # 32-bit interpreter on Windows
         log.error(
             "You are running OCRmyPDF in a 32-bit (x86) Python interpreter."

src/ocrmypdf/api.py

@@ -295,7 +295,7 @@ def ocr(  # pylint: disable=unused-argument
             text already, and settings did not tell us to proceed.
         ocrmypdf.InputFileError: Any other problem with the input file.
         ocrmypdf.SubprocessOutputError: Any error related to executing a subprocess.
-        ocrmypdf.EncryptedPdfERror: If the input PDF is encrypted (password protected).
+        ocrmypdf.EncryptedPdfError: If the input PDF is encrypted (password protected).
             OCRmyPDF does not remove passwords.
         ocrmypdf.TesseractConfigError: If Tesseract reported its configuration was not
             valid.

src/ocrmypdf/cli.py

@@ -228,7 +228,13 @@ Online documentation is located at:
         help=argparse.SUPPRESS,
     )
     jobcontrol.add_argument(
-        '--use-threads', action='store_true', help=argparse.SUPPRESS
+        '--use-threads', action='store_true', default=True, help=argparse.SUPPRESS
+    )
+    jobcontrol.add_argument(
+        '--no-use-threads',
+        action='store_false',
+        dest='use_threads',
+        help=argparse.SUPPRESS,
     )
 
     metadata = parser.add_argument_group(

src/ocrmypdf/helpers.py

@@ -220,11 +220,19 @@ def check_pdf(input_file: Path) -> bool:
     else:
         with pdf:
             messages = pdf.check()
+            success = True
             for msg in messages:
                 if 'error' in msg.lower():
                     log.error(msg)
+                    success = False
+                elif (
+                    "/DecodeParms: operation for dictionary attempted on object "
+                    "of type null" in msg
+                ):
+                    pass  # Ignore/spurious warning
                 else:
                     log.warning(msg)
+                    success = False
 
             sio = StringIO()
             linearize_msgs = ''
@@ -239,7 +247,7 @@ def check_pdf(input_file: Path) -> bool:
                 if linearize_msgs:
                     log.warning(linearize_msgs)
 
-            if not messages and not linearize_msgs:
+            if success and not linearize_msgs:
                 return True
             return False
 

src/ocrmypdf/optimize.py

@@ -8,6 +8,7 @@
 import logging
 import sys
 import tempfile
+import threading
 from collections import defaultdict
 from os import fspath
 from pathlib import Path
@@ -23,6 +24,7 @@ from typing import (
     Sequence,
     Tuple,
 )
+from zlib import compress
 
 import img2pdf
 from pikepdf import (
@@ -31,6 +33,7 @@ from pikepdf import (
     Object,
     ObjectStreamMode,
     Pdf,
+    PdfError,
     PdfImage,
     Stream,
     UnsupportedImageTypeError,
@@ -78,33 +81,49 @@ def extract_image_filter(
     if image.Subtype != Name.Image:
         return None
     if image.Length < 100:
-        log.debug(f"Skipping small image, xref {xref}")
+        log.debug(f"xref {xref}: skipping image with small stream size")
         return None
     if image.Width < 8 or image.Height < 8:  # Issue 732
-        log.debug(f"Skipping oddly sized image, xref {xref}")
+        log.debug(f"xref {xref}: skipping image with unusually small dimensions")
         return None
 
     pim = PdfImage(image)
 
     if len(pim.filter_decodeparms) > 1:
-        log.debug(f"Skipping multiply filtered image, xref {xref}")
+        first_filtdp = pim.filter_decodeparms[0]
-        return None
+        second_filtdp = pim.filter_decodeparms[1]
-    filtdp = pim.filter_decodeparms[0]
+        if (
+            len(pim.filter_decodeparms) == 2
+            and first_filtdp[0] == Name.FlateDecode
+            and first_filtdp[1].get(Name.Predictor, 1) == 1
+            and second_filtdp[0] == Name.DCTDecode
+            and not second_filtdp[1]
+        ):
+            log.debug(
+                f"xref {xref}: found image compressed as /FlateDecode /DCTDecode, "
+                "marked for JPEG optimization"
+            )
+            filtdp = pim.filter_decodeparms[1]
+        else:
+            log.debug(f"xref {xref}: skipping image with multiple compression filters")
+            return None
+    else:
+        filtdp = pim.filter_decodeparms[0]
 
     if pim.bits_per_component > 8:
-        log.debug(f"Skipping wide gamut image, xref {xref}")
+        log.debug(f"xref {xref}: skipping wide gamut image")
         return None  # Don't mess with wide gamut images
 
     if filtdp[0] == Name.JPXDecode:
-        log.debug(f"Skipping JPEG2000 image, xref {xref}")
+        log.debug(f"xref {xref}: skipping JPEG2000 image")
         return None  # Don't do JPEG2000
 
     if filtdp[0] == Name.CCITTFaxDecode and filtdp[1].get('/K', 0) >= 0:
-        log.debug(f"Skipping CCITT Group 3 image, xref {xref}")
+        log.debug(f"xref {xref}: skipping CCITT Group 3 image")
         return None  # pikepdf doesn't support Group 3 yet
 
     if Name.Decode in image:
-        log.debug(f"Skipping image with Decode table, xref {xref}")
+        log.debug(f"xref {xref}: skipping image with Decode table")
         return None  # Don't mess with custom Decode tables
 
     return pim, filtdp
@@ -172,14 +191,6 @@ def extract_image_generic(
         # jpeg_quality_estimate = 117.0 * (bytes_per_pixel ** 0.213)
         # if jpeg_quality_estimate < 65:
         #     return None
-
-        # We could get the ICC profile here, but there's no need to look at it
-        # for quality transcoding
-        # if icc:
-        #     stream = BytesIO(raw_jpeg.read_raw_bytes())
-        #     iccbytes = icc.read_bytes()
-        #     with Image.open(stream) as im:
-        #         im.save(jpg_name(root, xref), icc_profile=iccbytes)
         try:
             imgname = root / f'{xref:08d}'
             with imgname.open('wb') as f:
@@ -259,9 +270,9 @@ def extract_images(
                 # Ignore soft masks
                 smask_xref = Xref(image.SMask.objgen[0])
                 exclude_xrefs.add(smask_xref)
-                log.debug(f"Skipping image {smask_xref} because it is an SMask")
+                log.debug(f"xref {smask_xref}: skipping image because it is an SMask")
             include_xrefs.add(xref)
-            log.debug(f"Treating {xref} as an optimization candidate")
+            log.debug(f"xref {xref}: treating as an optimization candidate")
             if xref not in pageno_for_xref:
                 pageno_for_xref[xref] = pageno
 
@@ -273,7 +284,9 @@ def extract_images(
                 pike=pike, root=root, image=image, xref=xref, options=options
             )
         except Exception:  # pylint: disable=broad-except
-            log.exception(f"While extracting image xref {xref}, an error occurred")
+            log.exception(
+                f"xref {xref}: While extracting this image, an error occurred"
+            )
             errors += 1
         else:
             if result:
@@ -294,7 +307,7 @@ def extract_images_generic(
             pngs.append(xref_ext.xref)
         elif xref_ext.ext == '.jpg':
             jpegs.append(xref_ext.xref)
-    log.debug("Optimizable images: JPEGs: %s PNGs: %s", len(jpegs), len(pngs))
+    log.debug(f"Optimizable images: JPEGs: {len(jpegs)} PNGs: {len(pngs)}")
     return jpegs, pngs
 
 
@@ -306,7 +319,7 @@ def extract_images_jbig2(pike: Pdf, root: Path, options) -> Dict[int, List[XrefE
         group = pageno // options.jbig2_page_group_size
         jbig2_groups[group].append(xref_ext)
 
-    log.debug("Optimizable images: JBIG2 groups: %s", (len(jbig2_groups),))
+    log.debug(f"Optimizable images: JBIG2 groups: {len(jbig2_groups)}")
     return jbig2_groups
 
 
@@ -405,15 +418,11 @@ def convert_to_jbig2(
 def _optimize_jpeg(args: Tuple[Xref, Path, Path, int]) -> Tuple[Xref, Optional[Path]]:
     xref, in_jpg, opt_jpg, jpeg_quality = args
 
-    # This may produce a debug warning from PIL
-    # DEBUG:PIL.Image:Error closing: 'NoneType' object has no attribute
-    # 'close'.  Seems to be mostly harmless
-    # https://github.com/python-pillow/Pillow/issues/1144
     with Image.open(in_jpg) as im:
         im.save(opt_jpg, optimize=True, quality=jpeg_quality)
 
     if opt_jpg.stat().st_size > in_jpg.stat().st_size:
-        log.debug("xref %s, jpeg, made larger - skip", xref)
+        log.debug(f"xref {xref}, jpeg, made larger - skip")
         opt_jpg.unlink()
         return xref, None
     return xref, opt_jpg
@@ -440,7 +449,7 @@ def transcode_jpegs(
         use_threads=True,  # Processes are significantly slower at this task
         max_workers=options.jobs,
         tqdm_kwargs=dict(
-            desc="JPEGs",
+            desc="Recompressing JPEGs",
             total=len(jpegs),
             unit='image',
             disable=not options.progress_bar,
@@ -451,6 +460,73 @@ def transcode_jpegs(
     )
 
 
+def _find_deflatable_jpeg(
+    *, pike: Pdf, root: Path, image: Stream, xref: Xref, options
+) -> Optional[XrefExt]:
+    result = extract_image_filter(pike, root, image, xref)
+    if result is None:
+        return None
+    pim, filtdp = result
+
+    if filtdp[0] == Name.DCTDecode and not filtdp[1] and options.optimize >= 1:
+        return XrefExt(xref, '.memory')
+
+    return None
+
+
+def _deflate_jpeg(args: Tuple[Pdf, threading.Lock, Xref, int]) -> Tuple[Xref, bytes]:
+    pike, lock, xref, complevel = args
+    with lock:
+        xobj = pike.get_object(xref, 0)
+        try:
+            data = xobj.read_raw_bytes()
+        except PdfError:
+            return xref, b''
+    compdata = compress(data, complevel)
+    if len(compdata) >= len(data):
+        return xref, b''
+    return xref, compdata
+
+
+def deflate_jpegs(pike: Pdf, root: Path, options, executor: Executor) -> None:
+    jpegs = []
+    for _pageno, xref_ext in extract_images(pike, root, options, _find_deflatable_jpeg):
+        xref = xref_ext.xref
+        log.debug(f'xref {xref}: marking this JPEG as deflatable')
+        jpegs.append(xref)
+
+    complevel = 9 if options.optimize == 3 else 6
+
+    # Our calls to xobj.write() in finish() need coordination
+    lock = threading.Lock()
+
+    def deflate_args() -> Iterator:
+        for xref in jpegs:
+            yield pike, lock, xref, complevel
+
+    def finish(result, pbar):
+        xref, compdata = result
+        if len(compdata) > 0:
+            with lock:
+                xobj = pike.get_object(xref, 0)
+                xobj.write(compdata, filter=[Name.FlateDecode, Name.DCTDecode])
+        pbar.update()
+
+    executor(
+        use_threads=True,  # We're sharing the pdf directly, must use threads
+        max_workers=options.jobs,
+        tqdm_kwargs=dict(
+            desc="Deflating JPEGs",
+            total=len(jpegs),
+            unit='image',
+            disable=not options.progress_bar,
+        ),
+        task=_deflate_jpeg,
+        task_arguments=deflate_args(),
+        task_finished=finish,
+    )
+
+
 def _transcode_png(pike: Pdf, filename: Path, xref: Xref) -> bool:
     output = filename.with_suffix('.png.pdf')
     with output.open('wb') as f:
@@ -564,6 +640,7 @@ def optimize(
 
         jpegs, pngs = extract_images_generic(pike, root, options)
         transcode_jpegs(pike, jpegs, root, options, executor)
+        deflate_jpegs(pike, root, options, executor)
         # if options.optimize >= 2:
         # Try pngifying the jpegs
         #    transcode_pngs(pike, jpegs, jpg_name, root, options)

src/ocrmypdf/pdfinfo/info.py

@@ -25,6 +25,7 @@ from typing import (
     Mapping,
     NamedTuple,
     Optional,
+    Sequence,
     Tuple,
     Union,
 )
@@ -36,6 +37,7 @@ from pikepdf import (
     PdfImage,
     PdfInlineImage,
     PdfMatrix,
+    UnsupportedImageTypeError,
     parse_content_stream,
 )
 
@@ -349,13 +351,20 @@ class ImageInfo:
 
         if self._color == Colorspace.icc:
             # Check the ICC profile to determine actual colorspace
-            pim_icc = pim.icc
+            try:
-            if pim_icc.profile.xcolor_space == 'GRAY':
+                pim_icc = pim.icc
-                self._comp = 1
+                if pim_icc.profile.xcolor_space == 'GRAY':
-            elif pim_icc.profile.xcolor_space == 'CMYK':
+                    self._comp = 1
-                self._comp = 4
+                elif pim_icc.profile.xcolor_space == 'CMYK':
-            else:
+                    self._comp = 4
-                self._comp = 3
+                else:
+                    self._comp = 3
+            except UnsupportedImageTypeError as ex:
+                self._comp = None
+                logger.warning(
+                    f"An image with a corrupt or unreadable ICC profile was found. "
+                    f"The output PDF may not match the input PDF visually: {ex}. {self}"
+                )
         else:
             if isinstance(self._color, Colorspace):
                 self._comp = FRIENDLY_COMP.get(self._color)
@@ -408,15 +417,10 @@ class ImageInfo:
         return _get_dpi(self._shorthand, (self._width, self._height))
 
     def __repr__(self):
-        class_locals = {
-            attr: getattr(self, attr, None)
-            for attr in dir(self)
-            if not attr.startswith('_')
-        }
         return (
-            "<ImageInfo '{name}' {type_} {width}x{height} {color} "
+            f"<ImageInfo '{self.name}' {self.type_} {self.width}x{self.height} "
-            "{comp} {bpc} {enc} {dpi}>"
+            f"{self.color} {self.comp} {self.bpc} {self.enc} {self.dpi}>"
-        ).format(**class_locals)
+        )
 
 
 def _find_inline_images(contentsinfo: ContentsInfo) -> Iterator[ImageInfo]:
@@ -646,8 +650,8 @@ def _pdf_pageinfo_concurrent(
     max_workers,
     check_pages,
     detailed_analysis=False,
-):
+) -> Sequence[Optional['PageInfo']]:
-    pages = [None] * len(pdf.pages)
+    pages: Sequence[Optional['PageInfo']] = [None] * len(pdf.pages)
 
     def update_pageinfo(result, pbar):
         page = result
@@ -918,17 +922,17 @@ class PdfInfo:
                     self._has_acroform = True
 
     @property
-    def pages(self):
+    def pages(self) -> Sequence[Optional[PageInfo]]:
         return self._pages
 
     @property
     def min_version(self) -> str:
         # The minimum PDF is the maximum version that any particular page needs
-        return max(page.min_version for page in self.pages)
+        return max(page.min_version for page in self.pages if page)
 
     @property
     def has_userunit(self) -> bool:
-        return any(page.userunit != 1.0 for page in self.pages)
+        return any(page.userunit != 1.0 for page in self.pages if page)
 
     @property
     def has_acroform(self) -> bool:

src/ocrmypdf/pluginspec.py

@@ -132,6 +132,7 @@ def get_progressbar_class():
     Here is how OCRmyPDF will use the progress bar:
 
     Example:
+
         pbar_class = pm.hook.get_progressbar_class()
         with pbar_class(**tqdm_kwargs) as pbar:
             ...
@@ -235,9 +236,9 @@ def filter_page_image(page: 'PageContext', image_filename: Path) -> Path:
     ``image_filename``. The hook may overwrite ``image_filename`` with a new file.
 
     The output image should preserve the same physical unit dimensions, that is
-    (width * dpi_x, height * dpi_y). That is, if the image is resized, the DPI
+    ``(width * dpi_x, height * dpi_y)``. That is, if the image is resized, the DPI
     must be adjusted by the reciprocal. If this is not preserved, the PDF page
-    will be resized and the OCR layer misaligned. OCRmyPDF does not nothing
+    will be resized and the OCR layer misaligned. OCRmyPDF does nothing
     to enforce these constraints; it is up to the plugin to do sensible things.
 
     OCRmyPDF will create the PDF page based on the image format used (unless the
@@ -399,8 +400,7 @@ def get_ocr_engine() -> OcrEngine:
     """Returns an OcrEngine to use for processing this file.
 
     The OcrEngine may be instantiated multiple times, by both the main process
-    and child process. As such, it must be obtain store any state in ``options``
+    and child process.
-    or some common location.
 
     Note:
         This is a :ref:`firstresult hook<firstresult>`.

src/ocrmypdf/subprocess/_windows.py

@@ -4,6 +4,9 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+# type: ignore
+# Non-Windows mypy now breaks when trying to typecheck winreg
+
 import logging
 import os
 import shutil
@@ -17,6 +20,7 @@ try:
 except ModuleNotFoundError as e:
     raise ModuleNotFoundError("This module is for Windows only") from e
 
+
 log = logging.getLogger(__name__)
 
 T = TypeVar('T')

tests/conftest.py

@@ -134,8 +134,7 @@ def run_ocrmypdf(
 
     p = run(
         p_args,
-        stdout=PIPE,
+        capture_output=True,
-        stderr=PIPE,
         text=text,
         check=False,
     )

tests/plugins/tesseract_cache.py

@@ -50,6 +50,7 @@ import logging
 import platform
 import re
 import shutil
+import threading
 from functools import partial
 from pathlib import Path
 from subprocess import PIPE, CalledProcessError, CompletedProcess
@@ -176,26 +177,40 @@ def cached_run(options, run_args, **run_kwargs):
 
 
 class CacheOcrEngine(TesseractOcrEngine):
+    # Concurrent threads (with --use-threads) might try to use different parts
+    # of the OcrEngine, so we need a lock to protect the state of patched
+    # module whenever it's patched. Should refactor ocrmypdf._exec.tesseract so that
+    # it does not to be patched at all for testing.
+    lock = threading.Lock()
+
     @staticmethod
     def get_orientation(input_file, options):
-        with patch('ocrmypdf._exec.tesseract.run', new=partial(cached_run, options)):
+        with CacheOcrEngine.lock, patch(
+            'ocrmypdf._exec.tesseract.run', new=partial(cached_run, options)
+        ):
             return TesseractOcrEngine.get_orientation(input_file, options)
 
     @staticmethod
     def get_deskew(input_file, options) -> float:
-        with patch('ocrmypdf._exec.tesseract.run', new=partial(cached_run, options)):
+        with CacheOcrEngine.lock, patch(
+            'ocrmypdf._exec.tesseract.run', new=partial(cached_run, options)
+        ):
             return TesseractOcrEngine.get_deskew(input_file, options)
 
     @staticmethod
     def generate_hocr(input_file, output_hocr, output_text, options):
-        with patch('ocrmypdf._exec.tesseract.run', new=partial(cached_run, options)):
+        with CacheOcrEngine.lock, patch(
+            'ocrmypdf._exec.tesseract.run', new=partial(cached_run, options)
+        ):
             TesseractOcrEngine.generate_hocr(
                 input_file, output_hocr, output_text, options
             )
 
     @staticmethod
     def generate_pdf(input_file, output_pdf, output_text, options):
-        with patch('ocrmypdf._exec.tesseract.run', new=partial(cached_run, options)):
+        with CacheOcrEngine.lock, patch(
+            'ocrmypdf._exec.tesseract.run', new=partial(cached_run, options)
+        ):
             TesseractOcrEngine.generate_pdf(
                 input_file, output_pdf, output_text, options
             )

tests/test_completion.py

@@ -24,8 +24,7 @@ def test_fish():
             ['fish', '-n', 'misc/completion/ocrmypdf.fish'],
             check=True,
             encoding='utf-8',
-            stdout=PIPE,
+            capture_output=True,
-            stderr=PIPE,
         )
         assert proc.stderr == '', proc.stderr
     except FileNotFoundError:
@@ -41,8 +40,7 @@ def test_bash():
             ['bash', '-n', 'misc/completion/ocrmypdf.bash'],
             check=True,
             encoding='utf-8',
-            stdout=PIPE,
+            capture_output=True,
-            stderr=PIPE,
         )
         assert proc.stderr == '', proc.stderr
     except FileNotFoundError:

tests/test_concurrency.py

@@ -13,6 +13,7 @@ from ocrmypdf import ExitCode
 from .conftest import run_ocrmypdf_api
 
 
+@pytest.mark.skipif(True, reason="--use-threads is currently default")
 @pytest.mark.skipif(os.name == 'nt', reason="Windows doesn't have SIGKILL")
 def test_simulate_oom_killer(resources, no_outpdf):
     exitcode = run_ocrmypdf_api(

tests/test_main.py

@@ -465,12 +465,18 @@ def test_overlay(resources, outpdf):
     )
 
 
-def test_destination_not_writable(resources, outdir):
+@pytest.fixture
-    if os.name != 'nt' and (os.getuid() == 0 or os.geteuid() == 0):
+def protected_file(outdir):
-        pytest.xfail(reason="root can write to anything")
     protected_file = outdir / 'protected.pdf'
     protected_file.touch()
     protected_file.chmod(0o400)  # Read-only
+    yield protected_file
+
+
+@pytest.mark.skipif(
+    os.name == 'nt' or os.geteuid() == 0, reason="root can write to anything"
+)
+def test_destination_not_writable(resources, protected_file):
     p = run_ocrmypdf(
         resources / 'jbig2.pdf',
         protected_file,
@@ -480,7 +486,8 @@ def test_destination_not_writable(resources, outdir):
     assert p.returncode == ExitCode.file_access_error, "Expected error"
 
 
-def test_tesseract_config_valid(resources, outdir):
+@pytest.fixture
+def valid_tess_config(outdir):
     cfg_file = outdir / 'test.cfg'
     with cfg_file.open('w') as f:
         f.write(
@@ -490,20 +497,22 @@ language_model_penalty_non_dict_word 0
 language_model_penalty_non_freq_dict_word 0
 '''
         )
+    yield cfg_file
 
+
+def test_tesseract_config_valid(resources, valid_tess_config, outpdf):
     check_ocrmypdf(
         resources / '3small.pdf',
-        outdir / 'out.pdf',
+        outpdf,
         '--tesseract-config',
-        cfg_file,
+        valid_tess_config,
         '--pages',
         '1',
     )
 
 
-@pytest.mark.slow  # This test sometimes times out in CI
+@pytest.fixture
-@pytest.mark.parametrize('renderer', RENDERERS)
+def invalid_tess_config(outdir):
-def test_tesseract_config_invalid(renderer, resources, outdir):
     cfg_file = outdir / 'test.cfg'
     with cfg_file.open('w') as f:
         f.write(
@@ -511,14 +520,19 @@ def test_tesseract_config_invalid(renderer, resources, outdir):
 THIS FILE IS INVALID
 '''
         )
+    yield cfg_file
 
+
+@pytest.mark.slow  # This test sometimes times out in CI
+@pytest.mark.parametrize('renderer', RENDERERS)
+def test_tesseract_config_invalid(renderer, resources, invalid_tess_config, outpdf):
     p = run_ocrmypdf(
         resources / 'ccitt.pdf',
-        outdir / 'out.pdf',
+        outpdf,
         '--pdf-renderer',
         renderer,
         '--tesseract-config',
-        cfg_file,
+        invalid_tess_config,
     )
     assert (
         "parameter not found" in p.stderr.lower()
@@ -630,8 +644,7 @@ def test_compression_preserved(ocrmypdf_exec, resources, image, outpdf):
         ]
         p = run(
             p_args,
-            stdout=PIPE,
+            capture_output=True,
-            stderr=PIPE,
             stdin=input_stream,
             text=True,
             check=False,
@@ -691,8 +704,7 @@ def test_compression_changed(ocrmypdf_exec, resources, image, compression, outpd
         ]
         p = run(
             p_args,
-            stdout=PIPE,
+            capture_output=True,
-            stderr=PIPE,
             stdin=input_stream,
             text=True,
             check=False,
@@ -803,6 +815,9 @@ def test_text_curves(resources, outpdf):
         info = PdfInfo(outpdf)
         assert len(info.pages[0].images) == 0, "added images to the vector PDF"
 
+
+def test_text_curves_force(resources, outpdf):
+    with patch('ocrmypdf._pipeline.VECTOR_PAGE_DPI', 100):
         check_ocrmypdf(
             resources / 'vector.pdf',
             outpdf,
@@ -924,3 +939,25 @@ def test_outputtype_none(resources, outtxt):
         'tests/plugins/tesseract_noop.py',
     )
     assert p.returncode == ExitCode.ok
+
+
+@pytest.fixture
+def graph_bad_icc(resources, outdir):
+    synth_input_file = outdir / 'graph-bad-icc.pdf'
+    with pikepdf.open(resources / 'graph.pdf') as pdf:
+        icc = pdf.make_stream(
+            b'invalid icc profile', N=3, Alternate=pikepdf.Name.DeviceRGB
+        )
+        pdf.pages[0].Resources.XObject['/Im0'].ColorSpace = pikepdf.Array(
+            [pikepdf.Name.ICCBased, icc]
+        )
+        pdf.save(synth_input_file)
+        yield synth_input_file
+
+
+def test_corrupt_icc(graph_bad_icc, outpdf, caplog):
+    result = run_ocrmypdf_api(graph_bad_icc, outpdf)
+    assert result == ExitCode.ok
+    assert any(
+        'corrupt or unreadable ICC profile' in rec.message for rec in caplog.records
+    )

tests/test_metadata.py

@@ -6,10 +6,10 @@
 
 
 import datetime
+import warnings
 from datetime import timezone
 from os import fspath
 from shutil import copyfile
-from unittest.mock import patch
 
 import pikepdf
 import pytest
@@ -173,6 +173,19 @@ def test_creation_date_preserved(output_type, resources, infile, outpdf):
     assert seconds_between_dates(date_after, datetime.datetime.now(timezone.utc)) < 1000
 
 
+@pytest.fixture
+def libxmp_file_to_dict():
+    try:
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", DeprecationWarning)
+            from libxmp.utils import (
+                file_to_dict,  # pylint: disable=import-outside-toplevel
+            )
+    except Exception:  # pylint: disable=broad-except
+        pytest.skip("libxmp not available or libexempi3 not installed")
+    return file_to_dict
+
+
 @pytest.mark.parametrize(
     'test_file,output_type',
     [
@@ -182,15 +195,12 @@ def test_creation_date_preserved(output_type, resources, infile, outpdf):
         ('3small.pdf', 'pdfa'),
     ],
 )
-def test_xml_metadata_preserved(test_file, output_type, resources, outpdf):
+def test_xml_metadata_preserved(
+    libxmp_file_to_dict, test_file, output_type, resources, outpdf
+):
     input_file = resources / test_file
 
-    try:
+    before = libxmp_file_to_dict(str(input_file))
-        from libxmp.utils import file_to_dict  # pylint: disable=import-outside-toplevel
-    except Exception:  # pylint: disable=broad-except
-        pytest.skip("libxmp not available or libexempi3 not installed")
-
-    before = file_to_dict(str(input_file))
 
     check_ocrmypdf(
         input_file,
@@ -202,7 +212,7 @@ def test_xml_metadata_preserved(test_file, output_type, resources, outpdf):
         'tests/plugins/tesseract_noop.py',
     )
 
-    after = file_to_dict(str(outpdf))
+    after = libxmp_file_to_dict(str(outpdf))
 
     equal_properties = [
         'dc:contributor',

tests/test_optimize.py

@@ -81,7 +81,7 @@ def test_jbig2_lossy(lossy, resources, outpdf):
         '--image-dpi',
         '200',
         '--optimize',
-        3,
+        '3',
         '--jpg-quality',
         '50',
         '--png-quality',

tests/test_pdfa.py

@@ -4,23 +4,31 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import os
+
 import pikepdf
 import pytest
 
+from ocrmypdf.exceptions import MissingDependencyError
+
 from .conftest import check_ocrmypdf
 
 
 @pytest.mark.parametrize('optimize', (0, 3))
 @pytest.mark.parametrize('pdfa_level', (1, 2, 3))
 def test_pdfa(resources, outpdf, optimize, pdfa_level):
-    check_ocrmypdf(
+    try:
-        resources / 'francais.pdf',
+        check_ocrmypdf(
-        outpdf,
+            resources / 'francais.pdf',
-        '--plugin',
+            outpdf,
-        'tests/plugins/tesseract_noop.py',
+            '--plugin',
-        f'--output-type=pdfa-{pdfa_level}',
+            'tests/plugins/tesseract_noop.py',
-        f'--optimize={optimize}',
+            f'--output-type=pdfa-{pdfa_level}',
-    )
+            f'--optimize={optimize}',
+        )
+    except MissingDependencyError as e:
+        if 'pngquant' in str(e) and optimize in (2, 3) and os.name == 'nt':
+            pytest.xfail("pngquant currently not available on Windows")
     if pdfa_level in (2, 3):
         # PDF/A-2 allows ObjStm
         assert b'/ObjStm' in outpdf.read_bytes()

tests/test_pdfinfo.py

@@ -173,17 +173,15 @@ def test_stack_abuse():
     p = pikepdf.Pdf.new()
 
     stream = pikepdf.Stream(p, b'q ' * 35)
-    with pytest.warns(None) as record:
+    with pytest.warns(UserWarning, match="overflowed"):
         pdfinfo.info._interpret_contents(stream)
-    assert 'overflowed' in str(record[0].message)
 
     stream = pikepdf.Stream(p, b'q Q Q Q Q')
-    with pytest.warns(None) as record:
+    with pytest.warns(UserWarning, match="underflowed") as record:
         pdfinfo.info._interpret_contents(stream)
-    assert 'underflowed' in str(record[0].message)
 
     stream = pikepdf.Stream(p, b'q ' * 135)
-    with pytest.warns(None):
+    with pytest.warns(UserWarning):
         with pytest.raises(RuntimeError):
             pdfinfo.info._interpret_contents(stream)
 

tests/test_rotation.py

@@ -25,6 +25,12 @@ from .conftest import check_ocrmypdf, run_ocrmypdf
 
 # pylintx: disable=unused-variable
 
+# Remove this workaround when we require Pillow >= 10
+try:
+    Transpose = Image.Transpose  # type: ignore
+except AttributeError:
+    # Pillow 9 shim
+    Transpose = Image  # type: ignore
 
 RENDERERS = ['hocr', 'sandwich']
 
@@ -222,7 +228,7 @@ def test_rotate_page_level(image_angle, page_angle, resources, outdir):
         with Image.open(fspath(resources / 'typewriter.png')) as im:
             if image_angle != 0:
                 ccw_angle = -image_angle % 360
-                im = im.transpose(getattr(Image, f'ROTATE_{ccw_angle}'))
+                im = im.transpose(getattr(Transpose, f'ROTATE_{ccw_angle}'))
             im.save(memimg, format='PNG')
         memimg.seek(0)
         mempdf = BytesIO()

tests/test_stdio.py

@@ -30,12 +30,12 @@ def test_stdin(ocrmypdf_exec, resources, outpdf):
             '--plugin',
             'tests/plugins/tesseract_noop.py',
         ]
-        run(p_args, stdout=PIPE, stderr=PIPE, stdin=input_stream, check=True)
+        run(p_args, capture_output=True, stdin=input_stream, check=True)
 
 
 def test_stdout(ocrmypdf_exec, resources, outpdf):
     if 'COV_CORE_DATAFILE' in os.environ:
-        pytest.skip(msg="Coverage uses stdout")
+        pytest.skip("Coverage uses stdout")
 
     input_file = str(resources / 'francais.pdf')
     output_file = str(outpdf)
@@ -72,7 +72,7 @@ def test_bad_locale(monkeypatch):
 )
 def test_dev_null(resources):
     if 'COV_CORE_DATAFILE' in os.environ:
-        pytest.skip(msg="Coverage uses stdout")
+        pytest.skip("Coverage uses stdout")
 
     p = run_ocrmypdf(
         resources / 'trivial.pdf',