unpaper: super syntax

Fixes #974 when used in conjunction with Python 3.10.

Reviewed other uses of TemporaryDirectory in ocrmypdf and decided it
was not worth fixing them since neither are exactly production code.

.docker/Dockerfile

@@ -1,7 +1,7 @@
 # OCRmyPDF
 #
 
-FROM ubuntu:21.04 as base
+FROM ubuntu:22.04 as base
 
 ENV LANG=C.UTF-8
 ENV TZ=UTC

.github/ISSUE_TEMPLATE/2-problem-with-a-specific-input-file.md

@@ -22,7 +22,7 @@ Run with verbosity or higher `-v1` to see more detailed logging. This informatio
 **Example file**
 If your issue is a problem that affects only certain files, and we will require an input file (PDF or image) that demonstrates your issue.
 
-Please provide an input file with no personal or confidential information. At your option you may [GPG-encrypt the file](https://github.com/jbarlow83/OCRmyPDF/wiki) for OCRmyPDF's author only.
+Please provide an input file with no personal or confidential information. At your option you may [GPG-encrypt the file](https://github.com/ocrmypdf/OCRmyPDF/wiki) for OCRmyPDF's author only.
 
 Links to files hosted elsewhere are perfectly acceptable. You could also look in ``tests/resources`` and see if any of those files reproduce your issue.
 

.github/ISSUE_TEMPLATE/3-feature_request.md

@@ -19,7 +19,7 @@ A clear and concise description of any alternative solutions or features you've
 **Example file**
 If your issue concerns how OCRmyPDF processes certain files, and please provide an example file that helps illustrate how OCRmyPDF's output could be improve.
 
-Please provide an input file with no personal or confidential information. At your option you may [GPG-encrypt the file](https://github.com/jbarlow83/OCRmyPDF/wiki) for OCRmyPDF's author only.
+Please provide an input file with no personal or confidential information. At your option you may [GPG-encrypt the file](https://github.com/ocrmypdf/OCRmyPDF/wiki) for OCRmyPDF's author only.
 
 Links to files hosted elsewhere are perfectly acceptable. You could also look in ``tests/resources`` and see if any of those files reproduce your issue.
 

.github/workflows/build.yml

@@ -21,17 +21,19 @@ jobs:
       matrix:
         include:
           - os: ubuntu-18.04
-            python: 3.7
+            python: "3.7"
           - os: ubuntu-20.04
-            python: 3.8
+            python: "3.8"
           - os: ubuntu-20.04
-            python: 3.9
+            python: "3.9"
+          - os: ubuntu-20.04
+            python: "3.10"
           - os: ubuntu-latest
-            python: 3.9
+            python: "3.9"
           - os: ubuntu-latest
-            python: "pypy-3.7"
+            python: "pypy-3.8"
           - os: ubuntu-latest
-            python: 3.9
+            python: "3.9"
             tesseract5: true
 
     env:
@@ -120,7 +122,7 @@ jobs:
     strategy:
       matrix:
         os: [macos-latest]
-        python: ["3.9"]
+        python: ["3.9", "3.10"]
 
     env:
       OS: ${{ matrix.os }}
@@ -175,7 +177,7 @@ jobs:
     strategy:
       matrix:
         os: [windows-latest]
-        python: ["3.9"]
+        python: ["3.9", "3.10"]
 
     env:
       OS: ${{ matrix.os }}
@@ -194,7 +196,7 @@ jobs:
       - name: Install system packages
         run: |
           choco install --yes --no-progress --pre tesseract
-          choco install --yes --no-progress --ignore-checksums ghostscript pngquant
+          choco install --yes --no-progress --ignore-checksums ghostscript
 
       - name: Install Python packages
         run: |

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.0.1
+    rev: v4.2.0
     hooks:
       - id: check-case-conflict
       - id: check-merge-conflict
@@ -8,26 +8,26 @@ repos:
       - id: check-yaml
       - id: debug-statements
   - repo: https://github.com/pycqa/isort
-    rev: 5.9.3
+    rev: 5.10.1
     hooks:
       - id: isort
         args: ["--profile", "black"]
   - repo: https://github.com/psf/black
-    rev: 21.9b0
+    rev: 22.3.0
     hooks:
       - id: black
         language_version: python
   - repo: https://github.com/asottile/setup-cfg-fmt
-    rev: v1.19.0
+    rev: v1.20.1
     hooks:
       - id: setup-cfg-fmt
   - repo: https://github.com/asottile/pyupgrade
-    rev: v2.29.0
+    rev: v2.32.1
     hooks:
       - id: pyupgrade
         args: ["--py37-plus"]
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.910-1
+    rev: v0.950
     hooks:
       - id: mypy
         additional_dependencies:

README.md

@@ -1,9 +1,7 @@
 <img src="docs/images/logo.svg" width="240" alt="OCRmyPDF">
 
-[![Build Status](https://github.com/jbarlow83/OCRmyPDF/actions/workflows/build.yml/badge.svg)](https://github.com/jbarlow83/OCRmyPDF/actions/workflows/build.yml) [![PyPI version][pypi]](https://pypi.org/project/ocrmypdf/) ![Homebrew version][homebrew] ![ReadTheDocs][docs] ![Python versions][pyversions]
+[![Build Status](https://github.com/ocrmypdf/OCRmyPDF/actions/workflows/build.yml/badge.svg)](https://github.com/ocrmypdf/OCRmyPDF/actions/workflows/build.yml) [![PyPI version][pypi]](https://pypi.org/project/ocrmypdf/) ![Homebrew version][homebrew] ![ReadTheDocs][docs] ![Python versions][pyversions]
 
-[azure]: https://dev.azure.com/jim0585/ocrmypdf/_apis/build/status/jbarlow83.OCRmyPDF?branchName=master
-[travis]: https://travis-ci.org/jbarlow83/OCRmyPDF.svg?branch=master "Travis build status"
 [pypi]: https://img.shields.io/pypi/v/ocrmypdf.svg "PyPI version"
 [homebrew]: https://img.shields.io/homebrew/v/ocrmypdf.svg "Homebrew version"
 [docs]: https://readthedocs.org/projects/ocrmypdf/badge/?version=latest "RTD"
@@ -106,7 +104,7 @@ ocrmypdf --help
 
 Our [documentation is served on Read the Docs](https://ocrmypdf.readthedocs.io/en/latest/index.html).
 
-Please report issues on our [GitHub issues](https://github.com/jbarlow83/OCRmyPDF/issues) page, and follow the issue template for quick response.
+Please report issues on our [GitHub issues](https://github.com/ocrmypdf/OCRmyPDF/issues) page, and follow the issue template for quick response.
 
 ## Requirements
 

debian/copyright

@@ -1,7 +1,7 @@
 Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
 Upstream-Name: OCRmyPDF
 Upstream-Contact: James R. Barlow <barlow.jim@gmail.com>
-Source: https://github.com/jbarlow83/OCRmyPDF
+Source: https://github.com/ocrmypdf/OCRmyPDF
 
 Files: *
 Copyright:

docs/advanced.rst

@@ -67,11 +67,11 @@ without modifying the PDF. This is to ensure that PDFs that were
 previously OCRed or were "born digital" rather than scanned are not
 processed.
 
-If ``--skip-text`` is issued, then no OCR will be performed on pages
-that already have text. The page will be copied to the output. This may
-be useful for documents that contain both "born digital" and scanned
-content, or to use OCRmyPDF to normalize and convert to PDF/A regardless
-of their contents.
+If ``--skip-text`` is issued, then no image processing or OCR will be
+performed on pages that already have text. The page will be copied to
+the output. This may be useful for documents that contain both "born
+digital" and scanned content, or to use OCRmyPDF to normalize and
+convert to PDF/A regardless of their contents.
 
 If ``--redo-ocr`` is issued, then a detailed text analysis is performed.
 Text is categorized as either visible or invisible. Invisible text (OCR)

docs/conf.py

@@ -42,7 +42,7 @@ extensions = [
 # Extension settings
 intersphinx_mapping = {'https://docs.python.org/': None}
 napoleon_use_rtype = False
-issues_github_path = "jbarlow83/OCRmyPDF"
+issues_github_path = "ocrmypdf/OCRmyPDF"
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']

docs/cookbook.rst

@@ -243,10 +243,11 @@ You can also optimize all images without performing any OCR:
 
     ocrmypdf --tesseract-timeout=0 --optimize 3 --skip-text input.pdf output.pdf
 
-Perform OCR only certain pages
-------------------------------
+Process only certain pages
+--------------------------
 
-You can ask OCRmyPDF to only apply OCR to certain pages.
+You can ask OCRmyPDF to only apply `image processing <#image-processing>`__
+and OCR to certain pages.
 
 .. code-block:: bash
 
@@ -260,10 +261,10 @@ overlap pages. OCRmyPDF does not currently account for document page numbers,
 such as an introduction section of a book that uses Roman numerals. It simply
 counts the number of virtual pieces of paper since the start.
 
-Regardless of the argument to ``--pages``, OCRmyPDF will optimize all pages in
-the file and convert it to PDF/A, unless you disable those options. In this
-example, we want to OCR only the title and otherwise change the PDF as little
-as possible:
+Regardless of the argument to ``--pages``, OCRmyPDF will optimize all pages/images
+in the file and convert it to PDF/A, unless you disable those options. Both of these
+steps are "whole file" operations. In this example, we want to OCR only the title
+and otherwise change the PDF as little as possible:
 
 .. code-block:: bash
 

docs/installation.rst

@@ -375,7 +375,7 @@ languages you can optionally install them all:
 .. note::
 
    Users who previously installed OCRmyPDF from the private tap should
-   switch to the mainline version (``brew untap jbarlow83/ocrmypdf``)
+   switch to the mainline version (``brew untap ocrmypdf/OCRmyPDF``)
    and install from there.
 
 Manual installation on macOS
@@ -696,7 +696,7 @@ environment:
 
 .. code-block:: bash
 
-    pip3 install git+https://github.com/jbarlow83/OCRmyPDF.git
+    pip3 install git+https://github.com/ocrmypdf/OCRmyPDF.git
 
 Or, to install in `development
 mode <https://pythonhosted.org/setuptools/setuptools.html#development-mode>`__,
@@ -704,14 +704,14 @@ allowing customization of OCRmyPDF, use the ``-e`` flag:
 
 .. code-block:: bash
 
-    pip3 install -e git+https://github.com/jbarlow83/OCRmyPDF.git
+    pip3 install -e git+https://github.com/ocrmypdf/OCRmyPDF.git
 
 You may find it easiest to install in a virtual environment, rather than
 system-wide:
 
 .. code-block:: bash
 
-    git clone -b master https://github.com/jbarlow83/OCRmyPDF.git
+    git clone -b master https://github.com/ocrmypdf/OCRmyPDF.git
     python3 -m venv
     source venv/bin/activate
     cd OCRmyPDF
@@ -738,7 +738,7 @@ To install all of the development and test requirements:
 
 .. code-block:: bash
 
-    git clone -b master https://github.com/jbarlow83/OCRmyPDF.git
+    git clone -b master https://github.com/ocrmypdf/OCRmyPDF.git
     python3 -m venv
     source venv/bin/activate
     cd OCRmyPDF

docs/jbig2.rst

@@ -32,6 +32,9 @@ For all other Linux, you must build a JBIG2 encoder from source:
 
 .. _jbig2-lossy:
 
+Dependencies include libtoolize and libleptonica, which on Ubuntu systems 
+are packaged as libtool and libleptonica-dev.
+
 Lossy mode JBIG2
 ================
 

docs/languages.rst

@@ -54,6 +54,33 @@ to what languages it should search for. Multiple languages can be
 requested using either ``-l eng+fra`` (English and French) or
 ``-l eng -l fra``.
 
+Gentoo users
+============
+
+On Gentoo the package ``app-text/tessdata_fast``, which ``app-text/tesseract`` depends on, handles Tesseract languages.
+It accepts USE flags to select what languages should be installed, these can be set in ``/etc/portage/package.use``.
+Alternatively one can globally set the `L10N use extension <https://wiki.gentoo.org/wiki/Localization/Guide#L10N>`__ in ``/etc/portage/make.conf``.
+This enables these languages for all packages (e.g. including aspell).
+
+.. code-block:: bash
+
+   # Display a list of all Tesseract language packs
+   equery uses app-text/tessdata_fast
+   
+   # Add English and German language support for Tesseract only
+   echo 'app-text/tessdata_fast l10n_de l10n_en' >> /etc/portage/package.use
+   
+   # Add global English and German language support (the `l10n_` from equery has to be omited)
+   echo L10N="de en" >> /etc/portage/make.conf
+   
+   # update system to reflect changed USE flags
+   emerge --update --deep --newuse @world
+
+You can then pass the ``-l LANG`` argument to OCRmyPDF to give a hint as
+to what languages it should search for. Multiple languages can be
+requested using either ``-l eng+fra`` (English and French) or
+``-l eng -l fra``.
+
 macOS users
 ===========
 

docs/maintainers.rst

@@ -2,7 +2,7 @@
 Maintainer notes
 ================
 
-This is for those who package OCRmyPDF for downstream use.  (Thank you 
+This is for those who package OCRmyPDF for downstream use.  (Thank you
 for your hard work.)
 
 Known ports/packagers
@@ -25,7 +25,7 @@ Non-Python dependencies
 
 Note that we have non-Python dependencies. In particular, OCRmyPDF requires
 Ghostscript and Tesseract OCR to be installed and needs to be able to locate their
-binaries on the system PATH. On Windows, OCRmyPDF will also check the registry 
+binaries on the system PATH. On Windows, OCRmyPDF will also check the registry
 for their locations.
 
 Tesseract OCR relies on SIMD for performance and only has proper support for this
@@ -38,13 +38,13 @@ OCRmyPDF uses setuptools-scm for versioning, which derives the version from
 Git as a single source of truth. This may be unsuitable for some distributions, e.g.
 to indicate that your distribution modifies OCRmyPDF in some way.
 
-You can patch the ``__version__`` variable in ``src/ocrmypdf/_version.py`` if 
+You can patch the ``__version__`` variable in ``src/ocrmypdf/_version.py`` if
 necessary.
 
 OCRmyPDF uses setuptools-scm-git-archive to ensure that tarballs downloaded from
 GitHub contain version information. Unfortunately, these tarballs are not always
-deterministic. See this 
-`issue <https://github.com/jbarlow83/OCRmyPDF/issues/841#issuecomment-936562696>`_.
+deterministic. See this
+`issue <https://github.com/ocrmypdf/OCRmyPDF/issues/841#issuecomment-936562696>`_.
 
 jbig2enc
 --------

docs/release_notes.rst

@@ -10,7 +10,63 @@ that is, output messages may be improved at any release level, so parsing them
 may be unreliable. Use the API to depend on precise behavior.
 
 The public API may be useful in scripts that launch OCRmyPDF processes or that
-wish to use some of its features for working with PDFs..
+wish to use some of its features for working with PDFs.
+
+The most recent release of OCRmyPDF is |OCRmyPDF PyPI|. Any newer versions
+referred to in these notes may exist the main branch but have not been
+tagged yet.
+
+.. |OCRmyPDF PyPI| image:: https://img.shields.io/pypi/v/ocrmypdf.svg
+
+v13.4.7
+=======
+
+-  Fixed PermissionError when cleaning up temporary files in rare cases. :issue:`974`
+-  Fixed PermissionError when calling ``os.nice`` on platforms that lack it. :issue:`973`
+-  Suppressed some warnings from libxmp during tests.
+
+v13.4.6
+=======
+
+-  Convert error on corrupt ICC profiles into a warning. Thanks to @oscherler.
+
+v13.4.5
+=======
+
+-  Remove upper bound on pdfminer.six version.
+-  Documentation.
+
+v13.4.4
+=======
+
+-  Updated pdfminer.six version.
+-  Docker image changed to Ubuntu 22.04 now that it is released and provides the
+   dependencies we need. This seems more consistent than our recent change to
+   Debian.
+
+v13.4.3
+=======
+
+-  Fix error on pytest.skip() with older versions of pytest.
+-  Documentation updates.
+
+v13.4.2
+=======
+
+-  Worked around a
+   `major regression in Ghostscript 9.56.0 <https://bugs.ghostscript.com/show_bug.cgi?id=705187>`__
+   where **all OCR text is stripped out of the PDF**. It simply removes all text,
+   even generated by software other than OCRmyPDF. Fortunately, we can ask
+   Ghostscript 9.56.0 to use its old behavior that worked correctly for our purposes.
+   Users must avoid the combination (Ghostscript 9.56.0, ocrmypdf <13.4.2) since
+   older versions of OCRmyPDF have no way of detecting that this particular
+   version of Ghostscript removes all OCR text.
+-  Marked pdfminer 20220319 as supported.
+-  Fixed some deprecation warnings from recent versions of Pillow and pytest.
+-  Test suite now covers Python 3.10 (Python 3.10 worked fine before, but was not
+   being tested).
+-  Docker image now uses debian:bookworm-slim as the base image to fix the Docker
+   image build.
 
 v13.4.1
 =======
@@ -24,9 +80,9 @@ v13.4.0
 
 -  Fixed test failures when using pikepdf 5.0.0.
 -  Various improvements to the optimizer. In particular, we now recognize PDF images
-   that are encoded with both flate and DCT (JPEG), and also produce PDF with images
-   compressed with flate and DCT, since this often yields file size improvements
-   compared to plain DCT.
+   that are encoded with both deflate (PNG) and DCT (JPEG), and also produce PDF
+   with images compressed with deflate and DCT, since this often yields file size
+   improvements compared to plain DCT.
 
 v13.3.0
 =======

misc/webservice.py

@@ -59,7 +59,7 @@ def do_ocrmypdf(file):
         return Response("--sidecar not supported", 501, mimetype='text/plain')
 
     ocrmypdf_args = ["ocrmypdf", *cmd_args, up_file, down_file]
-    proc = run(ocrmypdf_args, stdout=PIPE, stderr=PIPE, encoding="utf-8")
+    proc = run(ocrmypdf_args, capture_output=True, encoding="utf-8")
     if proc.returncode != 0:
         stderr = proc.stderr
         return Response(stderr, 400, mimetype='text/plain')

setup.cfg

@@ -3,7 +3,7 @@ name = ocrmypdf
 description = OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched
 long_description = file: README.md
 long_description_content_type = text/markdown
-url = https://github.com/jbarlow83/OCRmyPDF
+url = https://github.com/ocrmypdf/OCRmyPDF
 author = James R. Barlow
 author_email = james@purplerock.ca
 license = MPL-2.0
@@ -39,8 +39,8 @@ keywords =
     scanning
 project_urls =
     Documentation = https://ocrmypdf.readthedocs.io/
-    Source = https://github.com/jbarlow83/ocrmypdf
-    Tracker = https://github.com/jbarlow83/ocrmypdf/issues
+    Source = https://github.com/ocrmypdf/OCRmyPDF
+    Tracker = https://github.com/ocrmypdf/OCRmyPDF/issues
 
 [options]
 packages = find:
@@ -49,8 +49,8 @@ install_requires =
     coloredlogs>=14.0  # strictly optional
     img2pdf>=0.3.0,<0.5  # pure Python
     packaging>=20
-    pdfminer.six!=20200720,>=20191110,<=20211012
-    pikepdf>=4.0.0,!=5.0.0
+    pdfminer.six!=20200720,>=20191110
+    pikepdf!=5.0.0,>=4.0.0
     pluggy>=0.13.0,<2
     reportlab>=3.5.66
     tqdm>=4

src/ocrmypdf/__main__.py

@@ -10,6 +10,7 @@ import logging
 import os
 import signal
 import sys
+from contextlib import suppress
 from multiprocessing import set_start_method
 
 from ocrmypdf import __version__
@@ -34,7 +35,7 @@ def sigbus(*args):
 def run(args=None):
     _parser, options, plugin_manager = get_parser_options_plugins(args=args)
 
-    if hasattr(os, 'nice'):
+    with suppress(AttributeError, PermissionError):
         os.nice(5)
 
     verbosity = options.verbose

src/ocrmypdf/_exec/ghostscript.py

@@ -24,6 +24,13 @@ from ocrmypdf.exceptions import MissingDependencyError, SubprocessOutputError
 from ocrmypdf.helpers import Resolution
 from ocrmypdf.subprocess import get_version, run, run_polling_stderr
 
+# Remove this workaround when we require Pillow >= 10
+try:
+    Transpose = Image.Transpose  # type: ignore
+except AttributeError:
+    # Pillow 9 shim
+    Transpose = Image  # type: ignore
+
 log = logging.getLogger(__name__)
 
 missing_gs_error = """
@@ -132,11 +139,11 @@ def rasterize_pdf(
                 # rotation is a clockwise angle and Image.ROTATE_* is
                 # counterclockwise so this cancels out the rotation
                 if rotation == 90:
-                    im = im.transpose(Image.ROTATE_90)
+                    im = im.transpose(Transpose.ROTATE_90)
                 elif rotation == 180:
-                    im = im.transpose(Image.ROTATE_180)
+                    im = im.transpose(Transpose.ROTATE_180)
                 elif rotation == 270:
-                    im = im.transpose(Image.ROTATE_270)
+                    im = im.transpose(Transpose.ROTATE_270)
                 if rotation % 180 == 90:
                     page_dpi = page_dpi.flip_axis()
             im.save(fspath(output_file), dpi=page_dpi)
@@ -210,14 +217,18 @@ def generate_pdfa(
     # Older versions of Ghostscript expect a leading slash in
     # sColorConversionStrategy, newer ones should not have it. See Ghostscript
     # git commit fe1c025d.
-    strategy = ('/' + strategy) if version() < '9.19' else strategy
+    gs_version = version()
+    strategy = ('/' + strategy) if gs_version < '9.19' else strategy
 
-    if version() == '9.23':
+    if gs_version == '9.23':
         # 9.23: added JPEG passthrough as a new feature, but with a bug that
         # incorrectly formats some images. Fixed as of 9.24. So we disable this
         # feature for 9.23.
         # https://bugs.ghostscript.com/show_bug.cgi?id=699216
         compression_args.append('-dPassThroughJPEGImages=false')
+    elif gs_version == '9.56.0':
+        # 9.56.0 breaks our OCR...?
+        compression_args.append('-dNEWPDF=false')
 
     # nb no need to specify ProcessColorModel when ColorConversionStrategy
     # is set; see:

src/ocrmypdf/_exec/unpaper.py

@@ -13,18 +13,35 @@
 import logging
 import os
 import shlex
+import sys
 from contextlib import contextmanager
 from decimal import Decimal
 from pathlib import Path
 from subprocess import PIPE, STDOUT
-from tempfile import TemporaryDirectory
-from typing import List, Optional, Tuple, Union
+from typing import Iterator, List, Optional, Tuple, Union
 
 from PIL import Image
 
 from ocrmypdf.exceptions import MissingDependencyError, SubprocessOutputError
 from ocrmypdf.subprocess import get_version, run
 
+if sys.version_info >= (3, 10):
+    from tempfile import TemporaryDirectory
+else:
+    from tempfile import TemporaryDirectory as _TemporaryDirectory
+
+    # Consume the ignore_cleanup_errors kwarg in Python 3.9 and older, without acting
+    # on this keyword. Users who need this issue full resolved should upgrade to Python
+    # 3.10.
+    # See: https://github.com/python/cpython/pull/24793
+
+    class TemporaryDirectory(_TemporaryDirectory):
+        def __init__(self, ignore_cleanup_errors=False, **kwargs):
+            super().__init__(**kwargs)
+
+    del _TemporaryDirectory
+
+
 UNPAPER_IMAGE_PIXEL_LIMIT = 256 * 1024 * 1024
 
 DecFloat = Union[Decimal, float]
@@ -76,13 +93,13 @@ def _convert_image(im: Image.Image) -> Tuple[Image.Image, bool, str]:
 
 
 @contextmanager
-def _setup_unpaper_io(input_file: Path) -> Tuple[Path, Path, Path]:
+def _setup_unpaper_io(input_file: Path) -> Iterator[Tuple[Path, Path, Path]]:
     with Image.open(input_file) as im:
         if im.width * im.height >= UNPAPER_IMAGE_PIXEL_LIMIT:
             raise UnpaperImageTooLargeError(w=im.width, h=im.height)
         im, im_modified, suffix = _convert_image(im)
 
-        with TemporaryDirectory() as tmpdir:
+        with TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir:
             tmppath = Path(tmpdir)
             if im_modified or input_file.suffix != '.pnm':
                 input_pnm = tmppath / 'input.pnm'

src/ocrmypdf/_pipeline.py

@@ -18,7 +18,7 @@ from typing import Dict, Iterable, Optional
 import img2pdf
 import pikepdf
 from pikepdf.models.metadata import encode_pdf_date
-from PIL import Image, ImageDraw
+from PIL import Image, ImageColor, ImageDraw
 
 from ocrmypdf._concurrent import Executor
 from ocrmypdf._exec import unpaper
@@ -38,6 +38,13 @@ from ocrmypdf.optimize import optimize
 from ocrmypdf.pdfa import generate_pdfa_ps
 from ocrmypdf.pdfinfo import Colorspace, Encoding, PdfInfo
 
+# Remove this workaround when we require Pillow >= 10
+try:
+    BICUBIC = Image.Resampling.BICUBIC  # type: ignore
+except AttributeError:
+    # Pillow 9 shim
+    BICUBIC = Image.BICUBIC  # type: ignore
+
 log = logging.getLogger(__name__)
 
 VECTOR_PAGE_DPI = 400
@@ -484,7 +491,9 @@ def preprocess_deskew(input_file: Path, page_context: PageContext):
         # According to Pillow docs, .rotate() will automatically use Image.NEAREST
         # resampling if image is mode '1' or 'P'
         deskewed = im.rotate(
-            deskew_angle_degrees, resample=Image.BICUBIC, fillcolor='white'
+            deskew_angle_degrees,
+            resample=BICUBIC,
+            fillcolor=ImageColor.getcolor('white', mode=im.mode),
         )
         deskewed.save(output_file, dpi=dpi)
 

src/ocrmypdf/_validation.py

@@ -41,7 +41,7 @@ log = logging.getLogger(__name__)
 
 
 def check_platform():
-    if os.name == 'nt' and sys.maxsize <= 2 ** 32:  # pragma: no cover
+    if os.name == 'nt' and sys.maxsize <= 2**32:  # pragma: no cover
         # 32-bit interpreter on Windows
         log.error(
             "You are running OCRmyPDF in a 32-bit (x86) Python interpreter."

src/ocrmypdf/api.py

@@ -295,7 +295,7 @@ def ocr(  # pylint: disable=unused-argument
             text already, and settings did not tell us to proceed.
         ocrmypdf.InputFileError: Any other problem with the input file.
         ocrmypdf.SubprocessOutputError: Any error related to executing a subprocess.
-        ocrmypdf.EncryptedPdfERror: If the input PDF is encrypted (password protected).
+        ocrmypdf.EncryptedPdfError: If the input PDF is encrypted (password protected).
             OCRmyPDF does not remove passwords.
         ocrmypdf.TesseractConfigError: If Tesseract reported its configuration was not
             valid.

src/ocrmypdf/pdfinfo/info.py

@@ -37,6 +37,7 @@ from pikepdf import (
     PdfImage,
     PdfInlineImage,
     PdfMatrix,
+    UnsupportedImageTypeError,
     parse_content_stream,
 )
 
@@ -350,13 +351,20 @@ class ImageInfo:
 
         if self._color == Colorspace.icc:
             # Check the ICC profile to determine actual colorspace
-            pim_icc = pim.icc
-            if pim_icc.profile.xcolor_space == 'GRAY':
-                self._comp = 1
-            elif pim_icc.profile.xcolor_space == 'CMYK':
-                self._comp = 4
-            else:
-                self._comp = 3
+            try:
+                pim_icc = pim.icc
+                if pim_icc.profile.xcolor_space == 'GRAY':
+                    self._comp = 1
+                elif pim_icc.profile.xcolor_space == 'CMYK':
+                    self._comp = 4
+                else:
+                    self._comp = 3
+            except UnsupportedImageTypeError as ex:
+                self._comp = None
+                logger.warning(
+                    f"An image with a corrupt or unreadable ICC profile was found. "
+                    f"The output PDF may not match the input PDF visually: {ex}. {self}"
+                )
         else:
             if isinstance(self._color, Colorspace):
                 self._comp = FRIENDLY_COMP.get(self._color)
@@ -409,15 +417,10 @@ class ImageInfo:
         return _get_dpi(self._shorthand, (self._width, self._height))
 
     def __repr__(self):
-        class_locals = {
-            attr: getattr(self, attr, None)
-            for attr in dir(self)
-            if not attr.startswith('_')
-        }
         return (
-            "<ImageInfo '{name}' {type_} {width}x{height} {color} "
-            "{comp} {bpc} {enc} {dpi}>"
-        ).format(**class_locals)
+            f"<ImageInfo '{self.name}' {self.type_} {self.width}x{self.height} "
+            f"{self.color} {self.comp} {self.bpc} {self.enc} {self.dpi}>"
+        )
 
 
 def _find_inline_images(contentsinfo: ContentsInfo) -> Iterator[ImageInfo]:
@@ -647,8 +650,8 @@ def _pdf_pageinfo_concurrent(
     max_workers,
     check_pages,
     detailed_analysis=False,
-) -> List[Optional['PageInfo']]:
-    pages = [None] * len(pdf.pages)
+) -> Sequence[Optional['PageInfo']]:
+    pages: Sequence[Optional['PageInfo']] = [None] * len(pdf.pages)
 
     def update_pageinfo(result, pbar):
         page = result
@@ -925,11 +928,11 @@ class PdfInfo:
     @property
     def min_version(self) -> str:
         # The minimum PDF is the maximum version that any particular page needs
-        return max(page.min_version for page in self.pages)
+        return max(page.min_version for page in self.pages if page)
 
     @property
     def has_userunit(self) -> bool:
-        return any(page.userunit != 1.0 for page in self.pages)
+        return any(page.userunit != 1.0 for page in self.pages if page)
 
     @property
     def has_acroform(self) -> bool:

src/ocrmypdf/subprocess/_windows.py

@@ -4,6 +4,9 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+# type: ignore
+# Non-Windows mypy now breaks when trying to typecheck winreg
+
 import logging
 import os
 import shutil
@@ -17,6 +20,7 @@ try:
 except ModuleNotFoundError as e:
     raise ModuleNotFoundError("This module is for Windows only") from e
 
+
 log = logging.getLogger(__name__)
 
 T = TypeVar('T')

tests/conftest.py

@@ -134,8 +134,7 @@ def run_ocrmypdf(
 
     p = run(
         p_args,
-        stdout=PIPE,
-        stderr=PIPE,
+        capture_output=True,
         text=text,
         check=False,
     )

tests/plugins/tesseract_cache.py

@@ -50,6 +50,7 @@ import logging
 import platform
 import re
 import shutil
+import threading
 from functools import partial
 from pathlib import Path
 from subprocess import PIPE, CalledProcessError, CompletedProcess
@@ -176,26 +177,40 @@ def cached_run(options, run_args, **run_kwargs):
 
 
 class CacheOcrEngine(TesseractOcrEngine):
+    # Concurrent threads (with --use-threads) might try to use different parts
+    # of the OcrEngine, so we need a lock to protect the state of patched
+    # module whenever it's patched. Should refactor ocrmypdf._exec.tesseract so that
+    # it does not to be patched at all for testing.
+    lock = threading.Lock()
+
     @staticmethod
     def get_orientation(input_file, options):
-        with patch('ocrmypdf._exec.tesseract.run', new=partial(cached_run, options)):
+        with CacheOcrEngine.lock, patch(
+            'ocrmypdf._exec.tesseract.run', new=partial(cached_run, options)
+        ):
             return TesseractOcrEngine.get_orientation(input_file, options)
 
     @staticmethod
     def get_deskew(input_file, options) -> float:
-        with patch('ocrmypdf._exec.tesseract.run', new=partial(cached_run, options)):
+        with CacheOcrEngine.lock, patch(
+            'ocrmypdf._exec.tesseract.run', new=partial(cached_run, options)
+        ):
             return TesseractOcrEngine.get_deskew(input_file, options)
 
     @staticmethod
     def generate_hocr(input_file, output_hocr, output_text, options):
-        with patch('ocrmypdf._exec.tesseract.run', new=partial(cached_run, options)):
+        with CacheOcrEngine.lock, patch(
+            'ocrmypdf._exec.tesseract.run', new=partial(cached_run, options)
+        ):
             TesseractOcrEngine.generate_hocr(
                 input_file, output_hocr, output_text, options
             )
 
     @staticmethod
     def generate_pdf(input_file, output_pdf, output_text, options):
-        with patch('ocrmypdf._exec.tesseract.run', new=partial(cached_run, options)):
+        with CacheOcrEngine.lock, patch(
+            'ocrmypdf._exec.tesseract.run', new=partial(cached_run, options)
+        ):
             TesseractOcrEngine.generate_pdf(
                 input_file, output_pdf, output_text, options
             )

tests/test_completion.py

@@ -24,8 +24,7 @@ def test_fish():
             ['fish', '-n', 'misc/completion/ocrmypdf.fish'],
             check=True,
             encoding='utf-8',
-            stdout=PIPE,
-            stderr=PIPE,
+            capture_output=True,
         )
         assert proc.stderr == '', proc.stderr
     except FileNotFoundError:
@@ -41,8 +40,7 @@ def test_bash():
             ['bash', '-n', 'misc/completion/ocrmypdf.bash'],
             check=True,
             encoding='utf-8',
-            stdout=PIPE,
-            stderr=PIPE,
+            capture_output=True,
         )
         assert proc.stderr == '', proc.stderr
     except FileNotFoundError:

tests/test_main.py

@@ -465,12 +465,18 @@ def test_overlay(resources, outpdf):
     )
 
 
-def test_destination_not_writable(resources, outdir):
-    if os.name != 'nt' and (os.getuid() == 0 or os.geteuid() == 0):
-        pytest.xfail(reason="root can write to anything")
+@pytest.fixture
+def protected_file(outdir):
     protected_file = outdir / 'protected.pdf'
     protected_file.touch()
     protected_file.chmod(0o400)  # Read-only
+    yield protected_file
+
+
+@pytest.mark.skipif(
+    os.name == 'nt' or os.geteuid() == 0, reason="root can write to anything"
+)
+def test_destination_not_writable(resources, protected_file):
     p = run_ocrmypdf(
         resources / 'jbig2.pdf',
         protected_file,
@@ -480,7 +486,8 @@ def test_destination_not_writable(resources, outdir):
     assert p.returncode == ExitCode.file_access_error, "Expected error"
 
 
-def test_tesseract_config_valid(resources, outdir):
+@pytest.fixture
+def valid_tess_config(outdir):
     cfg_file = outdir / 'test.cfg'
     with cfg_file.open('w') as f:
         f.write(
@@ -490,20 +497,22 @@ language_model_penalty_non_dict_word 0
 language_model_penalty_non_freq_dict_word 0
 '''
         )
+    yield cfg_file
 
+
+def test_tesseract_config_valid(resources, valid_tess_config, outpdf):
     check_ocrmypdf(
         resources / '3small.pdf',
-        outdir / 'out.pdf',
+        outpdf,
         '--tesseract-config',
-        cfg_file,
+        valid_tess_config,
         '--pages',
         '1',
     )
 
 
-@pytest.mark.slow  # This test sometimes times out in CI
-@pytest.mark.parametrize('renderer', RENDERERS)
-def test_tesseract_config_invalid(renderer, resources, outdir):
+@pytest.fixture
+def invalid_tess_config(outdir):
     cfg_file = outdir / 'test.cfg'
     with cfg_file.open('w') as f:
         f.write(
@@ -511,14 +520,19 @@ def test_tesseract_config_invalid(renderer, resources, outdir):
 THIS FILE IS INVALID
 '''
         )
+    yield cfg_file
 
+
+@pytest.mark.slow  # This test sometimes times out in CI
+@pytest.mark.parametrize('renderer', RENDERERS)
+def test_tesseract_config_invalid(renderer, resources, invalid_tess_config, outpdf):
     p = run_ocrmypdf(
         resources / 'ccitt.pdf',
-        outdir / 'out.pdf',
+        outpdf,
         '--pdf-renderer',
         renderer,
         '--tesseract-config',
-        cfg_file,
+        invalid_tess_config,
     )
     assert (
         "parameter not found" in p.stderr.lower()
@@ -630,8 +644,7 @@ def test_compression_preserved(ocrmypdf_exec, resources, image, outpdf):
         ]
         p = run(
             p_args,
-            stdout=PIPE,
-            stderr=PIPE,
+            capture_output=True,
             stdin=input_stream,
             text=True,
             check=False,
@@ -691,8 +704,7 @@ def test_compression_changed(ocrmypdf_exec, resources, image, compression, outpd
         ]
         p = run(
             p_args,
-            stdout=PIPE,
-            stderr=PIPE,
+            capture_output=True,
             stdin=input_stream,
             text=True,
             check=False,
@@ -803,6 +815,9 @@ def test_text_curves(resources, outpdf):
         info = PdfInfo(outpdf)
         assert len(info.pages[0].images) == 0, "added images to the vector PDF"
 
+
+def test_text_curves_force(resources, outpdf):
+    with patch('ocrmypdf._pipeline.VECTOR_PAGE_DPI', 100):
         check_ocrmypdf(
             resources / 'vector.pdf',
             outpdf,
@@ -924,3 +939,25 @@ def test_outputtype_none(resources, outtxt):
         'tests/plugins/tesseract_noop.py',
     )
     assert p.returncode == ExitCode.ok
+
+
+@pytest.fixture
+def graph_bad_icc(resources, outdir):
+    synth_input_file = outdir / 'graph-bad-icc.pdf'
+    with pikepdf.open(resources / 'graph.pdf') as pdf:
+        icc = pdf.make_stream(
+            b'invalid icc profile', N=3, Alternate=pikepdf.Name.DeviceRGB
+        )
+        pdf.pages[0].Resources.XObject['/Im0'].ColorSpace = pikepdf.Array(
+            [pikepdf.Name.ICCBased, icc]
+        )
+        pdf.save(synth_input_file)
+        yield synth_input_file
+
+
+def test_corrupt_icc(graph_bad_icc, outpdf, caplog):
+    result = run_ocrmypdf_api(graph_bad_icc, outpdf)
+    assert result == ExitCode.ok
+    assert any(
+        'corrupt or unreadable ICC profile' in rec.message for rec in caplog.records
+    )

tests/test_metadata.py

@@ -6,10 +6,10 @@
 
 
 import datetime
+import warnings
 from datetime import timezone
 from os import fspath
 from shutil import copyfile
-from unittest.mock import patch
 
 import pikepdf
 import pytest
@@ -173,6 +173,19 @@ def test_creation_date_preserved(output_type, resources, infile, outpdf):
     assert seconds_between_dates(date_after, datetime.datetime.now(timezone.utc)) < 1000
 
 
+@pytest.fixture
+def libxmp_file_to_dict():
+    try:
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", DeprecationWarning)
+            from libxmp.utils import (
+                file_to_dict,  # pylint: disable=import-outside-toplevel
+            )
+    except Exception:  # pylint: disable=broad-except
+        pytest.skip("libxmp not available or libexempi3 not installed")
+    return file_to_dict
+
+
 @pytest.mark.parametrize(
     'test_file,output_type',
     [
@@ -182,15 +195,12 @@ def test_creation_date_preserved(output_type, resources, infile, outpdf):
         ('3small.pdf', 'pdfa'),
     ],
 )
-def test_xml_metadata_preserved(test_file, output_type, resources, outpdf):
+def test_xml_metadata_preserved(
+    libxmp_file_to_dict, test_file, output_type, resources, outpdf
+):
     input_file = resources / test_file
 
-    try:
-        from libxmp.utils import file_to_dict  # pylint: disable=import-outside-toplevel
-    except Exception:  # pylint: disable=broad-except
-        pytest.skip("libxmp not available or libexempi3 not installed")
-
-    before = file_to_dict(str(input_file))
+    before = libxmp_file_to_dict(str(input_file))
 
     check_ocrmypdf(
         input_file,
@@ -202,7 +212,7 @@ def test_xml_metadata_preserved(test_file, output_type, resources, outpdf):
         'tests/plugins/tesseract_noop.py',
     )
 
-    after = file_to_dict(str(outpdf))
+    after = libxmp_file_to_dict(str(outpdf))
 
     equal_properties = [
         'dc:contributor',

tests/test_pdfa.py

@@ -4,23 +4,31 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import os
+
 import pikepdf
 import pytest
 
+from ocrmypdf.exceptions import MissingDependencyError
+
 from .conftest import check_ocrmypdf
 
 
 @pytest.mark.parametrize('optimize', (0, 3))
 @pytest.mark.parametrize('pdfa_level', (1, 2, 3))
 def test_pdfa(resources, outpdf, optimize, pdfa_level):
-    check_ocrmypdf(
-        resources / 'francais.pdf',
-        outpdf,
-        '--plugin',
-        'tests/plugins/tesseract_noop.py',
-        f'--output-type=pdfa-{pdfa_level}',
-        f'--optimize={optimize}',
-    )
+    try:
+        check_ocrmypdf(
+            resources / 'francais.pdf',
+            outpdf,
+            '--plugin',
+            'tests/plugins/tesseract_noop.py',
+            f'--output-type=pdfa-{pdfa_level}',
+            f'--optimize={optimize}',
+        )
+    except MissingDependencyError as e:
+        if 'pngquant' in str(e) and optimize in (2, 3) and os.name == 'nt':
+            pytest.xfail("pngquant currently not available on Windows")
     if pdfa_level in (2, 3):
         # PDF/A-2 allows ObjStm
         assert b'/ObjStm' in outpdf.read_bytes()

tests/test_pdfinfo.py

@@ -173,17 +173,15 @@ def test_stack_abuse():
     p = pikepdf.Pdf.new()
 
     stream = pikepdf.Stream(p, b'q ' * 35)
-    with pytest.warns(None) as record:
+    with pytest.warns(UserWarning, match="overflowed"):
         pdfinfo.info._interpret_contents(stream)
-    assert 'overflowed' in str(record[0].message)
 
     stream = pikepdf.Stream(p, b'q Q Q Q Q')
-    with pytest.warns(None) as record:
+    with pytest.warns(UserWarning, match="underflowed") as record:
         pdfinfo.info._interpret_contents(stream)
-    assert 'underflowed' in str(record[0].message)
 
     stream = pikepdf.Stream(p, b'q ' * 135)
-    with pytest.warns(None):
+    with pytest.warns(UserWarning):
         with pytest.raises(RuntimeError):
             pdfinfo.info._interpret_contents(stream)
 

tests/test_rotation.py

@@ -25,6 +25,12 @@ from .conftest import check_ocrmypdf, run_ocrmypdf
 
 # pylintx: disable=unused-variable
 
+# Remove this workaround when we require Pillow >= 10
+try:
+    Transpose = Image.Transpose  # type: ignore
+except AttributeError:
+    # Pillow 9 shim
+    Transpose = Image  # type: ignore
 
 RENDERERS = ['hocr', 'sandwich']
 
@@ -222,7 +228,7 @@ def test_rotate_page_level(image_angle, page_angle, resources, outdir):
         with Image.open(fspath(resources / 'typewriter.png')) as im:
             if image_angle != 0:
                 ccw_angle = -image_angle % 360
-                im = im.transpose(getattr(Image, f'ROTATE_{ccw_angle}'))
+                im = im.transpose(getattr(Transpose, f'ROTATE_{ccw_angle}'))
             im.save(memimg, format='PNG')
         memimg.seek(0)
         mempdf = BytesIO()

tests/test_stdio.py

@@ -30,12 +30,12 @@ def test_stdin(ocrmypdf_exec, resources, outpdf):
             '--plugin',
             'tests/plugins/tesseract_noop.py',
         ]
-        run(p_args, stdout=PIPE, stderr=PIPE, stdin=input_stream, check=True)
+        run(p_args, capture_output=True, stdin=input_stream, check=True)
 
 
 def test_stdout(ocrmypdf_exec, resources, outpdf):
     if 'COV_CORE_DATAFILE' in os.environ:
-        pytest.skip(msg="Coverage uses stdout")
+        pytest.skip("Coverage uses stdout")
 
     input_file = str(resources / 'francais.pdf')
     output_file = str(outpdf)
@@ -72,7 +72,7 @@ def test_bad_locale(monkeypatch):
 )
 def test_dev_null(resources):
     if 'COV_CORE_DATAFILE' in os.environ:
-        pytest.skip(msg="Coverage uses stdout")
+        pytest.skip("Coverage uses stdout")
 
     p = run_ocrmypdf(
         resources / 'trivial.pdf',