v16.9.0 release notes

Fixes #1453

.docker/Dockerfile

@@ -1,7 +1,7 @@
 # SPDX-FileCopyrightText: 2024 James R. Barlow
 # SPDX-License-Identifier: MPL-2.0
 
-FROM ubuntu:22.04 AS base
+FROM ubuntu:24.04 AS base
 
 ENV LANG=C.UTF-8
 ENV TZ=UTC
@@ -36,16 +36,27 @@ RUN \
   && cd .. \
   && rm -rf jbig2
 
-COPY . /app
 
 WORKDIR /app
 
-RUN curl -LsSf https://astral.sh/uv/0.4.27/install.sh | sh
+# Copy uv from ghcr
+COPY --from=ghcr.io/astral-sh/uv:0.5.5 /uv /uvx /bin/
 
 ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy
 
-# Instead of restarting the shell, use uv directly from its installed location.
-RUN /root/.cargo/bin/uv sync --extra test --extra webservice --extra watcher
+# Install the project's dependencies using the lockfile and settings
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    uv sync --frozen --no-install-project --no-dev
+
+# Then, add the rest of the project source code and install it
+# Installing separately from its dependencies allows optimal layer caching
+COPY . /app
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --frozen \
+        --extra test --extra webservice --extra watcher --no-dev \
+        --no-install-package pyarrow
 
 FROM base
 
@@ -65,6 +76,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   tesseract-ocr-fra \
   tesseract-ocr-por \
   tesseract-ocr-spa \
+  ttyd \
   unpaper \
   && rm -rf /var/lib/apt/lists/*
 

.docker/Dockerfile.alpine

@@ -3,12 +3,11 @@
 
 # Note: Alpine 3.20 builds tesseract with --enable-opencl, which is not
 # supported by anyone. OCRmyPDF is not compatible with Alpine 3.20.0
-# through 3.20.3. The Alpine issue should be fixed in 3.21.0. It is
-# not clear if 3.20.4+ will have the fix.
+# through 3.20.3. The issue is fixed in 3.21.
 # Details
 #  https://gitlab.alpinelinux.org/alpine/aports/-/issues/16143
 #  https://github.com/ocrmypdf/OCRmyPDF/issues/1395
-FROM alpine:3.19 AS base
+FROM alpine:3.21 AS base
 
 ENV LANG=C.UTF-8
 ENV TZ=UTC
@@ -19,22 +18,35 @@ RUN apk add --no-cache \
 
 FROM base AS builder
 
+# Yes it really is python3-dev, and py3-package
 RUN apk add --no-cache \
     ca-certificates \
     git \
     python3-dev \
+    py3-pyarrow \
     curl
 
-COPY . /app
-
 WORKDIR /app
 
-RUN curl -LsSf https://astral.sh/uv/0.4.27/install.sh | sh
+COPY --from=ghcr.io/astral-sh/uv:0.5.5 /uv /uvx /bin/
 
 ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy
 
-# Instead of restarting the shell, use uv directly from its installed location.
-RUN /root/.cargo/bin/uv sync --extra test --extra webservice --extra watcher
+RUN uv venv --system-site-packages .venv
+
+# Install the project's dependencies using the lockfile and settings
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    uv sync --frozen --no-install-project --no-dev
+
+# Then, add the rest of the project source code and install it
+# Installing separately from its dependencies allows optimal layer caching
+COPY . /app
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --frozen \
+        --extra test --extra webservice --extra watcher --no-dev \
+        --no-install-package pyarrow
 
 FROM base
 
@@ -52,6 +64,7 @@ RUN apk add --no-cache \
     tesseract-ocr-data-por \
     tesseract-ocr-data-spa \
     ttf-droid \
+    ttyd \
     unpaper \
     && rm -rf /var/cache/apk/*
 

.github/workflows/build.yml

@@ -40,9 +40,9 @@ jobs:
           fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v3
+        uses: astral-sh/setup-uv@v5
         with:
-          version: "0.4.27"
+          version: "0.5.x"
 
       - name: "Set up Python"
         uses: actions/setup-python@v5
@@ -84,7 +84,7 @@ jobs:
 
       - name: Install Python packages
         run: |
-          uv sync --extra test
+          uv sync --extra test --no-dev
 
       - name: Report versions
         run: |
@@ -92,14 +92,14 @@ jobs:
           gs --version
           pngquant --version
           unpaper --version
-          uv run img2pdf --version
+          uv run --no-dev img2pdf --version
 
       - name: Test
         run: |
-          uv run pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
+          uv run --no-dev pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
 
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:
@@ -136,9 +136,9 @@ jobs:
             tesseract
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v3
+        uses: astral-sh/setup-uv@v5
         with:
-          version: "0.4.27"
+          version: "0.5.x"
 
       - name: "Set up Python"
         uses: actions/setup-python@v5
@@ -147,21 +147,21 @@ jobs:
 
       - name: Install Python packages
         run: |
-          uv sync --extra test
+          uv sync --extra test --no-dev
 
       - name: Report versions
         run: |
           tesseract --version
           gs --version
           pngquant --version
-          uv run img2pdf --version
+          uv run --no-dev img2pdf --version
 
       - name: Test
         run: |
-          uv run pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
+          uv run --no-dev pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
 
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:
@@ -186,9 +186,9 @@ jobs:
           fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v3
+        uses: astral-sh/setup-uv@v5
         with:
-          version: "0.4.27"
+          version: "0.5.x"
 
       - name: "Set up Python"
         uses: actions/setup-python@v5
@@ -202,14 +202,14 @@ jobs:
 
       - name: Install Python packages
         run: |
-          uv sync --extra test
+          uv sync --extra test --no-dev
 
       - name: Test
         run: |
-          uv run pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
+          uv run --no-dev pytest --cov-report xml --cov=ocrmypdf --cov=tests/ -n0 tests/
 
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:
@@ -225,9 +225,9 @@ jobs:
           fetch-depth: "0" # 0=all, needed for setuptools-scm to resolve version tags
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v3
+        uses: astral-sh/setup-uv@v5
         with:
-          version: "0.4.27"
+          version: "0.5.x"
 
       - name: Make wheels and sdist
         run: |
@@ -275,14 +275,14 @@ jobs:
       - name: Sign the dists with Sigstore
         uses: sigstore/gh-action-sigstore-python@v3.0.0
         with:
-          inputs: >-
+          inputs: |
             ./dist/*.tar.gz
             ./dist/*.whl
 
       - name: Create GitHub Release
         env:
           GITHUB_TOKEN: ${{ github.token }}
-        run: >-
+        run: |
           gh release create
           '${{ github.ref_name }}'
           --repo '${{ github.repository }}'
@@ -294,7 +294,7 @@ jobs:
         # Upload to GitHub Release using the `gh` CLI.
         # `dist/` contains the built packages, and the
         # sigstore-produced signatures and certificates.
-        run: >-
+        run: |
           gh release upload
           '${{ github.ref_name }}' dist/**
           --repo '${{ github.repository }}'

.github/workflows/triage.yml

@@ -0,0 +1,32 @@
+# SPDX-FileCopyrightText: 2024 James R. Barlow
+# SPDX-License-Identifier: MPL-2.0
+
+name: Remove Triage Label on Reply
+
+on:
+  issue_comment:
+    types:
+      - created
+
+jobs:
+  remove-triage-label:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check if comment is by the repository owner
+        id: check_comment
+        run: |
+          echo "::set-output name=is_owner::$(
+            if [[ '${{ github.event.comment.user.login }}' == 'jbarlow83' ]]; then
+              echo 'true';
+            else
+              echo 'false';
+            fi
+          )"
+
+      - name: Remove 'triage' label
+        if: ${{ steps.check_comment.outputs.is_owner == 'true' }}
+        uses: actions-ecosystem/action-remove-labels@v1
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          labels: triage

.readthedocs.yaml

@@ -11,10 +11,6 @@ version: 2
 sphinx:
   configuration: docs/conf.py
 
-# Optionally build your docs in additional formats such as PDF
-formats:
-  - pdf
-
 # Optionally set the version of Python and requirements required to build your docs
 build:
   os: ubuntu-22.04

README.md

@@ -74,7 +74,6 @@ Linux, Windows, macOS and FreeBSD are supported. Docker images are also availabl
 | macOS (nix)                   | ``nix-env -i ocrmypdf``       |
 | LinuxBrew                     | ``brew install ocrmypdf``     |
 | FreeBSD                       | ``pkg install py-ocrmypdf``   |
-| Conda                         | ``conda install ocrmypdf``    |
 | Ubuntu Snap                   | ``snap install ocrmypdf``     |
 
 For everyone else, [see our documentation](https://ocrmypdf.readthedocs.io/en/latest/installation.html) for installation steps.
@@ -113,9 +112,33 @@ Our [documentation is served on Read the Docs](https://ocrmypdf.readthedocs.io/e
 
 Please report issues on our [GitHub issues](https://github.com/ocrmypdf/OCRmyPDF/issues) page, and follow the issue template for quick response.
 
+## Feature demo
+
+```bash
+# Add an OCR layer and convert to PDF/A
+ocrmypdf input.pdf output.pdf
+
+# Convert an image to single page PDF
+ocrmypdf input.jpg output.pdf
+
+# Add OCR to a file in place (only modifies file on success)
+ocrmypdf myfile.pdf myfile.pdf
+
+# OCR with non-English languages (look up your language's ISO 639-3 code)
+ocrmypdf -l fra LeParisien.pdf LeParisien.pdf
+
+# OCR multilingual documents
+ocrmypdf -l eng+fra Bilingual-English-French.pdf Bilingual-English-French.pdf
+
+# Deskew (straighten crooked pages)
+ocrmypdf --deskew input.pdf output.pdf
+```
+
+For more features, see the [documentation](https://ocrmypdf.readthedocs.io/en/latest/index.html).
+
 ## Requirements
 
-In addition to the required Python version (3.8+), OCRmyPDF requires external program installations of Ghostscript and Tesseract OCR. OCRmyPDF is pure Python, and runs on pretty much everything: Linux, macOS, Windows and FreeBSD.
+In addition to the required Python version, OCRmyPDF requires external program installations of Ghostscript and Tesseract OCR. OCRmyPDF is pure Python, and runs on pretty much everything: Linux, macOS, Windows and FreeBSD.
 
 ## Press & Media
 

docs/advanced.rst

@@ -235,7 +235,7 @@ The directive ``--tesseract-pagesegmode Nmode`` forwards the desired page segmen
 mode to Tesseract OCR. The default is 3.
 
 Page segmentation can improve OCR results when you know that a PDF ought to be
-analyzed a particular way, such as PDFs whose pages contain only a single line of 
+analyzed a particular way, such as PDFs whose pages contain only a single line of
 text. For the vast majority of users, changing the page segmentation mode will only
 make things worse.
 
@@ -244,37 +244,37 @@ As of June 2024, the Tesseract page segmentation modes are:
 +-----+----------------------------------------------------------------------------------+
 | ID  | Description                                                                      |
 +=====+==================================================================================+
-|  0  | Orientation and script detection (OSD) only.                                      |
+|  0  | Orientation and script detection (OSD) only.                                     |
 +-----+----------------------------------------------------------------------------------+
-|  1  | Automatic page segmentation with OSD.                                             |
+|  1  | Automatic page segmentation with OSD.                                            |
 +-----+----------------------------------------------------------------------------------+
 |  2  | Automatic page segmentation, but no OSD, or OCR. (not implemented)               |
 +-----+----------------------------------------------------------------------------------+
 |  3  | Fully automatic page segmentation, but no OSD. (Default)                         |
 +-----+----------------------------------------------------------------------------------+
-|  4  | Assume a single column of text of variable sizes.                                 |
+|  4  | Assume a single column of text of variable sizes.                                |
 +-----+----------------------------------------------------------------------------------+
-|  5  | Assume a single uniform block of vertically aligned text.                         |
+|  5  | Assume a single uniform block of vertically aligned text.                        |
 +-----+----------------------------------------------------------------------------------+
-|  6  | Assume a single uniform block of text.                                            |
+|  6  | Assume a single uniform block of text.                                           |
 +-----+----------------------------------------------------------------------------------+
-|  7  | Treat the image as a single text line.                                            |
+|  7  | Treat the image as a single text line.                                           |
 +-----+----------------------------------------------------------------------------------+
-|  8  | Treat the image as a single word.                                                 |
+|  8  | Treat the image as a single word.                                                |
 +-----+----------------------------------------------------------------------------------+
-|  9  | Treat the image as a single word in a circle.                                     |
+|  9  | Treat the image as a single word in a circle.                                    |
 +-----+----------------------------------------------------------------------------------+
-| 10  | Treat the image as a single character.                                            |
+| 10  | Treat the image as a single character.                                           |
 +-----+----------------------------------------------------------------------------------+
-| 11  | Sparse text. Find as much text as possible in no particular order.                |
+| 11  | Sparse text. Find as much text as possible in no particular order.               |
 +-----+----------------------------------------------------------------------------------+
 | 12  | Sparse text with OSD.                                                            |
 +-----+----------------------------------------------------------------------------------+
 | 13  | Raw line. Treat the image as a single text line, bypassing hacks that are        |
-|     | Tesseract-specific.                                                               |
+|     | Tesseract-specific.                                                              |
 +-----+----------------------------------------------------------------------------------+
 
-Modes 0, 1, 2, and 12 (all of those that enable orientation and script detection) 
+Modes 0, 1, 2, and 12 (all of those that enable orientation and script detection)
 are not compatible with OCRmyPDF, which performs OSD in a separate step from OCR.
 Their use may interfere with ``--rotate-pages`` and other features.
 
@@ -445,7 +445,7 @@ Debugging the intermediate files
 OCRmyPDF normally saves its intermediate results to a temporary folder
 and deletes this folder when it exits, whether it succeeded or failed.
 
-If the ``--keep-temporary-files`` (``-k```) argument is issued on the
+If the ``--keep-temporary-files`` (``-k``) argument is issued on the
 command line, OCRmyPDF will keep the temporary folder and print the location,
 whether it succeeded or failed. An example message is:
 

docs/apiref.rst

@@ -9,27 +9,12 @@ API reference
 This page summarizes the rest of the public API. Generally speaking this
 should be mainly of interest to plugin developers.
 
-ocrmypdf
-========
+ocrmypdf.api
+============
 
-.. autoclass:: ocrmypdf.PageContext
+.. automodule:: ocrmypdf.api
     :members:
 
-.. autoclass:: ocrmypdf.PdfContext
-    :members:
-
-.. autoclass:: ocrmypdf.Verbosity
-    :members:
-    :undoc-members:
-
-.. autofunction:: ocrmypdf.configure_logging
-
-.. autofunction:: ocrmypdf.ocr
-
-.. autofunction:: ocrmypdf.pdf_to_hocr
-
-.. autofunction:: ocrmypdf.hocr_to_ocr_pdf
-
 ocrmypdf.exceptions
 ===================
 

docs/batch.rst

@@ -135,7 +135,7 @@ Users may need to customize the script to meet their requirements.
     "OCR_ON_SUCCESS_ARCHIVE", "This will move the processed original file to ``OCR_ARCHIVE_DIRECTORY`` if the exit code is 0 (OK). Note that ``OCR_ON_SUCCESS_DELETE`` takes precedence over this option, i.e. if both options are set, the input file will be deleted."
     "OCR_OUTPUT_DIRECTORY_YEAR_MONTH", "This will place files in the output in ``{output}/{year}/{month}/{filename}``"
     "OCR_DESKEW", "Apply deskew to crooked input PDFs"
-    "OCR_JSON_SETTINGS", "A JSON string specifying any other arguments for ``ocrmypdf.ocr``, e.g. ``'OCR_JSON_SETTINGS={""rotate_pages"": true, ""optimize"": "3"}'``."
+    "OCR_JSON_SETTINGS", "A JSON string specifying any other arguments for ``ocrmypdf.ocr``, e.g. ``'OCR_JSON_SETTINGS={""rotate_pages"": true, ""optimize"": ""3""}'``."
     "OCR_POLL_NEW_FILE_SECONDS", "Polling interval"
     "OCR_LOGLEVEL", "Level of log messages to report"
 

docs/conf.py

@@ -30,6 +30,8 @@
 #
 # needs_sphinx = '1.0'
 
+import datetime
+
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
@@ -38,11 +40,12 @@ extensions = [
     'sphinx.ext.intersphinx',
     'sphinx.ext.autosummary',
     'sphinx.ext.napoleon',
+    'sphinx.ext.imgconverter',  # PDF docs needs this for SVG to PNG conversion
     'sphinx_issues',
 ]
 
 # Extension settings
-intersphinx_mapping = {'https://docs.python.org/': None}
+intersphinx_mapping = {'python': ('https://docs.python.org/3', None)}
 napoleon_use_rtype = False
 issues_github_path = "ocrmypdf/OCRmyPDF"
 
@@ -50,10 +53,7 @@ issues_github_path = "ocrmypdf/OCRmyPDF"
 templates_path = ['_templates']
 
 # The suffix(es) of source filenames.
-# You can specify multiple suffix as a list of string:
-#
-# source_suffix = ['.rst', '.md']
-source_suffix = '.rst'
+source_suffix = {'.rst': 'restructuredtext', '.md': 'markdown'}
 
 # The encoding of source files.
 #
@@ -64,8 +64,11 @@ master_doc = 'index'
 
 # General information about the project.
 project = 'ocrmypdf'
+
+year = str(datetime.date.today().year)
 copyright = (
-    '2023, James R. Barlow. Licensed under Creative Commons Attribution-ShareAlike 4.0.'
+    f'{year}, James R. Barlow. ',
+    'Licensed under Creative Commons Attribution-ShareAlike 4.0.',
 )
 author = 'James R. Barlow'
 
@@ -92,6 +95,7 @@ if on_rtd:
 
     MOCK_MODULES = [
         'pikepdf',
+        'pikepdf.canvas',
         'pikepdf.models',
         'pikepdf.models.metadata',
     ]
@@ -108,7 +112,7 @@ version = '.'.join(release.split('.')[:2])
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = 'en'
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
@@ -158,19 +162,18 @@ todo_include_todos = False
 
 # -- Options for HTML output ----------------------------------------------
 
-import sphinx_rtd_theme
+import sphinx_rtd_theme  # noqa: F401
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
 html_theme = 'sphinx_rtd_theme'
-html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
 #
-html_theme_options = {'display_version': False}
+html_theme_options = {}
 
 # Add any paths that contain custom themes here, relative to this directory.
 # html_theme_path = []
@@ -198,7 +201,7 @@ html_theme_options = {'display_version': False}
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+# html_static_path = ['_static']
 
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied

docs/cookbook.rst

@@ -399,7 +399,7 @@ Some users may consider enabling lossy JBIG2. See: :ref:`jbig2-lossy`.
 Digitally signed PDFs
 =====================
 
-OCRmyPDF cannot preserve digital signatures in PDFs and also add to OCR to them.
+OCRmyPDF cannot preserve digital signatures in PDFs and also add OCR to them.
 By default, it will refuse to modify a signed PDF regardless of other settings. You can
 override this behavior with ``--invalidate-digital-signatures``; as the name suggests,
 any digital signatures will be invalidated.

docs/docker.rst

@@ -35,7 +35,7 @@ execute the image:
    docker run hello-world
 
 .. list-table:: Docker images
-   :width: 30 20 50
+   :widths: 30 20 50
    :header-rows: 1
 
    * - Image
@@ -69,9 +69,9 @@ OCRmyPDF will use all available CPU cores. See the Docker documentation for
 if you are using Docker on macOS or Windows, where you may need to manually assign
 more resources. On Linux, all resources will be available automatically.
 
-The underlying operating system and other details in Docker images are subject
-to change at minor releases. If you are modifying the image, you should pin
-the version you intend to use.
+The underlying operating system and other details in Docker images are considered
+implementation details and **subject to change at minor releases**. If you are
+modifying the image, you should pin the version you intend to use.
 
 Using the Docker image on the command line
 ==========================================

docs/installation.rst

@@ -31,8 +31,6 @@ These platforms have one-liner installs:
 +-------------------------------+-----------------------------------------+
 | FreeBSD                       | ``pkg install textproc/py-ocrmypdf``    |
 +-------------------------------+-----------------------------------------+
-| Conda (WSL, macOS, Linux)     | ``conda install ocrmypdf``              |
-+-------------------------------+-----------------------------------------+
 | Snap (snapcraft packaging)    | ``snap install ocrmypdf``               |
 +-------------------------------+-----------------------------------------+
 

docs/jbig2.rst

@@ -37,8 +37,6 @@ For all other platforms, you would need to build the JBIG2 encoder from source:
    ./configure && make
    [sudo] make install
 
-.. _jbig2-lossy:
-
 Dependencies include libtoolize and libleptonica, which on Ubuntu systems
 are packaged as libtool and libleptonica-dev. On Fedora (35) they are packaged
 as libtool and leptonica-devel. For this to work, please make sure to install
@@ -48,8 +46,8 @@ installed.
 .. code-block:: bash
 
     [sudo] apt install autotools-dev automake libtool libleptonica-dev
-..
 
+.. _jbig2-lossy:
 
 Lossy mode JBIG2
 ================

docs/languages.rst

@@ -68,8 +68,8 @@ to what languages it should search for. Multiple languages can be
 requested using either ``-l eng+fra`` (English and French) or
 ``-l eng -l fra``.
 
-Archlinux
-------
+Arch Linux
+----------
 
 .. code-block:: bash
 

docs/optimizer.rst

@@ -14,8 +14,8 @@ perform other possible optimizations such as deduplicating resources,
 consolidating fonts, simplifying vector drawings, or anything of that nature.
 
 .. list-table:: Title
-   :widths: 33 6 60
-   :header-rows: 1
+    :widths: 33 6 60
+    :header-rows: 1
 
     * - Optimization level
       - Shorthand

docs/pdfsecurity.rst

@@ -2,6 +2,8 @@
 ..
 .. SPDX-License-Identifier: CC-BY-SA-4.0
 
+.. _security:
+
 ===================
 PDF security issues
 ===================

docs/release_notes.rst

@@ -30,6 +30,42 @@ OCRmyPDF typically supports the three most recent Python versions.
 
 .. |OCRmyPDF PyPI| image:: https://img.shields.io/pypi/v/ocrmypdf.svg
 
+v16.9.0
+=======
+
+-  Added hocr caption processing. Thanks @0dinD :issue:`1466`
+-  ocrmypdf-alpine Docker image is now built with Alpine 3.21.
+-  Fixed error handling of PDFs that contain invalid images with both ImageMask
+   and ColorSpace defined. :issue:`1453`
+-  Fixed test suite regression when only older Ghostscripts are installed.
+-  Improved documetnation of _progressbar.py. Thanks @QuentinFuxa. :issue:`1456`
+-  Disabling building of documentation as PDF on ReadTheDocs, as this caused
+   complex build issues deemed not worth solving.
+
+v16.8.0
+=======
+
+-  Upgraded webservice.py demonstration using streamlit. It's now possible to
+   exercise most of OCRmyPDF's functionality in a simple web UI.
+-  Added cache to Dockerfiles to improve build speed.
+-  Fixed numerous formatting errors in the documentation that prevented some
+   parts of documentation from generating correctly.
+-  Improved OCR text rendering by suppressing negative-width spaces. Thanks
+   @pajowu. :issue:`1446`
+-  Improved detecting of invisible text when using `--redo-ocr`. Thanks
+   @pajowu. :issue:`1448``
+
+v16.7.0
+=======
+
+-  Fixed further issues with Docker build and updated some versions.
+-  Main Docker image returned to Ubuntu 24.04 since the fix in v16.6.2 resolved
+   that concern.
+-  Code that previously sent Ghostscript output to stdout has been changed to
+   output to temporary files, since Ghostscript was doing that anyway internally.
+   This is a modest efficiency improvement.
+-  Fixed an issue with debug log output being parsed as rich markup. :issue:`1444`
+
 v16.6.2
 =======
 

misc/_webservice.py

@@ -0,0 +1,251 @@
+# SPDX-FileCopyrightText: 2025 James R. Barlow
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+"""This is a simple web service/HTTP wrapper for OCRmyPDF.
+
+This may be more convenient than the command line tool for some Docker users.
+Note that OCRmyPDF uses Ghostscript, which is licensed under AGPLv3+. While
+OCRmyPDF is under GPLv3, this file is distributed under the Affero GPLv3+ license,
+to emphasize that SaaS deployments should make sure they comply with
+Ghostscript's license as well as OCRmyPDF's.
+"""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+import time
+import urllib
+from functools import partial
+from operator import getitem
+from pathlib import Path
+from shutil import which
+from tempfile import NamedTemporaryFile
+
+import pikepdf
+import streamlit as st
+from port_for import get_port
+from streamlit.components.v1 import iframe
+
+from ocrmypdf._defaults import DEFAULT_ROTATE_PAGES_THRESHOLD
+
+
+def get_host_url_with_port(port: int) -> str:
+    """Get the host URL for the web service. Hacky."""
+    host_url = st.context.headers["host"]
+    try:
+        host, _streamlit_port = host_url.split(":", maxsplit=1)
+    except ValueError:
+        host = host_url
+    return f"//{host}:{port}"  # Use the same protocol
+
+
+st.title("OCRmyPDF Web Service")
+
+if not which("ttyd"):
+    st.error("Missing dependency: ttyd. Please install ttyd to the local environment.")
+    sys.exit(1)
+
+uploaded = st.file_uploader("Upload input PDF or image", type=["pdf"], key="file")
+
+mode = st.selectbox("Mode", options=["normal", "skip-text", "force-ocr", "redo-ocr"])
+
+with st.expander("Input options"):
+    invalidate_digital_signatures = st.checkbox(
+        "Invalidate digital signatures", value=False
+    )
+    language = st.selectbox("Language", options=["eng", "deu", "fra", "spa"])
+
+    image_dpi = st.slider(
+        "Image DPI", value=300, key="image_dpi", min_value=1, max_value=5000, step=50
+    )
+with st.expander("Preprocessing"):
+    skip_big = st.checkbox("Skip OCR on big pages", value=False, key="skip_big")
+    oversample = st.slider("Oversample", min_value=0, max_value=5000, value=0, step=50)
+    rotate_pages = st.checkbox("Rotate pages", value=False, key="rotate")
+    deskew = st.checkbox("Deskew pages", value=False, key="deskew")
+    clean = st.checkbox("Clean pages before OCR", value=False, key="clean")
+    clean_final = st.checkbox("Clean final", value=False, key="clean_final")
+    remove_vectors = st.checkbox("Remove vectors", value=False, key="remove_vectors")
+
+
+with st.expander("Output options"):
+    output_type = st.selectbox(
+        "Output type", options=["pdfa", "pdfa", "pdfa-1", "pdfa-2", "pdfa-3", "none"]
+    )
+
+    pdf_renderer = st.selectbox(
+        "PDF rendereer", options=["auto", "hocr", "hocrdebug", "sandwich"]
+    )
+
+    optimize = st.selectbox("Optimize", options=["0", "1", "2", "3"])
+
+    st.selectbox("PDF/A compression", options=["auto", "jpeg", "lossless"])
+
+with st.expander("Metadata"):
+    title = author = keywords = subject = None
+    if uploaded:
+        with pikepdf.open(uploaded) as pdf, pdf.open_metadata() as meta:
+            st.code(str(meta), language="xml")
+            title = st.text_input("Title", value=meta.get('dc:title', ''))
+            author = st.text_input("Author", value=meta.get('dc:creator', ''))
+            keywords = st.text_input("Keywords", value=meta.get('dc:subject', ''))
+            subject = st.text_input("Subject", value=meta.get('dc:description', ''))
+
+
+with st.expander("Optimization after OCR"):
+    jpeg_quality = st.slider(
+        "JPEG quality", min_value=0, max_value=100, value=75, key="jpeg_quality"
+    )
+    png_quality = st.slider(
+        "PNG quality", min_value=0, max_value=100, value=75, key="png_quality"
+    )
+    jbig2_lossy = st.checkbox("JBIG2 lossy (dangerous)", value=False, key="jbig2_lossy")
+    jbig2_threshold = st.number_input("JBIG2 threshold", value=0, key="jbig2_threshold")
+
+with st.expander("Advanced options"):
+    jobs = st.slider(
+        "Threads",
+        min_value=1,
+        max_value=os.cpu_count(),
+        value=os.cpu_count(),
+        key="threads",
+    )
+    pages = st.text_input(
+        "Pages", value="", help="Comma-separated list of pages to process"
+    )
+    max_image_mpixels = st.number_input(
+        "Max image size",
+        value=250.0,
+        min_value=0.0,
+        help="Maximum image size in megapixels",
+    )
+    rotate_pages_threshold = st.number_input(
+        "Rotate pages threshold",
+        value=DEFAULT_ROTATE_PAGES_THRESHOLD,
+        min_value=0.0,
+        max_value=1000.0,
+        help="Threshold for automatic page rotation",
+    )
+    fast_web_view = st.number_input(
+        "Fast web view",
+        value=1.0,
+        min_value=0.0,
+        help="Linearize files above this size in MB",
+    )
+    continue_on_soft_render_error = st.checkbox(
+        "Continue on soft render error", value=True
+    )
+    verbose_labels = ["quiet", "default", "debug", "debug_all"]
+    verbose = st.selectbox(
+        "Verbosity level",
+        options=[-1, 0, 1, 2],
+        index=1,
+        format_func=partial(getitem, verbose_labels),
+    )
+
+if uploaded:
+    args = []
+    if mode and mode != 'normal':
+        args.append(f"--{mode}")
+    if language:
+        args.append(f"--language={language}")
+    if not uploaded.name.lower().endswith(".pdf") and image_dpi:
+        args.append(f"--image-dpi={image_dpi}")
+    if skip_big:
+        args.append("--skip-big")
+    if oversample:
+        args.append(f"--oversample={oversample}")
+    if rotate_pages:
+        args.append("--rotate-pages")
+    if deskew:
+        args.append("--deskew")
+    if clean:
+        args.append("--clean")
+    if clean_final:
+        args.append("--clean-final")
+    if remove_vectors:
+        args.append("--remove-vectors")
+    if output_type:
+        args.append(f"--output-type={output_type}")
+    if pdf_renderer:
+        args.append(f"--pdf-renderer={pdf_renderer}")
+    if optimize:
+        args.append(f"--optimize={optimize}")
+    if title:
+        args.append(f"--title={title}")
+    if author:
+        args.append(f"--author={author}")
+    if keywords:
+        args.append(f"--keywords={keywords}")
+    if subject:
+        args.append(f"--subject={subject}")
+    if pages:
+        args.append(f"--pages={pages}")
+    if max_image_mpixels:
+        args.append(f"--max-image-mpixels={max_image_mpixels}")
+    if rotate_pages_threshold:
+        args.append(f"--rotate-pages-threshold={rotate_pages_threshold}")
+    if fast_web_view:
+        args.append(f"--fast-web-view={fast_web_view}")
+    if continue_on_soft_render_error:
+        args.append("--continue-on-soft-render-error")
+    if verbose:
+        args.append(f"--verbose={verbose}")
+    if optimize > '0' and jpeg_quality:
+        args.append(f"--jpeg-quality={jpeg_quality}")
+    if optimize > '0' and png_quality:
+        args.append(f"--png-quality={png_quality}")
+    if jbig2_lossy:
+        args.append("--jbig2-lossy")
+    if jbig2_threshold:
+        args.append(f"--jbig2-threshold={jbig2_threshold}")
+    if jobs:
+        args.append(f"--jobs={jobs}")
+    input_file = NamedTemporaryFile(delete=True, suffix=f"_{uploaded.name}")
+    input_file.write(uploaded.getvalue())
+    input_file.flush()
+    input_file.seek(0)
+    args.append(str(input_file.name))
+    output_file = NamedTemporaryFile(delete=True, suffix=".pdf")
+    args.append(str(output_file.name))
+
+    st.session_state['running'] = (
+        'run_button' in st.session_state and st.session_state.run_button
+    )
+    if st.button(
+        "Run OCRmyPDF",
+        disabled=st.session_state.get("running", False),
+        key='run_button',
+    ):
+        st.session_state['running'] = True
+        args = [sys.executable, '-m', "ocrmypdf"] + args
+        cmdline = " ".join(args)
+        st.code(cmdline, language="bash")
+
+        port = get_port((5000, 7000))
+        ttyd_args = ['ttyd', '--port', str(port), '--once', '--readonly']
+
+        ttyd_proc = subprocess.Popen(
+            ttyd_args + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+        )
+        iframe(src=get_host_url_with_port(port), height=400)
+
+        while ttyd_proc.poll() is None:
+            ttyd_proc.poll()
+            time.sleep(1)
+
+        if ttyd_proc.returncode == 0:
+            if Path(output_file.name).stat().st_size == 0:
+                st.error("No output PDF file was generated")
+            else:
+                st.download_button(
+                    label="Download output PDF",
+                    data=input_file.read(),
+                    file_name=uploaded.name,
+                    mime="application/pdf",
+                )
+        else:
+            st.error(f"ttyd failed with exit code {ttyd_proc.returncode}")
+        st.session_state['running'] = False

misc/ocrmypdf_compare.py

@@ -0,0 +1,123 @@
+# SPDX-FileCopyrightText: 2025 James R. Barlow
+# SPDX-License-Identifier: MIT
+
+"""Run OCRmyPDF on the same PDF with different options."""
+
+from __future__ import annotations
+
+import os
+import shlex
+from io import BytesIO
+from pathlib import Path
+from subprocess import check_output, run
+from tempfile import TemporaryDirectory
+
+import pikepdf
+import pymupdf
+import streamlit as st
+from lxml import etree
+from streamlit_pdf_viewer import pdf_viewer
+
+
+def main():
+    st.set_page_config(layout="wide")
+
+    st.title("OCRmyPDF Compare")
+    st.write("Run OCRmyPDF on the same PDF with different options.")
+
+    uploaded_pdf = st.file_uploader("Upload a PDF", type=["pdf"])
+    if uploaded_pdf is None:
+        return
+
+    pdf_bytes = uploaded_pdf.read()
+
+    with pikepdf.open(BytesIO(pdf_bytes)) as p, TemporaryDirectory() as d:
+        with st.expander("PDF Metadata"):
+            with p.open_metadata() as meta:
+                xml_txt = str(meta)
+                parser = etree.XMLParser(remove_blank_text=True)
+                tree = etree.fromstring(xml_txt, parser=parser)
+                st.code(
+                    etree.tostring(tree, pretty_print=True).decode("utf-8"),
+                    language="xml",
+                )
+            st.write(p.docinfo)
+            st.write("Number of pages:", len(p.pages))
+
+        col1, col2 = st.columns(2)
+        with col1:
+            cli1 = st.text_area(
+                "Command line arguments for A",
+                key="args1",
+                value="ocrmypdf {in_} {out}",
+            )
+            env1 = st.text_area("Environment variables for A", key="env1")
+            args1 = shlex.split(
+                cli1.format(
+                    in_=os.path.join(d, "input.pdf"),
+                    out=os.path.join(d, "output1.pdf"),
+                )
+            )
+            st.code(shlex.join(args1))
+        with col2:
+            cli2 = st.text_area(
+                "Command line arguments for B",
+                key="args2",
+                value="ocrmypdf {in_} {out}",
+            )
+            env2 = st.text_area("Environment variables for B", key="env2")
+            args2 = shlex.split(
+                cli2.format(
+                    in_=os.path.join(d, "input.pdf"),
+                    out=os.path.join(d, "output2.pdf"),
+                )
+            )
+            st.code(shlex.join(args2))
+
+        if not st.button("Execute and Compare"):
+            return
+        with st.spinner("Executing..."):
+            Path(d, "input.pdf").write_bytes(pdf_bytes)
+            run(args1, env=dict(os.environ, **eval(env1 or "{}")))
+            run(args2, env=dict(os.environ, **eval(env2 or "{}")))
+
+            col1, col2 = st.columns(2)
+            with col1:
+                st.text(
+                    "Ghostscript version A: "
+                    + check_output(
+                        ["gs", "--version"],
+                        env=dict(os.environ, **eval(env1 or "{}")),
+                        text=True,
+                    )
+                )
+            with col2:
+                st.text(
+                    "Ghostscript version B: "
+                    + check_output(
+                        ["gs", "--version"],
+                        env=dict(os.environ, **eval(env2 or "{}")),
+                        text=True,
+                    )
+                )
+
+            doc1 = pymupdf.open(os.path.join(d, "output1.pdf"))
+            doc2 = pymupdf.open(os.path.join(d, "output2.pdf"))
+            for i, page1_2 in enumerate(zip(doc1, doc2)):
+                st.write(f"Page {i+1}")
+                page1, page2 = page1_2
+                col1, col2 = st.columns(2)
+                with col1, st.container(border=True):
+                    st.write(page1.get_text())
+                with col2, st.container(border=True):
+                    st.write(page2.get_text())
+
+            col1, col2 = st.columns(2)
+            with col1, st.expander("PDF Viewer"):
+                pdf_viewer(Path(d, "output1.pdf"))
+            with col2, st.expander("PDF Viewer"):
+                pdf_viewer(Path(d, "output2.pdf"))
+
+
+if __name__ == "__main__":
+    main()

misc/pdf_compare.py

@@ -0,0 +1,83 @@
+# SPDX-FileCopyrightText: 2025 James R. Barlow
+# SPDX-License-Identifier: MIT
+
+"""Compare two PDFs."""
+
+from __future__ import annotations
+
+import os
+from io import BytesIO
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+import pikepdf
+import pymupdf
+import streamlit as st
+from lxml import etree
+from streamlit_pdf_viewer import pdf_viewer
+
+
+def do_metadata(pdf):
+    with pikepdf.open(pdf) as pdf:
+        with pdf.open_metadata() as meta:
+            xml_txt = str(meta)
+            parser = etree.XMLParser(remove_blank_text=True)
+            tree = etree.fromstring(xml_txt, parser=parser)
+            st.code(
+                etree.tostring(tree, pretty_print=True).decode("utf-8"),
+                language="xml",
+            )
+        st.write(pdf.docinfo)
+        st.write("Number of pages:", len(pdf.pages))
+
+
+def main():
+    st.set_page_config(layout="wide")
+
+    st.title("PDF Compare")
+    st.write("Compare two PDFs.")
+
+    col1, col2 = st.columns(2)
+    with col1:
+        uploaded_pdf1 = st.file_uploader("Upload a PDF", type=["pdf"], key='pdf1')
+    with col2:
+        uploaded_pdf2 = st.file_uploader("Upload a PDF", type=["pdf"], key='pdf2')
+    if uploaded_pdf1 is None or uploaded_pdf2 is None:
+        return
+
+    pdf_bytes1 = uploaded_pdf1.getvalue()
+    pdf_bytes2 = uploaded_pdf2.getvalue()
+
+    with st.expander("PDF Metadata"):
+        col1, col2 = st.columns(2)
+        with col1:
+            do_metadata(BytesIO(pdf_bytes1))
+        with col2:
+            do_metadata(BytesIO(pdf_bytes2))
+
+    with TemporaryDirectory() as d:
+        Path(d, "1.pdf").write_bytes(pdf_bytes1)
+        Path(d, "2.pdf").write_bytes(pdf_bytes2)
+
+        with st.expander("Text"):
+            doc1 = pymupdf.open(os.path.join(d, "1.pdf"))
+            doc2 = pymupdf.open(os.path.join(d, "2.pdf"))
+            for i, page1_2 in enumerate(zip(doc1, doc2)):
+                st.write(f"Page {i+1}")
+                page1, page2 = page1_2
+                col1, col2 = st.columns(2)
+                with col1, st.container(border=True):
+                    st.write(page1.get_text())
+                with col2, st.container(border=True):
+                    st.write(page2.get_text())
+
+        with st.expander("PDF Viewer"):
+            col1, col2 = st.columns(2)
+            with col1:
+                pdf_viewer(Path(d, "1.pdf"), key='pdf_viewer1', render_text=True)
+            with col2:
+                pdf_viewer(Path(d, "2.pdf"), key='pdf_viewer2', render_text=True)
+
+
+if __name__ == "__main__":
+    main()

misc/pdf_text_diff.py

@@ -0,0 +1,46 @@
+# SPDX-FileCopyrightText: 2025 James R. Barlow
+# SPDX-License-Identifier: MPL-2.0
+
+"""Compare text in PDFs."""
+
+from __future__ import annotations
+
+from subprocess import run
+from tempfile import NamedTemporaryFile
+from typing import Annotated
+
+import typer
+
+
+def main(
+    pdf1: Annotated[typer.FileBinaryRead, typer.Argument()],
+    pdf2: Annotated[typer.FileBinaryRead, typer.Argument()],
+    engine: Annotated[str, typer.Option()] = 'pdftotext',
+):
+    """Compare text in PDFs."""
+
+    text1 = run(
+        ['pdftotext', '-layout', '-', '-'], stdin=pdf1, capture_output=True, check=True
+    )
+    text2 = run(
+        ['pdftotext', '-layout', '-', '-'], stdin=pdf2, capture_output=True, check=True
+    )
+
+    with NamedTemporaryFile() as f1, NamedTemporaryFile() as f2:
+        f1.write(text1.stdout)
+        f1.flush()
+        f2.write(text2.stdout)
+        f2.flush()
+        diff = run(
+            ['diff', '--color=always', '--side-by-side', f1.name, f2.name],
+            capture_output=True,
+        )
+        run(['less', '-R'], input=diff.stdout, check=True)
+        if text1.stdout.strip() != text2.stdout.strip():
+            return 1
+
+    return 0
+
+
+if __name__ == '__main__':
+    typer.run(main)

misc/webservice.py

@@ -1,107 +1,29 @@
-#!/usr/bin/env python3
-# SPDX-FileCopyrightText: 2019 James R. Barlow
+#!/usr/bin/env python
+# SPDX-FileCopyrightText: 2025 James R. Barlow
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
-"""This is a simple web service/HTTP wrapper for OCRmyPDF.
-
-This may be more convenient than the command line tool for some Docker users.
-Note that OCRmyPDF uses Ghostscript, which is licensed under AGPLv3+. While
-OCRmyPDF is under GPLv3, this file is distributed under the Affero GPLv3+ license,
-to emphasize that SaaS deployments should make sure they comply with
-Ghostscript's license as well as OCRmyPDF's.
-"""
-
-from __future__ import annotations
+"""Run the OCRmyPDF web service."""
 
 import os
-import shlex
-from subprocess import run
-from tempfile import TemporaryDirectory
+import sys
 
-from flask import Flask, Response, request, send_from_directory
-from werkzeug.utils import secure_filename
+try:
+    import streamlit  # noqa: F401
+except ImportError:
+    raise ImportError(
+        'You need to install streamlit in the Python environment '
+        'to run the web service.\n'
+    )
 
-app = Flask(__name__)
-app.secret_key = "secret"
-app.config['MAX_CONTENT_LENGTH'] = 50_000_000
-app.config.from_envvar("OCRMYPDF_WEBSERVICE_SETTINGS", silent=True)
-
-ALLOWED_EXTENSIONS = {"pdf"}
-
-
-def allowed_file(filename):
-    return "." in filename and filename.rsplit(".", 1)[1].lower() in ALLOWED_EXTENSIONS
-
-
-def do_ocrmypdf(file):
-    uploaddir = TemporaryDirectory(prefix="ocrmypdf-upload")
-    downloaddir = TemporaryDirectory(prefix="ocrmypdf-download")
-
-    filename = secure_filename(file.filename)
-    up_file = os.path.join(uploaddir.name, filename)
-    file.save(up_file)
-
-    down_file = os.path.join(downloaddir.name, filename)
-
-    cmd_args = [arg for arg in shlex.split(request.form["params"])]
-    if "--sidecar" in cmd_args:
-        return Response("--sidecar not supported", 501, mimetype='text/plain')
-
-    ocrmypdf_args = ["ocrmypdf", *cmd_args, up_file, down_file]
-    proc = run(ocrmypdf_args, capture_output=True, encoding="utf-8", check=False)
-    if proc.returncode != 0:
-        stderr = proc.stderr
-        return Response(stderr, 400, mimetype='text/plain')
-
-    return send_from_directory(downloaddir.name, filename)
-
-
-@app.route("/", methods=["GET", "POST"])
-def upload_file():
-    if request.method == "POST":
-        if "file" not in request.files:
-            return Response("No file in POST", 400, mimetype='text/plain')
-        file = request.files["file"]
-        if file.filename == "":
-            return Response("Empty filename", 400, mimetype='text/plain')
-        if not allowed_file(file.filename):
-            return Response("Invalid filename", 400, mimetype='text/plain')
-        if file and allowed_file(file.filename):
-            return do_ocrmypdf(file)
-        return Response("Some other problem", 400, mimetype='text/plain')
-
-    return """
-    <!doctype html>
-    <title>OCRmyPDF webservice</title>
-    <h1>Upload a PDF (debug UI)</h1>
-    <form method=post enctype=multipart/form-data>
-      <label for="args">Command line parameters</label>
-      <input type=textbox name=params>
-      <label for="file">File to upload</label>
-      <input type=file name=file>
-      <input type=submit value=Upload>
-    </form>
-    <h4>Notice</h2>
-    <div style="font-size: 70%; max-width: 34em;">
-    <p>This is a webservice wrapper for OCRmyPDF.</p>
-    <p>Copyright 2019 James R. Barlow</p>
-    <p>This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Affero General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-    </p>
-    <p>This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-    </p>
-    <p>
-    You should have received a copy of the GNU Affero General Public License
-    along with this program.  If not, see &lt;http://www.gnu.org/licenses/&gt;.
-    </p>
-    </div>
-    """
-
-
-if __name__ == "__main__":
-    app.run(host='0.0.0.0', port=5000)
+if __name__ == '__main__':
+    os.execvp(
+        sys.executable,
+        [
+            sys.executable,
+            '-m',
+            'streamlit',
+            'run',
+            'misc/_webservice.py',
+            *sys.argv[1:],
+        ],
+    )

pyproject.toml

@@ -63,7 +63,10 @@ test = [
   "types-humanfriendly",
 ]
 watcher = ["watchdog>=1.0.2", "typer-slim[standard]", "python-dotenv"]
-webservice = ["Flask>=2.0.1"]
+webservice = [
+    "port-for>=0.7.4",
+    "streamlit>=1.41.0",
+]
 
 [project.scripts]
 ocrmypdf = "ocrmypdf.__main__:run"
@@ -118,8 +121,6 @@ filterwarnings = [
 [[tool.mypy.overrides]]
 module = [
   'pluggy',
-  'tqdm',
-  'coloredlogs',
   'img2pdf',
   'pdfminer.*',
   'reportlab.*',
@@ -157,4 +158,10 @@ convention = "google"
 quote-style = "preserve"
 
 [dependency-groups]
-dev = ["mypy>=1.13.0"]
+dev = [
+  "mypy>=1.13.0",
+  "pymupdf>=1.24.14",
+  "streamlit-pdf-viewer>=0.0.19",
+  "streamlit>=1.40.2",
+  "ipykernel>=6.29.5",
+]

src/ocrmypdf/_exec/ghostscript.py

@@ -17,7 +17,11 @@ from subprocess import PIPE, CalledProcessError
 from packaging.version import Version
 from PIL import Image, UnidentifiedImageError
 
-from ocrmypdf.exceptions import ColorConversionNeededError, SubprocessOutputError
+from ocrmypdf.exceptions import (
+    ColorConversionNeededError,
+    InputFileError,
+    SubprocessOutputError,
+)
 from ocrmypdf.helpers import Resolution
 from ocrmypdf.subprocess import get_version, run, run_polling_stderr
 
@@ -111,7 +115,6 @@ def rasterize_pdf(
     args_gs = (
         [
             GS,
-            '-dQUIET',
             '-dSAFER',
             '-dBATCH',
             '-dNOPAUSE',
@@ -125,7 +128,7 @@ def rasterize_pdf(
         + (['-dPDFSTOPONERROR'] if stop_on_error else [])
         + [
             '-o',
-            '-',
+            fspath(output_file),
             '-sstdout=%stderr',  # Literal %s, not string interpolation
             '-dAutoRotatePages=/None',  # Probably has no effect on raster
             '-f',
@@ -137,14 +140,23 @@ def rasterize_pdf(
         p = run(args_gs, stdout=PIPE, stderr=PIPE, check=True)
     except CalledProcessError as e:
         log.error(e.stderr.decode(errors='replace'))
-        raise SubprocessOutputError('Ghostscript rasterizing failed') from e
-    else:
-        stderr = p.stderr.decode(errors='replace')
-        if _gs_error_reported(stderr):
-            log.error(stderr)
+        Path(output_file).unlink(missing_ok=True)
+        raise SubprocessOutputError("Ghostscript rasterizing failed") from e
+
+    stderr = p.stderr.decode(errors='replace')
+    if _gs_error_reported(stderr):
+        log.error(stderr)
+        if stop_on_error and "recoverable image error" in stderr:
+            Path(output_file).unlink(missing_ok=True)
+            raise InputFileError(
+                "Ghostscript rasterizing failed. The input file contains errors that "
+                "cause PDF viewers to interpret it differently and incorrectly. "
+                "Try using --continue-on-soft-render-error and manually inspect the "
+                "input and output files to check for visual differences or errors."
+            )
 
     try:
-        with Image.open(BytesIO(p.stdout)) as im:
+        with Image.open(output_file) as im:
             if rotation is not None:
                 log.debug("Rotating output by %i", rotation)
                 # rotation is a clockwise angle and Image.ROTATE_* is
@@ -157,13 +169,19 @@ def rasterize_pdf(
                     im = im.transpose(Image.Transpose.ROTATE_270)
                 if rotation % 180 == 90:
                     page_dpi = page_dpi.flip_axis()
-            im.save(fspath(output_file), dpi=page_dpi)
+            im.save(output_file, dpi=page_dpi)
     except UnidentifiedImageError:
         log.error(
             f"Ghostscript (using {raster_device} at {raster_dpi} dpi) produced "
             "an invalid page image file."
         )
         raise
+    except OSError as e:
+        log.error(
+            f"Ghostscript (using {raster_device} at {raster_dpi} dpi) produced "
+            "an invalid page image file."
+        )
+        raise UnidentifiedImageError() from e
 
 
 class GhostscriptFollower:
@@ -271,19 +289,15 @@ def generate_pdfa(
             f"-dPDFA={pdfa_part}",
             "-dPDFACompatibilityPolicy=1",
             "-o",
-            "-",
+            fspath(output_file),
             "-sstdout=%stderr",  # Literal %s, not string interpolation
         ]
     )
     args_gs.extend(fspath(s) for s in pdf_pages)  # Stringify Path objs
     try:
-        with (
-            Path(output_file).open('wb') as output,
-            GhostscriptFollower(progressbar_class) as pbar,
-        ):
+        with GhostscriptFollower(progressbar_class) as pbar:
             p = run_polling_stderr(
                 args_gs,
-                stdout=output,
                 stderr=PIPE,
                 check=True,
                 text=True,

src/ocrmypdf/_graft.py

@@ -61,19 +61,31 @@ def strip_invisible_text(pdf: Pdf, page: Page):
     stream = []
     in_text_obj = False
     render_mode = 0
+    render_mode_stack = []
     text_objects = []
 
     for operands, operator in parse_content_stream(page, ''):
+        if operator == Operator('Tr'):
+            render_mode = operands[0]
+
+        if operator == Operator('q'):
+            render_mode_stack.append(render_mode)
+
+        if operator == Operator('Q'):
+            try:
+                render_mode = render_mode_stack.pop()
+            except IndexError:
+                # Stack underflow: content stream is malformed
+                # but try to carry on
+                pass
+
         if not in_text_obj:
             if operator == Operator('BT'):
                 in_text_obj = True
-                render_mode = 0
                 text_objects.append((operands, operator))
             else:
                 stream.append((operands, operator))
         else:
-            if operator == Operator('Tr'):
-                render_mode = operands[0]
             text_objects.append((operands, operator))
             if operator == Operator('ET'):
                 in_text_obj = False

src/ocrmypdf/_logging.py

@@ -26,5 +26,5 @@ class PageNumberFilter(logging.Filter):
 class RichLoggingHandler(RichHandler):
     def __init__(self, console: Console, **kwargs):
         super().__init__(
-            console=console, show_level=False, show_time=False, markup=True, **kwargs
+            console=console, show_level=False, show_time=False, markup=False, **kwargs
         )

src/ocrmypdf/_pipeline.py

@@ -79,6 +79,14 @@ def triage_image_file(input_file: Path, output_file: Path, options) -> None:
     except OSError as e:
         # Recover the original filename
         log.error(str(e).replace(str(input_file), str(options.input_file)))
+        if not input_file.exists():
+            log.error("Input file does not exist: %s", input_file)
+        if input_file.is_dir():
+            log.error("Input file is a directory: %s", input_file)
+        if input_file.is_file():
+            log.error("Input file is a file: %s", input_file)
+        if input_file.stat().st_size == 0:
+            log.error("Input file is empty: %s", input_file)
         raise UnsupportedImageFormatError() from e
 
     with im:

src/ocrmypdf/_progressbar.py

@@ -32,12 +32,76 @@ class ProgressBar(Protocol):
     The progress bar is held in the main process/thread and not updated by child
     process/threads. When a child notifies the parent of completed work, the
     parent updates the progress bar.
-
     Progress bars should never write to ``sys.stdout``, or they will corrupt the
     output if OCRmyPDF writes a PDF to standard output.
 
-    The type of events that OCRmyPDF reports to a progress bar may change in
+    Note:
+        The type of events that OCRmyPDF reports to a progress bar may change in
     minor releases.
+
+    Args:
+        total (int | float | None):
+            The total number of work units expected. If ``None``, the total is unknown.
+            For example, if you are processing pages, this might be the number of pages,
+            or if you are measuring overall progress in percent, this might be 100.
+        desc (str | None):
+            A brief description of the current step (e.g. "Scanning contents",
+            "OCR", "PDF/A conversion"). OCRmyPDF updates this before each major step.
+        unit (str | None):
+            A short label for the type of work being tracked (e.g. "page", "%", "image").
+        disable (bool):
+            If ``True``, progress updates are suppressed (no output). Defaults to ``False``.
+        **kwargs:
+            Future or extra parameters that OCRmyPDF might pass. Implementations
+            should accept and ignore unrecognized keywords gracefully.
+
+    Example:
+        A simple plugin implementation could look like this:
+
+        .. code-block:: python
+
+            from ocrmypdf.pluginspec import ProgressBar
+            from ocrmypdf import hookimpl
+
+            class ConsoleProgressBar(ProgressBar):
+                def __init__(self, *, total=None, desc=None, unit=None, disable=False, **kwargs):
+                    self.total = total
+                    self.desc = desc
+                    self.unit = unit
+                    self.disable = disable
+                    self.current = 0
+
+                def __enter__(self):
+                    if not self.disable:
+                        print(f"Starting {self.desc or 'an OCR task'} (total={self.total} {self.unit})")
+                    return self
+
+                def __exit__(self, exc_type, exc_value, traceback):
+                    if not self.disable:
+                        if exc_type is None:
+                            print("Completed successfully.")
+                        else:
+                            print(f"Task ended with error: {exc_value}")
+                    return False  # Let OCRmyPDF raise any exceptions
+
+                def update(self, n=1, *, completed=None):
+                    if completed is not None:
+                        # If 'completed' is given, you could set self.current = completed
+                        # but let's just read it to show usage
+                        print(f"Absolute completion reported: {completed}")
+                    # Otherwise, we increment by 'n'
+                    self.current += n
+                    if not self.disable:
+                        if self.total:
+                            percent = (self.current / self.total) * 100
+                            print(f"{self.desc}: {self.current}/{self.total} ({percent:.1f}%)")
+                        else:
+                            print(f"{self.desc}: {self.current} units done")
+
+            @hookimpl
+            def get_progressbar_class():
+                return MyProgressBar
+
     """
 
     def __init__(
@@ -51,13 +115,22 @@ class ProgressBar(Protocol):
     ):
         """Initialize a progress bar.
 
-        *total* indicates the total number of work units. If None, the total
-        number of work units is unknown. If *disable* is True, the progress bar
-        should be disabled. *unit* is a description of the work unit.
-        *desc* is a description of the overall task to be performed.
+        This is called once before any work is done. OCRmyPDF supplies the total
+        number of units (or None if unknown), a description of the work, and the
+        type of units. The ``disable`` parameter can be used to turn off progress
+        reporting. Unrecognized keyword arguments should be ignored.
 
-        Unrecognized keyword arguments must be ignored, as the list of keyword
-        arguments may grow with time.
+        Args:
+            total (int | float | None):
+                The total amount of work. If ``None``, the total is unknown.
+            desc (str | None):
+                A description of the current task. May change for different stages.
+            unit (str | None):
+                A short label for the unit of work.
+            disable (bool):
+                If ``True``, no output or logging should be displayed.
+            **kwargs:
+                Extra parameters that may be passed by OCRmyPDF in future versions.
         """
 
     def __enter__(self):
@@ -66,10 +139,32 @@ class ProgressBar(Protocol):
     def __exit__(self, *args):
         """Exit a progress bar context."""
 
-    def update(self, n=1, *, completed=None):
-        """Update the progress bar by an increment.
+    def update(self, n: float = 1, *, completed: float | None = None):
+        """Increment the progress bar by ``n`` units, or set an absolute completion.
 
-        For use within a progress bar context.
+        OCRmyPDF calls this method repeatedly while processing pages or other tasks.
+        If your total is known and you track it, you might do something like:
+
+        .. code-block:: python
+
+            self.current += n
+            percent = (self.current / total) * 100
+
+        The ``completed`` argument can indicate an absolute position, which is
+        particularly helpful if you're tracking a percentage of work (e.g., 0 to 100)
+        and want precise updates. In contrast, the incremental parameter ``n`` is
+        often more useful for page-based increments.
+
+        Args:
+            n (float, optional):
+                The amount to increment the progress by. Defaults to 1. May be
+                fractional if OCRmyPDF performs partial steps. If you are tracking
+                pages, this is typically how many pages have been processed in the
+                most recent step.
+            completed (float | None, optional):
+                The absolute amount of work completed so far. This can override or
+                supplement the simple increment logic. It's particularly useful
+                for percentage-based tracking (e.g., when ``total`` is 100).
         """
 
 

src/ocrmypdf/hocrtransform/_hocr.py

@@ -210,7 +210,7 @@ class HocrTransform:
                     for element in par.iterfind(self._child_xpath('span'))
                     if 'class' in element.attrib
                     and element.attrib['class']
-                    in {'ocr_header', 'ocr_line', 'ocr_textfloat'}
+                    in {'ocr_header', 'ocr_line', 'ocr_textfloat', 'ocr_caption'}
                 ):
                     found_lines = True
                     direction = self._get_text_direction(par)
@@ -396,7 +396,7 @@ class HocrTransform:
         )
         if hocr_next_box is None:
             return
-        # Render a space this word and the next word. The explicit space helps
+        # Render a space between this word and the next word. The explicit space helps
         # PDF viewers identify the word break, and horizontally scaling it to
         # occupy the space the between the words helps the PDF viewer
         # avoid combiningthewordstogether.
@@ -409,7 +409,7 @@ class HocrTransform:
             space_box = Rectangle(next_box.urx, box.lly, box.llx, next_box.ury)
         self._debug_draw_space_bbox(canvas, space_box)
         space_width = self._font.text_width(' ', fontsize)
-        if space_width > 0:
+        if space_width > 0 and space_box.width > 0:
             if text_direction == TextDirection.LTR:
                 text.text_transform(Matrix(1, 0, 0, -1, space_box.llx, 0))
             elif text_direction == TextDirection.RTL:

src/ocrmypdf/pluginspec.py

@@ -145,13 +145,15 @@ def get_progressbar_class() -> type[ProgressBar]:
     The class returned by this function must be compatible with the
     :class:`ProgressBar` protocol.
 
-    Here is how OCRmyPDF will use the progress bar:
-
     Example:
-        pbar_class = pm.hook.get_progressbar_class()
-        with pbar_class(**progress_kwargs) as pbar:
-            ...
-            pbar.update(1)
+        Here is how OCRmyPDF will use the progress bar:
+
+        .. code-block:: python
+
+            pbar_class = pm.hook.get_progressbar_class()
+            with pbar_class(**progress_kwargs) as pbar:
+                ... # do some work
+                pbar.update(1)
     """
 
 

tests/test_ghostscript.py

@@ -11,10 +11,12 @@ from unittest.mock import patch
 
 import pikepdf
 import pytest
+from packaging.version import Version
 from PIL import Image, UnidentifiedImageError
 
+from ocrmypdf._exec import ghostscript
 from ocrmypdf._exec.ghostscript import DuplicateFilter, rasterize_pdf
-from ocrmypdf.exceptions import ColorConversionNeededError, ExitCode
+from ocrmypdf.exceptions import ColorConversionNeededError, ExitCode, InputFileError
 from ocrmypdf.helpers import Resolution
 
 from .conftest import check_ocrmypdf, run_ocrmypdf_api
@@ -139,7 +141,7 @@ def test_ghostscript_mandatory_color_conversion(resources, outpdf):
 
 def test_rasterize_pdf_errors(resources, no_outpdf, caplog):
     with patch('ocrmypdf._exec.ghostscript.run') as mock:
-        # ghostscript can produce
+        # ghostscript can produce empty files with return code 0
         mock.return_value = subprocess.CompletedProcess(
             ['fakegs'], returncode=0, stdout=b'', stderr=b'error this is an error'
         )
@@ -208,3 +210,71 @@ class TestDuplicateFilter:
         assert caplog.records[1].msg == "another error message"
         assert caplog.records[2].msg == "(suppressed 5 repeated lines)"
         assert caplog.records[3].msg == "yet another error message"
+
+
+@pytest.fixture
+def pdf_with_invalid_image(outdir):
+    # issue 1451
+    Name = pikepdf.Name
+    pdf = pikepdf.new()
+    pdf.add_blank_page()
+    pdf.pages[0].Contents = pdf.make_stream(b'612 0 0 612 0 0 cm /Image Do')
+    # Create an invalid image object that has both ColorSpace and ImageMask set
+    pdf.pages[0].Resources = pikepdf.Dictionary(
+        XObject=pdf.make_indirect(
+            pikepdf.Dictionary(
+                Image=pdf.make_stream(
+                    b"\xf0\x0f" * 8,
+                    ColorSpace=Name.DeviceGray,
+                    BitsPerComponent=1,
+                    Width=8,
+                    Height=8,
+                    ImageMask=True,
+                    Subtype=Name.Image,
+                    Type=Name.XObject,
+                )
+            )
+        )
+    )
+    pdf.save(outdir / 'invalid_image.pdf')
+    pdf.save('invalid_image.pdf')
+    return outdir / 'invalid_image.pdf'
+
+
+@pytest.mark.xfail(
+    ghostscript.version() < Version('10.04.0'),
+    reason="Older Ghostscript behavior is different",
+)
+def test_recoverable_image_error(pdf_with_invalid_image, outdir, caplog):
+    # When stop_on_error is False, we expect Ghostscript to print an error
+    # but continue
+    rasterize_pdf(
+        outdir / 'invalid_image.pdf',
+        outdir / 'out.png',
+        raster_device='pngmono',
+        raster_dpi=Resolution(10, 10),
+        stop_on_error=False,
+    )
+    assert 'Image has both ImageMask and ColorSpace' in caplog.text
+
+
+@pytest.mark.xfail(
+    ghostscript.version() < Version('10.04.0'),
+    reason="Older Ghostscript behavior is different",
+)
+def test_recoverable_image_error_with_stop(pdf_with_invalid_image, outdir, caplog):
+    # When stop_on_error is True, Ghostscript will print an error and exit
+    # but still produce a viable image. We intercept this case and raise
+    # InputFileError because it will contain an image of the whole page minus
+    # the image we are rendering.
+    with pytest.raises(
+        InputFileError, match="Try using --continue-on-soft-render-error"
+    ):
+        rasterize_pdf(
+            outdir / 'invalid_image.pdf',
+            outdir / 'out.png',
+            raster_device='pngmono',
+            raster_dpi=Resolution(100, 100),
+            stop_on_error=True,
+        )
+    # out2.png will not be created; if it were it would be blank.

tests/test_graft.py

@@ -40,3 +40,72 @@ def test_links(resources, outpdf):
         p2 = pdf.pages[1]
         assert p1.Annots[0].A.D[0].objgen == p2.objgen
         assert p2.Annots[0].A.D[0].objgen == p1.objgen
+
+
+def test_strip_invisble_text():
+    pdf = pikepdf.Pdf.new()
+    print(pikepdf.parse_content_stream(pikepdf.Stream(pdf, b'3 Tr')))
+    page = pdf.add_blank_page()
+    visible_text = [
+        pikepdf.ContentStreamInstruction((), pikepdf.Operator('BT')),
+        pikepdf.ContentStreamInstruction(
+            (pikepdf.Name('/F0'), 12), pikepdf.Operator('Tf')
+        ),
+        pikepdf.ContentStreamInstruction((288, 720), pikepdf.Operator('Td')),
+        pikepdf.ContentStreamInstruction(
+            (pikepdf.String('visible'),), pikepdf.Operator('Tj')
+        ),
+        pikepdf.ContentStreamInstruction((), pikepdf.Operator('ET')),
+    ]
+    invisible_text = [
+        pikepdf.ContentStreamInstruction((), pikepdf.Operator('BT')),
+        pikepdf.ContentStreamInstruction(
+            (pikepdf.Name('/F0'), 12), pikepdf.Operator('Tf')
+        ),
+        pikepdf.ContentStreamInstruction((288, 720), pikepdf.Operator('Td')),
+        pikepdf.ContentStreamInstruction(
+            (pikepdf.String('invisible'),), pikepdf.Operator('Tj')
+        ),
+        pikepdf.ContentStreamInstruction((), pikepdf.Operator('ET')),
+    ]
+    invisible_text_setting_tr = [
+        pikepdf.ContentStreamInstruction((), pikepdf.Operator('BT')),
+        pikepdf.ContentStreamInstruction([3], pikepdf.Operator('Tr')),
+        pikepdf.ContentStreamInstruction(
+            (pikepdf.Name('/F0'), 12), pikepdf.Operator('Tf')
+        ),
+        pikepdf.ContentStreamInstruction((288, 720), pikepdf.Operator('Td')),
+        pikepdf.ContentStreamInstruction(
+            (pikepdf.String('invisible'),), pikepdf.Operator('Tj')
+        ),
+        pikepdf.ContentStreamInstruction((), pikepdf.Operator('ET')),
+    ]
+    stream = [
+        pikepdf.ContentStreamInstruction([], pikepdf.Operator('q')),
+        pikepdf.ContentStreamInstruction([3], pikepdf.Operator('Tr')),
+        *invisible_text,
+        pikepdf.ContentStreamInstruction([], pikepdf.Operator('Q')),
+        *visible_text,
+        *invisible_text_setting_tr,
+        *invisible_text,
+    ]
+    content_stream = pikepdf.unparse_content_stream(stream)
+    page.Contents = pikepdf.Stream(pdf, content_stream)
+
+    def count(string, page):
+        return len(
+            [
+                True
+                for operands, operator in pikepdf.parse_content_stream(page)
+                if operator == pikepdf.Operator('Tj')
+                and operands[0] == pikepdf.String(string)
+            ]
+        )
+
+    nr_visible_pre = count('visible', page)
+    ocrmypdf._graft.strip_invisible_text(pdf, page)
+    nr_visible_post = count('visible', page)
+    assert (
+        nr_visible_pre == nr_visible_post
+    ), 'Number of visible text elements did not change'
+    assert count('invisible', page) == 0, 'No invisible elems left'

tests/test_page_reducing.py

@@ -1,2 +0,0 @@
-import hypothesis
-import pytest