v16.3.1 release notes

Fixes #1316

.github/ISSUE_TEMPLATE/1-bug-report-general.yml

@@ -24,7 +24,7 @@ body:
   - type: dropdown
     id: packaging-system
     attributes:
-      label: Where are you installing from?
+      label: Where are you installing/running from?
       multiple: true
       options:
         - PyPI (pip, poetry, pipx, etc.)
@@ -37,6 +37,11 @@ body:
         - source build
     validations:
       required: true
+  - type: input
+    id: version
+    attributes:
+      label: OCRmyPDF version
+      description: Paste "ocrmypdf --version" here
   - type: dropdown
     id: operating-system
     attributes:
@@ -47,6 +52,17 @@ body:
         - Windows
         - macOS
         - BSD
+  - type: input
+    id: os_version
+    attributes:
+      label: Operating system details and version
+  - type: checkboxes
+    attributes:
+      label: Simple sanity checks
+      description: Select all that apply
+      options:
+        - label: Operating system is currently supported by its vendor (not end of life)
+        - label: Python version is compatible with OCRmyPDF
   - type: textarea
     id: logs
     attributes:

.github/workflows/build.yml

@@ -101,16 +101,19 @@ jobs:
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:
           files: ./coverage.xml
           env_vars: OS,PYTHON
 
+
   test_macos:
     name: Test macOS
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [macos-latest]
+        os: [macos-latest, macos-13]  # macos-latest is arm64, macos-13 is x86_64
         python: ["3.10", "3.11", "3.12"]
 
     env:
@@ -131,7 +134,6 @@ jobs:
             ghostscript \
             jbig2enc \
             openjpeg \
-            openssl \
             pngquant \
             tesseract
 
@@ -159,6 +161,8 @@ jobs:
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:
           files: ./coverage.xml
           env_vars: OS,PYTHON
@@ -202,6 +206,8 @@ jobs:
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:
           files: ./coverage.xml
           env_vars: OS,PYTHON

docs/languages.rst

@@ -68,6 +68,22 @@ to what languages it should search for. Multiple languages can be
 requested using either ``-l eng+fra`` (English and French) or
 ``-l eng -l fra``.
 
+Archlinux
+------
+
+.. code-block:: bash
+
+   # Display a list of all Tesseract language packs
+   pacman -Ss tesseract-data
+
+   # Install German language pack
+   pacman -S tesseract-data-deu
+
+You can then pass the ``-l LANG`` argument to OCRmyPDF to give a hint as
+to what languages it should search for. Multiple languages can be
+requested using either ``-l eng+fra`` (English and French) or
+``-l eng -l fra``.
+
 Gentoo
 ------
 
@@ -122,4 +138,4 @@ Custom language packs
 If you have fine-tuned or trained Tesseract and generated custom trained data, you can
 copy your ``customlang.traineddata`` file into your Tesseract "tessdata" folder, and
 then use the ``-l customlang`` argument to tell OCRmyPDF to pass that language on to
-Tesseract.
+Tesseract.

docs/release_notes.rst

@@ -30,6 +30,21 @@ OCRmyPDF typically supports the three most recent Python versions.
 
 .. |OCRmyPDF PyPI| image:: https://img.shields.io/pypi/v/ocrmypdf.svg
 
+v16.3.1
+=======
+
+-  Fixed a test suite failure with Ghostscript 10.03.0+. :issue:`1316`
+-  Fixed an issue with the presentation of the "OCR" progress bar. :issue:`1313`
+
+v16.3.0
+=======
+
+-  Fixed progress bar not displaying for Ghostscript PDF/A conversion. :issue:`1313`
+-  Added progress bar for linearization. :issue:`1313`
+-  If `--rotate-pages-threshold` issued without `--rotate-pages` we now exit with
+   an error since the user likely intended to use `--rotate-pages`. :issue:`1309`
+-  If Tesseract hOCR gives an invalid line box, print an error message instead of
+   exiting with an error. :issue:`1312`
 
 v16.2.0
 =======

src/ocrmypdf/_defaults.py

@@ -0,0 +1,8 @@
+# SPDX-FileCopyrightText: 2024 James R. Barlow
+# SPDX-License-Identifier: MPL-2.0
+
+# Enforce English hegemony
+DEFAULT_LANGUAGE = 'eng'
+
+# Default rotation threshold
+DEFAULT_ROTATE_PAGES_THRESHOLD = 14.0

src/ocrmypdf/_exec/ghostscript.py

@@ -177,6 +177,17 @@ class GhostscriptFollower:
         self.progressbar_class = progressbar_class
         self.progressbar = None
 
+    def __enter__(self):
+        # We can't actually set up the progressbar here, because we don't know
+        # how many pages there are until the first __call__() happens. So we
+        # do it in __call__().
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        if self.progressbar:
+            return self.progressbar.__exit__(exc_type, exc_value, traceback)
+        return False
+
     def __call__(self, line):
         if not self.progressbar_class:
             return
@@ -187,7 +198,8 @@ class GhostscriptFollower:
                 self.progressbar = self.progressbar_class(
                     total=self.count, desc="PDF/A conversion", unit='page'
                 )
-                return
+                # Now that we know the count, we can set up the progressbar.
+                self.progressbar.__enter__()
         else:
             if self.re_page.match(line.strip()):
                 self.progressbar.update()
@@ -265,7 +277,10 @@ def generate_pdfa(
     )
     args_gs.extend(fspath(s) for s in pdf_pages)  # Stringify Path objs
     try:
-        with Path(output_file).open('wb') as output:
+        with (
+            Path(output_file).open('wb') as output,
+            GhostscriptFollower(progressbar_class) as pbar,
+        ):
             p = run_polling_stderr(
                 args_gs,
                 stdout=output,
@@ -274,7 +289,7 @@ def generate_pdfa(
                 text=True,
                 encoding='utf-8',
                 errors='replace',
-                callback=GhostscriptFollower(progressbar_class),
+                callback=pbar,
             )
     except CalledProcessError as e:
         # Ghostscript does not change return code when it fails to create

src/ocrmypdf/_metadata.py

@@ -153,18 +153,45 @@ def _set_language(pdf: Pdf, languages: list[str]):
     pdf.Root.Lang = iso639_2
 
 
+class MetadataProgress:
+    def __init__(self, progressbar_class):
+        self.progressbar_class = progressbar_class
+        self.progressbar = self.progressbar_class(
+            total=100, desc="Linearizing", unit='%'
+        )
+
+    def __enter__(self):
+        self.progressbar.__enter__()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        return self.progressbar.__exit__(exc_type, exc_value, traceback)
+
+    def __call__(self, percent: int):
+        if not self.progressbar_class:
+            return
+        self.progressbar.update(completed=percent)
+
+
 def metadata_fixup(
     working_file: Path, context: PdfContext, pdf_save_settings: dict[str, Any]
 ) -> Path:
-    """Fix certain metadata fields after Ghostscript PDF/A conversion.
+    """Fix certain metadata fields whether PDF or PDF/A.
+
+    Override some of Ghostscript's metadata choices.
 
     Also report on metadata in the input file that was not retained during
-    PDF/A conversion.
+    conversion.
     """
     output_file = context.get_path('metafix.pdf')
     options = context.options
 
-    with Pdf.open(context.origin) as original, Pdf.open(working_file) as pdf:
+    pbar_class = context.plugin_manager.hook.get_progressbar_class()
+    with (
+        Pdf.open(context.origin) as original,
+        Pdf.open(working_file) as pdf,
+        MetadataProgress(pbar_class) as pbar,
+    ):
         docinfo = get_docinfo(original, context)
         with (
             original.open_metadata(
@@ -182,6 +209,6 @@ def metadata_fixup(
             report_on_metadata(options, meta_missing)
 
         _set_language(pdf, options.languages)
-        pdf.save(output_file, **pdf_save_settings)
+        pdf.save(output_file, progress=pbar, **pdf_save_settings)
 
     return output_file

src/ocrmypdf/_pipelines/ocr.py

@@ -103,14 +103,14 @@ def exec_concurrent(context: PdfContext, executor: Executor) -> Sequence[str]:
         try:
             set_thread_pageno(result.pageno + 1)
             sidecars[result.pageno] = result.text
-            pbar.update()
+            pbar.update(0.5)
             ocrgraft.graft_page(
                 pageno=result.pageno,
                 image=result.pdf_page_from_image,
                 textpdf=result.ocr,
                 autorotate_correction=result.orientation_correction,
             )
-            pbar.update()
+            pbar.update(0.5)
         finally:
             set_thread_pageno(None)
 
@@ -118,10 +118,9 @@ def exec_concurrent(context: PdfContext, executor: Executor) -> Sequence[str]:
         use_threads=options.use_threads,
         max_workers=max_workers,
         progress_kwargs=dict(
-            total=(2 * len(context.pdfinfo)),
+            total=len(context.pdfinfo),
             desc='OCR' if options.tesseract_timeout > 0 else 'Image processing',
             unit='page',
-            unit_scale=0.5,
             disable=not options.progress_bar,
         ),
         worker_initializer=partial(worker_init, PIL.Image.MAX_IMAGE_PIXELS),

src/ocrmypdf/_progressbar.py

@@ -66,7 +66,7 @@ class ProgressBar(Protocol):
     def __exit__(self, *args):
         """Exit a progress bar context."""
 
-    def update(self, n=1):
+    def update(self, n=1, *, completed=None):
         """Update the progress bar by an increment.
 
         For use within a progress bar context.
@@ -85,7 +85,7 @@ class NullProgressBar:
     def __exit__(self, exc_type, exc_value, traceback):
         return False
 
-    def update(self, _arg=None):
+    def update(self, _arg=None, *, completed=None):
         return
 
 
@@ -103,6 +103,7 @@ class RichProgressBar:
         disable: bool = False,
         **kwargs,
     ):
+        self._entered = False
         self.progress = Progress(
             TextColumn(
                 "[progress.description]{task.description}",
@@ -130,6 +131,7 @@ class RichProgressBar:
 
     def __enter__(self):
         self.progress.start()
+        self._entered = True
         return self
 
     def __exit__(self, exc_type, exc_value, traceback):
@@ -137,6 +139,10 @@ class RichProgressBar:
         self.progress.stop()
         return False
 
-    def update(self, value=None):
-        advance = self.unit_scale if value is None else value
-        self.progress.update(self.progress_bar, advance=advance)
+    def update(self, n=1, *, completed=None):
+        assert self._entered, "Progress bar must be entered before updating"
+        if completed is None:
+            advance = self.unit_scale if n is None else n
+            self.progress.update(self.progress_bar, advance=advance)
+        else:
+            self.progress.update(self.progress_bar, completed=completed)

src/ocrmypdf/_validation.py

@@ -20,6 +20,7 @@ import pikepdf
 import PIL
 from pluggy import PluginManager
 
+from ocrmypdf._defaults import DEFAULT_LANGUAGE, DEFAULT_ROTATE_PAGES_THRESHOLD
 from ocrmypdf._exec import unpaper
 from ocrmypdf.exceptions import (
     BadArgsError,
@@ -30,17 +31,9 @@ from ocrmypdf.exceptions import (
 from ocrmypdf.helpers import is_file_writable, monotonic, safe_symlink
 from ocrmypdf.subprocess import check_external_program
 
-# -------------
-# External dependencies
-
-DEFAULT_LANGUAGE = 'eng'  # Enforce English hegemony
-
 log = logging.getLogger(__name__)
 
 
-# --------
-
-
 def check_platform() -> None:
     if sys.maxsize <= 2**32:  # pragma: no cover
         log.warning(
@@ -130,6 +123,11 @@ def check_options_preprocessing(options: Namespace) -> None:
         options.clean = True
     if options.unpaper_args and not options.clean:
         raise BadArgsError("--clean is required for --unpaper-args")
+    if (
+        options.rotate_pages_threshold != DEFAULT_ROTATE_PAGES_THRESHOLD
+        and not options.rotate_pages
+    ):
+        raise BadArgsError("--rotate-pages is required for --rotate-pages-threshold")
     if options.clean:
         check_external_program(
             program='unpaper',

src/ocrmypdf/cli.py

@@ -9,6 +9,7 @@ import argparse
 from collections.abc import Callable, Mapping
 from typing import Any, TypeVar
 
+from ocrmypdf._defaults import DEFAULT_ROTATE_PAGES_THRESHOLD
 from ocrmypdf._version import PROGRAM_NAME as _PROGRAM_NAME
 from ocrmypdf._version import __version__ as _VERSION
 
@@ -403,7 +404,7 @@ Online documentation is located at:
     )
     advanced.add_argument(
         '--rotate-pages-threshold',
-        default=14.0,
+        default=DEFAULT_ROTATE_PAGES_THRESHOLD,
         type=numeric(float, 0, 1000),
         metavar='CONFIDENCE',
         help="Only rotate pages when confidence is above this value (arbitrary "

src/ocrmypdf/hocrtransform/_hocr.py

@@ -117,15 +117,12 @@ class HocrTransform:
             # Stop after first div that has page coordinates
             break
 
-    def _get_element_text(self, element: Element):
+    def _get_element_text(self, element: Element) -> str:
         """Return the textual content of the element and its children."""
-        text = ''
-        if element.text is not None:
-            text += element.text
+        text = element.text if element.text is not None else ''
         for child in element:
             text += self._get_element_text(child)
-        if element.tail is not None:
-            text += element.tail
+        text += element.tail if element.tail is not None else ''
         return text
 
     @classmethod
@@ -286,7 +283,10 @@ class HocrTransform:
         line_box = self.element_coordinates(line)
         if not line_box:
             return
-        assert line_box.ury > line_box.lly  # lly is top, ury is bottom
+        if line_box.ury <= line_box.lly:
+            log.error("line box is invalid so we cannot render it: box=%s text=%s",
+                      line_box, self._get_element_text(line))
+            return
 
         self._debug_draw_line_bbox(canvas, line_box)
 

src/ocrmypdf/pdfinfo/info.py

@@ -239,7 +239,13 @@ def _interpret_contents(contentstream: Object, initial_shorthand=UNIT_SQUARE):
                 # to do. Just pretend nothing happened, keep calm and carry on.
                 warn("PDF graphics stack underflowed - PDF may be malformed")
         elif operator == 'cm':
-            ctm = Matrix(operands) @ ctm
+            try:
+                ctm = Matrix(operands) @ ctm
+            except ValueError:
+                raise InputFileError(
+                    "PDF content stream is corrupt - this PDF is malformed. "
+                    "Use a PDF editor that is capable of visually inspecting the PDF."
+                )
         elif operator == 'Do':
             image_name = operands[0]
             settings = XobjectSettings(

tests/test_semfree.py

@@ -23,4 +23,4 @@ def test_semfree(resources, outpdf):
         '--plugin',
         'tests/plugins/tesseract_noop.py',
     )
-    assert exitcode == ExitCode.ok
+    assert exitcode in (ExitCode.ok, ExitCode.pdfa_conversion_failed)