v16.4.2 release notes

Ghostscript now expects the pdfa.ps file to precede other files. Fixes #1359.

.docker/Dockerfile.alpine

@@ -1,7 +1,7 @@
 # SPDX-FileCopyrightText: 2023 James R. Barlow
 # SPDX-License-Identifier: MPL-2.0
 
-FROM alpine:3.19 as base
+FROM alpine:3.19.1 as base
 
 ENV LANG=C.UTF-8
 ENV TZ=UTC

.github/ISSUE_TEMPLATE/1-bug-report-general.yml

@@ -1,7 +1,7 @@
 name: Installation, packaging, dependencies
 description: Installation, packages, dependencies, "nothing works", test suite failures...
 title: "[Bug]: "
-labels: ["bug", "triage"]
+labels: ["triage"]
 assignees:
   - jbarlow83
 body:
@@ -24,7 +24,7 @@ body:
   - type: dropdown
     id: packaging-system
     attributes:
-      label: Where are you installing from?
+      label: Where are you installing/running from?
       multiple: true
       options:
         - PyPI (pip, poetry, pipx, etc.)
@@ -37,6 +37,11 @@ body:
         - source build
     validations:
       required: true
+  - type: input
+    id: version
+    attributes:
+      label: OCRmyPDF version
+      description: Paste "ocrmypdf --version" here
   - type: dropdown
     id: operating-system
     attributes:
@@ -47,6 +52,18 @@ body:
         - Windows
         - macOS
         - BSD
+  - type: input
+    id: os_version
+    attributes:
+      label: Operating system details and version
+  - type: checkboxes
+    attributes:
+      label: Simple sanity checks
+      description: Select all that apply
+      options:
+        - label: Operating system is currently supported by its vendor (not end of life)
+        - label: Python version is compatible with OCRmyPDF
+        - label: This issue is not about a specific input file
   - type: textarea
     id: logs
     attributes:

.github/ISSUE_TEMPLATE/2-problem-with-specific-file.yml

@@ -1,7 +1,7 @@
 name: Problem with specific file
 description: Something went wrong while trying to OCR a specific file
 title: "[Bug]: "
-labels: ["bug", "triage"]
+labels: ["triage"]
 assignees:
   - jbarlow83
 body:
@@ -39,7 +39,7 @@ body:
         causing the issue. There's really no substitute for a test file.
 
         We understand files may contain personal or sensitive information. Here are some options:
-        - Try reproducing the issue with a file from the test suite. (See tests/resources)
+        - Try reproducing the issue with a file from the OCRmyPDF test suite. (See tests/resources)
         - Try to create another file in the same way as your private file.
         - Encrypt the file to OCRmyPDF's private GPG key, and then zip the GPG file.
         - Use ``qpdf --json yourfile.pdf`` to produce a JSON representation of your file that

.github/ISSUE_TEMPLATE/3-app.yml

@@ -0,0 +1,83 @@
+name: Problem with third party app that uses OCRmyPDF
+description: |
+  For PDF generation issues with third party software such as Paperless-ngx that
+  uses OCRmyPDF to perform OCR or generate PDFs.
+title: "[3rdparty]: "
+labels: ["triage"]
+assignees:
+  - jbarlow83
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to describe this issue with a particular file
+        and third party app.
+
+        If you are comfortable using OCRmyPDF, please trying to install OCRmyPDF,
+        run it on your file, and see if it works. It's easier for everyone
+        if you can confirm that the issue occurs with OCRmyPDF and not with
+        the third party app.
+  - type: checkboxes
+    attributes:
+      label: Simple sanity checks
+      description: Select all that apply
+      options:
+        - label: This is an issue with an app that uses OCRmyPDF for OCR
+        - label: I am using a recent version of the third party app
+        - label: I will include a file that reproduces the issuse
+  - type: input
+    id: thirdparty-app-name-version
+    attributes:
+      label: Third party app name and version
+      description: e.g. Paperless-ngx 2.9.0
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: Describe the bug
+      description: A clear and concise description of what the bug is.
+      placeholder: Tell us what you see!
+    validations:
+      required: true
+  - type: textarea
+    id: reproduce
+    attributes:
+      label: Steps to reproduce
+      description: Please include steps to reproduce.
+      value: |
+        1. Import attached file into Paperless-ngx
+        2. Trigger OCR
+        3. Check log file
+        4. ...
+      render: plain text
+  - type: textarea
+    id: files
+    attributes:
+      label: Files
+      description: |
+        Please attach the input and output files, or any screenshots that may be helpful.
+
+        If you cannot provide a test file, we probably won't be able to help with the issue.
+        PDF is a complex file format, and there may be technical details in the PDF that are
+        causing the issue. There's really no substitute for a test file.
+
+        We understand files may contain personal or sensitive information. Here are some options:
+        - Try reproducing the issue with a file from the test suite. (See tests/resources)
+        - Try to create another file in the same way as your private file.
+        - Encrypt the file to OCRmyPDF's private GPG key, and then zip the GPG file.
+        - Use ``qpdf --json yourfile.pdf`` to produce a JSON representation of your file that
+          omits personal information.
+      placeholder: |
+        Drag and drop files here.
+  - type: input
+    id: version
+    attributes:
+      label: OCRmyPDF version
+      description: Paste "ocrmypdf --version" here
+      placeholder: ocrmypdf --version
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant log output
+      description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
+      placeholder: Run OCRmyPDF with verbosity `-v1` to get more detailed logging output.
+      render: plain text

.github/workflows/build.yml

@@ -61,6 +61,7 @@ jobs:
           sudo apt-get install -y --no-install-recommends \
             curl \
             ghostscript \
+            jbig2dec \
             img2pdf \
             libexempi8 \
             libffi-dev \
@@ -101,16 +102,19 @@ jobs:
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:
           files: ./coverage.xml
           env_vars: OS,PYTHON
 
+
   test_macos:
     name: Test macOS
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [macos-latest]
+        os: [macos-latest, macos-13]  # macos-latest is arm64, macos-13 is x86_64
         python: ["3.10", "3.11", "3.12"]
 
     env:
@@ -131,7 +135,6 @@ jobs:
             ghostscript \
             jbig2enc \
             openjpeg \
-            openssl \
             pngquant \
             tesseract
 
@@ -159,6 +162,8 @@ jobs:
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:
           files: ./coverage.xml
           env_vars: OS,PYTHON
@@ -202,6 +207,8 @@ jobs:
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:
           files: ./coverage.xml
           env_vars: OS,PYTHON
@@ -352,9 +359,6 @@ jobs:
           username: jbarlow83
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
       - name: Set up Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v3
@@ -366,6 +370,6 @@ jobs:
         run: |
           docker buildx build \
             --push \
-            --platform linux/amd64  \
+            --platform linux/amd64,linux/arm64  \
             --tag "${DOCKER_REPOSITORY}/${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}" \
             --file .docker/Dockerfile.alpine .

docs/advanced.rst

@@ -228,6 +228,59 @@ then run ocrmypdf as follows (along with any other desired arguments):
    Some combinations of control parameters will break Tesseract or break
    assumptions that OCRmyPDF makes about Tesseract's output.
 
+Changing page segmentation mode
+-------------------------------
+
+The directive ``--tesseract-pagesegmode Nmode`` forwards the desired page segmentation
+mode to Tesseract OCR. The default is 3.
+
+Page segmentation can improve OCR results when you know that a PDF ought to be
+analyzed a particular way, such as PDFs whose pages contain only a single line of 
+text. For the vast majority of users, changing the page segmentation mode will only
+make things worse.
+
+As of June 2024, the Tesseract page segmentation modes are:
+
++-----+----------------------------------------------------------------------------------+
+| ID  | Description                                                                      |
++=====+==================================================================================+
+|  0  | Orientation and script detection (OSD) only.                                      |
++-----+----------------------------------------------------------------------------------+
+|  1  | Automatic page segmentation with OSD.                                             |
++-----+----------------------------------------------------------------------------------+
+|  2  | Automatic page segmentation, but no OSD, or OCR. (not implemented)               |
++-----+----------------------------------------------------------------------------------+
+|  3  | Fully automatic page segmentation, but no OSD. (Default)                         |
++-----+----------------------------------------------------------------------------------+
+|  4  | Assume a single column of text of variable sizes.                                 |
++-----+----------------------------------------------------------------------------------+
+|  5  | Assume a single uniform block of vertically aligned text.                         |
++-----+----------------------------------------------------------------------------------+
+|  6  | Assume a single uniform block of text.                                            |
++-----+----------------------------------------------------------------------------------+
+|  7  | Treat the image as a single text line.                                            |
++-----+----------------------------------------------------------------------------------+
+|  8  | Treat the image as a single word.                                                 |
++-----+----------------------------------------------------------------------------------+
+|  9  | Treat the image as a single word in a circle.                                     |
++-----+----------------------------------------------------------------------------------+
+| 10  | Treat the image as a single character.                                            |
++-----+----------------------------------------------------------------------------------+
+| 11  | Sparse text. Find as much text as possible in no particular order.                |
++-----+----------------------------------------------------------------------------------+
+| 12  | Sparse text with OSD.                                                            |
++-----+----------------------------------------------------------------------------------+
+| 13  | Raw line. Treat the image as a single text line, bypassing hacks that are        |
+|     | Tesseract-specific.                                                               |
++-----+----------------------------------------------------------------------------------+
+
+Modes 0, 1, 2, and 12 (all of those that enable orientation and script detection) 
+are not compatible with OCRmyPDF, which performs OSD in a separate step from OCR.
+Their use may interfere with ``--rotate-pages`` and other features.
+
+It is currently not possible to use advanced Tesseract OCR features, such as creating
+OCR information, when using Tesseract through OCRmyPDF.
+
 Changing the PDF renderer
 =========================
 

docs/docker.rst

@@ -42,7 +42,7 @@ execute the image:
      - Architecture
      - Description
    * - ``jbarlow83/ocrmypdf-alpine``
-     - x86_64 only
+     - x86_64 and arm64
      - Recommended image, based on Alpine Linux.
    * - ``jbarlow83/ocrmypdf-ubuntu``
      - x86_64 and arm64
@@ -81,7 +81,7 @@ To start a Docker container (instance of the image):
 
 .. code-block:: bash
 
-   docker tag jbarlow83/ocrmypdf ocrmypdf
+   docker tag jbarlow83/ocrmypdf-alpine ocrmypdf
    docker run --rm -i ocrmypdf (... all other arguments here...) - -
 
 For convenience, create a shell alias to hide the Docker command. It is

docs/installation.rst

@@ -224,7 +224,7 @@ standard tooling needed to build packages, such as a compiler and binary tools.
 
 .. code-block:: bash
 
-   sudo pacman -S base-devel
+   sudo pacman -S --needed base-devel
 
 Now you are ready to install the OCRmyPDF package.
 
@@ -681,7 +681,7 @@ To install all of the development and test requirements:
 .. code-block:: bash
 
     git clone -b main https://github.com/ocrmypdf/OCRmyPDF.git
-    python -m .venv
+    python -m venv .venv
     source .venv/bin/activate
     cd OCRmyPDF
     pip install -e .[test]

docs/languages.rst

@@ -68,6 +68,22 @@ to what languages it should search for. Multiple languages can be
 requested using either ``-l eng+fra`` (English and French) or
 ``-l eng -l fra``.
 
+Archlinux
+------
+
+.. code-block:: bash
+
+   # Display a list of all Tesseract language packs
+   pacman -Ss tesseract-data
+
+   # Install German language pack
+   pacman -S tesseract-data-deu
+
+You can then pass the ``-l LANG`` argument to OCRmyPDF to give a hint as
+to what languages it should search for. Multiple languages can be
+requested using either ``-l eng+fra`` (English and French) or
+``-l eng -l fra``.
+
 Gentoo
 ------
 
@@ -122,4 +138,4 @@ Custom language packs
 If you have fine-tuned or trained Tesseract and generated custom trained data, you can
 copy your ``customlang.traineddata`` file into your Tesseract "tessdata" folder, and
 then use the ``-l customlang`` argument to tell OCRmyPDF to pass that language on to
-Tesseract.
+Tesseract.

docs/release_notes.rst

@@ -30,6 +30,60 @@ OCRmyPDF typically supports the three most recent Python versions.
 
 .. |OCRmyPDF PyPI| image:: https://img.shields.io/pypi/v/ocrmypdf.svg
 
+v16.4.2
+=======
+
+-  Fixed order of filenames passed to Ghostscript for PDF/A generation. :issue:`1359`
+-  Suppressed missing jbig2dec warning message. :issue:`1358`
+-  Fixed calculation of image size when soft mask dimensions don't match image
+   dimension. :issue:`1351`
+-  Several fixes to documentation. Thanks to users Iris and JoKalliauer
+   who contributed these changes.
+-  Fixed error on processing PDFs that are missing certain image metadata. :issue:`1315`
+
+v16.4.1
+=======
+
+-  Fixed calculation of image printed area (used in finding weighted DPI for OCR).
+   :issue:`1334`
+-  Fixed "NotImplementedError: not sure how to get colorspace" error
+   messages in logs which simply records a failure to optimize images with
+   print production colorspaces. :issue:`1315`
+
+v16.4.0
+=======
+
+-  Selecting the ``osd`` and ``equ`` pseudo-languages with ``-l/--language`` now
+   exits with an error when using Tesseract OCR, because these are not
+   regular Tesseract languages but implementation details implemented.
+   Using them can cause Tesseract to crash.
+-  The hOCR renderer is more tolerant of extra whitespace in input files.
+-  watcher.py now changes the output file extension to .pdf when the input is not
+   .pdf.
+-  Improved handling of PDFs that contain circularly referenced Form XObjects.
+   :issue:`1321`
+-  Fixed Alpine Docker image for ARM64, which was not building correctly.
+-  Docker images now use pikepdf 9.0.0.
+-  Prevent use of Tesseract OCR 5.4.0, a version with known regressions.
+-  Disabled progressbar for "Linearizing" when ``--no-progress-bar`` set.
+-  Fixed some tests that warn about missing JBIG2 decoding via pikepdf, by
+   installing the necessary libraries during tests.
+
+v16.3.1
+=======
+
+-  Fixed a test suite failure with Ghostscript 10.03.0+. :issue:`1316`
+-  Fixed an issue with the presentation of the "OCR" progress bar. :issue:`1313`
+
+v16.3.0
+=======
+
+-  Fixed progress bar not displaying for Ghostscript PDF/A conversion. :issue:`1313`
+-  Added progress bar for linearization. :issue:`1313`
+-  If `--rotate-pages-threshold` issued without `--rotate-pages` we now exit with
+   an error since the user likely intended to use `--rotate-pages`. :issue:`1309`
+-  If Tesseract hOCR gives an invalid line box, print an error message instead of
+   exiting with an error. :issue:`1312`
 
 v16.2.0
 =======

misc/batch.py

@@ -14,12 +14,12 @@ You should edit this script to meet your needs.
 
 from __future__ import annotations
 
+import filecmp
 import logging
-import sys
 import os
 import posixpath
 import shutil
-import filecmp
+import sys
 from pathlib import Path
 
 import ocrmypdf
@@ -70,7 +70,7 @@ for filename in start_dir.glob("**/*.pdf"):
             logging.info(f"Archiving document to {archive_filename}")
             try:
                 shutil.copy2(filename, posixpath.dirname(archive_filename))
-            except IOError as io_err:
+            except OSError:
                 os.makedirs(posixpath.dirname(archive_filename))
                 shutil.copy2(filename, posixpath.dirname(archive_filename))
         try:
@@ -86,6 +86,6 @@ for filename in start_dir.glob("**/*.pdf"):
             logging.info(
                 "Skipped document because it does not need ocr as it is tagged"
             )
-        except:
+        except Exception:
             logging.error("Unhandled error occured")
         logging.info("OCR complete")

misc/watcher.py

@@ -46,15 +46,18 @@ class LoggingLevelEnum(str, Enum):
     CRITICAL = "CRITICAL"
 
 
-def get_output_dir(root: Path, basename: str, output_dir_year_month: bool) -> Path:
+def get_output_path(root: Path, basename: str, output_dir_year_month: bool) -> Path:
+    assert '/' not in basename, "basename must not contain '/'"
     if output_dir_year_month:
         today = datetime.today()
         output_directory_year_month = root / str(today.year) / f'{today.month:02d}'
         if not output_directory_year_month.exists():
             output_directory_year_month.mkdir(parents=True, exist_ok=True)
-        output_path = Path(output_directory_year_month) / basename
+        output_path = Path(output_directory_year_month) / Path(basename).with_suffix(
+            '.pdf'
+        )
     else:
-        output_path = root / basename
+        output_path = root / Path(basename).with_suffix('.pdf')
     return output_path
 
 
@@ -98,7 +101,7 @@ def execute_ocrmypdf(
     retries_loading_file: int,
     output_dir_year_month: bool,
 ):
-    output_path = get_output_dir(output_dir, file_path.name, output_dir_year_month)
+    output_path = get_output_path(output_dir, file_path.name, output_dir_year_month)
 
     log.info("-" * 20)
     log.info(f'New file: {file_path}. Waiting until fully written...')

snapcraft.yaml

@@ -18,8 +18,8 @@ architectures: [amd64]
 
 environment:
   TESSDATA_PREFIX: $SNAP/usr/share/tesseract-ocr/4.00/tessdata
-  GS_LIB: $SNAP/usr/share/ghostscript/9.55/Resource/Init
-  GS_FONTPATH: $SNAP/usr/share/ghostscript/9.55/Resource/Font
+  GS_LIB: $SNAP/usr/share/ghostscript/9.55.0/Resource/Init
+  GS_FONTPATH: $SNAP/usr/share/ghostscript/9.55.0/Resource/Font
   LD_LIBRARY_PATH: $SNAP/usr/lib/x86_64-linux-gnu
 
 apps:

src/ocrmypdf/_defaults.py

@@ -0,0 +1,8 @@
+# SPDX-FileCopyrightText: 2024 James R. Barlow
+# SPDX-License-Identifier: MPL-2.0
+
+# Enforce English hegemony
+DEFAULT_LANGUAGE = 'eng'
+
+# Default rotation threshold
+DEFAULT_ROTATE_PAGES_THRESHOLD = 14.0

src/ocrmypdf/_exec/ghostscript.py

@@ -177,6 +177,17 @@ class GhostscriptFollower:
         self.progressbar_class = progressbar_class
         self.progressbar = None
 
+    def __enter__(self):
+        # We can't actually set up the progressbar here, because we don't know
+        # how many pages there are until the first __call__() happens. So we
+        # do it in __call__().
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        if self.progressbar:
+            return self.progressbar.__exit__(exc_type, exc_value, traceback)
+        return False
+
     def __call__(self, line):
         if not self.progressbar_class:
             return
@@ -187,7 +198,8 @@ class GhostscriptFollower:
                 self.progressbar = self.progressbar_class(
                     total=self.count, desc="PDF/A conversion", unit='page'
                 )
-                return
+                # Now that we know the count, we can set up the progressbar.
+                self.progressbar.__enter__()
         else:
             if self.re_page.match(line.strip()):
                 self.progressbar.update()
@@ -265,7 +277,10 @@ def generate_pdfa(
     )
     args_gs.extend(fspath(s) for s in pdf_pages)  # Stringify Path objs
     try:
-        with Path(output_file).open('wb') as output:
+        with (
+            Path(output_file).open('wb') as output,
+            GhostscriptFollower(progressbar_class) as pbar,
+        ):
             p = run_polling_stderr(
                 args_gs,
                 stdout=output,
@@ -274,7 +289,7 @@ def generate_pdfa(
                 text=True,
                 encoding='utf-8',
                 errors='replace',
-                callback=GhostscriptFollower(progressbar_class),
+                callback=pbar,
             )
     except CalledProcessError as e:
         # Ghostscript does not change return code when it fails to create

src/ocrmypdf/_metadata.py

@@ -153,18 +153,45 @@ def _set_language(pdf: Pdf, languages: list[str]):
     pdf.Root.Lang = iso639_2
 
 
+class MetadataProgress:
+    def __init__(self, progressbar_class, enable: bool = True):
+        self.progressbar_class = progressbar_class
+        self.progressbar = self.progressbar_class(
+            total=100, desc="Linearizing", unit='%', disable=not enable
+        )
+
+    def __enter__(self):
+        self.progressbar.__enter__()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        return self.progressbar.__exit__(exc_type, exc_value, traceback)
+
+    def __call__(self, percent: int):
+        if not self.progressbar_class:
+            return
+        self.progressbar.update(completed=percent)
+
+
 def metadata_fixup(
     working_file: Path, context: PdfContext, pdf_save_settings: dict[str, Any]
 ) -> Path:
-    """Fix certain metadata fields after Ghostscript PDF/A conversion.
+    """Fix certain metadata fields whether PDF or PDF/A.
+
+    Override some of Ghostscript's metadata choices.
 
     Also report on metadata in the input file that was not retained during
-    PDF/A conversion.
+    conversion.
     """
     output_file = context.get_path('metafix.pdf')
     options = context.options
 
-    with Pdf.open(context.origin) as original, Pdf.open(working_file) as pdf:
+    pbar_class = context.plugin_manager.hook.get_progressbar_class()
+    with (
+        Pdf.open(context.origin) as original,
+        Pdf.open(working_file) as pdf,
+        MetadataProgress(pbar_class, options.progress_bar) as pbar,
+    ):
         docinfo = get_docinfo(original, context)
         with (
             original.open_metadata(
@@ -182,6 +209,6 @@ def metadata_fixup(
             report_on_metadata(options, meta_missing)
 
         _set_language(pdf, options.languages)
-        pdf.save(output_file, **pdf_save_settings)
+        pdf.save(output_file, progress=pbar, **pdf_save_settings)
 
     return output_file

src/ocrmypdf/_pipeline.py

@@ -475,7 +475,7 @@ def calculate_raster_dpi(page_context: PageContext):
     page_dpi = get_page_square_dpi(page_context, image_dpi)
     if dpi_profile and dpi_profile.average_to_max_dpi_ratio < 0.8:
         log.warning(
-            "Weight average image DPI is %0.1f, max DPI is %0.1f. "
+            "Weighted average image DPI is %0.1f, max DPI is %0.1f. "
             "The discrepancy may indicate a high detail region on this page, "
             "but could also indicate a problem with the input PDF file. "
             "Page image will be rendered at %0.1f DPI.",

src/ocrmypdf/_pipelines/ocr.py

@@ -103,14 +103,14 @@ def exec_concurrent(context: PdfContext, executor: Executor) -> Sequence[str]:
         try:
             set_thread_pageno(result.pageno + 1)
             sidecars[result.pageno] = result.text
-            pbar.update()
+            pbar.update(0.5)
             ocrgraft.graft_page(
                 pageno=result.pageno,
                 image=result.pdf_page_from_image,
                 textpdf=result.ocr,
                 autorotate_correction=result.orientation_correction,
             )
-            pbar.update()
+            pbar.update(0.5)
         finally:
             set_thread_pageno(None)
 
@@ -118,10 +118,9 @@ def exec_concurrent(context: PdfContext, executor: Executor) -> Sequence[str]:
         use_threads=options.use_threads,
         max_workers=max_workers,
         progress_kwargs=dict(
-            total=(2 * len(context.pdfinfo)),
+            total=len(context.pdfinfo),
             desc='OCR' if options.tesseract_timeout > 0 else 'Image processing',
             unit='page',
-            unit_scale=0.5,
             disable=not options.progress_bar,
         ),
         worker_initializer=partial(worker_init, PIL.Image.MAX_IMAGE_PIXELS),

src/ocrmypdf/_progressbar.py

@@ -66,7 +66,7 @@ class ProgressBar(Protocol):
     def __exit__(self, *args):
         """Exit a progress bar context."""
 
-    def update(self, n=1):
+    def update(self, n=1, *, completed=None):
         """Update the progress bar by an increment.
 
         For use within a progress bar context.
@@ -85,7 +85,7 @@ class NullProgressBar:
     def __exit__(self, exc_type, exc_value, traceback):
         return False
 
-    def update(self, _arg=None):
+    def update(self, _arg=None, *, completed=None):
         return
 
 
@@ -103,6 +103,7 @@ class RichProgressBar:
         disable: bool = False,
         **kwargs,
     ):
+        self._entered = False
         self.progress = Progress(
             TextColumn(
                 "[progress.description]{task.description}",
@@ -130,6 +131,7 @@ class RichProgressBar:
 
     def __enter__(self):
         self.progress.start()
+        self._entered = True
         return self
 
     def __exit__(self, exc_type, exc_value, traceback):
@@ -137,6 +139,10 @@ class RichProgressBar:
         self.progress.stop()
         return False
 
-    def update(self, value=None):
-        advance = self.unit_scale if value is None else value
-        self.progress.update(self.progress_bar, advance=advance)
+    def update(self, n=1, *, completed=None):
+        assert self._entered, "Progress bar must be entered before updating"
+        if completed is None:
+            advance = self.unit_scale if n is None else n
+            self.progress.update(self.progress_bar, advance=advance)
+        else:
+            self.progress.update(self.progress_bar, completed=completed)

src/ocrmypdf/_validation.py

@@ -20,6 +20,7 @@ import pikepdf
 import PIL
 from pluggy import PluginManager
 
+from ocrmypdf._defaults import DEFAULT_LANGUAGE, DEFAULT_ROTATE_PAGES_THRESHOLD
 from ocrmypdf._exec import unpaper
 from ocrmypdf.exceptions import (
     BadArgsError,
@@ -30,17 +31,9 @@ from ocrmypdf.exceptions import (
 from ocrmypdf.helpers import is_file_writable, monotonic, safe_symlink
 from ocrmypdf.subprocess import check_external_program
 
-# -------------
-# External dependencies
-
-DEFAULT_LANGUAGE = 'eng'  # Enforce English hegemony
-
 log = logging.getLogger(__name__)
 
 
-# --------
-
-
 def check_platform() -> None:
     if sys.maxsize <= 2**32:  # pragma: no cover
         log.warning(
@@ -61,6 +54,7 @@ def check_options_languages(
             log.debug("No language specified; assuming --language %s", DEFAULT_LANGUAGE)
     if not ocr_engine_languages:
         return
+
     missing_languages = set(options.languages) - set(ocr_engine_languages)
     if missing_languages:
         lang_text = '\n'.join(lang for lang in missing_languages)
@@ -130,6 +124,11 @@ def check_options_preprocessing(options: Namespace) -> None:
         options.clean = True
     if options.unpaper_args and not options.clean:
         raise BadArgsError("--clean is required for --unpaper-args")
+    if (
+        options.rotate_pages_threshold != DEFAULT_ROTATE_PAGES_THRESHOLD
+        and not options.rotate_pages
+    ):
+        raise BadArgsError("--rotate-pages is required for --rotate-pages-threshold")
     if options.clean:
         check_external_program(
             program='unpaper',

src/ocrmypdf/builtin_plugins/ghostscript.py

@@ -129,7 +129,7 @@ def generate_pdfa(
 ):
     """Generate a PDF/A from the list of PDF pages and PDF/A metadata."""
     ghostscript.generate_pdfa(
-        pdf_pages=[*pdf_pages, pdfmark],
+        pdf_pages=[pdfmark, *pdf_pages],
         output_file=output_file,
         compression=context.options.pdfa_image_compression,
         color_conversion_strategy=context.options.color_conversion_strategy,

src/ocrmypdf/builtin_plugins/tesseract_ocr.py

@@ -14,6 +14,7 @@ from ocrmypdf import hookimpl
 from ocrmypdf._exec import tesseract
 from ocrmypdf._jobcontext import PageContext
 from ocrmypdf.cli import numeric, str_to_int
+from ocrmypdf.exceptions import BadArgsError, MissingDependencyError
 from ocrmypdf.helpers import clamp
 from ocrmypdf.imageops import calculate_downsample, downsample_image
 from ocrmypdf.pluginspec import OcrEngine
@@ -144,6 +145,12 @@ def check_options(options):
         need_version='4.1.1',  # Ubuntu 22.04 version (also 20.04)
         version_parser=tesseract.TesseractVersion,
     )
+    tess_version = tesseract.version()
+    if tess_version == tesseract.TesseractVersion('5.4.0'):
+        raise MissingDependencyError(
+            "Tesseract 5.4.0 is not supported due to regressions in this version. "
+            "Please upgrade to a newer or supported older version."
+        )
 
     # Decide on what renderer to use
     if options.pdf_renderer == 'auto':
@@ -164,6 +171,14 @@ def check_options(options):
             "The --tesseract-pagesegmode argument you select will disable OCR. "
             "This may cause processing to fail."
         )
+    DENIED_LANGUAGES = {'equ', 'osd'}
+    if DENIED_LANGUAGES & set(options.languages):
+        raise BadArgsError(
+            "The following languages for Tesseract's internal use and should not "
+            "be issued explicitly: "
+            f"{', '.join(DENIED_LANGUAGES & set(options.languages))}\n"
+            "Remove them from the -l/--language argument."
+        )
 
 
 @hookimpl

src/ocrmypdf/cli.py

@@ -9,6 +9,7 @@ import argparse
 from collections.abc import Callable, Mapping
 from typing import Any, TypeVar
 
+from ocrmypdf._defaults import DEFAULT_ROTATE_PAGES_THRESHOLD
 from ocrmypdf._version import PROGRAM_NAME as _PROGRAM_NAME
 from ocrmypdf._version import __version__ as _VERSION
 
@@ -403,7 +404,7 @@ Online documentation is located at:
     )
     advanced.add_argument(
         '--rotate-pages-threshold',
-        default=14.0,
+        default=DEFAULT_ROTATE_PAGES_THRESHOLD,
         type=numeric(float, 0, 1000),
         metavar='CONFIDENCE',
         help="Only rotate pages when confidence is above this value (arbitrary "

src/ocrmypdf/helpers.py

@@ -268,7 +268,9 @@ def check_pdf(input_file: Path) -> bool:
         return False
     else:
         with pdf:
-            messages = pdf.check()
+            with warnings.catch_warnings():
+                warnings.filterwarnings('ignore', message=r'pikepdf.*JBIG2.*')
+                messages = pdf.check()
             success = True
             for msg in messages:
                 if 'error' in msg.lower():

src/ocrmypdf/hocrtransform/_hocr.py

@@ -61,15 +61,25 @@ class HocrTransform:
     """A class for converting documents from the hOCR format.
 
     For details of the hOCR format, see:
-    http://kba.cloud/hocr-spec/.
+    http://kba.github.io/hocr-spec/1.2/.
     """
 
-    box_pattern = re.compile(r'bbox (\d+) (\d+) (\d+) (\d+)')
+    box_pattern = re.compile(
+        r'''
+        bbox \s+
+        (\d+) \s+   # left: uint
+        (\d+) \s+   # top: uint
+        (\d+) \s+   # right: uint
+        (\d+)       # bottom: uint
+        ''',
+        re.VERBOSE,
+    )
     baseline_pattern = re.compile(
         r'''
         baseline \s+
         ([\-\+]?\d*\.?\d*) \s+  # +/- decimal float
-        ([\-\+]?\d+)            # +/- int''',
+        ([\-\+]?\d+)            # +/- int
+        ''',
         re.VERBOSE,
     )
 
@@ -117,15 +127,12 @@ class HocrTransform:
             # Stop after first div that has page coordinates
             break
 
-    def _get_element_text(self, element: Element):
+    def _get_element_text(self, element: Element) -> str:
         """Return the textual content of the element and its children."""
-        text = ''
-        if element.text is not None:
-            text += element.text
+        text = element.text if element.text is not None else ''
         for child in element:
             text += self._get_element_text(child)
-        if element.tail is not None:
-            text += element.tail
+        text += element.tail if element.tail is not None else ''
         return text
 
     @classmethod
@@ -286,7 +293,13 @@ class HocrTransform:
         line_box = self.element_coordinates(line)
         if not line_box:
             return
-        assert line_box.ury > line_box.lly  # lly is top, ury is bottom
+        if line_box.ury <= line_box.lly:
+            log.error(
+                "line box is invalid so we cannot render it: box=%s text=%s",
+                line_box,
+                self._get_element_text(line),
+            )
+            return
 
         self._debug_draw_line_bbox(canvas, line_box)
 

src/ocrmypdf/optimize.py

@@ -28,6 +28,7 @@ from pikepdf import (
     Stream,
     UnsupportedImageTypeError,
 )
+from pikepdf.models.image import HifiPrintImageNotTranscodableError
 from PIL import Image
 
 from ocrmypdf._concurrent import Executor, SerialExecutor
@@ -200,7 +201,7 @@ def extract_image_generic(
             with imgname.open('wb') as f:
                 ext = pim.extract_to(stream=f)
             imgname.rename(imgname.with_suffix(ext))
-        except UnsupportedImageTypeError:
+        except (UnsupportedImageTypeError, HifiPrintImageNotTranscodableError):
             return None
         return XrefExt(xref, ext)
     elif (
@@ -256,6 +257,9 @@ def _find_image_xrefs_container(
     for _imname, image in dict(xobjs).items():
         if image.objgen[1] != 0:
             continue  # Ignore images in an incremental PDF
+        xref = Xref(image.objgen[0])
+        if xref in include_xrefs or xref in exclude_xrefs:
+            continue  # Already processed
         if Name.Subtype in image and image.Subtype == Name.Form:
             # Recurse into Form XObjects
             log.debug(f"Recursing into Form XObject {_imname} in page {pageno}")
@@ -269,7 +273,6 @@ def _find_image_xrefs_container(
                 depth + 1,
             )
             continue
-        xref = Xref(image.objgen[0])
         if Name.SMask in image:
             # Ignore soft masks
             smask_xref = Xref(image.SMask.objgen[0])

src/ocrmypdf/pdfinfo/info.py

@@ -239,7 +239,13 @@ def _interpret_contents(contentstream: Object, initial_shorthand=UNIT_SQUARE):
                 # to do. Just pretend nothing happened, keep calm and carry on.
                 warn("PDF graphics stack underflowed - PDF may be malformed")
         elif operator == 'cm':
-            ctm = Matrix(operands) @ ctm
+            try:
+                ctm = Matrix(operands) @ ctm
+            except ValueError:
+                raise InputFileError(
+                    "PDF content stream is corrupt - this PDF is malformed. "
+                    "Use a PDF editor that is capable of visually inspecting the PDF."
+                )
         elif operator == 'Do':
             image_name = operands[0]
             settings = XobjectSettings(
@@ -363,8 +369,18 @@ class ImageInfo:
             pim = PdfImage(pdfimage)
         else:
             raise ValueError("Either pdfimage or inline must be set")
-        self._width = pim.width
-        self._height = pim.height
+        if pim.obj.get(Name.SMask, None) is not None:
+            # SMask is pretty much an alpha channel, but in PDF it's possible
+            # for channel to have different dimensions than the image
+            # itself. Some PDF writers use this to create a grayscale stencil
+            # mask. For our purposes, the effective size is the size of the
+            # larger component (image or smask).
+            smask = pim.obj[Name.SMask]
+            self._width = max(smask.get(Name.Width, 0), pim.width)
+            self._height = max(smask.get(Name.Height, 0), pim.height)
+        else:
+            self._width = pim.width
+            self._height = pim.height
 
         # If /ImageMask is true, then this image is a stencil mask
         # (Images that draw with this stencil mask will have a reference to
@@ -485,7 +501,7 @@ class ImageInfo:
         """Physical area of the image in square inches."""
         if not self.renderable:
             return 0.0
-        return float(self.width * self.dpi.x * self.height * self.dpi.y)
+        return float((self.width / self.dpi.x) * (self.height / self.dpi.y))
 
     def __repr__(self):
         """Return a string representation of the image."""
@@ -567,7 +583,7 @@ def _find_form_xobject_images(pdf: Pdf, container: Object, contentsinfo: Content
     xobjs = resources[Name.XObject].as_dict()
     for xobj in xobjs:
         candidate = xobjs[xobj]
-        if candidate is None or candidate[Name.Subtype] != Name.Form:
+        if candidate is None or candidate.get(Name.Subtype) != Name.Form:
             continue
 
         form_xobject = candidate
@@ -1065,7 +1081,6 @@ class PageInfo:
 
         arg_max_dpi = image_dpis.index(max_dpi)
         max_area_ratio = image_areas[arg_max_dpi] / total_drawn_area
-
         return PageResolutionProfile(
             weighted_dpi,
             max_dpi,

tests/test_semfree.py

@@ -23,4 +23,4 @@ def test_semfree(resources, outpdf):
         '--plugin',
         'tests/plugins/tesseract_noop.py',
     )
-    assert exitcode == ExitCode.ok
+    assert exitcode in (ExitCode.ok, ExitCode.pdfa_conversion_failed)

tests/test_tesseract.py

@@ -13,9 +13,9 @@ import pytest
 
 from ocrmypdf import pdfinfo
 from ocrmypdf._exec import tesseract
-from ocrmypdf.exceptions import MissingDependencyError
+from ocrmypdf.exceptions import BadArgsError, ExitCode, MissingDependencyError
 
-from .conftest import check_ocrmypdf
+from .conftest import check_ocrmypdf, run_ocrmypdf_api
 
 # pylint: disable=redefined-outer-name
 
@@ -144,3 +144,10 @@ def test_tesseract_log_output_raises(caplog):
     with pytest.raises(tesseract.TesseractConfigError):
         tesseract.tesseract_log_output(b'parameter not found: moo')
     assert 'not found' in caplog.text
+
+
+def test_blocked_language(resources, no_outpdf):
+    infile = resources / 'masks.pdf'
+    for bad_lang in ['osd', 'equ']:
+        with pytest.raises(BadArgsError):
+            run_ocrmypdf_api(infile, no_outpdf, '-l', bad_lang)