v16.4.0 release notes (3)

Fixes #1332

.docker/Dockerfile.alpine

@@ -1,7 +1,7 @@
 # SPDX-FileCopyrightText: 2023 James R. Barlow
 # SPDX-License-Identifier: MPL-2.0
 
-FROM alpine:3.19 as base
+FROM alpine:3.19.1 as base
 
 ENV LANG=C.UTF-8
 ENV TZ=UTC

.github/ISSUE_TEMPLATE/1-bug-report-general.yml

@@ -1,7 +1,7 @@
 name: Installation, packaging, dependencies
 description: Installation, packages, dependencies, "nothing works", test suite failures...
 title: "[Bug]: "
-labels: ["bug", "triage"]
+labels: ["triage"]
 assignees:
   - jbarlow83
 body:
@@ -63,6 +63,7 @@ body:
       options:
         - label: Operating system is currently supported by its vendor (not end of life)
         - label: Python version is compatible with OCRmyPDF
+        - label: This issue is not about a specific input file
   - type: textarea
     id: logs
     attributes:

.github/ISSUE_TEMPLATE/2-problem-with-specific-file.yml

@@ -1,7 +1,7 @@
 name: Problem with specific file
 description: Something went wrong while trying to OCR a specific file
 title: "[Bug]: "
-labels: ["bug", "triage"]
+labels: ["triage"]
 assignees:
   - jbarlow83
 body:
@@ -39,7 +39,7 @@ body:
         causing the issue. There's really no substitute for a test file.
 
         We understand files may contain personal or sensitive information. Here are some options:
-        - Try reproducing the issue with a file from the test suite. (See tests/resources)
+        - Try reproducing the issue with a file from the OCRmyPDF test suite. (See tests/resources)
         - Try to create another file in the same way as your private file.
         - Encrypt the file to OCRmyPDF's private GPG key, and then zip the GPG file.
         - Use ``qpdf --json yourfile.pdf`` to produce a JSON representation of your file that

.github/ISSUE_TEMPLATE/3-app.yml

@@ -0,0 +1,83 @@
+name: Problem with third party app that uses OCRmyPDF
+description: |
+  For PDF generation issues with third party software such as Paperless-ngx that
+  uses OCRmyPDF to perform OCR or generate PDFs.
+title: "[3rdparty]: "
+labels: ["triage"]
+assignees:
+  - jbarlow83
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to describe this issue with a particular file
+        and third party app.
+
+        If you are comfortable using OCRmyPDF, please trying to install OCRmyPDF,
+        run it on your file, and see if it works. It's easier for everyone
+        if you can confirm that the issue occurs with OCRmyPDF and not with
+        the third party app.
+  - type: checkboxes
+    attributes:
+      label: Simple sanity checks
+      description: Select all that apply
+      options:
+        - label: This is an issue with an app that uses OCRmyPDF for OCR
+        - label: I am using a recent version of the third party app
+        - label: I will include a file that reproduces the issuse
+  - type: input
+    id: thirdparty-app-name-version
+    attributes:
+      label: Third party app name and version
+      description: e.g. Paperless-ngx 2.9.0
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: Describe the bug
+      description: A clear and concise description of what the bug is.
+      placeholder: Tell us what you see!
+    validations:
+      required: true
+  - type: textarea
+    id: reproduce
+    attributes:
+      label: Steps to reproduce
+      description: Please include steps to reproduce.
+      value: |
+        1. Import attached file into Paperless-ngx
+        2. Trigger OCR
+        3. Check log file
+        4. ...
+      render: plain text
+  - type: textarea
+    id: files
+    attributes:
+      label: Files
+      description: |
+        Please attach the input and output files, or any screenshots that may be helpful.
+
+        If you cannot provide a test file, we probably won't be able to help with the issue.
+        PDF is a complex file format, and there may be technical details in the PDF that are
+        causing the issue. There's really no substitute for a test file.
+
+        We understand files may contain personal or sensitive information. Here are some options:
+        - Try reproducing the issue with a file from the test suite. (See tests/resources)
+        - Try to create another file in the same way as your private file.
+        - Encrypt the file to OCRmyPDF's private GPG key, and then zip the GPG file.
+        - Use ``qpdf --json yourfile.pdf`` to produce a JSON representation of your file that
+          omits personal information.
+      placeholder: |
+        Drag and drop files here.
+  - type: input
+    id: version
+    attributes:
+      label: OCRmyPDF version
+      description: Paste "ocrmypdf --version" here
+      placeholder: ocrmypdf --version
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant log output
+      description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
+      placeholder: Run OCRmyPDF with verbosity `-v1` to get more detailed logging output.
+      render: plain text

.github/workflows/build.yml

@@ -61,6 +61,7 @@ jobs:
           sudo apt-get install -y --no-install-recommends \
             curl \
             ghostscript \
+            jbig2dec \
             img2pdf \
             libexempi8 \
             libffi-dev \
@@ -358,9 +359,6 @@ jobs:
           username: jbarlow83
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
       - name: Set up Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v3
@@ -372,6 +370,6 @@ jobs:
         run: |
           docker buildx build \
             --push \
-            --platform linux/amd64  \
+            --platform linux/amd64,linux/arm64  \
             --tag "${DOCKER_REPOSITORY}/${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}" \
             --file .docker/Dockerfile.alpine .

docs/docker.rst

@@ -42,7 +42,7 @@ execute the image:
      - Architecture
      - Description
    * - ``jbarlow83/ocrmypdf-alpine``
-     - x86_64 only
+     - x86_64 and arm64
      - Recommended image, based on Alpine Linux.
    * - ``jbarlow83/ocrmypdf-ubuntu``
      - x86_64 and arm64
@@ -81,7 +81,7 @@ To start a Docker container (instance of the image):
 
 .. code-block:: bash
 
-   docker tag jbarlow83/ocrmypdf ocrmypdf
+   docker tag jbarlow83/ocrmypdf-alpine ocrmypdf
    docker run --rm -i ocrmypdf (... all other arguments here...) - -
 
 For convenience, create a shell alias to hide the Docker command. It is

docs/release_notes.rst

@@ -30,6 +30,25 @@ OCRmyPDF typically supports the three most recent Python versions.
 
 .. |OCRmyPDF PyPI| image:: https://img.shields.io/pypi/v/ocrmypdf.svg
 
+v16.4.0
+=======
+
+-  Selecting the ``osd`` and ``equ`` pseudo-languages with ``-l/--language`` now
+   exits with an error when using Tesseract OCR, because these are not
+   regular Tesseract languages but implementation details implemented.
+   Using them can cause Tesseract to crash.
+-  The hOCR renderer is more tolerant of extra whitespace in input files.
+-  watcher.py now changes the output file extension to .pdf when the input is not
+   .pdf.
+-  Improved handling of PDFs that contain circularly referenced Form XObjects.
+   :issue:`1321`
+-  Fixed Alpine Docker image for ARM64, which was not building correctly.
+-  Docker images now use pikepdf 9.0.0.
+-  Prevent use of Tesseract OCR 5.4.0, a version with known regressions.
+-  Disabled progressbar for "Linearizing" when ``--no-progress-bar`` set.
+-  Fixed some tests that warn about missing JBIG2 decoding via pikepdf, by
+   installing the necessary libraries during tests.
+
 v16.3.1
 =======
 

misc/batch.py

@@ -14,12 +14,12 @@ You should edit this script to meet your needs.
 
 from __future__ import annotations
 
+import filecmp
 import logging
-import sys
 import os
 import posixpath
 import shutil
-import filecmp
+import sys
 from pathlib import Path
 
 import ocrmypdf
@@ -70,7 +70,7 @@ for filename in start_dir.glob("**/*.pdf"):
             logging.info(f"Archiving document to {archive_filename}")
             try:
                 shutil.copy2(filename, posixpath.dirname(archive_filename))
-            except IOError as io_err:
+            except OSError:
                 os.makedirs(posixpath.dirname(archive_filename))
                 shutil.copy2(filename, posixpath.dirname(archive_filename))
         try:
@@ -86,6 +86,6 @@ for filename in start_dir.glob("**/*.pdf"):
             logging.info(
                 "Skipped document because it does not need ocr as it is tagged"
             )
-        except:
+        except Exception:
             logging.error("Unhandled error occured")
         logging.info("OCR complete")

misc/watcher.py

@@ -46,15 +46,18 @@ class LoggingLevelEnum(str, Enum):
     CRITICAL = "CRITICAL"
 
 
-def get_output_dir(root: Path, basename: str, output_dir_year_month: bool) -> Path:
+def get_output_path(root: Path, basename: str, output_dir_year_month: bool) -> Path:
+    assert '/' not in basename, "basename must not contain '/'"
     if output_dir_year_month:
         today = datetime.today()
         output_directory_year_month = root / str(today.year) / f'{today.month:02d}'
         if not output_directory_year_month.exists():
             output_directory_year_month.mkdir(parents=True, exist_ok=True)
-        output_path = Path(output_directory_year_month) / basename
+        output_path = Path(output_directory_year_month) / Path(basename).with_suffix(
+            '.pdf'
+        )
     else:
-        output_path = root / basename
+        output_path = root / Path(basename).with_suffix('.pdf')
     return output_path
 
 
@@ -98,7 +101,7 @@ def execute_ocrmypdf(
     retries_loading_file: int,
     output_dir_year_month: bool,
 ):
-    output_path = get_output_dir(output_dir, file_path.name, output_dir_year_month)
+    output_path = get_output_path(output_dir, file_path.name, output_dir_year_month)
 
     log.info("-" * 20)
     log.info(f'New file: {file_path}. Waiting until fully written...')

snapcraft.yaml

@@ -18,8 +18,8 @@ architectures: [amd64]
 
 environment:
   TESSDATA_PREFIX: $SNAP/usr/share/tesseract-ocr/4.00/tessdata
-  GS_LIB: $SNAP/usr/share/ghostscript/9.55/Resource/Init
+  GS_LIB: $SNAP/usr/share/ghostscript/9.55.0/Resource/Init
-  GS_FONTPATH: $SNAP/usr/share/ghostscript/9.55/Resource/Font
+  GS_FONTPATH: $SNAP/usr/share/ghostscript/9.55.0/Resource/Font
   LD_LIBRARY_PATH: $SNAP/usr/lib/x86_64-linux-gnu
 
 apps:

src/ocrmypdf/_metadata.py

@@ -154,10 +154,10 @@ def _set_language(pdf: Pdf, languages: list[str]):
 
 
 class MetadataProgress:
-    def __init__(self, progressbar_class):
+    def __init__(self, progressbar_class, enable: bool = True):
         self.progressbar_class = progressbar_class
         self.progressbar = self.progressbar_class(
-            total=100, desc="Linearizing", unit='%'
+            total=100, desc="Linearizing", unit='%', disable=not enable
         )
 
     def __enter__(self):
@@ -190,7 +190,7 @@ def metadata_fixup(
     with (
         Pdf.open(context.origin) as original,
         Pdf.open(working_file) as pdf,
-        MetadataProgress(pbar_class) as pbar,
+        MetadataProgress(pbar_class, options.progress_bar) as pbar,
     ):
         docinfo = get_docinfo(original, context)
         with (

src/ocrmypdf/_validation.py

@@ -54,6 +54,7 @@ def check_options_languages(
             log.debug("No language specified; assuming --language %s", DEFAULT_LANGUAGE)
     if not ocr_engine_languages:
         return
+
     missing_languages = set(options.languages) - set(ocr_engine_languages)
     if missing_languages:
         lang_text = '\n'.join(lang for lang in missing_languages)

src/ocrmypdf/builtin_plugins/tesseract_ocr.py

@@ -14,6 +14,7 @@ from ocrmypdf import hookimpl
 from ocrmypdf._exec import tesseract
 from ocrmypdf._jobcontext import PageContext
 from ocrmypdf.cli import numeric, str_to_int
+from ocrmypdf.exceptions import BadArgsError, MissingDependencyError
 from ocrmypdf.helpers import clamp
 from ocrmypdf.imageops import calculate_downsample, downsample_image
 from ocrmypdf.pluginspec import OcrEngine
@@ -144,6 +145,12 @@ def check_options(options):
         need_version='4.1.1',  # Ubuntu 22.04 version (also 20.04)
         version_parser=tesseract.TesseractVersion,
     )
+    tess_version = tesseract.version()
+    if tess_version == tesseract.TesseractVersion('5.4.0'):
+        raise MissingDependencyError(
+            "Tesseract 5.4.0 is not supported due to regressions in this version. "
+            "Please upgrade to a newer or supported older version."
+        )
 
     # Decide on what renderer to use
     if options.pdf_renderer == 'auto':
@@ -164,6 +171,14 @@ def check_options(options):
             "The --tesseract-pagesegmode argument you select will disable OCR. "
             "This may cause processing to fail."
         )
+    DENIED_LANGUAGES = {'equ', 'osd'}
+    if DENIED_LANGUAGES & set(options.languages):
+        raise BadArgsError(
+            "The following languages for Tesseract's internal use and should not "
+            "be issued explicitly: "
+            f"{', '.join(DENIED_LANGUAGES & set(options.languages))}\n"
+            "Remove them from the -l/--language argument."
+        )
 
 
 @hookimpl

src/ocrmypdf/hocrtransform/_hocr.py

@@ -61,15 +61,25 @@ class HocrTransform:
     """A class for converting documents from the hOCR format.
 
     For details of the hOCR format, see:
-    http://kba.cloud/hocr-spec/.
+    http://kba.github.io/hocr-spec/1.2/.
     """
 
-    box_pattern = re.compile(r'bbox (\d+) (\d+) (\d+) (\d+)')
+    box_pattern = re.compile(
+        r'''
+        bbox \s+
+        (\d+) \s+   # left: uint
+        (\d+) \s+   # top: uint
+        (\d+) \s+   # right: uint
+        (\d+)       # bottom: uint
+        ''',
+        re.VERBOSE,
+    )
     baseline_pattern = re.compile(
         r'''
         baseline \s+
         ([\-\+]?\d*\.?\d*) \s+  # +/- decimal float
-        ([\-\+]?\d+)            # +/- int''',
+        ([\-\+]?\d+)            # +/- int
+        ''',
         re.VERBOSE,
     )
 
@@ -284,8 +294,11 @@ class HocrTransform:
         if not line_box:
             return
         if line_box.ury <= line_box.lly:
-            log.error("line box is invalid so we cannot render it: box=%s text=%s",
+            log.error(
-                      line_box, self._get_element_text(line))
+                "line box is invalid so we cannot render it: box=%s text=%s",
+                line_box,
+                self._get_element_text(line),
+            )
             return
 
         self._debug_draw_line_bbox(canvas, line_box)

src/ocrmypdf/optimize.py

@@ -256,6 +256,9 @@ def _find_image_xrefs_container(
     for _imname, image in dict(xobjs).items():
         if image.objgen[1] != 0:
             continue  # Ignore images in an incremental PDF
+        xref = Xref(image.objgen[0])
+        if xref in include_xrefs or xref in exclude_xrefs:
+            continue  # Already processed
         if Name.Subtype in image and image.Subtype == Name.Form:
             # Recurse into Form XObjects
             log.debug(f"Recursing into Form XObject {_imname} in page {pageno}")
@@ -269,7 +272,6 @@ def _find_image_xrefs_container(
                 depth + 1,
             )
             continue
-        xref = Xref(image.objgen[0])
         if Name.SMask in image:
             # Ignore soft masks
             smask_xref = Xref(image.SMask.objgen[0])

tests/test_tesseract.py

@@ -13,9 +13,9 @@ import pytest
 
 from ocrmypdf import pdfinfo
 from ocrmypdf._exec import tesseract
-from ocrmypdf.exceptions import MissingDependencyError
+from ocrmypdf.exceptions import BadArgsError, ExitCode, MissingDependencyError
 
-from .conftest import check_ocrmypdf
+from .conftest import check_ocrmypdf, run_ocrmypdf_api
 
 # pylint: disable=redefined-outer-name
 
@@ -144,3 +144,10 @@ def test_tesseract_log_output_raises(caplog):
     with pytest.raises(tesseract.TesseractConfigError):
         tesseract.tesseract_log_output(b'parameter not found: moo')
     assert 'not found' in caplog.text
+
+
+def test_blocked_language(resources, no_outpdf):
+    infile = resources / 'masks.pdf'
+    for bad_lang in ['osd', 'equ']:
+        with pytest.raises(BadArgsError):
+            run_ocrmypdf_api(infile, no_outpdf, '-l', bad_lang)