v16.6.1 release notes

Fixes [Bug]: HOCRResult.from_json() not unpickling correctly #1427

.docker/Dockerfile

@@ -1,7 +1,7 @@
 # SPDX-FileCopyrightText: 2024 James R. Barlow
 # SPDX-License-Identifier: MPL-2.0
 
-FROM ubuntu:24.04 AS base
+FROM ubuntu:22.04 AS base
 
 ENV LANG=C.UTF-8
 ENV TZ=UTC
@@ -51,7 +51,7 @@ FROM base
 
 RUN apt-get update && apt-get install -y software-properties-common
 
-RUN add-apt-repository -y ppa:alex-p/tesseract-ocr-devel
+RUN add-apt-repository -y ppa:alex-p/tesseract-ocr5
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
   ghostscript \
@@ -75,6 +75,10 @@ COPY --from=builder /usr/local/bin/ /usr/local/bin/
 
 COPY --from=builder --chown=app:app /app /app
 
+RUN rm -rf /app/.git && \
+ln -s /app/misc/webservice.py /app/webservice.py && \
+ln -s /app/misc/watcher.py /app/watcher.py
+
 ENV PATH="/app/.venv/bin:${PATH}"
 
 ENTRYPOINT ["/app/.venv/bin/ocrmypdf"]

.docker/Dockerfile.alpine

@@ -59,6 +59,10 @@ WORKDIR /app
 
 COPY --from=builder --chown=app:app /app /app
 
+RUN rm -rf /app/.git && \
+    ln -s /app/misc/webservice.py /app/webservice.py && \
+    ln -s /app/misc/watcher.py /app/watcher.py
+
 ENV PATH="/app/.venv/bin:${PATH}"
 
 ENTRYPOINT ["/app/.venv/bin/ocrmypdf"]

.github/workflows/build.yml

@@ -50,9 +50,9 @@ jobs:
           python-version: ${{ matrix.python }}
 
       - name: Install Tesseract from PPA
-        if: matrix.tesseract_ppa
+        if: matrix.tesseract_ppa == 'ppa'
         run: |
-          sudo add-apt-repository -y ppa:alex-p/tesseract-ocr-devel
+          sudo add-apt-repository -y ppa:alex-p/tesseract-ocr5.3
 
       - name: Install common packages
         run: |
@@ -106,7 +106,6 @@ jobs:
           files: ./coverage.xml
           env_vars: OS,PYTHON
 
-
   test_macos:
     name: Test macOS
     runs-on: ${{ matrix.os }}

docs/batch.rst

@@ -44,7 +44,7 @@ place, and printing each filename in between runs:
 
 .. code-block:: bash
 
-   find . -printf '%p\n' -name '*.pdf' -exec ocrmypdf '{}' '{}' \;
+   find . -name '*.pdf' -printf '%p\n' -exec ocrmypdf '{}' '{}' \;
 
 This only runs one ``ocrmypdf`` process at a time. This variation uses
 ``find`` to create a directory list and ``parallel`` to parallelize runs

docs/release_notes.rst

@@ -30,6 +30,16 @@ OCRmyPDF typically supports the three most recent Python versions.
 
 .. |OCRmyPDF PyPI| image:: https://img.shields.io/pypi/v/ocrmypdf.svg
 
+v16.6.1
+=======
+
+-  Fixed some issues with Docker build, such as removing unnecessary content and using
+   a stable Tesseract version.
+-  Reverted Docker image to Ubuntu 22.04 to access older/more stable Ghostscript
+   for now.
+-  Clarified batch commands in documentation.
+-  Fixed an issue with JSON serialization and pickling of HOCRResult. :issue:`1427`
+
 v16.6.0
 =======
 

misc/bisect_pdf.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: 2024 James R. Barlow
+# SPDX-License-Identifier: MIT
+
+"""Helper script for bisecting PDFs to find a page with an issue."""
+
+import sys
+
+import pikepdf
+
+if len(sys.argv) != 2:
+    print(f"Usage: {sys.argv[0]} <input.pdf>")
+    sys.exit(1)
+
+with pikepdf.open(sys.argv[1]) as pdf:
+    num_pages = len(pdf.pages)
+    low = 0
+    high = num_pages - 1
+    while low <= high:
+        mid = (low + high) // 2
+        with pikepdf.new() as new_pdf:
+            new_pdf.pages.extend(pdf.pages[low : mid + 1])
+            new_pdf.save(f"bisect-issue-{low + 1}-{mid + 1}.pdf")
+        print(f"Is bisect-issue-{low + 1}-{mid + 1}.pdf good or bad?", end=" ")
+        while True:
+            response = input().lower()
+            if response == "good":
+                low = mid + 1
+                break
+            elif response == "bad":
+                high = mid - 1
+                break
+            else:
+                print("Please respond with 'good' or 'bad'.")
+    print(f"The issue is on page {low + 1} of the original PDF.")
+    with pikepdf.new() as new_pdf:
+        new_pdf.pages.extend(pdf.pages[low])
+        new_pdf.save(f"bisect-issue-bad-{low + 1}.pdf")
+    with pikepdf.new() as new_pdf:
+        new_pdf.pages.extend(pdf.pages[:low])
+        new_pdf.pages.extend(pdf.pages[low + 1 :])
+        new_pdf.save(f"bisect-issue-good-{low + 1}.pdf")

src/ocrmypdf/_pipelines/_common.py

@@ -104,6 +104,23 @@ class PageResult(NamedTuple):
     """Orientation correction in degrees."""
 
 
+class HOCRResultEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, Path):
+            return {'Path': str(obj)}
+        return super().default(obj)
+
+
+class HOCRResultDecoder(json.JSONDecoder):
+    def __init__(self, *args, **kwargs):
+        super().__init__(object_hook=self.dict_to_object, *args, **kwargs)
+
+    def dict_to_object(self, d):
+        if 'Path' in d:
+            return Path(d['Path'])
+        return d
+
+
 @dataclass
 class HOCRResult:
     """Result when hOCR is finished processing."""
@@ -123,38 +140,14 @@ class HOCRResult:
     orientation_correction: int = 0
     """Orientation correction in degrees."""
 
-    def __getstate__(self):
-        """Return state values to be pickled."""
-        return {
-            k: (
-                ('Path://' + str(v))
-                if k in ('pdf_page_from_image', 'hocr', 'textpdf') and v is not None
-                else v
-            )
-            for k, v in self.__dict__.items()
-        }
-
-    def __setstate__(self, state):
-        """Restore state from the unpickled state values."""
-        self.__dict__.update(
-            {
-                k: (
-                    Path(v.removeprefix('Path://'))
-                    if k in ('pdf_page_from_image', 'hocr', 'textpdf') and v is not None
-                    else v
-                )
-                for k, v in state.items()
-            }
-        )
-
     @classmethod
     def from_json(cls, json_str: str) -> HOCRResult:
         """Create an instance from a dict."""
-        return cls(**json.loads(json_str))
+        return cls(**json.loads(json_str, cls=HOCRResultDecoder))
 
     def to_json(self) -> str:
         """Serialize to a JSON string."""
-        return json.dumps(self.__getstate__())
+        return json.dumps(self.__dict__, cls=HOCRResultEncoder)
 
 
 def configure_debug_logging(

tests/test_api.py

@@ -3,6 +3,7 @@
 
 from __future__ import annotations
 
+import pickle
 from io import BytesIO
 from pathlib import Path
 
@@ -10,6 +11,7 @@ import pytest
 from pdfminer.high_level import extract_text
 
 import ocrmypdf
+import ocrmypdf._pipelines
 import ocrmypdf.api
 
 
@@ -35,7 +37,7 @@ def test_sidecar_stringio(resources: Path, outdir: Path, outpdf: Path):
         resources / 'ccitt.pdf',
         outpdf,
         plugins=['tests/plugins/tesseract_cache.py'],
-        sidecar=s
+        sidecar=s,
     )
     s.seek(0)
     assert b'the' in s.getvalue()
@@ -75,3 +77,29 @@ def test_hocr_to_pdf_api(resources: Path, outdir: Path, outpdf: Path):
     text = extract_text(outpdf)
     assert 'hocr' in text and 'the' not in text
 
+
+def test_hocr_result_json():
+    result = ocrmypdf._pipelines._common.HOCRResult(
+        pageno=1,
+        pdf_page_from_image=Path('a'),
+        hocr=Path('b'),
+        textpdf=Path('c'),
+        orientation_correction=180,
+    )
+    assert (
+        result.to_json()
+        == '{"pageno": 1, "pdf_page_from_image": {"Path": "a"}, "hocr": {"Path": "b"}, '
+        '"textpdf": {"Path": "c"}, "orientation_correction": 180}'
+    )
+    assert ocrmypdf._pipelines._common.HOCRResult.from_json(result.to_json()) == result
+
+
+def test_hocr_result_pickle():
+    result = ocrmypdf._pipelines._common.HOCRResult(
+        pageno=1,
+        pdf_page_from_image=Path('a'),
+        hocr=Path('b'),
+        textpdf=Path('c'),
+        orientation_correction=180,
+    )
+    assert result == pickle.loads(pickle.dumps(result))