Disable oom killer test for --use-threads

Re: issue Hanging on Random Files  #814

docs/docker.rst

@@ -59,7 +59,7 @@ Using the Docker image on the command line
 ==========================================
 
 **Unlike typical Docker containers**, in this section the OCRmyPDF Docker
-container is emphemeral – it runs for one OCR job and terminates, just like a
+container is ephemeral – it runs for one OCR job and terminates, just like a
 command line program. We are using Docker to deliver an application (as opposed
 to the more conventional case, where a Docker container runs as a server).
 

docs/plugins.rst

@@ -152,6 +152,16 @@ hooks. As such, you cannot "chain" a series of plugin filters together in this
 way. Instead, a single hook implementation should be responsible for any such
 chaining operations.
 
+Examples
+========
+
+* OCRmyPDF's test suite contains several plugins that are used to simulate certain
+  test conditions.
+* `ocrmypdf-papermerge <https://github.com/papermerge/OCRmyPDF_papermerge>`_ is
+  a production plugin that integrates OCRmyPDF and the Papermerge document
+  management system.
+
+
 Custom command line arguments
 -----------------------------
 

docs/release_notes.rst

@@ -12,6 +12,22 @@ may be unreliable. Use the API to depend on precise behavior.
 The public API may be useful in scripts that launch OCRmyPDF processes or that
 wish to use some of its features for working with PDFs..
 
+v13.4.1
+=======
+
+-  Temporarily make threads rather than processes the default executor worker, due
+   to a persistent deadlock issue when processes are used. Add a new command line
+   argument ``--no-use-threads`` to disable this.
+
+v13.4.0
+=======
+
+-  Fixed test failures when using pikepdf 5.0.0.
+-  Various improvements to the optimizer. In particular, we now recognize PDF images
+   that are encoded with both flate and DCT (JPEG), and also produce PDF with images
+   compressed with flate and DCT, since this often yields file size improvements
+   compared to plain DCT.
+
 v13.3.0
 =======
 
@@ -21,8 +37,8 @@ v13.3.0
    C library used by unpaper so it cannot be rectified easily.
 -  We now use better default settings when calling img2pdf.
 -  We no longer try to optimize images that we failed to save in certain situations.
--  We now account for some differences in text output from Tesseract 5 that differs
-   from Tesseract 4.
+-  We now account for some differences in text output from Tesseract 5 compared to
+   Tesseract 4.
 -  Better handling of Ghostscript producing empty images when attempting to rasterize
    page images.
 
@@ -73,7 +89,7 @@ v13.0.0
    Tesseract 5.x has implemented improvements to thresholding, so this feature will be
    redundant anyway.
 -  ``--deskew`` was previous calculated by a Leptonica algorithm. We now use a feature
-   of Tesseract to term the appropriate the angle to deskew a page. The deskew angle
+   of Tesseract to find the appropriate the angle to deskew a page. The deskew angle
    according to Tesseract may differ from Leptonica's algorithm. At least in theory,
    Tesseract's deskew angle is informed by a more complex analysis than Leptonica,
    so this should improve results in general. We also use Pillow to perform the

setup.cfg

@@ -50,7 +50,7 @@ install_requires =
     img2pdf>=0.3.0,<0.5  # pure Python
     packaging>=20
     pdfminer.six!=20200720,>=20191110,<=20211012
-    pikepdf>=4.0.0
+    pikepdf>=4.0.0,!=5.0.0
     pluggy>=0.13.0,<2
     reportlab>=3.5.66
     tqdm>=4

src/ocrmypdf/cli.py

@@ -228,7 +228,13 @@ Online documentation is located at:
         help=argparse.SUPPRESS,
     )
     jobcontrol.add_argument(
-        '--use-threads', action='store_true', help=argparse.SUPPRESS
+        '--use-threads', action='store_true', default=True, help=argparse.SUPPRESS
+    )
+    jobcontrol.add_argument(
+        '--no-use-threads',
+        action='store_false',
+        dest='use_threads',
+        help=argparse.SUPPRESS,
     )
 
     metadata = parser.add_argument_group(

src/ocrmypdf/helpers.py

@@ -220,11 +220,19 @@ def check_pdf(input_file: Path) -> bool:
     else:
         with pdf:
             messages = pdf.check()
+            success = True
             for msg in messages:
                 if 'error' in msg.lower():
                     log.error(msg)
+                    success = False
+                elif (
+                    "/DecodeParms: operation for dictionary attempted on object "
+                    "of type null" in msg
+                ):
+                    pass  # Ignore/spurious warning
                 else:
                     log.warning(msg)
+                    success = False
 
             sio = StringIO()
             linearize_msgs = ''
@@ -239,7 +247,7 @@ def check_pdf(input_file: Path) -> bool:
                 if linearize_msgs:
                     log.warning(linearize_msgs)
 
-            if not messages and not linearize_msgs:
+            if success and not linearize_msgs:
                 return True
             return False
 

src/ocrmypdf/optimize.py

@@ -8,6 +8,7 @@
 import logging
 import sys
 import tempfile
+import threading
 from collections import defaultdict
 from os import fspath
 from pathlib import Path
@@ -23,6 +24,7 @@ from typing import (
     Sequence,
     Tuple,
 )
+from zlib import compress
 
 import img2pdf
 from pikepdf import (
@@ -31,6 +33,7 @@ from pikepdf import (
     Object,
     ObjectStreamMode,
     Pdf,
+    PdfError,
     PdfImage,
     Stream,
     UnsupportedImageTypeError,
@@ -78,33 +81,49 @@ def extract_image_filter(
     if image.Subtype != Name.Image:
         return None
     if image.Length < 100:
-        log.debug(f"Skipping small image, xref {xref}")
+        log.debug(f"xref {xref}: skipping image with small stream size")
         return None
     if image.Width < 8 or image.Height < 8:  # Issue 732
-        log.debug(f"Skipping oddly sized image, xref {xref}")
+        log.debug(f"xref {xref}: skipping image with unusually small dimensions")
         return None
 
     pim = PdfImage(image)
 
     if len(pim.filter_decodeparms) > 1:
-        log.debug(f"Skipping multiply filtered image, xref {xref}")
-        return None
-    filtdp = pim.filter_decodeparms[0]
+        first_filtdp = pim.filter_decodeparms[0]
+        second_filtdp = pim.filter_decodeparms[1]
+        if (
+            len(pim.filter_decodeparms) == 2
+            and first_filtdp[0] == Name.FlateDecode
+            and first_filtdp[1].get(Name.Predictor, 1) == 1
+            and second_filtdp[0] == Name.DCTDecode
+            and not second_filtdp[1]
+        ):
+            log.debug(
+                f"xref {xref}: found image compressed as /FlateDecode /DCTDecode, "
+                "marked for JPEG optimization"
+            )
+            filtdp = pim.filter_decodeparms[1]
+        else:
+            log.debug(f"xref {xref}: skipping image with multiple compression filters")
+            return None
+    else:
+        filtdp = pim.filter_decodeparms[0]
 
     if pim.bits_per_component > 8:
-        log.debug(f"Skipping wide gamut image, xref {xref}")
+        log.debug(f"xref {xref}: skipping wide gamut image")
         return None  # Don't mess with wide gamut images
 
     if filtdp[0] == Name.JPXDecode:
-        log.debug(f"Skipping JPEG2000 image, xref {xref}")
+        log.debug(f"xref {xref}: skipping JPEG2000 image")
         return None  # Don't do JPEG2000
 
     if filtdp[0] == Name.CCITTFaxDecode and filtdp[1].get('/K', 0) >= 0:
-        log.debug(f"Skipping CCITT Group 3 image, xref {xref}")
+        log.debug(f"xref {xref}: skipping CCITT Group 3 image")
         return None  # pikepdf doesn't support Group 3 yet
 
     if Name.Decode in image:
-        log.debug(f"Skipping image with Decode table, xref {xref}")
+        log.debug(f"xref {xref}: skipping image with Decode table")
         return None  # Don't mess with custom Decode tables
 
     return pim, filtdp
@@ -172,14 +191,6 @@ def extract_image_generic(
         # jpeg_quality_estimate = 117.0 * (bytes_per_pixel ** 0.213)
         # if jpeg_quality_estimate < 65:
         #     return None
-
-        # We could get the ICC profile here, but there's no need to look at it
-        # for quality transcoding
-        # if icc:
-        #     stream = BytesIO(raw_jpeg.read_raw_bytes())
-        #     iccbytes = icc.read_bytes()
-        #     with Image.open(stream) as im:
-        #         im.save(jpg_name(root, xref), icc_profile=iccbytes)
         try:
             imgname = root / f'{xref:08d}'
             with imgname.open('wb') as f:
@@ -259,9 +270,9 @@ def extract_images(
                 # Ignore soft masks
                 smask_xref = Xref(image.SMask.objgen[0])
                 exclude_xrefs.add(smask_xref)
-                log.debug(f"Skipping image {smask_xref} because it is an SMask")
+                log.debug(f"xref {smask_xref}: skipping image because it is an SMask")
             include_xrefs.add(xref)
-            log.debug(f"Treating {xref} as an optimization candidate")
+            log.debug(f"xref {xref}: treating as an optimization candidate")
             if xref not in pageno_for_xref:
                 pageno_for_xref[xref] = pageno
 
@@ -273,7 +284,9 @@ def extract_images(
                 pike=pike, root=root, image=image, xref=xref, options=options
             )
         except Exception:  # pylint: disable=broad-except
-            log.exception(f"While extracting image xref {xref}, an error occurred")
+            log.exception(
+                f"xref {xref}: While extracting this image, an error occurred"
+            )
             errors += 1
         else:
             if result:
@@ -294,7 +307,7 @@ def extract_images_generic(
             pngs.append(xref_ext.xref)
         elif xref_ext.ext == '.jpg':
             jpegs.append(xref_ext.xref)
-    log.debug("Optimizable images: JPEGs: %s PNGs: %s", len(jpegs), len(pngs))
+    log.debug(f"Optimizable images: JPEGs: {len(jpegs)} PNGs: {len(pngs)}")
     return jpegs, pngs
 
 
@@ -306,7 +319,7 @@ def extract_images_jbig2(pike: Pdf, root: Path, options) -> Dict[int, List[XrefE
         group = pageno // options.jbig2_page_group_size
         jbig2_groups[group].append(xref_ext)
 
-    log.debug("Optimizable images: JBIG2 groups: %s", (len(jbig2_groups),))
+    log.debug(f"Optimizable images: JBIG2 groups: {len(jbig2_groups)}")
     return jbig2_groups
 
 
@@ -405,15 +418,11 @@ def convert_to_jbig2(
 def _optimize_jpeg(args: Tuple[Xref, Path, Path, int]) -> Tuple[Xref, Optional[Path]]:
     xref, in_jpg, opt_jpg, jpeg_quality = args
 
-    # This may produce a debug warning from PIL
-    # DEBUG:PIL.Image:Error closing: 'NoneType' object has no attribute
-    # 'close'.  Seems to be mostly harmless
-    # https://github.com/python-pillow/Pillow/issues/1144
     with Image.open(in_jpg) as im:
         im.save(opt_jpg, optimize=True, quality=jpeg_quality)
 
     if opt_jpg.stat().st_size > in_jpg.stat().st_size:
-        log.debug("xref %s, jpeg, made larger - skip", xref)
+        log.debug(f"xref {xref}, jpeg, made larger - skip")
         opt_jpg.unlink()
         return xref, None
     return xref, opt_jpg
@@ -440,7 +449,7 @@ def transcode_jpegs(
         use_threads=True,  # Processes are significantly slower at this task
         max_workers=options.jobs,
         tqdm_kwargs=dict(
-            desc="JPEGs",
+            desc="Recompressing JPEGs",
             total=len(jpegs),
             unit='image',
             disable=not options.progress_bar,
@@ -451,6 +460,73 @@ def transcode_jpegs(
     )
 
 
+def _find_deflatable_jpeg(
+    *, pike: Pdf, root: Path, image: Stream, xref: Xref, options
+) -> Optional[XrefExt]:
+    result = extract_image_filter(pike, root, image, xref)
+    if result is None:
+        return None
+    pim, filtdp = result
+
+    if filtdp[0] == Name.DCTDecode and not filtdp[1] and options.optimize >= 1:
+        return XrefExt(xref, '.memory')
+
+    return None
+
+
+def _deflate_jpeg(args: Tuple[Pdf, threading.Lock, Xref, int]) -> Tuple[Xref, bytes]:
+    pike, lock, xref, complevel = args
+    with lock:
+        xobj = pike.get_object(xref, 0)
+        try:
+            data = xobj.read_raw_bytes()
+        except PdfError:
+            return xref, b''
+    compdata = compress(data, complevel)
+    if len(compdata) >= len(data):
+        return xref, b''
+    return xref, compdata
+
+
+def deflate_jpegs(pike: Pdf, root: Path, options, executor: Executor) -> None:
+    jpegs = []
+    for _pageno, xref_ext in extract_images(pike, root, options, _find_deflatable_jpeg):
+        xref = xref_ext.xref
+        log.debug(f'xref {xref}: marking this JPEG as deflatable')
+        jpegs.append(xref)
+
+    complevel = 9 if options.optimize == 3 else 6
+
+    # Our calls to xobj.write() in finish() need coordination
+    lock = threading.Lock()
+
+    def deflate_args() -> Iterator:
+        for xref in jpegs:
+            yield pike, lock, xref, complevel
+
+    def finish(result, pbar):
+        xref, compdata = result
+        if len(compdata) > 0:
+            with lock:
+                xobj = pike.get_object(xref, 0)
+                xobj.write(compdata, filter=[Name.FlateDecode, Name.DCTDecode])
+        pbar.update()
+
+    executor(
+        use_threads=True,  # We're sharing the pdf directly, must use threads
+        max_workers=options.jobs,
+        tqdm_kwargs=dict(
+            desc="Deflating JPEGs",
+            total=len(jpegs),
+            unit='image',
+            disable=not options.progress_bar,
+        ),
+        task=_deflate_jpeg,
+        task_arguments=deflate_args(),
+        task_finished=finish,
+    )
+
+
 def _transcode_png(pike: Pdf, filename: Path, xref: Xref) -> bool:
     output = filename.with_suffix('.png.pdf')
     with output.open('wb') as f:
@@ -564,6 +640,7 @@ def optimize(
 
         jpegs, pngs = extract_images_generic(pike, root, options)
         transcode_jpegs(pike, jpegs, root, options, executor)
+        deflate_jpegs(pike, root, options, executor)
         # if options.optimize >= 2:
         # Try pngifying the jpegs
         #    transcode_pngs(pike, jpegs, jpg_name, root, options)

src/ocrmypdf/pdfinfo/info.py

@@ -25,6 +25,7 @@ from typing import (
     Mapping,
     NamedTuple,
     Optional,
+    Sequence,
     Tuple,
     Union,
 )
@@ -646,7 +647,7 @@ def _pdf_pageinfo_concurrent(
     max_workers,
     check_pages,
     detailed_analysis=False,
-):
+) -> List[Optional['PageInfo']]:
     pages = [None] * len(pdf.pages)
 
     def update_pageinfo(result, pbar):
@@ -918,7 +919,7 @@ class PdfInfo:
                     self._has_acroform = True
 
     @property
-    def pages(self):
+    def pages(self) -> Sequence[Optional[PageInfo]]:
         return self._pages
 
     @property

src/ocrmypdf/pluginspec.py

@@ -132,6 +132,7 @@ def get_progressbar_class():
     Here is how OCRmyPDF will use the progress bar:
 
     Example:
+
         pbar_class = pm.hook.get_progressbar_class()
         with pbar_class(**tqdm_kwargs) as pbar:
             ...
@@ -235,9 +236,9 @@ def filter_page_image(page: 'PageContext', image_filename: Path) -> Path:
     ``image_filename``. The hook may overwrite ``image_filename`` with a new file.
 
     The output image should preserve the same physical unit dimensions, that is
-    (width * dpi_x, height * dpi_y). That is, if the image is resized, the DPI
+    ``(width * dpi_x, height * dpi_y)``. That is, if the image is resized, the DPI
     must be adjusted by the reciprocal. If this is not preserved, the PDF page
-    will be resized and the OCR layer misaligned. OCRmyPDF does not nothing
+    will be resized and the OCR layer misaligned. OCRmyPDF does nothing
     to enforce these constraints; it is up to the plugin to do sensible things.
 
     OCRmyPDF will create the PDF page based on the image format used (unless the
@@ -399,8 +400,7 @@ def get_ocr_engine() -> OcrEngine:
     """Returns an OcrEngine to use for processing this file.
 
     The OcrEngine may be instantiated multiple times, by both the main process
-    and child process. As such, it must be obtain store any state in ``options``
-    or some common location.
+    and child process.
 
     Note:
         This is a :ref:`firstresult hook<firstresult>`.

tests/test_concurrency.py

@@ -13,6 +13,7 @@ from ocrmypdf import ExitCode
 from .conftest import run_ocrmypdf_api
 
 
+@pytest.mark.skipif(True, reason="--use-threads is currently default")
 @pytest.mark.skipif(os.name == 'nt', reason="Windows doesn't have SIGKILL")
 def test_simulate_oom_killer(resources, no_outpdf):
     exitcode = run_ocrmypdf_api(

tests/test_optimize.py

@@ -81,7 +81,7 @@ def test_jbig2_lossy(lossy, resources, outpdf):
         '--image-dpi',
         '200',
         '--optimize',
-        3,
+        '3',
         '--jpg-quality',
         '50',
         '--png-quality',