Disable oom killer test for --use-threads

Re: issue Hanging on Random Files  #814

docs/docker.rst

@@ -59,7 +59,7 @@ Using the Docker image on the command line
 ==========================================
 
 **Unlike typical Docker containers**, in this section the OCRmyPDF Docker
-container is emphemeral – it runs for one OCR job and terminates, just like a
+container is ephemeral – it runs for one OCR job and terminates, just like a
 command line program. We are using Docker to deliver an application (as opposed
 to the more conventional case, where a Docker container runs as a server).
 

docs/plugins.rst

@@ -152,6 +152,16 @@ hooks. As such, you cannot "chain" a series of plugin filters together in this
 way. Instead, a single hook implementation should be responsible for any such
 chaining operations.
 
+Examples
+========
+
+* OCRmyPDF's test suite contains several plugins that are used to simulate certain
+  test conditions.
+* `ocrmypdf-papermerge <https://github.com/papermerge/OCRmyPDF_papermerge>`_ is
+  a production plugin that integrates OCRmyPDF and the Papermerge document
+  management system.
+
+
 Custom command line arguments
 -----------------------------
 

docs/release_notes.rst

@@ -10,13 +10,37 @@ that is, output messages may be improved at any release level, so parsing them
 may be unreliable. Use the API to depend on precise behavior.
 
 The public API may be useful in scripts that launch OCRmyPDF processes or that
-wish to use some of its features for working with PDFs.
+wish to use some of its features for working with PDFs..
 
-.. note::
+v13.4.1
+=======
 
-   Python 3.6 reaches end of life on December 23, 2021. We have already ended support
+-  Temporarily make threads rather than processes the default executor worker, due
-   for Python 3.6 but might release fixes for critical issues if necessary before that
+   to a persistent deadlock issue when processes are used. Add a new command line
-   date.
+   argument ``--no-use-threads`` to disable this.
+
+v13.4.0
+=======
+
+-  Fixed test failures when using pikepdf 5.0.0.
+-  Various improvements to the optimizer. In particular, we now recognize PDF images
+   that are encoded with both flate and DCT (JPEG), and also produce PDF with images
+   compressed with flate and DCT, since this often yields file size improvements
+   compared to plain DCT.
+
+v13.3.0
+=======
+
+-  Made a harmless but "scary" exception after failing to optimize an image less scary.
+-  Added a warning if a page image is too large for unpaper to clean. The image is
+   passed through without cleaning. This is due to a hard-coded limitation in a
+   C library used by unpaper so it cannot be rectified easily.
+-  We now use better default settings when calling img2pdf.
+-  We no longer try to optimize images that we failed to save in certain situations.
+-  We now account for some differences in text output from Tesseract 5 compared to
+   Tesseract 4.
+-  Better handling of Ghostscript producing empty images when attempting to rasterize
+   page images.
 
 v13.2.0
 =======
@@ -65,7 +89,7 @@ v13.0.0
    Tesseract 5.x has implemented improvements to thresholding, so this feature will be
    redundant anyway.
 -  ``--deskew`` was previous calculated by a Leptonica algorithm. We now use a feature
-   of Tesseract to term the appropriate the angle to deskew a page. The deskew angle
+   of Tesseract to find the appropriate the angle to deskew a page. The deskew angle
    according to Tesseract may differ from Leptonica's algorithm. At least in theory,
    Tesseract's deskew angle is informed by a more complex analysis than Leptonica,
    so this should improve results in general. We also use Pillow to perform the

setup.cfg

@@ -50,7 +50,7 @@ install_requires =
     img2pdf>=0.3.0,<0.5  # pure Python
     packaging>=20
     pdfminer.six!=20200720,>=20191110,<=20211012
-    pikepdf>=4.0.0
+    pikepdf>=4.0.0,!=5.0.0
     pluggy>=0.13.0,<2
     reportlab>=3.5.66
     tqdm>=4

src/ocrmypdf/_exec/ghostscript.py

@@ -18,7 +18,7 @@ from shutil import which
 from subprocess import PIPE, CalledProcessError
 from typing import Optional
 
-from PIL import Image
+from PIL import Image, UnidentifiedImageError
 
 from ocrmypdf.exceptions import MissingDependencyError, SubprocessOutputError
 from ocrmypdf.helpers import Resolution
@@ -71,7 +71,8 @@ def jpeg_passthrough_available() -> bool:
 
 
 def _gs_error_reported(stream) -> bool:
-    return True if re.search(r'error', stream, flags=re.IGNORECASE) else False
+    match = re.search(r'error', stream, flags=re.IGNORECASE)
+    return bool(match)
 
 
 def rasterize_pdf(
@@ -124,20 +125,27 @@ def rasterize_pdf(
         if _gs_error_reported(stderr):
             log.error(stderr)
 
-    with Image.open(BytesIO(p.stdout)) as im:
+    try:
-        if rotation is not None:
+        with Image.open(BytesIO(p.stdout)) as im:
-            log.debug("Rotating output by %i", rotation)
+            if rotation is not None:
-            # rotation is a clockwise angle and Image.ROTATE_* is
+                log.debug("Rotating output by %i", rotation)
-            # counterclockwise so this cancels out the rotation
+                # rotation is a clockwise angle and Image.ROTATE_* is
-            if rotation == 90:
+                # counterclockwise so this cancels out the rotation
-                im = im.transpose(Image.ROTATE_90)
+                if rotation == 90:
-            elif rotation == 180:
+                    im = im.transpose(Image.ROTATE_90)
-                im = im.transpose(Image.ROTATE_180)
+                elif rotation == 180:
-            elif rotation == 270:
+                    im = im.transpose(Image.ROTATE_180)
-                im = im.transpose(Image.ROTATE_270)
+                elif rotation == 270:
-            if rotation % 180 == 90:
+                    im = im.transpose(Image.ROTATE_270)
-                page_dpi = page_dpi.flip_axis()
+                if rotation % 180 == 90:
-        im.save(fspath(output_file), dpi=page_dpi)
+                    page_dpi = page_dpi.flip_axis()
+            im.save(fspath(output_file), dpi=page_dpi)
+    except UnidentifiedImageError:
+        log.error(
+            f"Ghostscript (using {raster_device} at {raster_dpi} dpi) produced "
+            "an invalid page image file."
+        )
+        raise
 
 
 class GhostscriptFollower:
@@ -161,8 +169,7 @@ class GhostscriptFollower:
                 )
                 return
         else:
-            m = self.re_page.match(line.strip())
+            if self.re_page.match(line.strip()):
-            if m:
                 self.progressbar.update()
 
 

src/ocrmypdf/_exec/tesseract.py

@@ -344,7 +344,7 @@ def generate_hocr(
         _generate_null_hocr(output_hocr, output_text, input_file)
     except CalledProcessError as e:
         tesseract_log_output(e.output)
-        if b'Image too large' in e.output:
+        if b'Image too large' in e.output or b'Empty page!!' in e.output:
             _generate_null_hocr(output_hocr, output_text, input_file)
             return
 
@@ -416,7 +416,7 @@ def generate_pdf(
         use_skip_page(output_pdf, output_text)
     except CalledProcessError as e:
         tesseract_log_output(e.output)
-        if b'Image too large' in e.output:
+        if b'Image too large' in e.output or b'Empty page!!' in e.output:
             use_skip_page(output_pdf, output_text)
             return
         raise SubprocessOutputError() from e

src/ocrmypdf/_exec/unpaper.py

@@ -13,6 +13,7 @@
 import logging
 import os
 import shlex
+from contextlib import contextmanager
 from decimal import Decimal
 from pathlib import Path
 from subprocess import PIPE, STDOUT
@@ -22,60 +23,84 @@ from typing import List, Optional, Tuple, Union
 from PIL import Image
 
 from ocrmypdf.exceptions import MissingDependencyError, SubprocessOutputError
-from ocrmypdf.subprocess import get_version
+from ocrmypdf.subprocess import get_version, run
-from ocrmypdf.subprocess import run as external_run
+
+UNPAPER_IMAGE_PIXEL_LIMIT = 256 * 1024 * 1024
 
 DecFloat = Union[Decimal, float]
 
 log = logging.getLogger(__name__)
 
 
+class UnpaperImageTooLargeError(Exception):
+    def __init__(
+        self,
+        w,
+        h,
+        message="Image with size {}x{} is too large for cleaning with 'unpaper'.",
+    ):
+        self.w = w
+        self.h = h
+        self.message = message.format(w, h)
+        super().__init__(self.message)
+
+
 def version() -> str:
     return get_version('unpaper')
 
 
-def _setup_unpaper_io(tmpdir: Path, input_file: Path) -> Tuple[Path, Path]:
+def _convert_image(im: Image.Image) -> Tuple[Image.Image, bool, str]:
     SUFFIXES = {'1': '.pbm', 'L': '.pgm', 'RGB': '.ppm'}
-    with Image.open(input_file) as im:
+    im_modified = False
-        im_modified = False
+
-        if im.mode not in SUFFIXES:
+    if im.mode not in SUFFIXES:
-            log.info("Converting image to other colorspace")
+        log.info("Converting image to other colorspace")
-            try:
-                if im.mode == 'P' and len(im.getcolors()) == 2:
-                    im = im.convert(mode='1')
-                else:
-                    im = im.convert(mode='RGB')
-            except OSError as e:
-                raise MissingDependencyError(
-                    "Could not convert image with type " + im.mode
-                ) from e
-            else:
-                im_modified = True
         try:
-            suffix = SUFFIXES[im.mode]
+            if im.mode == 'P' and len(im.getcolors()) == 2:
-        except KeyError:
+                im = im.convert(mode='1')
+            else:
+                im = im.convert(mode='RGB')
+        except OSError as e:
             raise MissingDependencyError(
-                "Failed to convert image to a supported format."
+                "Could not convert image with type " + im.mode
-            ) from None
+            ) from e
-
-        if im_modified or input_file.suffix != '.pnm':
-            input_pnm = tmpdir / 'input.pnm'
-            im.save(input_pnm, format='PPM')
         else:
-            # No changes, PNG input, just use the file we already have
+            im_modified = True
-            input_pnm = input_file
+    try:
-        output_pnm = tmpdir / f'output{suffix}'
+        suffix = SUFFIXES[im.mode]
-    return input_pnm, output_pnm
+    except KeyError:
+        raise MissingDependencyError(
+            "Failed to convert image to a supported format."
+        ) from None
+    return im, im_modified, suffix
 
 
-def run(
+@contextmanager
+def _setup_unpaper_io(input_file: Path) -> Tuple[Path, Path, Path]:
+    with Image.open(input_file) as im:
+        if im.width * im.height >= UNPAPER_IMAGE_PIXEL_LIMIT:
+            raise UnpaperImageTooLargeError(w=im.width, h=im.height)
+        im, im_modified, suffix = _convert_image(im)
+
+        with TemporaryDirectory() as tmpdir:
+            tmppath = Path(tmpdir)
+            if im_modified or input_file.suffix != '.pnm':
+                input_pnm = tmppath / 'input.pnm'
+                im.save(input_pnm, format='PPM')
+            else:
+                # No changes, PNG input, just use the file we already have
+                input_pnm = input_file
+
+            output_pnm = tmppath / f'output{suffix}'
+            yield input_pnm, output_pnm, tmppath
+
+
+def run_unpaper(
     input_file: Path, output_file: Path, *, dpi: DecFloat, mode_args: List[str]
 ) -> None:
     args_unpaper = ['unpaper', '-v', '--dpi', str(round(dpi, 6))] + mode_args
 
-    with TemporaryDirectory() as tmpdir:
+    with _setup_unpaper_io(input_file) as (input_pnm, output_pnm, tmpdir):
-        input_pnm, output_pnm = _setup_unpaper_io(Path(tmpdir), input_file)
-
         # To prevent any shenanigans from accepting arbitrary parameters in
         # --unpaper-args, we:
         # 1) run with cwd set to a tmpdir with only unpaper's files
@@ -84,7 +109,7 @@ def run(
         # This should ensure that a user cannot clobber some other file with
         # their unpaper arguments (whether intentionally or otherwise)
         args_unpaper.extend([os.fspath(input_pnm), os.fspath(output_pnm)])
-        external_run(
+        run(
             args_unpaper,
             close_fds=True,
             check=True,
@@ -117,7 +142,7 @@ def clean(
     *,
     dpi: DecFloat,
     unpaper_args: Optional[List[str]] = None,
-):
+) -> Path:
     default_args = [
         '--layout',
         'none',
@@ -131,4 +156,9 @@ def clean(
     ]
     if not unpaper_args:
         unpaper_args = default_args
-    run(input_file, output_file, dpi=dpi, mode_args=unpaper_args)
+    try:
+        run_unpaper(input_file, output_file, dpi=dpi, mode_args=unpaper_args)
+        return output_file
+    except UnpaperImageTooLargeError as e:
+        log.warning(str(e))
+        return input_file

src/ocrmypdf/_pipeline.py

@@ -18,7 +18,7 @@ from typing import Dict, Iterable, Optional
 import img2pdf
 import pikepdf
 from pikepdf.models.metadata import encode_pdf_date
-from PIL import Image, ImageColor, ImageDraw
+from PIL import Image, ImageDraw
 
 from ocrmypdf._concurrent import Executor
 from ocrmypdf._exec import unpaper
@@ -32,7 +32,7 @@ from ocrmypdf.exceptions import (
     PriorOcrFoundError,
     UnsupportedImageFormatError,
 )
-from ocrmypdf.helpers import Resolution, safe_symlink
+from ocrmypdf.helpers import IMG2PDF_KWARGS, Resolution, safe_symlink
 from ocrmypdf.hocrtransform import HocrTransform
 from ocrmypdf.optimize import optimize
 from ocrmypdf.pdfa import generate_pdfa_ps
@@ -98,8 +98,8 @@ def triage_image_file(input_file, output_file, options):
             img2pdf.convert(
                 os.fspath(input_file),
                 layout_fun=layout_fun,
-                with_pdfrw=False,
                 outputstream=outf,
+                **IMG2PDF_KWARGS,
             )
         log.info("Successfully converted to PDF, processing...")
     except img2pdf.ImageOpenError as e:
@@ -494,13 +494,12 @@ def preprocess_deskew(input_file: Path, page_context: PageContext):
 def preprocess_clean(input_file: Path, page_context: PageContext):
     output_file = page_context.get_path('pp_clean.png')
     dpi = get_page_square_dpi(page_context.pageinfo, page_context.options)
-    unpaper.clean(
+    return unpaper.clean(
         input_file,
         output_file,
         dpi=dpi.x,
         unpaper_args=page_context.options.unpaper_args,
     )
-    return output_file
 
 
 def create_ocr_image(image: Path, page_context: PageContext):
@@ -613,7 +612,7 @@ def create_pdf_page_from_image(
 
         layout_fun = img2pdf.get_layout_fun(pagesize)
         img2pdf.convert(
-            imfile, with_pdfrw=False, layout_fun=layout_fun, outputstream=pdf
+            imfile, layout_fun=layout_fun, outputstream=pdf, **IMG2PDF_KWARGS
         )
         log.debug('convert done')
 

src/ocrmypdf/cli.py

@@ -142,7 +142,7 @@ Online documentation is located at:
         'output_file',
         metavar="output_pdf",
         help="Output searchable PDF file (or '-' to write to standard output). "
-        "Existing files will be ovewritten. If same as input file, the "
+        "Existing files will be overwritten. If same as input file, the "
         "input file will be updated only if processing is successful.",
     )
     parser.add_argument(
@@ -228,7 +228,13 @@ Online documentation is located at:
         help=argparse.SUPPRESS,
     )
     jobcontrol.add_argument(
-        '--use-threads', action='store_true', help=argparse.SUPPRESS
+        '--use-threads', action='store_true', default=True, help=argparse.SUPPRESS
+    )
+    jobcontrol.add_argument(
+        '--no-use-threads',
+        action='store_false',
+        dest='use_threads',
+        help=argparse.SUPPRESS,
     )
 
     metadata = parser.add_argument_group(

src/ocrmypdf/helpers.py

@@ -19,10 +19,21 @@ from math import isclose, isfinite
 from pathlib import Path
 from typing import Any, Sequence
 
+import img2pdf
 import pikepdf
+from packaging.version import Version
 
 log = logging.getLogger(__name__)
 
+if Version(img2pdf.__version__) < Version('0.4.0'):
+    IMG2PDF_KWARGS = dict(without_pdfw=True)
+elif Version(img2pdf.__version__) < Version('0.4.3'):
+    IMG2PDF_KWARGS = dict(engine=img2pdf.Engine.pikepdf)
+else:
+    IMG2PDF_KWARGS = dict(
+        engine=img2pdf.Engine.pikepdf, rotation=img2pdf.Rotation.ifvalid
+    )
+
 
 class Resolution(namedtuple('Resolution', ('x', 'y'))):
     """The number of pixels per inch in each 2D direction.
@@ -209,11 +220,19 @@ def check_pdf(input_file: Path) -> bool:
     else:
         with pdf:
             messages = pdf.check()
+            success = True
             for msg in messages:
                 if 'error' in msg.lower():
                     log.error(msg)
+                    success = False
+                elif (
+                    "/DecodeParms: operation for dictionary attempted on object "
+                    "of type null" in msg
+                ):
+                    pass  # Ignore/spurious warning
                 else:
                     log.warning(msg)
+                    success = False
 
             sio = StringIO()
             linearize_msgs = ''
@@ -228,7 +247,7 @@ def check_pdf(input_file: Path) -> bool:
                 if linearize_msgs:
                     log.warning(linearize_msgs)
 
-            if not messages and not linearize_msgs:
+            if success and not linearize_msgs:
                 return True
             return False
 

src/ocrmypdf/optimize.py

@@ -8,6 +8,7 @@
 import logging
 import sys
 import tempfile
+import threading
 from collections import defaultdict
 from os import fspath
 from pathlib import Path
@@ -23,6 +24,7 @@ from typing import (
     Sequence,
     Tuple,
 )
+from zlib import compress
 
 import img2pdf
 from pikepdf import (
@@ -31,6 +33,7 @@ from pikepdf import (
     Object,
     ObjectStreamMode,
     Pdf,
+    PdfError,
     PdfImage,
     Stream,
     UnsupportedImageTypeError,
@@ -41,7 +44,7 @@ from ocrmypdf._concurrent import Executor, SerialExecutor
 from ocrmypdf._exec import jbig2enc, pngquant
 from ocrmypdf._jobcontext import PdfContext
 from ocrmypdf.exceptions import OutputFileAccessError
-from ocrmypdf.helpers import safe_symlink
+from ocrmypdf.helpers import IMG2PDF_KWARGS, safe_symlink
 
 log = logging.getLogger(__name__)
 
@@ -78,33 +81,49 @@ def extract_image_filter(
     if image.Subtype != Name.Image:
         return None
     if image.Length < 100:
-        log.debug(f"Skipping small image, xref {xref}")
+        log.debug(f"xref {xref}: skipping image with small stream size")
         return None
     if image.Width < 8 or image.Height < 8:  # Issue 732
-        log.debug(f"Skipping oddly sized image, xref {xref}")
+        log.debug(f"xref {xref}: skipping image with unusually small dimensions")
         return None
 
     pim = PdfImage(image)
 
     if len(pim.filter_decodeparms) > 1:
-        log.debug(f"Skipping multiply filtered image, xref {xref}")
+        first_filtdp = pim.filter_decodeparms[0]
-        return None
+        second_filtdp = pim.filter_decodeparms[1]
-    filtdp = pim.filter_decodeparms[0]
+        if (
+            len(pim.filter_decodeparms) == 2
+            and first_filtdp[0] == Name.FlateDecode
+            and first_filtdp[1].get(Name.Predictor, 1) == 1
+            and second_filtdp[0] == Name.DCTDecode
+            and not second_filtdp[1]
+        ):
+            log.debug(
+                f"xref {xref}: found image compressed as /FlateDecode /DCTDecode, "
+                "marked for JPEG optimization"
+            )
+            filtdp = pim.filter_decodeparms[1]
+        else:
+            log.debug(f"xref {xref}: skipping image with multiple compression filters")
+            return None
+    else:
+        filtdp = pim.filter_decodeparms[0]
 
     if pim.bits_per_component > 8:
-        log.debug(f"Skipping wide gamut image, xref {xref}")
+        log.debug(f"xref {xref}: skipping wide gamut image")
         return None  # Don't mess with wide gamut images
 
     if filtdp[0] == Name.JPXDecode:
-        log.debug(f"Skipping JPEG2000 image, xref {xref}")
+        log.debug(f"xref {xref}: skipping JPEG2000 image")
         return None  # Don't do JPEG2000
 
     if filtdp[0] == Name.CCITTFaxDecode and filtdp[1].get('/K', 0) >= 0:
-        log.debug(f"Skipping CCITT Group 3 image, xref {xref}")
+        log.debug(f"xref {xref}: skipping CCITT Group 3 image")
         return None  # pikepdf doesn't support Group 3 yet
 
     if Name.Decode in image:
-        log.debug(f"Skipping image with Decode table, xref {xref}")
+        log.debug(f"xref {xref}: skipping image with Decode table")
         return None  # Don't mess with custom Decode tables
 
     return pim, filtdp
@@ -172,14 +191,6 @@ def extract_image_generic(
         # jpeg_quality_estimate = 117.0 * (bytes_per_pixel ** 0.213)
         # if jpeg_quality_estimate < 65:
         #     return None
-
-        # We could get the ICC profile here, but there's no need to look at it
-        # for quality transcoding
-        # if icc:
-        #     stream = BytesIO(raw_jpeg.read_raw_bytes())
-        #     iccbytes = icc.read_bytes()
-        #     with Image.open(stream) as im:
-        #         im.save(jpg_name(root, xref), icc_profile=iccbytes)
         try:
             imgname = root / f'{xref:08d}'
             with imgname.open('wb') as f:
@@ -200,7 +211,11 @@ def extract_image_generic(
     elif not pim.indexed and pim.colorspace in pim.SIMPLE_COLORSPACES:
         # An optimization opportunity here, not currently taken, is directly
         # generating a PNG from compressed data
-        pim.as_pil_image().save(png_name(root, xref))
+        try:
+            pim.as_pil_image().save(png_name(root, xref))
+        except NotImplementedError:
+            log.warning("PDF contains an atypical image that cannot be optimized.")
+            return None
         return XrefExt(xref, '.png')
     elif (
         not pim.indexed
@@ -255,9 +270,9 @@ def extract_images(
                 # Ignore soft masks
                 smask_xref = Xref(image.SMask.objgen[0])
                 exclude_xrefs.add(smask_xref)
-                log.debug(f"Skipping image {smask_xref} because it is an SMask")
+                log.debug(f"xref {smask_xref}: skipping image because it is an SMask")
             include_xrefs.add(xref)
-            log.debug(f"Treating {xref} as an optimization candidate")
+            log.debug(f"xref {xref}: treating as an optimization candidate")
             if xref not in pageno_for_xref:
                 pageno_for_xref[xref] = pageno
 
@@ -269,7 +284,9 @@ def extract_images(
                 pike=pike, root=root, image=image, xref=xref, options=options
             )
         except Exception:  # pylint: disable=broad-except
-            log.exception(f"While extracting image xref {xref}, an error occurred")
+            log.exception(
+                f"xref {xref}: While extracting this image, an error occurred"
+            )
             errors += 1
         else:
             if result:
@@ -290,7 +307,7 @@ def extract_images_generic(
             pngs.append(xref_ext.xref)
         elif xref_ext.ext == '.jpg':
             jpegs.append(xref_ext.xref)
-    log.debug("Optimizable images: JPEGs: %s PNGs: %s", len(jpegs), len(pngs))
+    log.debug(f"Optimizable images: JPEGs: {len(jpegs)} PNGs: {len(pngs)}")
     return jpegs, pngs
 
 
@@ -302,7 +319,7 @@ def extract_images_jbig2(pike: Pdf, root: Path, options) -> Dict[int, List[XrefE
         group = pageno // options.jbig2_page_group_size
         jbig2_groups[group].append(xref_ext)
 
-    log.debug("Optimizable images: JBIG2 groups: %s", (len(jbig2_groups),))
+    log.debug(f"Optimizable images: JBIG2 groups: {len(jbig2_groups)}")
     return jbig2_groups
 
 
@@ -401,15 +418,11 @@ def convert_to_jbig2(
 def _optimize_jpeg(args: Tuple[Xref, Path, Path, int]) -> Tuple[Xref, Optional[Path]]:
     xref, in_jpg, opt_jpg, jpeg_quality = args
 
-    # This may produce a debug warning from PIL
-    # DEBUG:PIL.Image:Error closing: 'NoneType' object has no attribute
-    # 'close'.  Seems to be mostly harmless
-    # https://github.com/python-pillow/Pillow/issues/1144
     with Image.open(in_jpg) as im:
         im.save(opt_jpg, optimize=True, quality=jpeg_quality)
 
     if opt_jpg.stat().st_size > in_jpg.stat().st_size:
-        log.debug("xref %s, jpeg, made larger - skip", xref)
+        log.debug(f"xref {xref}, jpeg, made larger - skip")
         opt_jpg.unlink()
         return xref, None
     return xref, opt_jpg
@@ -436,7 +449,7 @@ def transcode_jpegs(
         use_threads=True,  # Processes are significantly slower at this task
         max_workers=options.jobs,
         tqdm_kwargs=dict(
-            desc="JPEGs",
+            desc="Recompressing JPEGs",
             total=len(jpegs),
             unit='image',
             disable=not options.progress_bar,
@@ -447,10 +460,77 @@ def transcode_jpegs(
     )
 
 
+def _find_deflatable_jpeg(
+    *, pike: Pdf, root: Path, image: Stream, xref: Xref, options
+) -> Optional[XrefExt]:
+    result = extract_image_filter(pike, root, image, xref)
+    if result is None:
+        return None
+    pim, filtdp = result
+
+    if filtdp[0] == Name.DCTDecode and not filtdp[1] and options.optimize >= 1:
+        return XrefExt(xref, '.memory')
+
+    return None
+
+
+def _deflate_jpeg(args: Tuple[Pdf, threading.Lock, Xref, int]) -> Tuple[Xref, bytes]:
+    pike, lock, xref, complevel = args
+    with lock:
+        xobj = pike.get_object(xref, 0)
+        try:
+            data = xobj.read_raw_bytes()
+        except PdfError:
+            return xref, b''
+    compdata = compress(data, complevel)
+    if len(compdata) >= len(data):
+        return xref, b''
+    return xref, compdata
+
+
+def deflate_jpegs(pike: Pdf, root: Path, options, executor: Executor) -> None:
+    jpegs = []
+    for _pageno, xref_ext in extract_images(pike, root, options, _find_deflatable_jpeg):
+        xref = xref_ext.xref
+        log.debug(f'xref {xref}: marking this JPEG as deflatable')
+        jpegs.append(xref)
+
+    complevel = 9 if options.optimize == 3 else 6
+
+    # Our calls to xobj.write() in finish() need coordination
+    lock = threading.Lock()
+
+    def deflate_args() -> Iterator:
+        for xref in jpegs:
+            yield pike, lock, xref, complevel
+
+    def finish(result, pbar):
+        xref, compdata = result
+        if len(compdata) > 0:
+            with lock:
+                xobj = pike.get_object(xref, 0)
+                xobj.write(compdata, filter=[Name.FlateDecode, Name.DCTDecode])
+        pbar.update()
+
+    executor(
+        use_threads=True,  # We're sharing the pdf directly, must use threads
+        max_workers=options.jobs,
+        tqdm_kwargs=dict(
+            desc="Deflating JPEGs",
+            total=len(jpegs),
+            unit='image',
+            disable=not options.progress_bar,
+        ),
+        task=_deflate_jpeg,
+        task_arguments=deflate_args(),
+        task_finished=finish,
+    )
+
+
 def _transcode_png(pike: Pdf, filename: Path, xref: Xref) -> bool:
     output = filename.with_suffix('.png.pdf')
     with output.open('wb') as f:
-        img2pdf.convert(fspath(filename), outputstream=f)
+        img2pdf.convert(fspath(filename), outputstream=f, **IMG2PDF_KWARGS)
 
     with Pdf.open(output) as pdf_image:
         foreign_image = next(iter(pdf_image.pages[0].images.values()))
@@ -560,6 +640,7 @@ def optimize(
 
         jpegs, pngs = extract_images_generic(pike, root, options)
         transcode_jpegs(pike, jpegs, root, options, executor)
+        deflate_jpegs(pike, root, options, executor)
         # if options.optimize >= 2:
         # Try pngifying the jpegs
         #    transcode_pngs(pike, jpegs, jpg_name, root, options)

src/ocrmypdf/pdfinfo/info.py

@@ -25,6 +25,7 @@ from typing import (
     Mapping,
     NamedTuple,
     Optional,
+    Sequence,
     Tuple,
     Union,
 )
@@ -646,7 +647,7 @@ def _pdf_pageinfo_concurrent(
     max_workers,
     check_pages,
     detailed_analysis=False,
-):
+) -> List[Optional['PageInfo']]:
     pages = [None] * len(pdf.pages)
 
     def update_pageinfo(result, pbar):
@@ -918,7 +919,7 @@ class PdfInfo:
                     self._has_acroform = True
 
     @property
-    def pages(self):
+    def pages(self) -> Sequence[Optional[PageInfo]]:
         return self._pages
 
     @property

src/ocrmypdf/pluginspec.py

@@ -132,6 +132,7 @@ def get_progressbar_class():
     Here is how OCRmyPDF will use the progress bar:
 
     Example:
+
         pbar_class = pm.hook.get_progressbar_class()
         with pbar_class(**tqdm_kwargs) as pbar:
             ...
@@ -235,9 +236,9 @@ def filter_page_image(page: 'PageContext', image_filename: Path) -> Path:
     ``image_filename``. The hook may overwrite ``image_filename`` with a new file.
 
     The output image should preserve the same physical unit dimensions, that is
-    (width * dpi_x, height * dpi_y). That is, if the image is resized, the DPI
+    ``(width * dpi_x, height * dpi_y)``. That is, if the image is resized, the DPI
     must be adjusted by the reciprocal. If this is not preserved, the PDF page
-    will be resized and the OCR layer misaligned. OCRmyPDF does not nothing
+    will be resized and the OCR layer misaligned. OCRmyPDF does nothing
     to enforce these constraints; it is up to the plugin to do sensible things.
 
     OCRmyPDF will create the PDF page based on the image format used (unless the
@@ -399,8 +400,7 @@ def get_ocr_engine() -> OcrEngine:
     """Returns an OcrEngine to use for processing this file.
 
     The OcrEngine may be instantiated multiple times, by both the main process
-    and child process. As such, it must be obtain store any state in ``options``
+    and child process.
-    or some common location.
 
     Note:
         This is a :ref:`firstresult hook<firstresult>`.

tests/test_concurrency.py

@@ -13,6 +13,7 @@ from ocrmypdf import ExitCode
 from .conftest import run_ocrmypdf_api
 
 
+@pytest.mark.skipif(True, reason="--use-threads is currently default")
 @pytest.mark.skipif(os.name == 'nt', reason="Windows doesn't have SIGKILL")
 def test_simulate_oom_killer(resources, no_outpdf):
     exitcode = run_ocrmypdf_api(

tests/test_ghostscript.py

@@ -6,11 +6,13 @@
 
 
 import logging
+import subprocess
 from decimal import Decimal
+from unittest.mock import patch
 
 import pikepdf
 import pytest
-from PIL import Image
+from PIL import Image, UnidentifiedImageError
 
 from ocrmypdf._exec.ghostscript import rasterize_pdf
 from ocrmypdf.exceptions import ExitCode
@@ -124,3 +126,20 @@ def test_ghostscript_feature_elision(resources, outpdf):
         '--plugin',
         'tests/plugins/gs_feature_elision.py',
     )
+
+
+def test_rasterize_pdf_errors(resources, no_outpdf, caplog):
+    with patch('ocrmypdf._exec.ghostscript.run') as mock:
+        # ghostscript can produce
+        mock.return_value = subprocess.CompletedProcess(
+            ['fakegs'], returncode=0, stdout=b'', stderr=b'error this is an error'
+        )
+        with pytest.raises(UnidentifiedImageError):
+            rasterize_pdf(
+                resources / 'francais.pdf',
+                no_outpdf,
+                raster_device='pngmono',
+                raster_dpi=Resolution(100, 100),
+            )
+        assert "this is an error" in caplog.text
+        assert "invalid page image file" in caplog.text

tests/test_optimize.py

@@ -17,7 +17,7 @@ from PIL import Image, ImageDraw
 from ocrmypdf import optimize as opt
 from ocrmypdf._exec import jbig2enc, pngquant
 from ocrmypdf._exec.ghostscript import rasterize_pdf
-from ocrmypdf.helpers import Resolution
+from ocrmypdf.helpers import IMG2PDF_KWARGS, Resolution
 
 from .conftest import check_ocrmypdf
 
@@ -81,7 +81,7 @@ def test_jbig2_lossy(lossy, resources, outpdf):
         '--image-dpi',
         '200',
         '--optimize',
-        3,
+        '3',
         '--jpg-quality',
         '50',
         '--png-quality',
@@ -139,8 +139,8 @@ def test_multiple_pngs(resources, outdir):
         img2pdf.convert(
             fspath(resources / 'baiona_colormapped.png'),
             fspath(resources / 'baiona_gray.png'),
-            with_pdfrw=False,
             outputstream=inpdf,
+            **IMG2PDF_KWARGS,
         )
 
     def mockquant(input_file, output_file, *_args):

tests/test_pdfinfo.py

@@ -17,7 +17,7 @@ from reportlab.pdfgen.canvas import Canvas
 
 from ocrmypdf import pdfinfo
 from ocrmypdf.exceptions import InputFileError
-from ocrmypdf.helpers import Resolution
+from ocrmypdf.helpers import IMG2PDF_KWARGS, Resolution
 from ocrmypdf.pdfinfo import Colorspace, Encoding
 from ocrmypdf.pdfinfo.layout import PDFPage
 
@@ -67,9 +67,9 @@ def test_single_page_image(eight_by_eight, outpdf):
         img2pdf.convert(
             bio,
             producer="img2pdf",
-            with_pdfrw=False,
             layout_fun=layout_fun,
             outputstream=f,
+            **IMG2PDF_KWARGS,
         )
     info = pdfinfo.PdfInfo(outpdf)
 

tests/test_rotation.py

@@ -18,7 +18,7 @@ from reportlab.pdfgen.canvas import Canvas
 
 from ocrmypdf._exec import ghostscript
 from ocrmypdf._plugin_manager import get_plugin_manager
-from ocrmypdf.helpers import Resolution
+from ocrmypdf.helpers import IMG2PDF_KWARGS, Resolution
 from ocrmypdf.pdfinfo import PdfInfo
 
 from .conftest import check_ocrmypdf, run_ocrmypdf
@@ -230,6 +230,7 @@ def test_rotate_page_level(image_angle, page_angle, resources, outdir):
             memimg.read(),
             layout_fun=img2pdf.get_fixed_dpi_layout_fun((200, 200)),
             outputstream=mempdf,
+            **IMG2PDF_KWARGS,
         )
         mempdf.seek(0)
         pike = pikepdf.open(mempdf)

tests/test_unpaper.py

@@ -5,19 +5,24 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 
+import logging
 from os import fspath
 from unittest.mock import patch
 
 import pytest
+from PIL import Image
 
+from ocrmypdf._exec import unpaper
 from ocrmypdf._plugin_manager import get_parser_options_plugins
 from ocrmypdf._validation import check_options
 from ocrmypdf.exceptions import ExitCode, MissingDependencyError
 
-from .conftest import check_ocrmypdf, have_unpaper, run_ocrmypdf
+from .conftest import check_ocrmypdf, have_unpaper, ocrmypdf_exec, run_ocrmypdf
 
 # pylint: disable=redefined-outer-name
 
+needs_unpaper = pytest.mark.skipif(not have_unpaper(), reason="requires unpaper")
+
 
 def test_no_unpaper(resources, no_outpdf):
     input_ = fspath(resources / "c02-22.pdf")
@@ -45,7 +50,7 @@ def test_old_unpaper(resources, no_outpdf):
         mock.assert_called()
 
 
-@pytest.mark.skipif(not have_unpaper(), reason="requires unpaper")
+@needs_unpaper
 def test_clean(resources, outpdf):
     check_ocrmypdf(
         resources / "skew.pdf",
@@ -56,7 +61,7 @@ def test_clean(resources, outpdf):
     )
 
 
-@pytest.mark.skipif(not have_unpaper(), reason="requires unpaper")
+@needs_unpaper
 def test_unpaper_args_valid(resources, outpdf):
     check_ocrmypdf(
         resources / "skew.pdf",
@@ -69,7 +74,7 @@ def test_unpaper_args_valid(resources, outpdf):
     )
 
 
-@pytest.mark.skipif(not have_unpaper(), reason="requires unpaper")
+@needs_unpaper
 def test_unpaper_args_invalid_filename(resources, outpdf):
     p = run_ocrmypdf(
         resources / "skew.pdf",
@@ -84,7 +89,7 @@ def test_unpaper_args_invalid_filename(resources, outpdf):
     assert p.returncode == ExitCode.bad_args
 
 
-@pytest.mark.skipif(not have_unpaper(), reason="requires unpaper")
+@needs_unpaper
 def test_unpaper_args_invalid(resources, outpdf):
     p = run_ocrmypdf(
         resources / "skew.pdf",
@@ -98,3 +103,16 @@ def test_unpaper_args_invalid(resources, outpdf):
     # Can't tell difference between unpaper choking on bad arguments or some
     # other unpaper failure
     assert p.returncode == ExitCode.child_process_error
+
+
+@needs_unpaper
+def test_unpaper_image_too_big(resources, outdir, caplog):
+    with patch('ocrmypdf._exec.unpaper.UNPAPER_IMAGE_PIXEL_LIMIT', 42):
+        infile = resources / 'crom.png'
+        unpaper.clean(infile, outdir / 'out.png', dpi=300) == infile
+
+        assert any(
+            'too large for cleaning' in rec.message
+            for rec in caplog.get_records('call')
+            if rec.levelno == logging.WARNING
+        )