v11.3.0 release notes

Commit 1327ab3 introduced a fix for a regression, which was reported
in #581, #634. It appears that the actual cause of this issue was
default parameters to rasterize_pdf_page in pluggy not working as
expected, causing a default rotation=0 even when a rotation was needed.
As such the OCR image was generated with the wrong orientation,
causing the initial regression and fix in commit 1327ab3.

Now that the real problem is identified, it's apparent that the logic
prior to 1327ab3 was found and we can revert to 1327ab3 since it fixes
all known cases including #658.

This reverts 1327ab3 except for retaining improves to rotation output.

docs/release_notes.rst

@@ -12,10 +12,43 @@ may be unreliable. Use the API to depend on precise behavior.
 The public API may be useful in scripts that launch OCRmyPDF processes or that
 wish to use some of its features for working with PDFs.
 
+v11.3.0
+=======
+
+-  The "OCR" step is describing as "Image processing" in the output messages when
+   OCR is disabled, to better explain the application's behavior.
+-  Debug logs are now only created when run as a command line, and not when OCR
+   is performed for an API call. It is the calling application's responsibility
+   to set up logging.
+-  For PDFs with a low number of pages, we gathered information about the input PDF
+   in a thread rather than process (when there are more pages). When run as a
+   thread, we did not close the file handle to the working PDF, leaking one file
+   handle per call of ``ocrmypdf.ocr``.
+-  Fixed an issue where debug messages send by child worker processes did not match
+   the log settings of parent process, causing messages to be dropped. This affected
+   macOS and Windows only where the parent process is not forked.
+-  Fixed the hookspec of rasterize_pdf_page to remove default parameters that
+   were not handled in an expected way by pluggy.
+-  Fixed another issue with automatic page rotation (#658) due to the issue above.
+
+v11.2.1
+=======
+
+-  Fixed an issue where optimization of a 1-bit image with a color palette or
+   associated ICC that was optimized to JBIG2 could have its colors inverted.
+
+v11.2.0
+=======
+
+-  Fixed an issue with optimizing PNG-type images that had soft masks or image masks.
+   This is a regression introduced in (or about) v11.1.0.
+-  Improved type checking of the ``plugins`` parameter for the ``ocrmypdf.ocr``
+   API call.
+
 v11.1.2
 =======
 
--  Fix hOCR renderer writing the text in roughly reverse order. This should not
+-  Fixed hOCR renderer writing the text in roughly reverse order. This should not
    affect reasonably smart PDF readers that properly locate the position of all
    text, but may confuse those that rely on the order of objects in the content
    stream. (#642)

misc/example_plugin.py

@@ -18,6 +18,25 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
+"""
+An example of an OCRmyPDF plugin.
+
+This plugin adds two new command line arguments
+    --grayscale-ocr: converts the image to grayscale before performing OCR on it
+        (This is occasionally useful for images whose color confounds OCR. It only
+        affects the image shown to OCR. The image is not saved.)
+    --mono-page: converts pages all pages in the output file to black and white
+
+To use this from the command line:
+    ocrmypdf --plugin path/to/example_plugin.py --mono-page input.pdf output.pdf
+
+To use this as an API:
+    import ocrmypdf
+    ocrmypdf.ocr('input.pdf', 'output.pdf',
+        plugins=['path/to/example_plugin.py'], mono_page=True
+    )
+"""
+
 import logging
 
 from PIL import Image

setup.py

@@ -76,7 +76,7 @@ setup(
         'pdfminer.six >= 20191110, != 20200720, <= 20200726',
         'pikepdf >= 1.14.0, < 2',
         'Pillow >= 7.0.0',
-        'pluggy >= 0.13.0',
+        'pluggy >= 0.13.0, < 1.0',
         'reportlab >= 3.3.0',  # oldest released version with sane image handling
         'tqdm >= 4',
     ],

src/ocrmypdf/_concurrent.py

@@ -49,7 +49,7 @@ def process_sigbus(*args):
     raise InputFileError("A worker process lost access to an input file")
 
 
-def process_init(queue, user_init):
+def process_init(queue, user_init, loglevel):
     """Initialize a process pool worker"""
 
     # Ignore SIGINT (our parent process will kill us gracefully)
@@ -62,6 +62,7 @@ def process_init(queue, user_init):
     # Reconfigure the root logger for this process to send all messages to a queue
     h = logging.handlers.QueueHandler(queue)
     root = logging.getLogger()
+    root.setLevel(loglevel)
     root.handlers = []
     root.addHandler(h)
 
@@ -69,7 +70,7 @@ def process_init(queue, user_init):
         user_init()
 
 
-def thread_init(_queue, user_init):
+def thread_init(_queue, user_init, _loglevel):
     # As a thread, block SIGBUS so the main thread deals with it...
     if hasattr(signal, 'SIGBUS'):
         signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGBUS})
@@ -102,7 +103,7 @@ def exec_progress_pool(
         pool = pool_class(
             processes=max_workers,
             initializer=initializer,
-            initargs=(log_queue, task_initializer),
+            initargs=(log_queue, task_initializer, logging.getLogger("").level),
         )
         try:
             results = pool.imap_unordered(task, task_arguments)

src/ocrmypdf/_exec/ghostscript.py

@@ -81,8 +81,8 @@ def rasterize_pdf(
     raster_device: str,
     raster_dpi: Resolution,
     pageno: int = 1,
-    page_dpi: Resolution = None,
-    rotation: int = None,
+    page_dpi: Optional[Resolution] = None,
+    rotation: Optional[int] = None,
     filter_vector: bool = False,
 ):
     """Rasterize one page of a PDF at resolution raster_dpi in canvas units."""

src/ocrmypdf/_graft.py

@@ -109,6 +109,7 @@ class OcrGrafter:
         if textpdf and not self.font:
             self.font, self.font_key = self._find_font(textpdf)
 
+        emplaced_page = False
         content_rotation = self.pdfinfo[pageno].rotation
         path_image = Path(image).resolve() if image else None
         if path_image is not None and path_image != self.path_base:
@@ -122,24 +123,21 @@ class OcrGrafter:
                 local_image_page = self.pdf_base.pages[-1]
                 self.pdf_base.pages[pageno].emplace(local_image_page)
                 del self.pdf_base.pages[-1]
-            # The pdf_image_page will always be created with any /Rotate applied
-            # applied already
-            content_rotation = 0
+            emplaced_page = True
 
-        if content_rotation != 0:
-            # Text can be misaligned on a /Rotate'd page.
-            # That is because we rasterize pages with /Rotate applied,
-            # so that the OCR image text is upright and comes back upright.
-            text_misaligned = (autorotate_correction - content_rotation) % 360
-            log.debug(
-                f"Text rotation: (autorotate, content) -> text misalignment = "
-                f"({autorotate_correction}, {content_rotation}) -> {text_misaligned}"
-            )
-        else:
-            text_misaligned = 0
+        # Calculate if the text is misaligned compared to the content
+        if emplaced_page:
+            content_rotation = autorotate_correction
+        text_rotation = autorotate_correction
+        text_misaligned = (text_rotation - content_rotation) % 360
+        log.debug(
+            f"Text rotation: (text, autorotate, content) -> text misalignment = "
+            f"({text_rotation}, {autorotate_correction}, {content_rotation}) -> {text_misaligned}"
+        )
 
         if textpdf and self.font:
-            # Graft the text layer onto this page, whether new or old
+            # Graft the text layer onto this page, whether new or old, possibly
+            # rotating the text layer by the amount is misaligned.
             strip_old = self.context.options.redo_ocr
             self._graft_text_layer(
                 page_num=pageno + 1,
@@ -151,14 +149,14 @@ class OcrGrafter:
                 strip_old_text=strip_old,
             )
 
-        # Correct the page rotation
+        # Correct the overall page rotation if needed, now that the text and content
+        # are aligned
         page_rotation = (content_rotation - autorotate_correction) % 360
         self.pdf_base.pages[pageno].Rotate = page_rotation
         log.debug(
             f"Page rotation: (content, auto) -> page = "
             f"({content_rotation}, {autorotate_correction}) -> {page_rotation}"
         )
-
         if self.emplacements % MAX_REPLACE_PAGES == 0:
             self.save_and_reload()
 

src/ocrmypdf/_pipeline.py

@@ -332,8 +332,10 @@ def rasterize_preview(input_file: Path, page_context: PageContext):
         output_file=output_file,
         raster_device='jpeggray',
         raster_dpi=canvas_dpi,
-        page_dpi=page_dpi,
         pageno=page_context.pageinfo.pageno + 1,
+        page_dpi=page_dpi,
+        rotation=0,
+        filter_vector=False,
     )
     return output_file
 
@@ -433,7 +435,7 @@ def rasterize(
 
     device = colorspaces[device_idx]
 
-    log.debug(f"Rasterize with {device}")
+    log.debug(f"Rasterize with {device}, rotation {correction}")
 
     # Produce the page image with square resolution or else deskew and OCR
     # will not work properly.
@@ -534,6 +536,9 @@ def create_ocr_image(image: Path, page_context: PageContext):
 
         # Pillow requires integer DPI
         dpi = tuple(round(coord) for coord in im.info['dpi'])
+        if page_context.pageinfo.rotation != 0:
+            log.info(f"Rotating {page_context.pageinfo.rotation}")
+            im = im.rotate(page_context.pageinfo.rotation)
         im.save(output_file, dpi=dpi)
     return output_file
 

src/ocrmypdf/_sync.py

@@ -209,9 +209,10 @@ def exec_page_sync(page_context: PageContext):
     if options.pdf_renderer == 'hocr':
         (hocr_out, text_out) = ocr_engine_hocr(ocr_image_out, page_context)
         ocr_out = render_hocr_page(hocr_out, page_context)
-
-    if options.pdf_renderer == 'sandwich':
+    elif options.pdf_renderer == 'sandwich':
         (ocr_out, text_out) = ocr_engine_textonly_pdf(ocr_image_out, page_context)
+    else:
+        raise NotImplementedError(f"pdf_renderer {options.pdf_renderer}")
 
     return PageResult(
         pageno=page_context.pageno,
@@ -244,7 +245,8 @@ def exec_concurrent(context: PdfContext):
     """Execute the pipeline concurrently"""
 
     # Run exec_page_sync on every page context
-    max_workers = min(len(context.pdfinfo), context.options.jobs)
+    options = context.options
+    max_workers = min(len(context.pdfinfo), options.jobs)
     if max_workers > 1:
         log.info("Start processing %d pages concurrently", max_workers)
 
@@ -267,14 +269,14 @@ def exec_concurrent(context: PdfContext):
             tls.pageno = None
 
     exec_progress_pool(
-        use_threads=context.options.use_threads,
+        use_threads=options.use_threads,
         max_workers=max_workers,
         tqdm_kwargs=dict(
             total=(2 * len(context.pdfinfo)),
-            desc='OCR',
+            desc='OCR' if options.tesseract_timeout > 0 else 'Image processing',
             unit='page',
             unit_scale=0.5,
-            disable=not context.options.progress_bar,
+            disable=not options.progress_bar,
         ),
         task_initializer=partial(worker_init, PIL.Image.MAX_IMAGE_PIXELS),
         task=exec_page_sync,
@@ -283,10 +285,10 @@ def exec_concurrent(context: PdfContext):
     )
 
     # Output sidecar text
-    if context.options.sidecar:
+    if options.sidecar:
         text = merge_sidecars(sidecars, context)
         # Copy text file to destination
-        copy_final(text, context.options.sidecar, context)
+        copy_final(text, options.sidecar, context)
 
     # Merge layers to one single pdf
     pdf = ocrgraft.finalize()
@@ -296,7 +298,7 @@ def exec_concurrent(context: PdfContext):
     pdf = post_process(pdf, context)
 
     # Copy PDF file to destination
-    copy_final(pdf, context.options.output_file, context)
+    copy_final(pdf, options.output_file, context)
 
 
 class NeverRaise(Exception):
@@ -328,9 +330,12 @@ def run_pipeline(options, *, plugin_manager, api=False):
 
     work_folder = Path(mkdtemp(prefix="com.github.ocrmypdf."))
     debug_log_handler = None
-    if (options.keep_temporary_files or options.verbose >= 1) and not os.environ.get(
-        'PYTEST_CURRENT_TEST', ''
+    if (
+        (options.keep_temporary_files or options.verbose >= 1)
+        and not os.environ.get('PYTEST_CURRENT_TEST', '')
+        and not api
     ):
+        # Debug log for command line interface only with verbose output
         debug_log_handler = configure_debug_logging(Path(work_folder) / "debug.log")
 
     pikepdf_enable_mmap()

src/ocrmypdf/api.py

@@ -11,6 +11,7 @@ import sys
 from enum import IntEnum
 from pathlib import Path
 from typing import BinaryIO, Iterable, Union
+from warnings import warn
 
 from ocrmypdf._logging import PageNumberFilter, TqdmConsole
 from ocrmypdf._plugin_manager import get_plugin_manager
@@ -44,16 +45,28 @@ def configure_logging(
 ):
     """Set up logging.
 
-    Library users may wish to use this function if they want their log output to be
-    similar to ocrmypdf command line interface. If not used, the external application
-    should configure logging on its own.
+    Before calling :func:`ocrmypdf.ocr()`, you can use this function to
+    configure logging, if you want ocrmypdf's output to look like the ocrmypdf
+    command line interface. It will register log handlers, log filters, and
+    formatters, configure color logging to standard error, and adjust the log
+    levels of third party libraries. Details of this are fine-tuned and subject
+    to change. The ``verbosity`` argument is equivalent to the argument
+    ``--verbose`` and applies those settings.
 
-    ocrmypdf will perform all of its logging under the ``"ocrmypdf"`` logging namespace.
-    In addition, ocrmypdf imports pdfminer, which logs under ``"pdfminer"``. A library
-    user may wish to configure both; note that pdfminer is extremely chatty at the log
-    level ``logging.INFO``.
+    If this function is not called, ocrmypdf will not configure logging, and it
+    is up to the caller of ``ocrmypdf.ocr()`` to set up logging as it wishes using
+    the Python standard library's logging module. If this function is called,
+    the caller may of course make further adjustments to logging.
 
-    Library users may perform additional configuration afterwards.
+    Regardless of whether this function is called, ocrmypdf will perform all of
+    its logging under the ``"ocrmypdf"`` logging namespace. In addition,
+    ocrmypdf imports pdfminer, which logs under ``"pdfminer"``. A library user
+    may wish to configure both; note that pdfminer is extremely chatty at the
+    log level ``logging.INFO``.
+
+    This function does not set up the ``debug.log`` log file that the command
+    line interface does at certain verbosity levels. Applications should configure
+    their own debug logging.
 
     Args:
         verbosity (Verbosity): Verbosity level.
@@ -226,7 +239,7 @@ def ocr(  # pylint: disable=unused-argument
     user_words: os.PathLike = None,
     user_patterns: os.PathLike = None,
     fast_web_view: float = None,
-    plugins: Iterable[str] = None,
+    plugins: Iterable[Union[str, Path]] = None,
     keep_temporary_files: bool = None,
     progress_bar: bool = None,
     **kwargs,
@@ -280,6 +293,8 @@ def ocr(  # pylint: disable=unused-argument
     """
     if not plugins:
         plugins = []
+    elif isinstance(plugins, (str, Path)):
+        plugins = [plugins]
     else:
         plugins = list(plugins)
 
@@ -292,6 +307,9 @@ def ocr(  # pylint: disable=unused-argument
     }
     create_options_kwargs.update(kwargs)
 
+    if 'verbose' in kwargs:
+        warn("ocrmypdf.ocr(verbose=) is ignored. Use ocrmypdf.configure_logging().")
+
     options = create_options(**create_options_kwargs)
     check_options(options, _plugin_manager)
     return run_pipeline(options=options, plugin_manager=_plugin_manager, api=True)

src/ocrmypdf/builtin_plugins/ghostscript.py

@@ -61,9 +61,9 @@ def rasterize_pdf_page(
     raster_device,
     raster_dpi,
     pageno,
-    page_dpi=None,
-    rotation=None,
-    filter_vector=False,
+    page_dpi,
+    rotation,
+    filter_vector,
 ):
     ghostscript.rasterize_pdf(
         input_file,

src/ocrmypdf/builtin_plugins/tesseract_ocr.py

@@ -121,9 +121,7 @@ def validate(pdfinfo, options):
         os.environ['OMP_THREAD_LIMIT'] = str(tess_threads)
     else:
         tess_threads = int(os.environ['OMP_THREAD_LIMIT'])
-
-    if tess_threads > 1:
-        log.info("Using Tesseract OpenMP thread limit %d", tess_threads)
+    log.debug("Using Tesseract OpenMP thread limit %d", tess_threads)
 
 
 class TesseractOcrEngine(OcrEngine):

src/ocrmypdf/optimize.py

@@ -77,23 +77,26 @@ def extract_image_filter(
     if image.Subtype != Name.Image:
         return None
     if image.Length < 100:
-        log.debug("Skipping small image, xref %s", xref)
+        log.debug(f"Skipping small image, xref {xref}")
         return None
 
     pim = PdfImage(image)
 
     if len(pim.filter_decodeparms) > 1:
-        log.debug("Skipping multiply filtered, xref %s", xref)
+        log.debug(f"Skipping multiply filtered image, xref {xref}")
         return None
     filtdp = pim.filter_decodeparms[0]
 
     if pim.bits_per_component > 8:
+        log.debug(f"Skipping wide gamut image, xref {xref}")
         return None  # Don't mess with wide gamut images
 
     if filtdp[0] == Name.JPXDecode:
+        log.debug(f"Skipping JPEG2000 iamge, xref {xref}")
         return None  # Don't do JPEG2000
 
     if Name.Decode in image:
+        log.debug(f"Skipping image with Decode table, xref {xref}")
         return None  # Don't mess with custom Decode tables
 
     return pim, filtdp
@@ -113,12 +116,23 @@ def extract_image_jbig2(
         and jbig2enc.available()
     ):
         try:
+            # Save any colorspace associated with the image, so that we
+            # will export a pure 1-bit PNG with no palette or ICC profile.
+            # Showing the palette or ICC to jbig2enc will cause it to perform
+            # colorspace transform to 1bpp, which will conflict the palette or
+            # ICC if it exists.
+            colorspace = pim.obj.ColorSpace
+            # Set to DeviceGray temporarily; we already in 1 bpc.
+            pim.obj.ColorSpace = pikepdf.Name.DeviceGray
             imgname = root / f'{xref:08d}'
             with imgname.open('wb') as f:
                 ext = pim.extract_to(stream=f)
             imgname.rename(imgname.with_suffix(ext))
         except pikepdf.UnsupportedImageTypeError:
             return None
+        finally:
+            # Restore image colorspace after temporarily setting it to DeviceGray
+            pim.obj.ColorSpace = colorspace
         return XrefExt(xref, ext)
     return None
 
@@ -229,7 +243,9 @@ def extract_images(
                 # Ignore soft masks
                 smask_xref = Xref(image.SMask.objgen[0])
                 exclude_xrefs.add(smask_xref)
+                log.debug(f"Skipping image {smask_xref} because it is an SMask")
             include_xrefs.add(xref)
+            log.debug(f"Treating {xref} as an optimization candidate")
             if xref not in pageno_for_xref:
                 pageno_for_xref[xref] = pageno
 
@@ -411,9 +427,25 @@ def _transcode_png(pike: Pdf, filename: Path, xref: Xref) -> bool:
             decode_parms=local_image.DecodeParms,
         )
 
+        # Don't copy keys from the new image...
         del_keys = set(im_obj.keys()) - set(local_image.keys())
+        # ...except for the keep_fields, which are essential to displaying
+        # the image correctly and preserving its metadata. (/Decode arrays
+        # and /SMaskInData are implicitly discarded prior to this point.)
+        keep_fields = {
+            '/ID',
+            '/Intent',
+            '/Interpolate',
+            '/Mask',
+            '/Metadata',
+            '/OC',
+            '/OPI',
+            '/SMask',
+            '/StructParent',
+        }
+        del_keys -= keep_fields
         for key in local_image.keys():
-            if key != Name.Length:
+            if key != Name.Length and str(key) not in keep_fields:
                 im_obj[key] = local_image[key]
         for key in del_keys:
             del im_obj[key]

src/ocrmypdf/pdfinfo/info.py

@@ -630,6 +630,9 @@ worker_pdf = None
 def _pdf_pageinfo_sync_init(infile):
     global worker_pdf  # pylint: disable=global-statement
     pikepdf_enable_mmap()
+    # If this function is called as a thread initializer, we need a messy hack
+    # to close worker_pdf. If called as a process, it will be released when the
+    # process is terminated.
     worker_pdf = pikepdf.open(infile)
 
 
@@ -643,6 +646,7 @@ def _pdf_pageinfo_sync(args):
 def _pdf_pageinfo_concurrent(
     pdf, infile, progbar, max_workers, check_pages, detailed_analysis=False
 ):
+    global worker_pdf  # pylint: disable=global-statement
     pages = [None] * len(pdf.pages)
 
     def update_pageinfo(result, pbar):
@@ -663,17 +667,23 @@ def _pdf_pageinfo_concurrent(
         # a separate process.
         use_threads = True
 
-    exec_progress_pool(
-        use_threads=use_threads,
-        max_workers=n_workers,
-        tqdm_kwargs=dict(
-            total=total, desc="Scanning contents", unit='page', disable=not progbar
-        ),
-        task_initializer=partial(_pdf_pageinfo_sync_init, infile),
-        task=_pdf_pageinfo_sync,
-        task_arguments=contexts,
-        task_finished=update_pageinfo,
-    )
+    try:
+        exec_progress_pool(
+            use_threads=use_threads,
+            max_workers=n_workers,
+            tqdm_kwargs=dict(
+                total=total, desc="Scanning contents", unit='page', disable=not progbar
+            ),
+            task_initializer=partial(_pdf_pageinfo_sync_init, infile),
+            task=_pdf_pageinfo_sync,
+            task_arguments=contexts,
+            task_finished=update_pageinfo,
+        )
+    finally:
+        if worker_pdf and use_threads:
+            assert n_workers == 1, "Should have only one worker when threaded"
+            # This is messy, but if we ran in thread, close worker_pdf
+            worker_pdf.close()
     return pages
 
 

src/ocrmypdf/pluginspec.py

@@ -89,9 +89,9 @@ def rasterize_pdf_page(
     raster_device: str,
     raster_dpi: Resolution,
     pageno: int,
-    page_dpi: Optional[Resolution] = None,
-    rotation: Optional[int] = None,
-    filter_vector: bool = False,
+    page_dpi: Optional[Resolution],
+    rotation: Optional[int],
+    filter_vector: bool,
 ) -> Path:
     """Rasterize one page of a PDF at resolution raster_dpi in canvas units.
 

tests/test_rotation.py

@@ -15,6 +15,7 @@ from PIL import Image
 
 from ocrmypdf import leptonica
 from ocrmypdf._exec import ghostscript, tesseract
+from ocrmypdf._plugin_manager import get_plugin_manager
 from ocrmypdf.helpers import Resolution
 from ocrmypdf.pdfinfo import PdfInfo
 
@@ -256,3 +257,33 @@ def test_tesseract_orientation(resources, tmp_path):
     tesseract.get_orientation(  # Test results of this are unreliable
         tmp_path / '000001.png', engine_mode='3', timeout=10
     )
+
+
+def test_rasterize_rotates(resources, tmp_path):
+    pm = get_plugin_manager([])
+
+    img = tmp_path / 'img90.png'
+    pm.hook.rasterize_pdf_page(
+        input_file=resources / 'graph.pdf',
+        output_file=img,
+        raster_device='pngmono',
+        raster_dpi=Resolution(20, 20),
+        page_dpi=Resolution(20, 20),
+        pageno=1,
+        rotation=90,
+        filter_vector=False,
+    )
+    assert Image.open(img).size == (123, 151), "Image not rotated"
+
+    img = tmp_path / 'img180.png'
+    pm.hook.rasterize_pdf_page(
+        input_file=resources / 'graph.pdf',
+        output_file=img,
+        raster_device='pngmono',
+        raster_dpi=Resolution(20, 20),
+        page_dpi=Resolution(20, 20),
+        pageno=1,
+        rotation=180,
+        filter_vector=False,
+    )
+    assert Image.open(img).size == (151, 123), "Image not rotated"