Merge api (without plugins)

src/ocrmypdf/__init__.py

@@ -44,3 +44,4 @@ from . import hocrtransform
 from . import leptonica
 from . import pdfa
 from . import pdfinfo
+from .api import ocr, configure_logging, Verbosity

src/ocrmypdf/_graft.py

@@ -15,17 +15,12 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
-from contextlib import suppress
-from itertools import groupby
-from pathlib import Path
 import os
+from contextlib import suppress
+from pathlib import Path
 
 import pikepdf
 
-from .exec import tesseract
-from .helpers import flatten_groups, page_number
-
-
 MAX_REPLACE_PAGES = int(os.environ.get('_OCRMYPDF_MAX_REPLACE_PAGES', 100))
 
 
@@ -48,7 +43,7 @@ def _update_page_resources(*, page, font, font_key, procset):
     resources['/ProcSet'] = procset
 
 
-def strip_invisible_text(pdf, page, log):
+def strip_invisible_text(pdf, page):
     stream = []
     in_text_obj = False
     render_mode = 0
@@ -93,7 +88,7 @@ def strip_invisible_text(pdf, page, log):
     page.Contents = pikepdf.Stream(pdf, content_stream)
 
 
-def _weave_layers_graft(
+def _graft_text_layer(
     *, pdf_base, page_num, text, font, font_key, procset, rotation, strip_old_text, log
 ):
     """Insert the text layer from text page 0 on to pdf_base at page_num"""
@@ -106,16 +101,6 @@ def _weave_layers_graft(
     pdf_text = pikepdf.open(text)
     pdf_text_contents = pdf_text.pages[0].Contents.read_bytes()
 
-    if not tesseract.has_textonly_pdf():
-        # If we don't have textonly_pdf, edit the stream to delete the
-        # instruction to draw the image Tesseract generated, which we do not
-        # use.
-        stream = bytearray(pdf_text_contents)
-        pattern = b'/Im1 Do'
-        idx = stream.find(pattern)
-        stream[idx : (idx + len(pattern))] = b' ' * len(pattern)
-        pdf_text_contents = bytes(stream)
-
     base_page = pdf_base.pages.p(page_num)
 
     # The text page always will be oriented up by this stage but the original
@@ -130,6 +115,7 @@ def _weave_layers_graft(
 
     translate = pikepdf.PdfMatrix().translated(-wt / 2, -ht / 2)
     untranslate = pikepdf.PdfMatrix().translated(wp / 2, hp / 2)
+    corner = pikepdf.PdfMatrix().translated(mediabox[0], mediabox[1])
     # -rotation because the input is a clockwise angle and this formula
     # uses CCW
     rotation = -rotation % 360
@@ -143,19 +129,20 @@ def _weave_layers_graft(
     scale_x = wp / wt
     scale_y = hp / ht
 
-    log.debug('%r', (scale_x, scale_y))
+    # log.debug('%r', scale_x, scale_y)
     scale = pikepdf.PdfMatrix().scaled(scale_x, scale_y)
 
     # Translate the text so it is centered at (0, 0), rotate it there, adjust
-    # for a size different between initial and text PDF, then untranslate
-    ctm = translate @ rotate @ scale @ untranslate
+    # for a size different between initial and text PDF, then untranslate, and
+    # finally move the lower left corner to match the mediabox
+    ctm = translate @ rotate @ scale @ untranslate @ corner
 
     pdf_text_contents = b'q %s cm\n' % ctm.encode() + pdf_text_contents + b'\nQ\n'
 
     new_text_layer = pikepdf.Stream(pdf_base, pdf_text_contents)
 
     if strip_old_text:
-        strip_invisible_text(pdf_base, base_page, log)
+        strip_invisible_text(pdf_base, base_page)
 
     base_page.page_contents_add(new_text_layer, prepend=True)
 
@@ -189,144 +176,109 @@ def _find_font(text, pdf_base):
         return None, None
 
 
-def weave_layers(infiles, output_file, log, context):
-    """Apply text layer and/or image layer changes to baseline file
+class OcrGrafter:
+    def __init__(self, context):
+        self.context = context
+        self.log = context.log
+        self.path_base = Path(context.origin).resolve()
 
-    This is where the magic happens. infiles will be the main PDF to modify,
-    and optional .text.pdf and .image-layer.pdf files, organized however ruffus
-    organizes them.
+        self.pdf_base = pikepdf.open(self.path_base)
+        self.font, self.font_key = None, None
 
-    From .text.pdf, we copy the content stream (which contains the Tesseract
-    OCR results), and rotate it into place. The first time we do this, we also
-    copy the GlyphlessFont, and then reference that font again.
+        self.pdfinfo = context.pdfinfo
+        self.output_file = context.get_path('graft_layers.pdf')
 
-    For .image-layer.pdf, we check if this is a "pointer" to the original file,
-    or a new file. If a new file, we replace the page and remember that we
-    replaced this page.
+        self.procset = self.pdf_base.make_indirect(
+            pikepdf.Object.parse(b'[ /PDF /Text /ImageB /ImageC /ImageI ]')
+        )
 
-    Every 100 open files, we save intermediate results, to avoid any resource
-    limits, since pikepdf/qpdf need to keep a lot of open file handles in the
-    background. When objects are copied from one file to another qpdf, qpdf
-    doesn't actually copy the data until asked to write, so all the resources
-    it may need to remain available.
+        self.emplacements = 1
+        self.interim_count = 0
 
-    For completeness, we set up a /ProcSet on every page, although it's
-    unlikely any PDF viewer cares about this anymore.
-
-    """
-
-    def input_sorter(key):
-        try:
-            return page_number(key)
-        except ValueError:
-            return -1
-
-    flat_inputs = sorted(flatten_groups(infiles), key=input_sorter)
-    groups = groupby(flat_inputs, key=input_sorter)
-
-    # Extract first item
-    _, basegroup = next(groups)
-    base = list(basegroup)[0]
-    path_base = Path(base).resolve()
-    pdf_base = pikepdf.open(path_base)
-    font, font_key, procset = None, None, None
-    pdfinfo = context.get_pdfinfo()
-
-    procset = pdf_base.make_indirect(
-        pikepdf.Object.parse(b'[ /PDF /Text /ImageB /ImageC /ImageI ]')
-    )
-
-    emplacements = 1
-    interim_count = 0
-
-    # Iterate rest
-    for page_num, layers in groups:
-        layers = list(layers)
-        log.debug(page_num)
-        log.debug(layers)
-
-        text = next((ii for ii in layers if ii.endswith('.text.pdf')), None)
-        image = next((ii for ii in layers if ii.endswith('.image-layer.pdf')), None)
-
-        if text and not font:
-            font, font_key = _find_font(text, pdf_base)
+    def graft_page(self, page_result):
+        pageno, image, text, _sidecar, autorotate_correction = page_result
+        if text and not self.font:
+            self.font, self.font_key = _find_font(text, self.pdf_base)
 
         emplaced_page = False
-        content_rotation = pdfinfo[page_num - 1].rotation
-
+        content_rotation = self.pdfinfo[pageno].rotation
         path_image = Path(image).resolve() if image else None
-        if path_image is not None and path_image != path_base:
+        if path_image is not None and path_image != self.path_base:
             # We are updating the old page with a rasterized PDF of the new
             # page (without changing objgen, to preserve references)
-            log.debug("Emplacement update")
+            self.log.debug("Emplacement update")
             with pikepdf.open(image) as pdf_image:
-                emplacements += 1
+                self.emplacements += 1
                 foreign_image_page = pdf_image.pages[0]
-                pdf_base.pages.append(foreign_image_page)
-                local_image_page = pdf_base.pages[-1]
-                pdf_base.pages[page_num - 1].emplace(local_image_page)
-                del pdf_base.pages[-1]
+                self.pdf_base.pages.append(foreign_image_page)
+                local_image_page = self.pdf_base.pages[-1]
+                self.pdf_base.pages[pageno].emplace(local_image_page)
+                del self.pdf_base.pages[-1]
             emplaced_page = True
 
-        autorotate_correction = context.get_rotation(page_num - 1)
         if emplaced_page:
             content_rotation = autorotate_correction
         text_rotation = autorotate_correction
         text_misaligned = (text_rotation - content_rotation) % 360
-        log.debug(
-            '%r',
-            [text_rotation, autorotate_correction, text_misaligned, content_rotation],
+        self.log.debug(
+            f"Rotations for page {pageno}: [text, auto, misalign, content] = "
+            f"{text_rotation}, {autorotate_correction}, "
+            f"{text_misaligned}, {content_rotation}"
         )
 
-        if text and font:
+        if text and self.font:
             # Graft the text layer onto this page, whether new or old
-            strip_old = context.get_options().redo_ocr
-            _weave_layers_graft(
-                pdf_base=pdf_base,
-                page_num=page_num,
+            strip_old = self.context.options.redo_ocr
+            _graft_text_layer(
+                pdf_base=self.pdf_base,
+                page_num=pageno + 1,
                 text=text,
-                font=font,
-                font_key=font_key,
+                font=self.font,
+                font_key=self.font_key,
                 rotation=text_misaligned,
-                procset=procset,
+                procset=self.procset,
                 strip_old_text=strip_old,
-                log=log,
+                log=self.log,
             )
 
         # Correct the rotation if applicable
-        pdf_base.pages[page_num - 1].Rotate = (
+        self.pdf_base.pages[pageno].Rotate = (
             content_rotation - autorotate_correction
         ) % 360
 
-        if emplacements % MAX_REPLACE_PAGES == 0:
-            # Periodically save and reload the Pdf object. This will keep a
-            # lid on our memory usage for very large files. Attach the font to
-            # page 1 even if page 1 doesn't use it, so we have a way to get it
-            # back.
-            # TODO refactor this to outside the loop
-            page0 = pdf_base.pages[0]
-            _update_page_resources(
-                page=page0, font=font, font_key=font_key, procset=procset
-            )
+        if self.emplacements % MAX_REPLACE_PAGES == 0:
+            self.save_and_reload()
 
-            # We cannot read and write the same file, that will corrupt it
-            # but we don't to keep more copies than we need to. Delete intermediates.
-            # {interim_count} is the opened file we were updateing
-            # {interim_count - 1} can be deleted
-            # {interim_count + 1} is the new file will produce and open
-            old_file = output_file + f'_working{interim_count - 1}.pdf'
-            if not context.get_options().keep_temporary_files:
-                with suppress(FileNotFoundError):
-                    os.unlink(old_file)
+    def save_and_reload(self):
+        # Periodically save and reload the Pdf object. This will keep a
+        # lid on our memory usage for very large files. Attach the font to
+        # page 1 even if page 1 doesn't use it, so we have a way to get it
+        # back.
+        page0 = self.pdf_base.pages[0]
+        _update_page_resources(
+            page=page0, font=self.font, font_key=self.font_key, procset=self.procset
+        )
 
-            next_file = output_file + f'_working{interim_count + 1}.pdf'
-            pdf_base.save(next_file)
-            pdf_base.close()
+        # We cannot read and write the same file, that will corrupt it
+        # but we don't to keep more copies than we need to. Delete intermediates.
+        # {interim_count} is the opened file we were updateing
+        # {interim_count - 1} can be deleted
+        # {interim_count + 1} is the new file will produce and open
+        old_file = self.output_file + f'_working{self.interim_count - 1}.pdf'
+        if not self.context.options.keep_temporary_files:
+            with suppress(FileNotFoundError):
+                os.unlink(old_file)
 
-            pdf_base = pikepdf.open(next_file)
-            procset = pdf_base.pages[0].Resources.ProcSet
-            font, font_key = None, None  # Ensure we reacquire this information
-            interim_count += 1
+        next_file = self.output_file + f'_working{self.interim_count + 1}.pdf'
+        self.pdf_base.save(next_file)
+        self.pdf_base.close()
 
-    pdf_base.save(output_file)
-    pdf_base.close()
+        self.pdf_base = pikepdf.open(next_file)
+        self.procset = self.pdf_base.pages[0].Resources.ProcSet
+        self.font, self.font_key = None, None  # Ensure we reacquire this information
+        self.interim_count += 1
+
+    def finalize(self):
+        self.pdf_base.save(self.output_file)
+        self.pdf_base.close()
+        return self.output_file

src/ocrmypdf/_jobcontext.py

@@ -15,69 +15,110 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
+import logging
 import shutil
 import sys
-from contextlib import suppress
-from multiprocessing.managers import SyncManager
-
-from .pdfinfo import PdfInfo
+import os
 
 
-class JobContext:
-    """Holds our context for a particular run of the pipeline
+class PicklableLoggerMixin:
+    def __init__(self):
+        self._log = None
 
-    A multiprocessing manager effectively creates a separate process
-    that keeps the master job context object.  Other threads access
-    job context via multiprocessing proxy objects.
+    @property
+    def log(self):
+        if not self._log:
+            self._log = self.get_logger()
+        return self._log
 
-    While this would naturally lend itself @property's it seems to make
-    a little more sense to use functions to make it explicitly that the
-    invocation requires marshalling data across a process boundary.
+    def __getstate__(self):
+        # Python 3.6 is incapable of pickling a logger and marshalling it to another
+        # process (threading._RLock error), so we disconnect it before pickling,
+        # and create a new logger in the worker process.
+        state = self.__dict__.copy()
+        state['_log'] = None
+        return state
 
+
+class PDFContext(PicklableLoggerMixin):
+    """Holds our context for a particular run of the pipeline"""
+
+    def __init__(self, options, work_folder, origin, pdfinfo):
+        PicklableLoggerMixin.__init__(self)
+        self.options = options
+        self.work_folder = work_folder
+        self.origin = origin
+        self.pdfinfo = pdfinfo
+        if options:
+            self.name = os.path.basename(options.input_file)
+        else:
+            self.name = 'origin.pdf'
+        if self.name == '-':
+            self.name = 'stdin'
+
+    def get_logger(self):
+        return make_logger(self.options, filename=self.name)
+
+    def get_path(self, name):
+        return os.path.join(self.work_folder, name)
+
+    def get_page_contexts(self):
+        npages = len(self.pdfinfo)
+        for n in range(npages):
+            yield PageContext(self, n)
+
+
+class PageContext(PicklableLoggerMixin):
+    """Holds our context for a page
+
+    Must be pickable, so only store intrinsic/simple data elements
     """
 
-    def __init__(self):
-        self.pdfinfo = None
-        self.options = None
-        self.work_folder = None
-        self.rotations = {}
+    def __init__(self, pdf_context, pageno):
+        PicklableLoggerMixin.__init__(self)
+        self.work_folder = pdf_context.work_folder
+        self.origin = pdf_context.origin
+        self.options = pdf_context.options
+        self.name = pdf_context.name
+        self.pageno = pageno
+        self.pageinfo = pdf_context.pdfinfo[pageno]
+        self._log = None
 
-    def generate_pdfinfo(self, infile):
-        self.pdfinfo = PdfInfo(infile)
+    def get_logger(self):
+        return make_logger(self.options, filename=self.name, page=self.pageno + 1)
 
-    def get_pdfinfo(self):
-        "What we know about the input PDF"
-        return self.pdfinfo
-
-    def set_pdfinfo(self, pdfinfo):
-        self.pdfinfo = pdfinfo
-
-    def get_options(self):
-        return self.options
-
-    def set_options(self, options):
-        self.options = options
-
-    def get_work_folder(self):
-        return self.work_folder
-
-    def set_work_folder(self, work_folder):
-        self.work_folder = work_folder
-
-    def get_rotation(self, pageno):
-        return self.rotations.get(pageno, 0)
-
-    def set_rotation(self, pageno, value):
-        self.rotations[pageno] = value
-
-
-class JobContextManager(SyncManager):
-    pass
+    def get_path(self, name):
+        return os.path.join(self.work_folder, "%06d_%s" % (self.pageno + 1, name))
 
 
 def cleanup_working_files(work_folder, options):
     if options.keep_temporary_files:
-        print(f"Temporary working files saved at:\n{work_folder}", file=sys.stderr)
+        print(f"Temporary working files retained at:\n{work_folder}", file=sys.stderr)
     else:
-        with suppress(FileNotFoundError):
-            shutil.rmtree(work_folder)
+        shutil.rmtree(work_folder, ignore_errors=True)
+
+
+class LogNameAdapter(logging.LoggerAdapter):
+    def process(self, msg, kwargs):
+        # return '[%s] %s' % (self.extra['filename'], msg), kwargs
+        return '%s' % (msg,), kwargs
+
+
+class LogNamePageAdapter(logging.LoggerAdapter):
+    def process(self, msg, kwargs):
+        return (
+            #'[%s:%05u] %s' % (self.extra['filename'], self.extra['page'], msg),
+            '%4u: %s' % (self.extra['page'], msg),
+            kwargs,
+        )
+
+
+def make_logger(options=None, prefix='ocrmypdf', filename=None, page=None):
+    log = logging.getLogger(prefix)
+    if filename and page:
+        adapter = LogNamePageAdapter(log, dict(filename=filename, page=page))
+    elif filename:
+        adapter = LogNameAdapter(log, dict(filename=filename))
+    else:
+        adapter = log
+    return adapter

src/ocrmypdf/_sync.py

@@ -0,0 +1,381 @@
+# © 2016 James R. Barlow: github.com/jbarlow83
+#
+# This file is part of OCRmyPDF.
+#
+# OCRmyPDF is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# OCRmyPDF is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+import logging.handlers
+import multiprocessing
+import os
+import signal
+import sys
+import threading
+from collections import namedtuple
+from tempfile import mkdtemp
+
+from tqdm import tqdm
+
+from . import __version__
+from ._jobcontext import PDFContext, cleanup_working_files, make_logger
+from ._pipeline import (
+    convert_to_pdfa,
+    copy_final,
+    create_ocr_image,
+    create_pdf_page_from_image,
+    create_visible_page_jpg,
+    generate_postscript_stub,
+    get_orientation_correction,
+    get_pdfinfo,
+    is_ocr_required,
+    merge_sidecars,
+    metadata_fixup,
+    ocr_tesseract_hocr,
+    ocr_tesseract_textonly_pdf,
+    optimize_pdf,
+    preprocess_clean,
+    preprocess_deskew,
+    preprocess_remove_background,
+    rasterize,
+    rasterize_preview,
+    render_hocr_page,
+    should_visible_page_image_use_jpg,
+    triage,
+    validate_pdfinfo_options,
+)
+from ._validation import (
+    check_requested_output_file,
+    create_input_file,
+    report_output_file_size,
+)
+from ._graft import OcrGrafter
+from .exceptions import ExitCode, ExitCodeException
+from .exec import qpdf
+from .helpers import available_cpu_count
+from .pdfa import file_claims_pdfa
+
+PageResult = namedtuple(
+    'PageResult', 'pageno, pdf_page_from_image, ocr, text, orientation_correction'
+)
+
+
+def preprocess(page_context, image, remove_background, deskew, clean):
+    if remove_background:
+        image = preprocess_remove_background(image, page_context)
+    if deskew:
+        image = preprocess_deskew(image, page_context)
+    if clean:
+        image = preprocess_clean(image, page_context)
+    return image
+
+
+def exec_page_sync(page_context):
+    options = page_context.options
+    orientation_correction = 0
+    pdf_page_from_image_out = None
+    ocr_out = None
+    text_out = None
+    if is_ocr_required(page_context):
+        if options.rotate_pages:
+            # Rasterize
+            rasterize_preview_out = rasterize_preview(page_context.origin, page_context)
+            orientation_correction = get_orientation_correction(
+                rasterize_preview_out, page_context
+            )
+
+        rasterize_out = rasterize(
+            page_context.origin,
+            page_context,
+            correction=orientation_correction,
+            remove_vectors=False,
+        )
+
+        if not any([options.clean, options.clean_final, options.remove_vectors]):
+            ocr_image = preprocess_out = preprocess(
+                page_context,
+                rasterize_out,
+                options.remove_background,
+                options.deskew,
+                clean=False,
+            )
+        else:
+            if not options.lossless_reconstruction:
+                preprocess_out = preprocess(
+                    page_context,
+                    rasterize_out,
+                    options.remove_background,
+                    options.deskew,
+                    clean=options.clean_final,
+                )
+            if options.remove_vectors:
+                rasterize_ocr_out = rasterize(
+                    page_context.origin,
+                    page_context,
+                    correction=orientation_correction,
+                    remove_vectors=True,
+                    output_tag='_ocr',
+                )
+            else:
+                rasterize_ocr_out = rasterize_out
+            ocr_image = preprocess(
+                page_context,
+                rasterize_ocr_out,
+                options.remove_background,
+                options.deskew,
+                clean=options.clean,
+            )
+
+        ocr_image_out = create_ocr_image(ocr_image, page_context)
+
+        pdf_page_from_image_out = None
+        if not options.lossless_reconstruction:
+            visible_image_out = preprocess_out
+            if should_visible_page_image_use_jpg(page_context.pageinfo):
+                visible_image_out = create_visible_page_jpg(
+                    visible_image_out, page_context
+                )
+            pdf_page_from_image_out = create_pdf_page_from_image(
+                visible_image_out, page_context
+            )
+
+        if options.pdf_renderer == 'hocr':
+            (hocr_out, text_out) = ocr_tesseract_hocr(ocr_image_out, page_context)
+            ocr_out = render_hocr_page(hocr_out, page_context)
+
+        if options.pdf_renderer == 'sandwich':
+            (ocr_out, text_out) = ocr_tesseract_textonly_pdf(
+                ocr_image_out, page_context
+            )
+
+    return PageResult(
+        pageno=page_context.pageno,
+        pdf_page_from_image=pdf_page_from_image_out,
+        ocr=ocr_out,
+        text=text_out,
+        orientation_correction=orientation_correction,
+    )
+
+
+def post_process(pdf_file, context):
+    pdf_out = pdf_file
+    if context.options.output_type.startswith('pdfa'):
+        ps_stub_out = generate_postscript_stub(context)
+        pdf_out = convert_to_pdfa(pdf_out, ps_stub_out, context)
+
+    pdf_out = metadata_fixup(pdf_out, context)
+    return optimize_pdf(pdf_out, context)
+
+
+def worker_init(queue):
+    """Initialize a process pool worker"""
+
+    # Ignore SIGINT (our parent process will kill us gracefully)
+    signal.signal(signal.SIGINT, signal.SIG_IGN)
+
+    # Reconfigure the root logger for this process to send all messages to a queue
+    h = logging.handlers.QueueHandler(queue)
+    root = logging.getLogger()
+    root.handlers = []
+    root.addHandler(h)
+
+
+def worker_thread_init(queue):
+    pass
+
+
+def log_listener(queue):
+    """Listen to the worker processes and forward the messages to logging
+
+    For simplicity this is a thread rather than a process. Only one process
+    should actually write to sys.stderr or whatever we're using, so if this is
+    made into a process the main application needs to be directed to it.
+
+    See https://docs.python.org/3/howto/logging-cookbook.html#logging-to-a-single-file-from-multiple-processes
+    """
+
+    while True:
+        try:
+            record = queue.get()
+            if record is None:
+                break
+            logger = logging.getLogger(record.name)
+            logger.handle(record)
+        except Exception:
+            import traceback
+
+            print("Logging problem", file=sys.stderr)
+            traceback.print_exc(file=sys.stderr)
+
+
+def exec_concurrent(context):
+    """Execute the pipeline concurrently"""
+
+    # Run exec_page_sync on every page context
+    max_workers = min(len(context.pdfinfo), context.options.jobs)
+    if max_workers > 1:
+        context.log.info("Start processing %d pages concurrent", max_workers)
+
+    # Tesseract 4.0 is multithreaded, and we also run multiple workers. We want to
+    # avoid the situation where we end up trying to run NxN jobs on N CPU cores,
+    # as that gives poor performance. Performance testing shows we're better off
+    # parallelizing ocrmypdf and forcing Tesseract to be single threaded, which we
+    # get by setting the envvar OMP_THREAD_LIMIT to 1. But if the page count of the
+    # input file is small, then we allow Tesseract to use threads, subject to the
+    # constraint: (ocrmypdf workers) * (tesseract threads) <= max_workers and limiting
+    # Tesseract to 4 threads.
+    tess_threads = min(4, context.options.jobs // max_workers)
+    if context.options.tesseract_env is None:
+        context.options.tesseract_env = os.environ.copy()
+    context.options.tesseract_env.setdefault('OMP_THREAD_LIMIT', str(tess_threads))
+    if tess_threads > 1:
+        context.log.info("Using Tesseract OpenMP thread limit %d", tess_threads)
+
+    if context.options.use_threads:
+        from multiprocessing.dummy import Pool
+
+        initializer = worker_thread_init
+    else:
+        Pool = multiprocessing.Pool
+        initializer = worker_init
+
+    sidecars = [None] * len(context.pdfinfo)
+    ocrgraft = OcrGrafter(context)
+
+    log_queue = multiprocessing.Queue(-1)
+    listener = threading.Thread(target=log_listener, args=(log_queue,))
+    listener.start()
+    with tqdm(
+        total=(2 * len(context.pdfinfo)),
+        desc='OCR',
+        unit='page',
+        unit_scale=0.5,
+        disable=not context.options.progress_bar,
+    ) as pbar, Pool(
+        processes=max_workers, initializer=initializer, initargs=(log_queue,)
+    ) as pool:
+        results = pool.imap_unordered(exec_page_sync, context.get_page_contexts())
+        while True:
+            try:
+                page_result = results.next()
+                sidecars[page_result.pageno] = page_result.text
+                pbar.update()
+                ocrgraft.graft_page(page_result)
+                pbar.update()
+            except StopIteration:
+                break
+            except (Exception, KeyboardInterrupt):
+                pool.terminate()
+                log_queue.put_nowait(None)  # Terminate log listener
+                # Don't try listener.join() here, will deadlock
+                raise
+
+    log_queue.put_nowait(None)
+    listener.join()
+
+    # Output sidecar text
+    if context.options.sidecar:
+        text = merge_sidecars(sidecars, context)
+        # Copy text file to destination
+        copy_final(text, context.options.sidecar, context)
+
+    # Merge layers to one single pdf
+    pdf = ocrgraft.finalize()
+
+    # PDF/A and metadata
+    pdf = post_process(pdf, context)
+
+    # Copy PDF file to destination
+    copy_final(pdf, context.options.output_file, context)
+
+
+class NeverRaise(Exception):
+    """An exception that is never raised"""
+
+    pass
+
+
+def run_pipeline(options, api=False):
+    log = make_logger(options, __name__)
+
+    # Any changes to options will not take effect for options that are already
+    # bound to function parameters in the pipeline. (For example
+    # options.input_file, options.pdf_renderer are already bound.)
+    if not options.jobs:
+        options.jobs = available_cpu_count()
+
+    work_folder = mkdtemp(prefix="com.github.ocrmypdf.")
+    try:
+        check_requested_output_file(options)
+        start_input_file = create_input_file(options, work_folder)
+
+        # Triage image or pdf
+        origin_pdf = triage(
+            start_input_file, os.path.join(work_folder, 'origin.pdf'), options, log
+        )
+
+        # Gather pdfinfo and create context
+        pdfinfo = get_pdfinfo(
+            origin_pdf,
+            detailed_page_analysis=options.redo_ocr,
+            progbar=options.progress_bar,
+        )
+        context = PDFContext(options, work_folder, origin_pdf, pdfinfo)
+
+        # Validate options are okay for this pdf
+        validate_pdfinfo_options(context)
+
+        # Execute the pipeline
+        exec_concurrent(context)
+
+        if options.output_file == '-':
+            log.info("Output sent to stdout")
+        elif os.path.samefile(options.output_file, os.devnull):
+            pass  # Say nothing when sending to dev null
+        else:
+            if options.output_type.startswith('pdfa'):
+                pdfa_info = file_claims_pdfa(options.output_file)
+                if pdfa_info['pass']:
+                    log.info(
+                        "Output file is a %s (as expected)", pdfa_info['conformance']
+                    )
+                else:
+                    log.warning(
+                        "Output file is okay but is not PDF/A (seems to be %s)",
+                        pdfa_info['conformance'],
+                    )
+                    return ExitCode.pdfa_conversion_failed
+            if not qpdf.check(options.output_file, log):
+                log.warning('Output file: The generated PDF is INVALID')
+                return ExitCode.invalid_output_pdf
+            report_output_file_size(options, start_input_file, options.output_file)
+
+    except (KeyboardInterrupt if not api else NeverRaise) as e:
+        if options.verbose >= 1:
+            log.exception("KeyboardInterrupt")
+        else:
+            log.error("KeyboardInterrupt")
+        return ExitCode.ctrl_c
+    except (ExitCodeException if not api else NeverRaise) as e:
+        if str(e):
+            log.error("%s: %s", type(e).__name__, str(e))
+        else:
+            log.error(type(e).__name__)
+        return e.exit_code
+    except (Exception if not api else NeverRaise) as e:
+        log.exception("An exception occurred while executing the pipeline")
+        return ExitCode.other_error
+    finally:
+        cleanup_working_files(work_folder, options)
+
+    return ExitCode.ok

src/ocrmypdf/_validation.py

@@ -0,0 +1,445 @@
+#!/usr/bin/env python3
+# © 2015-17 James R. Barlow: github.com/jbarlow83
+#
+# This file is part of OCRmyPDF.
+#
+# OCRmyPDF is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# OCRmyPDF is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import logging
+import os
+import sys
+from pathlib import Path
+from shutil import copyfileobj
+
+import PIL
+
+from ._unicodefun import verify_python3_env
+from .exceptions import (
+    BadArgsError,
+    InputFileError,
+    MissingDependencyError,
+    OutputFileAccessError,
+)
+from .exec import (
+    check_external_program,
+    ghostscript,
+    jbig2enc,
+    pngquant,
+    qpdf,
+    tesseract,
+    unpaper,
+)
+from .helpers import is_file_writable, re_symlink, is_iterable_notstr, monotonic
+
+# -------------
+# External dependencies
+
+HOCR_OK_LANGS = frozenset(['eng', 'deu', 'spa', 'ita', 'por'])
+
+log = logging.getLogger(__name__)
+
+
+# --------
+# Critical environment tests
+verify_python3_env()
+
+
+def check_options_languages(options):
+    if not options.language:
+        options.language = ['eng']  # Enforce English hegemony
+
+    # Support v2.x "eng+deu" language syntax
+    if '+' in options.language[0]:
+        options.language = options.language[0].split('+')
+
+    languages = set(options.language)
+    if not languages.issubset(tesseract.languages()):
+        msg = (
+            "The installed version of tesseract does not have language "
+            "data for the following requested languages: \n"
+        )
+        for lang in languages - tesseract.languages():
+            msg += lang + '\n'
+        raise MissingDependencyError(msg)
+
+
+def check_options_output(options):
+    # We have these constraints to check for.
+    # 1. Ghostscript < 9.20 mangles multibyte Unicode
+    # 2. hocr doesn't work on non-Latin languages (so don't select it)
+
+    languages = set(options.language)
+    is_latin = languages.issubset(HOCR_OK_LANGS)
+
+    if options.pdf_renderer == 'hocr' and not is_latin:
+        msg = (
+            "The 'hocr' PDF renderer is known to cause problems with one "
+            "or more of the languages in your document.  Use "
+            "--pdf-renderer auto (the default) to avoid this issue."
+        )
+        log.warning(msg)
+
+    if ghostscript.version() < '9.20' and options.output_type != 'pdf' and not is_latin:
+        # https://bugs.ghostscript.com/show_bug.cgi?id=696874
+        # Ghostscript < 9.20 fails to encode multibyte characters properly
+        msg = (
+            "The installed version of Ghostscript does not work correctly "
+            "with the OCR languages you specified. Use --output-type pdf or "
+            "upgrade to Ghostscript 9.20 or later to avoid this issue."
+        )
+        msg += f"Found Ghostscript {ghostscript.version()}"
+        log.warning(msg)
+
+    # Decide on what renderer to use
+    if options.pdf_renderer == 'auto':
+        options.pdf_renderer = 'sandwich'
+
+    if options.pdf_renderer == 'sandwich' and not tesseract.has_textonly_pdf(
+        options.tesseract_env
+    ):
+        raise MissingDependencyError(
+            "You are using an alpha version of Tesseract 4.0 that does not support "
+            "the textonly_pdf parameter. We don't support versions this old."
+        )
+
+    if options.output_type == 'pdfa':
+        options.output_type = 'pdfa-2'
+
+    if options.output_type == 'pdfa-3' and ghostscript.version() < '9.19':
+        raise MissingDependencyError(
+            "--output-type pdfa-3 requires Ghostscript 9.19 or later"
+        )
+
+    lossless_reconstruction = False
+    if not any(
+        (
+            options.deskew,
+            options.clean_final,
+            options.force_ocr,
+            options.remove_background,
+        )
+    ):
+        lossless_reconstruction = True
+    options.lossless_reconstruction = lossless_reconstruction
+
+    if not options.lossless_reconstruction and options.redo_ocr:
+        raise BadArgsError(
+            "--redo-ocr is not currently compatible with --deskew, "
+            "--clean-final, and --remove-background"
+        )
+
+
+def check_options_sidecar(options):
+    if options.sidecar == '\0':
+        if options.output_file == '-':
+            raise BadArgsError(
+                "--sidecar filename must be specified when output file is stdout."
+            )
+        options.sidecar = options.output_file + '.txt'
+
+
+def check_options_preprocessing(options):
+    if options.clean_final:
+        options.clean = True
+    if options.unpaper_args and not options.clean:
+        raise BadArgsError("--clean is required for --unpaper-args")
+    if options.clean:
+        check_external_program(
+            program='unpaper',
+            package='unpaper',
+            version_checker=unpaper.version,
+            need_version='6.1',
+            required_for=['--clean, --clean-final'],
+        )
+        try:
+            if options.unpaper_args:
+                options.unpaper_args = unpaper.validate_custom_args(
+                    options.unpaper_args
+                )
+        except Exception as e:
+            raise BadArgsError(str(e))
+
+
+def _pages_from_ranges(ranges):
+    if is_iterable_notstr(ranges):
+        return set(ranges)
+    pages = []
+    page_groups = ranges.replace(' ', '').split(',')
+    for g in page_groups:
+        if not g:
+            continue
+        try:
+            start, end = g.split('-')
+        except ValueError:
+            pages.append(int(g) - 1)
+        else:
+            pages.extend(range(int(start) - 1, int(end)))
+
+    if not monotonic(pages):
+        log.warning(
+            "List of pages to process contains duplicate pages, or pages that are "
+            "out of order"
+        )
+    if any(page < 0 for page in pages):
+        raise BadArgsError("pages refers to a page number less than 1")
+
+    log.debug("OCRing only these pages: %s", pages)
+    return set(pages)
+
+
+def check_options_ocr_behavior(options):
+    exclusive_options = sum(
+        [
+            (1 if opt else 0)
+            for opt in (options.force_ocr, options.skip_text, options.redo_ocr)
+        ]
+    )
+    if exclusive_options >= 2:
+        raise BadArgsError("Choose only one of --force-ocr, --skip-text, --redo-ocr.")
+    if options.pages and options.sidecar:
+        raise BadArgsError("--pages and --sidecar are mutually exclusive")
+    if options.pages:
+        options.pages = _pages_from_ranges(options.pages)
+
+
+def check_options_optimizing(options):
+    if options.optimize >= 2:
+        check_external_program(
+            program='pngquant',
+            package='pngquant',
+            version_checker=pngquant.version,
+            need_version='2.0.1',
+            required_for='--optimize {2,3}',
+        )
+
+    if options.optimize >= 2:
+        # Although we use JBIG2 for optimize=1, don't nag about it unless the
+        # user is asking for more optimization
+        check_external_program(
+            program='jbig2',
+            package='jbig2enc',
+            version_checker=jbig2enc.version,
+            need_version='0.28',
+            required_for='--optimize {2,3} | --jbig2-lossy',
+            recommended=True if not options.jbig2_lossy else False,
+        )
+
+    if options.optimize == 0 and any(
+        [options.jbig2_lossy, options.png_quality, options.jpeg_quality]
+    ):
+        log.warning(
+            "The arguments --jbig2-lossy, --png-quality, and --jpeg-quality "
+            "will be ignored because --optimize=0."
+        )
+
+
+def check_options_advanced(options):
+    if options.pdfa_image_compression != 'auto' and options.output_type.startswith(
+        'pdfa'
+    ):
+        log.warning(
+            "--pdfa-image-compression argument has no effect when "
+            "--output-type is not 'pdfa', 'pdfa-1', or 'pdfa-2'"
+        )
+    if tesseract.v4(options.tesseract_env) and (
+        options.user_words or options.user_patterns
+    ):
+        log.warning('Tesseract 4.x ignores --user-words, so this has no effect')
+
+
+def check_options_metadata(options):
+    import unicodedata
+
+    docinfo = [options.title, options.author, options.keywords, options.subject]
+    for s in (m for m in docinfo if m):
+        for c in s:
+            if unicodedata.category(c) == 'Co' or ord(c) >= 0x10000:
+                raise ValueError(
+                    "One of the metadata strings contains "
+                    "an unsupported Unicode character: '{}' (U+{})".format(
+                        c, hex(ord(c))[2:].upper()
+                    )
+                )
+
+
+def check_options_pillow(options):
+    PIL.Image.MAX_IMAGE_PIXELS = int(options.max_image_mpixels * 1_000_000)
+    if PIL.Image.MAX_IMAGE_PIXELS == 0:
+        PIL.Image.MAX_IMAGE_PIXELS = None
+
+
+def check_options(options):
+    check_options_languages(options)
+    check_options_metadata(options)
+    check_options_output(options)
+    check_options_sidecar(options)
+    check_options_preprocessing(options)
+    check_options_ocr_behavior(options)
+    check_options_optimizing(options)
+    check_options_advanced(options)
+    check_options_pillow(options)
+    check_dependency_versions(options)
+
+
+def check_closed_streams(options):
+    """Work around Python issue with multiprocessing forking on closed streams
+
+    https://bugs.python.org/issue28326
+
+    Attempting to a fork/exec a new Python process when any of std{in,out,err}
+    are closed or not flushable for some reason may raise an exception.
+    Fix this by opening devnull if the handle seems to be closed.  Do this
+    globally to avoid tracking places all places that fork.
+
+    Seems to be specific to multiprocessing.Process not all Python process
+    forkers.
+
+    The error actually occurs when the stream object is not flushable,
+    but replacing an open stream object that is not flushable with
+    /dev/null is a bad idea since it will create a silent failure.  Replacing
+    a closed handle with /dev/null seems safe.
+
+    """
+
+    if sys.version_info[0:3] >= (3, 6, 4):
+        return True  # Issued fixed in Python 3.6.4+
+
+    if sys.stderr is None:
+        sys.stderr = open(os.devnull, 'w')
+
+    if sys.stdin is None:
+        if options.input_file == '-':
+            log.error("Trying to read from stdin but stdin seems closed")
+            return False
+        sys.stdin = open(os.devnull, 'r')
+
+    if sys.stdout is None:
+        if options.output_file == '-':
+            # Can't replace stdout if the user is piping
+            # If this case can even happen, it must be some kind of weird
+            # stream.
+            log.error(
+                "Output was set to stdout '-' but the stream attached to "
+                "stdout does not support the flush() system call.  This "
+                "will fail."
+            )
+            return False
+        sys.stdout = open(os.devnull, 'w')
+
+    return True
+
+
+def log_page_orientations(pdfinfo):
+    direction = {0: 'n', 90: 'e', 180: 's', 270: 'w'}
+    orientations = []
+    for n, page in enumerate(pdfinfo):
+        angle = page.rotation or 0
+        if angle != 0:
+            orientations.append('{0}{1}'.format(n + 1, direction.get(angle, '')))
+    if orientations:
+        log.info('Page orientations detected: %s', ' '.join(orientations))
+
+
+def create_input_file(options, work_folder):
+    if options.input_file == '-':
+        # stdin
+        log.info('reading file from standard input')
+        target = os.path.join(work_folder, 'stdin')
+        with open(target, 'wb') as stream_buffer:
+            copyfileobj(sys.stdin.buffer, stream_buffer)
+        return target
+    else:
+        try:
+            target = os.path.join(work_folder, 'origin')
+            re_symlink(options.input_file, target)
+            return target
+        except FileNotFoundError:
+            raise InputFileError(f"File not found - {options.input_file}")
+
+
+def check_requested_output_file(options):
+    if options.output_file == '-':
+        if sys.stdout.isatty():
+            raise BadArgsError(
+                "Output was set to stdout '-' but it looks like stdout "
+                "is connected to a terminal.  Please redirect stdout to a "
+                "file."
+            )
+    elif not is_file_writable(options.output_file):
+        raise OutputFileAccessError(
+            f"Output file location ({options.output_file}) is not a writable file."
+        )
+
+
+def report_output_file_size(options, input_file, output_file):
+    try:
+        output_size = Path(output_file).stat().st_size
+        input_size = Path(input_file).stat().st_size
+    except FileNotFoundError:
+        return  # Outputting to stream or something
+    ratio = output_size / input_size
+    if ratio < 1.35 or input_size < 25000:
+        return  # Seems fine
+
+    reasons = []
+    image_preproc = {
+        'deskew',
+        'clean_final',
+        'remove_background',
+        'oversample',
+        'force_ocr',
+    }
+    for arg in image_preproc:
+        if getattr(options, arg, False):
+            reasons.append(
+                f"The argument --{arg.replace('_', '-')} was issued, causing transcoding."
+            )
+
+    if reasons:
+        explanation = "Possible reasons for this include:\n" + '\n'.join(reasons) + "\n"
+    else:
+        explanation = "No reason for this increase is known.  Please report this issue."
+
+    log.warning(
+        f"The output file size is {ratio:.2f}× larger than the input file.\n"
+        f"{explanation}"
+    )
+
+
+def check_dependency_versions(options):
+    check_external_program(
+        program='tesseract',
+        package={'darwin': 'tesseract', 'linux': 'tesseract-ocr'},
+        version_checker=tesseract.version,
+        need_version='4.0.0',  # using backport for Travis CI
+    )
+    check_external_program(
+        program='gs',
+        package='ghostscript',
+        version_checker=ghostscript.version,
+        need_version='9.15',  # limited by Travis CI / Ubuntu 14.04 backports
+    )
+    if ghostscript.version() == '9.24':
+        raise MissingDependencyError(
+            "Ghostscript 9.24 contains serious regressions and is not "
+            "supported. Please upgrade to Ghostscript 9.25 or use an older "
+            "version."
+        )
+    check_external_program(
+        program='qpdf',
+        package='qpdf',
+        version_checker=qpdf.version,
+        need_version='8.0.2',
+    )

src/ocrmypdf/api.py

@@ -0,0 +1,239 @@
+# © 2019 James R. Barlow: github.com/jbarlow83
+#
+# This file is part of OCRmyPDF.
+#
+# OCRmyPDF is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# OCRmyPDF is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+import os
+import sys
+from enum import IntEnum
+from pathlib import Path
+
+from tqdm import tqdm
+
+from .cli import parser
+from ._sync import run_pipeline
+from ._validation import check_options
+
+
+class TqdmConsole:
+    """Wrapper to log messages in a way that is compatible with tqdm progress bar"""
+
+    def __init__(self, file):
+        self.file = file
+        self.py36 = sys.version_info >= (3, 6)
+
+    def write(self, msg):
+        # When no progress bar is active, tqdm.write() routes to print()
+        if self.py36:
+            if msg.strip() != '':
+                tqdm.write(msg.rstrip(), end='\n', file=self.file)
+        else:
+            tqdm.write(msg.rstrip(), end='\n', file=self.file)
+
+    def flush(self):
+        if hasattr(self.file, "flush"):
+            self.file.flush()
+
+
+class Verbosity(IntEnum):
+    """Verbosity level for configure_logging."""
+
+    quiet = -1  #: Suppress most messages
+    default = 0  #: Default level of logging
+    debug = 1  #: Output ocrmypdf debug messages
+    debug_all = 2  #: More detailed debugging from ocrmypdf and dependent modules
+
+
+def configure_logging(verbosity, progress_bar_friendly=True, manage_root_logger=False):
+    """Set up logging.
+
+    Library users may wish to use this function if they want their log output to be
+    similar to ocrmypdf command line interface. If not used, the external application
+    should configure logging on its own.
+
+    ocrmypdf will perform all of its logging under the `"ocrmypdf"` logging namespace.
+    In addition, ocrmypdf imports pdfminer, which logs under `"pdfminer"`. A library
+    user may wish to configure both; note that pdfminer is extremely chatty at the log
+    level logging.INFO.
+
+    Library users may perform additional configuration afterwards.
+
+    Args:
+        verbosity (Verbosity): Verbosity level.
+        progress_bar_friendly (bool): Install the TqdmConsole log handler, which is
+            compatible with the tqdm progress bar; without this log messages will
+            overwrite the progress bar
+        manage_root_logger (bool): Configure the process's root logger, to ensure
+            all log output is sent through
+    """
+
+    prefix = '' if manage_root_logger else 'ocrmypdf'
+    log = logging.getLogger(prefix)
+    log.setLevel(logging.INFO)
+
+    if progress_bar_friendly:
+        console = logging.StreamHandler(stream=TqdmConsole(sys.stderr))
+    else:
+        console = logging.StreamHandler(stream=sys.stderr)
+
+    if verbosity < 0:
+        console.setLevel(logging.ERROR)
+    elif verbosity >= 1:
+        console.setLevel(logging.DEBUG)
+    else:
+        console.setLevel(logging.INFO)
+
+    formatter = logging.Formatter('%(levelname)7s - %(message)s')
+    if verbosity >= 1:
+        log.setLevel(logging.DEBUG)
+    if verbosity >= 2:
+        formatter = logging.Formatter('%(name)s - %(levelname)7s - %(message)s')
+
+    console.setFormatter(formatter)
+    log.addHandler(console)
+
+    if verbosity <= 1:
+        pdfminer_log = logging.getLogger('pdfminer')
+        pdfminer_log.setLevel(logging.ERROR)
+        pil_log = logging.getLogger('PIL')
+        pil_log.setLevel(logging.INFO)
+
+    if manage_root_logger:
+        logging.captureWarnings(True)
+
+
+def create_options(*, input_file, output_file, **kwargs):
+    cmdline = []
+    deferred = []
+
+    for arg, val in kwargs.items():
+        if val is None:
+            continue
+        if arg == 'tesseract_env':
+            deferred.append((arg, val))
+            continue
+        cmd_style_arg = arg.replace('_', '-')
+        cmdline.append(f"--{cmd_style_arg}")
+        if isinstance(val, bool):
+            continue
+        if isinstance(val, (int, float)):
+            cmdline.append(str(val))
+        elif isinstance(val, str):
+            cmdline.append(val)
+        elif isinstance(val, Path):
+            cmdline.append(str(val))
+        else:
+            raise TypeError(f"{arg}: {val} ({type(val)})")
+
+    cmdline.append(str(input_file))
+    cmdline.append(str(output_file))
+
+    parser.api_mode = True
+    options = parser.parse_args(cmdline)
+    for keyword, val in deferred:
+        setattr(options, keyword, val)
+
+    # If we are running a Tesseract spoof, ensure it knows what the input file is
+    if os.environ.get('PYTEST_CURRENT_TEST') and options.tesseract_env:
+        options.tesseract_env['_OCRMYPDF_TEST_INFILE'] = input_file
+
+    return options
+
+
+def ocr(  # pylint: disable=unused-argument
+    input_file,
+    output_file,
+    *,
+    language=None,
+    image_dpi=None,
+    output_type=None,
+    sidecar=None,
+    jobs=None,
+    use_threads=None,
+    title=None,
+    author=None,
+    subject=None,
+    keywords=None,
+    rotate_pages=None,
+    remove_background=None,
+    deskew=None,
+    clean=None,
+    clean_final=None,
+    unpaper_args=None,
+    oversample=None,
+    remove_vectors=None,
+    threshold=None,
+    force_ocr=None,
+    skip_text=None,
+    redo_ocr=None,
+    skip_big=None,
+    optimize=None,
+    jpg_quality=None,
+    png_quality=None,
+    jbig2_lossy=None,
+    jbig2_page_group_size=None,
+    pages=None,
+    max_image_mpixels=None,
+    tesseract_config=None,
+    tesseract_pagesegmode=None,
+    tesseract_oem=None,
+    pdf_renderer=None,
+    tesseract_timeout=None,
+    rotate_pages_threshold=None,
+    pdfa_image_compression=None,
+    user_words=None,
+    user_patterns=None,
+    keep_temporary_files=None,
+    progress_bar=None,
+    tesseract_env=None,
+):
+    """Run OCRmyPDF on one PDF or image.
+
+    For most arguments, see documentation for the equivalent command line parameter.
+    A few specific arguments are discussed here:
+
+    Args:
+        use_threads (bool): Use worker threads instead of processes. This reduces
+            performance but may make debugging easier since it is easier to set
+            breakpoints.
+        tesseract_env (dict): Override environment variables for Tesseract
+    Raises:
+        ocrmypdf.PdfMergeFailedError: If the input PDF is malformed, preventing merging
+            with the OCR layer.
+        ocrmypdf.MissingDependencyError: If a required dependency program is missing or
+            was not found on PATH.
+        ocrmypdf.UnsupportedImageFormatError: If the input file type was an image that
+            could not be read, or some other file type that is not a PDF.
+        ocrmypdf.DpiError: If the input file is an image, but the resolution of the
+            image is not credible (allowing it to proceed would cause poor OCR).
+        ocrmypdf.OutputFileAccessError: If an attempt to write to the intended output
+            file failed.
+        ocrmypdf.PriorOcrFoundError: If the input PDF seems to have OCR or digital
+            text already, and settings did not tell us to proceed.
+        ocrmypdf.InputFileError: Any other problem with the input file.
+        ocrmypdf.SubprocessOutputError: Any error related to executing a subprocess.
+        ocrmypdf.EncryptedPdfERror: If the input PDF is encrypted (password protected).
+            OCRmyPDF does not remove passwords.
+        ocrmypdf.TesseractConfigError: If Tesseract reported its configuration was not
+            valid.
+
+    Returns:
+        :class:`ocrmypdf.ExitCode`
+    """
+
+    options = create_options(**locals())
+    check_options(options)
+    return run_pipeline(options, api=True)

src/ocrmypdf/cli.py

@@ -0,0 +1,478 @@
+# © 2015-19 James R. Barlow: github.com/jbarlow83
+#
+# This file is part of OCRmyPDF.
+#
+# OCRmyPDF is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# OCRmyPDF is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
+
+import argparse
+
+from . import PROGRAM_NAME, VERSION
+
+
+def numeric(basetype, min_=None, max_=None):
+    """Validator for numeric params"""
+    min_ = basetype(min_) if min_ is not None else None
+    max_ = basetype(max_) if max_ is not None else None
+
+    def _numeric(string):
+        value = basetype(string)
+        if (min_ is not None and value < min_) or (max_ is not None and value > max_):
+            msg = "%r not in valid range %r" % (string, (min_, max_))
+            raise argparse.ArgumentTypeError(msg)
+        return value
+
+    _numeric.__name__ = basetype.__name__
+    return _numeric
+
+
+class ArgumentParser(argparse.ArgumentParser):
+    """Override parser's default behavior of calling sys.exit()
+
+    https://stackoverflow.com/questions/5943249/python-argparse-and-controlling-overriding-the-exit-status-code
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.api_mode = False
+
+    def error(self, message):
+        if not self.api_mode:
+            super().error(message)
+            return
+        raise ValueError(message)
+
+
+parser = ArgumentParser(
+    prog=PROGRAM_NAME,
+    fromfile_prefix_chars='@',
+    formatter_class=argparse.RawDescriptionHelpFormatter,
+    description="""\
+Generates a searchable PDF or PDF/A from a regular PDF.
+
+OCRmyPDF rasterizes each page of the input PDF, optionally corrects page
+rotation and performs image processing, runs the Tesseract OCR engine on the
+image, and then creates a PDF from the OCR information.
+""",
+    epilog="""\
+OCRmyPDF attempts to keep the output file at about the same size.  If a file
+contains losslessly compressed images, and output file will be losslessly
+compressed as well.
+
+PDF is a page description file that attempts to preserve a layout exactly.
+A PDF can contain vector objects (such as text or lines) and raster objects
+(images).  A page might have multiple images.  OCRmyPDF is prepared to deal
+with the wide variety of PDFs that exist in the wild.
+
+When a PDF page contains text, OCRmyPDF assumes that the page has already
+been OCRed or is a "born digital" page that should not be OCRed.  The default
+behavior is to exit in this case without producing a file.  You can use the
+option --skip-text to ignore pages with text, or --force-ocr to rasterize
+all objects on the page and produce an image-only PDF as output.
+
+    ocrmypdf --skip-text file_with_some_text_pages.pdf output.pdf
+
+    ocrmypdf --force-ocr word_document.pdf output.pdf
+
+If you are concerned about long-term archiving of PDFs, use the default option
+--output-type pdfa which converts the PDF to a standardized PDF/A-2b.  This
+converts images to sRGB colorspace, removes some features from the PDF such
+as Javascript or forms. If you want to minimize the number of changes made to
+your PDF, use --output-type pdf.
+
+If OCRmyPDF is given an image file as input, it will attempt to convert the
+image to a PDF before processing.  For more control over the conversion of
+images to PDF, use the Python package img2pdf or other image to PDF software.
+
+For example, this command uses img2pdf to convert all .png files beginning
+with the 'page' prefix to a PDF, fitting each image on A4-sized paper, and
+sending the result to OCRmyPDF through a pipe.  img2pdf is a dependency of
+ocrmypdf so it is already installed.
+
+    img2pdf --pagesize A4 page*.png | ocrmypdf - myfile.pdf
+
+Online documentation is located at:
+    https://ocrmypdf.readthedocs.io/en/latest/introduction.html
+
+""",
+)
+
+parser.add_argument(
+    'input_file',
+    metavar="input_pdf_or_image",
+    help="PDF file containing the images to be OCRed (or '-' to read from "
+    "standard input)",
+)
+parser.add_argument(
+    'output_file',
+    metavar="output_pdf",
+    help="Output searchable PDF file (or '-' to write to standard output). "
+    "Existing files will be ovewritten. If same as input file, the "
+    "input file will be updated only if processing is successful.",
+)
+parser.add_argument(
+    '-l',
+    '--language',
+    action='append',
+    help="Language(s) of the file to be OCRed (see tesseract --list-langs for "
+    "all language packs installed in your system). Use -l eng+deu for "
+    "multiple languages.",
+)
+parser.add_argument(
+    '--image-dpi',
+    metavar='DPI',
+    type=int,
+    help="For input image instead of PDF, use this DPI instead of file's.",
+)
+parser.add_argument(
+    '--output-type',
+    choices=['pdfa', 'pdf', 'pdfa-1', 'pdfa-2', 'pdfa-3'],
+    default='pdfa',
+    help="Choose output type. 'pdfa' creates a PDF/A-2b compliant file for "
+    "long term archiving (default, recommended) but may not suitable "
+    "for users who want their file altered as little as possible. 'pdfa' "
+    "also has problems with full Unicode text. 'pdf' attempts to "
+    "preserve file contents as much as possible. 'pdf-a1' creates a "
+    "PDF/A1-b file. 'pdf-a2' is equivalent to 'pdfa'. 'pdf-a3' creates a "
+    "PDF/A3-b file.",
+)
+
+# Use null string '\0' as sentinel to indicate the user supplied no argument,
+# since that is the only invalid character for filepaths on all platforms
+# bool('\0') is True in Python
+parser.add_argument(
+    '--sidecar',
+    nargs='?',
+    const='\0',
+    default=None,
+    metavar='FILE',
+    help="Generate sidecar text files that contain the same text recognized "
+    "by Tesseract. This may be useful for building a OCR text database. "
+    "If FILE is omitted, the sidecar file be named {output_file}.txt "
+    "If FILE is set to '-', the sidecar is written to stdout (a "
+    "convenient way to preview OCR quality). The output file and sidecar "
+    "may not both use stdout at the same time.",
+)
+
+parser.add_argument(
+    '--version',
+    action='version',
+    version=VERSION,
+    help="Print program version and exit",
+)
+
+jobcontrol = parser.add_argument_group("Job control options")
+jobcontrol.add_argument(
+    '-j',
+    '--jobs',
+    metavar='N',
+    type=numeric(int, 0, 256),
+    help="Use up to N CPU cores simultaneously (default: use all).",
+)
+jobcontrol.add_argument(
+    '-q', '--quiet', action='store_true', help="Suppress INFO messages"
+)
+jobcontrol.add_argument(
+    '-v',
+    '--verbose',
+    type=numeric(int, 0, 2),
+    default=0,
+    const=1,
+    nargs='?',
+    help="Print more verbose messages for each additional verbose level. Use "
+    "`-v 1` typically for much more detailed logging. Higher numbers "
+    "are probably only useful in debugging.",
+)
+jobcontrol.add_argument(
+    '--no-progress-bar',
+    action='store_false',
+    dest='progress_bar',
+    help=argparse.SUPPRESS,
+)
+jobcontrol.add_argument('--use-threads', action='store_true', help=argparse.SUPPRESS)
+
+metadata = parser.add_argument_group(
+    "Metadata options",
+    "Set output PDF/A metadata (default: copy input document's metadata)",
+)
+metadata.add_argument(
+    '--title', type=str, help="Set document title (place multiple words in quotes)"
+)
+metadata.add_argument('--author', type=str, help="Set document author")
+metadata.add_argument('--subject', type=str, help="Set document subject description")
+metadata.add_argument('--keywords', type=str, help="Set document keywords")
+
+preprocessing = parser.add_argument_group(
+    "Image preprocessing options",
+    "Options to improve the quality of the final PDF and OCR",
+)
+preprocessing.add_argument(
+    '-r',
+    '--rotate-pages',
+    action='store_true',
+    help="Automatically rotate pages based on detected text orientation",
+)
+preprocessing.add_argument(
+    '--remove-background',
+    action='store_true',
+    help="Attempt to remove background from gray or color pages, setting it "
+    "to white ",
+)
+preprocessing.add_argument(
+    '-d', '--deskew', action='store_true', help="Deskew each page before performing OCR"
+)
+preprocessing.add_argument(
+    '-c',
+    '--clean',
+    action='store_true',
+    help="Clean pages from scanning artifacts before performing OCR, and send "
+    "the cleaned page to OCR, but do not include the cleaned page in "
+    "the output",
+)
+preprocessing.add_argument(
+    '-i',
+    '--clean-final',
+    action='store_true',
+    help="Clean page as above, and incorporate the cleaned image in the final "
+    "PDF.  Might remove desired content.",
+)
+preprocessing.add_argument(
+    '--unpaper-args',
+    type=str,
+    default=None,
+    help="A quoted string of arguments to pass to unpaper. Requires --clean. "
+    "Example: --unpaper-args '--layout double'.",
+)
+preprocessing.add_argument(
+    '--oversample',
+    metavar='DPI',
+    type=numeric(int, 0, 5000),
+    default=0,
+    help="Oversample images to at least the specified DPI, to improve OCR "
+    "results slightly",
+)
+preprocessing.add_argument(
+    '--remove-vectors',
+    action='store_true',
+    help="EXPERIMENTAL. Mask out any vector objects in the PDF so that they "
+    "will not be included in OCR. This can eliminate false characters.",
+)
+preprocessing.add_argument(
+    '--threshold',
+    action='store_true',
+    help="EXPERIMENTAL. Threshold image to 1bpp before sending it to Tesseract for OCR. Can "
+    "improve OCR quality compared to Tesseract's thresholder.",
+)
+
+ocrsettings = parser.add_argument_group("OCR options", "Control how OCR is applied")
+ocrsettings.add_argument(
+    '-f',
+    '--force-ocr',
+    action='store_true',
+    help="Rasterize any text or vector objects on each page, apply OCR, and "
+    "save the rastered output (this rewrites the PDF)",
+)
+ocrsettings.add_argument(
+    '-s',
+    '--skip-text',
+    action='store_true',
+    help="Skip OCR on any pages that already contain text, but include the "
+    "page in final output; useful for PDFs that contain a mix of "
+    "images, text pages, and/or previously OCRed pages",
+)
+ocrsettings.add_argument(
+    '--redo-ocr',
+    action='store_true',
+    help="Attempt to detect and remove the hidden OCR layer from files that "
+    "were previously OCRed with OCRmyPDF or another program. Apply OCR "
+    "to text found in raster images. Existing visible text objects will "
+    "not be changed. If there is no existing OCR, OCR will be added.",
+)
+ocrsettings.add_argument(
+    '--skip-big',
+    type=numeric(float, 0, 5000),
+    metavar='MPixels',
+    help="Skip OCR on pages larger than the specified amount of megapixels, "
+    "but include skipped pages in final output",
+)
+
+optimizing = parser.add_argument_group(
+    "Optimization options", "Control how the PDF is optimized after OCR"
+)
+optimizing.add_argument(
+    '-O',
+    '--optimize',
+    type=int,
+    choices=range(0, 4),
+    default=1,
+    help=(
+        "Control how PDF is optimized after processing:"
+        "0 - do not optimize; "
+        "1 - do safe, lossless optimizations (default); "
+        "2 - do some lossy optimizations; "
+        "3 - do aggressive lossy optimizations (including lossy JBIG2)"
+    ),
+)
+optimizing.add_argument(
+    '--jpeg-quality',
+    type=numeric(int, 0, 100),
+    default=0,
+    metavar='Q',
+    help=(
+        "Adjust JPEG quality level for JPEG optimization. "
+        "100 is best quality and largest output size; "
+        "1 is lowest quality and smallest output; "
+        "0 uses the default."
+    ),
+)
+optimizing.add_argument(
+    '--jpg-quality',
+    type=numeric(int, 0, 100),
+    default=0,
+    metavar='Q',
+    dest='jpeg_quality',
+    help=argparse.SUPPRESS,  # Alias for --jpeg-quality
+)
+optimizing.add_argument(
+    '--png-quality',
+    type=numeric(int, 0, 100),
+    default=0,
+    metavar='Q',
+    help=(
+        "Adjust PNG quality level to use when quantizing PNGs. "
+        "Values have same meaning as with --jpeg-quality"
+    ),
+)
+optimizing.add_argument(
+    '--jbig2-lossy',
+    action='store_true',
+    help=(
+        "Enable JBIG2 lossy mode (better compression, not suitable for some "
+        "use cases - see documentation)."
+    ),
+)
+optimizing.add_argument(
+    '--jbig2-page-group-size',
+    type=numeric(int, 1, 10000),
+    default=0,
+    metavar='N',
+    # Adjust number of pages to consider at once for JBIG2 compression
+    help=argparse.SUPPRESS,
+)
+
+advanced = parser.add_argument_group(
+    "Advanced", "Advanced options to control Tesseract's OCR behavior"
+)
+advanced.add_argument(
+    '--pages',
+    type=str,
+    help="Limit OCR to the specified pages (ranges or comma separated), skipping others",
+)
+advanced.add_argument(
+    '--max-image-mpixels',
+    action='store',
+    type=numeric(float, 0),
+    metavar='MPixels',
+    help="Set maximum number of pixels to unpack before treating an image as a "
+    "decompression bomb",
+    default=128.0,
+)
+advanced.add_argument(
+    '--tesseract-config',
+    action='append',
+    metavar='CFG',
+    default=[],
+    help="Additional Tesseract configuration files -- see documentation",
+)
+advanced.add_argument(
+    '--tesseract-pagesegmode',
+    action='store',
+    type=int,
+    metavar='PSM',
+    choices=range(0, 14),
+    help="Set Tesseract page segmentation mode (see tesseract --help)",
+)
+advanced.add_argument(
+    '--tesseract-oem',
+    action='store',
+    type=int,
+    metavar='MODE',
+    choices=range(0, 4),
+    help=(
+        "Set Tesseract 4.0 OCR engine mode: "
+        "0 - original Tesseract only; "
+        "1 - neural nets LSTM only; "
+        "2 - Tesseract + LSTM; "
+        "3 - default."
+    ),
+)
+advanced.add_argument(
+    '--pdf-renderer',
+    choices=['auto', 'hocr', 'sandwich'],
+    default='auto',
+    help="Choose OCR PDF renderer - the default option is to let OCRmyPDF "
+    "choose.  See documentation for discussion.",
+)
+advanced.add_argument(
+    '--tesseract-timeout',
+    default=180.0,
+    type=numeric(float, 0),
+    metavar='SECONDS',
+    help='Give up on OCR after the timeout, but copy the preprocessed page '
+    'into the final output',
+)
+advanced.add_argument(
+    '--rotate-pages-threshold',
+    default=14.0,
+    type=numeric(float, 0, 1000),
+    metavar='CONFIDENCE',
+    help="Only rotate pages when confidence is above this value (arbitrary "
+    "units reported by tesseract)",
+)
+advanced.add_argument(
+    '--pdfa-image-compression',
+    choices=['auto', 'jpeg', 'lossless'],
+    default='auto',
+    help="Specify how to compress images in the output PDF/A. 'auto' lets "
+    "OCRmyPDF decide.  'jpeg' changes all grayscale and color images to "
+    "JPEG compression.  'lossless' uses PNG-style lossless compression "
+    "for all images.  Monochrome images are always compressed using a "
+    "lossless codec.  Compression settings "
+    "are applied to all pages, including those for which OCR was "
+    "skipped.  Not supported for --output-type=pdf ; that setting "
+    "preserves the original compression of all images.",
+)
+advanced.add_argument(
+    '--user-words',
+    metavar='FILE',
+    help="Specify the location of the Tesseract user words file. This is a "
+    "list of words Tesseract should consider while performing OCR in "
+    "addition to its standard language dictionaries. This can improve "
+    "OCR quality especially for specialized and technical documents.",
+)
+advanced.add_argument(
+    '--user-patterns',
+    metavar='FILE',
+    help="Specify the location of the Tesseract user patterns file.",
+)
+
+debugging = parser.add_argument_group(
+    "Debugging", "Arguments to help with troubleshooting and debugging"
+)
+debugging.add_argument(
+    '-k',
+    '--keep-temporary-files',
+    action='store_true',
+    help="Keep temporary files (helpful for debugging)",
+)
+debugging.add_argument('--tesseract-env', type=str, help=argparse.SUPPRESS)

src/ocrmypdf/exec/__init__.py

@@ -17,6 +17,7 @@
 
 """Wrappers to manage subprocess calls"""
 
+import logging
 import os
 import re
 import sys
@@ -24,8 +25,10 @@ from subprocess import run, STDOUT, PIPE, CalledProcessError
 from ..exceptions import MissingDependencyError, ExitCode
 from collections.abc import Mapping
 
+log = logging.Logger(__name__)
 
-def get_version(program, *, version_arg='--version', regex=r'(\d+(\.\d+)*)'):
+
+def get_version(program, *, version_arg='--version', regex=r'(\d+(\.\d+)*)', env=None):
     "Get the version of the specified program"
     args_prog = [program, version_arg]
     try:
@@ -36,6 +39,7 @@ def get_version(program, *, version_arg='--version', regex=r'(\d+(\.\d+)*)'):
             stdout=PIPE,
             stderr=STDOUT,
             check=True,
+            env=env,
         )
         output = proc.stdout
     except FileNotFoundError as e:
@@ -115,7 +119,7 @@ def _get_platform():
     return sys.platform
 
 
-def _error_trailer(log, program, package, **kwargs):
+def _error_trailer(program, package, **kwargs):
     if isinstance(package, Mapping):
         package = package[_get_platform()]
 
@@ -125,7 +129,7 @@ def _error_trailer(log, program, package, **kwargs):
         log.info(linux_install_advice.format(**locals()))
 
 
-def _error_missing_program(log, program, package, required_for, recommended):
+def _error_missing_program(program, package, required_for, recommended):
     if required_for:
         log.error(missing_optional_program.format(**locals()))
     elif recommended:
@@ -135,9 +139,7 @@ def _error_missing_program(log, program, package, required_for, recommended):
     _error_trailer(**locals())
 
 
-def _error_old_version(
-    log, program, package, need_version, found_version, required_for
-):
+def _error_old_version(program, package, need_version, found_version, required_for):
     if required_for:
         log.error(old_version_required_for.format(**locals()))
     else:
@@ -147,27 +149,28 @@ def _error_old_version(
 
 def check_external_program(
     *,
-    log,
     program,
     package,
     version_checker,
     need_version,
     required_for=None,
     recommended=False,
+    **kwargs,  # To consume log parameter
 ):
+    if kwargs:
+        if not 'log' in kwargs:
+            log.warning('check_external_program(log=...) is deprecated')
     try:
         found_version = version_checker()
     except (CalledProcessError, FileNotFoundError, MissingDependencyError):
-        _error_missing_program(log, program, package, required_for, recommended)
+        _error_missing_program(program, package, required_for, recommended)
         if not recommended:
-            sys.exit(ExitCode.missing_dependency)
+            raise MissingDependencyError()
         return
 
     if found_version < need_version:
-        _error_old_version(
-            log, program, package, need_version, found_version, required_for
-        )
+        _error_old_version(program, package, need_version, found_version, required_for)
         if not recommended:
-            sys.exit(ExitCode.missing_dependency)
+            raise MissingDependencyError()
 
-    log.debug(f'Found {program} {found_version}')
+    log.debug('Found %s %s', program, found_version)

src/ocrmypdf/exec/ghostscript.py

@@ -15,6 +15,7 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
+import logging
 import re
 from functools import lru_cache
 from os import fspath
@@ -24,8 +25,11 @@ from tempfile import NamedTemporaryFile
 
 from PIL import Image
 
-from . import get_version
 from ..exceptions import SubprocessOutputError
+from . import get_version
+
+
+gslog = logging.getLogger()
 
 
 @lru_cache(maxsize=1)
@@ -132,6 +136,8 @@ def rasterize_pdf(
     res = round(xres, 6), round(yres, 6)
     if not page_dpi:
         page_dpi = res
+    if not log:
+        log = gslog
 
     with NamedTemporaryFile(delete=True) as tmp:
         args_gs = (
@@ -160,12 +166,11 @@ def rasterize_pdf(
         p = run(args_gs, stdout=PIPE, stderr=STDOUT, universal_newlines=True)
         if _gs_error_reported(p.stdout):
             log.error(p.stdout)
-        else:
+        elif p.stdout:
             log.debug(p.stdout)
 
         if p.returncode != 0:
-            log.error('Ghostscript rasterizing failed')
-            raise SubprocessOutputError()
+            raise SubprocessOutputError('Ghostscript rasterizing failed')
 
         tmp.seek(0)
         with Image.open(tmp) as im:
@@ -210,6 +215,9 @@ def generate_pdfa(
     images entirely. (The feature was added in 9.23 but broken, and the 9.24
     release of Ghostscript had regressions, so we don't support it until 9.25.)
     """
+    if not log:
+        log = gslog
+
     compression_args = []
     if compression == 'jpeg':
         compression_args = [
@@ -287,5 +295,4 @@ def generate_pdfa(
             # PDF/A - check PDF/A status elsewhere
             copy(gs_pdf.name, fspath(output_file))
         else:
-            log.error('Ghostscript PDF/A rendering failed')
-            raise SubprocessOutputError()
+            raise SubprocessOutputError('Ghostscript PDF/A rendering failed')

src/ocrmypdf/exec/tesseract.py

@@ -22,18 +22,14 @@ from collections import namedtuple
 from contextlib import suppress
 from functools import lru_cache
 from os import fspath
-from subprocess import (
-    PIPE,
-    STDOUT,
-    CalledProcessError,
-    TimeoutExpired,
-    check_output,
-    run,
-)
-from textwrap import dedent
+from subprocess import PIPE, STDOUT, CalledProcessError, TimeoutExpired, run
 
 from . import get_version
-from ..exceptions import MissingDependencyError, TesseractConfigError
+from ..exceptions import (
+    MissingDependencyError,
+    TesseractConfigError,
+    SubprocessOutputError,
+)
 from ..helpers import page_number
 
 OrientationConfidence = namedtuple('OrientationConfidence', ('angle', 'confidence'))
@@ -56,18 +52,16 @@ HOCR_TEMPLATE = """<?xml version="1.0" encoding="UTF-8"?>
 """
 
 
-@lru_cache(maxsize=1)
-def version():
-    return get_version('tesseract', regex=r'tesseract\s(.+)')
+def version(tesseract_env=None):
+    return get_version('tesseract', regex=r'tesseract\s(.+)', env=tesseract_env)
 
 
-def v4():
+def v4(tesseract_env=None):
     "Is this Tesseract v4.0?"
-    return version() >= '4'
+    return version(tesseract_env) >= '4'
 
 
-@lru_cache(maxsize=1)
-def has_textonly_pdf():
+def has_textonly_pdf(tesseract_env=None):
     """Does Tesseract have textonly_pdf capability?
 
     Available in v4.00.00alpha since January 2017. Best to
@@ -76,41 +70,51 @@ def has_textonly_pdf():
     args_tess = ['tesseract', '--print-parameters', 'pdf']
     params = ''
     try:
-        params = check_output(args_tess, universal_newlines=True, stderr=STDOUT)
+        proc = run(
+            args_tess,
+            check=True,
+            universal_newlines=True,
+            stdout=PIPE,
+            stderr=STDOUT,
+            env=tesseract_env,
+        )
+        params = proc.stdout
     except CalledProcessError as e:
-        print("Could not --print-parameters from tesseract", file=sys.stderr)
-        raise MissingDependencyError from e
+        raise MissingDependencyError(
+            "Could not --print-parameters from tesseract"
+        ) from e
     if 'textonly_pdf' in params:
         return True
     return False
 
 
-@lru_cache(maxsize=1)
-def languages():
+def languages(tesseract_env=None):
     def lang_error(output):
-        msg = dedent(
-            """Tesseract failed to report available languages.
-        Output from Tesseract:
-        -----------
-        """
+        msg = (
+            "Tesseract failed to report available languages.\n"
+            "Output from Tesseract:\n"
+            "-----------\n"
         )
         msg += output
-        print(msg, file=sys.stderr)
+        return msg
 
     args_tess = ['tesseract', '--list-langs']
     try:
         proc = run(
-            args_tess, universal_newlines=True, stdout=PIPE, stderr=STDOUT, check=True
+            args_tess,
+            universal_newlines=True,
+            stdout=PIPE,
+            stderr=STDOUT,
+            check=True,
+            env=tesseract_env,
         )
         output = proc.stdout
     except CalledProcessError as e:
-        lang_error(e.output)
-        raise MissingDependencyError from e
+        raise MissingDependencyError(lang_error(e.output)) from e
 
     header, *rest = output.splitlines()
     if not header.startswith('List of available languages'):
-        lang_error(output)
-        raise MissingDependencyError
+        raise MissingDependencyError(lang_error(output))
     return set(lang.strip() for lang in rest)
 
 
@@ -123,7 +127,7 @@ def tess_base_args(langs, engine_mode):
     return args
 
 
-def get_orientation(input_file, engine_mode, timeout: float, log):
+def get_orientation(input_file, engine_mode, timeout: float, log, tesseract_env=None):
     args_tesseract = tess_base_args(['osd'], engine_mode) + [
         '--psm',
         '0',
@@ -132,7 +136,15 @@ def get_orientation(input_file, engine_mode, timeout: float, log):
     ]
 
     try:
-        stdout = check_output(args_tesseract, stderr=STDOUT, timeout=timeout)
+        p = run(
+            args_tesseract,
+            stdout=PIPE,
+            stderr=STDOUT,
+            timeout=timeout,
+            check=True,
+            env=tesseract_env,
+        )
+        stdout = p.stdout
     except TimeoutExpired:
         return OrientationConfidence(angle=0, confidence=0.0)
     except CalledProcessError as e:
@@ -142,7 +154,7 @@ def get_orientation(input_file, engine_mode, timeout: float, log):
             or b'Image too large' in e.output
         ):
             return OrientationConfidence(0, 0)
-        raise e from e
+        raise SubprocessOutputError() from e
     else:
         osd = {}
         for line in stdout.decode().splitlines():
@@ -159,7 +171,7 @@ def get_orientation(input_file, engine_mode, timeout: float, log):
 
 
 def tesseract_log_output(log, stdout, input_file):
-    prefix = f"{(page_number(input_file)):4d}: [tesseract] "
+    prefix = "[tesseract] "
 
     try:
         text = stdout.decode()
@@ -231,6 +243,7 @@ def generate_hocr(
     pagesegmode: int,
     user_words,
     user_patterns,
+    tesseract_env,
     log,
 ):
 
@@ -254,7 +267,15 @@ def generate_hocr(
     args_tesseract.extend([input_file, prefix, 'hocr', 'txt'] + tessconfig)
     try:
         log.debug(args_tesseract)
-        stdout = check_output(args_tesseract, stderr=STDOUT, timeout=timeout)
+        p = run(
+            args_tesseract,
+            stdout=PIPE,
+            stderr=STDOUT,
+            timeout=timeout,
+            check=True,
+            env=tesseract_env,
+        )
+        stdout = p.stdout
     except TimeoutExpired:
         # Generate a HOCR file with no recognized text if tesseract times out
         # Temporary workaround to hocrTransform not being able to function if
@@ -267,7 +288,7 @@ def generate_hocr(
             _generate_null_hocr(output_hocr, output_sidecar, input_file)
             return
 
-        raise e from e
+        raise SubprocessOutputError() from e
     else:
         tesseract_log_output(log, stdout, input_file)
         # The sidecar text file will get the suffix .txt; rename it to
@@ -306,9 +327,10 @@ def generate_pdf(
     pagesegmode: int,
     user_words,
     user_patterns,
+    tesseract_env,
     log,
 ):
-    '''Use Tesseract to render a PDF.
+    """Use Tesseract to render a PDF.
 
     input_image -- image to analyze
     skip_pdf -- if we time out, use this file as output
@@ -320,14 +342,14 @@ def generate_pdf(
     tessconfig -- tesseract configuration
     timeout -- timeout (seconds)
     log -- logger object
-    '''
+    """
 
     args_tesseract = tess_base_args(language, engine_mode)
 
     if pagesegmode is not None:
         args_tesseract.extend(['--psm', str(pagesegmode)])
 
-    if text_only and has_textonly_pdf():
+    if text_only and has_textonly_pdf(tesseract_env):
         args_tesseract.extend(['-c', 'textonly_pdf=1'])
 
     if user_words:
@@ -342,10 +364,17 @@ def generate_pdf(
     # to the number of order parameters here
 
     args_tesseract.extend([input_image, prefix, 'pdf', 'txt'] + tessconfig)
-
     try:
         log.debug(args_tesseract)
-        stdout = check_output(args_tesseract, stderr=STDOUT, timeout=timeout)
+        p = run(
+            args_tesseract,
+            stdout=PIPE,
+            stderr=STDOUT,
+            timeout=timeout,
+            check=True,
+            env=tesseract_env,
+        )
+        stdout = p.stdout
         if os.path.exists(prefix + '.txt'):
             shutil.move(prefix + '.txt', output_text)
     except TimeoutExpired:
@@ -356,6 +385,6 @@ def generate_pdf(
         if b'Image too large' in e.output:
             use_skip_page(text_only, skip_pdf, output_pdf, output_text)
             return
-        raise e from e
+        raise SubprocessOutputError() from e
     else:
         tesseract_log_output(log, stdout, input_image)

src/ocrmypdf/exec/unpaper.py

@@ -29,11 +29,7 @@ from tempfile import TemporaryDirectory
 from . import get_version
 from ..exceptions import MissingDependencyError, SubprocessOutputError
 
-try:
-    from PIL import Image
-except ImportError:
-    print("Could not find Python3 imaging library", file=sys.stderr)
-    raise
+from PIL import Image
 
 
 @lru_cache(maxsize=1)
@@ -55,16 +51,18 @@ def run(input_file, output_file, dpi, log, mode_args):
             else:
                 im = im.convert(mode='RGB')
         except IOError as e:
-            log.error("Could not convert image with type " + im.mode)
             im.close()
-            raise MissingDependencyError() from e
+            raise MissingDependencyError(
+                "Could not convert image with type " + im.mode
+            ) from e
 
     try:
         suffix = SUFFIXES[im.mode]
     except KeyError:
-        log.error("Failed to convert image to a supported format.")
         im.close()
-        raise MissingDependencyError() from e
+        raise MissingDependencyError(
+            "Failed to convert image to a supported format."
+        ) from e
 
     with TemporaryDirectory() as tmpdir:
         input_pnm = os.path.join(tmpdir, f'input{suffix}')

src/ocrmypdf/filters.py

@@ -0,0 +1,10 @@
+from PIL import Image
+import PIL.ImageOps
+
+
+def invert(im):
+    return PIL.ImageOps.invert(im.convert('L'))
+
+
+def whiteout(im):
+    return Image.new(im.mode, im.size)

src/ocrmypdf/helpers.py

@@ -15,32 +15,35 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
+import logging
 import multiprocessing
 import os
-import sys
 import warnings
 from collections.abc import Iterable
 from contextlib import suppress
-from functools import partial, wraps
+from functools import wraps
 from pathlib import Path
 
+log = logging.getLogger(__name__)
 
-def re_symlink(input_file, soft_link_name, log=None):
+
+def re_symlink(input_file, soft_link_name, *args, **kwargs):
     """
     Helper function: relinks soft symbolic link if necessary
     """
+    if len(args) == 1 and isinstance(args[0], logging.Logger):
+        log.warning("Deprecated: re_symlink(,log)")
+    if 'log' in kwargs:
+        log.warning('Deprecated: re_symlink(...log=)')
+
     input_file = os.fspath(input_file)
     soft_link_name = os.fspath(soft_link_name)
-    if log is None:
-        prdebug = partial(print, file=sys.stderr)
-    else:
-        prdebug = log.debug
 
     # Guard against soft linking to oneself
     if input_file == soft_link_name:
-        prdebug(
-            "Warning: No symbolic link made. You are using "
-            + "the original data directory as the working directory."
+        log.warning(
+            "No symbolic link made. You are using "
+            "the original data directory as the working directory."
         )
         return
 
@@ -48,16 +51,16 @@ def re_symlink(input_file, soft_link_name, log=None):
     if os.path.lexists(soft_link_name):
         # do not delete or overwrite real (non-soft link) file
         if not os.path.islink(soft_link_name):
-            raise FileExistsError("%s exists and is not a link" % soft_link_name)
+            raise FileExistsError(f"{soft_link_name} exists and is not a link")
         try:
             os.unlink(soft_link_name)
         except OSError:
-            prdebug("Can't unlink %s" % (soft_link_name))
+            log.debug("Can't unlink %s", soft_link_name)
 
     if not os.path.exists(input_file):
-        raise FileNotFoundError("trying to create a broken symlink to %s" % input_file)
+        raise FileNotFoundError(f"trying to create a broken symlink to {input_file}")
 
-    prdebug("os.symlink(%s, %s)" % (input_file, soft_link_name))
+    log.debug("os.symlink(%s, %s)", input_file, soft_link_name)
 
     # Create symbolic link using absolute path
     os.symlink(os.path.abspath(input_file), soft_link_name)
@@ -67,6 +70,11 @@ def is_iterable_notstr(thing):
     return isinstance(thing, Iterable) and not isinstance(thing, str)
 
 
+def monotonic(L):
+    """Does list increase monotonically?"""
+    return all(b > a for a, b in zip(L, L[1:]))
+
+
 def page_number(input_file):
     """Get one-based page number implied by filename (000002.pdf -> 2)"""
     return int(os.path.basename(os.fspath(input_file))[0:6])
@@ -77,14 +85,6 @@ def available_cpu_count():
         return multiprocessing.cpu_count()
     except NotImplementedError:
         pass
-
-    try:
-        import psutil
-
-        return psutil.cpu_count()
-    except (ImportError, AttributeError):
-        pass
-
     warnings.warn(
         "Could not get CPU count.  Assuming one (1) CPU." "Use -j N to set manually."
     )
@@ -122,14 +122,6 @@ def is_file_writable(test_file):
         return True
 
 
-def flatten_groups(groups):
-    for obj in groups:
-        if is_iterable_notstr(obj):
-            yield from obj
-        else:
-            yield obj
-
-
 def deprecated(func):
     """Warn that function is deprecated"""
 

src/ocrmypdf/optimize.py

@@ -16,20 +16,21 @@
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
 import concurrent.futures
-import logging
 import sys
+import tempfile
 from collections import defaultdict
 from os import fspath
 from pathlib import Path
 
 from PIL import Image
-
+from tqdm import tqdm
 import pikepdf
-from pikepdf import Name, Dictionary, Array
+from pikepdf import Name, Dictionary
 
 from . import leptonica
-from ._jobcontext import JobContext
+from ._jobcontext import PDFContext
 from .exec import jbig2enc, pngquant
+from .exceptions import OutputFileAccessError
 from .helpers import re_symlink
 
 DEFAULT_JPEG_QUALITY = 75
@@ -267,9 +268,17 @@ def _produce_jbig2_images(jbig2_groups, root, log, options):
 
     with concurrent.futures.ThreadPoolExecutor(max_workers=options.jobs) as executor:
         futures = jbig2_futures(executor, root, jbig2_groups)
-        for future in concurrent.futures.as_completed(futures):
-            proc = future.result()
-            log.debug(proc.stderr.decode())
+        with tqdm(
+            total=len(jbig2_groups),
+            desc="JBIG2",
+            unit='item',
+            disable=not options.progress_bar,
+        ) as pbar:
+            for future in concurrent.futures.as_completed(futures):
+                proc = future.result()
+                if proc.stderr:
+                    log.debug(proc.stderr.decode())
+                pbar.update()
 
 
 def convert_to_jbig2(pike, jbig2_groups, root, log, options):
@@ -311,7 +320,9 @@ def convert_to_jbig2(pike, jbig2_groups, root, log, options):
 
 
 def transcode_jpegs(pike, jpegs, root, log, options):
-    for xref in jpegs:
+    for xref in tqdm(
+        jpegs, desc="JPEGs", unit='image', disable=not options.progress_bar
+    ):
         in_jpg = Path(jpg_name(root, xref))
         opt_jpg = in_jpg.with_suffix('.opt.jpg')
 
@@ -340,15 +351,26 @@ def transcode_pngs(pike, images, image_name_fn, root, log, options):
         with concurrent.futures.ThreadPoolExecutor(
             max_workers=options.jobs
         ) as executor:
+            futures = []
             for xref in images:
                 log.debug(image_name_fn(root, xref))
-                executor.submit(
-                    pngquant.quantize,
-                    image_name_fn(root, xref),
-                    png_name(root, xref),
-                    png_quality[0],
-                    png_quality[1],
+                futures.append(
+                    executor.submit(
+                        pngquant.quantize,
+                        image_name_fn(root, xref),
+                        png_name(root, xref),
+                        png_quality[0],
+                        png_quality[1],
+                    )
                 )
+            with tqdm(
+                desc="PNGs",
+                total=len(futures),
+                unit='image',
+                disable=not options.progress_bar,
+            ) as pbar:
+                for _future in concurrent.futures.as_completed(futures):
+                    pbar.update()
 
     for xref in images:
         im_obj = pike.get_object(xref, 0)
@@ -427,11 +449,11 @@ def transcode_pngs(pike, images, image_name_fn, root, log, options):
         im_obj.write(compdata.read(), filter=Name.FlateDecode, decode_parms=dparms)
 
 
-def optimize(input_file, output_file, log, context):
-
-    options = context.get_options()
+def optimize(input_file, output_file, context):
+    log = context.log
+    options = context.options
     if options.optimize == 0:
-        re_symlink(input_file, output_file, log)
+        re_symlink(input_file, output_file)
         return
 
     if options.jpeg_quality == 0:
@@ -441,40 +463,44 @@ def optimize(input_file, output_file, log, context):
     if options.jbig2_page_group_size == 0:
         options.jbig2_page_group_size = 10 if options.jbig2_lossy else 1
 
-    pike = pikepdf.Pdf.open(input_file)
+    with pikepdf.Pdf.open(input_file) as pike:
+        root = Path(output_file).parent / 'images'
+        root.mkdir(exist_ok=True)
 
-    root = Path(output_file).parent / 'images'
-    root.mkdir(exist_ok=True)
+        jpegs, pngs = extract_images_generic(pike, root, log, options)
+        transcode_jpegs(pike, jpegs, root, log, options)
+        # if options.optimize >= 2:
+        # Try pngifying the jpegs
+        #    transcode_pngs(pike, jpegs, jpg_name, root, log, options)
+        transcode_pngs(pike, pngs, png_name, root, log, options)
 
-    jpegs, pngs = extract_images_generic(pike, root, log, options)
-    transcode_jpegs(pike, jpegs, root, log, options)
-    # if options.optimize >= 2:
-    # Try pngifying the jpegs
-    #    transcode_pngs(pike, jpegs, jpg_name, root, log, options)
-    transcode_pngs(pike, pngs, png_name, root, log, options)
+        jbig2_groups = extract_images_jbig2(pike, root, log, options)
+        convert_to_jbig2(pike, jbig2_groups, root, log, options)
 
-    jbig2_groups = extract_images_jbig2(pike, root, log, options)
-    convert_to_jbig2(pike, jbig2_groups, root, log, options)
-
-    target_file = Path(output_file).with_suffix('.opt.pdf')
-    pike.remove_unreferenced_resources()
-    pike.save(
-        target_file,
-        preserve_pdfa=True,
-        object_stream_mode=pikepdf.ObjectStreamMode.generate,
-    )
+        target_file = Path(output_file).with_suffix('.opt.pdf')
+        pike.remove_unreferenced_resources()
+        pike.save(
+            target_file,
+            preserve_pdfa=True,
+            object_stream_mode=pikepdf.ObjectStreamMode.generate,
+        )
 
     input_size = Path(input_file).stat().st_size
     output_size = Path(target_file).stat().st_size
+    if output_size == 0:
+        raise OutputFileAccessError(
+            f"Output file not created after optimizing. We probably ran "
+            f"out of disk space in the temporary folder: {tempfile.gettempdir()}."
+        )
     ratio = input_size / output_size
     savings = 1 - output_size / input_size
     log.info(f"Optimize ratio: {ratio:.2f} savings: {(100 * savings):.1f}%")
 
     if savings < 0:
         log.info("Optimize did not improve the file - discarded")
-        re_symlink(input_file, output_file, log)
+        re_symlink(input_file, output_file)
     else:
-        re_symlink(target_file, output_file, log)
+        re_symlink(target_file, output_file)
 
 
 def main(infile, outfile, level, jobs=1):
@@ -484,30 +510,32 @@ def main(infile, outfile, level, jobs=1):
     class OptimizeOptions:
         """Emulate ocrmypdf's options"""
 
-        def __init__(self, jobs, optimize, jpeg_quality, png_quality, jb2lossy):
+        def __init__(
+            self, input_file, jobs, optimize, jpeg_quality, png_quality, jb2lossy
+        ):
+            self.input_file = input_file
             self.jobs = jobs
             self.optimize = optimize
             self.jpeg_quality = jpeg_quality
             self.png_quality = png_quality
             self.jbig2_page_group_size = 0
             self.jbig2_lossy = jb2lossy
+            self.quiet = True
+            self.progress_bar = False
 
-    logging.basicConfig(level=logging.DEBUG)
-    log = logging.getLogger()
-
-    ctx = JobContext()
     options = OptimizeOptions(
+        input_file=infile,
         jobs=jobs,
         optimize=int(level),
         jpeg_quality=0,  # Use default
         png_quality=0,
         jb2lossy=False,
     )
-    ctx.set_options(options)
 
     with TemporaryDirectory() as td:
+        context = PDFContext(options, td, infile, None)
         tmpout = Path(td) / 'out.pdf'
-        optimize(infile, tmpout, log, ctx)
+        optimize(infile, tmpout, context)
         copy(fspath(tmpout), fspath(outfile))
 
 

src/ocrmypdf/pdfinfo/__init__.py

@@ -16,795 +16,4 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
-from collections import namedtuple
-from decimal import Decimal
-from enum import Enum
-from math import hypot, isclose
-from os import fspath
-from pathlib import Path
-from unittest.mock import Mock
-from warnings import warn
-import re
-
-from pikepdf import PdfMatrix
-import pikepdf
-
-from . import ghosttext
-from .layout import get_page_analysis, get_text_boxes
-
-from ..exceptions import EncryptedPdfError, MissingDependencyError
-
-
-Colorspace = Enum('Colorspace', 'gray rgb cmyk lab icc index sep devn pattern jpeg2000')
-
-Encoding = Enum(
-    'Encoding', 'ccitt jpeg jpeg2000 jbig2 asciihex ascii85 lzw flate ' + 'runlength'
-)
-
-FRIENDLY_COLORSPACE = {
-    '/DeviceGray': Colorspace.gray,
-    '/CalGray': Colorspace.gray,
-    '/DeviceRGB': Colorspace.rgb,
-    '/CalRGB': Colorspace.rgb,
-    '/DeviceCMYK': Colorspace.cmyk,
-    '/Lab': Colorspace.lab,
-    '/ICCBased': Colorspace.icc,
-    '/Indexed': Colorspace.index,
-    '/Separation': Colorspace.sep,
-    '/DeviceN': Colorspace.devn,
-    '/Pattern': Colorspace.pattern,
-    '/G': Colorspace.gray,  # Abbreviations permitted in inline images
-    '/RGB': Colorspace.rgb,
-    '/CMYK': Colorspace.cmyk,
-    '/I': Colorspace.index,
-}
-
-FRIENDLY_ENCODING = {
-    '/CCITTFaxDecode': Encoding.ccitt,
-    '/DCTDecode': Encoding.jpeg,
-    '/JPXDecode': Encoding.jpeg2000,
-    '/JBIG2Decode': Encoding.jbig2,
-    '/CCF': Encoding.ccitt,  # Abbreviations permitted in inline images
-    '/DCT': Encoding.jpeg,
-    '/AHx': Encoding.asciihex,
-    '/A85': Encoding.ascii85,
-    '/LZW': Encoding.lzw,
-    '/Fl': Encoding.flate,
-    '/RL': Encoding.runlength,
-}
-
-FRIENDLY_COMP = {
-    Colorspace.gray: 1,
-    Colorspace.rgb: 3,
-    Colorspace.cmyk: 4,
-    Colorspace.lab: 3,
-    Colorspace.index: 1,
-}
-
-
-UNIT_SQUARE = (1.0, 0.0, 0.0, 1.0, 0.0, 0.0)
-
-
-def _is_unit_square(shorthand):
-    values = map(float, shorthand)
-    pairwise = zip(values, UNIT_SQUARE)
-    return all([isclose(a, b, rel_tol=1e-3) for a, b in pairwise])
-
-
-XobjectSettings = namedtuple('XobjectSettings', ['name', 'shorthand', 'stack_depth'])
-
-InlineSettings = namedtuple('InlineSettings', ['iimage', 'shorthand', 'stack_depth'])
-
-ContentsInfo = namedtuple(
-    'ContentsInfo', ['xobject_settings', 'inline_images', 'found_vector']
-)
-
-TextboxInfo = namedtuple('TextboxInfo', ['bbox', 'is_visible', 'is_corrupt'])
-
-
-class VectorInfo:
-    def __init__(self):
-        pass
-
-
-def _normalize_stack(graphobjs):
-    """Convert runs of qQ's in the stack into single graphobjs"""
-    for operands, operator in graphobjs:
-        operator = str(operator)
-        if re.match(r'Q*q+$', operator):  # Zero or more Q, one or more q
-            for char in operator:  # Split into individual
-                yield ([], char)  # Yield individual
-        else:
-            yield (operands, operator)
-
-
-def _interpret_contents(contentstream, initial_shorthand=UNIT_SQUARE):
-    """Interpret the PDF content stream.
-
-    The stack represents the state of the PDF graphics stack.  We are only
-    interested in the current transformation matrix (CTM) so we only track
-    this object; a full implementation would need to track many other items.
-
-    The CTM is initialized to the mapping from user space to device space.
-    PDF units are 1/72".  In a PDF viewer or printer this matrix is initialized
-    to the transformation to device space.  For example if set to
-    (1/72, 0, 0, 1/72, 0, 0) then all units would be calculated in inches.
-
-    Images are always considered to be (0, 0) -> (1, 1).  Before drawing an
-    image there should be a 'cm' that sets up an image coordinate system
-    where drawing from (0, 0) -> (1, 1) will draw on the desired area of the
-    page.
-
-    PDF units suit our needs so we initialize ctm to the identity matrix.
-
-    According to the PDF specification, the maximum stack depth is 32. Other
-    viewers tolerate some amount beyond this.  We issue a warning if the
-    stack depth exceeds the spec limit and set a hard limit beyond this to
-    bound our memory requirements.  If the stack underflows behavior is
-    undefined in the spec, but we just pretend nothing happened and leave the
-    CTM unchanged.
-    """
-
-    stack = []
-    ctm = PdfMatrix(initial_shorthand)
-    xobject_settings = []
-    inline_images = []
-    found_vector = False
-    vector_ops = set('S s f F f* B B* b b*'.split())
-    image_ops = set('BI ID EI q Q Do cm'.split())
-    operator_whitelist = ' '.join(vector_ops | image_ops)
-
-    for n, graphobj in enumerate(
-        _normalize_stack(
-            pikepdf.parse_content_stream(contentstream, operator_whitelist)
-        )
-    ):
-        operands, operator = graphobj
-        if operator == 'q':
-            stack.append(ctm)
-            if len(stack) > 32:  # See docstring
-                if len(stack) > 128:
-                    raise RuntimeError(
-                        "PDF graphics stack overflowed hard limit, operator %i" % n
-                    )
-                warn("PDF graphics stack overflowed spec limit")
-        elif operator == 'Q':
-            try:
-                ctm = stack.pop()
-            except IndexError:
-                # Keeping the ctm the same seems to be the only sensible thing
-                # to do. Just pretend nothing happened, keep calm and carry on.
-                warn("PDF graphics stack underflowed - PDF may be malformed")
-        elif operator == 'cm':
-            ctm = PdfMatrix(operands) @ ctm
-        elif operator == 'Do':
-            image_name = operands[0]
-            settings = XobjectSettings(
-                name=image_name, shorthand=ctm.shorthand, stack_depth=len(stack)
-            )
-            xobject_settings.append(settings)
-        elif operator == 'INLINE IMAGE':  # BI/ID/EI are grouped into this
-            iimage = operands[0]
-            inline = InlineSettings(
-                iimage=iimage, shorthand=ctm.shorthand, stack_depth=len(stack)
-            )
-            inline_images.append(inline)
-        elif operator in vector_ops:
-            found_vector = True
-
-    return ContentsInfo(
-        xobject_settings=xobject_settings,
-        inline_images=inline_images,
-        found_vector=found_vector,
-    )
-
-
-def _get_dpi(ctm_shorthand, image_size):
-    """Given the transformation matrix and image size, find the image DPI.
-
-    PDFs do not include image resolution information within image data.
-    Instead, the PDF page content stream describes the location where the
-    image will be rasterized, and the effective resolution is the ratio of the
-    pixel size to raster target size.
-
-    Normally a scanned PDF has the paper size set appropriately but this is
-    not guaranteed. The most common case is a cropped image will change the
-    page size (/CropBox) without altering the page content stream. That means
-    it is not sufficient to assume that the image fills the page, even though
-    that is the most common case.
-
-    A PDF image may be scaled (always), cropped, translated, rotated in place
-    to an arbitrary angle (rarely) and skewed. Only equal area mappings can
-    be expressed, that is, it is not necessary to consider distortions where
-    the effective DPI varies with position.
-
-    To determine the image scale, transform an offset axis vector v0 (0, 0),
-    width-axis vector v0 (1, 0), height-axis vector vh (0, 1) with the matrix,
-    which gives the dimensions of the image in PDF units. From there we can
-    compare to actual image dimensions. PDF uses
-    row vector * matrix_tranposed unlike the traditional
-    matrix * column vector.
-
-    The offset, width and height vectors can be combined in a matrix and
-    multiplied by the transform matrix. Then we want to calculated
-        magnitude(width_vector - offset_vector)
-    and
-        magnitude(height_vector - offset_vector)
-
-    When the above is worked out algebraically, the effect of translation
-    cancels out, and the vector magnitudes become functions of the nonzero
-    transformation matrix indices. The results of the derivation are used
-    in this code.
-
-    pdfimages -list does calculate the DPI in some way that is not completely
-    naive, but it does not get the DPI of rotated images right, so cannot be
-    used anymore to validate this. Photoshop works, or using Acrobat to
-    rotate the image back to normal.
-
-    It does not matter if the image is partially cropped, or even out of the
-    /MediaBox.
-
-    """
-
-    a, b, c, d, _, _ = ctm_shorthand
-
-    # Calculate the width and height of the image in PDF units
-    image_drawn_width = hypot(a, b)
-    image_drawn_height = hypot(c, d)
-
-    # The scale of the image is pixels per unit of default user space (1/72")
-    scale_w = image_size[0] / image_drawn_width
-    scale_h = image_size[1] / image_drawn_height
-
-    # DPI = scale * 72
-    dpi_w = scale_w * 72.0
-    dpi_h = scale_h * 72.0
-
-    return dpi_w, dpi_h
-
-
-class ImageInfo:
-    DPI_PREC = Decimal('1.000')
-
-    def __init__(self, *, name='', pdfimage=None, inline=None, shorthand=None):
-
-        self._name = str(name)
-        self._shorthand = shorthand
-
-        if inline is not None:
-            self._origin = 'inline'
-            pim = inline.iimage
-        elif pdfimage is not None:
-            self._origin = 'xobject'
-            pim = pikepdf.PdfImage(pdfimage)
-        self._width = pim.width
-        self._height = pim.height
-
-        # If /ImageMask is true, then this image is a stencil mask
-        # (Images that draw with this stencil mask will have a reference to
-        # it in their /Mask, but we don't actually need that information)
-        if pim.image_mask:
-            self._type = 'stencil'
-        else:
-            self._type = 'image'
-
-        self._bpc = int(pim.bits_per_component)
-        try:
-            self._enc = FRIENDLY_ENCODING.get(pim.filters[0], 'image')
-        except IndexError:
-            self._enc = '?'
-
-        try:
-            self._color = FRIENDLY_COLORSPACE.get(pim.colorspace, '?')
-        except NotImplementedError:
-            self._color = '?'
-        if self._enc == Encoding.jpeg2000:
-            self._color = Colorspace.jpeg2000
-
-        self._comp = FRIENDLY_COMP.get(self._color, '?')
-
-        # Bit of a hack... infer grayscale if component count is uncertain
-        # but encoding must be monochrome. This happens if a monochrome image
-        # has an ICC profile attached. Better solution would be to examine
-        # the ICC profile.
-        if self._comp == '?' and self._enc in (Encoding.ccitt, 'jbig2'):
-            self._comp = FRIENDLY_COMP[Colorspace.gray]
-
-    @property
-    def name(self):
-        return self._name
-
-    @property
-    def type_(self):
-        return self._type
-
-    @property
-    def width(self):
-        return self._width
-
-    @property
-    def height(self):
-        return self._height
-
-    @property
-    def bpc(self):
-        return self._bpc
-
-    @property
-    def color(self):
-        return self._color
-
-    @property
-    def comp(self):
-        return self._comp
-
-    @property
-    def enc(self):
-        return self._enc
-
-    @property
-    def xres(self):
-        return _get_dpi(self._shorthand, (self._width, self._height))[0]
-
-    @property
-    def yres(self):
-        return _get_dpi(self._shorthand, (self._width, self._height))[1]
-
-    def __repr__(self):
-        class_locals = {
-            attr: getattr(self, attr, None)
-            for attr in dir(self)
-            if not attr.startswith('_')
-        }
-        return (
-            "<ImageInfo '{name}' {type_} {width}x{height} {color} "
-            "{comp} {bpc} {enc} {xres}x{yres}>"
-        ).format(**class_locals)
-
-
-def _find_inline_images(contentsinfo):
-    "Find inline images in the contentstream"
-
-    for n, inline in enumerate(contentsinfo.inline_images):
-        yield ImageInfo(
-            name='inline-%02d' % n, shorthand=inline.shorthand, inline=inline
-        )
-
-
-def _image_xobjects(container):
-    """Search for all XObject-based images in the container
-
-    Usually the container is a page, but it could also be a Form XObject
-    that contains images. Filter out the Form XObjects which are dealt with
-    elsewhere.
-
-    Generate a sequence of tuples (image, xobj container), where container,
-    where xobj is the name of the object and image is the object itself,
-    since the object does not know its own name.
-
-    """
-
-    if '/Resources' not in container:
-        return
-    resources = container['/Resources']
-    if '/XObject' not in resources:
-        return
-    xobjs = resources['/XObject'].as_dict()
-    for xobj in xobjs:
-        candidate = xobjs[xobj]
-        if not '/Subtype' in candidate:
-            continue
-        if candidate['/Subtype'] == '/Image':
-            pdfimage = candidate
-            yield (pdfimage, xobj)
-
-
-def _find_regular_images(container, contentsinfo):
-    """Find images stored in the container's /Resources /XObject
-
-    Usually the container is a page, but it could also be a Form XObject
-    that contains images.
-
-    Generates images with their DPI at time of drawing.
-    """
-
-    for pdfimage, xobj in _image_xobjects(container):
-
-        # For each image that is drawn on this, check if we drawing the
-        # current image - yes this is O(n^2), but n == 1 almost always
-        for draw in contentsinfo.xobject_settings:
-            if draw.name != xobj:
-                continue
-
-            if draw.stack_depth == 0 and _is_unit_square(draw.shorthand):
-                # At least one PDF in the wild (and test suite) draws an image
-                # when the graphics stack depth is 0, meaning that the image
-                # gets drawn into a square of 1x1 PDF units (or 1/72",
-                # or 0.35 mm).  The equivalent DPI will be >100,000.  Exclude
-                # these from our DPI calculation for the page.
-                continue
-
-            yield ImageInfo(name=draw.name, pdfimage=pdfimage, shorthand=draw.shorthand)
-
-
-def _find_form_xobject_images(pdf, container, contentsinfo):
-    """Find any images that are in Form XObjects in the container
-
-    The container may be a page, or a parent Form XObject.
-
-    """
-    if '/Resources' not in container:
-        return
-    resources = container['/Resources']
-    if '/XObject' not in resources:
-        return
-    xobjs = resources['/XObject'].as_dict()
-    for xobj in xobjs:
-        candidate = xobjs[xobj]
-        if candidate['/Subtype'] != '/Form':
-            continue
-
-        form_xobject = candidate
-        for settings in contentsinfo.xobject_settings:
-            if settings.name != xobj:
-                continue
-
-            # Find images once for each time this Form XObject is drawn.
-            # This could be optimized to cache the multiple drawing events
-            # but in practice both Form XObjects and multiple drawing of the
-            # same object are both very rare.
-            ctm_shorthand = settings.shorthand
-            yield from _process_content_streams(
-                pdf=pdf, container=form_xobject, shorthand=ctm_shorthand
-            )
-
-
-def _process_content_streams(*, pdf, container, shorthand=None):
-    """Find all individual instances of images drawn in the container
-
-    Usually the container is a page, but it may also be a Form XObject.
-
-    On a typical page images are stored inline or as regular images
-    in an XObject.
-
-    Form XObjects may include inline images, XObject images,
-    and recursively, other Form XObjects; and also vector graphic objects.
-
-    Every instance of an image being drawn somewhere is flattened and
-    treated as a unique image, since if the same image is drawn multiple times
-    on one page it may be drawn at differing resolutions, and our objective
-    is to find the resolution at which the page can be rastered without
-    downsampling.
-
-    """
-
-    if container.get('/Type') == '/Page' and '/Contents' in container:
-        initial_shorthand = shorthand or UNIT_SQUARE
-    elif container.get('/Type') == '/XObject' and container['/Subtype'] == '/Form':
-        # Set the CTM to the state it was when the "Do" operator was
-        # encountered that is drawing this instance of the Form XObject
-        ctm = PdfMatrix(shorthand) if shorthand else PdfMatrix.identity()
-
-        # A Form XObject may provide its own matrix to map form space into
-        # user space. Get this if one exists
-        form_shorthand = container.get('/Matrix', PdfMatrix.identity())
-        form_matrix = PdfMatrix(form_shorthand)
-
-        # Concatenate form matrix with CTM to ensure CTM is correct for
-        # drawing this instance of the XObject
-        ctm = form_matrix @ ctm
-        initial_shorthand = ctm.shorthand
-    else:
-        return
-
-    contentsinfo = _interpret_contents(container, initial_shorthand)
-
-    if contentsinfo.found_vector:
-        yield VectorInfo()
-    yield from _find_inline_images(contentsinfo)
-    yield from _find_regular_images(container, contentsinfo)
-    yield from _find_form_xobject_images(pdf, container, contentsinfo)
-
-
-def _page_has_text(text_blocks, page_width, page_height):
-    """Smarter text detection that ignores text in margins"""
-
-    pw, ph = float(page_width), float(page_height)
-
-    margin_ratio = 0.125
-    interior_bbox = (
-        margin_ratio * pw,  # left
-        (1 - margin_ratio) * ph,  # top
-        (1 - margin_ratio) * pw,  # right
-        margin_ratio * ph,  # bottom  (first quadrant: bottom < top)
-    )
-
-    def rects_intersect(a, b):
-        """
-        Where (a,b) are 4-tuple rects (left-0, top-1, right-2, bottom-3)
-        https://stackoverflow.com/questions/306316/determine-if-two-rectangles-overlap-each-other
-        Formula assumes all boxes are in first quadrant
-        """
-        return a[0] < b[2] and a[2] > b[0] and a[1] > b[3] and a[3] < b[1]
-
-    has_text = False
-    for bbox in text_blocks:
-        if rects_intersect(bbox, interior_bbox):
-            has_text = True
-            break
-    return has_text
-
-
-def simplify_textboxes(miner, textbox_getter):
-    """Extract only limited content from text boxes
-
-    We do this to save memory and ensure that our objects are pickleable.
-    """
-    for box in textbox_getter(miner):
-        first_line = box._objs[0]
-        first_char = first_line._objs[0]
-
-        visible = first_char.rendermode != 3
-        corrupt = first_char.get_text() == '\ufffd'
-        yield TextboxInfo(box.bbox, visible, corrupt)
-
-
-def _pdf_get_pageinfo(pdf, pageno: int, infile, xmltext):
-    pageinfo = {}
-    pageinfo['pageno'] = pageno
-    pageinfo['images'] = []
-
-    page = pdf.pages[pageno]
-    mediabox = [Decimal(d) for d in page.MediaBox.as_list()]
-    width_pt = mediabox[2] - mediabox[0]
-    height_pt = mediabox[3] - mediabox[1]
-
-    if xmltext is not None:
-        bboxes = ghosttext.page_get_textblocks(
-            fspath(infile), pageno, xmltext=xmltext, height=height_pt
-        )
-        pageinfo['bboxes'] = bboxes
-    else:
-        pscript5_mode = str(pdf.docinfo.get('/Creator')).startswith('PScript5')
-        miner = get_page_analysis(infile, pageno, pscript5_mode)
-        pageinfo['textboxes'] = list(simplify_textboxes(miner, get_text_boxes))
-        bboxes = (box.bbox for box in pageinfo['textboxes'])
-
-    pageinfo['has_text'] = _page_has_text(bboxes, width_pt, height_pt)
-
-    userunit = page.get('/UserUnit', Decimal(1.0))
-    if not isinstance(userunit, Decimal):
-        userunit = Decimal(userunit)
-    pageinfo['userunit'] = userunit
-    pageinfo['width_inches'] = width_pt * userunit / Decimal(72.0)
-    pageinfo['height_inches'] = height_pt * userunit / Decimal(72.0)
-
-    try:
-        pageinfo['rotate'] = int(page['/Rotate'])
-    except KeyError:
-        pageinfo['rotate'] = 0
-
-    userunit_shorthand = (userunit, 0, 0, userunit, 0, 0)
-    contentsinfo = [
-        ci
-        for ci in _process_content_streams(
-            pdf=pdf, container=page, shorthand=userunit_shorthand
-        )
-    ]
-
-    pageinfo['has_vector'] = False
-    if any(isinstance(ci, VectorInfo) for ci in contentsinfo):
-        pageinfo['has_vector'] = True
-
-    pageinfo['images'] = [im for im in contentsinfo if isinstance(im, ImageInfo)]
-    if pageinfo['images']:
-        xres = Decimal(max(image.xres for image in pageinfo['images']))
-        yres = Decimal(max(image.yres for image in pageinfo['images']))
-        pageinfo['xres'], pageinfo['yres'] = xres, yres
-        pageinfo['width_pixels'] = int(round(xres * pageinfo['width_inches']))
-        pageinfo['height_pixels'] = int(round(yres * pageinfo['height_inches']))
-
-    return pageinfo
-
-
-def _pdf_get_all_pageinfo(infile, detailed_analysis=False, log=None):
-    if not log:
-        log = Mock()
-
-    pdf = pikepdf.open(infile)  # Do not close in this function
-    if pdf.is_encrypted:
-        pdf.close()
-        raise EncryptedPdfError()  # Triggered by encryption with empty passwd
-    if detailed_analysis:
-        pages_xml = None
-    else:
-        pages_xml = ghosttext.extract_text_xml(infile, pdf, pageno=None, log=log)
-
-    pages = []
-    for n in range(len(pdf.pages)):
-        page_xml = pages_xml[n] if pages_xml else None
-        page = PageInfo(pdf, n, infile, page_xml, detailed_analysis)
-        pages.append(page)
-
-    return pages, pdf
-
-
-class PageInfo:
-    def __init__(self, pdf, pageno, infile, xmltext, detailed_analysis=False):
-        self._pageno = pageno
-        self._infile = infile
-        self._pageinfo = _pdf_get_pageinfo(pdf, pageno, infile, xmltext)
-        self._detailed_analysis = detailed_analysis
-
-    @property
-    def pageno(self):
-        return self._pageno
-
-    @property
-    def has_text(self):
-        return self._pageinfo['has_text']
-
-    @property
-    def has_corrupt_text(self):
-        if not self._detailed_analysis:
-            raise NotImplementedError('Did not do detailed analysis')
-        return any(tbox.is_corrupt for tbox in self._pageinfo['textboxes'])
-
-    @property
-    def has_vector(self):
-        return self._pageinfo['has_vector']
-
-    @property
-    def width_inches(self):
-        return self._pageinfo['width_inches']
-
-    @property
-    def height_inches(self):
-        return self._pageinfo['height_inches']
-
-    @property
-    def width_pixels(self):
-        return int(round(self.width_inches * self.xres))
-
-    @property
-    def height_pixels(self):
-        return int(round(self.height_inches * self.yres))
-
-    @property
-    def rotation(self):
-        return self._pageinfo.get('rotate', None)
-
-    @rotation.setter
-    def rotation(self, value):
-        if value in (0, 90, 180, 270, 360, -90, -180, -270):
-            self._pageinfo['rotate'] = value
-        else:
-            raise ValueError("rotation must be a cardinal angle")
-
-    @property
-    def images(self):
-        return self._pageinfo['images']
-
-    def get_textareas(self, visible=None, corrupt=None):
-        def predicate(obj, want_visible, want_corrupt):
-            result = True
-            if want_visible is not None:
-                if obj.is_visible != want_visible:
-                    result = False
-            if want_corrupt is not None:
-                if obj.is_corrupt != want_corrupt:
-                    result = False
-            return result
-
-        if 'textboxes' not in self._pageinfo:
-            if visible is not None and corrupt is not None:
-                raise NotImplementedError('Ghostscript textboxes cannot be classified')
-            return self._pageinfo['bboxes']
-
-        return (
-            obj.bbox
-            for obj in self._pageinfo['textboxes']
-            if predicate(obj, visible, corrupt)
-        )
-
-    @property
-    def xres(self):
-        return self._pageinfo.get('xres', None)
-
-    @property
-    def yres(self):
-        return self._pageinfo.get('yres', None)
-
-    @property
-    def userunit(self):
-        return self._pageinfo.get('userunit', None)
-
-    @property
-    def min_version(self):
-        if self.userunit is not None:
-            return '1.6'
-        else:
-            return '1.5'
-
-    def __repr__(self):
-        return (
-            '<PageInfo ' 'pageno={} {}"x{}" rotation={} res={}x{} has_text={}>'
-        ).format(
-            self.pageno,
-            self.width_inches,
-            self.height_inches,
-            self.rotation,
-            self.xres,
-            self.yres,
-            self.has_text,
-        )
-
-
-class PdfInfo:
-    """Get summary information about a PDF"""
-
-    def __init__(self, infile, detailed_page_analysis=False, log=None):
-        self._infile = infile
-        self._pages, pdf = _pdf_get_all_pageinfo(
-            infile, detailed_page_analysis, log=log
-        )
-        self._needs_rendering = pdf.root.get('/NeedsRendering', False)
-        self._has_acroform = False
-        if '/AcroForm' in pdf.root:
-            if len(pdf.root.AcroForm.get('/Fields', [])) > 0:
-                self._has_acroform = True
-            elif '/XFA' in pdf.root.AcroForm:
-                self._has_acroform = True
-        pdf.close()
-
-    @property
-    def pages(self):
-        return self._pages
-
-    @property
-    def min_version(self):
-        # The minimum PDF is the maximum version that any particular page needs
-        return max(page.min_version for page in self.pages)
-
-    @property
-    def has_userunit(self):
-        return any(page.userunit != 1.0 for page in self.pages)
-
-    @property
-    def has_acroform(self):
-        return self._has_acroform
-
-    @property
-    def filename(self):
-        if not isinstance(self._infile, (str, Path)):
-            raise NotImplementedError("can't get filename from stream")
-        return self._infile
-
-    @property
-    def needs_rendering(self):
-        return self._needs_rendering
-
-    def __getitem__(self, item):
-        return self._pages[item]
-
-    def __len__(self):
-        return len(self._pages)
-
-    def __repr__(self):
-        return f"<PdfInfo('...'), page count={len(self)}>"
-
-
-def main():
-    import argparse
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument('infile')
-    args = parser.parse_args()
-    info = _pdf_get_all_pageinfo(args.infile)
-    from pprint import pprint
-
-    pprint(info)
-
-
-if __name__ == '__main__':
-    main()
+from .info import PdfInfo, Colorspace, Encoding

src/ocrmypdf/pdfinfo/ghosttext.py

@@ -15,11 +15,14 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
+import logging
 import re
 import xml.etree.ElementTree as ET
 
 from ..exec import ghostscript
 
+gslog = logging.getLogger()
+
 # Forgive me for I have sinned
 # I am using regular expressions to parse XML. However the XML in this case,
 # generated by Ghostscript, is self-consistent enough to be parseable.
@@ -74,7 +77,7 @@ def page_get_textblocks(infile, pageno, xmltext, height):
     return [block for block in joined_blocks()]
 
 
-def extract_text_xml(infile, pdf, pageno=None, log=None):
+def extract_text_xml(infile, pdf, pageno=None, log=gslog):
     existing_text = ghostscript.extract_text(infile, pageno=None)
     existing_text = regex_remove_char_tags.sub(b' ', existing_text)
 

src/ocrmypdf/pdfinfo/info.py

@@ -0,0 +1,814 @@
+#!/usr/bin/env python3
+# © 2015 James R. Barlow: github.com/jbarlow83
+#
+# This file is part of OCRmyPDF.
+#
+# OCRmyPDF is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# OCRmyPDF is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
+
+from collections import namedtuple
+from decimal import Decimal
+from enum import Enum
+import logging
+from math import hypot, isclose
+from os import fspath
+from pathlib import Path
+from warnings import warn
+import re
+
+from pikepdf import PdfMatrix
+import pikepdf
+from tqdm import tqdm
+
+from . import ghosttext
+from .layout import get_page_analysis, get_text_boxes
+from ocrmypdf.exceptions import EncryptedPdfError, MissingDependencyError
+
+logger = logging.getLogger()
+
+Colorspace = Enum('Colorspace', 'gray rgb cmyk lab icc index sep devn pattern jpeg2000')
+
+Encoding = Enum(
+    'Encoding', 'ccitt jpeg jpeg2000 jbig2 asciihex ascii85 lzw flate ' + 'runlength'
+)
+
+FRIENDLY_COLORSPACE = {
+    '/DeviceGray': Colorspace.gray,
+    '/CalGray': Colorspace.gray,
+    '/DeviceRGB': Colorspace.rgb,
+    '/CalRGB': Colorspace.rgb,
+    '/DeviceCMYK': Colorspace.cmyk,
+    '/Lab': Colorspace.lab,
+    '/ICCBased': Colorspace.icc,
+    '/Indexed': Colorspace.index,
+    '/Separation': Colorspace.sep,
+    '/DeviceN': Colorspace.devn,
+    '/Pattern': Colorspace.pattern,
+    '/G': Colorspace.gray,  # Abbreviations permitted in inline images
+    '/RGB': Colorspace.rgb,
+    '/CMYK': Colorspace.cmyk,
+    '/I': Colorspace.index,
+}
+
+FRIENDLY_ENCODING = {
+    '/CCITTFaxDecode': Encoding.ccitt,
+    '/DCTDecode': Encoding.jpeg,
+    '/JPXDecode': Encoding.jpeg2000,
+    '/JBIG2Decode': Encoding.jbig2,
+    '/CCF': Encoding.ccitt,  # Abbreviations permitted in inline images
+    '/DCT': Encoding.jpeg,
+    '/AHx': Encoding.asciihex,
+    '/A85': Encoding.ascii85,
+    '/LZW': Encoding.lzw,
+    '/Fl': Encoding.flate,
+    '/RL': Encoding.runlength,
+}
+
+FRIENDLY_COMP = {
+    Colorspace.gray: 1,
+    Colorspace.rgb: 3,
+    Colorspace.cmyk: 4,
+    Colorspace.lab: 3,
+    Colorspace.index: 1,
+}
+
+
+UNIT_SQUARE = (1.0, 0.0, 0.0, 1.0, 0.0, 0.0)
+
+
+def _is_unit_square(shorthand):
+    values = map(float, shorthand)
+    pairwise = zip(values, UNIT_SQUARE)
+    return all([isclose(a, b, rel_tol=1e-3) for a, b in pairwise])
+
+
+XobjectSettings = namedtuple('XobjectSettings', ['name', 'shorthand', 'stack_depth'])
+
+InlineSettings = namedtuple('InlineSettings', ['iimage', 'shorthand', 'stack_depth'])
+
+ContentsInfo = namedtuple(
+    'ContentsInfo', ['xobject_settings', 'inline_images', 'found_vector']
+)
+
+TextboxInfo = namedtuple('TextboxInfo', ['bbox', 'is_visible', 'is_corrupt'])
+
+
+class VectorInfo:
+    def __init__(self):
+        pass
+
+
+def _normalize_stack(graphobjs):
+    """Convert runs of qQ's in the stack into single graphobjs"""
+    for operands, operator in graphobjs:
+        operator = str(operator)
+        if re.match(r'Q*q+$', operator):  # Zero or more Q, one or more q
+            for char in operator:  # Split into individual
+                yield ([], char)  # Yield individual
+        else:
+            yield (operands, operator)
+
+
+def _interpret_contents(contentstream, initial_shorthand=UNIT_SQUARE):
+    """Interpret the PDF content stream.
+
+    The stack represents the state of the PDF graphics stack.  We are only
+    interested in the current transformation matrix (CTM) so we only track
+    this object; a full implementation would need to track many other items.
+
+    The CTM is initialized to the mapping from user space to device space.
+    PDF units are 1/72".  In a PDF viewer or printer this matrix is initialized
+    to the transformation to device space.  For example if set to
+    (1/72, 0, 0, 1/72, 0, 0) then all units would be calculated in inches.
+
+    Images are always considered to be (0, 0) -> (1, 1).  Before drawing an
+    image there should be a 'cm' that sets up an image coordinate system
+    where drawing from (0, 0) -> (1, 1) will draw on the desired area of the
+    page.
+
+    PDF units suit our needs so we initialize ctm to the identity matrix.
+
+    According to the PDF specification, the maximum stack depth is 32. Other
+    viewers tolerate some amount beyond this.  We issue a warning if the
+    stack depth exceeds the spec limit and set a hard limit beyond this to
+    bound our memory requirements.  If the stack underflows behavior is
+    undefined in the spec, but we just pretend nothing happened and leave the
+    CTM unchanged.
+    """
+
+    stack = []
+    ctm = PdfMatrix(initial_shorthand)
+    xobject_settings = []
+    inline_images = []
+    found_vector = False
+    vector_ops = set('S s f F f* B B* b b*'.split())
+    image_ops = set('BI ID EI q Q Do cm'.split())
+    operator_whitelist = ' '.join(vector_ops | image_ops)
+
+    for n, graphobj in enumerate(
+        _normalize_stack(
+            pikepdf.parse_content_stream(contentstream, operator_whitelist)
+        )
+    ):
+        operands, operator = graphobj
+        if operator == 'q':
+            stack.append(ctm)
+            if len(stack) > 32:  # See docstring
+                if len(stack) > 128:
+                    raise RuntimeError(
+                        "PDF graphics stack overflowed hard limit, operator %i" % n
+                    )
+                warn("PDF graphics stack overflowed spec limit")
+        elif operator == 'Q':
+            try:
+                ctm = stack.pop()
+            except IndexError:
+                # Keeping the ctm the same seems to be the only sensible thing
+                # to do. Just pretend nothing happened, keep calm and carry on.
+                warn("PDF graphics stack underflowed - PDF may be malformed")
+        elif operator == 'cm':
+            ctm = PdfMatrix(operands) @ ctm
+        elif operator == 'Do':
+            image_name = operands[0]
+            settings = XobjectSettings(
+                name=image_name, shorthand=ctm.shorthand, stack_depth=len(stack)
+            )
+            xobject_settings.append(settings)
+        elif operator == 'INLINE IMAGE':  # BI/ID/EI are grouped into this
+            iimage = operands[0]
+            inline = InlineSettings(
+                iimage=iimage, shorthand=ctm.shorthand, stack_depth=len(stack)
+            )
+            inline_images.append(inline)
+        elif operator in vector_ops:
+            found_vector = True
+
+    return ContentsInfo(
+        xobject_settings=xobject_settings,
+        inline_images=inline_images,
+        found_vector=found_vector,
+    )
+
+
+def _get_dpi(ctm_shorthand, image_size):
+    """Given the transformation matrix and image size, find the image DPI.
+
+    PDFs do not include image resolution information within image data.
+    Instead, the PDF page content stream describes the location where the
+    image will be rasterized, and the effective resolution is the ratio of the
+    pixel size to raster target size.
+
+    Normally a scanned PDF has the paper size set appropriately but this is
+    not guaranteed. The most common case is a cropped image will change the
+    page size (/CropBox) without altering the page content stream. That means
+    it is not sufficient to assume that the image fills the page, even though
+    that is the most common case.
+
+    A PDF image may be scaled (always), cropped, translated, rotated in place
+    to an arbitrary angle (rarely) and skewed. Only equal area mappings can
+    be expressed, that is, it is not necessary to consider distortions where
+    the effective DPI varies with position.
+
+    To determine the image scale, transform an offset axis vector v0 (0, 0),
+    width-axis vector v0 (1, 0), height-axis vector vh (0, 1) with the matrix,
+    which gives the dimensions of the image in PDF units. From there we can
+    compare to actual image dimensions. PDF uses
+    row vector * matrix_tranposed unlike the traditional
+    matrix * column vector.
+
+    The offset, width and height vectors can be combined in a matrix and
+    multiplied by the transform matrix. Then we want to calculated
+        magnitude(width_vector - offset_vector)
+    and
+        magnitude(height_vector - offset_vector)
+
+    When the above is worked out algebraically, the effect of translation
+    cancels out, and the vector magnitudes become functions of the nonzero
+    transformation matrix indices. The results of the derivation are used
+    in this code.
+
+    pdfimages -list does calculate the DPI in some way that is not completely
+    naive, but it does not get the DPI of rotated images right, so cannot be
+    used anymore to validate this. Photoshop works, or using Acrobat to
+    rotate the image back to normal.
+
+    It does not matter if the image is partially cropped, or even out of the
+    /MediaBox.
+
+    """
+
+    a, b, c, d, _, _ = ctm_shorthand
+
+    # Calculate the width and height of the image in PDF units
+    image_drawn_width = hypot(a, b)
+    image_drawn_height = hypot(c, d)
+
+    # The scale of the image is pixels per unit of default user space (1/72")
+    scale_w = image_size[0] / image_drawn_width
+    scale_h = image_size[1] / image_drawn_height
+
+    # DPI = scale * 72
+    dpi_w = scale_w * 72.0
+    dpi_h = scale_h * 72.0
+
+    return dpi_w, dpi_h
+
+
+class ImageInfo:
+    DPI_PREC = Decimal('1.000')
+
+    def __init__(self, *, name='', pdfimage=None, inline=None, shorthand=None):
+
+        self._name = str(name)
+        self._shorthand = shorthand
+
+        if inline is not None:
+            self._origin = 'inline'
+            pim = inline.iimage
+        elif pdfimage is not None:
+            self._origin = 'xobject'
+            pim = pikepdf.PdfImage(pdfimage)
+        self._width = pim.width
+        self._height = pim.height
+
+        # If /ImageMask is true, then this image is a stencil mask
+        # (Images that draw with this stencil mask will have a reference to
+        # it in their /Mask, but we don't actually need that information)
+        if pim.image_mask:
+            self._type = 'stencil'
+        else:
+            self._type = 'image'
+
+        self._bpc = int(pim.bits_per_component)
+        try:
+            self._enc = FRIENDLY_ENCODING.get(pim.filters[0], 'image')
+        except IndexError:
+            self._enc = '?'
+
+        try:
+            self._color = FRIENDLY_COLORSPACE.get(pim.colorspace, '?')
+        except NotImplementedError:
+            self._color = '?'
+        if self._enc == Encoding.jpeg2000:
+            self._color = Colorspace.jpeg2000
+
+        self._comp = FRIENDLY_COMP.get(self._color, '?')
+
+        # Bit of a hack... infer grayscale if component count is uncertain
+        # but encoding must be monochrome. This happens if a monochrome image
+        # has an ICC profile attached. Better solution would be to examine
+        # the ICC profile.
+        if self._comp == '?' and self._enc in (Encoding.ccitt, Encoding.jbig2):
+            self._comp = FRIENDLY_COMP[Colorspace.gray]
+
+    @property
+    def name(self):
+        return self._name
+
+    @property
+    def type_(self):
+        return self._type
+
+    @property
+    def width(self):
+        return self._width
+
+    @property
+    def height(self):
+        return self._height
+
+    @property
+    def bpc(self):
+        return self._bpc
+
+    @property
+    def color(self):
+        return self._color
+
+    @property
+    def comp(self):
+        return self._comp
+
+    @property
+    def enc(self):
+        return self._enc
+
+    @property
+    def xres(self):
+        return _get_dpi(self._shorthand, (self._width, self._height))[0]
+
+    @property
+    def yres(self):
+        return _get_dpi(self._shorthand, (self._width, self._height))[1]
+
+    def __repr__(self):
+        class_locals = {
+            attr: getattr(self, attr, None)
+            for attr in dir(self)
+            if not attr.startswith('_')
+        }
+        return (
+            "<ImageInfo '{name}' {type_} {width}x{height} {color} "
+            "{comp} {bpc} {enc} {xres}x{yres}>"
+        ).format(**class_locals)
+
+
+def _find_inline_images(contentsinfo):
+    "Find inline images in the contentstream"
+
+    for n, inline in enumerate(contentsinfo.inline_images):
+        yield ImageInfo(
+            name='inline-%02d' % n, shorthand=inline.shorthand, inline=inline
+        )
+
+
+def _image_xobjects(container):
+    """Search for all XObject-based images in the container
+
+    Usually the container is a page, but it could also be a Form XObject
+    that contains images. Filter out the Form XObjects which are dealt with
+    elsewhere.
+
+    Generate a sequence of tuples (image, xobj container), where container,
+    where xobj is the name of the object and image is the object itself,
+    since the object does not know its own name.
+
+    """
+
+    if '/Resources' not in container:
+        return
+    resources = container['/Resources']
+    if '/XObject' not in resources:
+        return
+    xobjs = resources['/XObject'].as_dict()
+    for xobj in xobjs:
+        candidate = xobjs[xobj]
+        if not '/Subtype' in candidate:
+            continue
+        if candidate['/Subtype'] == '/Image':
+            pdfimage = candidate
+            yield (pdfimage, xobj)
+
+
+def _find_regular_images(container, contentsinfo):
+    """Find images stored in the container's /Resources /XObject
+
+    Usually the container is a page, but it could also be a Form XObject
+    that contains images.
+
+    Generates images with their DPI at time of drawing.
+    """
+
+    for pdfimage, xobj in _image_xobjects(container):
+
+        # For each image that is drawn on this, check if we drawing the
+        # current image - yes this is O(n^2), but n == 1 almost always
+        for draw in contentsinfo.xobject_settings:
+            if draw.name != xobj:
+                continue
+
+            if draw.stack_depth == 0 and _is_unit_square(draw.shorthand):
+                # At least one PDF in the wild (and test suite) draws an image
+                # when the graphics stack depth is 0, meaning that the image
+                # gets drawn into a square of 1x1 PDF units (or 1/72",
+                # or 0.35 mm).  The equivalent DPI will be >100,000.  Exclude
+                # these from our DPI calculation for the page.
+                continue
+
+            yield ImageInfo(name=draw.name, pdfimage=pdfimage, shorthand=draw.shorthand)
+
+
+def _find_form_xobject_images(pdf, container, contentsinfo):
+    """Find any images that are in Form XObjects in the container
+
+    The container may be a page, or a parent Form XObject.
+
+    """
+    if '/Resources' not in container:
+        return
+    resources = container['/Resources']
+    if '/XObject' not in resources:
+        return
+    xobjs = resources['/XObject'].as_dict()
+    for xobj in xobjs:
+        candidate = xobjs[xobj]
+        if candidate['/Subtype'] != '/Form':
+            continue
+
+        form_xobject = candidate
+        for settings in contentsinfo.xobject_settings:
+            if settings.name != xobj:
+                continue
+
+            # Find images once for each time this Form XObject is drawn.
+            # This could be optimized to cache the multiple drawing events
+            # but in practice both Form XObjects and multiple drawing of the
+            # same object are both very rare.
+            ctm_shorthand = settings.shorthand
+            yield from _process_content_streams(
+                pdf=pdf, container=form_xobject, shorthand=ctm_shorthand
+            )
+
+
+def _process_content_streams(*, pdf, container, shorthand=None):
+    """Find all individual instances of images drawn in the container
+
+    Usually the container is a page, but it may also be a Form XObject.
+
+    On a typical page images are stored inline or as regular images
+    in an XObject.
+
+    Form XObjects may include inline images, XObject images,
+    and recursively, other Form XObjects; and also vector graphic objects.
+
+    Every instance of an image being drawn somewhere is flattened and
+    treated as a unique image, since if the same image is drawn multiple times
+    on one page it may be drawn at differing resolutions, and our objective
+    is to find the resolution at which the page can be rastered without
+    downsampling.
+
+    """
+
+    if container.get('/Type') == '/Page' and '/Contents' in container:
+        initial_shorthand = shorthand or UNIT_SQUARE
+    elif container.get('/Type') == '/XObject' and container['/Subtype'] == '/Form':
+        # Set the CTM to the state it was when the "Do" operator was
+        # encountered that is drawing this instance of the Form XObject
+        ctm = PdfMatrix(shorthand) if shorthand else PdfMatrix.identity()
+
+        # A Form XObject may provide its own matrix to map form space into
+        # user space. Get this if one exists
+        form_shorthand = container.get('/Matrix', PdfMatrix.identity())
+        form_matrix = PdfMatrix(form_shorthand)
+
+        # Concatenate form matrix with CTM to ensure CTM is correct for
+        # drawing this instance of the XObject
+        ctm = form_matrix @ ctm
+        initial_shorthand = ctm.shorthand
+    else:
+        return
+
+    contentsinfo = _interpret_contents(container, initial_shorthand)
+
+    if contentsinfo.found_vector:
+        yield VectorInfo()
+    yield from _find_inline_images(contentsinfo)
+    yield from _find_regular_images(container, contentsinfo)
+    yield from _find_form_xobject_images(pdf, container, contentsinfo)
+
+
+def _page_has_text(text_blocks, page_width, page_height):
+    """Smarter text detection that ignores text in margins"""
+
+    pw, ph = float(page_width), float(page_height)
+
+    margin_ratio = 0.125
+    interior_bbox = (
+        margin_ratio * pw,  # left
+        (1 - margin_ratio) * ph,  # top
+        (1 - margin_ratio) * pw,  # right
+        margin_ratio * ph,  # bottom  (first quadrant: bottom < top)
+    )
+
+    def rects_intersect(a, b):
+        """
+        Where (a,b) are 4-tuple rects (left-0, top-1, right-2, bottom-3)
+        https://stackoverflow.com/questions/306316/determine-if-two-rectangles-overlap-each-other
+        Formula assumes all boxes are in first quadrant
+        """
+        return a[0] < b[2] and a[2] > b[0] and a[1] > b[3] and a[3] < b[1]
+
+    has_text = False
+    for bbox in text_blocks:
+        if rects_intersect(bbox, interior_bbox):
+            has_text = True
+            break
+    return has_text
+
+
+def simplify_textboxes(miner, textbox_getter):
+    """Extract only limited content from text boxes
+
+    We do this to save memory and ensure that our objects are pickleable.
+    """
+    for box in textbox_getter(miner):
+        first_line = box._objs[0]
+        first_char = first_line._objs[0]
+
+        visible = first_char.rendermode != 3
+        corrupt = first_char.get_text() == '\ufffd'
+        yield TextboxInfo(box.bbox, visible, corrupt)
+
+
+def _pdf_get_pageinfo(pdf, pageno: int, infile, xmltext):
+    pageinfo = {}
+    pageinfo['pageno'] = pageno
+    pageinfo['images'] = []
+
+    page = pdf.pages[pageno]
+    mediabox = [Decimal(d) for d in page.MediaBox.as_list()]
+    width_pt = mediabox[2] - mediabox[0]
+    height_pt = mediabox[3] - mediabox[1]
+
+    if xmltext is not None:
+        bboxes = ghosttext.page_get_textblocks(
+            fspath(infile), pageno, xmltext=xmltext, height=height_pt
+        )
+        pageinfo['bboxes'] = bboxes
+    else:
+        pscript5_mode = str(pdf.docinfo.get('/Creator')).startswith('PScript5')
+        miner = get_page_analysis(infile, pageno, pscript5_mode)
+        pageinfo['textboxes'] = list(simplify_textboxes(miner, get_text_boxes))
+        bboxes = (box.bbox for box in pageinfo['textboxes'])
+
+    pageinfo['has_text'] = _page_has_text(bboxes, width_pt, height_pt)
+
+    userunit = page.get('/UserUnit', Decimal(1.0))
+    if not isinstance(userunit, Decimal):
+        userunit = Decimal(userunit)
+    pageinfo['userunit'] = userunit
+    pageinfo['width_inches'] = width_pt * userunit / Decimal(72.0)
+    pageinfo['height_inches'] = height_pt * userunit / Decimal(72.0)
+
+    try:
+        pageinfo['rotate'] = int(page['/Rotate'])
+    except KeyError:
+        pageinfo['rotate'] = 0
+
+    userunit_shorthand = (userunit, 0, 0, userunit, 0, 0)
+    contentsinfo = [
+        ci
+        for ci in _process_content_streams(
+            pdf=pdf, container=page, shorthand=userunit_shorthand
+        )
+    ]
+
+    pageinfo['has_vector'] = False
+    if any(isinstance(ci, VectorInfo) for ci in contentsinfo):
+        pageinfo['has_vector'] = True
+
+    pageinfo['images'] = [im for im in contentsinfo if isinstance(im, ImageInfo)]
+    if pageinfo['images']:
+        xres = Decimal(max(image.xres for image in pageinfo['images']))
+        yres = Decimal(max(image.yres for image in pageinfo['images']))
+        pageinfo['xres'], pageinfo['yres'] = xres, yres
+        pageinfo['width_pixels'] = int(round(xres * pageinfo['width_inches']))
+        pageinfo['height_pixels'] = int(round(yres * pageinfo['height_inches']))
+
+    return pageinfo
+
+
+def _pdf_get_all_pageinfo(infile, detailed_analysis=False, log=None, progbar=False):
+    pdf = pikepdf.open(infile)  # Do not close in this function
+    if pdf.is_encrypted:
+        pdf.close()
+        raise EncryptedPdfError()  # Triggered by encryption with empty passwd
+    if detailed_analysis:
+        pages_xml = None
+    else:
+        pages_xml = ghosttext.extract_text_xml(infile, pdf, pageno=None, log=log)
+
+    pages = []
+    for n, _ in tqdm(
+        enumerate(pdf.pages),
+        total=len(pdf.pages),
+        desc="Scan",
+        unit='page',
+        disable=not progbar,
+    ):
+        page_xml = pages_xml[n] if pages_xml else None
+        page = PageInfo(pdf, n, infile, page_xml, detailed_analysis)
+        pages.append(page)
+
+    return pages, pdf
+
+
+class PageInfo:
+    def __init__(self, pdf, pageno, infile, xmltext, detailed_analysis=False):
+        self._pageno = pageno
+        self._infile = infile
+        self._pageinfo = _pdf_get_pageinfo(pdf, pageno, infile, xmltext)
+        self._detailed_analysis = detailed_analysis
+
+    @property
+    def pageno(self):
+        return self._pageno
+
+    @property
+    def has_text(self):
+        return self._pageinfo['has_text']
+
+    @property
+    def has_corrupt_text(self):
+        if not self._detailed_analysis:
+            raise NotImplementedError('Did not do detailed analysis')
+        return any(tbox.is_corrupt for tbox in self._pageinfo['textboxes'])
+
+    @property
+    def has_vector(self):
+        return self._pageinfo['has_vector']
+
+    @property
+    def width_inches(self):
+        return self._pageinfo['width_inches']
+
+    @property
+    def height_inches(self):
+        return self._pageinfo['height_inches']
+
+    @property
+    def width_pixels(self):
+        return int(round(self.width_inches * self.xres))
+
+    @property
+    def height_pixels(self):
+        return int(round(self.height_inches * self.yres))
+
+    @property
+    def rotation(self):
+        return self._pageinfo.get('rotate', None)
+
+    @rotation.setter
+    def rotation(self, value):
+        if value in (0, 90, 180, 270, 360, -90, -180, -270):
+            self._pageinfo['rotate'] = value
+        else:
+            raise ValueError("rotation must be a cardinal angle")
+
+    @property
+    def images(self):
+        return self._pageinfo['images']
+
+    def get_textareas(self, visible=None, corrupt=None):
+        def predicate(obj, want_visible, want_corrupt):
+            result = True
+            if want_visible is not None:
+                if obj.is_visible != want_visible:
+                    result = False
+            if want_corrupt is not None:
+                if obj.is_corrupt != want_corrupt:
+                    result = False
+            return result
+
+        if 'textboxes' not in self._pageinfo:
+            if visible is not None and corrupt is not None:
+                raise NotImplementedError('Ghostscript textboxes cannot be classified')
+            return self._pageinfo['bboxes']
+
+        return (
+            obj.bbox
+            for obj in self._pageinfo['textboxes']
+            if predicate(obj, visible, corrupt)
+        )
+
+    @property
+    def xres(self):
+        return self._pageinfo.get('xres', None)
+
+    @property
+    def yres(self):
+        return self._pageinfo.get('yres', None)
+
+    @property
+    def userunit(self):
+        return self._pageinfo.get('userunit', None)
+
+    @property
+    def min_version(self):
+        if self.userunit is not None:
+            return '1.6'
+        else:
+            return '1.5'
+
+    def __repr__(self):
+        return (
+            '<PageInfo ' 'pageno={} {}"x{}" rotation={} res={}x{} has_text={}>'
+        ).format(
+            self.pageno,
+            self.width_inches,
+            self.height_inches,
+            self.rotation,
+            self.xres,
+            self.yres,
+            self.has_text,
+        )
+
+
+class PdfInfo:
+    """Get summary information about a PDF"""
+
+    def __init__(self, infile, detailed_page_analysis=False, log=logger, progbar=False):
+        self._infile = infile
+        self._pages, pdf = _pdf_get_all_pageinfo(
+            infile, detailed_page_analysis, log=log, progbar=progbar
+        )
+        self._needs_rendering = pdf.root.get('/NeedsRendering', False)
+        self._has_acroform = False
+        if '/AcroForm' in pdf.root:
+            if len(pdf.root.AcroForm.get('/Fields', [])) > 0:
+                self._has_acroform = True
+            elif '/XFA' in pdf.root.AcroForm:
+                self._has_acroform = True
+        pdf.close()
+
+    @property
+    def pages(self):
+        return self._pages
+
+    @property
+    def min_version(self):
+        # The minimum PDF is the maximum version that any particular page needs
+        return max(page.min_version for page in self.pages)
+
+    @property
+    def has_userunit(self):
+        return any(page.userunit != 1.0 for page in self.pages)
+
+    @property
+    def has_acroform(self):
+        return self._has_acroform
+
+    @property
+    def filename(self):
+        if not isinstance(self._infile, (str, Path)):
+            raise NotImplementedError("can't get filename from stream")
+        return self._infile
+
+    @property
+    def needs_rendering(self):
+        return self._needs_rendering
+
+    def __getitem__(self, item):
+        return self._pages[item]
+
+    def __len__(self):
+        return len(self._pages)
+
+    def __repr__(self):
+        return f"<PdfInfo('...'), page count={len(self)}>"
+
+
+def main():
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('infile')
+    args = parser.parse_args()
+    info = _pdf_get_all_pageinfo(args.infile)
+    from pprint import pprint
+
+    pprint(info)
+
+
+if __name__ == '__main__':
+    main()

src/ocrmypdf/pdfinfo/layout.py

@@ -172,6 +172,7 @@ class LTStateAwareChar(LTChar):
             - the Unicode mapping is known, and both have the same render mode
             - the Unicode mapping is unknown but both are part of the same font
         """
+        # pylint: disable=protected-access
         both_unicode_mapped = isinstance(self._text, str) and isinstance(obj._text, str)
         try:
             if both_unicode_mapped:
@@ -184,7 +185,7 @@ class LTStateAwareChar(LTChar):
 
     def get_text(self):
         if isinstance(self._text, tuple):
-            return '�'
+            return '\ufffd'  # standard 'Unknown symbol'
         return self._text
 
     def __repr__(self):