v8.1.0 release notes

If using --tesseract-timeout 0 and any image processing on a file with
more than 100 pages, the weave handoff will occur. Ensure this
works correctly even if no Glyphless font is present.

Closes #347

.docker/webservice.dockerfile

@@ -16,4 +16,9 @@ COPY .docker/webservice.py /application
 
 USER docker
 
+VOLUME ["/config"]
+
+# This config file is optional
+ENV OCRMYPDF_WEBSERVICE_SETTINGS "/config/config.py"
+
 ENTRYPOINT ["python3", "/application/webservice.py"]

.docker/webservice.py

@@ -32,11 +32,9 @@ import shlex
 
 app = Flask(__name__)
 app.secret_key = "secret"
+app.config['MAX_CONTENT_LENGTH'] = 50_000_000
+app.config.from_envvar("OCRMYPDF_WEBSERVICE_SETTINGS", silent=True)
 
-uploaddir = TemporaryDirectory(prefix="ocrmypdf-upload")
-downloaddir = TemporaryDirectory(prefix="ocrmypdf-download")
-
-app.config["UPLOAD_FOLDER"] = uploaddir
 ALLOWED_EXTENSIONS = set(["pdf"])
 
 
@@ -45,9 +43,13 @@ def allowed_file(filename):
 
 
 def do_ocrmypdf(file):
+    uploaddir = TemporaryDirectory(prefix="ocrmypdf-upload")
+    downloaddir = TemporaryDirectory(prefix="ocrmypdf-download")
+
     filename = secure_filename(file.filename)
     up_file = os.path.join(uploaddir.name, filename)
     file.save(up_file)
+
     down_file = os.path.join(downloaddir.name, filename)
 
     cmd_args = [arg for arg in shlex.split(request.form["params"])]

.pre-commit-config.yaml

@@ -0,0 +1,6 @@
+repos:
+-   repo: https://github.com/ambv/black
+    rev: stable
+    hooks:
+    - id: black
+      language_version: python3.6

docs/advanced.rst

@@ -1,6 +1,34 @@
 Advanced features
 =================
 
+Control of unpaper
+------------------
+
+OCRmyPDF uses ``unpaper`` to provide the implementation of the ``--clean`` and ``--clean-final`` arguments. `unpaper <https://github.com/Flameeyes/unpaper/blob/master/doc/basic-concepts.md>`_ provides a variety of image processing filters to improve images.
+
+By default, OCRmyPDF uses only ``unpaper`` arguments that were found to be safe to use on almost all files without having to inspect every page of the file afterwards. This is particularly true when only ``--clean`` is used, since that instructs OCRmyPDF to only clean the image before OCR and not the final image.
+
+However, if you wish to use the more aggressive options in ``unpaper``, you may use ``--unpaper-args '...'`` to override the OCRmyPDF's defaults and forward other arguments to unpaper. This option will forward arguments to ``unpaper`` without any knowledge of what that program considers to be valid arguments. The string of arguments must be quoted as shown in the examples below. No filename arguments may be included. OCRmyPDF will assume it can append input and output filename of intermediate images to the ``--unpaper-args`` string.
+
+In this example, we tell ``unpaper`` to expect two pages of text on a sheet (image), such as occurs when two facing pages of a book are scanned. ``unpaper`` uses this information to deskew each independently and clean up the margins of both.
+
+.. code-block:: bash
+
+    ocrmypdf --clean --clean-final --unpaper-args '--layout double' input.pdf output.pdf
+    ocrmypdf --clean --clean-final --unpaper-args '--layout double --no-noisefilter' input.pdf output.pdf
+
+.. warning::
+
+    Some ``unpaper`` features will reposition text within the image. ``--clean-final`` is recommended to avoid this issue.
+
+.. warning::
+
+    Some ``unpaper`` features cause multiple input or output files to be consumed or produced. OCRmyPDF requires ``unpaper`` to consume one file and produce one file. An deviation from that condition will result in errors.
+
+.. note::
+
+    ``unpaper`` uses uncompressed PBM/PGM/PPM files for its intermediate files. For large images or documents, it can take a lot of temporary disk space.
+
 Control of OCR options
 ----------------------
 
@@ -17,7 +45,6 @@ If ``--redo-ocr`` is issued, then a detailed text analysis is performed. Text is
 
 If ``--force-ocr`` is issued, then all pages will be rasterized to images, discarding any hidden OCR text, and rasterizing any printable text. This is useful for redoing OCR, for fixing OCR text with a damaged character map (text is selectable but not searchable), and destroying redacted information. Any forms and vector graphics will be rasterized as well.
 
-
 Time and image size limits
 """"""""""""""""""""""""""
 
@@ -189,3 +216,25 @@ user interface.  They may be imported from ``ocrmypdf.exceptions``.
     *	- 130
         - ``ExitCode.ctrl_c``
         - The program was interrupted by pressing Ctrl+C.
+
+
+Debugging the intermediate files
+--------------------------------
+
+OCRmyPDF normally saves its intermediate results to a temporary folder and deletes this folder when it exits, whether it succeeded or failed.
+
+If the ``-k`` argument is issued on the command line, OCRmyPDF will keep the temporary folder and print the location, whether it succeeded or failed (provided the Python interpreter did not crash). An example message is:
+
+.. code-block::
+
+    Temporary working files saved at:
+    /tmp/com.github.ocrmypdf.u20wpz07
+
+The organization of this folder is an implementation detail and subject to change between releases. However the general organization is that working files on a per page basis have the page number as a prefix (starting with page 1), an infix indicates the processing stage, and a suffix indicates the file type. Some important files include:
+
+* ``.page.png`` - what the input page looks like
+* ``.image`` - the image we will show the user if we are in a mode that changes the final appearance; may be in one of several image formats
+* ``.text.pdf`` - the OCR file; this will load as a blank page but should have visible text if checked with a tool like pdftotext or pdfminder.six
+* ``.ocr.png`` - the file that is sent to Tesseract for OCR; depending on arguments this may differ from the presentation image
+* ``layers.rendered.pdf`` - the composite PDF, before metadata repair and optimization
+* ``images/*`` - images extracted during the optimization process; here the prefix indicates a PDF object ID not a page number

docs/batch.rst

@@ -18,12 +18,12 @@ The ``--tag`` argument tells parallel to print the filename as a prefix whenever
 
 	parallel --tag -j 2 ocrmypdf '{}' 'output/{}' ::: *.pdf
 
-OCRmyPDF automatically repairs PDFs before parsing and gathering information from them.  If you are already repairing PDFs with ``qpdf`` prior to attempting OCR, or you can use ``--skip-repair`` to skip this step.  It may improve performance for large files, since repairing PDFs is single-threaded.
+OCRmyPDF automatically repairs PDFs before parsing and gathering information from them.
 
 Directory trees
 ---------------
 
-This will walk through a directory tree and run OCR on all files in place, printing the output in a way that makes 
+This will walk through a directory tree and run OCR on all files in place, printing the output in a way that makes
 
 .. code-block:: bash
 
@@ -66,7 +66,7 @@ This user contributed script also provides an example of batch processing.
 	    log_file = script_dir + '/ocr-tree.log'
 
 	logging.basicConfig(
-			level=logging.INFO, format='%(asctime)s %(message)s', 
+			level=logging.INFO, format='%(asctime)s %(message)s',
 			filename=log_file, filemode='w')
 
 	for dir_name, subdirs, file_list in os.walk(start_dir):
@@ -144,11 +144,11 @@ This is only possible for x86-based Synology products. Some Synology products us
 	            timestamp_OCR = time.strftime("%Y-%m-%d-%H%M_OCR_")
 	            filename_OCR = timestamp_OCR + file_noext + '.pdf'
 	            docker_mount = dir_name + ':/home/docker'
-	# create string for pdf processing 
+	# create string for pdf processing
 	# diskstation needs a user:group docker:docker. find uid:gid of your diskstation docker:docker with id docker.
 	# use this uid:gid in -u flag
 	# rw rights for docker:docker at source dir are also necessary
-	# the script is processed as root user via chron 
+	# the script is processed as root user via chron
 	            cmd = ['docker', 'run', '--rm', '-v', docker_mount, '-u=1030:65538', 'jbarlow83/ocrmypdf', , '--deskew' , filename, filename_OCR]
 	            logging.info(cmd)
 	            proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
@@ -210,5 +210,3 @@ Alternatives
 """"""""""""
 
 * `Watchman <https://facebook.github.io/watchman/>`_ is a more powerful alternative to ``watchmedo``.
-
-

docs/installation.rst

@@ -6,9 +6,12 @@ Installation
 
 |latest|
 
-The easiest way to install OCRmyPDF to follow the steps for your operating system/platform.
+The easiest way to install OCRmyPDF is to follow the steps for your operating
+system/platform, although sometimes this version may be out of date.
 
-If you want to use the latest version of OCRmyPDF, your best bet is to install the most recent version your platform provides, and then upgrade that version by installing the Python binary wheels.
+If you want to use the latest version of OCRmyPDF, your best bet is to install
+the most recent version your platform provides, and then upgrade that version by
+installing the Python binary wheels.
 
 .. contents:: Platform-specific steps
     :depth: 2
@@ -136,22 +139,56 @@ To add JBIG2 encoding, see :ref:`jbig2`.
 Ubuntu 16.04 LTS
 ^^^^^^^^^^^^^^^^
 
-No package is currently available for Ubuntu 16.04, but you can install the dependencies manually:
+No package is available for Ubuntu 16.04. OCRmyPDF 8.0 and newer require Python
+3.6. Ubuntu 16.04 ships Python 3.5, but you can install Python 3.6 on it. Or,
+you can skip Python 3.6 and install OCRmyPDF 7.x or older - for that procedure,
+please see the installation documentation for the version of OCRmyPDF you plan
+to use.
+
+**Install system packages for OCRmyPDF**
 
 .. code-block:: bash
 
     sudo apt-get update
-    sudo apt-get install \
+    sudo apt-get install -y software-properties-common python-software-properties
+    sudo add-apt-repository -y \
+        ppa:jonathonf/python-3.6 \
+        ppa:alex-p/tesseract-ocr
+    sudo apt-get update
+    sudo apt-get install -y \
         ghostscript \
         libexempi3 \
+        libffi6 \
         pngquant \
-        python3-cffi \
-        python3-pip \
+        python3.6 \
         qpdf \
         tesseract-ocr \
         unpaper
 
-If you wish install OCRmyPDF for the current user, and ensure that the ``PATH``
+This will install a Python 3.6 binary at ``/usr/bin/python3.6`` alongside the
+system's Python 3.5. Do not remove the system Python. This will also install
+Tesseract 4.0 from a PPA, since the version available in Ubuntu 16.04 is too old
+for OCRmyPDF.
+
+Now install pip for Python 3.6. This will install the Python 3.6 version of
+``pip`` at ``/usr/local/bin/pip``.
+
+.. code-block:: bash
+
+    curl https://bootstrap.pypa.io/get-pip.py | sudo python3.6
+
+**Install OCRmyPDF**
+
+OCRmyPDF requires the locale to be set for UTF-8. **On some minimal Ubuntu
+installations systems**, it may be necessary to set the locale.
+
+.. code-block:: bash
+
+    # Optional: Only need to set these if they are not already set
+    export LC_ALL=C.UTF-8
+    export LANG=C.UTF-8
+
+Now install OCRmyPDF for the current user, and ensure that the ``PATH``
 environment variable contains ``$HOME/.local/bin``.
 
 .. code-block:: bash
@@ -159,38 +196,20 @@ environment variable contains ``$HOME/.local/bin``.
     export PATH=$HOME/.local/bin:$PATH
     pip3 install --user ocrmypdf
 
-Alternately, you can install ocrmypdf system-wide. (Not recommended.)
-
-.. code-block:: bash
-
-    sudo pip3 install ocrmypdf
-
-At your option, you may upgrade Ubuntu 16.04 LTS to Tesseract 4.0 for improved OCR results.
-
-.. code-block:: bash
-
-    sudo apt-get install -y software-properties-common python-software-properties
-    sudo add-apt-repository ppa:alex-p/tesseract-ocr -y
-    sudo apt-get update
-    sudo apt-get upgrade tesseract-ocr
-
 To add JBIG2 encoding, see :ref:`jbig2`.
 
 Ubuntu 14.04 LTS
 ^^^^^^^^^^^^^^^^
 
-Installing on Ubuntu 14.04 LTS (trusty) is more difficult than some other options, because it is older and does not provide ``pip``.
-
-Update apt-get:
-
-.. code-block:: bash
-
-    sudo apt-get update
+Installing on Ubuntu 14.04 LTS (trusty) is more difficult than some other
+options, because of its age. Several backports are required. For explanations of
+some steps of this procedure, see the similar steps for Ubuntu 16.04.
 
 Install system dependencies:
 
 .. code-block:: bash
 
+    sudo apt-get update
     sudo apt-get install \
         software-properties-common python-software-properties \
         zlib1g-dev \
@@ -200,9 +219,13 @@ Install system dependencies:
         pngquant \
         qpdf
 
-We will need backports of Ghostscript 9.16, libav-11 (for unpaper 6.1), Tesseract 4.00 (alpha), and Python 3.6. This will replace Ghostscript and Tesseract 3.x on your system. Python 3.6 will be installed alongside the system Python 3.4.
+We will need backports of Ghostscript 9.16, libav-11 (for unpaper 6.1),
+Tesseract 4.00 (alpha), and Python 3.6. This will replace Ghostscript and
+Tesseract 3.x on your system. Python 3.6 will be installed alongside the system
+Python 3.4.
 
-If you prefer to not modify your system in this matter, consider using a Docker container.
+If you prefer to not modify your system in this matter, consider using a Docker
+container.
 
 .. code-block:: bash
 
@@ -228,8 +251,6 @@ Now we need to install ``pip`` and let it install ocrmypdf:
     curl https://bootstrap.pypa.io/ez_setup.py -o - | python3.6 && python3.6 -m easy_install pip
     pip3.6 install ocrmypdf
 
-The ``wget`` command will download a program and run it.
-
 These installation instructions omit the optional dependency ``unpaper``, which is only available at version 0.4.2 in Ubuntu 14.04. The author could not find a backport of ``unpaper``, and created a .deb package to do the job of installing unpaper 6.1 (for x86 64-bit only):
 
 .. code-block:: bash
@@ -239,14 +260,14 @@ These installation instructions omit the optional dependency ``unpaper``, which
 
 To add JBIG2 encoding, see :ref:`jbig2`.
 
-ArchLinux
-^^^^^^^^^
+ArchLinux (AUR)
+^^^^^^^^^^^^^^^
 
 .. image:: https://repology.org/badge/version-for-repo/aur/ocrmypdf.svg
     :alt: ArchLinux
     :target: https://repology.org/metapackage/ocrmypdf
 
-The author is aware of an `ArchLinux User Repository package for ocrmypdf <https://aur.archlinux.org/packages/ocrmypdf/>`_. You can use the following command.
+There is an `ArchLinux User Repository package for ocrmypdf <https://aur.archlinux.org/packages/ocrmypdf/>`_. You can use the following command.
 
 .. code-block:: bash
 
@@ -378,7 +399,7 @@ Assuming you have a Docker engine running, you can download one of the three ava
         - ``docker pull jbarlow83/ocrmypdf-polyglot``
         - As above, with all available language packs.
     *   - ocrmypdf-webservice
-        - ``docker pull jbarlow83/ocrmypdf-polyglot``
+        - ``docker pull jbarlow83/ocrmypdf-webservice``
         - All language packs, and a simple HTTP wrapper allowing OCRmyPDF to be used as a web service. Note that this component is licensed under AGPLv3.
 
 For example:

docs/release_notes.rst

@@ -13,6 +13,25 @@ Note that it is licensed under GPLv3, so scripts that ``import ocrmypdf`` and ar
    find:    [^`]\#([0-9]{1,3})[^0-9]
    replace: `#$1 <https://github.com/jbarlow83/OCRmyPDF/issues/$1>`_
 
+v8.1.0
+------
+
+-   Added a feature, ``--unpaper-args``, which allows passing arbitrary arguments to ``unpaper`` when using ``--clean`` or ``--clean-final``. The default, very conservative unpaper settings are suppressed.
+
+-   The argument ``--clean-final`` now implies ``--clean``. It was possible to issue ``--clean-final`` on its before this, but it would have no useful effect.
+
+-   Fixed an exception on traversing corrupt table of contents entries (specifically, those with invalid destination objects)
+
+-   Fixed an issue when using ``--tesseract-timeout`` and image processing features on a file with more than 100 pages. `#347 <https://github.com/jbarlow83/OCRmyPDF/issues/347>`_
+
+-   OCRmyPDF now always calls ``os.nice(5)`` to signal to operating systems that it is a background process.
+
+v8.0.1
+------
+
+-   Fixed an exception when parsing PDFs that are missing a required field. `#325 <https://github.com/jbarlow83/OCRmyPDF/issues/325>`_
+
+-   pikepdf 1.0.5 is now required, to address some other PDF parsing issues.
 
 v8.0.0
 ------

pyproject.toml

@@ -0,0 +1,22 @@
+[tool.black]
+line-length = 88
+py36 = true
+skip-string-normalization = true
+include = '\.pyi?$'
+exclude = '''
+/(
+    \.eggs
+  | \.git
+  | \.hg
+  | \.mypy_cache
+  | \.tox
+  | \.venv
+  | _build
+  | buck-out
+  | build
+  | dist
+  | docs
+  | misc
+  | \.egg-info
+)/
+'''

setup.py

@@ -251,7 +251,7 @@ setup(
         'cffi >= 1.9.1',          # must be a setup and install requirement
         'img2pdf >= 0.3.0, < 0.4',       # pure Python, so track HEAD closely
         'pdfminer.six == 20181108 ; sys_platform != "darwin"',
-        'pikepdf >= 1.0.2, < 2',
+        'pikepdf >= 1.0.5, < 2',
         'Pillow >= 4.0.0, != 5.1.0 ; sys_platform == "darwin"',
                                   # Pillow < 4 has BytesIO/TIFF bug w/img2pdf 0.2.3
                                   # block 5.1.0, broken wheels

src/ocrmypdf/__main__.py

@@ -278,6 +278,13 @@ preprocessing.add_argument(
     help="Clean page as above, and incorporate the cleaned image in the final "
     "PDF.  Might remove desired content.",
 )
+preprocessing.add_argument(
+    '--unpaper-args',
+    type=str,
+    default=None,
+    help="A quoted string of arguments to pass to unpaper. Requires --clean. "
+    "Example: --unpaper-args '--layout double'.",
+)
 preprocessing.add_argument(
     '--oversample',
     metavar='DPI',
@@ -623,12 +630,25 @@ def _optional_program_recommended(name, version_fn, min_version, for_argument):
 
 
 def check_options_preprocessing(options, log):
+    if options.clean_final:
+        options.clean = True
+    if options.unpaper_args and not options.clean:
+        raise argparse.ArgumentError(
+            None, "--clean is required for --unpaper-args"
+        )
     if any((options.clean, options.clean_final)):
         from .exec import unpaper
 
         _optional_program_required(
             'unpaper', unpaper.version, '6.1', '--clean, --clean-final'
         )
+        try:
+            if options.unpaper_args:
+                options.unpaper_args = unpaper.validate_custom_args(
+                    options.unpaper_args
+                )
+        except Exception as e:
+            raise argparse.ArgumentError(None, str(e))
 
 
 def check_options_ocr_behavior(options, log):
@@ -1081,6 +1101,8 @@ def run_pipeline(args=None):
 
         build_pipeline(options, work_folder, _log, context)
         atexit.register(cleanup_working_files, work_folder, options)
+        if hasattr(os, 'nice'):
+            os.nice(5)
         cmdline.run(options)
     except ruffus_exceptions.RethrownJobError as e:
         if options.verbose:

src/ocrmypdf/_pipeline.py

@@ -566,7 +566,7 @@ def preprocess_clean(input_file, output_file, log, context):
     pageinfo = get_pageinfo(input_file, context)
     dpi = get_page_square_dpi(pageinfo, options)
 
-    unpaper.clean(input_file, output_file, dpi, log)
+    unpaper.clean(input_file, output_file, dpi, log, options.unpaper_args)
 
 
 def select_ocr_image(infiles, output_file, log, context):
@@ -648,7 +648,7 @@ def ocr_tesseract_hocr(input_file, output_files, log, context):
 
 
 def select_visible_page_image(infiles, output_file, log, context):
-    "Selects a whole page image that we can show the user (if necessary)"
+    """Selects a whole page image that we can show the user (if necessary)"""
 
     options = context.get_options()
     if options.clean_final:

src/ocrmypdf/_weave.py

@@ -17,6 +17,7 @@
 
 from itertools import groupby
 from pathlib import Path
+import os
 
 import pikepdf
 
@@ -24,8 +25,11 @@ from .exec import tesseract
 from .helpers import flatten_groups, page_number
 
 
+MAX_OPEN_PAGE_PDFS = int(os.environ.get('_OCRMYPDF_MAX_OPEN_PAGE_PDFS', 100))
+
+
 def _update_page_resources(*, page, font, font_key, procset):
-    "Update this page's fonts with a reference to the Glyphless font"
+    """Update this page's fonts with a reference to the Glyphless font"""
 
     if '/Resources' not in page:
         page['/Resources'] = pikepdf.Dictionary({})
@@ -34,7 +38,7 @@ def _update_page_resources(*, page, font, font_key, procset):
         fonts = resources['/Font']
     except KeyError:
         fonts = pikepdf.Dictionary({})
-    if font_key not in fonts:
+    if font_key is not None and font_key not in fonts:
         fonts[font_key] = font
     resources['/Font'] = fonts
 
@@ -160,7 +164,7 @@ def _weave_layers_graft(
 
 
 def _find_font(text, pdf_base):
-    "Copy a font from the filename text into pdf_base"
+    """Copy a font from the filename text into pdf_base"""
 
     font, font_key = None, None
     possible_font_names = ('/f-0-0', '/F1')
@@ -177,6 +181,8 @@ def _find_font(text, pdf_base):
             break
     if pdf_text_font:
         font = pdf_base.copy_foreign(pdf_text_font)
+    if font_key is None:
+        print('font_key is None')
     return font, font_key
 
 
@@ -245,12 +251,14 @@ def _fix_toc(pdf_base, pageref_remap, log):
         Inner helper function: change the objgen for any page from the old we
         invalidated to its new one.
         """
-        if not isinstance(dest_node, pikepdf.Array):
-            return
-        pageref = dest_node[0]
-        if pageref['/Type'] == '/Page' and pageref.objgen in pageref_remap:
-            new_objgen = pageref_remap[pageref.objgen]
-            dest_node[0] = pdf_base.get_object(new_objgen)
+        try:
+            pageref = dest_node[0]
+            if pageref['/Type'] == '/Page' and pageref.objgen in pageref_remap:
+                new_objgen = pageref_remap[pageref.objgen]
+                dest_node[0] = pdf_base.get_object(new_objgen)
+        except (IndexError, TypeError) as e:
+            log.warning("This file may contain invalid table of contents entries")
+            log.debug(e)
 
     def visit_remap_dest(pdf_base, node, log):
         """
@@ -390,7 +398,7 @@ def weave_layers(infiles, output_file, log, context):
             content_rotation - autorotate_correction
         ) % 360
 
-        if len(keep_open) > 100:
+        if len(keep_open) > MAX_OPEN_PAGE_PDFS:
             # qpdf limitations require us to keep files open when we intend
             # to copy content from them before saving. However, we want to keep
             # a lid on file handles and memory usage, so for big files we're
@@ -407,7 +415,7 @@ def weave_layers(infiles, output_file, log, context):
 
             pdf_base = pikepdf.open(interim)
             procset = pdf_base.pages[0].Resources.ProcSet
-            font = pdf_base.pages[0].Resources.Font.get(font_key)
+            font, font_key = None, None  # Reacquire this information
 
     _fix_toc(pdf_base, pagerefs, log)
     pdf_base.save(output_file)

src/ocrmypdf/exec/tesseract.py

@@ -200,7 +200,9 @@ def tesseract_log_output(log, stdout, input_file):
             log.info(prefix + line.strip())
 
 
-def page_timedout(log, input_file):
+def page_timedout(log, input_file, timeout):
+    if timeout == 0:
+        return
     prefix = f"{(page_number(input_file)):4d}: [tesseract] "
     log.warning(prefix + " took too long to OCR - skipping")
 
@@ -257,7 +259,7 @@ def generate_hocr(
         # Generate a HOCR file with no recognized text if tesseract times out
         # Temporary workaround to hocrTransform not being able to function if
         # it does not have a valid hOCR file.
-        page_timedout(log, input_file)
+        page_timedout(log, input_file, timeout)
         _generate_null_hocr(output_hocr, output_sidecar, input_file)
     except CalledProcessError as e:
         tesseract_log_output(log, e.output, input_file)
@@ -347,7 +349,7 @@ def generate_pdf(
         if os.path.exists(prefix + '.txt'):
             shutil.move(prefix + '.txt', output_text)
     except TimeoutExpired:
-        page_timedout(log, input_image)
+        page_timedout(log, input_image, timeout)
         use_skip_page(text_only, skip_pdf, output_pdf, output_text)
     except CalledProcessError as e:
         tesseract_log_output(log, e.output, input_image)

src/ocrmypdf/exec/unpaper.py

@@ -19,13 +19,15 @@
 # https://github.com/Flameeyes/unpaper/blob/master/doc/basic-concepts.md
 
 import os
+import shlex
+import subprocess
 import sys
 from functools import lru_cache
-from subprocess import STDOUT, CalledProcessError, check_output
-from tempfile import NamedTemporaryFile
+from subprocess import PIPE, STDOUT, CalledProcessError
+from tempfile import TemporaryDirectory
 
 from . import get_version
-from ..exceptions import MissingDependencyError
+from ..exceptions import MissingDependencyError, SubprocessOutputError
 
 try:
     from PIL import Image
@@ -64,43 +66,64 @@ def run(input_file, output_file, dpi, log, mode_args):
         im.close()
         raise MissingDependencyError() from e
 
-    with NamedTemporaryFile(suffix=suffix) as input_pnm, NamedTemporaryFile(
-        suffix=suffix, mode="r+b"
-    ) as output_pnm:
+    with TemporaryDirectory() as tmpdir:
+        input_pnm = os.path.join(tmpdir, f'input{suffix}')
+        output_pnm = os.path.join(tmpdir, f'output{suffix}')
         im.save(input_pnm, format='PPM')
         im.close()
 
-        os.unlink(output_pnm.name)
-
-        args_unpaper.extend([input_pnm.name, output_pnm.name])
+        # To prevent any shenanigans from accepting arbitrary parameters in
+        # --unpaper-args, we:
+        # 1) run with cwd set to a tmpdir with only unpaper's files
+        # 2) forbid the use of '/' in arguments, to prevent changing paths
+        # 3) append absolute paths for the input and output file
+        # This should ensure that a user cannot clobber some other file with
+        # their unpaper arguments (whether intentionally or otherwise)
+        args_unpaper.extend([input_pnm, output_pnm])
         try:
-            stdout = check_output(
-                args_unpaper, close_fds=True, universal_newlines=True, stderr=STDOUT
+            proc = subprocess.run(
+                args_unpaper,
+                check=True,
+                close_fds=True,
+                universal_newlines=True,
+                stderr=STDOUT,
+                cwd=tmpdir,
+                stdout=PIPE,
             )
         except CalledProcessError as e:
             log.debug(e.output)
             raise e from e
         else:
-            log.debug(stdout)
-            # unpaper sets dpi to 72
-            Image.open(output_pnm.name).save(output_file, dpi=(dpi, dpi))
+            log.debug(proc.stdout)
+            # unpaper sets dpi to 72; fix this
+            try:
+                Image.open(output_pnm).save(output_file, dpi=(dpi, dpi))
+            except (FileNotFoundError, OSError):
+                raise SubprocessOutputError(
+                    "unpaper: failed to produce the expected output file. Called with: "
+                    + str(args_unpaper)
+                ) from None
 
 
-def clean(input_file, output_file, dpi, log):
-    run(
-        input_file,
-        output_file,
-        dpi,
-        log,
-        [
-            '--layout',
-            'none',
-            '--mask-scan-size',
-            '100',  # don't blank out narrow columns
-            '--no-border-align',  # don't align visible content to borders
-            '--no-mask-center',  # don't center visible content within page
-            '--no-grayfilter',  # don't remove light gray areas
-            '--no-blackfilter',  # don't remove solid black areas
-            '--no-deskew',  # don't deskew
-        ],
-    )
+def validate_custom_args(args: str):
+    unpaper_args = shlex.split(args)
+    if any('/' in arg for arg in unpaper_args):
+        raise ValueError('No filenames allowed in --unpaper-args')
+    return unpaper_args
+
+
+def clean(input_file, output_file, dpi, log, unpaper_args=None):
+    default_args = [
+        '--layout',
+        'none',
+        '--mask-scan-size',
+        '100',  # don't blank out narrow columns
+        '--no-border-align',  # don't align visible content to borders
+        '--no-mask-center',  # don't center visible content within page
+        '--no-grayfilter',  # don't remove light gray areas
+        '--no-blackfilter',  # don't remove solid black areas
+        '--no-deskew',  # don't deskew
+    ]
+    if not unpaper_args:
+        unpaper_args = default_args
+    run(input_file, output_file, dpi, log, unpaper_args)

src/ocrmypdf/pdfinfo/__init__.py

@@ -391,6 +391,8 @@ def _image_xobjects(container):
     xobjs = resources['/XObject'].as_dict()
     for xobj in xobjs:
         candidate = xobjs[xobj]
+        if not '/Subtype' in candidate:
+            continue
         if candidate['/Subtype'] == '/Image':
             pdfimage = candidate
             yield (pdfimage, xobj)

tests/conftest.py

@@ -147,7 +147,7 @@ def no_outpdf(tmpdir):
 
 @pytest.helpers.register
 def check_ocrmypdf(input_file, output_file, *args, env=None):
-    "Run ocrmypdf and confirmed that a valid file was created"
+    """Run ocrmypdf and confirmed that a valid file was created"""
 
     p, out, err = run_ocrmypdf(input_file, output_file, *args, env=env)
     # ensure py.test collects the output, use -s to view

tests/test_unpaper.py

@@ -15,11 +15,16 @@
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
 
+import argparse
+from os import fspath
 from pathlib import Path
+from unittest.mock import MagicMock, patch
 
 import pytest
 
+from ocrmypdf import __main__ as main
 from ocrmypdf.exceptions import ExitCode
+from ocrmypdf.exec import unpaper
 
 # pytest.helpers is dynamic
 # pylint: disable=no-member
@@ -30,26 +35,79 @@ run_ocrmypdf = pytest.helpers.run_ocrmypdf
 spoof = pytest.helpers.spoof
 
 
-@pytest.fixture(scope='session')
+def have_unpaper():
+    try:
+        unpaper.version()
+    except Exception:
+        return False
+    else:
+        return True
+
+
+@pytest.fixture(scope="session")
 def spoof_unpaper_oldversion(tmpdir_factory):
-    return spoof(tmpdir_factory, unpaper='unpaper_oldversion.py')
+    return spoof(tmpdir_factory, unpaper="unpaper_oldversion.py")
 
 
-@pytest.mark.skipif(True, reason="needs new fixture implementation")
 def test_no_unpaper(resources, no_outpdf):
-    # <disable unpaper here>
-    p, out, err = run_ocrmypdf(
-        resources / 'c02-22.pdf', no_outpdf, '--clean', env=os.environ
-    )
-    assert p.returncode == ExitCode.missing_dependency
+    input_ = fspath(resources / "c02-22.pdf")
+    output = fspath(no_outpdf)
+    options = main.parser.parse_args(args=["--clean", input_, output])
+
+    with patch("ocrmypdf.exec.unpaper.version") as mock_unpaper_version:
+        mock_unpaper_version.side_effect = FileNotFoundError("unpaper")
+        with pytest.raises(SystemExit):
+            main.check_options(options, log=MagicMock())
 
 
 def test_old_unpaper(spoof_unpaper_oldversion, resources, no_outpdf):
     p, out, err = run_ocrmypdf(
-        resources / 'c02-22.pdf', no_outpdf, '--clean', env=spoof_unpaper_oldversion
+        resources / "c02-22.pdf", no_outpdf, "--clean", env=spoof_unpaper_oldversion
     )
     assert p.returncode == ExitCode.missing_dependency
 
 
+@pytest.mark.skipif(not have_unpaper(), reason="requires unpaper")
 def test_clean(spoof_tesseract_noop, resources, outpdf):
-    check_ocrmypdf(resources / 'skew.pdf', outpdf, '-c', env=spoof_tesseract_noop)
+    check_ocrmypdf(resources / "skew.pdf", outpdf, "-c", env=spoof_tesseract_noop)
+
+
+@pytest.mark.skipif(not have_unpaper(), reason="requires unpaper")
+def test_unpaper_args_valid(spoof_tesseract_noop, resources, outpdf):
+    check_ocrmypdf(
+        resources / "skew.pdf",
+        outpdf,
+        "-c",
+        "--unpaper-args",
+        "--layout double",  # Spaces required here
+        env=spoof_tesseract_noop,
+    )
+
+
+@pytest.mark.skipif(not have_unpaper(), reason="requires unpaper")
+def test_unpaper_args_invalid_filename(spoof_tesseract_noop, resources, outpdf):
+    p, out, err = run_ocrmypdf(
+        resources / "skew.pdf",
+        outpdf,
+        "-c",
+        "--unpaper-args",
+        "/etc/passwd",
+        env=spoof_tesseract_noop,
+    )
+    assert "No filenames allowed" in err
+    assert p.returncode == ExitCode.bad_args
+
+
+@pytest.mark.skipif(not have_unpaper(), reason="requires unpaper")
+def test_unpaper_args_invalid(spoof_tesseract_noop, resources, outpdf):
+    p, out, err = run_ocrmypdf(
+        resources / "skew.pdf",
+        outpdf,
+        "-c",
+        "--unpaper-args",
+        "unpaper is not going to like these arguments",
+        env=spoof_tesseract_noop,
+    )
+    # Can't tell difference between unpaper choking on bad arguments or some
+    # other unpaper failure
+    assert p.returncode == ExitCode.child_process_error

tests/test_weave.py

@@ -0,0 +1,64 @@
+# © 2019 James R. Barlow: github.com/jbarlow83
+#
+# This file is part of OCRmyPDF.
+#
+# OCRmyPDF is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# OCRmyPDF is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
+
+from unittest.mock import MagicMock
+import logging
+import os
+
+import pytest
+
+import pikepdf
+from ocrmypdf._weave import _fix_toc, _update_page_resources
+
+check_ocrmypdf = pytest.helpers.check_ocrmypdf
+
+
+def test_invalid_toc(resources, outdir, caplog):
+    pdf = pikepdf.open(resources / 'toc.pdf')
+
+    # Corrupt a TOC entry
+    pdf.Root.Outlines.Last.Dest = pikepdf.Array([None, 0.0, 0.1, 0.2])
+    pdf.save(outdir / 'test.pdf')
+
+    pdf = pikepdf.open(outdir / 'test.pdf')
+    remap = {}
+    remap[pdf.pages[0].objgen] = pdf.pages[0].objgen  # Dummy remap
+
+    # Confirm we complain about the TOC and don't throw an exception
+    log = logging.getLogger()
+    _fix_toc(pdf, remap, log)
+    assert 'invalid table of contents entries' in caplog.text
+
+
+def test_no_glyphless_weave(resources, outdir):
+    pdf = pikepdf.open(resources / 'francais.pdf')
+    pdf_aspect = pikepdf.open(resources / 'aspect.pdf')
+    pdf_cmyk = pikepdf.open(resources / 'cmyk.pdf')
+    pdf.pages.extend(pdf_aspect.pages)
+    pdf.pages.extend(pdf_cmyk.pages)
+    pdf.save(outdir / 'test.pdf')
+
+    env = os.environ.copy()
+    env['_OCRMYPDF_MAX_OPEN_PAGE_PDFS'] = '2'
+    check_ocrmypdf(
+        outdir / 'test.pdf',
+        outdir / 'out.pdf',
+        '--deskew',
+        '--tesseract-timeout',
+        '0',
+        env=env,
+    )