Fixed setup.py syntax error

Better get the last one out

.travis.yml

@@ -16,13 +16,25 @@ matrix:
       sudo: required
       language: python
       python: 3.5
+      env: EXTRAS=
     - os: linux
       sudo: required
       language: python
       python: 3.6
+      env: EXTRAS=
+    - os: linux
+      sudo: required
+      language: python
+      python: 3.6
+      env: EXTRAS=[fitz]
     - os: osx
       osx_image: xcode8
       language: generic
+      env: EXTRAS=
+    - os: osx
+      osx_image: xcode8
+      language: generic
+      env: EXTRAS=[fitz]
 
 before_cache:
 - rm -f $HOME/.cache/pip/log/debug.log
@@ -37,7 +49,7 @@ before_install: |
   fi
 
 install:
-- pip3 install .
+- pip3 install ".$EXTRAS"
 - pip3 install -r requirements.txt -r test_requirements.txt
 
 script:
@@ -47,12 +59,11 @@ script:
 
 deploy:
   # release for main pypi
-  # 3.6 is considered the build leader and does the deploy, otherwise there is
+  # 3.6 w/o fitz is considered the build leader and does the deploy, otherwise there is
   # a race and all versions will try to deploy
   # OTOH if we ever need separate binary wheels then each version needs its
   # own deploy
 - provider: pypi
-  server: https://upload.pypi.org/legacy/
   user: ocrmypdf-travis
   password:
     secure: "DTFOmmNL6olA0+yXvp4u9jXZlZeqrJsJ0526jzqf4a3gZ6jnGTq5UI6WzRsslSyoMMfXKtHQebqHM6ogSgCZinyZ3ufHJo8fn9brxbEc2gsiWkbj5o3bGwdWMT1vNNE7XW0VCpw87rZ1EEwjl4FJHFudMlPR1yfU5+uq0k0PACo="
@@ -60,7 +71,7 @@ deploy:
   on:
     branch: master
     tags: true
-    condition: $TRAVIS_PYTHON_VERSION == "3.6" && $TRAVIS_OS_NAME == "linux"
+    condition: $TRAVIS_PYTHON_VERSION == "3.6" && $TRAVIS_OS_NAME == "linux" && $EXTRAS == ""
   skip_upload_docs: true
 
   # test pypi

docs/batch.rst

@@ -18,6 +18,8 @@ The ``--tag`` argument tells parallel to print the filename as a prefix whenever
 
 	parallel --tag -j 2 ocrmypdf '{}' 'output/{}' ::: *.pdf
 
+OCRmyPDF automaticaly repairs PDFs before parsing and gathering information from them.  If you are already repairing PDFs with ``qpdf`` prior to attempting OCR, or you can use ``--skip-repair`` to skip this step.  It may improve performance for large files, since repairing PDFs is single-threaded.
+
 Directory trees
 ---------------
 

docs/introduction.rst

@@ -104,5 +104,11 @@ To the author's knowledge, OCRmyPDF is the most feature-rich and thoroughly test
 * pypdfocr
 * pdfbeads
 
-If you are looking for a micro web-frontend for OCRmyPDF, consider the third-party `OCRmyPDF-web <https://github.com/sseemayer/OCRmyPDF-web>`_. Bear in mind that OCRmyPDF is not designed to be secure against malware-bearing PDFs (see `Using OCRmyPDF online`_).
+Web front-ends
+--------------
+
+* `Nextcloud OCR <https://github.com/janis91/ocr>`_ is a free software plugin for the Nextcloud private cloud software
+* `OCRmyPDF-web <https://github.com/sseemayer/OCRmyPDF-web>`_, a micro web-frontend for OCRmyPDF (third-party, not actively maintained)
+
+Bear in mind that OCRmyPDF is not designed to be secure against malware-bearing PDFs (see `Using OCRmyPDF online`_).
 

docs/release_notes.rst

@@ -5,9 +5,36 @@ OCRmyPDF uses `semantic versioning <http://semver.org/>`_ for its command line i
 
 The OCRmyPDF package itself does not contain a public API, although it is fairly stable and breaking changes are usually timed with a major release. A future release will clearly define the stable public API.
 
-v6.0.0
+v6.1.2
 ------
 
+-   Upgrade to PyMuPDF v1.12.5 which includes a more complete fix to #239.
+
+-   Add ``defusedxml`` dependency.
+
+
+v6.1.1
+------
+
+-   Fix text being reported as found on all pages if PyMuPDF is not installed.
+
+
+v6.1.0
+------
+
+-   PyMuPDF is now an optional but recommended dependency, to alleviate installation difficulties on platforms that have less access to PyMuPDF than the author anticipated.  Install OCRmyPDF with ``pip install ocrmypdf[fitz]`` to use it to its full potential.
+
+-   Fix ``FileExistsError`` that could occur if OCR timed out while it was generating the output file. (#218)
+
+-   Fix table of contents/bookmarks all being redirected to page 1 when generating a PDF/A (with PyMuPDF).  (Without PyMuPDF the table of contents is removed in PDF/A mode.)
+
+-   Fix "RuntimeError: invalid key in dict" when table of contents/bookmarks titles contained the character ``)``. (#239)
+
+-   Added a new argument ``--skip-repair`` to skip the initial PDF repair step if the PDF is already well-formed (because another program repaired it).
+
+
+v6.0.0
+------
 
 -   The software license has been changed to GPLv3. Test resource files and some individual sources may have other licenses.
 
@@ -23,11 +50,13 @@ v6.0.0
 
     + The deprecated module ``ocrmypdf.pageinfo`` was removed.
 
+    + The ``--pdf-renderer tess4`` alias for ``sandwich`` was removed.
+
 -   Fixed an issue where OCRmyPDF failed to detect existing text on pages, depending on how the text and fonts were encoded within the PDF. (#233, #232)
 
 -   Fixed an issue that caused dramatic inflation of file sizes when ``--skip-text --output-type pdf`` was used. OCRmyPDF now removes duplicate resources such as fonts, images and other objects that it generates. (#237)
 
--   Improved performance of the inital page splitting step. Originally this step was not believed to be expensive and ran in a process. Large file testing revealed it to be a bottleneck, so it is now parallelized. On a 700 page file with quad core machine, this change saves about 2 minutes. (#234)
+-   Improved performance of the initial page splitting step. Originally this step was not believed to be expensive and ran in a process. Large file testing revealed it to be a bottleneck, so it is now parallelized. On a 700 page file with quad core machine, this change saves about 2 minutes. (#234)
 
 -   The test suite now includes a cache that can be used to speed up test runs across platforms. This also does not require computing checksums, so it's faster. (#217)
 

setup.cfg

@@ -11,4 +11,7 @@ ignore =
 [tool:pytest]
 norecursedirs = lib .pc .git output cache resources
 testpaths = tests
-addopts = -n auto
+addopts = -n auto
+
+[metadata]
+license_file = LICENSE

setup.py

@@ -232,25 +232,28 @@ setup(
         "Topic :: Text Processing :: Indexing",
         "Topic :: Text Processing :: Linguistic",
         ],
-    python_requires='>=3.5',
+    python_requires=' >= 3.5',
     setup_requires=[
+        'cffi >= 1.9.1',        # to build the leptonica module
+        'pytest-runner',        # to enable python setup.py test
         'setuptools_scm',       # so that version will work
-        'cffi>=1.9.1',          # to build the leptonica module
-        'pytest-runner'         # to enable python setup.py test
     ],
     use_scm_version={'version_scheme': 'post-release'},
     cffi_modules=[
         'src/ocrmypdf/lib/compile_leptonica.py:ffibuilder'
     ],
     install_requires=[
-        'ruffus==2.6.3',        # pinned - ocrmypdf implements a 2.6.3 workaround
-        'Pillow>=4.0.0',        # Pillow < 4 has BytesIO/TIFF bug w/img2pdf 0.2.3
-        'reportlab>=3.3.0',     # oldest released version with sane image handling
-        'PyPDF2>=1.26',         # pure Python, so track HEAD closely
-        'img2pdf>=0.2.3',       # pure Python, so track HEAD closely
-        'cffi>=1.9.1',          # must be a setup and install requirement
-        'PyMuPDF == 1.12.4'     # pinned to avoid problems with 1.12.4.x
+        'cffi >= 1.9.1',          # must be a setup and install requirement
+        'defusedxml >= 0.5.0',    # pure Python, so track HEAD closely
+        'img2pdf >= 0.2.4',       # pure Python, so track HEAD closely
+        'Pillow >= 4.0.0',        # Pillow < 4 has BytesIO/TIFF bug w/img2pdf 0.2.3
+        'PyPDF2 >= 1.26',         # pure Python, so track HEAD closely
+        'reportlab >= 3.3.0',     # oldest released version with sane image handling
+        'ruffus == 2.6.3',        # pinned - ocrmypdf implements a 2.6.3 workaround
     ],
+    extras_require={
+        'fitz': ['PyMuPDF >= 1.12.5']     # for table of contents bug
+    },
     tests_require=tests_require,
     entry_points={
         'console_scripts': [

src/ocrmypdf/__main__.py

@@ -18,6 +18,7 @@
 
 from tempfile import mkdtemp
 from collections.abc import Sequence
+from pathlib import Path
 import sys
 import os
 import re
@@ -41,6 +42,7 @@ from .pdfa import file_claims_pdfa
 from .helpers import is_iterable_notstr, re_symlink, is_file_writable, \
     available_cpu_count
 from .exec import tesseract, qpdf, ghostscript
+from .lib import fitz
 from . import PROGRAM_NAME, VERSION
 
 from .exceptions import ExitCode, ExitCodeException, MissingDependencyError, \
@@ -282,7 +284,7 @@ advanced.add_argument(
     )
 advanced.add_argument(
     '--pdf-renderer',
-    choices=['auto', 'tesseract', 'hocr', 'tess4', 'sandwich'], default='auto',
+    choices=['auto', 'tesseract', 'hocr', 'sandwich'], default='auto',
     help="Choose OCR PDF renderer - the default option is to let OCRmyPDF "
          "choose."
          "auto - let OCRmyPDF choose; "
@@ -291,7 +293,6 @@ advanced.add_argument(
          "tesseract - gives better results for non-Latin languages and "
          "Tesseract older than 3.05.01 but has problems with some versions "
          " of Ghostscript; deprecated"
-         "tess4 - deprecated alias for 'sandwich'"
     )
 advanced.add_argument(
     '--tesseract-timeout', default=180.0, type=float, metavar='SECONDS',
@@ -321,6 +322,14 @@ advanced.add_argument(
 advanced.add_argument(
     '--user-patterns', metavar='FILE',
     help="Specify the location of the Tesseract user patterns file.")
+advanced.add_argument(
+    '--skip-repair', action='store_true',
+    help="Normally OCRmyPDF automatically repairs PDFs using qpdf before "
+         "processing.  If you have already run qpdf or a similar program "
+         "that repairs PDF errors, you can tell OCRmyPDF to skip repair with "
+         "this option.  This may be helpful in batch processing where all "
+         "files are repaired prior to OCR occurs, since repair is single "
+         "threaded and time consuming for large files.")
 
 debugging = parser.add_argument_group(
     "Debugging",
@@ -366,11 +375,6 @@ def check_options_output(options, log):
             "The 'sandwich' renderer requires Tesseract 3.05.01 or newer; "
             "or Tesseract 4.00 alpha newer than February 2017.")
 
-    if options.pdf_renderer == 'tess4':
-        log.warning("The 'tess4' PDF renderer has been renamed to 'sandwich'. "
-                    "Please use --pdf-renderer=sandwich.")
-        options.pdf_renderer = 'sandwich'
-
     if options.pdf_renderer == 'tesseract':
         if tesseract.version() < '3.05' and \
                 options.output_type.startswith('pdfa'):
@@ -464,15 +468,12 @@ def check_options_ocr_behavior(options, log):
 
 
 def check_options_advanced(options, log):
-    if tesseract.v4():
-        log.info(
-            "Tesseract v4.x.alpha found.")
     if options.tesseract_oem and not tesseract.v4():
         log.warning(
             "--tesseract-oem requires Tesseract 4.x -- argument ignored")
-    if options.pdf_renderer == 'tess4' and not tesseract.has_textonly_pdf():
+    if options.pdf_renderer == 'sandwich' and not tesseract.has_textonly_pdf():
         raise MissingDependencyError(
-            "--pdf-renderer tess4 requires Tesseract 4.x "
+            "--pdf-renderer sandwich requires Tesseract 4.x "
             "commit 3d9fb3b or later")
     if options.pdfa_image_compression != 'auto' and \
             options.output_type.startswith('pdfa'):
@@ -723,6 +724,11 @@ def preamble(_log):
     _log.debug('ocrmypdf ' + VERSION)
     _log.debug('tesseract ' + tesseract.version())
     _log.debug('qpdf ' + qpdf.version())
+    if fitz:
+        _log.debug('PyMuPDF ' + fitz.version[0])
+        _log.debug('libmupdf ' + fitz.version[1])
+    else:
+        _log.debug('PyMuPDF not installed')
 
 
 def check_environ(options, _log):
@@ -753,7 +759,7 @@ def check_input_file(options, _log, start_input_file):
             raise InputFileError()
 
 
-def check_output_file(options, _log):
+def check_requested_output_file(options, _log):
     if options.output_file == '-':
         if sys.stdout.isatty():
             _log.error(textwrap.dedent("""\
@@ -768,6 +774,35 @@ def check_output_file(options, _log):
         raise OutputFileAccessError()
 
 
+def report_output_file_size(options, _log, input_file, output_file):
+    try:
+        output_size = Path(output_file).stat().st_size
+        input_size = Path(input_file).stat().st_size
+    except FileNotFoundError:
+        return  # Outputting to stream or something
+    ratio = output_size / input_size
+    if ratio < 1.35 or input_size < 25000:
+        return  # Seems fine
+    
+    reasons = []
+    if not fitz:
+        reasons.append("The optional dependency PyMuPDF is not installed.")
+    if options.force_ocr:
+        reasons.append("The argument --force-ocr was issued.")
+
+    if reasons:
+        explanation = (
+            "Possible reasons for this include:\n" + '\n'.join(reasons) + "\n")
+    else:
+        explanation = (
+            "No reason for this increase is known.  Please report this issue.")
+
+    _log.warning(textwrap.dedent("""\
+        The output file size is {:.2f}× larger than the input file.
+        {}
+        """.format(ratio, explanation)))
+
+
 def run_pipeline():
     options = parser.parse_args()
     options.verbose_abbreviated_path = 1
@@ -814,7 +849,7 @@ def run_pipeline():
             work_folder, 'origin')
 
         check_input_file(options, _log, start_input_file)
-        check_output_file(options, _log)
+        check_requested_output_file(options, _log)
 
         manager = JobContextManager()
         manager.register('JobContext', JobContext)  # pylint: disable=no-member
@@ -862,6 +897,9 @@ def run_pipeline():
             _log.warning('Output file: The generated PDF is INVALID')
             return ExitCode.invalid_output_pdf
 
+        report_output_file_size(options, _log, start_input_file, 
+                                options.output_file)
+
     pdfinfo = context.get_pdfinfo()
     if options.verbose:
         from pprint import pformat

src/ocrmypdf/exec/tesseract.py

@@ -22,9 +22,11 @@ import shutil
 from functools import lru_cache
 from collections import namedtuple
 from textwrap import dedent
-import PyPDF2 as pypdf
 from subprocess import PIPE, CalledProcessError, \
     TimeoutExpired, check_output, STDOUT
+from contextlib import suppress
+
+import PyPDF2 as pypdf
 
 from ..exceptions import MissingDependencyError, TesseractConfigError
 from ..helpers import page_number
@@ -281,6 +283,8 @@ def use_skip_page(text_only, skip_pdf, output_pdf, output_text):
         f.write('[skipped page]')
 
     if not text_only:
+        with suppress(FileNotFoundError):
+            os.remove(output_pdf)  # In case it was partially created
         os.symlink(skip_pdf, output_pdf)
         return
 

src/ocrmypdf/lib/__init__.py

@@ -1,4 +1,3 @@
-"""Bindings to external libraries"""
 # © 2017 James R. Barlow: github.com/jbarlow83
 #
 # This file is part of OCRmyPDF.
@@ -15,3 +14,15 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Bindings to external libraries"""
+
+import os as _os
+
+try:
+    import fitz
+except ImportError:
+    fitz = None
+
+if _os.environ.get('_OCRMYPDF_NO_FITZ'):
+    fitz = None

src/ocrmypdf/pageinfo.py

@@ -1,553 +0,0 @@
-#!/usr/bin/env python3
-# © 2015 James R. Barlow: github.com/jbarlow83
-#
-# This file is part of OCRmyPDF.
-#
-# OCRmyPDF is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# OCRmyPDF is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
-
-from subprocess import Popen, PIPE
-from decimal import Decimal
-from math import hypot, isclose
-import re
-import sys
-import PyPDF2 as pypdf
-from collections import namedtuple
-import warnings
-
-
-warnings.warn("ocrmypdf.pageinfo is deprecated'; use ocrmypdf.pdfinfo",
-              DeprecationWarning)
-
-matrix_mult = pypdf.pdf.utils.matrixMultiply
-
-FRIENDLY_COLORSPACE = {
-    '/DeviceGray': 'gray',
-    '/CalGray': 'gray',
-    '/DeviceRGB': 'rgb',
-    '/CalRGB': 'rgb',
-    '/DeviceCMYK': 'cmyk',
-    '/Lab': 'lab',
-    '/ICCBased': 'icc',
-    '/Indexed': 'index',
-    '/Separation': 'sep',
-    '/DeviceN': 'devn',
-    '/Pattern': '-',
-    '/G': 'gray',  # Abbreviations permitted in inline images
-    '/RGB': 'rgb',
-    '/CMYK': 'cmyk',
-    '/I': 'index',
-}
-
-FRIENDLY_ENCODING = {
-    '/CCITTFaxDecode': 'ccitt',
-    '/DCTDecode': 'jpeg',
-    '/JPXDecode': 'jpx',
-    '/JBIG2Decode': 'jbig2',
-    '/CCF': 'ccitt',  # Abbreviations permitted in inline images
-    '/DCT': 'jpeg',
-    '/AHx': 'asciihex',
-    '/A85': 'ascii85',
-    '/LZW': 'lzw',
-    '/Fl': 'flate',
-    '/RL': 'runlength'
-}
-
-FRIENDLY_COMP = {
-    'gray': 1,
-    'rgb': 3,
-    'cmyk': 4,
-    'lab': 3,
-    'index': 1
-}
-
-
-UNIT_SQUARE = (1.0, 0.0, 0.0, 1.0, 0.0, 0.0)
-
-
-def _matrix_from_shorthand(shorthand):
-    """Convert from PDF matrix shorthand to full matrix
-
-    PDF 1.7 spec defines a shorthand for describing the entries of a matrix
-    since the last column is always (0, 0, 1).
-    """
-
-    a, b, c, d, e, f = map(float, shorthand)
-    return ((a, b, 0),
-            (c, d, 0),
-            (e, f, 1))
-
-
-def _shorthand_from_matrix(matrix):
-    """Convert from transformation matrix to PDF shorthand."""
-    a, b = matrix[0][0], matrix[0][1]
-    c, d = matrix[1][0], matrix[1][1]
-    e, f = matrix[2][0], matrix[2][1]
-    return tuple(map(float, (a, b, c, d, e, f)))
-
-
-def _is_unit_square(shorthand):
-    values = map(float, shorthand)
-    pairwise = zip(values, UNIT_SQUARE)
-    return all([isclose(a, b, rel_tol=1e-3) for a, b in pairwise])
-
-XobjectSettings = namedtuple('XobjectSettings',
-    ['name', 'shorthand', 'stack_depth'])
-
-InlineSettings = namedtuple('InlineSettings',
-    ['settings', 'shorthand', 'stack_depth'])
-
-ContentsInfo = namedtuple('ContentsInfo', ['xobject_settings', 'inline_images'])
-
-
-def _normalize_stack(operations):
-    """Fix runs of qQ's in the stack
-
-    For some reason PyPDF2 converts runs of qqq, QQ, QQQq, etc. into single
-    operations.  Break this silliness up and issue each stack operation
-    individually so we don't lose count.
-
-    """
-    for operands, command in operations:
-        if re.match(br'Q*q+$', command):   # Zero or more Q, one or more q
-            for char in command:           # Split into individual bytes
-                yield ([], bytes([char]))  # Yield individual bytes
-        else:
-            yield (operands, command)
-
-
-def _interpret_contents(contentstream, initial_shorthand=UNIT_SQUARE):
-    """Interpret the PDF content stream
-
-    The stack represents the state of the PDF graphics stack.  We are only
-    interested in the current transformation matrix (CTM) so we only track
-    this object; a full implementation would need to track many other items.
-
-    The CTM is initialized to the mapping from user space to device space.
-    PDF units are 1/72".  In a PDF viewer or printer this matrix is initialized
-    to the transformation to device space.  For example if set to
-    (1/72, 0, 0, 1/72, 0, 0) then all units would be calculated in inches.
-
-    Images are always considered to be (0, 0) -> (1, 1).  Before drawing an
-    image there should be a 'cm' that sets up an image coordinate system
-    where drawing from (0, 0) -> (1, 1) will draw on the desired area of the
-    page.
-
-    PDF units suit our needs so we initialize ctm to the identity matrix.
-
-    PyPDF2 replaces inline images with a fake "INLINE IMAGE" operator.
-
-    """
-
-    operations = contentstream.operations
-    stack = []
-    ctm = _matrix_from_shorthand(initial_shorthand)
-    xobject_settings = []
-    inline_images = []
-
-    for n, op in enumerate(_normalize_stack(operations)):
-        operands, command = op
-        if command == b'q':
-            stack.append(ctm)
-            if len(stack) > 32:
-                raise RuntimeError(
-                    "PDF graphics stack overflow, command %i" % n)
-        elif command == b'Q':
-            try:
-                ctm = stack.pop()
-            except IndexError:
-                raise RuntimeError(
-                    "PDF graphics stack underflow, command %i" % n)
-        elif command == b'cm':
-            ctm = matrix_mult(
-                _matrix_from_shorthand(operands), ctm)
-        elif command == b'Do':
-            image_name = operands[0]
-            settings = XobjectSettings(
-                name=image_name, shorthand=_shorthand_from_matrix(ctm),
-                stack_depth=len(stack))
-            xobject_settings.append(settings)
-        elif command == b'INLINE IMAGE':
-            settings = operands['settings']
-            inline = InlineSettings(
-                settings=settings, shorthand=_shorthand_from_matrix(ctm),
-                stack_depth=len(stack))
-            inline_images.append(inline)
-
-    return ContentsInfo(
-        xobject_settings=xobject_settings,
-        inline_images=inline_images)
-
-
-def _get_dpi(ctm_shorthand, image_size):
-    """Given the transformation matrix and image size, find the image DPI.
-
-    PDFs do not include image resolution information within image data.
-    Instead, the PDF page content stream describes the location where the
-    image will be rasterized, and the effective resolution is the ratio of the
-    pixel size to raster target size.
-
-    Normally a scanned PDF has the paper size set appropriately but this is
-    not guaranteed. The most common case is a cropped image will change the
-    page size (/CropBox) without altering the page content stream. That means
-    it is not sufficient to assume that the image fills the page, even though
-    that is the most common case.
-
-    A PDF image may be scaled (always), cropped, translated, rotated in place
-    to an arbitrary angle (rarely) and skewed. Only equal area mappings can
-    be expressed, that is, it is not necessary to consider distortions where
-    the effective DPI varies with position.
-
-    To determine the image scale, transform an offset axis vector v0 (0, 0),
-    width-axis vector v0 (1, 0), height-axis vector vh (0, 1) with the matrix,
-    which gives the dimensions of the image in PDF units. From there we can
-    compare to actual image dimensions. PDF uses
-    row vector * matrix_tranposed unlike the traditional
-    matrix * column vector.
-
-    The offset, width and height vectors can be combined in a matrix and
-    multiplied by the transform matrix. Then we want to calculated
-        magnitude(width_vector - offset_vector)
-    and
-        magnitude(height_vector - offset_vector)
-
-    When the above is worked out algebraically, the effect of translation
-    cancels out, and the vector magnitudes become functions of the nonzero
-    transformation matrix indices. The results of the derivation are used
-    in this code.
-
-    pdfimages -list does calculate the DPI in some way that is not completely
-    naive, but it does not get the DPI of rotated images right, so cannot be
-    used anymore to validate this. Photoshop works, or using Acrobat to
-    rotate the image back to normal.
-
-    It does not matter if the image is partially cropped, or even out of the
-    /MediaBox.
-
-    """
-
-    a, b, c, d, _, _ = ctm_shorthand
-
-    # Calculate the width and height of the image in PDF units
-    image_drawn_width = hypot(a, b)
-    image_drawn_height = hypot(c, d)
-
-    # The scale of the image is pixels per PDF unit (1/72")
-    scale_w = image_size[0] / image_drawn_width
-    scale_h = image_size[1] / image_drawn_height
-
-    # DPI = scale * 72
-    dpi_w = scale_w * 72.0
-    dpi_h = scale_h * 72.0
-
-    return (dpi_w, dpi_h)
-
-
-def _find_inline_images(contentsinfo):
-    "Find inline images in the contentstream"
-
-    for n, inline in enumerate(contentsinfo.inline_images):
-        image = {}
-        image['name'] = str('inline-%02d' % n)
-        image['width'] = inline.settings['/W']
-        image['height'] = inline.settings['/H']
-        if '/BPC' in inline.settings:
-            image['bpc'] = inline.settings['/BPC']
-        else:
-            image['bpc'] = 8
-        if '/CS' in inline.settings:
-            image['color'] = FRIENDLY_COLORSPACE.get(inline.settings['/CS'], '-')
-        else:
-            image['color'] = '-'
-        image['comp'] = FRIENDLY_COMP.get(image['color'], '?')
-        if '/F' in inline.settings:
-            filter_ = inline.settings['/F']
-            if isinstance(filter_, pypdf.generic.ArrayObject):
-                filter_ = filter_[0]
-            image['enc'] = FRIENDLY_ENCODING.get(filter_, 'image')
-        else:
-            image['enc'] = 'image'
-
-        dpi_w, dpi_h = _get_dpi(
-            inline.shorthand, (image['width'], image['height']))
-        image['dpi_w'], image['dpi_h'] = Decimal(dpi_w), Decimal(dpi_h)
-        yield image
-
-
-def _image_xobjects(container):
-    """Search for all XObject-based images in the container
-
-    Usually the container is a page, but it could also be a Form XObject
-    that contains images. Filter out the Form XObjects which are dealt with
-    elsewhere.
-
-    Generate a sequence of tuples (image, xobj container), where container,
-    where xobj is the name of the object and image is the object itself,
-    since the object does not know its own name.
-
-    """
-
-    if '/Resources' not in container:
-        return
-    resources = container['/Resources']
-    if '/XObject' not in resources:
-        return
-    for xobj in resources['/XObject']:
-        candidate = resources['/XObject'][xobj]
-        if candidate['/Subtype'] == '/Image':
-            image = candidate
-            yield (image, xobj)
-
-
-def _find_regular_images(container, contentsinfo):
-    """Find images stored in the container's /Resources /XObject
-
-    Usually the container is a page, but it could also be a Form XObject
-    that contains images.
-
-    Generates images with their DPI at time of drawing.
-
-    """
-
-    for pdfimage, xobj in _image_xobjects(container):
-        image = {}
-        image['name'] = xobj
-        image['width'] = pdfimage['/Width']
-        image['height'] = pdfimage['/Height']
-        if '/BitsPerComponent' in pdfimage:
-            image['bpc'] = pdfimage['/BitsPerComponent']
-        else:
-            image['bpc'] = 8
-
-        # Fixme: this is incorrectly treats explicit masks as stencil masks,
-        # but good enough for now. Explicit masks have /ImageMask true but are
-        # never called for in content stream, instead are drawn as a /Mask on
-        # other images. For our purposes finding out the details of /Mask
-        # will seldom matter.
-        if '/ImageMask' in pdfimage:
-            image['type'] = 'stencil' if pdfimage['/ImageMask'].value \
-                            else 'image'
-        else:
-            image['type'] = 'image'
-        if '/Filter' in pdfimage:
-            filter_ = pdfimage['/Filter']
-            if isinstance(filter_, pypdf.generic.ArrayObject):
-                filter_ = filter_[0]
-            image['enc'] = FRIENDLY_ENCODING.get(filter_, 'image')
-        else:
-            image['enc'] = 'image'
-        if '/ColorSpace' in pdfimage:
-            cs = pdfimage['/ColorSpace']
-            if isinstance(cs, pypdf.generic.ArrayObject):
-                cs = cs[0]
-            image['color'] = FRIENDLY_COLORSPACE.get(cs, '-')
-        else:
-            image['color'] = 'jpx' if image['enc'] == 'jpx' else '?'
-
-        image['comp'] = FRIENDLY_COMP.get(image['color'], '?')
-
-        # Bit of a hack... infer grayscale if component count is uncertain
-        # but encoding must be monochrome. This happens if a monochrome image
-        # has an ICC profile attached. Better solution would be to examine
-        # the ICC profile.
-        if image['comp'] == '?' and image['enc'] in ('ccitt', 'jbig2'):
-            image['comp'] = FRIENDLY_COMP['gray']
-
-        image['dpi_w'] = image['dpi_h'] = 0
-
-        for xobj in contentsinfo.xobject_settings:
-            # Loop in case the same image is display multiple times on a page
-            if xobj.name != image['name']:
-                continue
-
-            if xobj.stack_depth == 0 and _is_unit_square(xobj.shorthand):
-                # At least one PDF in the wild (and test suite) draws an image
-                # when the graphics stack depth is 0, meaning that the image
-                # gets drawn into a square of 1x1 PDF units (or 1/72",
-                # or 0.35 mm).  The equivalent DPI will be >100,000.  Exclude
-                # these from our DPI calculation for the page.
-                continue
-
-            dpi_w, dpi_h = _get_dpi(
-                xobj.shorthand, (image['width'], image['height']))
-
-            # When image is used multiple times take the highest DPI it is
-            # rendered at
-            image['dpi_w'] = max(dpi_w, image.get('dpi_w', 0))
-            image['dpi_h'] = max(dpi_h, image.get('dpi_h', 0))
-
-        DPI_PREC = Decimal('1.000')
-        dpi = Decimal(image['dpi_w'] * image['dpi_h']).sqrt()
-        image['dpi_w'] = Decimal(image['dpi_w']).quantize(DPI_PREC)
-        image['dpi_h'] = Decimal(image['dpi_h']).quantize(DPI_PREC)
-        image['dpi'] = dpi.quantize(DPI_PREC)
-        yield image
-
-
-def _find_form_xobject_images(pdf, container, contentsinfo):
-    """Find any images that are in Form XObjects in the container
-
-    The container may be a page, or a parent Form XObject.
-
-    """
-    if '/Resources' not in container:
-        return
-    resources = container['/Resources']
-    if '/XObject' not in resources:
-        return
-    for xobj in resources['/XObject']:
-        candidate = resources['/XObject'][xobj]
-        if candidate['/Subtype'] != '/Form':
-            continue
-
-        form_xobject = candidate
-        for settings in contentsinfo.xobject_settings:
-            if settings.name != xobj:
-                continue
-
-            # Find images once for each time this Form XObject is drawn.
-            # This could be optimized to cache the multiple drawing events
-            # but in practice both Form XObjects and multiple drawing of the
-            # same object are both very rare.
-            ctm_shorthand = settings.shorthand
-            yield from _find_images(pdf, form_xobject, ctm_shorthand)
-
-
-def _find_images(pdf, container, shorthand=None):
-    """Find all individual instances of images drawn in the container
-
-    Usually the container is a page, but it may also be a Form XObject.
-
-    On a typical page images are stored inline or as regular images
-    in an XObject.
-
-    Form XObjects may include inline images, XObject images,
-    and recursively, other Form XObjects; and also vector drawing commands.
-
-    Every instance of an image being drawn somewhere is flattened and
-    treated as a unique image, since if the same image is drawn multiple times
-    on one page it may be drawn at differing resolutions, and our objective
-    is to find the resolution at which the page can be rastered without
-    downsampling.
-
-    """
-
-    if container.get('/Type') == '/Page' and '/Contents' in container:
-        # For a /Page the content stream is attached to the page's /Contents
-        page = container
-        contentstream = pypdf.pdf.ContentStream(page.getContents(), pdf)
-        initial_shorthand = shorthand or UNIT_SQUARE
-    elif container.get('/Type') == '/XObject' and \
-            container['/Subtype'] == '/Form':
-        # For a Form XObject that content stream is attached to the XObject
-        contentstream = pypdf.pdf.ContentStream(container, pdf)
-
-        # Set the CTM to the state it was when the "Do" operator was
-        # encountered that is drawing this instance of the Form XObject
-        ctm = _matrix_from_shorthand(shorthand or UNIT_SQUARE)
-
-        # A Form XObject may provide its own matrix to map form space into
-        # user space. Get this if one exists
-        form_matrix = _matrix_from_shorthand(
-                container.get('/Matrix', UNIT_SQUARE))
-
-        # Concatenate form matrix with CTM to ensure CTM is correct for
-        # drawing this instance of the XObject
-        ctm = matrix_mult(form_matrix, ctm)
-        initial_shorthand = _shorthand_from_matrix(ctm)
-    else:
-        return
-
-    contentsinfo = _interpret_contents(contentstream, initial_shorthand)
-
-    yield from _find_inline_images(contentsinfo)
-    yield from _find_regular_images(container, contentsinfo)
-    yield from _find_form_xobject_images(pdf, container, contentsinfo)
-
-
-def _page_has_text(pdf, page):
-    if not '/Contents' in page:
-        return False
-
-    # Simple test
-    text = page.extractText()
-    if text.strip() != '':
-        return True
-
-    # More nuanced test to deal with quirks of Tesseract PDF generation
-    # Check if there's a Glyphless font
-    try:
-        font = page['/Resources']['/Font']
-    except KeyError:
-        pass
-    else:
-        font_objects = list(font.keys())
-        for font_object in font_objects:
-            basefont = font[font_object]['/BaseFont']
-            if basefont.endswith('GlyphLessFont'):
-                return True
-
-    return False
-
-
-def _pdf_get_pageinfo(infile, pageno: int):
-    pageinfo = {}
-    pageinfo['pageno'] = pageno
-    pageinfo['images'] = []
-
-    pdf = pypdf.PdfFileReader(infile)
-    page = pdf.pages[pageno]
-
-    pageinfo['has_text'] = _page_has_text(pdf, page)
-
-    width_pt = page.mediaBox.getWidth()
-    height_pt = page.mediaBox.getHeight()
-    pageinfo['width_inches'] = width_pt / Decimal(72.0)
-    pageinfo['height_inches'] = height_pt / Decimal(72.0)
-
-    try:
-        pageinfo['rotate'] = int(page['/Rotate'])
-    except KeyError:
-        pageinfo['rotate'] = 0
-
-    pageinfo['images'] = [im for im in
-                          _find_images(pdf, page)]
-    if pageinfo['images']:
-        xres = max(image['dpi_w'] for image in pageinfo['images'])
-        yres = max(image['dpi_h'] for image in pageinfo['images'])
-        pageinfo['xres'], pageinfo['yres'] = xres, yres
-        pageinfo['width_pixels'] = \
-            int(round(xres * pageinfo['width_inches']))
-        pageinfo['height_pixels'] = \
-            int(round(yres * pageinfo['height_inches']))
-
-    return pageinfo
-
-
-def pdf_get_all_pageinfo(infile):
-    pdf = pypdf.PdfFileReader(infile)
-    return [_pdf_get_pageinfo(infile, n) for n in range(pdf.numPages)]
-
-
-def main():
-    import argparse
-    parser = argparse.ArgumentParser()
-    parser.add_argument('infile')
-    args = parser.parse_args()
-    info = pdf_get_all_pageinfo(args.infile)
-    from pprint import pprint
-    pprint(info)
-
-
-if __name__ == '__main__':
-    main()

src/ocrmypdf/pdfa.py

@@ -18,7 +18,7 @@
 # Generate a PDFA_def.ps file for Ghostscript >= 9.14
 
 from string import Template
-import codecs
+from binascii import hexlify
 import pkg_resources
 import PyPDF2 as pypdf
 
@@ -93,7 +93,7 @@ def encode_text_string(s: str) -> str:
         return ''
 
     utf16_bytes = s.encode('utf-16be')
-    ascii_hex_bytes = codecs.encode(b'\xfe\xff' + utf16_bytes, 'hex')
+    ascii_hex_bytes = hexlify(b'\xfe\xff' + utf16_bytes)
     ascii_hex_str = ascii_hex_bytes.decode('ascii').lower()
     return ascii_hex_str
 

src/ocrmypdf/pdfinfo.py

@@ -29,9 +29,8 @@ from enum import Enum
 from contextlib import contextmanager
 
 import PyPDF2 as pypdf
-from fitz import Document
-
-from .helpers import universal_open
+from .lib import fitz
+from .helpers import universal_open, fspath
 
 
 
@@ -121,7 +120,8 @@ XobjectSettings = namedtuple('XobjectSettings',
 InlineSettings = namedtuple('InlineSettings',
     ['settings', 'shorthand', 'stack_depth'])
 
-ContentsInfo = namedtuple('ContentsInfo', ['xobject_settings', 'inline_images'])
+ContentsInfo = namedtuple('ContentsInfo', 
+    ['xobject_settings', 'inline_images', 'found_text'])
 
 
 def _normalize_stack(operations):
@@ -168,6 +168,7 @@ def _interpret_contents(contentstream, initial_shorthand=UNIT_SQUARE):
     ctm = _matrix_from_shorthand(initial_shorthand)
     xobject_settings = []
     inline_images = []
+    found_text = False
 
     for n, op in enumerate(_normalize_stack(operations)):
         operands, command = op
@@ -197,10 +198,14 @@ def _interpret_contents(contentstream, initial_shorthand=UNIT_SQUARE):
                 settings=settings, shorthand=_shorthand_from_matrix(ctm),
                 stack_depth=len(stack))
             inline_images.append(inline)
+        elif command in (b'Tj', b'TJ', b'"', b"'"):
+            found_text = True
+
 
     return ContentsInfo(
         xobject_settings=xobject_settings,
-        inline_images=inline_images)
+        inline_images=inline_images,
+        found_text=found_text)
 
 
 def _get_dpi(ctm_shorthand, image_size):
@@ -545,17 +550,40 @@ def _find_images(*, pdf, container, shorthand=None):
     yield from _find_form_xobject_images(pdf, container, contentsinfo)
 
 
-@contextmanager
-def borrow_stream(stream):
-    "Borrow a file stream from elsewhere and restore the offset when done"
-    offset = stream.tell()
-    stream.seek(0)
-    yield stream
-    stream.seek(offset)
+def _naive_find_text(*, pdf, page):
+    if not(page.get('/Type') == '/Page' and '/Contents' in page):
+        # Not a page, or has no /Contents => no text
+        return False
+
+    # First we check the main content stream    
+    contentstream = pypdf.pdf.ContentStream(page.getContents(), pdf)
+    contentsinfo = _interpret_contents(contentstream, UNIT_SQUARE)
+    if contentsinfo.found_text:
+        return True
+
+    # Then see if there is a Form XObject with with a content stream
+    # that might have text.  For full completeness we should recursively
+    # search nested Form XObjects, as we do with images.  But that is
+    # rare.
+    if '/Resources' in page:
+        resources = page['/Resources']
+        if '/XObject' in resources:    
+            for xobj in resources['/XObject']:
+                candidate = resources['/XObject'][xobj]
+                if candidate['/Subtype'] != '/Form':
+                    continue
+                form_xobject = candidate                
+                # Content stream is attached to Form XObject dictionary
+                contentstream = pypdf.pdf.ContentStream(form_xobject, pdf)
+                sub_contentsinfo = _interpret_contents(
+                    contentstream, UNIT_SQUARE)
+                if sub_contentsinfo.found_text:
+                    return True
+    return False
 
 
 def _page_has_text(infile, pageno):
-    doc = Document(infile)
+    doc = fitz.Document(infile)
     text = doc.getPageText(pageno)
     if text.strip() != '':
         return True
@@ -574,7 +602,10 @@ def _pdf_get_pageinfo(pdf, pageno: int, infile):
 
     page = pdf.pages[pageno]
 
-    pageinfo['has_text'] = _page_has_text(str(infile), pageno)
+    if fitz:
+        pageinfo['has_text'] = _page_has_text(str(infile), pageno)
+    else:
+        pageinfo['has_text'] = _naive_find_text(pdf=pdf, page=page)
 
     width_pt = page.mediaBox.getWidth()
     height_pt = page.mediaBox.getHeight()
@@ -692,6 +723,10 @@ class PdfInfo:
     def __init__(self, infile):
         self._infile = infile
         self._pages = _pdf_get_all_pageinfo(infile)
+        if fitz:
+            self._toc = fitz.Document(fspath(infile)).getToC()
+        else:
+            self._toc = []
 
     @property
     def pages(self):
@@ -712,6 +747,10 @@ class PdfInfo:
             raise NotImplementedError("can't get filename from stream")
         return self._infile
 
+    @property
+    def table_of_contents(self):
+        return self._toc
+
     def __getitem__(self, item):
         return self._pages[item]
 

src/ocrmypdf/pipeline.py

@@ -25,7 +25,6 @@ import re
 
 import img2pdf
 import PyPDF2 as pypdf
-import fitz
 
 from PIL import Image
 from ruffus import formatter, regex, Pipeline, suffix
@@ -35,12 +34,14 @@ from .pdfinfo import PdfInfo, Encoding, Colorspace
 from .pdfa import generate_pdfa_ps
 from .helpers import re_symlink, is_iterable_notstr, page_number
 from .exec import ghostscript, tesseract, qpdf
+from .lib import fitz
 from .exceptions import PdfMergeFailedError, UnsupportedImageFormatError, \
     DpiError, PriorOcrFoundError, InputFileError
 from . import leptonica
 from . import PROGRAM_NAME, VERSION
 
 
+
 VECTOR_PAGE_DPI = 400
 
 # -------------
@@ -205,13 +206,19 @@ def triage(
     triage_image_file(input_file, output_file, log, options)
 
 
-def repair_pdf(
+def repair_and_parse_pdf(
         input_file,
         output_file,
         log,
         context):
     options = context.get_options()
-    qpdf.repair(input_file, output_file, log)
+    if not options.skip_repair:
+        log.debug("Beginning qpdf repair...")
+        qpdf.repair(input_file, output_file, log)
+        log.debug("Repair OK; beginning parse...")
+    else:
+        re_symlink(input_file, output_file, log)
+
     pdfinfo = PdfInfo(output_file)
 
     if pdfinfo.has_userunit and options.output_type == 'pdfa':
@@ -963,11 +970,17 @@ def merge_pages_ghostscript(
     ghostscript.generate_pdfa(
         pdf_version=input_pdfinfo.min_version,
         pdf_pages=pdf_pages,
-        output_file=output_file,
+        output_file=output_file + '_toc.pdf',
         compression=options.pdfa_image_compression,
         log=log,
         threads=options.jobs or 1,
         pdfa_part=('1' if options.output_type == 'pdfa-1' else '2'))
+    if fitz:
+        doc = fitz.Document(output_file + '_toc.pdf')
+        doc.setToC(input_pdfinfo.table_of_contents)
+        doc.save(output_file)
+    else:
+        os.replace(output_file + '_toc.pdf', output_file)
 
 
 def merge_pages_qpdf(
@@ -1006,6 +1019,8 @@ def merge_pages_mupdf(
         output_file,
         log,
         context):
+    assert fitz
+
     options = context.get_options()
 
     pdf_pages, metadata_file = _merge_pages_common(
@@ -1023,7 +1038,8 @@ def merge_pages_mupdf(
         doc.insertPDF(page)
 
     metadata = fitz.open(metadata_file)
-    doc.setToC(metadata.getToC())
+    toc = metadata.getToC(simple=False)
+    doc.setToC(toc)
     doc.setMetadata(pymupdf_metadata)
     doc.save(output_file, garbage=4, deflate=True)
 
@@ -1100,8 +1116,8 @@ def build_pipeline(options, work_folder, log, context):
         output=os.path.join(work_folder, 'origin.pdf'),
         extras=[log, context])
 
-    task_repair_pdf = main_pipeline.transform(
-        task_func=repair_pdf,
+    task_repair_and_parse_pdf = main_pipeline.transform(
+        task_func=repair_and_parse_pdf,
         input=task_triage,
         filter=suffix('.pdf'),
         output='.repaired.pdf',
@@ -1111,7 +1127,7 @@ def build_pipeline(options, work_folder, log, context):
     # Split (kwargs for split seems to be broken, so pass plain args)
     task_pre_split_pages = main_pipeline.split(
         pre_split_pages,
-        task_repair_pdf,
+        task_repair_and_parse_pdf,
         os.path.join(work_folder, '*.presplit.pdf'),
         extras=[log, context])
 
@@ -1275,7 +1291,7 @@ def build_pipeline(options, work_folder, log, context):
     # PDF/A
     task_generate_postscript_stub = main_pipeline.transform(
         task_func=generate_postscript_stub,
-        input=task_repair_pdf,
+        input=task_repair_and_parse_pdf,
         filter=formatter(r'\.repaired\.pdf'),
         output=os.path.join(work_folder, 'pdfa.ps'),
         extras=[log, context])
@@ -1301,7 +1317,20 @@ def build_pipeline(options, work_folder, log, context):
                task_generate_postscript_stub],
         output=os.path.join(work_folder, 'merged.pdf'),
         extras=[log, context])
-    task_merge_pages_ghostscript.active_if(options.output_type.startswith('pdfa'))
+    task_merge_pages_ghostscript.active_if(
+        options.output_type.startswith('pdfa'))
+
+    task_merge_pages_qpdf = main_pipeline.merge(
+        task_func=merge_pages_qpdf,
+        input=[task_combine_layers,
+               task_render_hocr_debug_page,
+               task_skip_page,
+               task_ocr_tesseract_and_render_pdf,
+               task_repair_and_parse_pdf],
+        output=os.path.join(work_folder, 'merged.pdf'),
+        extras=[log, context])
+    task_merge_pages_qpdf.active_if(
+        options.output_type == 'pdf' and not fitz)
 
     task_merge_pages_mupdf = main_pipeline.merge(
         task_func=merge_pages_mupdf,
@@ -1309,10 +1338,11 @@ def build_pipeline(options, work_folder, log, context):
                task_render_hocr_debug_page,
                task_skip_page,
                task_ocr_tesseract_and_render_pdf,
-               task_repair_pdf],
+               task_repair_and_parse_pdf],
         output=os.path.join(work_folder, 'merged.pdf'),
         extras=[log, context])
-    task_merge_pages_mupdf.active_if(options.output_type == 'pdf')
+    task_merge_pages_mupdf.active_if(
+        options.output_type == 'pdf' and fitz)
 
     task_merge_sidecars = main_pipeline.merge(
         task_func=merge_sidecars,
@@ -1326,6 +1356,8 @@ def build_pipeline(options, work_folder, log, context):
     # Finalize
     main_pipeline.merge(
         task_func=copy_final,
-        input=[task_merge_pages_ghostscript, task_merge_pages_mupdf],
+        input=[task_merge_pages_ghostscript,
+               task_merge_pages_mupdf,
+               task_merge_pages_qpdf],
         output=options.output_file,
         extras=[log, context])

tests/resources/README.rst

@@ -128,6 +128,7 @@ Assemblies
 
 These test resources are assemblies or derivatives from other previously mentioned files, released under the same license terms as their input files.
 
+- baiona_gray.png (from baiona.png)
 - cardinal.pdf (four cardinal directions, baked-in rotated copies of LinnSequencer.jpg)
 - ccitt.pdf (LinnSequencer.jpg, converted to CCITT encoding)
 - encrypted_algo4.pdf (congress.jpg, encrypted with algorithm 4 - not supported by PyPDF2)
@@ -135,12 +136,11 @@ These test resources are assemblies or derivatives from other previously mention
 - jbig2.pdf (congress.jpg, converted to JBIG2 encoding)
 - multipage.pdf (from several other files)
 - palette.pdf (congress.jpg, converted to a 256-color palette)
-- rotated_skew.pdf (a /Rotate'd and skewed document from LinnSequencer.jpg)
-- skew.pdf (from LinnSequencer.jpg, skew simulated by adjusting the transformation matrix)
-- skew-encrypted.pdf (skew.pdf with encryption - access supported by PyPDF2, password is "password")
-- baiona_gray.png (from baiona.png)
 - poster.pdf (from LinnSequencer.jpg)
-
+- rotated_skew.pdf (a /Rotate'd and skewed document from LinnSequencer.jpg)
+- skew-encrypted.pdf (skew.pdf with encryption - access supported by PyPDF2, password is "password")
+- skew.pdf (from LinnSequencer.jpg, skew simulated by adjusting the transformation matrix)
+- toc.pdf (from formxobject.pdf, trivial.pdf)
 
 
 .. _`Wikimedia: LinnSequencer`: https://upload.wikimedia.org/wikipedia/en/b/b7/LinnSequencer_hardware_MIDI_sequencer_brochure_page_2_300dpi.jpg

tests/test_main.py

@@ -26,8 +26,8 @@ from ocrmypdf.pdfinfo import PdfInfo, Colorspace, Encoding
 import PyPDF2 as pypdf
 from ocrmypdf.exceptions import ExitCode
 from ocrmypdf import leptonica
-from ocrmypdf.pdfa import file_claims_pdfa
 from ocrmypdf.exec import ghostscript, tesseract, qpdf
+from ocrmypdf.pdfa import file_claims_pdfa
 import logging
 from math import isclose
 
@@ -174,71 +174,6 @@ def test_exotic_image(spoof_tesseract_cache, pdf, renderer, output_type,
     assert outfile.with_suffix('.pdf.txt').exists()
 
 
-@pytest.mark.parametrize("output_type", [
-    'pdfa', 'pdf'
-    ])
-def test_preserve_metadata(spoof_tesseract_noop, output_type,
-                           resources, outpdf):
-    pdf_before = pypdf.PdfFileReader(str(resources / 'graph.pdf'))
-
-    output = check_ocrmypdf(
-            resources / 'graph.pdf', outpdf,
-            '--output-type', output_type,
-            env=spoof_tesseract_noop)
-
-    pdf_after = pypdf.PdfFileReader(str(output))
-
-    for key in ('/Title', '/Author'):
-        assert pdf_before.documentInfo[key] == pdf_after.documentInfo[key]
-
-    pdfa_info = file_claims_pdfa(str(output))
-    assert pdfa_info['output'] == output_type
-
-
-@pytest.mark.parametrize("output_type", [
-    'pdfa', 'pdf'
-    ])
-def test_override_metadata(spoof_tesseract_noop, output_type, resources,
-                           outpdf):
-    input_file = resources / 'c02-22.pdf'
-    german = 'Du siehst den Wald vor lauter Bäumen nicht.'
-    chinese = '孔子'
-
-    p, out, err = run_ocrmypdf(
-        input_file, outpdf,
-        '--title', german,
-        '--author', chinese,
-        '--output-type', output_type,
-        env=spoof_tesseract_noop)
-
-    assert p.returncode == ExitCode.ok, err
-
-    reader = pypdf.PdfFileReader(outpdf)
-
-    assert reader.documentInfo['/Title'] == german
-    assert reader.documentInfo['/Author'] == chinese
-    assert reader.documentInfo.get('/Keywords', '') == ''
-
-    pdfa_info = file_claims_pdfa(outpdf)
-    assert pdfa_info['output'] == output_type
-
-
-def test_high_unicode(spoof_tesseract_noop, resources, no_outpdf):
-
-    # Ghostscript doesn't support high Unicode, so neither do we, to be
-    # safe
-    input_file = resources / 'c02-22.pdf'
-    high_unicode = 'U+1030C is: 𐌌'
-
-    p, out, err = run_ocrmypdf(
-        input_file, no_outpdf,
-        '--subject', high_unicode,
-        '--output-type', 'pdfa',
-        env=spoof_tesseract_noop)
-
-    assert p.returncode == ExitCode.bad_args, err
-
-
 @pytest.mark.parametrize('renderer', RENDERERS)
 def test_oversample(spoof_tesseract_cache, renderer, resources, outpdf):
     oversampled_pdf = check_ocrmypdf(
@@ -1133,4 +1068,12 @@ def test_output_is_symlink(spoof_tesseract_noop, resources, outdir):
     )
     assert p.returncode == ExitCode.ok, err
     assert (outdir / 'out.pdf').stat().st_size > 0, 'target file not created'
-    
+
+
+def test_skip_repair(spoof_tesseract_noop, resources, outpdf):
+    check_ocrmypdf(
+        resources / 'trivial.pdf',
+        outpdf,
+        '--skip-repair',
+        env=spoof_tesseract_noop
+    )

tests/test_metadata.py

@@ -0,0 +1,117 @@
+# © 2018 James R. Barlow: github.com/jbarlow83
+#
+# This file is part of OCRmyPDF.
+#
+# OCRmyPDF is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# OCRmyPDF is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import pytest
+import PyPDF2 as pypdf
+
+from ocrmypdf.pdfa import file_claims_pdfa
+from ocrmypdf.exceptions import ExitCode
+from ocrmypdf.lib import fitz
+
+# pytest.helpers is dynamic
+# pylint: disable=no-member
+# pylint: disable=w0612
+
+check_ocrmypdf = pytest.helpers.check_ocrmypdf
+run_ocrmypdf = pytest.helpers.run_ocrmypdf
+spoof = pytest.helpers.spoof
+
+
+@pytest.mark.parametrize("output_type", [
+    'pdfa', 'pdf'
+    ])
+def test_preserve_metadata(spoof_tesseract_noop, output_type,
+                           resources, outpdf):
+    pdf_before = pypdf.PdfFileReader(str(resources / 'graph.pdf'))
+
+    output = check_ocrmypdf(
+            resources / 'graph.pdf', outpdf,
+            '--output-type', output_type,
+            env=spoof_tesseract_noop)
+
+    pdf_after = pypdf.PdfFileReader(str(output))
+
+    for key in ('/Title', '/Author'):
+        assert pdf_before.documentInfo[key] == pdf_after.documentInfo[key]
+
+    pdfa_info = file_claims_pdfa(str(output))
+    assert pdfa_info['output'] == output_type
+
+
+@pytest.mark.parametrize("output_type", [
+    'pdfa', 'pdf'
+    ])
+def test_override_metadata(spoof_tesseract_noop, output_type, resources,
+                           outpdf):
+    input_file = resources / 'c02-22.pdf'
+    german = 'Du siehst den Wald vor lauter Bäumen nicht.'
+    chinese = '孔子'
+
+    p, out, err = run_ocrmypdf(
+        input_file, outpdf,
+        '--title', german,
+        '--author', chinese,
+        '--output-type', output_type,
+        env=spoof_tesseract_noop)
+
+    assert p.returncode == ExitCode.ok, err
+
+    reader = pypdf.PdfFileReader(outpdf)
+
+    assert reader.documentInfo['/Title'] == german
+    assert reader.documentInfo['/Author'] == chinese
+    assert reader.documentInfo.get('/Keywords', '') == ''
+
+    pdfa_info = file_claims_pdfa(outpdf)
+    assert pdfa_info['output'] == output_type
+
+
+def test_high_unicode(spoof_tesseract_noop, resources, no_outpdf):
+
+    # Ghostscript doesn't support high Unicode, so neither do we, to be
+    # safe
+    input_file = resources / 'c02-22.pdf'
+    high_unicode = 'U+1030C is: 𐌌'
+
+    p, out, err = run_ocrmypdf(
+        input_file, no_outpdf,
+        '--subject', high_unicode,
+        '--output-type', 'pdfa',
+        env=spoof_tesseract_noop)
+
+    assert p.returncode == ExitCode.bad_args, err
+
+
+@pytest.mark.xfail(not fitz, reason="needs fitz")
+@pytest.mark.parametrize('ocr_option', ['--skip-text', '--force-ocr'])
+@pytest.mark.parametrize('output_type', ['pdf', 'pdfa'])
+def test_bookmarks_preserved(spoof_tesseract_noop, output_type, ocr_option,
+                             resources, outpdf):
+    input_file = resources / 'toc.pdf'
+    before_toc = fitz.Document(str(input_file)).getToC()
+
+    check_ocrmypdf(
+        input_file, outpdf,
+        ocr_option,
+        '--output-type', output_type,
+        env=spoof_tesseract_noop)
+
+    after_toc = fitz.Document(str(outpdf)).getToC()
+    print(before_toc)
+    print(after_toc)
+    assert before_toc == after_toc

tests/test_pageinfo.py

@@ -28,6 +28,7 @@ import pytest
 import img2pdf
 import pytest
 import sys
+import PyPDF2 as pypdf
 
 
 def test_single_page_text(outdir):
@@ -125,6 +126,13 @@ def test_form_xobject(resources):
     assert pdfimage.width == 50
 
 
+def test_naive_find_text(resources):
+    filename = resources / 'formxobject.pdf'
+    reader = pypdf.PdfFileReader(str(filename))
+    page = reader.getPage(0)
+    assert pdfinfo._naive_find_text(pdf=reader, page=page)
+
+
 def test_no_contents(resources):
     filename = resources / 'no_contents.pdf'
 

tests/test_tess4.py

@@ -86,7 +86,7 @@ spoof = pytest.helpers.spoof
 def test_textonly_pdf(ensure_tess4, resources, outdir):
     check_ocrmypdf(
         resources / 'linn.pdf',
-        outdir / 'linn_textonly.pdf', '--pdf-renderer', 'tess4',
+        outdir / 'linn_textonly.pdf', '--pdf-renderer', 'sandwich',
         '--sidecar', outdir / 'foo.txt',
         env=ensure_tess4)
 
@@ -100,7 +100,7 @@ def test_pagesize_consistency_tess4(ensure_tess4, resources, outpdf):
 
     check_ocrmypdf(
         infile,
-        outpdf, '--pdf-renderer', 'tess4',
+        outpdf, '--pdf-renderer', 'sandwich',
         '--clean', '--deskew', '--remove-background', '--clean-final',
         env=ensure_tess4)
 
@@ -118,7 +118,7 @@ def test_skip_pages_does_not_replicate(
 
     check_ocrmypdf(
         infile,
-        outpdf, '--pdf-renderer', 'tess4', '--force-ocr',
+        outpdf, '--pdf-renderer', 'sandwich', '--force-ocr',
         '--tesseract-timeout', '0',
         env=ensure_tess4
     )
@@ -138,7 +138,7 @@ def test_content_preservation(ensure_tess4, resources, outpdf):
 
     check_ocrmypdf(
         infile,
-        outpdf, '--pdf-renderer', 'tess4', '--tesseract-timeout', '0',
+        outpdf, '--pdf-renderer', 'sandwich', '--tesseract-timeout', '0',
         env=ensure_tess4
     )