v4.1 release notes

I found this issue in ruffus 2.6.3
https://github.com/bunbun/ruffus/issues/65
also discussed here
https://github.com/bunbun/ruffus/pull/67

ruffus 2.6.3 RethrownJobError don't follow the normal conventions and
so its exception causes problems when they cross process boundaries.
This change carefully examines the various forms of ruffus exception
objects that can appear in 2.6.3 and parses them more carefully. It
also removes any direct posting of the exception to the logger because
this triggers another serializing of the exception object, mutating it
further.

.travis.yml

@@ -5,7 +5,7 @@ cache: pip
 cache:
     directories:
         - $HOME/.ccache
-        - tarballs
+        - packages
         - tests/cache
 
 python:
@@ -21,17 +21,15 @@ before_install:
     - sudo add-apt-repository ppa:b-eltzner/qpdfview-exp -y  # for QPDF 5
     - sudo add-apt-repository ppa:itachi-san/ffmpeg -y  # for libav 11.2 (for unpaper)
     - sudo apt-get update -qq  # must go after all add-apt-repo
-    - sudo apt-get install -y ghostscript tesseract-ocr tesseract-ocr-deu tesseract-ocr-eng tesseract-ocr-fra qpdf poppler-utils gcc libavformat-dev libavcodec-dev libavutil-dev automake make pkg-config xsltproc libffi-dev
+    - sudo apt-get install -y ghostscript tesseract-ocr tesseract-ocr-deu tesseract-ocr-eng tesseract-ocr-fra qpdf poppler-utils libavformat56 libavcodec56 libavutil54 libffi-dev
 
     # pip
     - pip install --upgrade pip
 
     # Download, make and install unpaper (using ccache)
-    - mkdir -p tarballs
+    - mkdir -p packages
-    - "[ -f tarballs/unpaper-6.1.tar.xz ] || wget -q https://www.flameeyes.eu/files/unpaper-6.1.tar.xz -O tarballs/unpaper-6.1.tar.xz"
+    - "[ -f packages/unpaper_6.1-1.deb ] || wget -q https://dl.dropboxusercontent.com/u/28971240/unpaper_6.1-1.deb -O packages/unpaper_6.1-1.deb"
-    - tar -xvf tarballs/unpaper-6.1.tar.xz
+    - sudo dpkg -i packages/unpaper_6.1-1.deb
-    - export PATH="/usr/lib/ccache:$PATH"
-    - pushd unpaper-6.1 && ./configure --prefix=/usr && make -j && sudo make install && popd
 
 install:
     - pip install -r requirements.txt

LICENSE.rst

@@ -18,3 +18,9 @@ IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+----
+
+sRGB_IEC61966-2-1_black_scaled.icc 
+
+To anyone who acknowledges that the file "sRGB_IEC61966-2-1_black scaled.icc" is provided "AS IS" WITH NO EXPRESS OR IMPLIED WARRANTY, permission to use, copy and distribute these file for any purpose is hereby granted without fee, provided that the file is not changed including the ICC copyright notice tag, and that the name of ICC shall not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. ICC makes no representations about the suitability of this software for any purpose.

README.rst

@@ -48,6 +48,12 @@ Download OCRmyPDF here: https://github.com/jbarlow83/OCRmyPDF/releases
 
 You can install it to a Python virtual environment or system-wide. 
 
+Debian and Ubuntu
+~~~~~~~~~~~~~~~~~
+
+Users of Debian 9 or later or Ubuntu 16.10 or later may simply
+``apt-get install ocrmypdf``.
+
 Installing the Docker image
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -148,6 +154,7 @@ Install system dependencies::
    sudo apt-get install \
       zlib1g-dev \
       libjpeg-dev \
+      libffi-dev \
       ghostscript \
       tesseract-ocr \
       qpdf \

RELEASE_NOTES.rst

@@ -6,6 +6,20 @@ Please always read this file before installing the package
 Download software here: https://github.com/jbarlow83/OCRmyPDF/tags
 
 
+v4.1:
+=====
+
+-  ``--rotate-pages`` now only rotates pages when reasonably confidence in the orientation. This behavior can be adjusted with the new argument ``--rotate-pages-threshold``
+-  Fixed problems in error checking if ``unpaper`` is uninstalled or missing at run-time
+-  Fixed problems with "RethrownJobError" errors during error handling that suppressed the useful error messages
+
+
+v4.0.7:
+=======
+
+-  Minor correction to Ghostscript output settings
+
+
 v4.0.6:
 =======
 

ocrmypdf/ghostscript.py

@@ -53,7 +53,6 @@ def generate_pdfa(pdf_pages, output_file, threads=1):
             "-dJPEGQ=95",
             "-dPDFA=2",
             "-sPDFACompatibilityPolicy=2",
-            "-sOutputICCProfile=" + SRGB_ICC_PROFILE,
             "-sOutputFile=" + gs_pdf.name,
         ]
         args_gs.extend(pdf_pages)

ocrmypdf/leptonica.py

@@ -110,8 +110,8 @@ class Pix:
             return "<leptonica.Pix image NULL>"
 
     def __getstate__(self):
-        data = ffi.new('l_uint32 *[]', 1)
+        data = ffi.new('l_uint32 **')
-        size = ffi.new('size_t *', 0)
+        size = ffi.new('size_t *')
 
         err = lept.pixSerializeToMemory(self.cpix, data, size)
         if err != 0:
@@ -195,16 +195,40 @@ class Pix:
             else:
                 return (None, None)
 
+    def otsu_adaptive_threshold(
+            self, tile_size=(300, 300), kernel_size=(4, 4), scorefract=0.1):
+        with LeptonicaErrorTrap():
+            sx, sy = tile_size
+            smoothx, smoothy = kernel_size
+            p_cpix = ffi.new('PIX **')
+
+            result = lept.pixOtsuAdaptiveThreshold(
+                self.cpix,
+                sx, sy,
+                smoothx, smoothy,
+                scorefract,
+                ffi.NULL,
+                p_cpix)
+            if result == 0:
+                return Pix(p_cpix[0])
+            else:
+                return None
+
+    @staticmethod
+    @lru_cache(maxsize=1)
+    def make_pixel_sum_tab8():
+        return lept.makePixelSumTab8()
+
     @staticmethod
     def correlation_binary(pix1, pix2):
         if get_leptonica_version() < 'leptonica-1.72':
             # Older versions of Leptonica (pre-1.72) have a buggy
             # implementation of pixCorrelationBinary that overflows on larger
             # images.
-            pix1_count = ffi.new('l_int32 *', 0)
+            pix1_count = ffi.new('l_int32 *')
-            pix2_count = ffi.new('l_int32 *', 0)
+            pix2_count = ffi.new('l_int32 *')
-            pixn_count = ffi.new('l_int32 *', 0)
+            pixn_count = ffi.new('l_int32 *')
-            tab8 = lept.makePixelSumTab8()  # Small memory leak on each call
+            tab8 = Pix.make_pixel_sum_tab8()
 
             lept.pixCountPixels(pix1.cpix, pix1_count, tab8)
             lept.pixCountPixels(pix2.cpix, pix2_count, tab8)

ocrmypdf/lib/compile_leptonica.py

@@ -62,6 +62,16 @@ l_int32 * makePixelSumTab8 ( void );
 PIX * pixDeserializeFromMemory ( const l_uint32 *data, size_t nbytes );
 l_int32 pixSerializeToMemory ( PIX *pixs, l_uint32 **pdata, size_t *pnbytes );
 
+l_int32
+pixOtsuAdaptiveThreshold(PIX       *pixs,
+                         l_int32    sx,
+                         l_int32    sy,
+                         l_int32    smoothx,
+                         l_int32    smoothy,
+                         l_float32  scorefract,
+                         PIX      **ppixth,
+                         PIX      **ppixd);
+
 void lept_free(void *ptr);
 """)
 

ocrmypdf/main.py

@@ -188,6 +188,10 @@ advanced.add_argument(
     '--tesseract-timeout', default=180.0, type=float, metavar='SECONDS',
     help='give up on OCR after the timeout, but copy the preprocessed page '
          'into the final output')
+advanced.add_argument(
+    '--rotate-pages-threshold', default=14.0, type=float, metavar='CONFIDENCE',
+    help="only rotate pages when confidence is above this value (arbitrary "
+         "units reported by tesseract)")
 
 debugging = parser.add_argument_group(
     "Debugging",
@@ -234,13 +238,18 @@ if options.pdf_renderer == 'tesseract' and tesseract.version() < '3.04.01' \
         "Some PDF viewers will fail to find searchable text.\n"
         "--pdf-renderer=tesseract is not recommended.")
 
-if any((options.deskew, options.clean, options.clean_final)):
+if any((options.clean, options.clean_final)):
     try:
         from . import unpaper
-    except ImportError:
+        if unpaper.version() < '6.1':
+            complain(
+                "The installed 'unpaper' is not supported. "
+                "Install version 6.1 or newer.")
+            sys.exit(ExitCode.missing_dependency)
+    except FileNotFoundError:
         complain(
             "Install the 'unpaper' program to use --deskew or --clean.")
-        sys.exit(ExitCode.bad_args)
+        sys.exit(ExitCode.missing_dependency)
 else:
     unpaper = None
 
@@ -529,15 +538,29 @@ def orient_page(
         270: '⇦'
     }
 
+    apply_correction = False
+    description = ''
+    if orient_conf.confidence >= options.rotate_pages_threshold:
+        if orient_conf.angle != 0:
+            apply_correction = True
+            description = ' - will rotate'
+        else:
+            description = ' - rotation appears correct'
+    else:
+        if orient_conf.angle != 0:
+            description = ' - confidence too low to rotate'
+        else:
+            description = ' - no change'
+
     log.info(
         '{0:4d}: page is facing {1}, confidence {2:.2f}{3}'.format(
             page_number(preview),
             direction.get(orient_conf.angle, '?'),
             orient_conf.confidence,
-            ' - correcting rotation' if orient_conf.angle != 0 else '')
+            description)
     )
 
-    if orient_conf.angle == 0:
+    if not apply_correction:
         re_symlink(page_pdf, output_file)
     else:
         writer = pypdf.PdfFileWriter()
@@ -1010,51 +1033,82 @@ def cleanup_ruffus_error_message(msg):
     return msg
 
 
+def do_ruffus_exception(ruffus_five_tuple):
+    """Replace the elaborate ruffus stack trace with a user friendly
+    description of the error message that occurred."""
+
+    task_name, job_name, exc_name, exc_value, exc_stack = ruffus_five_tuple
+    if exc_name == 'builtins.SystemExit':
+        match = re.search(r"\.(.+?)\)", exc_value)
+        exit_code_name = match.groups()[0]
+        exit_code = getattr(ExitCode, exit_code_name, 'other_error')
+        return exit_code
+    elif exc_name == 'ruffus.ruffus_exceptions.MissingInputFileError':
+        _log.error(cleanup_ruffus_error_message(exc_value))
+        return ExitCode.input_file
+    elif exc_name == 'builtins.TypeError':
+        # Even though repair_pdf will fail, ruffus will still try
+        # to call split_pages with no input files, likely due to a bug
+        if task_name == 'split_pages':
+            _log.error("Input file '{0}' is not a valid PDF".format(
+                options.input_file))
+            return ExitCode.input_file
+    elif exc_name == 'subprocess.CalledProcessError':
+        # It's up to the subprocess handler to report something useful
+        msg = "Error occurred while running this command:"
+        _log.error(msg + '\n' + exc_value)
+        return ExitCode.child_process_error
+    elif not options.verbose:
+        _log.error(exc_stack)
+        return ExitCode.other_error
+
+
+def traverse_ruffus_exception(e):
+    """Walk through a RethrownJobError and find the first exception.
+
+    The exit code will be based on this, even if multiple exceptions occurred
+    at the same time."""
+
+    if isinstance(e[0], str) and len(e) == 5:
+        return do_ruffus_exception(e)
+    elif hasattr(e, '__iter__'):
+        for exc in e:
+            return traverse_ruffus_exception(exc)
+
+
 def run_pipeline():
     if not options.jobs:
         options.jobs = available_cpu_count()
     try:
-        options.history_file = os.path.join(work_folder, 'ruffus_history.sqlite')
+        options.history_file = os.path.join(
+            work_folder, 'ruffus_history.sqlite')
         cmdline.run(options)
     except ruffus_exceptions.RethrownJobError as e:
         if options.verbose:
-            _log.debug(e)
+            _log.debug(str(e))  # stringify exception so logger doesn't have to
 
-        # Yuck. Hunt through the ruffus exception to find out what the
+        # Ruffus flattens exception to 5 element tuples. Because of a bug
-        # return code is supposed to be.
+        # in <= 2.6.3 it may present either the single:
-        # Ruffus flattens the exception to a string, throwing away all kinds
+        #   (task, job, exc, value, stack)
-        # of helpful details
+        # or something like:
-        # task_name, job_name - ruffus status
+        #   [[(task, job, exc, value, stack)]]
-        # exc_name - class name of exception
+        #
-        # exc_value - irritating string that makes impossible to recover
+        # Generally cross-process exception marshalling doesn't work well
-        #   exception object
+        # and ruffus doesn't support because BaseException has its own
-        # exc_stack - string that contains traceback of exception
+        # implementation of __reduce__ that attempts to reconstruct the
-        for exc in e.args:
+        # exception based on e.__init__(e.args).
-            task_name, job_name, exc_name, exc_value, exc_stack = exc
+        #
-            if exc_name == 'builtins.SystemExit':
+        # Attempting to log the exception directly marshalls it to the logger
-                match = re.search(r"\.(.+?)\)", exc_value)
+        # which is probably in another process, so it's better to log only
-                exit_code_name = match.groups()[0]
+        # data from the exception at this point.
-                exit_code = getattr(ExitCode, exit_code_name, 'other_error')
-                return exit_code
-            elif exc_name == 'ruffus.ruffus_exceptions.MissingInputFileError':
-                _log.error(cleanup_ruffus_error_message(exc_value))
-                return ExitCode.input_file
-            elif exc_name == 'builtins.TypeError':
-                # Even though repair_pdf will fail, ruffus will still try
-                # to call split_pages with no input files, likely due to a bug
-                if task_name == 'split_pages':
-                    _log.error("Input file '{0}' is not a valid PDF".format(
-                        options.input_file))
-                    return ExitCode.input_file
-            elif exc_name == 'subprocess.CalledProcessError':
-                # It's up to the subprocess handler to report something useful
-                msg = "Error occurred while running this command:"
-                _log.error(msg + '\n' + exc_value)
-                return ExitCode.child_process_error
-            elif not options.verbose:
-                _log.error(e)
 
-        return ExitCode.other_error
+        exitcode = traverse_ruffus_exception(e.args)
+        if exitcode is None:
+            _log.error("Unexpected ruffus exception: " + str(e))
+            _log.error(repr(e))
+            return ExitCode.other_error
+        else:
+            return exitcode
     except Exception as e:
         _log.error(e)
         return ExitCode.other_error

ocrmypdf/pageinfo.py

@@ -3,6 +3,7 @@
 
 from subprocess import Popen, PIPE
 from decimal import Decimal, getcontext
+from math import hypot
 import re
 import sys
 import PyPDF2 as pypdf
@@ -34,7 +35,12 @@ FRIENDLY_ENCODING = {
     '/JPXDecode': 'jpx',
     '/JBIG2Decode': 'jbig2',
     '/CCF': 'ccitt',  # Abbreviations permitted in inline images
-    '/DCT': 'jpeg'
+    '/DCT': 'jpeg',
+    '/AHx': 'asciihex',
+    '/A85': 'ascii85',
+    '/LZW': 'lzw',
+    '/Fl': 'flate',
+    '/RL': 'runlength'
 }
 
 FRIENDLY_COMP = {
@@ -67,14 +73,9 @@ def _shorthand_from_matrix(matrix):
     return tuple(map(float, (a, b, c, d, e, f)))
 
 
-def euclidean_distance(rowvec1, rowvec2):
+ContentsInfo = namedtuple('ContentsInfo', ['raster_settings', 'inline_images'])
-    return ((rowvec1[0] - rowvec2[0]) ** 2
-        + (rowvec1[1] - rowvec2[1]) ** 2) ** 0.5
 
 
-ContentsInfo = namedtuple('ContentsInfo',
-    ['raster_settings', 'inline_images'])
-
 def _interpret_contents(contentstream):
     operations = contentstream.operations
     stack = []
@@ -86,6 +87,8 @@ def _interpret_contents(contentstream):
         operands, command = op
         if command == b'q':
             stack.append(ctm)
+            if len(stack) > 32:
+                raise RuntimeError("PDF graphics stack overflow")
         elif command == b'Q':
             ctm = stack.pop()
         elif command == b'cm':
@@ -119,21 +122,31 @@ def _get_dpi(ctm_shorthand, image_size):
     it is not sufficient to assume that the image fills the page, even though
     that is the most common case.
 
-    This code solves the general case where the image may be scaled (always),
+    A PDF image may be scaled (always), cropped, translated, rotated in place
-    cropped, translated (often), and rotated in place (occasionally) to an
+    to an arbitrary angle (rarely) and skewed. Only equal area mappings can
-    arbitrary angle (rare). It will work as long as the image is a
+    be expressed, that is, it is not necessary to consider distortions where
-    parallelogram from the perspective of a rectilinear coordinate system.
+    the effective DPI varies with position.
-    It does not work for arbitrarily quadrilaterals that might be produced
-    by shearing, but by that point DPI becomes a linear gradient rather than
-    constant over the image.
 
-    The transformation matrix describes the coordinate system at the time of
+    To determine the image scale, transform an offset axis vector v0 (0, 0),
-    rendering. We transform the image corner locations into the coordinate
+    width-axis vector v0 (1, 0), height-axis vector vh (0, 1) with the matrix,
-    system and measure the width and height within the system, expressed in
+    which gives the dimensions of the image in PDF units. From there we can
-    PDF units. From there we can compare to the actual image dimensions.
+    compare to actual image dimensions. PDF uses
+    row vector * matrix_tranposed unlike the traditional
+    matrix * column vector.
+
+    The offset, width and height vectors can be combined in a matrix and
+    multiplied by the transform matrix. Then we want to calculated
+        magnitude(width_vector - offset_vector)
+    and
+        magnitude(height_vector - offset_vector)
+
+    When the above is worked out algebraically, the effect of translation
+    cancels out, and the vector magnitudes become functions of the nonzero
+    transformation matrix indices. The results of the derivation are used
+    in this code.
 
     pdfimages -list does calculate the DPI in some way that is not completely
-    naive, but it does not the DPI of rotated images right, so cannot be
+    naive, but it does not get the DPI of rotated images right, so cannot be
     used anymore to validate this. Photoshop works, or using Acrobat to
     rotate the image back to normal.
 
@@ -141,31 +154,12 @@ def _get_dpi(ctm_shorthand, image_size):
     /MediaBox.
 
     """
-    matrix = _matrix_from_shorthand(ctm_shorthand)
 
-    # Corners of the image in untransformed unit space; last
+    a, b, c, d, _, _ = ctm_shorthand
-    # column is a dummy to assist matrix math
-    corners = [[0, 0, 1],
-               [1, 0, 1],
-               [0, 1, 1],
-               [1, 1, 1]]
 
-    # Rotate/translate/scale the corners into PDF coords (1/72")
+    # Calculate the width and height of the image in PDF units
-    # ordering of points may change, e.g. if rotation is 180 then
+    image_drawn_width = hypot(a, b)
-    # the point (0, 0) may become the top right
+    image_drawn_height = hypot(c, d)
-    # The row vectors can all be transformed together here by building
-    # a matrix of them
-    page_unit_corners = matrix_mult(corners, matrix)
-
-    # Calculate the width and height of the rotated image
-    # the transformation matrix so the corner that was originally
-    # (1, 1) can be ignored
-    image_drawn_width = euclidean_distance(
-        page_unit_corners[0], page_unit_corners[1])
-    image_drawn_height = euclidean_distance(
-        page_unit_corners[0], page_unit_corners[2])
-
-    # print((image_drawn_width, image_drawn_height))
 
     # The scale of the image is pixels per PDF unit (1/72")
     scale_w = image_size[0] / image_drawn_width
@@ -178,7 +172,8 @@ def _get_dpi(ctm_shorthand, image_size):
     return (dpi_w, dpi_h)
 
 
-def _find_page_images(page, pageinfo, contentsinfo):
+def _find_page_inline_images(page, pageinfo, contentsinfo):
+    "Find inline images on the page"
 
     for n, im in enumerate(contentsinfo.inline_images):
         settings, shorthand = im
@@ -189,12 +184,22 @@ def _find_page_images(page, pageinfo, contentsinfo):
         image['bpc'] = settings['/BPC']
         image['color'] = FRIENDLY_COLORSPACE.get(settings['/CS'], '-')
         image['comp'] = FRIENDLY_COMP.get(image['color'], '?')
+        if '/F' in settings:
+            filter_ = settings['/F']
+            if isinstance(filter_, pypdf.generic.ArrayObject):
+                filter_ = filter_[0]
+            image['enc'] = FRIENDLY_ENCODING.get(filter_, 'image')
+        else:
+            image['enc'] = 'image'
 
         dpi_w, dpi_h = _get_dpi(shorthand, (image['width'], image['height']))
         image['dpi_w'], image['dpi_h'] = Decimal(dpi_w), Decimal(dpi_h)
         yield image
 
-    # Look for XObject (out of line images)
+
+def _find_page_regular_images(page, pageinfo, contentsinfo):
+    "Find images stored in XObject resources"
+
     try:
         page['/Resources']['/XObject']
     except KeyError:
@@ -204,9 +209,6 @@ def _find_page_images(page, pageinfo, contentsinfo):
         pdfimage = page['/Resources']['/XObject'][xobj]
         if pdfimage['/Subtype'] != '/Image':
             continue
-        if '/ImageMask' in pdfimage:
-            if pdfimage['/ImageMask']:
-                continue
         image = {}
         image['name'] = str(xobj)
         image['width'] = pdfimage['/Width']
@@ -250,6 +252,11 @@ def _find_page_images(page, pageinfo, contentsinfo):
         yield image
 
 
+def _find_page_images(page, pageinfo, contentsinfo):
+    yield from _find_page_inline_images(page, pageinfo, contentsinfo)
+    yield from _find_page_regular_images(page, pageinfo, contentsinfo)
+
+
 def _page_has_text(pdf, page):
     # Simple test
     text = page.extractText()

ocrmypdf/unpaper.py

@@ -3,7 +3,7 @@
 # unpaper documentation:
 # https://github.com/Flameeyes/unpaper/blob/master/doc/basic-concepts.md
 
-from subprocess import Popen, PIPE
+from subprocess import CalledProcessError, STDOUT, check_output, check_call
 from tempfile import NamedTemporaryFile
 import sys
 import os
@@ -17,10 +17,9 @@ def version():
         get_program('unpaper'),
         '--version'
     ]
-    p_unpaper = Popen(args_unpaper, close_fds=True, universal_newlines=True,
+    version = check_output(
-                      stdout=PIPE, stderr=PIPE)
+        args_unpaper, close_fds=True, universal_newlines=True,
-    version, _ = p_unpaper.communicate(timeout=5)
+        stderr=STDOUT, timeout=5)
-
     return version.strip()
 
 
@@ -68,15 +67,17 @@ def run(input_file, output_file, dpi, log, mode_args):
         os.unlink(output_pnm.name)
 
         args_unpaper.extend([input_pnm.name, output_pnm.name])
-        p_unpaper = Popen(
+        try:
-            args_unpaper, close_fds=True,
+            stdout = check_output(
-            universal_newlines=True, stdout=PIPE, stderr=PIPE
+                args_unpaper, close_fds=True,
-            )
+                universal_newlines=True, stderr=STDOUT,
-        out, err = p_unpaper.communicate()
+                )
-        log.debug(out)
+        except CalledProcessError as e:
-        log.debug(err)
+            log.debug(e.output)
-
+            raise e from e
-        Image.open(output_pnm.name).save(output_file)
+        else:
+            log.debug(stdout)
+            Image.open(output_pnm.name).save(output_file)
 
 
 def deskew(input_file, output_file, dpi, log):

setup.py

@@ -217,7 +217,7 @@ setup(
         'ocrmypdf/lib/compile_leptonica.py:ffi'
     ],
     install_requires=[
-        'ruffus>=2.6.3',
+        'ruffus==2.6.3',
         'Pillow>=3.0.0',
         'reportlab>=3.1.44',
         'PyPDF2>=1.25.1',

tests/spoof/unpaper_oldversion.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+import sys
+
+def main():
+    if sys.argv[1] == '--version':
+        print('0.5')
+        sys.exit(0)
+
+    print("Only supports --version")
+    sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

tests/test_main.py

@@ -325,11 +325,6 @@ def test_monochrome_correlation():
     'tesseract',
     ])
 def test_autorotate(spoof_tesseract_cache, renderer):
-    import ocrmypdf.ghostscript as ghostscript
-    import logging
-
-    gslog = logging.getLogger()
-
     # cardinal.pdf contains four copies of an image rotated in each cardinal
     # direction - these ones are "burned in" not tagged with /Rotate
     out = check_ocrmypdf('cardinal.pdf', 'test_autorotate_%s.pdf' % renderer,
@@ -343,6 +338,36 @@ def test_autorotate(spoof_tesseract_cache, renderer):
         assert correlation > 0.80
 
 
+def test_autorotate_threshold_low(spoof_tesseract_cache):
+    out = check_ocrmypdf('cardinal.pdf', 'test_autorotate_threshold_low.pdf',
+                         '--rotate-pages-threshold', '1',
+                         '-r', '-v', '1', env=spoof_tesseract_cache)
+
+    # Low threshold -> always rotate -> expect high correlation between
+    # reference page and test page
+    correlation = check_monochrome_correlation(
+        reference_pdf=_infile('cardinal.pdf'),
+        reference_pageno=1,
+        test_pdf=out,
+        test_pageno=3)
+    assert correlation > 0.80
+
+
+def test_autorotate_threshold_high(spoof_tesseract_cache):
+    out = check_ocrmypdf('cardinal.pdf', 'test_autorotate_threshold_high.pdf',
+                         '--rotate-pages-threshold', '99',
+                         '-r', '-v', '1', env=spoof_tesseract_cache)
+
+    # High threshold -> never rotate -> expect low correlation since
+    # test page will not be rotated
+    correlation = check_monochrome_correlation(
+        reference_pdf=_infile('cardinal.pdf'),
+        reference_pageno=1,
+        test_pdf=out,
+        test_pageno=3)
+    assert correlation < 0.10
+
+
 @pytest.mark.parametrize('renderer', [
     'hocr',
     'tesseract',
@@ -493,6 +518,8 @@ def test_tesseract_crash_autorotate(spoof_tesseract_crash):
     assert sh.returncode == ExitCode.child_process_error
     assert not os.path.exists(_outfile('wontwork.pdf'))
     assert "ERROR" in err
+    print(out)
+    print(err)
 
 
 @pytest.mark.parametrize('renderer', [
@@ -503,3 +530,20 @@ def test_tesseract_image_too_big(renderer, spoof_tesseract_big_image_error):
     check_ocrmypdf(
         'hugemono.pdf', 'hugemono_%s.pdf' % renderer, '-r',
         '--pdf-renderer', renderer, env=spoof_tesseract_big_image_error)
+
+
+def test_no_unpaper():
+    env = os.environ.copy()
+    env['OCRMYPDF_UNPAPER'] = os.path.abspath('./spoof/no_unpaper_here.py')
+    sh, out, err = run_ocrmypdf_env(
+        'c02-22.pdf', 'wont_be_created.pdf', '--clean', env=env)
+    assert sh.returncode == ExitCode.missing_dependency
+
+
+def test_old_unpaper():
+    env = os.environ.copy()
+    env['OCRMYPDF_UNPAPER'] = os.path.abspath('./spoof/unpaper_oldversion.py')
+    sh, out, err = run_ocrmypdf_env(
+        'c02-22.pdf', 'wont_be_created.pdf', '--clean', env=env)
+    assert sh.returncode == ExitCode.missing_dependency
+