v4.2.4 release notes

More thorough testing showed that Acrobat do not presume that images
fill the page if the CTM is unspecified, as tests/resources/masks.pdf
seems to want.  Instead they treat it literally and draw the image
as 1x1 PDF units or 1/72" square in the bottom left corner of the page.

Seems like the best thing to do is ignore any such images for the purpose
of DPI calculation.  masks.pdf still works out okay because it has
other images.

For more robustness we could consider invalidating any DPI above some
limit, or warning the user about these microdot thumbnails.

RELEASE_NOTES.rst

@@ -3,6 +3,13 @@ RELEASE NOTES
 
 OCRmyPDF uses `semantic versioning <http://semver.org/>`_.
 
+
+v4.2.4:
+=======
+
+-  Fixed an error (#90) caused by PDFs that use stencil masks properly
+-  Fixed handling of PDFs that try to draw images or stencil masks without properly setting up the graphics state (such images are now ignored for the purposes of calculating DPI)
+
 v4.2.3:
 =======
 

ocrmypdf/pageinfo.py

@@ -72,11 +72,38 @@ def _shorthand_from_matrix(matrix):
     e, f = matrix[2][0], matrix[2][1]
     return tuple(map(float, (a, b, c, d, e, f)))
 
+RasterSettings = namedtuple('RasterSettings',
+    ['name', 'shorthand', 'stack_depth'])
+
+InlineSettings = namedtuple('InlineSettings',
+    ['settings', 'shorthand', 'stack_depth'])
 
 ContentsInfo = namedtuple('ContentsInfo', ['raster_settings', 'inline_images'])
 
 
 def _interpret_contents(contentstream):
+    """Interpret the PDF content stream
+
+    The stack represents the state of the PDF graphics stack.  We are only
+    interested in the current transformation matrix (CTM) so we only track
+    this object; a full implementation would need to track many other items.
+
+    The CTM is initialized to the mapping from user space to device space.
+    PDF units are 1/72".  In a PDF viewer or printer this matrix is initialized
+    to the transformation to device space.  For example if set to
+    (1/72, 0, 0, 1/72, 0, 0) then all units would be calculated in inches.
+
+    Images are always considered to be (0, 0) -> (1, 1).  Before drawing an
+    image there should be a 'cm' that sets up an image coordinate system
+    where drawing from (0, 0) -> (1, 1) will draw on the desired area of the
+    page.
+
+    PDF units suit our needs so we initialize ctm to the identity matrix.
+
+    PyPDF2 replaces inline images with a fake "INLINE IMAGE" operator.
+
+    """
+
     operations = contentstream.operations
     stack = []
     ctm = _matrix_from_shorthand((1, 0, 0, 1, 0, 0))
@@ -96,12 +123,16 @@ def _interpret_contents(contentstream):
                 _matrix_from_shorthand(operands), ctm)
         elif command == b'Do':
             image_name = operands[0]
-            image_raster_settings.append(
+            raster = RasterSettings(
-                (image_name, _shorthand_from_matrix(ctm)))
+                name=image_name, shorthand=_shorthand_from_matrix(ctm),
+                stack_depth=len(stack))
+            image_raster_settings.append(raster)
         elif command == b'INLINE IMAGE':
             settings = operands['settings']
-            inline_images.append(
+            inline = InlineSettings(
-                (settings, _shorthand_from_matrix(ctm)))
+                settings=settings, shorthand=_shorthand_from_matrix(ctm),
+                stack_depth=len(stack))
+            inline_images.append(inline)
 
     return ContentsInfo(
         raster_settings=image_raster_settings,
@@ -175,24 +206,24 @@ def _get_dpi(ctm_shorthand, image_size):
 def _find_page_inline_images(page, pageinfo, contentsinfo):
     "Find inline images on the page"
 
-    for n, im in enumerate(contentsinfo.inline_images):
+    for n, inline in enumerate(contentsinfo.inline_images):
-        settings, shorthand = im
         image = {}
         image['name'] = str('inline-%02d' % n)
-        image['width'] = settings['/W']
+        image['width'] = inline.settings['/W']
-        image['height'] = settings['/H']
+        image['height'] = inline.settings['/H']
-        image['bpc'] = settings['/BPC']
+        image['bpc'] = inline.settings['/BPC']
-        image['color'] = FRIENDLY_COLORSPACE.get(settings['/CS'], '-')
+        image['color'] = FRIENDLY_COLORSPACE.get(inline.settings['/CS'], '-')
         image['comp'] = FRIENDLY_COMP.get(image['color'], '?')
-        if '/F' in settings:
+        if '/F' in inline.settings:
-            filter_ = settings['/F']
+            filter_ = inline.settings['/F']
             if isinstance(filter_, pypdf.generic.ArrayObject):
                 filter_ = filter_[0]
             image['enc'] = FRIENDLY_ENCODING.get(filter_, 'image')
         else:
             image['enc'] = 'image'
 
-        dpi_w, dpi_h = _get_dpi(shorthand, (image['width'], image['height']))
+        dpi_w, dpi_h = _get_dpi(
+            inline.shorthand, (image['width'], image['height']))
         image['dpi_w'], image['dpi_h'] = Decimal(dpi_w), Decimal(dpi_h)
         yield image
 
@@ -253,26 +284,19 @@ def _find_page_regular_images(page, pageinfo, contentsinfo):
 
         for raster in contentsinfo.raster_settings:
             # Loop in case the same image is display multiple times on a page
-            if raster[0] != image['name']:
+            if raster.name != image['name']:
                 continue
-            shorthand = raster[1]
 
-            if image['type'] == 'stencil':
+            if raster.stack_depth == 0:
-                # Stencil masks are implicitly scaled over the whole page
+                # At least one PDF in the wild (and test suite) draws an image
-                # Images that are used in explicit masks are not drawn directly
+                # when the graphics stack depth is 0, meaning that the image
-                # but drawn by the image they mask over, so they will never
+                # gets drawn into a square of 1x1 PDF units (or 1/72",
-                # be called for in raster settings
+                # or 0.35 mm).  The equivalent DPI will be >100,000.  Exclude
-                if shorthand != (1, 0, 0, 1, 0, 0):
+                # these from our DPI calculation for the page.
-                    raise NotImplementedError(
+                continue
-                        "Don't know how to handle "
-                        "stencil masks when graphics stack depth > 0.")
-                page_w = float(pageinfo['width_inches']) * 72.0
-                page_h = float(pageinfo['height_inches']) * 72.0
-                shorthand = (page_w, 0.0, 0.0,
-                             page_h, 0.0, 0.0)
 
             dpi_w, dpi_h = _get_dpi(
-                shorthand, (image['width'], image['height']))
+                raster.shorthand, (image['width'], image['height']))
 
             # When image is used multiple times take the highest DPI it is
             # rendered at

tasks.py

@@ -5,13 +5,20 @@
 
 import argparse
 from subprocess import run, PIPE, DEVNULL, STDOUT, CalledProcessError
-from git import Repo, Remote
+from git import Repo, Remote, PushInfo
 import logging
 import re
 import sys
 import os
 
 
+logging.basicConfig(level=logging.INFO)
+
+REMOTE_ERROR_FLAGS = \
+    PushInfo.REJECTED | PushInfo.NO_MATCH | PushInfo.REMOTE_REJECTED | \
+    PushInfo.REMOTE_FAILURE | PushInfo.DELETED | PushInfo.ERROR
+
+
 def test_repo(repo):
     assert not repo.is_dirty(), "Repository is dirty"
     if repo.untracked_files:
@@ -40,8 +47,10 @@ def travis(args):
 
     origin = Remote(repo, 'jbarlow')
     result = origin.push(refspec='master:master')[0]
-    if result.flags & (1024|4|8|32|16):
+
+    if result.flags & REMOTE_ERROR_FLAGS:
         logging.error(result.summary)
+        sys.exit(1)
     else:
         logging.info(result.summary)
 
@@ -55,7 +64,7 @@ def release(args):
 
     git_describe = repo.git.describe()
 
-    assert not git_describe.startswith('v') and not '-' in git_describe and not '+ng' in git_describe, \
+    assert git_describe.startswith('v') and not '-' in git_describe and not '+ng' in git_describe, \
         "Not tagged properly for release: " + git_describe
 
     plain_version = git_describe[1:]  # without 'v' prefix
@@ -70,8 +79,9 @@ def release(args):
 
     origin = Remote(repo, 'jbarlow')
     result = origin.push(refspec='master:master', tags=True)[0]
-    if result.flags & (1024|4|8|32|16):
+    if result.flags & REMOTE_ERROR_FLAGS:
         logging.error(result.summary)
+        sys.exit(1)
     else:
         logging.info(result.summary)
 

tests/resources/README.rst

@@ -48,8 +48,9 @@ under the terms of the license in LICENSE.rst.
 +---------------------+-----------------------+---------------------------------------------------------+
 | invalid.pdf         | @jbarlow83            | a PDF file header followed by EOF marker                |
 +---------------------+-----------------------+---------------------------------------------------------+
-| masks.pdf           | @supergrobi           | file containing stencil masks; printout of a German     |
+| masks.pdf           | @supergrobi           | file containing explicit masks and a stencil mask       |
-|                     |                       | Wikipedia article (Creative Commons BY-SA)              |
+|                     |                       | drawn without a proper transformation matrix; printout  |
+|                     |                       | of a German Wikipedia article (Creative Commons BY-SA)  |
 +---------------------+-----------------------+---------------------------------------------------------+
 | milk.pdf            | @lowesjam             | linearized PDF containing some indirect objects         |
 +---------------------+-----------------------+---------------------------------------------------------+