v4.2.5: update release notes, fix silly typo in pageinfo.py

`brew install tesseract` just installed the english language pack not French, German or Spanish

.gitignore

@@ -3,6 +3,7 @@
 *.sublime-*
 venv-*/
 pyvenv.cfg
+tasks.py
 
 # Package building
 *.egg-info/

MANIFEST.in

@@ -32,6 +32,7 @@ recursive-exclude .github *
 recursive-include ocrmypdf/data *
 recursive-include share *
 include *.py
+exclude tasks.py
 
 # code
 recursive-include ocrmypdf *.py

README.rst

@@ -19,7 +19,7 @@ Main features
 -  Processes pages in parallel when more than one CPU core is
    available
 -  Uses `Tesseract OCR <https://github.com/tesseract-ocr/tesseract>`_ engine
--  Supports the `39 languages <https://code.google.com/p/tesseract-ocr/downloads/list>`_ recognized by Tesseract
+-  Supports more than `100 languages <https://github.com/tesseract-ocr/tessdata>`_ recognized by Tesseract
 -  Battle-tested on thousands of PDFs, a test suite and continuous integration
 
 For details: please consult the `release notes <RELEASE_NOTES.rst>`_.
@@ -162,7 +162,7 @@ Install the required Tesseract OCR engine with the language packs you plan to us
    
 .. code-block:: bash
 
-   brew install tesseract                       # Option 1: for English, French, German, Spanish
+   brew install tesseract                       # Option 1: for English
    
 .. code-block:: bash
    

RELEASE_NOTES.rst

@@ -3,6 +3,20 @@ RELEASE NOTES
 
 OCRmyPDF uses `semantic versioning <http://semver.org/>`_.
 
+
+v4.2.5:
+=======
+
+-  Fixed an issue (#100) with PDFs that omit the optional /BitsPerComponent parameter on images
+-  Removed non-free file milk.pdf
+
+
+v4.2.4:
+=======
+
+-  Fixed an error (#90) caused by PDFs that use stencil masks properly
+-  Fixed handling of PDFs that try to draw images or stencil masks without properly setting up the graphics state (such images are now ignored for the purposes of calculating DPI)
+
 v4.2.3:
 =======
 

ocrmypdf/pageinfo.py

@@ -72,11 +72,38 @@ def _shorthand_from_matrix(matrix):
     e, f = matrix[2][0], matrix[2][1]
     return tuple(map(float, (a, b, c, d, e, f)))
 
+RasterSettings = namedtuple('RasterSettings',
+    ['name', 'shorthand', 'stack_depth'])
+
+InlineSettings = namedtuple('InlineSettings',
+    ['settings', 'shorthand', 'stack_depth'])
 
 ContentsInfo = namedtuple('ContentsInfo', ['raster_settings', 'inline_images'])
 
 
 def _interpret_contents(contentstream):
+    """Interpret the PDF content stream
+
+    The stack represents the state of the PDF graphics stack.  We are only
+    interested in the current transformation matrix (CTM) so we only track
+    this object; a full implementation would need to track many other items.
+
+    The CTM is initialized to the mapping from user space to device space.
+    PDF units are 1/72".  In a PDF viewer or printer this matrix is initialized
+    to the transformation to device space.  For example if set to
+    (1/72, 0, 0, 1/72, 0, 0) then all units would be calculated in inches.
+
+    Images are always considered to be (0, 0) -> (1, 1).  Before drawing an
+    image there should be a 'cm' that sets up an image coordinate system
+    where drawing from (0, 0) -> (1, 1) will draw on the desired area of the
+    page.
+
+    PDF units suit our needs so we initialize ctm to the identity matrix.
+
+    PyPDF2 replaces inline images with a fake "INLINE IMAGE" operator.
+
+    """
+
     operations = contentstream.operations
     stack = []
     ctm = _matrix_from_shorthand((1, 0, 0, 1, 0, 0))
@@ -96,12 +123,16 @@ def _interpret_contents(contentstream):
                 _matrix_from_shorthand(operands), ctm)
         elif command == b'Do':
             image_name = operands[0]
-            image_raster_settings.append(
+            raster = RasterSettings(
-                (image_name, _shorthand_from_matrix(ctm)))
+                name=image_name, shorthand=_shorthand_from_matrix(ctm),
+                stack_depth=len(stack))
+            image_raster_settings.append(raster)
         elif command == b'INLINE IMAGE':
             settings = operands['settings']
-            inline_images.append(
+            inline = InlineSettings(
-                (settings, _shorthand_from_matrix(ctm)))
+                settings=settings, shorthand=_shorthand_from_matrix(ctm),
+                stack_depth=len(stack))
+            inline_images.append(inline)
 
     return ContentsInfo(
         raster_settings=image_raster_settings,
@@ -175,24 +206,27 @@ def _get_dpi(ctm_shorthand, image_size):
 def _find_page_inline_images(page, pageinfo, contentsinfo):
     "Find inline images on the page"
 
-    for n, im in enumerate(contentsinfo.inline_images):
+    for n, inline in enumerate(contentsinfo.inline_images):
-        settings, shorthand = im
         image = {}
         image['name'] = str('inline-%02d' % n)
-        image['width'] = settings['/W']
+        image['width'] = inline.settings['/W']
-        image['height'] = settings['/H']
+        image['height'] = inline.settings['/H']
-        image['bpc'] = settings['/BPC']
+        if '/BPC' in inline.settings:
-        image['color'] = FRIENDLY_COLORSPACE.get(settings['/CS'], '-')
+            image['bpc'] = inline.settings['/BPC']
+        else:
+            image['bpc'] = 8
+        image['color'] = FRIENDLY_COLORSPACE.get(inline.settings['/CS'], '-')
         image['comp'] = FRIENDLY_COMP.get(image['color'], '?')
-        if '/F' in settings:
+        if '/F' in inline.settings:
-            filter_ = settings['/F']
+            filter_ = inline.settings['/F']
             if isinstance(filter_, pypdf.generic.ArrayObject):
                 filter_ = filter_[0]
             image['enc'] = FRIENDLY_ENCODING.get(filter_, 'image')
         else:
             image['enc'] = 'image'
 
-        dpi_w, dpi_h = _get_dpi(shorthand, (image['width'], image['height']))
+        dpi_w, dpi_h = _get_dpi(
+            inline.shorthand, (image['width'], image['height']))
         image['dpi_w'], image['dpi_h'] = Decimal(dpi_w), Decimal(dpi_h)
         yield image
 
@@ -213,7 +247,10 @@ def _find_page_regular_images(page, pageinfo, contentsinfo):
         image['name'] = str(xobj)
         image['width'] = pdfimage['/Width']
         image['height'] = pdfimage['/Height']
-        image['bpc'] = pdfimage['/BitsPerComponent']
+        if '/BitsPerComponent' in pdfimage:
+            image['bpc'] = pdfimage['/BitsPerComponent']
+        else:
+            image['bpc'] = 8
 
         # Fixme: this is incorrectly treats explicit masks as stencil masks,
         # but good enough for now. Explicit masks have /ImageMask true but are
@@ -253,26 +290,19 @@ def _find_page_regular_images(page, pageinfo, contentsinfo):
 
         for raster in contentsinfo.raster_settings:
             # Loop in case the same image is display multiple times on a page
-            if raster[0] != image['name']:
+            if raster.name != image['name']:
                 continue
-            shorthand = raster[1]
 
-            if image['type'] == 'stencil':
+            if raster.stack_depth == 0:
-                # Stencil masks are implicitly scaled over the whole page
+                # At least one PDF in the wild (and test suite) draws an image
-                # Images that are used in explicit masks are not drawn directly
+                # when the graphics stack depth is 0, meaning that the image
-                # but drawn by the image they mask over, so they will never
+                # gets drawn into a square of 1x1 PDF units (or 1/72",
-                # be called for in raster settings
+                # or 0.35 mm).  The equivalent DPI will be >100,000.  Exclude
-                if shorthand != (1, 0, 0, 1, 0, 0):
+                # these from our DPI calculation for the page.
-                    raise NotImplementedError(
+                continue
-                        "Don't know how to handle "
-                        "stencil masks when graphics stack depth > 0.")
-                page_w = float(pageinfo['width_inches']) * 72.0
-                page_h = float(pageinfo['height_inches']) * 72.0
-                shorthand = (page_w, 0.0, 0.0,
-                             page_h, 0.0, 0.0)
 
             dpi_w, dpi_h = _get_dpi(
-                shorthand, (image['width'], image['height']))
+                raster.shorthand, (image['width'], image['height']))
 
             # When image is used multiple times take the highest DPI it is
             # rendered at

pytest.ini

@@ -1,2 +1,2 @@
 [pytest]
-norecursedirs = lib .pc
+norecursedirs = lib .pc .git

tasks.py

@@ -1,101 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# © 2016 James R. Barlow: github.com/jbarlow83
-# Release sanity checking
-
-import argparse
-from subprocess import run, PIPE, DEVNULL, STDOUT, CalledProcessError
-from git import Repo, Remote
-import logging
-import re
-import sys
-import os
-
-
-def test_repo(repo):
-    assert not repo.is_dirty(), "Repository is dirty"
-    if repo.untracked_files:
-        logging.warning('Some files are untracked:')
-        logging.warning('\n' + '\n'.join(repo.untracked_files))
-    assert repo.active_branch.name == 'master', 'Not on branch master'
-
-
-def travis(args):
-    repo = Repo('.')
-    test_repo(repo)
-
-    git_describe = repo.git.describe()
-
-    try:
-        env = os.environ.copy()
-        env['SETUPTOOLS_SCM_PRETEND_VERSION'] = git_describe
-        proc = run(['check-manifest'], check=True, universal_newlines=True, stdout=PIPE, stderr=STDOUT, env=env)
-        logging.info(proc.stdout)
-    except CalledProcessError as e:
-        logging.error('MANIFEST.in error')
-        logging.error(e.stdout)
-        sys.exit(1)
-
-    run(['python3', 'setup.py', 'build'], check=True)
-
-    origin = Remote(repo, 'jbarlow')
-    result = origin.push(refspec='master:master')[0]
-    if result.flags & (1024|4|8|32|16):
-        logging.error(result.summary)
-    else:
-        logging.info(result.summary)
-
-    logging.info("Pushed to Travis CI")
-    logging.info("If this passes, git tag and release")
-
-
-def release(args):
-    repo = Repo('.')
-    test_repo(repo)
-
-    git_describe = repo.git.describe()
-
-    assert not git_describe.startswith('v') and not '-' in git_describe and not '+ng' in git_describe, \
-        "Not tagged properly for release: " + git_describe
-
-    plain_version = git_describe[1:]  # without 'v' prefix
-
-    with open('RELEASE_NOTES.rst') as f:
-        notes = f.read()
-        assert plain_version in notes, "Version not mentioned in release notes"
-
-    proc = run(['python3', 'setup.py', 'sdist', 'bdist_wheel'], universal_newlines=True, check=True, stdout=PIPE, stderr=STDOUT)
-    logging.info(proc.stdout)
-
-
-    origin = Remote(repo, 'jbarlow')
-    result = origin.push(refspec='master:master', tags=True)[0]
-    if result.flags & (1024|4|8|32|16):
-        logging.error(result.summary)
-    else:
-        logging.info(result.summary)
-
-    run(['twine', 'upload', '-r', 'pypitest',
-        'dist/ocrmypdf-{}.tar.gz'.format(plain_version),
-        'dist/ocrmypdf-{}-py34-none-any.whl'.format(plain_version)], check=True, universal_newlines=True, stdout=PIPE)
-
-
-parser = argparse.ArgumentParser(description="ocrmypdf release tasks")
-subparsers = parser.add_subparsers()
-
-push_travis = subparsers.add_parser(
-    'push-travis', description="Push master to travis for testing")
-push_travis.set_defaults(func=travis)
-
-release_parser = subparsers.add_parser(
-    'release', description="Release to PyPI etc")
-release_parser.set_defaults(func=release)
-
-
-def main():
-    args = parser.parse_args()
-    args.func(args)
-
-
-if __name__ == '__main__':
-    main()

tests/resources/README.rst

@@ -9,20 +9,28 @@ Files derived from free sources
 These test resources come from free sources, under either public domain or Creative Commons licenses.
 In some cases they were converted from one image format to another without other changes.
 
-+---------------------+--------------------------------------------------------------------------------+
+.. list-table:: 
-| File                | Source                                                                         |
+    :widths: 20 50 30
-+=====================+================================================================================+
+    :header-rows: 1
-| c02-22.pdf          | `Project Gutenberg`_, Adventures of Huckleberry Finn, page 22                  |
+
-+---------------------+--------------------------------------------------------------------------------+
+    *   - File
-| congress.jpg        | `US Congressional Records`_ (Public Domain)                                    |
+        - Source
-+---------------------+--------------------------------------------------------------------------------+
+        - License
-| graph.pdf           | `Wikimedia: Pandas text analysis.png`_ (Public Domain)                         |
+    *   - c02-22.pdf
-+---------------------+--------------------------------------------------------------------------------+
+        - `Project Gutenberg`_, Adventures of Huckleberry Finn, page 22
-| lichtenstein.pdf    | `Wikimedia: JPEG2000 Lichtenstein`_ (Creative Commons BY-SA 3.0)               |
+        - Public Domain
-+---------------------+--------------------------------------------------------------------------------+
+    *   - congress.jpg
-| LinnSequencer.jpg,  | `Wikimedia: LinnSequencer`_ (Creative Commons BY-SA 3.0)                       |
+        - `US Congressional Records`_
-| linn.pdf, linn.txt  |                                                                                |
+        - Public Domain
-+---------------------+--------------------------------------------------------------------------------+
+    *   - graph.pdf
+        - `Wikimedia: Pandas text analysis.png`_
+        - Public Domain
+    *   - lichtenstein.pdf
+        - `Wikimedia: JPEG2000 Lichtenstein`_
+        - Creative Commons BY-SA 3.0
+    *   - LinnSequencer.jpg, linn.pdf, linn.txt
+        - `Wikimedia: LinnSequencer`_
+        - Creative Commons BY-SA 3.0
 
 
 Files generated for this project
@@ -31,30 +39,43 @@ Files generated for this project
 The following test resources were crafted specifically for this project, and can be used
 under the terms of the license in LICENSE.rst.
 
-+---------------------+-----------------------+---------------------------------------------------------+
+.. list-table:: 
-| File                | Contributor           | Purpose                                                 |
+    :widths: 20 20 60
-+---------------------+-----------------------+---------------------------------------------------------+
+    :header-rows: 1
-| aspect.pdf          | @jbarlow83            | test image with 200 x 100 DPI resolution                |
+
-+---------------------+-----------------------+---------------------------------------------------------+
+    *   - File
-| blank.pdf           | @jbarlow83            | blank PDF                                               |
+        - Contributor
-+---------------------+-----------------------+---------------------------------------------------------+
+        - Purpose
-| cmyk.pdf            | @jbarlow83            | a CMYK image created in Photoshop                       |
+    *   - aspect.pdf
-+---------------------+-----------------------+---------------------------------------------------------+
+        - @jbarlow83
-| enormous.pdf        | @jbarlow83            | very large PDF page                                     |
+        - test image with 200 x 100 DPI resolution
-+---------------------+-----------------------+---------------------------------------------------------+
+    *   - blank.pdf
-| francais.pdf        | @jbarlow83            | a page containing French accents (diacritics)           |
+        - @jbarlow83
-+---------------------+-----------------------+---------------------------------------------------------+
+        - blank PDF
-| hugemono.pdf        | @jbarlow83            | large monochrome 35000x35000 image in JBIG2 encoding    |
+    *   - cmyk.pdf
-+---------------------+-----------------------+---------------------------------------------------------+
+        - @jbarlow83
-| invalid.pdf         | @jbarlow83            | a PDF file header followed by EOF marker                |
+        - a CMYK image created in Photoshop
-+---------------------+-----------------------+---------------------------------------------------------+
+    *   - enormous.pdf
-| masks.pdf           | @supergrobi           | file containing stencil masks; printout of a German     |
+        - @jbarlow83
-|                     |                       | Wikipedia article (Creative Commons BY-SA)              |
+        - very large PDF page
-+---------------------+-----------------------+---------------------------------------------------------+
+    *   - epson.pdf
-| milk.pdf            | @lowesjam             | linearized PDF containing some indirect objects         |
+        - @lowesjam
-+---------------------+-----------------------+---------------------------------------------------------+
+        - a linearized PDF containing some unusual indirect objects, created by an Epson printer; printout of a Wikipedia article (CC BY-SA)
-| missing_docinfo.pdf | @jbarlow83            | PDF file with no /DocumentInfo section                  |
+    *   - francais.pdf
-+---------------------+-----------------------+---------------------------------------------------------+
+        - @jbarlow83
+        - a page containing French accents (diacritics)  
+    *   - hugemono.pdf
+        - @jbarlow83
+        - large monochrome 35000x35000 image in JBIG2 encoding 
+    *   - invalid.pdf
+        - @jbarlow83
+        - a PDF file header followed by EOF marker
+    *   - masks.pdf
+        - @supergrobi
+        - file containing explicit masks and a stencil mask drawn without a proper transformation matrix; printout of a German Wikipedia article (CC BY-SA)
+    *   - missing_docinfo.pdf
+        - @jbarlow83
+        - PDF file with no /DocumentInfo section 
 
 Assemblies
 ==========

tests/test_main.py

@@ -124,8 +124,8 @@ def spoof_tesseract_big_image_error():
     return spoof('tesseract', 'tesseract_big_image_error.py')
 
 
-def test_quick(spoof_tesseract_noop):
+def test_quick(spoof_tesseract_cache):
-    check_ocrmypdf('c02-22.pdf', 'test_quick.pdf', env=spoof_tesseract_noop)
+    check_ocrmypdf('ccitt.pdf', 'test_quick.pdf', env=spoof_tesseract_cache)
 
 
 def test_deskew(spoof_tesseract_noop):
@@ -635,4 +635,6 @@ def test_masks(spoof_tesseract_noop):
 
 
 def test_linearized_pdf_and_indirect_object(spoof_tesseract_noop):
-    check_ocrmypdf('milk.pdf', 'test_milk.pdf', env=spoof_tesseract_noop)
+    check_ocrmypdf(
+        'epson.pdf', 'test_epson.pdf',
+        env=spoof_tesseract_noop)