v4.2.5: update release notes, fix silly typo in pageinfo.py

`brew install tesseract` just installed the english language pack not French, German or Spanish

.github/ISSUE_TEMPLATE

@@ -0,0 +1,3 @@
+Please include the command line and (if needed) a test file with your issue report.
+
+If possible, please use a test file that we can include in future test cases (no personal information, no copyrighted material).

.gitignore

@@ -3,6 +3,7 @@
 *.sublime-*
 venv-*/
 pyvenv.cfg
+tasks.py
 
 # Package building
 *.egg-info/
@@ -26,5 +27,5 @@ log/
 .ipynb_checkpoints/
 tests/cache/
 tests/output/
-tests/resources/private
+tests/resources/private/
 tmp/

MANIFEST.in

@@ -1,4 +1,43 @@
-recursive-exclude tests/output *
+# requirements
 include requirements.txt
 include test_requirements.txt
-include share/*
+include dev_requirements.txt
+
+# git
+include .git_archival.txt
+
+# docker
+include Dockerfile
+include Dockerfile.polyglot
+include .dockerignore
+
+# tests
+include pytest.ini
+recursive-include tests *.jpg
+recursive-include tests *.pdf
+recursive-include tests *.py
+recursive-include tests *.rst
+recursive-include tests *.txt
+recursive-exclude tests/output *
+recursive-exclude tests/output_pageinfo *
+recursive-exclude tests/resources/private *
+recursive-exclude tests/cache *
+
+# documentation
+include *.rst
+include pipeline.svg
+recursive-exclude .github * 
+
+# support files
+recursive-include ocrmypdf/data *
+recursive-include share *
+include *.py
+exclude tasks.py
+
+# code
+recursive-include ocrmypdf *.py
+exclude ocrmypdf/lib/_leptonica.py
+include OCRmyPDF.sh
+include docker-wrapper.sh
+
+

OCRmyPDF.sh

@@ -3,4 +3,6 @@
 # Copyright (c) 2013-14: fritz-hh from Github (https://github.com/fritz-hh)
 ##############################################################################
 
+echo "This script is deprecated. Use 'ocrmypdf' instead."
+
 python3 -m ocrmypdf.main "$@"

README.rst

@@ -19,7 +19,7 @@ Main features
 -  Processes pages in parallel when more than one CPU core is
    available
 -  Uses `Tesseract OCR <https://github.com/tesseract-ocr/tesseract>`_ engine
--  Supports the `39 languages <https://code.google.com/p/tesseract-ocr/downloads/list>`_ recognized by Tesseract
+-  Supports more than `100 languages <https://github.com/tesseract-ocr/tessdata>`_ recognized by Tesseract
 -  Battle-tested on thousands of PDFs, a test suite and continuous integration
 
 For details: please consult the `release notes <RELEASE_NOTES.rst>`_.
@@ -162,7 +162,7 @@ Install the required Tesseract OCR engine with the language packs you plan to us
    
 .. code-block:: bash
 
-   brew install tesseract                       # Option 1: for English, French, German, Spanish
+   brew install tesseract                       # Option 1: for English
    
 .. code-block:: bash
    

RELEASE_NOTES.rst

@@ -1,6 +1,35 @@
 RELEASE NOTES
 =============
 
+OCRmyPDF uses `semantic versioning <http://semver.org/>`_.
+
+
+v4.2.5:
+=======
+
+-  Fixed an issue (#100) with PDFs that omit the optional /BitsPerComponent parameter on images
+-  Removed non-free file milk.pdf
+
+
+v4.2.4:
+=======
+
+-  Fixed an error (#90) caused by PDFs that use stencil masks properly
+-  Fixed handling of PDFs that try to draw images or stencil masks without properly setting up the graphics state (such images are now ignored for the purposes of calculating DPI)
+
+v4.2.3:
+=======
+
+-  Fixed an issue with PDFs that store page rotation (/Rotate) in an indirect object
+-  Integrated a few fixes to simplify downstream packaging (Debian)
+
+   +  The test suite no longer assumes it is installed
+   +  If running Linux, skip a test that passes Unicode on the command line
+
+-  Added a test case to check explicit masks and stencil masks
+-  Added a test case for indirect objects and linearized PDFs
+-  Deprecated the OCRmyPDF.sh shell script
+
 
 v4.2.2:
 =======

dev_requirements.txt

@@ -0,0 +1,4 @@
+check-manifest>=0.33
+setuptools-scm>=1.11.1
+twine>=1.8.1
+coverage>=4.2

ocrmypdf/pageinfo.py

@@ -72,11 +72,38 @@ def _shorthand_from_matrix(matrix):
     e, f = matrix[2][0], matrix[2][1]
     return tuple(map(float, (a, b, c, d, e, f)))
 
+RasterSettings = namedtuple('RasterSettings',
+    ['name', 'shorthand', 'stack_depth'])
+
+InlineSettings = namedtuple('InlineSettings',
+    ['settings', 'shorthand', 'stack_depth'])
 
 ContentsInfo = namedtuple('ContentsInfo', ['raster_settings', 'inline_images'])
 
 
 def _interpret_contents(contentstream):
+    """Interpret the PDF content stream
+
+    The stack represents the state of the PDF graphics stack.  We are only
+    interested in the current transformation matrix (CTM) so we only track
+    this object; a full implementation would need to track many other items.
+
+    The CTM is initialized to the mapping from user space to device space.
+    PDF units are 1/72".  In a PDF viewer or printer this matrix is initialized
+    to the transformation to device space.  For example if set to
+    (1/72, 0, 0, 1/72, 0, 0) then all units would be calculated in inches.
+
+    Images are always considered to be (0, 0) -> (1, 1).  Before drawing an
+    image there should be a 'cm' that sets up an image coordinate system
+    where drawing from (0, 0) -> (1, 1) will draw on the desired area of the
+    page.
+
+    PDF units suit our needs so we initialize ctm to the identity matrix.
+
+    PyPDF2 replaces inline images with a fake "INLINE IMAGE" operator.
+
+    """
+
     operations = contentstream.operations
     stack = []
     ctm = _matrix_from_shorthand((1, 0, 0, 1, 0, 0))
@@ -96,12 +123,16 @@ def _interpret_contents(contentstream):
                 _matrix_from_shorthand(operands), ctm)
         elif command == b'Do':
             image_name = operands[0]
-            image_raster_settings.append(
-                (image_name, _shorthand_from_matrix(ctm)))
+            raster = RasterSettings(
+                name=image_name, shorthand=_shorthand_from_matrix(ctm),
+                stack_depth=len(stack))
+            image_raster_settings.append(raster)
         elif command == b'INLINE IMAGE':
             settings = operands['settings']
-            inline_images.append(
-                (settings, _shorthand_from_matrix(ctm)))
+            inline = InlineSettings(
+                settings=settings, shorthand=_shorthand_from_matrix(ctm),
+                stack_depth=len(stack))
+            inline_images.append(inline)
 
     return ContentsInfo(
         raster_settings=image_raster_settings,
@@ -175,24 +206,27 @@ def _get_dpi(ctm_shorthand, image_size):
 def _find_page_inline_images(page, pageinfo, contentsinfo):
     "Find inline images on the page"
 
-    for n, im in enumerate(contentsinfo.inline_images):
-        settings, shorthand = im
+    for n, inline in enumerate(contentsinfo.inline_images):
         image = {}
         image['name'] = str('inline-%02d' % n)
-        image['width'] = settings['/W']
-        image['height'] = settings['/H']
-        image['bpc'] = settings['/BPC']
-        image['color'] = FRIENDLY_COLORSPACE.get(settings['/CS'], '-')
+        image['width'] = inline.settings['/W']
+        image['height'] = inline.settings['/H']
+        if '/BPC' in inline.settings:
+            image['bpc'] = inline.settings['/BPC']
+        else:
+            image['bpc'] = 8
+        image['color'] = FRIENDLY_COLORSPACE.get(inline.settings['/CS'], '-')
         image['comp'] = FRIENDLY_COMP.get(image['color'], '?')
-        if '/F' in settings:
-            filter_ = settings['/F']
+        if '/F' in inline.settings:
+            filter_ = inline.settings['/F']
             if isinstance(filter_, pypdf.generic.ArrayObject):
                 filter_ = filter_[0]
             image['enc'] = FRIENDLY_ENCODING.get(filter_, 'image')
         else:
             image['enc'] = 'image'
 
-        dpi_w, dpi_h = _get_dpi(shorthand, (image['width'], image['height']))
+        dpi_w, dpi_h = _get_dpi(
+            inline.shorthand, (image['width'], image['height']))
         image['dpi_w'], image['dpi_h'] = Decimal(dpi_w), Decimal(dpi_h)
         yield image
 
@@ -213,7 +247,10 @@ def _find_page_regular_images(page, pageinfo, contentsinfo):
         image['name'] = str(xobj)
         image['width'] = pdfimage['/Width']
         image['height'] = pdfimage['/Height']
-        image['bpc'] = pdfimage['/BitsPerComponent']
+        if '/BitsPerComponent' in pdfimage:
+            image['bpc'] = pdfimage['/BitsPerComponent']
+        else:
+            image['bpc'] = 8
 
         # Fixme: this is incorrectly treats explicit masks as stencil masks,
         # but good enough for now. Explicit masks have /ImageMask true but are
@@ -253,26 +290,19 @@ def _find_page_regular_images(page, pageinfo, contentsinfo):
 
         for raster in contentsinfo.raster_settings:
             # Loop in case the same image is display multiple times on a page
-            if raster[0] != image['name']:
+            if raster.name != image['name']:
                 continue
-            shorthand = raster[1]
 
-            if image['type'] == 'stencil':
-                # Stencil masks are implicitly scaled over the whole page
-                # Images that are used in explicit masks are not drawn directly
-                # but drawn by the image they mask over, so they will never
-                # be called for in raster settings
-                if shorthand != (1, 0, 0, 1, 0, 0):
-                    raise NotImplementedError(
-                        "Don't know how to handle "
-                        "stencil masks when graphics stack depth > 0.")
-                page_w = float(pageinfo['width_inches']) * 72.0
-                page_h = float(pageinfo['height_inches']) * 72.0
-                shorthand = (page_w, 0.0, 0.0,
-                             page_h, 0.0, 0.0)
+            if raster.stack_depth == 0:
+                # At least one PDF in the wild (and test suite) draws an image
+                # when the graphics stack depth is 0, meaning that the image
+                # gets drawn into a square of 1x1 PDF units (or 1/72",
+                # or 0.35 mm).  The equivalent DPI will be >100,000.  Exclude
+                # these from our DPI calculation for the page.
+                continue
 
             dpi_w, dpi_h = _get_dpi(
-                shorthand, (image['width'], image['height']))
+                raster.shorthand, (image['width'], image['height']))
 
             # When image is used multiple times take the highest DPI it is
             # rendered at

ocrmypdf/tesseract.py

@@ -10,11 +10,7 @@ from . import ExitCode, get_program, page_number
 from collections import namedtuple
 
 from subprocess import Popen, PIPE, CalledProcessError, \
-    TimeoutExpired, check_output, STDOUT
-try:
-    from subprocess import DEVNULL
-except ImportError:
-    DEVNULL = open(os.devnull, 'wb')
+    TimeoutExpired, check_output, STDOUT, DEVNULL
 
 
 OrientationConfidence = namedtuple(

pytest.ini

@@ -1,2 +1,2 @@
 [pytest]
-norecursedirs = lib 
+norecursedirs = lib .pc .git

setup.cfg

@@ -1,4 +1,9 @@
 [bdist_wheel]
 python-tag = py34
+
 [aliases]
-test=pytest
+test=pytest
+
+[check-manifest]
+ignore =
+	.github

setup.py

@@ -227,7 +227,7 @@ setup(
     tests_require=tests_require,
     entry_points={
         'console_scripts': [
-            'ocrmypdf = ocrmypdf.main:run_pipeline'
+            'ocrmypdf = ocrmypdf.__main__:run_pipeline'
         ],
     },
     package_data={'ocrmypdf': ['data/sRGB.icc']},

tests/resources/README.rst

@@ -9,21 +9,29 @@ Files derived from free sources
 These test resources come from free sources, under either public domain or Creative Commons licenses.
 In some cases they were converted from one image format to another without other changes.
 
-+---------------------+--------------------------------------------------------------------------------+
-| File                | Source                                                                         |
-+=====================+================================================================================+
-| c02-22.pdf          | `Project Gutenberg`_, Adventures of Huckleberry Finn, page 22                  |
-+---------------------+--------------------------------------------------------------------------------+
-| congress.jpg        | `US Congressional Records`_ (Public Domain)                                    |
-+---------------------+--------------------------------------------------------------------------------+
-| graph.pdf           | `Wikimedia: Pandas text analysis.png`_ (Public Domain)                         |
-+---------------------+--------------------------------------------------------------------------------+
-| lichtenstein.pdf    | `Wikimedia: JPEG2000 Lichtenstein`_ (Creative Commons BY-SA 3.0)               |
-+---------------------+--------------------------------------------------------------------------------+
-| LinnSequencer.jpg,  | `Wikimedia: LinnSequencer`_ (Creative Commons Attribution-ShareAlike 3.0)      |
-| linn.pdf, linn.txt  |                                                                                |
-+---------------------+--------------------------------------------------------------------------------+
- 
+.. list-table:: 
+    :widths: 20 50 30
+    :header-rows: 1
+
+    *   - File
+        - Source
+        - License
+    *   - c02-22.pdf
+        - `Project Gutenberg`_, Adventures of Huckleberry Finn, page 22
+        - Public Domain
+    *   - congress.jpg
+        - `US Congressional Records`_
+        - Public Domain
+    *   - graph.pdf
+        - `Wikimedia: Pandas text analysis.png`_
+        - Public Domain
+    *   - lichtenstein.pdf
+        - `Wikimedia: JPEG2000 Lichtenstein`_
+        - Creative Commons BY-SA 3.0
+    *   - LinnSequencer.jpg, linn.pdf, linn.txt
+        - `Wikimedia: LinnSequencer`_
+        - Creative Commons BY-SA 3.0
+
 
 Files generated for this project
 ================================
@@ -31,15 +39,43 @@ Files generated for this project
 The following test resources were crafted specifically for this project, and can be used
 under the terms of the license in LICENSE.rst.
 
-- aspect.pdf (a page with an image with 200 x 100 DPI resolution)
-- blank.pdf (a blank PDF page)
-- cmyk.pdf (a CMYK image created in Photoshop)
-- enormous.pdf (a very lage page)
-- francais.pdf (a page containing French accented characters)
-- hugemono.pdf (large monochrome JBIG2 page with pixel dimensions of 35000x35000)
-- invalid.pdf (a PDF file header followed by EOF marker)
-- missing_docinfo.pdf (PDF file with no /DocumentInfo section)
+.. list-table:: 
+    :widths: 20 20 60
+    :header-rows: 1
 
+    *   - File
+        - Contributor
+        - Purpose
+    *   - aspect.pdf
+        - @jbarlow83
+        - test image with 200 x 100 DPI resolution
+    *   - blank.pdf
+        - @jbarlow83
+        - blank PDF
+    *   - cmyk.pdf
+        - @jbarlow83
+        - a CMYK image created in Photoshop
+    *   - enormous.pdf
+        - @jbarlow83
+        - very large PDF page
+    *   - epson.pdf
+        - @lowesjam
+        - a linearized PDF containing some unusual indirect objects, created by an Epson printer; printout of a Wikipedia article (CC BY-SA)
+    *   - francais.pdf
+        - @jbarlow83
+        - a page containing French accents (diacritics)  
+    *   - hugemono.pdf
+        - @jbarlow83
+        - large monochrome 35000x35000 image in JBIG2 encoding 
+    *   - invalid.pdf
+        - @jbarlow83
+        - a PDF file header followed by EOF marker
+    *   - masks.pdf
+        - @supergrobi
+        - file containing explicit masks and a stencil mask drawn without a proper transformation matrix; printout of a German Wikipedia article (CC BY-SA)
+    *   - missing_docinfo.pdf
+        - @jbarlow83
+        - PDF file with no /DocumentInfo section 
 
 Assemblies
 ==========
@@ -66,3 +102,5 @@ These test resources are assemblies from other previously mentioned files, relea
 .. _`Wikimedia: Pandas text analysis.png`: https://en.wikipedia.org/wiki/File:Pandas_text_analysis.png
 
 .. _`Wikimedia: JPEG2000 Lichtenstein`: https://en.wikipedia.org/wiki/JPEG_2000#/media/File:Jpeg2000_2-level_wavelet_transform-lichtenstein.png
+
+.. _`Linux (Wikipedia Article)`: https://de.wikipedia.org/wiki/Linux 

tests/spoof/tesseract_big_image_error.py

@@ -8,13 +8,7 @@ VERSION_STRING = '''tesseract 3.04.00
 SPOOFED: return error claiming image too big
 '''
 
-"""Simulates a Tesseract crash
-
-It isn't strictly necessary to crash the process and that has unwanted
-side effects like triggering core dumps or error reporting, logging and such.
-It's enough to dump some text to stderr and return an error code.
-
-Follows the POSIX? convention of returning 128 + signal number.
+"""Simulates an error of Tesseract failing on attempts to process large images
 
 """
 

tests/spoof/tesseract_cache.py

@@ -6,6 +6,25 @@ import shutil
 import subprocess
 
 
+"""Cache output of tesseract to speed up test suite
+
+The cache is keyed by a hash that includes the tesseract version, some of
+the command line, and the binary dump of the input file. The output file,
+stdout, and stderr are replicated on a cache hit.
+
+Page orientation checks are also cached (-psm 0 stdout)
+
+Errors and crashes are not cached.
+
+Things not checked:
+-changes to tesseract installation that don't affect --version
+
+Will fail on Tesseract 3.02.02 in "hocr" mode because it doesn't produce
+the incorrect file extension.
+
+"""
+
+
 CACHE_PATH = os.path.abspath(os.path.join(
         os.path.dirname(__file__), '..', 'cache'))
 

tests/spoof/tesseract_crash.py

@@ -10,13 +10,13 @@ VERSION_STRING = '''tesseract 3.04.00
 SPOOFED: CRASH ON OCR or -psm 0
 '''
 
-"""Simulates a Tesseract crash
+"""Simulates a Tesseract crash when asked to run OCR
 
 It isn't strictly necessary to crash the process and that has unwanted
 side effects like triggering core dumps or error reporting, logging and such.
 It's enough to dump some text to stderr and return an error code.
 
-Follows the POSIX? convention of returning 128 + signal number.
+Follows the POSIX(?) convention of returning 128 + signal number.
 
 """
 

tests/spoof/tesseract_noop.py

@@ -4,6 +4,18 @@ import img2pdf
 from PIL import Image
 
 
+"""Tesseract no-op spoof
+
+To quickly run tests where getting OCR output is not necessary.
+
+In 'hocr' mode, create a .hocr file that specifies no text found.
+
+In 'pdf' mode, convert the image to PDF using another program.
+
+In orientation check mode, report the orientation is upright.
+"""
+
+
 VERSION_STRING = '''tesseract 3.04.00
  leptonica-1.72
   libjpeg 8d : libpng 1.6.19 : libtiff 4.0.6 : zlib 1.2.5

tests/test_main.py

@@ -13,6 +13,7 @@ import PyPDF2 as pypdf
 from ocrmypdf import ExitCode
 from ocrmypdf import leptonica
 from ocrmypdf.pdfa import file_claims_pdfa
+import platform
 
 
 if sys.version_info.major < 3:
@@ -22,11 +23,11 @@ if sys.version_info.major < 3:
 TESTS_ROOT = os.path.abspath(os.path.dirname(__file__))
 SPOOF_PATH = os.path.join(TESTS_ROOT, 'spoof')
 PROJECT_ROOT = os.path.dirname(TESTS_ROOT)
-OCRMYPDF = os.path.join(PROJECT_ROOT, 'OCRmyPDF.sh')
 TEST_RESOURCES = os.path.join(PROJECT_ROOT, 'tests', 'resources')
 TEST_OUTPUT = os.environ.get(
     'OCRMYPDF_TEST_OUTPUT',
     default=os.path.join(PROJECT_ROOT, 'tests', 'output', 'main'))
+OCRMYPDF = [sys.executable, '-m', 'ocrmypdf']
 
 
 def running_in_docker():
@@ -34,6 +35,10 @@ def running_in_docker():
     return os.path.exists('/.dockerinit')
 
 
+def is_linux():
+    return platform.system() == 'Linux'
+
+
 def setup_module():
     with suppress(FileNotFoundError):
         shutil.rmtree(TEST_OUTPUT)
@@ -41,15 +46,6 @@ def setup_module():
         os.makedirs(TEST_OUTPUT)
 
 
-def run_ocrmypdf_sh(input_file, output_file, *args, env=None):
-    sh_args = ['sh', OCRMYPDF] + list(args) + [input_file, output_file]
-    sh = Popen(
-        sh_args, close_fds=True, stdout=PIPE, stderr=PIPE,
-        universal_newlines=True, env=env)
-    out, err = sh.communicate()
-    return sh, out, err
-
-
 def _infile(input_basename):
     return os.path.join(TEST_RESOURCES, input_basename)
 
@@ -59,29 +55,31 @@ def _outfile(output_basename):
 
 
 def check_ocrmypdf(input_basename, output_basename, *args, env=None):
+    "Run ocrmypdf and confirmed that a valid file was created"
     input_file = _infile(input_basename)
     output_file = _outfile(output_basename)
 
-    sh, out, err = run_ocrmypdf_sh(input_file, output_file, *args, env=env)
-    if sh.returncode != 0:
+    p, out, err = run_ocrmypdf(input_basename, output_basename, *args, env=env)
+    if p.returncode != 0:
         print('stdout\n======')
         print(out)
         print('stderr\n======')
         print(err)
-    assert sh.returncode == 0
+    assert p.returncode == 0
     assert os.path.exists(output_file), "Output file not created"
     assert os.stat(output_file).st_size > 100, "PDF too small or empty"
     return output_file
 
 
-def run_ocrmypdf_env(input_basename, output_basename, *args, env=None):
+def run_ocrmypdf(input_basename, output_basename, *args, env=None):
+    "Run ocrmypdf and let caller deal with results"
     input_file = _infile(input_basename)
     output_file = _outfile(output_basename)
 
     if env is None:
         env = os.environ
 
-    p_args = ['ocrmypdf'] + list(args) + [input_file, output_file]
+    p_args = OCRMYPDF + list(args) + [input_file, output_file]
     p = Popen(
         p_args, close_fds=True, stdout=PIPE, stderr=PIPE,
         universal_newlines=True, env=env)
@@ -126,8 +124,8 @@ def spoof_tesseract_big_image_error():
     return spoof('tesseract', 'tesseract_big_image_error.py')
 
 
-def test_quick(spoof_tesseract_noop):
-    check_ocrmypdf('c02-22.pdf', 'test_quick.pdf', env=spoof_tesseract_noop)
+def test_quick(spoof_tesseract_cache):
+    check_ocrmypdf('ccitt.pdf', 'test_quick.pdf', env=spoof_tesseract_cache)
 
 
 def test_deskew(spoof_tesseract_noop):
@@ -199,6 +197,9 @@ def test_preserve_metadata(spoof_tesseract_noop, output_type):
     assert pdfa_info['output'] == output_type
 
 
+@pytest.mark.skipif(
+    is_linux() and not running_in_docker(),
+    reason="likely to fail if Linux locale is not configured correctly")
 @pytest.mark.parametrize("output_type", [
     'pdfa', 'pdf'
     ])
@@ -210,7 +211,7 @@ def test_override_metadata(spoof_tesseract_noop, output_type):
     chinese = '孔子'
     high_unicode = 'U+1030C is: 𐌌'
 
-    p, out, err = run_ocrmypdf_env(
+    p, out, err = run_ocrmypdf(
         input_file, output_file,
         '--title', german,
         '--author', chinese,
@@ -255,8 +256,8 @@ def test_oversample(spoof_tesseract_cache, renderer):
 
 
 def test_repeat_ocr():
-    sh, _, _ = run_ocrmypdf_sh('graph_ocred.pdf', 'wontwork.pdf')
-    assert sh.returncode != 0
+    p, _, _ = run_ocrmypdf('graph_ocred.pdf', 'wontwork.pdf')
+    assert p.returncode != 0
 
 
 def test_force_ocr(spoof_tesseract_cache):
@@ -413,19 +414,19 @@ def test_tesseract_missing_tessdata():
     env = os.environ.copy()
     env['TESSDATA_PREFIX'] = '/tmp'
 
-    p, _, err = run_ocrmypdf_env(
+    p, _, err = run_ocrmypdf(
         'graph_ocred.pdf', 'not_a_pdfa.pdf', '-v', '1', '--skip-text', env=env)
     assert p.returncode == ExitCode.missing_dependency, err
 
 
 def test_invalid_input_pdf():
-    p, out, err = run_ocrmypdf_env(
+    p, out, err = run_ocrmypdf(
         'invalid.pdf', 'wont_be_created.pdf')
     assert p.returncode == ExitCode.input_file, err
 
 
 def test_blank_input_pdf():
-    p, out, err = run_ocrmypdf_env(
+    p, out, err = run_ocrmypdf(
         'blank.pdf', 'still_blank.pdf')
     assert p.returncode == ExitCode.ok
 
@@ -434,7 +435,7 @@ def test_force_ocr_on_pdf_with_no_images(spoof_tesseract_crash):
     # As a correctness test, make sure that --force-ocr on a PDF with no
     # content still triggers tesseract. If tesseract crashes, then it was
     # called.
-    p, _, err = run_ocrmypdf_env(
+    p, _, err = run_ocrmypdf(
         'blank.pdf', 'wont_be_created.pdf', '--force-ocr',
         env=spoof_tesseract_crash)
     assert p.returncode == ExitCode.child_process_error, err
@@ -442,20 +443,20 @@ def test_force_ocr_on_pdf_with_no_images(spoof_tesseract_crash):
 
 
 def test_french(spoof_tesseract_cache):
-    p, out, err = run_ocrmypdf_env(
+    p, out, err = run_ocrmypdf(
         'francais.pdf', 'francais.pdf', '-l', 'fra', env=spoof_tesseract_cache)
     assert p.returncode == ExitCode.ok, \
         "This test may fail if Tesseract language packs are missing"
 
 
 def test_klingon():
-    p, out, err = run_ocrmypdf_env(
+    p, out, err = run_ocrmypdf(
         'francais.pdf', 'francais.pdf', '-l', 'klz')
     assert p.returncode == ExitCode.bad_args
 
 
 def test_missing_docinfo(spoof_tesseract_noop):
-    p, out, err = run_ocrmypdf_env(
+    p, out, err = run_ocrmypdf(
         'missing_docinfo.pdf', 'missing_docinfo.pdf', '-l', 'eng', '-c',
         env=spoof_tesseract_noop)
     assert p.returncode == ExitCode.ok, err
@@ -474,26 +475,26 @@ def test_uppercase_extension(spoof_tesseract_noop):
 
 def test_input_file_not_found():
     input_file = "does not exist.pdf"
-    sh, out, err = run_ocrmypdf_sh(
+    p, out, err = run_ocrmypdf(
         _infile(input_file),
         _outfile("will not happen.pdf"))
-    assert sh.returncode == ExitCode.input_file
+    assert p.returncode == ExitCode.input_file
     assert (input_file in out or input_file in err)
 
 
 def test_input_file_not_a_pdf():
     input_file = __file__  # Try to OCR this file
-    sh, out, err = run_ocrmypdf_sh(
+    p, out, err = run_ocrmypdf(
         _infile(input_file),
         _outfile("will not happen.pdf"))
-    assert sh.returncode == ExitCode.input_file
+    assert p.returncode == ExitCode.input_file
     assert (input_file in out or input_file in err)
 
 
 def test_qpdf_repair_fails():
     env = os.environ.copy()
     env['OCRMYPDF_QPDF'] = os.path.abspath('./spoof/qpdf_dummy_return2.py')
-    p, out, err = run_ocrmypdf_env(
+    p, out, err = run_ocrmypdf(
         '-v', '1',
         'c02-22.pdf', 'wont_be_created.pdf', env=env)
     print(out)
@@ -502,7 +503,7 @@ def test_qpdf_repair_fails():
 
 
 def test_encrypted():
-    p, out, err = run_ocrmypdf_env('skew-encrypted.pdf', 'wont_be_created.pdf')
+    p, out, err = run_ocrmypdf('skew-encrypted.pdf', 'wont_be_created.pdf')
     assert p.returncode == ExitCode.input_file
     assert out.find('password')
 
@@ -524,19 +525,19 @@ def test_pagesegmode(renderer, spoof_tesseract_cache):
     'tesseract',
     ])
 def test_tesseract_crash(renderer, spoof_tesseract_crash):
-    sh, out, err = run_ocrmypdf_env(
+    p, out, err = run_ocrmypdf(
         'ccitt.pdf', 'wontwork.pdf', '-v', '1',
         '--pdf-renderer', renderer, env=spoof_tesseract_crash)
-    assert sh.returncode == ExitCode.child_process_error
+    assert p.returncode == ExitCode.child_process_error
     assert not os.path.exists(_outfile('wontwork.pdf'))
     assert "ERROR" in err
 
 
 def test_tesseract_crash_autorotate(spoof_tesseract_crash):
-    sh, out, err = run_ocrmypdf_env(
+    p, out, err = run_ocrmypdf(
         'ccitt.pdf', 'wontwork.pdf',
         '-r', env=spoof_tesseract_crash)
-    assert sh.returncode == ExitCode.child_process_error
+    assert p.returncode == ExitCode.child_process_error
     assert not os.path.exists(_outfile('wontwork.pdf'))
     assert "ERROR" in err
     print(out)
@@ -556,22 +557,22 @@ def test_tesseract_image_too_big(renderer, spoof_tesseract_big_image_error):
 def test_no_unpaper():
     env = os.environ.copy()
     env['OCRMYPDF_UNPAPER'] = os.path.abspath('./spoof/no_unpaper_here.py')
-    sh, out, err = run_ocrmypdf_env(
+    p, out, err = run_ocrmypdf(
         'c02-22.pdf', 'wont_be_created.pdf', '--clean', env=env)
-    assert sh.returncode == ExitCode.missing_dependency
+    assert p.returncode == ExitCode.missing_dependency
 
 
 def test_old_unpaper():
     env = os.environ.copy()
     env['OCRMYPDF_UNPAPER'] = os.path.abspath('./spoof/unpaper_oldversion.py')
-    sh, out, err = run_ocrmypdf_env(
+    p, out, err = run_ocrmypdf(
         'c02-22.pdf', 'wont_be_created.pdf', '--clean', env=env)
-    assert sh.returncode == ExitCode.missing_dependency
+    assert p.returncode == ExitCode.missing_dependency
 
 
 def test_algo4():
-    sh, _, _ = run_ocrmypdf_env('encrypted_algo4.pdf', 'wontwork.pdf')
-    assert sh.returncode == ExitCode.encrypted_pdf
+    p, _, _ = run_ocrmypdf('encrypted_algo4.pdf', 'wontwork.pdf')
+    assert p.returncode == ExitCode.encrypted_pdf
 
 
 @pytest.mark.parametrize('renderer', [
@@ -614,10 +615,12 @@ def test_stdin(spoof_tesseract_noop):
     input_file = _infile('francais.pdf')
     output_file = _outfile('test_stdin.pdf')
 
+    # Runs: cat testfile.pdf | ocrmypdf - output.pdf
+
     p1_args = ['cat', input_file]
     p1 = Popen(p1_args, close_fds=True, stdin=DEVNULL, stdout=PIPE)
 
-    p2_args = ['ocrmypdf', '-', output_file]
+    p2_args = OCRMYPDF + ['-', output_file]
     p2 = Popen(
         p2_args, close_fds=True, stdout=PIPE, stderr=PIPE,
         stdin=p1.stdout, env=spoof_tesseract_noop)
@@ -626,3 +629,12 @@ def test_stdin(spoof_tesseract_noop):
 
     assert p2.returncode == ExitCode.ok
 
+
+def test_masks(spoof_tesseract_noop):
+    check_ocrmypdf('masks.pdf', 'test_masks.pdf', env=spoof_tesseract_noop)
+
+
+def test_linearized_pdf_and_indirect_object(spoof_tesseract_noop):
+    check_ocrmypdf(
+        'epson.pdf', 'test_epson.pdf',
+        env=spoof_tesseract_noop)