v4.2.4 release notes

More thorough testing showed that Acrobat do not presume that images
fill the page if the CTM is unspecified, as tests/resources/masks.pdf
seems to want.  Instead they treat it literally and draw the image
as 1x1 PDF units or 1/72" square in the bottom left corner of the page.

Seems like the best thing to do is ignore any such images for the purpose
of DPI calculation.  masks.pdf still works out okay because it has
other images.

For more robustness we could consider invalidating any DPI above some
limit, or warning the user about these microdot thumbnails.

.github/ISSUE_TEMPLATE

@@ -0,0 +1,3 @@
+Please include the command line and (if needed) a test file with your issue report.
+
+If possible, please use a test file that we can include in future test cases (no personal information, no copyrighted material).

.gitignore

@@ -26,5 +26,5 @@ log/
 .ipynb_checkpoints/
 tests/cache/
 tests/output/
-tests/resources/private
+tests/resources/private/
 tmp/

MANIFEST.in

@@ -1,4 +1,42 @@
-recursive-exclude tests/output *
+# requirements
 include requirements.txt
 include test_requirements.txt
-include share/*
+include dev_requirements.txt
+
+# git
+include .git_archival.txt
+
+# docker
+include Dockerfile
+include Dockerfile.polyglot
+include .dockerignore
+
+# tests
+include pytest.ini
+recursive-include tests *.jpg
+recursive-include tests *.pdf
+recursive-include tests *.py
+recursive-include tests *.rst
+recursive-include tests *.txt
+recursive-exclude tests/output *
+recursive-exclude tests/output_pageinfo *
+recursive-exclude tests/resources/private *
+recursive-exclude tests/cache *
+
+# documentation
+include *.rst
+include pipeline.svg
+recursive-exclude .github * 
+
+# support files
+recursive-include ocrmypdf/data *
+recursive-include share *
+include *.py
+
+# code
+recursive-include ocrmypdf *.py
+exclude ocrmypdf/lib/_leptonica.py
+include OCRmyPDF.sh
+include docker-wrapper.sh
+
+

OCRmyPDF.sh

@@ -3,4 +3,6 @@
 # Copyright (c) 2013-14: fritz-hh from Github (https://github.com/fritz-hh)
 ##############################################################################
 
+echo "This script is deprecated. Use 'ocrmypdf' instead."
+
 python3 -m ocrmypdf.main "$@"

RELEASE_NOTES.rst

@@ -1,6 +1,41 @@
 RELEASE NOTES
 =============
 
+OCRmyPDF uses `semantic versioning <http://semver.org/>`_.
+
+
+v4.2.4:
+=======
+
+-  Fixed an error (#90) caused by PDFs that use stencil masks properly
+-  Fixed handling of PDFs that try to draw images or stencil masks without properly setting up the graphics state (such images are now ignored for the purposes of calculating DPI)
+
+v4.2.3:
+=======
+
+-  Fixed an issue with PDFs that store page rotation (/Rotate) in an indirect object
+-  Integrated a few fixes to simplify downstream packaging (Debian)
+
+   +  The test suite no longer assumes it is installed
+   +  If running Linux, skip a test that passes Unicode on the command line
+
+-  Added a test case to check explicit masks and stencil masks
+-  Added a test case for indirect objects and linearized PDFs
+-  Deprecated the OCRmyPDF.sh shell script
+
+
+v4.2.2:
+=======
+
+-  Improvements to documentation
+
+
+v4.2.1:
+=======
+
+-  Fixed an issue where PDF pages that contained stencil masks would report an incorrect DPI and cause Ghostscript to abort
+-  Implemented stdin streaming
+
 
 v4.2:
 =====

dev_requirements.txt

@@ -0,0 +1,4 @@
+check-manifest>=0.33
+setuptools-scm>=1.11.1
+twine>=1.8.1
+coverage>=4.2

ocrmypdf/pageinfo.py

@@ -72,11 +72,38 @@ def _shorthand_from_matrix(matrix):
     e, f = matrix[2][0], matrix[2][1]
     return tuple(map(float, (a, b, c, d, e, f)))
 
+RasterSettings = namedtuple('RasterSettings',
+    ['name', 'shorthand', 'stack_depth'])
+
+InlineSettings = namedtuple('InlineSettings',
+    ['settings', 'shorthand', 'stack_depth'])
 
 ContentsInfo = namedtuple('ContentsInfo', ['raster_settings', 'inline_images'])
 
 
 def _interpret_contents(contentstream):
+    """Interpret the PDF content stream
+
+    The stack represents the state of the PDF graphics stack.  We are only
+    interested in the current transformation matrix (CTM) so we only track
+    this object; a full implementation would need to track many other items.
+
+    The CTM is initialized to the mapping from user space to device space.
+    PDF units are 1/72".  In a PDF viewer or printer this matrix is initialized
+    to the transformation to device space.  For example if set to
+    (1/72, 0, 0, 1/72, 0, 0) then all units would be calculated in inches.
+
+    Images are always considered to be (0, 0) -> (1, 1).  Before drawing an
+    image there should be a 'cm' that sets up an image coordinate system
+    where drawing from (0, 0) -> (1, 1) will draw on the desired area of the
+    page.
+
+    PDF units suit our needs so we initialize ctm to the identity matrix.
+
+    PyPDF2 replaces inline images with a fake "INLINE IMAGE" operator.
+
+    """
+
     operations = contentstream.operations
     stack = []
     ctm = _matrix_from_shorthand((1, 0, 0, 1, 0, 0))
@@ -96,12 +123,16 @@ def _interpret_contents(contentstream):
                 _matrix_from_shorthand(operands), ctm)
         elif command == b'Do':
             image_name = operands[0]
-            image_raster_settings.append(
-                (image_name, _shorthand_from_matrix(ctm)))
+            raster = RasterSettings(
+                name=image_name, shorthand=_shorthand_from_matrix(ctm),
+                stack_depth=len(stack))
+            image_raster_settings.append(raster)
         elif command == b'INLINE IMAGE':
             settings = operands['settings']
-            inline_images.append(
-                (settings, _shorthand_from_matrix(ctm)))
+            inline = InlineSettings(
+                settings=settings, shorthand=_shorthand_from_matrix(ctm),
+                stack_depth=len(stack))
+            inline_images.append(inline)
 
     return ContentsInfo(
         raster_settings=image_raster_settings,
@@ -175,24 +206,24 @@ def _get_dpi(ctm_shorthand, image_size):
 def _find_page_inline_images(page, pageinfo, contentsinfo):
     "Find inline images on the page"
 
-    for n, im in enumerate(contentsinfo.inline_images):
-        settings, shorthand = im
+    for n, inline in enumerate(contentsinfo.inline_images):
         image = {}
         image['name'] = str('inline-%02d' % n)
-        image['width'] = settings['/W']
-        image['height'] = settings['/H']
-        image['bpc'] = settings['/BPC']
-        image['color'] = FRIENDLY_COLORSPACE.get(settings['/CS'], '-')
+        image['width'] = inline.settings['/W']
+        image['height'] = inline.settings['/H']
+        image['bpc'] = inline.settings['/BPC']
+        image['color'] = FRIENDLY_COLORSPACE.get(inline.settings['/CS'], '-')
         image['comp'] = FRIENDLY_COMP.get(image['color'], '?')
-        if '/F' in settings:
-            filter_ = settings['/F']
+        if '/F' in inline.settings:
+            filter_ = inline.settings['/F']
             if isinstance(filter_, pypdf.generic.ArrayObject):
                 filter_ = filter_[0]
             image['enc'] = FRIENDLY_ENCODING.get(filter_, 'image')
         else:
             image['enc'] = 'image'
 
-        dpi_w, dpi_h = _get_dpi(shorthand, (image['width'], image['height']))
+        dpi_w, dpi_h = _get_dpi(
+            inline.shorthand, (image['width'], image['height']))
         image['dpi_w'], image['dpi_h'] = Decimal(dpi_w), Decimal(dpi_h)
         yield image
 
@@ -253,26 +284,19 @@ def _find_page_regular_images(page, pageinfo, contentsinfo):
 
         for raster in contentsinfo.raster_settings:
             # Loop in case the same image is display multiple times on a page
-            if raster[0] != image['name']:
+            if raster.name != image['name']:
                 continue
-            shorthand = raster[1]
 
-            if image['type'] == 'stencil':
-                # Stencil masks are implicitly scaled over the whole page
-                # Images that are used in explicit masks are not drawn directly
-                # but drawn by the image they mask over, so they will never
-                # be called for in raster settings
-                if shorthand != (1, 0, 0, 1, 0, 0):
-                    raise NotImplementedError(
-                        "Don't know how to handle "
-                        "stencil masks when graphics stack depth > 0.")
-                page_w = float(pageinfo['width_inches']) * 72.0
-                page_h = float(pageinfo['height_inches']) * 72.0
-                shorthand = (page_w, 0.0, 0.0,
-                             page_h, 0.0, 0.0)
+            if raster.stack_depth == 0:
+                # At least one PDF in the wild (and test suite) draws an image
+                # when the graphics stack depth is 0, meaning that the image
+                # gets drawn into a square of 1x1 PDF units (or 1/72",
+                # or 0.35 mm).  The equivalent DPI will be >100,000.  Exclude
+                # these from our DPI calculation for the page.
+                continue
 
             dpi_w, dpi_h = _get_dpi(
-                shorthand, (image['width'], image['height']))
+                raster.shorthand, (image['width'], image['height']))
 
             # When image is used multiple times take the highest DPI it is
             # rendered at

ocrmypdf/pdfa.py

@@ -135,7 +135,8 @@ def file_claims_pdfa(filename):
             aboutUri='',
             namespace='http://www.aiim.org/pdfa/ns/id/')
     except AttributeError:
-        return {'pass': False, 'output': 'pdf', 'message': 'No XMP metadata'}
+        return {'pass': False, 'output': 'pdf',
+                'conformance': 'No XMP metadata'}
 
     pdfa_dict = {attr.localName: attr.value for attr in pdfa_nodes}
     pdfa_dict['pass'] = False
@@ -144,15 +145,15 @@ def file_claims_pdfa(filename):
         part_conformance = pdfa_dict['part'] + pdfa_dict['conformance']
         valid_part_conforms = {'1A', '1B', '2A', '2B', '2U', '3A', '3B', '3U'}
 
-        message = 'File claims to be PDF/A-{}'.format(
+        conformance = 'PDF/A-{}'.format(
             part_conformance)
 
         if part_conformance in valid_part_conforms:
             pdfa_dict['pass'] = True
             pdfa_dict['output'] = 'pdfa'
-        pdfa_dict['message'] = message
+        pdfa_dict['conformance'] = conformance
     else:
-        pdfa_dict['message'] = 'File is a regular PDF'
+        pdfa_dict['conformance'] = 'PDF'
 
     return pdfa_dict
 

ocrmypdf/tesseract.py

@@ -10,11 +10,7 @@ from . import ExitCode, get_program, page_number
 from collections import namedtuple
 
 from subprocess import Popen, PIPE, CalledProcessError, \
-    TimeoutExpired, check_output, STDOUT
-try:
-    from subprocess import DEVNULL
-except ImportError:
-    DEVNULL = open(os.devnull, 'wb')
+    TimeoutExpired, check_output, STDOUT, DEVNULL
 
 
 OrientationConfidence = namedtuple(

pytest.ini

@@ -1,2 +1,2 @@
 [pytest]
-norecursedirs = lib 
+norecursedirs = lib .pc

setup.cfg

@@ -1,4 +1,9 @@
 [bdist_wheel]
 python-tag = py34
+
 [aliases]
-test=pytest
+test=pytest
+
+[check-manifest]
+ignore =
+	.github

setup.py

@@ -6,7 +6,7 @@ from __future__ import print_function, unicode_literals
 
 import sys
 if sys.version_info < (3, 4):
-    print("Python 3.4 or newer is required")
+    print("Python 3.4 or newer is required", file=sys.stderr)
     sys.exit(1)
 
 from setuptools import setup  # nopep8
@@ -227,7 +227,7 @@ setup(
     tests_require=tests_require,
     entry_points={
         'console_scripts': [
-            'ocrmypdf = ocrmypdf.main:run_pipeline'
+            'ocrmypdf = ocrmypdf.__main__:run_pipeline'
         ],
     },
     package_data={'ocrmypdf': ['data/sRGB.icc']},

tasks.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# © 2016 James R. Barlow: github.com/jbarlow83
+# Release sanity checking
+
+import argparse
+from subprocess import run, PIPE, DEVNULL, STDOUT, CalledProcessError
+from git import Repo, Remote, PushInfo
+import logging
+import re
+import sys
+import os
+
+
+logging.basicConfig(level=logging.INFO)
+
+REMOTE_ERROR_FLAGS = \
+    PushInfo.REJECTED | PushInfo.NO_MATCH | PushInfo.REMOTE_REJECTED | \
+    PushInfo.REMOTE_FAILURE | PushInfo.DELETED | PushInfo.ERROR
+
+
+def test_repo(repo):
+    assert not repo.is_dirty(), "Repository is dirty"
+    if repo.untracked_files:
+        logging.warning('Some files are untracked:')
+        logging.warning('\n' + '\n'.join(repo.untracked_files))
+    assert repo.active_branch.name == 'master', 'Not on branch master'
+
+
+def travis(args):
+    repo = Repo('.')
+    test_repo(repo)
+
+    git_describe = repo.git.describe()
+
+    try:
+        env = os.environ.copy()
+        env['SETUPTOOLS_SCM_PRETEND_VERSION'] = git_describe
+        proc = run(['check-manifest'], check=True, universal_newlines=True, stdout=PIPE, stderr=STDOUT, env=env)
+        logging.info(proc.stdout)
+    except CalledProcessError as e:
+        logging.error('MANIFEST.in error')
+        logging.error(e.stdout)
+        sys.exit(1)
+
+    run(['python3', 'setup.py', 'build'], check=True)
+
+    origin = Remote(repo, 'jbarlow')
+    result = origin.push(refspec='master:master')[0]
+
+    if result.flags & REMOTE_ERROR_FLAGS:
+        logging.error(result.summary)
+        sys.exit(1)
+    else:
+        logging.info(result.summary)
+
+    logging.info("Pushed to Travis CI")
+    logging.info("If this passes, git tag and release")
+
+
+def release(args):
+    repo = Repo('.')
+    test_repo(repo)
+
+    git_describe = repo.git.describe()
+
+    assert git_describe.startswith('v') and not '-' in git_describe and not '+ng' in git_describe, \
+        "Not tagged properly for release: " + git_describe
+
+    plain_version = git_describe[1:]  # without 'v' prefix
+
+    with open('RELEASE_NOTES.rst') as f:
+        notes = f.read()
+        assert plain_version in notes, "Version not mentioned in release notes"
+
+    proc = run(['python3', 'setup.py', 'sdist', 'bdist_wheel'], universal_newlines=True, check=True, stdout=PIPE, stderr=STDOUT)
+    logging.info(proc.stdout)
+
+
+    origin = Remote(repo, 'jbarlow')
+    result = origin.push(refspec='master:master', tags=True)[0]
+    if result.flags & REMOTE_ERROR_FLAGS:
+        logging.error(result.summary)
+        sys.exit(1)
+    else:
+        logging.info(result.summary)
+
+    run(['twine', 'upload', '-r', 'pypitest',
+        'dist/ocrmypdf-{}.tar.gz'.format(plain_version),
+        'dist/ocrmypdf-{}-py34-none-any.whl'.format(plain_version)], check=True, universal_newlines=True, stdout=PIPE)
+
+
+parser = argparse.ArgumentParser(description="ocrmypdf release tasks")
+subparsers = parser.add_subparsers()
+
+push_travis = subparsers.add_parser(
+    'push-travis', description="Push master to travis for testing")
+push_travis.set_defaults(func=travis)
+
+release_parser = subparsers.add_parser(
+    'release', description="Release to PyPI etc")
+release_parser.set_defaults(func=release)
+
+
+def main():
+    args = parser.parse_args()
+    args.func(args)
+
+
+if __name__ == '__main__':
+    main()

tests/resources/README.rst

@@ -20,10 +20,10 @@ In some cases they were converted from one image format to another without other
 +---------------------+--------------------------------------------------------------------------------+
 | lichtenstein.pdf    | `Wikimedia: JPEG2000 Lichtenstein`_ (Creative Commons BY-SA 3.0)               |
 +---------------------+--------------------------------------------------------------------------------+
-| LinnSequencer.jpg,  | `Wikimedia: LinnSequencer`_ (Creative Commons Attribution-ShareAlike 3.0)      |
+| LinnSequencer.jpg,  | `Wikimedia: LinnSequencer`_ (Creative Commons BY-SA 3.0)                       |
 | linn.pdf, linn.txt  |                                                                                |
 +---------------------+--------------------------------------------------------------------------------+
- 
+
 
 Files generated for this project
 ================================
@@ -31,15 +31,31 @@ Files generated for this project
 The following test resources were crafted specifically for this project, and can be used
 under the terms of the license in LICENSE.rst.
 
-- aspect.pdf (a page with an image with 200 x 100 DPI resolution)
-- blank.pdf (a blank PDF page)
-- cmyk.pdf (a CMYK image created in Photoshop)
-- enormous.pdf (a very lage page)
-- francais.pdf (a page containing French accented characters)
-- hugemono.pdf (large monochrome JBIG2 page with pixel dimensions of 35000x35000)
-- invalid.pdf (a PDF file header followed by EOF marker)
-- missing_docinfo.pdf (PDF file with no /DocumentInfo section)
-
++---------------------+-----------------------+---------------------------------------------------------+
+| File                | Contributor           | Purpose                                                 |
++---------------------+-----------------------+---------------------------------------------------------+
+| aspect.pdf          | @jbarlow83            | test image with 200 x 100 DPI resolution                |
++---------------------+-----------------------+---------------------------------------------------------+
+| blank.pdf           | @jbarlow83            | blank PDF                                               |
++---------------------+-----------------------+---------------------------------------------------------+
+| cmyk.pdf            | @jbarlow83            | a CMYK image created in Photoshop                       |
++---------------------+-----------------------+---------------------------------------------------------+
+| enormous.pdf        | @jbarlow83            | very large PDF page                                     |
++---------------------+-----------------------+---------------------------------------------------------+
+| francais.pdf        | @jbarlow83            | a page containing French accents (diacritics)           |
++---------------------+-----------------------+---------------------------------------------------------+
+| hugemono.pdf        | @jbarlow83            | large monochrome 35000x35000 image in JBIG2 encoding    |
++---------------------+-----------------------+---------------------------------------------------------+
+| invalid.pdf         | @jbarlow83            | a PDF file header followed by EOF marker                |
++---------------------+-----------------------+---------------------------------------------------------+
+| masks.pdf           | @supergrobi           | file containing explicit masks and a stencil mask       |
+|                     |                       | drawn without a proper transformation matrix; printout  |
+|                     |                       | of a German Wikipedia article (Creative Commons BY-SA)  |
++---------------------+-----------------------+---------------------------------------------------------+
+| milk.pdf            | @lowesjam             | linearized PDF containing some indirect objects         |
++---------------------+-----------------------+---------------------------------------------------------+
+| missing_docinfo.pdf | @jbarlow83            | PDF file with no /DocumentInfo section                  |
++---------------------+-----------------------+---------------------------------------------------------+
 
 Assemblies
 ==========
@@ -66,3 +82,5 @@ These test resources are assemblies from other previously mentioned files, relea
 .. _`Wikimedia: Pandas text analysis.png`: https://en.wikipedia.org/wiki/File:Pandas_text_analysis.png
 
 .. _`Wikimedia: JPEG2000 Lichtenstein`: https://en.wikipedia.org/wiki/JPEG_2000#/media/File:Jpeg2000_2-level_wavelet_transform-lichtenstein.png
+
+.. _`Linux (Wikipedia Article)`: https://de.wikipedia.org/wiki/Linux 

tests/spoof/tesseract_big_image_error.py

@@ -8,13 +8,7 @@ VERSION_STRING = '''tesseract 3.04.00
 SPOOFED: return error claiming image too big
 '''
 
-"""Simulates a Tesseract crash
-
-It isn't strictly necessary to crash the process and that has unwanted
-side effects like triggering core dumps or error reporting, logging and such.
-It's enough to dump some text to stderr and return an error code.
-
-Follows the POSIX? convention of returning 128 + signal number.
+"""Simulates an error of Tesseract failing on attempts to process large images
 
 """
 

tests/spoof/tesseract_cache.py

@@ -6,6 +6,25 @@ import shutil
 import subprocess
 
 
+"""Cache output of tesseract to speed up test suite
+
+The cache is keyed by a hash that includes the tesseract version, some of
+the command line, and the binary dump of the input file. The output file,
+stdout, and stderr are replicated on a cache hit.
+
+Page orientation checks are also cached (-psm 0 stdout)
+
+Errors and crashes are not cached.
+
+Things not checked:
+-changes to tesseract installation that don't affect --version
+
+Will fail on Tesseract 3.02.02 in "hocr" mode because it doesn't produce
+the incorrect file extension.
+
+"""
+
+
 CACHE_PATH = os.path.abspath(os.path.join(
         os.path.dirname(__file__), '..', 'cache'))
 

tests/spoof/tesseract_crash.py

@@ -10,13 +10,13 @@ VERSION_STRING = '''tesseract 3.04.00
 SPOOFED: CRASH ON OCR or -psm 0
 '''
 
-"""Simulates a Tesseract crash
+"""Simulates a Tesseract crash when asked to run OCR
 
 It isn't strictly necessary to crash the process and that has unwanted
 side effects like triggering core dumps or error reporting, logging and such.
 It's enough to dump some text to stderr and return an error code.
 
-Follows the POSIX? convention of returning 128 + signal number.
+Follows the POSIX(?) convention of returning 128 + signal number.
 
 """
 

tests/spoof/tesseract_noop.py

@@ -4,6 +4,18 @@ import img2pdf
 from PIL import Image
 
 
+"""Tesseract no-op spoof
+
+To quickly run tests where getting OCR output is not necessary.
+
+In 'hocr' mode, create a .hocr file that specifies no text found.
+
+In 'pdf' mode, convert the image to PDF using another program.
+
+In orientation check mode, report the orientation is upright.
+"""
+
+
 VERSION_STRING = '''tesseract 3.04.00
  leptonica-1.72
   libjpeg 8d : libpng 1.6.19 : libtiff 4.0.6 : zlib 1.2.5

tests/test_main.py

@@ -2,7 +2,7 @@
 # © 2015 James R. Barlow: github.com/jbarlow83
 
 from __future__ import print_function
-from subprocess import Popen, PIPE, check_output, check_call
+from subprocess import Popen, PIPE, check_output, check_call, DEVNULL
 import os
 import shutil
 from contextlib import suppress
@@ -13,6 +13,7 @@ import PyPDF2 as pypdf
 from ocrmypdf import ExitCode
 from ocrmypdf import leptonica
 from ocrmypdf.pdfa import file_claims_pdfa
+import platform
 
 
 if sys.version_info.major < 3:
@@ -22,11 +23,11 @@ if sys.version_info.major < 3:
 TESTS_ROOT = os.path.abspath(os.path.dirname(__file__))
 SPOOF_PATH = os.path.join(TESTS_ROOT, 'spoof')
 PROJECT_ROOT = os.path.dirname(TESTS_ROOT)
-OCRMYPDF = os.path.join(PROJECT_ROOT, 'OCRmyPDF.sh')
 TEST_RESOURCES = os.path.join(PROJECT_ROOT, 'tests', 'resources')
 TEST_OUTPUT = os.environ.get(
     'OCRMYPDF_TEST_OUTPUT',
     default=os.path.join(PROJECT_ROOT, 'tests', 'output', 'main'))
+OCRMYPDF = [sys.executable, '-m', 'ocrmypdf']
 
 
 def running_in_docker():
@@ -34,6 +35,10 @@ def running_in_docker():
     return os.path.exists('/.dockerinit')
 
 
+def is_linux():
+    return platform.system() == 'Linux'
+
+
 def setup_module():
     with suppress(FileNotFoundError):
         shutil.rmtree(TEST_OUTPUT)
@@ -41,15 +46,6 @@ def setup_module():
         os.makedirs(TEST_OUTPUT)
 
 
-def run_ocrmypdf_sh(input_file, output_file, *args, env=None):
-    sh_args = ['sh', OCRMYPDF] + list(args) + [input_file, output_file]
-    sh = Popen(
-        sh_args, close_fds=True, stdout=PIPE, stderr=PIPE,
-        universal_newlines=True, env=env)
-    out, err = sh.communicate()
-    return sh, out, err
-
-
 def _infile(input_basename):
     return os.path.join(TEST_RESOURCES, input_basename)
 
@@ -59,29 +55,31 @@ def _outfile(output_basename):
 
 
 def check_ocrmypdf(input_basename, output_basename, *args, env=None):
+    "Run ocrmypdf and confirmed that a valid file was created"
     input_file = _infile(input_basename)
     output_file = _outfile(output_basename)
 
-    sh, out, err = run_ocrmypdf_sh(input_file, output_file, *args, env=env)
-    if sh.returncode != 0:
+    p, out, err = run_ocrmypdf(input_basename, output_basename, *args, env=env)
+    if p.returncode != 0:
         print('stdout\n======')
         print(out)
         print('stderr\n======')
         print(err)
-    assert sh.returncode == 0
+    assert p.returncode == 0
     assert os.path.exists(output_file), "Output file not created"
     assert os.stat(output_file).st_size > 100, "PDF too small or empty"
     return output_file
 
 
-def run_ocrmypdf_env(input_basename, output_basename, *args, env=None):
+def run_ocrmypdf(input_basename, output_basename, *args, env=None):
+    "Run ocrmypdf and let caller deal with results"
     input_file = _infile(input_basename)
     output_file = _outfile(output_basename)
 
     if env is None:
         env = os.environ
 
-    p_args = ['ocrmypdf'] + list(args) + [input_file, output_file]
+    p_args = OCRMYPDF + list(args) + [input_file, output_file]
     p = Popen(
         p_args, close_fds=True, stdout=PIPE, stderr=PIPE,
         universal_newlines=True, env=env)
@@ -199,6 +197,9 @@ def test_preserve_metadata(spoof_tesseract_noop, output_type):
     assert pdfa_info['output'] == output_type
 
 
+@pytest.mark.skipif(
+    is_linux() and not running_in_docker(),
+    reason="likely to fail if Linux locale is not configured correctly")
 @pytest.mark.parametrize("output_type", [
     'pdfa', 'pdf'
     ])
@@ -210,7 +211,7 @@ def test_override_metadata(spoof_tesseract_noop, output_type):
     chinese = '孔子'
     high_unicode = 'U+1030C is: 𐌌'
 
-    p, out, err = run_ocrmypdf_env(
+    p, out, err = run_ocrmypdf(
         input_file, output_file,
         '--title', german,
         '--author', chinese,
@@ -255,8 +256,8 @@ def test_oversample(spoof_tesseract_cache, renderer):
 
 
 def test_repeat_ocr():
-    sh, _, _ = run_ocrmypdf_sh('graph_ocred.pdf', 'wontwork.pdf')
-    assert sh.returncode != 0
+    p, _, _ = run_ocrmypdf('graph_ocred.pdf', 'wontwork.pdf')
+    assert p.returncode != 0
 
 
 def test_force_ocr(spoof_tesseract_cache):
@@ -413,19 +414,19 @@ def test_tesseract_missing_tessdata():
     env = os.environ.copy()
     env['TESSDATA_PREFIX'] = '/tmp'
 
-    p, _, err = run_ocrmypdf_env(
+    p, _, err = run_ocrmypdf(
         'graph_ocred.pdf', 'not_a_pdfa.pdf', '-v', '1', '--skip-text', env=env)
     assert p.returncode == ExitCode.missing_dependency, err
 
 
 def test_invalid_input_pdf():
-    p, out, err = run_ocrmypdf_env(
+    p, out, err = run_ocrmypdf(
         'invalid.pdf', 'wont_be_created.pdf')
     assert p.returncode == ExitCode.input_file, err
 
 
 def test_blank_input_pdf():
-    p, out, err = run_ocrmypdf_env(
+    p, out, err = run_ocrmypdf(
         'blank.pdf', 'still_blank.pdf')
     assert p.returncode == ExitCode.ok
 
@@ -434,7 +435,7 @@ def test_force_ocr_on_pdf_with_no_images(spoof_tesseract_crash):
     # As a correctness test, make sure that --force-ocr on a PDF with no
     # content still triggers tesseract. If tesseract crashes, then it was
     # called.
-    p, _, err = run_ocrmypdf_env(
+    p, _, err = run_ocrmypdf(
         'blank.pdf', 'wont_be_created.pdf', '--force-ocr',
         env=spoof_tesseract_crash)
     assert p.returncode == ExitCode.child_process_error, err
@@ -442,20 +443,20 @@ def test_force_ocr_on_pdf_with_no_images(spoof_tesseract_crash):
 
 
 def test_french(spoof_tesseract_cache):
-    p, out, err = run_ocrmypdf_env(
+    p, out, err = run_ocrmypdf(
         'francais.pdf', 'francais.pdf', '-l', 'fra', env=spoof_tesseract_cache)
     assert p.returncode == ExitCode.ok, \
         "This test may fail if Tesseract language packs are missing"
 
 
 def test_klingon():
-    p, out, err = run_ocrmypdf_env(
+    p, out, err = run_ocrmypdf(
         'francais.pdf', 'francais.pdf', '-l', 'klz')
     assert p.returncode == ExitCode.bad_args
 
 
 def test_missing_docinfo(spoof_tesseract_noop):
-    p, out, err = run_ocrmypdf_env(
+    p, out, err = run_ocrmypdf(
         'missing_docinfo.pdf', 'missing_docinfo.pdf', '-l', 'eng', '-c',
         env=spoof_tesseract_noop)
     assert p.returncode == ExitCode.ok, err
@@ -474,26 +475,26 @@ def test_uppercase_extension(spoof_tesseract_noop):
 
 def test_input_file_not_found():
     input_file = "does not exist.pdf"
-    sh, out, err = run_ocrmypdf_sh(
+    p, out, err = run_ocrmypdf(
         _infile(input_file),
         _outfile("will not happen.pdf"))
-    assert sh.returncode == ExitCode.input_file
+    assert p.returncode == ExitCode.input_file
     assert (input_file in out or input_file in err)
 
 
 def test_input_file_not_a_pdf():
     input_file = __file__  # Try to OCR this file
-    sh, out, err = run_ocrmypdf_sh(
+    p, out, err = run_ocrmypdf(
         _infile(input_file),
         _outfile("will not happen.pdf"))
-    assert sh.returncode == ExitCode.input_file
+    assert p.returncode == ExitCode.input_file
     assert (input_file in out or input_file in err)
 
 
 def test_qpdf_repair_fails():
     env = os.environ.copy()
     env['OCRMYPDF_QPDF'] = os.path.abspath('./spoof/qpdf_dummy_return2.py')
-    p, out, err = run_ocrmypdf_env(
+    p, out, err = run_ocrmypdf(
         '-v', '1',
         'c02-22.pdf', 'wont_be_created.pdf', env=env)
     print(out)
@@ -502,7 +503,7 @@ def test_qpdf_repair_fails():
 
 
 def test_encrypted():
-    p, out, err = run_ocrmypdf_env('skew-encrypted.pdf', 'wont_be_created.pdf')
+    p, out, err = run_ocrmypdf('skew-encrypted.pdf', 'wont_be_created.pdf')
     assert p.returncode == ExitCode.input_file
     assert out.find('password')
 
@@ -524,19 +525,19 @@ def test_pagesegmode(renderer, spoof_tesseract_cache):
     'tesseract',
     ])
 def test_tesseract_crash(renderer, spoof_tesseract_crash):
-    sh, out, err = run_ocrmypdf_env(
+    p, out, err = run_ocrmypdf(
         'ccitt.pdf', 'wontwork.pdf', '-v', '1',
         '--pdf-renderer', renderer, env=spoof_tesseract_crash)
-    assert sh.returncode == ExitCode.child_process_error
+    assert p.returncode == ExitCode.child_process_error
     assert not os.path.exists(_outfile('wontwork.pdf'))
     assert "ERROR" in err
 
 
 def test_tesseract_crash_autorotate(spoof_tesseract_crash):
-    sh, out, err = run_ocrmypdf_env(
+    p, out, err = run_ocrmypdf(
         'ccitt.pdf', 'wontwork.pdf',
         '-r', env=spoof_tesseract_crash)
-    assert sh.returncode == ExitCode.child_process_error
+    assert p.returncode == ExitCode.child_process_error
     assert not os.path.exists(_outfile('wontwork.pdf'))
     assert "ERROR" in err
     print(out)
@@ -556,22 +557,22 @@ def test_tesseract_image_too_big(renderer, spoof_tesseract_big_image_error):
 def test_no_unpaper():
     env = os.environ.copy()
     env['OCRMYPDF_UNPAPER'] = os.path.abspath('./spoof/no_unpaper_here.py')
-    sh, out, err = run_ocrmypdf_env(
+    p, out, err = run_ocrmypdf(
         'c02-22.pdf', 'wont_be_created.pdf', '--clean', env=env)
-    assert sh.returncode == ExitCode.missing_dependency
+    assert p.returncode == ExitCode.missing_dependency
 
 
 def test_old_unpaper():
     env = os.environ.copy()
     env['OCRMYPDF_UNPAPER'] = os.path.abspath('./spoof/unpaper_oldversion.py')
-    sh, out, err = run_ocrmypdf_env(
+    p, out, err = run_ocrmypdf(
         'c02-22.pdf', 'wont_be_created.pdf', '--clean', env=env)
-    assert sh.returncode == ExitCode.missing_dependency
+    assert p.returncode == ExitCode.missing_dependency
 
 
 def test_algo4():
-    sh, _, _ = run_ocrmypdf_env('encrypted_algo4.pdf', 'wontwork.pdf')
-    assert sh.returncode == ExitCode.encrypted_pdf
+    p, _, _ = run_ocrmypdf('encrypted_algo4.pdf', 'wontwork.pdf')
+    assert p.returncode == ExitCode.encrypted_pdf
 
 
 @pytest.mark.parametrize('renderer', [
@@ -609,3 +610,29 @@ def test_jbig2_passthrough(spoof_tesseract_cache):
     out_pageinfo = pdf_get_all_pageinfo(out)
     assert out_pageinfo[0]['images'][0]['enc'] == 'jbig2'
 
+
+def test_stdin(spoof_tesseract_noop):
+    input_file = _infile('francais.pdf')
+    output_file = _outfile('test_stdin.pdf')
+
+    # Runs: cat testfile.pdf | ocrmypdf - output.pdf
+
+    p1_args = ['cat', input_file]
+    p1 = Popen(p1_args, close_fds=True, stdin=DEVNULL, stdout=PIPE)
+
+    p2_args = OCRMYPDF + ['-', output_file]
+    p2 = Popen(
+        p2_args, close_fds=True, stdout=PIPE, stderr=PIPE,
+        stdin=p1.stdout, env=spoof_tesseract_noop)
+    p1.stdout.close()
+    out, err = p2.communicate()
+
+    assert p2.returncode == ExitCode.ok
+
+
+def test_masks(spoof_tesseract_noop):
+    check_ocrmypdf('masks.pdf', 'test_masks.pdf', env=spoof_tesseract_noop)
+
+
+def test_linearized_pdf_and_indirect_object(spoof_tesseract_noop):
+    check_ocrmypdf('milk.pdf', 'test_milk.pdf', env=spoof_tesseract_noop)