Add release helper script

Executing a package with python -m packagename will check for
__main__.py inside the package.  In other words main.py should have
always been named __main__.py.

In the unlikely event that someone depends on "import ocrmypdf.main"
being meaningful, main.py continues to exist and replicates the
behavior of __main__.  (It's unlikely because import ocrmypdf.main does
unpythonic ruffus-related things at things import time, essentially
configuring itself to work with sys.argv.  To fix another day.)

This should solve the problem of Debian needing to run test suites
before installation and afterwards for continuous integration without
having to patch either file, as python -m ocrmypdf will follow import
order.  That is, if the current directory contains "ocrmypdf/" (e.g.
staging a new version) then that will be tested, else sys.path will
be checked.

.github/ISSUE_TEMPLATE

@@ -0,0 +1,3 @@
+Please include the command line and (if needed) a test file with your issue report.
+
+If possible, please use a test file that we can include in future test cases (no personal information, no copyrighted material).

.gitignore

@@ -26,5 +26,5 @@ log/
 .ipynb_checkpoints/
 tests/cache/
 tests/output/
-tests/resources/private
+tests/resources/private/
 tmp/

MANIFEST.in

@@ -1,4 +1,42 @@
-recursive-exclude tests/output *
+# requirements
 include requirements.txt
 include test_requirements.txt
-include share/*
+include dev_requirements.txt
+
+# git
+include .git_archival.txt
+
+# docker
+include Dockerfile
+include Dockerfile.polyglot
+include .dockerignore
+
+# tests
+include pytest.ini
+recursive-include tests *.jpg
+recursive-include tests *.pdf
+recursive-include tests *.py
+recursive-include tests *.rst
+recursive-include tests *.txt
+recursive-exclude tests/output *
+recursive-exclude tests/output_pageinfo *
+recursive-exclude tests/resources/private *
+recursive-exclude tests/cache *
+
+# documentation
+include *.rst
+include pipeline.svg
+recursive-exclude .github * 
+
+# support files
+recursive-include ocrmypdf/data *
+recursive-include share *
+include *.py
+
+# code
+recursive-include ocrmypdf *.py
+exclude ocrmypdf/lib/_leptonica.py
+include OCRmyPDF.sh
+include docker-wrapper.sh
+
+

OCRmyPDF.sh

@@ -3,4 +3,6 @@
 # Copyright (c) 2013-14: fritz-hh from Github (https://github.com/fritz-hh)
 ##############################################################################
 
+echo "This script is deprecated. Use 'ocrmypdf' instead."
+
 python3 -m ocrmypdf.main "$@"

RELEASE_NOTES.rst

@@ -1,6 +1,34 @@
 RELEASE NOTES
 =============
 
+OCRmyPDF uses `semantic versioning <http://semver.org/>`_.
+
+v4.2.3:
+=======
+
+-  Fixed an issue with PDFs that store page rotation (/Rotate) in an indirect object
+-  Integrated a few fixes to simplify downstream packaging (Debian)
+
+   +  The test suite no longer assumes it is installed
+   +  If running Linux, skip a test that passes Unicode on the command line
+
+-  Added a test case to check explicit masks and stencil masks
+-  Added a test case for indirect objects and linearized PDFs
+-  Deprecated the OCRmyPDF.sh shell script
+
+
+v4.2.2:
+=======
+
+-  Improvements to documentation
+
+
+v4.2.1:
+=======
+
+-  Fixed an issue where PDF pages that contained stencil masks would report an incorrect DPI and cause Ghostscript to abort
+-  Implemented stdin streaming
+
 
 v4.2:
 =====

dev_requirements.txt

@@ -0,0 +1,4 @@
+check-manifest>=0.33
+setuptools-scm>=1.11.1
+twine>=1.8.1
+coverage>=4.2

ocrmypdf/pdfa.py

@@ -135,7 +135,8 @@ def file_claims_pdfa(filename):
             aboutUri='',
             namespace='http://www.aiim.org/pdfa/ns/id/')
     except AttributeError:
-        return {'pass': False, 'output': 'pdf', 'message': 'No XMP metadata'}
+        return {'pass': False, 'output': 'pdf',
+                'conformance': 'No XMP metadata'}
 
     pdfa_dict = {attr.localName: attr.value for attr in pdfa_nodes}
     pdfa_dict['pass'] = False
@@ -144,15 +145,15 @@ def file_claims_pdfa(filename):
         part_conformance = pdfa_dict['part'] + pdfa_dict['conformance']
         valid_part_conforms = {'1A', '1B', '2A', '2B', '2U', '3A', '3B', '3U'}
 
-        message = 'File claims to be PDF/A-{}'.format(
+        conformance = 'PDF/A-{}'.format(
             part_conformance)
 
         if part_conformance in valid_part_conforms:
             pdfa_dict['pass'] = True
             pdfa_dict['output'] = 'pdfa'
-        pdfa_dict['message'] = message
+        pdfa_dict['conformance'] = conformance
     else:
-        pdfa_dict['message'] = 'File is a regular PDF'
+        pdfa_dict['conformance'] = 'PDF'
 
     return pdfa_dict
 

ocrmypdf/tesseract.py

@@ -10,11 +10,7 @@ from . import ExitCode, get_program, page_number
 from collections import namedtuple
 
 from subprocess import Popen, PIPE, CalledProcessError, \
-    TimeoutExpired, check_output, STDOUT
-try:
-    from subprocess import DEVNULL
-except ImportError:
-    DEVNULL = open(os.devnull, 'wb')
+    TimeoutExpired, check_output, STDOUT, DEVNULL
 
 
 OrientationConfidence = namedtuple(

pytest.ini

@@ -1,2 +1,2 @@
 [pytest]
-norecursedirs = lib 
+norecursedirs = lib .pc

setup.cfg

@@ -1,4 +1,9 @@
 [bdist_wheel]
 python-tag = py34
+
 [aliases]
-test=pytest
+test=pytest
+
+[check-manifest]
+ignore =
+	.github

setup.py

@@ -6,7 +6,7 @@ from __future__ import print_function, unicode_literals
 
 import sys
 if sys.version_info < (3, 4):
-    print("Python 3.4 or newer is required")
+    print("Python 3.4 or newer is required", file=sys.stderr)
     sys.exit(1)
 
 from setuptools import setup  # nopep8
@@ -227,7 +227,7 @@ setup(
     tests_require=tests_require,
     entry_points={
         'console_scripts': [
-            'ocrmypdf = ocrmypdf.main:run_pipeline'
+            'ocrmypdf = ocrmypdf.__main__:run_pipeline'
         ],
     },
     package_data={'ocrmypdf': ['data/sRGB.icc']},

tasks.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# © 2016 James R. Barlow: github.com/jbarlow83
+# Release sanity checking
+
+import argparse
+from subprocess import run, PIPE, DEVNULL, STDOUT, CalledProcessError
+from git import Repo, Remote
+import logging
+import re
+import sys
+import os
+
+
+def test_repo(repo):
+    assert not repo.is_dirty(), "Repository is dirty"
+    if repo.untracked_files:
+        logging.warning('Some files are untracked:')
+        logging.warning('\n' + '\n'.join(repo.untracked_files))
+    assert repo.active_branch.name == 'master', 'Not on branch master'
+
+
+def travis(args):
+    repo = Repo('.')
+    test_repo(repo)
+
+    git_describe = repo.git.describe()
+
+    try:
+        env = os.environ.copy()
+        env['SETUPTOOLS_SCM_PRETEND_VERSION'] = git_describe
+        proc = run(['check-manifest'], check=True, universal_newlines=True, stdout=PIPE, stderr=STDOUT, env=env)
+        logging.info(proc.stdout)
+    except CalledProcessError as e:
+        logging.error('MANIFEST.in error')
+        logging.error(e.stdout)
+        sys.exit(1)
+
+    run(['python3', 'setup.py', 'build'], check=True)
+
+    origin = Remote(repo, 'jbarlow')
+    result = origin.push(refspec='master:master')[0]
+    if result.flags & (1024|4|8|32|16):
+        logging.error(result.summary)
+    else:
+        logging.info(result.summary)
+
+    logging.info("Pushed to Travis CI")
+    logging.info("If this passes, git tag and release")
+
+
+def release(args):
+    repo = Repo('.')
+    test_repo(repo)
+
+    git_describe = repo.git.describe()
+
+    assert not git_describe.startswith('v') and not '-' in git_describe and not '+ng' in git_describe, \
+        "Not tagged properly for release: " + git_describe
+
+    plain_version = git_describe[1:]  # without 'v' prefix
+
+    with open('RELEASE_NOTES.rst') as f:
+        notes = f.read()
+        assert plain_version in notes, "Version not mentioned in release notes"
+
+    proc = run(['python3', 'setup.py', 'sdist', 'bdist_wheel'], universal_newlines=True, check=True, stdout=PIPE, stderr=STDOUT)
+    logging.info(proc.stdout)
+
+
+    origin = Remote(repo, 'jbarlow')
+    result = origin.push(refspec='master:master', tags=True)[0]
+    if result.flags & (1024|4|8|32|16):
+        logging.error(result.summary)
+    else:
+        logging.info(result.summary)
+
+    run(['twine', 'upload', '-r', 'pypitest',
+        'dist/ocrmypdf-{}.tar.gz'.format(plain_version),
+        'dist/ocrmypdf-{}-py34-none-any.whl'.format(plain_version)], check=True, universal_newlines=True, stdout=PIPE)
+
+
+parser = argparse.ArgumentParser(description="ocrmypdf release tasks")
+subparsers = parser.add_subparsers()
+
+push_travis = subparsers.add_parser(
+    'push-travis', description="Push master to travis for testing")
+push_travis.set_defaults(func=travis)
+
+release_parser = subparsers.add_parser(
+    'release', description="Release to PyPI etc")
+release_parser.set_defaults(func=release)
+
+
+def main():
+    args = parser.parse_args()
+    args.func(args)
+
+
+if __name__ == '__main__':
+    main()

tests/resources/README.rst

@@ -20,10 +20,10 @@ In some cases they were converted from one image format to another without other
 +---------------------+--------------------------------------------------------------------------------+
 | lichtenstein.pdf    | `Wikimedia: JPEG2000 Lichtenstein`_ (Creative Commons BY-SA 3.0)               |
 +---------------------+--------------------------------------------------------------------------------+
-| LinnSequencer.jpg,  | `Wikimedia: LinnSequencer`_ (Creative Commons Attribution-ShareAlike 3.0)      |
+| LinnSequencer.jpg,  | `Wikimedia: LinnSequencer`_ (Creative Commons BY-SA 3.0)                       |
 | linn.pdf, linn.txt  |                                                                                |
 +---------------------+--------------------------------------------------------------------------------+
- 
+
 
 Files generated for this project
 ================================
@@ -31,15 +31,30 @@ Files generated for this project
 The following test resources were crafted specifically for this project, and can be used
 under the terms of the license in LICENSE.rst.
 
-- aspect.pdf (a page with an image with 200 x 100 DPI resolution)
-- blank.pdf (a blank PDF page)
-- cmyk.pdf (a CMYK image created in Photoshop)
-- enormous.pdf (a very lage page)
-- francais.pdf (a page containing French accented characters)
-- hugemono.pdf (large monochrome JBIG2 page with pixel dimensions of 35000x35000)
-- invalid.pdf (a PDF file header followed by EOF marker)
-- missing_docinfo.pdf (PDF file with no /DocumentInfo section)
-
++---------------------+-----------------------+---------------------------------------------------------+
+| File                | Contributor           | Purpose                                                 |
++---------------------+-----------------------+---------------------------------------------------------+
+| aspect.pdf          | @jbarlow83            | test image with 200 x 100 DPI resolution                |
++---------------------+-----------------------+---------------------------------------------------------+
+| blank.pdf           | @jbarlow83            | blank PDF                                               |
++---------------------+-----------------------+---------------------------------------------------------+
+| cmyk.pdf            | @jbarlow83            | a CMYK image created in Photoshop                       |
++---------------------+-----------------------+---------------------------------------------------------+
+| enormous.pdf        | @jbarlow83            | very large PDF page                                     |
++---------------------+-----------------------+---------------------------------------------------------+
+| francais.pdf        | @jbarlow83            | a page containing French accents (diacritics)           |
++---------------------+-----------------------+---------------------------------------------------------+
+| hugemono.pdf        | @jbarlow83            | large monochrome 35000x35000 image in JBIG2 encoding    |
++---------------------+-----------------------+---------------------------------------------------------+
+| invalid.pdf         | @jbarlow83            | a PDF file header followed by EOF marker                |
++---------------------+-----------------------+---------------------------------------------------------+
+| masks.pdf           | @supergrobi           | file containing stencil masks; printout of a German     |
+|                     |                       | Wikipedia article (Creative Commons BY-SA)              |
++---------------------+-----------------------+---------------------------------------------------------+
+| milk.pdf            | @lowesjam             | linearized PDF containing some indirect objects         |
++---------------------+-----------------------+---------------------------------------------------------+
+| missing_docinfo.pdf | @jbarlow83            | PDF file with no /DocumentInfo section                  |
++---------------------+-----------------------+---------------------------------------------------------+
 
 Assemblies
 ==========
@@ -66,3 +81,5 @@ These test resources are assemblies from other previously mentioned files, relea
 .. _`Wikimedia: Pandas text analysis.png`: https://en.wikipedia.org/wiki/File:Pandas_text_analysis.png
 
 .. _`Wikimedia: JPEG2000 Lichtenstein`: https://en.wikipedia.org/wiki/JPEG_2000#/media/File:Jpeg2000_2-level_wavelet_transform-lichtenstein.png
+
+.. _`Linux (Wikipedia Article)`: https://de.wikipedia.org/wiki/Linux 

tests/spoof/tesseract_big_image_error.py

@@ -8,13 +8,7 @@ VERSION_STRING = '''tesseract 3.04.00
 SPOOFED: return error claiming image too big
 '''
 
-"""Simulates a Tesseract crash
-
-It isn't strictly necessary to crash the process and that has unwanted
-side effects like triggering core dumps or error reporting, logging and such.
-It's enough to dump some text to stderr and return an error code.
-
-Follows the POSIX? convention of returning 128 + signal number.
+"""Simulates an error of Tesseract failing on attempts to process large images
 
 """
 

tests/spoof/tesseract_cache.py

@@ -6,6 +6,25 @@ import shutil
 import subprocess
 
 
+"""Cache output of tesseract to speed up test suite
+
+The cache is keyed by a hash that includes the tesseract version, some of
+the command line, and the binary dump of the input file. The output file,
+stdout, and stderr are replicated on a cache hit.
+
+Page orientation checks are also cached (-psm 0 stdout)
+
+Errors and crashes are not cached.
+
+Things not checked:
+-changes to tesseract installation that don't affect --version
+
+Will fail on Tesseract 3.02.02 in "hocr" mode because it doesn't produce
+the incorrect file extension.
+
+"""
+
+
 CACHE_PATH = os.path.abspath(os.path.join(
         os.path.dirname(__file__), '..', 'cache'))
 

tests/spoof/tesseract_crash.py

@@ -10,13 +10,13 @@ VERSION_STRING = '''tesseract 3.04.00
 SPOOFED: CRASH ON OCR or -psm 0
 '''
 
-"""Simulates a Tesseract crash
+"""Simulates a Tesseract crash when asked to run OCR
 
 It isn't strictly necessary to crash the process and that has unwanted
 side effects like triggering core dumps or error reporting, logging and such.
 It's enough to dump some text to stderr and return an error code.
 
-Follows the POSIX? convention of returning 128 + signal number.
+Follows the POSIX(?) convention of returning 128 + signal number.
 
 """
 

tests/spoof/tesseract_noop.py

@@ -4,6 +4,18 @@ import img2pdf
 from PIL import Image
 
 
+"""Tesseract no-op spoof
+
+To quickly run tests where getting OCR output is not necessary.
+
+In 'hocr' mode, create a .hocr file that specifies no text found.
+
+In 'pdf' mode, convert the image to PDF using another program.
+
+In orientation check mode, report the orientation is upright.
+"""
+
+
 VERSION_STRING = '''tesseract 3.04.00
  leptonica-1.72
   libjpeg 8d : libpng 1.6.19 : libtiff 4.0.6 : zlib 1.2.5

tests/test_main.py

@@ -2,7 +2,7 @@
 # © 2015 James R. Barlow: github.com/jbarlow83
 
 from __future__ import print_function
-from subprocess import Popen, PIPE, check_output, check_call
+from subprocess import Popen, PIPE, check_output, check_call, DEVNULL
 import os
 import shutil
 from contextlib import suppress
@@ -13,6 +13,7 @@ import PyPDF2 as pypdf
 from ocrmypdf import ExitCode
 from ocrmypdf import leptonica
 from ocrmypdf.pdfa import file_claims_pdfa
+import platform
 
 
 if sys.version_info.major < 3:
@@ -22,11 +23,11 @@ if sys.version_info.major < 3:
 TESTS_ROOT = os.path.abspath(os.path.dirname(__file__))
 SPOOF_PATH = os.path.join(TESTS_ROOT, 'spoof')
 PROJECT_ROOT = os.path.dirname(TESTS_ROOT)
-OCRMYPDF = os.path.join(PROJECT_ROOT, 'OCRmyPDF.sh')
 TEST_RESOURCES = os.path.join(PROJECT_ROOT, 'tests', 'resources')
 TEST_OUTPUT = os.environ.get(
     'OCRMYPDF_TEST_OUTPUT',
     default=os.path.join(PROJECT_ROOT, 'tests', 'output', 'main'))
+OCRMYPDF = [sys.executable, '-m', 'ocrmypdf']
 
 
 def running_in_docker():
@@ -34,6 +35,10 @@ def running_in_docker():
     return os.path.exists('/.dockerinit')
 
 
+def is_linux():
+    return platform.system() == 'Linux'
+
+
 def setup_module():
     with suppress(FileNotFoundError):
         shutil.rmtree(TEST_OUTPUT)
@@ -41,15 +46,6 @@ def setup_module():
         os.makedirs(TEST_OUTPUT)
 
 
-def run_ocrmypdf_sh(input_file, output_file, *args, env=None):
-    sh_args = ['sh', OCRMYPDF] + list(args) + [input_file, output_file]
-    sh = Popen(
-        sh_args, close_fds=True, stdout=PIPE, stderr=PIPE,
-        universal_newlines=True, env=env)
-    out, err = sh.communicate()
-    return sh, out, err
-
-
 def _infile(input_basename):
     return os.path.join(TEST_RESOURCES, input_basename)
 
@@ -59,29 +55,31 @@ def _outfile(output_basename):
 
 
 def check_ocrmypdf(input_basename, output_basename, *args, env=None):
+    "Run ocrmypdf and confirmed that a valid file was created"
     input_file = _infile(input_basename)
     output_file = _outfile(output_basename)
 
-    sh, out, err = run_ocrmypdf_sh(input_file, output_file, *args, env=env)
-    if sh.returncode != 0:
+    p, out, err = run_ocrmypdf(input_basename, output_basename, *args, env=env)
+    if p.returncode != 0:
         print('stdout\n======')
         print(out)
         print('stderr\n======')
         print(err)
-    assert sh.returncode == 0
+    assert p.returncode == 0
     assert os.path.exists(output_file), "Output file not created"
     assert os.stat(output_file).st_size > 100, "PDF too small or empty"
     return output_file
 
 
-def run_ocrmypdf_env(input_basename, output_basename, *args, env=None):
+def run_ocrmypdf(input_basename, output_basename, *args, env=None):
+    "Run ocrmypdf and let caller deal with results"
     input_file = _infile(input_basename)
     output_file = _outfile(output_basename)
 
     if env is None:
         env = os.environ
 
-    p_args = ['ocrmypdf'] + list(args) + [input_file, output_file]
+    p_args = OCRMYPDF + list(args) + [input_file, output_file]
     p = Popen(
         p_args, close_fds=True, stdout=PIPE, stderr=PIPE,
         universal_newlines=True, env=env)
@@ -199,6 +197,9 @@ def test_preserve_metadata(spoof_tesseract_noop, output_type):
     assert pdfa_info['output'] == output_type
 
 
+@pytest.mark.skipif(
+    is_linux() and not running_in_docker(),
+    reason="likely to fail if Linux locale is not configured correctly")
 @pytest.mark.parametrize("output_type", [
     'pdfa', 'pdf'
     ])
@@ -210,7 +211,7 @@ def test_override_metadata(spoof_tesseract_noop, output_type):
     chinese = '孔子'
     high_unicode = 'U+1030C is: 𐌌'
 
-    p, out, err = run_ocrmypdf_env(
+    p, out, err = run_ocrmypdf(
         input_file, output_file,
         '--title', german,
         '--author', chinese,
@@ -255,8 +256,8 @@ def test_oversample(spoof_tesseract_cache, renderer):
 
 
 def test_repeat_ocr():
-    sh, _, _ = run_ocrmypdf_sh('graph_ocred.pdf', 'wontwork.pdf')
-    assert sh.returncode != 0
+    p, _, _ = run_ocrmypdf('graph_ocred.pdf', 'wontwork.pdf')
+    assert p.returncode != 0
 
 
 def test_force_ocr(spoof_tesseract_cache):
@@ -413,19 +414,19 @@ def test_tesseract_missing_tessdata():
     env = os.environ.copy()
     env['TESSDATA_PREFIX'] = '/tmp'
 
-    p, _, err = run_ocrmypdf_env(
+    p, _, err = run_ocrmypdf(
         'graph_ocred.pdf', 'not_a_pdfa.pdf', '-v', '1', '--skip-text', env=env)
     assert p.returncode == ExitCode.missing_dependency, err
 
 
 def test_invalid_input_pdf():
-    p, out, err = run_ocrmypdf_env(
+    p, out, err = run_ocrmypdf(
         'invalid.pdf', 'wont_be_created.pdf')
     assert p.returncode == ExitCode.input_file, err
 
 
 def test_blank_input_pdf():
-    p, out, err = run_ocrmypdf_env(
+    p, out, err = run_ocrmypdf(
         'blank.pdf', 'still_blank.pdf')
     assert p.returncode == ExitCode.ok
 
@@ -434,7 +435,7 @@ def test_force_ocr_on_pdf_with_no_images(spoof_tesseract_crash):
     # As a correctness test, make sure that --force-ocr on a PDF with no
     # content still triggers tesseract. If tesseract crashes, then it was
     # called.
-    p, _, err = run_ocrmypdf_env(
+    p, _, err = run_ocrmypdf(
         'blank.pdf', 'wont_be_created.pdf', '--force-ocr',
         env=spoof_tesseract_crash)
     assert p.returncode == ExitCode.child_process_error, err
@@ -442,20 +443,20 @@ def test_force_ocr_on_pdf_with_no_images(spoof_tesseract_crash):
 
 
 def test_french(spoof_tesseract_cache):
-    p, out, err = run_ocrmypdf_env(
+    p, out, err = run_ocrmypdf(
         'francais.pdf', 'francais.pdf', '-l', 'fra', env=spoof_tesseract_cache)
     assert p.returncode == ExitCode.ok, \
         "This test may fail if Tesseract language packs are missing"
 
 
 def test_klingon():
-    p, out, err = run_ocrmypdf_env(
+    p, out, err = run_ocrmypdf(
         'francais.pdf', 'francais.pdf', '-l', 'klz')
     assert p.returncode == ExitCode.bad_args
 
 
 def test_missing_docinfo(spoof_tesseract_noop):
-    p, out, err = run_ocrmypdf_env(
+    p, out, err = run_ocrmypdf(
         'missing_docinfo.pdf', 'missing_docinfo.pdf', '-l', 'eng', '-c',
         env=spoof_tesseract_noop)
     assert p.returncode == ExitCode.ok, err
@@ -474,26 +475,26 @@ def test_uppercase_extension(spoof_tesseract_noop):
 
 def test_input_file_not_found():
     input_file = "does not exist.pdf"
-    sh, out, err = run_ocrmypdf_sh(
+    p, out, err = run_ocrmypdf(
         _infile(input_file),
         _outfile("will not happen.pdf"))
-    assert sh.returncode == ExitCode.input_file
+    assert p.returncode == ExitCode.input_file
     assert (input_file in out or input_file in err)
 
 
 def test_input_file_not_a_pdf():
     input_file = __file__  # Try to OCR this file
-    sh, out, err = run_ocrmypdf_sh(
+    p, out, err = run_ocrmypdf(
         _infile(input_file),
         _outfile("will not happen.pdf"))
-    assert sh.returncode == ExitCode.input_file
+    assert p.returncode == ExitCode.input_file
     assert (input_file in out or input_file in err)
 
 
 def test_qpdf_repair_fails():
     env = os.environ.copy()
     env['OCRMYPDF_QPDF'] = os.path.abspath('./spoof/qpdf_dummy_return2.py')
-    p, out, err = run_ocrmypdf_env(
+    p, out, err = run_ocrmypdf(
         '-v', '1',
         'c02-22.pdf', 'wont_be_created.pdf', env=env)
     print(out)
@@ -502,7 +503,7 @@ def test_qpdf_repair_fails():
 
 
 def test_encrypted():
-    p, out, err = run_ocrmypdf_env('skew-encrypted.pdf', 'wont_be_created.pdf')
+    p, out, err = run_ocrmypdf('skew-encrypted.pdf', 'wont_be_created.pdf')
     assert p.returncode == ExitCode.input_file
     assert out.find('password')
 
@@ -524,19 +525,19 @@ def test_pagesegmode(renderer, spoof_tesseract_cache):
     'tesseract',
     ])
 def test_tesseract_crash(renderer, spoof_tesseract_crash):
-    sh, out, err = run_ocrmypdf_env(
+    p, out, err = run_ocrmypdf(
         'ccitt.pdf', 'wontwork.pdf', '-v', '1',
         '--pdf-renderer', renderer, env=spoof_tesseract_crash)
-    assert sh.returncode == ExitCode.child_process_error
+    assert p.returncode == ExitCode.child_process_error
     assert not os.path.exists(_outfile('wontwork.pdf'))
     assert "ERROR" in err
 
 
 def test_tesseract_crash_autorotate(spoof_tesseract_crash):
-    sh, out, err = run_ocrmypdf_env(
+    p, out, err = run_ocrmypdf(
         'ccitt.pdf', 'wontwork.pdf',
         '-r', env=spoof_tesseract_crash)
-    assert sh.returncode == ExitCode.child_process_error
+    assert p.returncode == ExitCode.child_process_error
     assert not os.path.exists(_outfile('wontwork.pdf'))
     assert "ERROR" in err
     print(out)
@@ -556,22 +557,22 @@ def test_tesseract_image_too_big(renderer, spoof_tesseract_big_image_error):
 def test_no_unpaper():
     env = os.environ.copy()
     env['OCRMYPDF_UNPAPER'] = os.path.abspath('./spoof/no_unpaper_here.py')
-    sh, out, err = run_ocrmypdf_env(
+    p, out, err = run_ocrmypdf(
         'c02-22.pdf', 'wont_be_created.pdf', '--clean', env=env)
-    assert sh.returncode == ExitCode.missing_dependency
+    assert p.returncode == ExitCode.missing_dependency
 
 
 def test_old_unpaper():
     env = os.environ.copy()
     env['OCRMYPDF_UNPAPER'] = os.path.abspath('./spoof/unpaper_oldversion.py')
-    sh, out, err = run_ocrmypdf_env(
+    p, out, err = run_ocrmypdf(
         'c02-22.pdf', 'wont_be_created.pdf', '--clean', env=env)
-    assert sh.returncode == ExitCode.missing_dependency
+    assert p.returncode == ExitCode.missing_dependency
 
 
 def test_algo4():
-    sh, _, _ = run_ocrmypdf_env('encrypted_algo4.pdf', 'wontwork.pdf')
-    assert sh.returncode == ExitCode.encrypted_pdf
+    p, _, _ = run_ocrmypdf('encrypted_algo4.pdf', 'wontwork.pdf')
+    assert p.returncode == ExitCode.encrypted_pdf
 
 
 @pytest.mark.parametrize('renderer', [
@@ -609,3 +610,29 @@ def test_jbig2_passthrough(spoof_tesseract_cache):
     out_pageinfo = pdf_get_all_pageinfo(out)
     assert out_pageinfo[0]['images'][0]['enc'] == 'jbig2'
 
+
+def test_stdin(spoof_tesseract_noop):
+    input_file = _infile('francais.pdf')
+    output_file = _outfile('test_stdin.pdf')
+
+    # Runs: cat testfile.pdf | ocrmypdf - output.pdf
+
+    p1_args = ['cat', input_file]
+    p1 = Popen(p1_args, close_fds=True, stdin=DEVNULL, stdout=PIPE)
+
+    p2_args = OCRMYPDF + ['-', output_file]
+    p2 = Popen(
+        p2_args, close_fds=True, stdout=PIPE, stderr=PIPE,
+        stdin=p1.stdout, env=spoof_tesseract_noop)
+    p1.stdout.close()
+    out, err = p2.communicate()
+
+    assert p2.returncode == ExitCode.ok
+
+
+def test_masks(spoof_tesseract_noop):
+    check_ocrmypdf('masks.pdf', 'test_masks.pdf', env=spoof_tesseract_noop)
+
+
+def test_linearized_pdf_and_indirect_object(spoof_tesseract_noop):
+    check_ocrmypdf('milk.pdf', 'test_milk.pdf', env=spoof_tesseract_noop)