v4.5.3 release notes update

If tess4 renderer needed to skip OCR on a page it would end up
duplicating the page contents onto the new page, rather than creating
a blank OCR layer and placing it on the output page. This created
duplicated content in output files.

.gitignore

@@ -4,6 +4,8 @@
 venv*/
 pyvenv.cfg
 tasks.py
+.bash_history
+.ruffus_history.sqlite
 
 # Package building
 *.egg-info/
@@ -27,8 +29,11 @@ htmlcov/
 # Testing
 log/
 /*.pdf
+/*.qdf
+*.ipynb
 .ipynb_checkpoints/
 tests/cache/
 tests/output/
 tests/resources/private/
 tmp/
+pdfbox-app*.jar

.travis.yml

@@ -1,4 +1,3 @@
-sudo: required
 dist: trusty
 language: python
 cache:
@@ -6,39 +5,45 @@ cache:
   directories:
   - packages
   - tests/cache
+  - $HOME/Library/Caches/Homebrew
 
-python:
+matrix:
-- '3.4'
+  include:
-- '3.5'
+    - os: linux
-- '3.6'
+      sudo: required
-
+      language: python
-os:
+      python: 3.4
-- linux
+    - os: linux
+      sudo: required
+      language: python
+      python: 3.5
+    - os: linux
+      sudo: required
+      language: python
+      python: 3.6
+    - os: osx
+      osx_image: xcode8
+      language: generic
 
 before_cache:
 - rm -f $HOME/.cache/pip/log/debug.log
 
 before_install:
-- sudo add-apt-repository ppa:vshn/ghostscript -y
+- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then bash .travis/linux_before_install.sh ; fi
-- sudo add-apt-repository ppa:heyarje/libav-11 -y
+- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then bash .travis/osx_before_install.sh ; fi
-- sudo apt-get update -qq
-- sudo apt-get install -y ghostscript tesseract-ocr tesseract-ocr-deu tesseract-ocr-eng
-  tesseract-ocr-fra qpdf poppler-utils libavformat56 libavcodec56 libavutil54 libffi-dev
-- pip install --upgrade pip
-- mkdir -p packages
-- "[ -f packages/unpaper_6.1-1.deb ] || wget -q https://dl.dropboxusercontent.com/u/28971240/unpaper_6.1-1.deb
-  -O packages/unpaper_6.1-1.deb"
-- sudo dpkg -i packages/unpaper_6.1-1.deb
 
 install:
-- pip install .
+- pip3 install .
-- pip install -r requirements.txt -r test_requirements.txt
+- pip3 install -r requirements.txt -r test_requirements.txt
 
 script:
 - mv ocrmypdf dont_import_this_ocrmypdf
 - pytest
 - mv dont_import_this_ocrmypdf ocrmypdf
 
+after_success:
+- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then bash .travis/osx_brew.sh ; fi
+
 # See https://www.appneta.com/blog/pypi-deployment-with-travis-ci/ for
 # steps to set up testpypi deploy for untagged builds if desired
 
@@ -51,5 +56,5 @@ deploy:
   on:
     branch: master
     tags: true
-    condition: $TRAVIS_PYTHON_VERSION = "3.6"
+    condition: $TRAVIS_PYTHON_VERSION == "3.6" && $TRAVIS_OS_NAME == "linux"
   skip_upload_docs: true

.travis/autobrew.py

@@ -0,0 +1,75 @@
+from string import Template
+from subprocess import run, PIPE
+import re
+
+recipe_template = Template("""
+class Ocrmypdf < Formula
+  include Language::Python::Virtualenv
+
+  desc "Adds an OCR text layer to scanned PDF files"
+  homepage "https://github.com/jbarlow83/OCRmyPDF"
+  ${ocrmypdf_url}
+  ${ocrmypdf_sha256}
+
+  depends_on :python3
+  depends_on "pkg-config" => :build
+  depends_on "zlib"
+  depends_on "libffi"
+  depends_on "tesseract"
+  depends_on "ghostscript"
+  depends_on "unpaper"
+  depends_on "qpdf"
+
+  # For Pillow source install
+  depends_on "openjpeg"
+  depends_on "freetype"
+  depends_on "libpng"
+
+${resources}
+
+  def install
+    ENV.append ["SETUPTOOLS_SCM_PRETEND_VERSION"], "v${ocrmypdf_version}"
+    ENV.each do |key, value|
+      puts "#{key}:#{value}"
+    end
+    virtualenv_install_with_resources
+  end
+
+  test do
+    # `test do` will create, run in and delete a temporary directory.
+    #
+    # The installed folder is not in the path, so use the entire path to any
+    # executables being tested: `system "#{bin}/program", "do", "something"`.
+    system "#{bin}/ocrmypdf", "--version"
+  end
+end
+""")
+
+def main():
+    p = run(['poet', '--single', 'ocrmypdf'],
+            encoding='utf-8', stdout=PIPE, check=True)
+
+    ocrmypdf_lines = p.stdout.splitlines()
+    ocrmypdf_url = ocrmypdf_lines[1].strip()
+    ocrmypdf_sha256 = ocrmypdf_lines[2].strip()
+
+    ocrmypdf_version = re.search(
+        r'ocrmypdf-(.+)\.tar.*', ocrmypdf_url).group(1)
+    print(f"Autobrewing {ocrmypdf_version}")
+
+    p = run(['poet', '--resources', 'ocrmypdf'],
+            encoding='utf-8', stdout=PIPE, check=True)
+
+    poet_resources = p.stdout
+
+    # Remove the duplicate "ocrmypdf" resource block
+    all_resources = poet_resources.split('resource')
+    kept_resources = [block for block in all_resources if 'ocrmypdf' not in block]
+    resources = 'resource'.join(kept_resources)
+
+    with open('ocrmypdf.rb', 'w') as out:
+        out.write(recipe_template.substitute(**locals()))
+
+
+if __name__ == '__main__':
+    main()

.travis/linux_before_install.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+set -euo pipefail
+set -x
+
+sudo add-apt-repository ppa:vshn/ghostscript -y
+sudo add-apt-repository ppa:heyarje/libav-11 -y
+sudo apt-get update -qq
+sudo apt-get install -y \
+	ghostscript \
+	tesseract-ocr \
+	tesseract-ocr-deu \
+	tesseract-ocr-eng \
+	tesseract-ocr-fra \
+	qpdf \
+	poppler-utils \
+	libavformat56 \
+	libavcodec56 \
+	libavutil54 \
+	libffi-dev
+
+pip install --upgrade pip
+mkdir -p packages
+[ -f packages/unpaper_6.1-1.deb ] || wget -q https://dl.dropboxusercontent.com/u/28971240/unpaper_6.1-1.deb -O packages/unpaper_6.1-1.deb
+sudo dpkg -i packages/unpaper_6.1-1.deb

.travis/osx_before_install.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -euo pipefail
+set -x
+
+brew update
+
+brew install openjpeg jbig2dec libtiff     # image libraries
+brew install qpdf
+brew install ghostscript
+brew install python3
+brew install libxml2 libffi leptonica
+brew install unpaper   # optional
+brew install tesseract
+
+pip3 install --upgrade pip
+pip3 install wheel

.travis/osx_brew.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -euo pipefail
+set -x
+
+pip3 install homebrew-pypi-poet
+python3 .travis/autobrew.py
+brew audit ocrmypdf.rb
+cat ocrmypdf.rb

README.rst

@@ -60,7 +60,17 @@ Installation
 Linux, UNIX, and macOS are supported. Windows is not directly supported but there is a Docker image available that runs on Windows.
 
 Users of Debian 9 or later or Ubuntu 16.10 or later may simply
-``apt-get install ocrmypdf``.
+
+.. code-block:: bash
+
+   apt-get install ocrmypdf
+
+and macOS users may simply
+
+.. code-block:: bash
+
+   brew tap jbarlow83/ocrmypdf
+   brew install ocrmypdf
 
 For everyone else, `see our documentation <https://ocrmypdf.readthedocs.io/en/latest/installation.html>`_ for installation steps.
 

RELEASE_NOTES.rst

@@ -4,6 +4,29 @@ RELEASE NOTES
 OCRmyPDF uses `semantic versioning <http://semver.org/>`_.
 
 
+v4.5.3
+======
+
+-  Added a workaround for Ghostscript 9.21 and probably earlier versions would fail with the error message "VMerror -25", due to a Ghostscript bug in XMP metadata handling
+-  High Unicode characters (U+10000 and up) are no longer accepted for setting metadata on the command line, as Ghostscript may not handle them correctly.
+-  Fixed an issue where the ``tess4`` renderer would duplicate content onto output pages if tesseract failed or timed out
+-  Fixed ``tess4`` renderer not recognized when lossless reconstruction is possible
+
+
+v4.5.2
+======
+
+-  Fix issue #147. ``--pdf-renderer tess4 --clean`` will produce an oversized page containing the original image in the bottom left corner, due to loss DPI information.
+-  Make "using Tesseract 4.0" warning less ominous
+-  Set up machinery for homebrew OCRmyPDF tap
+
+
+v4.5.1
+======
+
+-  Fix issue #137, proportions of images with a non-square pixel aspect ratio would be distorted in output for ``--force-ocr`` and some other combinations of flags
+
+
 v4.5
 ====
 

docs/installation.rst

@@ -10,6 +10,18 @@ Users of Debian 9 ("stretch") or later or Ubuntu 16.10 or later may simply
 
 	apt-get install ocrmypdf
 
+
+Installing on macOS
+-------------------
+
+.. code-block:: bash
+
+   brew tap jbarlow83/ocrmypdf
+   brew install ocrmypdf
+
+
+Users who previously installed OCRmyPDF on macOS may need to remove the ``pip`` based installation (``pip3 uninstall ocrmypdf``).
+
 .. _Docker:
 
 Installing the Docker image
@@ -103,8 +115,8 @@ Or in the wonderful `fish shell <https://fishshell.com/>`_:
    funcsave ocrmypdf
 
 
-Installing on macOS
+Manual installation on macOS
--------------------
+----------------------------
 
 These instructions probably work on all macOS supported by Homebrew.
 
@@ -175,10 +187,6 @@ No package is currently available for Ubuntu 16.04, but you can install the depe
       tesseract-ocr \
       qpdf \
       python3-pip \
-      python3-pil \
-      python3-pytest \
-      python3-reportlab \
-      python3-ruffus \
       python3-cffi
 
 If you wish install OCRmyPDF to the system Python, then install as follows (note this installs new packages
@@ -193,7 +201,7 @@ follow these steps.
 
 .. code-block:: bash
 
-   python3 -m venv --system-site-packages venv-ocrmypdf
+   python3 -m venv venv-ocrmypdf
    source venv-ocrmypdf/bin/activate
    pip3 install ocrmypdf
 

ocrmypdf/__main__.py

@@ -53,7 +53,6 @@ if tesseract.version() < MINIMUM_TESS_VERSION:
             MINIMUM_TESS_VERSION, tesseract.version()))
     sys.exit(ExitCode.missing_dependency)
 
-
 # -------------
 # Parser
 
@@ -275,12 +274,15 @@ def check_options_output(options, log):
     if options.pdf_renderer in ('tesseract', 'tess4'):
         if tesseract.version() < '3.05':
             log.warning(
-                "tesseract < 3.05 may corrupt PDF output. "
+                "The setting --pdf-renderer=tesseract is not recommend for "
-                "--pdf-renderer=tesseract is not recommend.")
+                " use with tesseract versions less than 3.05, because it "
+                " produces OCR text that is incompatible with Ghostscript and "
+                " some other software.")
         elif tesseract.version() == '4.00.00alpha':
             log.warning(
-                "tesseract 4.00.00alpha may corrupt PDF output. "
+                "The setting --pdf-renderer={tesseract,tess4} is not"
-                "--pdf-renderer={tesseract,tess4} is not recommend.")
+                " recommended for builds of tesseract 4.00.00alpha older than"
+                " February 2017. Make sure you are using a recent build.")
 
     if options.debug_rendering and options.pdf_renderer == 'tesseract':
         log.info(
@@ -288,7 +290,7 @@ def check_options_output(options, log):
             "--pdf-renderer=tesseract.")
 
     lossless_reconstruction = False
-    if options.pdf_renderer == 'hocr':
+    if options.pdf_renderer in ('hocr', 'tess4'):
         if not any((options.deskew, options.clean_final, options.force_ocr,
                    options.remove_background)):
             lossless_reconstruction = True
@@ -341,13 +343,31 @@ def check_options_advanced(options, log):
             "commit 3d9fb3b or later")
 
 
+def check_options_metadata(options, log):
+    import unicodedata
+    metadata = [options.title, options.author, options.keywords,
+                options.subject]
+    for s in (m for m in metadata if m):
+        for c in s:
+            if unicodedata.category(c) == 'Co' or ord(c) >= 0x10000:
+                raise ValueError(
+                    "One of the metadata strings contains "
+                    "an unsupported Unicode character: '{}' (U+{})".format(
+                        c, hex(ord(c))[2:].upper()
+                ))
+
+
 def check_options(options, log):
     try:
         check_options_languages(options, log)
+        check_options_metadata(options, log)
         check_options_output(options, log)
         check_options_preprocessing(options, log)
         check_options_ocr_behavior(options, log)
         check_options_advanced(options, log)
+    except ValueError as e:
+        log.error(e)
+        sys.exit(ExitCode.bad_args)
     except argparse.ArgumentError as e:
         log.error(e)
         sys.exit(ExitCode.bad_args)
@@ -478,10 +498,58 @@ def traverse_ruffus_exception(e_args, options, log):
             return traverse_ruffus_exception(exc, options, log)
 
 
+def check_closed_streams(options):
+    """Work around Python issue with multiprocessing forking on closed streams
+
+    https://bugs.python.org/issue28326
+
+    Attempting to a fork/exec a new Python process when any of std{in,out,err}
+    are closed or not flushable for some reason may raise an exception.
+    Fix this by opening devnull if the handle seems to be closed.  Do this
+    globally to avoid tracking places all places that fork.
+
+    Seems to be specific to multiprocessing.Process not all Python process
+    forkers.
+
+    The error actually occurs when the stream object is not flushable,
+    but replacing an open stream object that is not flushable with
+    /dev/null is a bad idea since it will create a silent failure.  Replacing
+    a closed handle with /dev/null seems safe.
+
+    """
+
+    if sys.stderr is None:
+        sys.stderr = open(os.devnull, 'w')
+
+    if sys.stdin is None:
+        if options.input_file == '-':
+            print("Trying to read from stdin but stdin seems closed",
+                  file=sys.stderr)
+            return False
+        sys.stdin = open(os.devnull, 'r')
+
+    if sys.stdout is None:
+        if options.output_file == '-':
+            # Can't replace stdout if the user is piping
+            # If this case can even happen, it must be some kind of weird
+            # stream.
+            print(textwrap.dedent("""\
+                Output was set to stdout '-' but the stream attached to
+                stdout does not support the flush() system call.  This
+                will fail."""), file=sys.stderr)
+            return False
+        sys.stdout = open(os.devnull, 'w')
+
+    return True
+
+
 def run_pipeline():
     options = parser.parse_args()
     options.verbose_abbreviated_path = 1
 
+    if not check_closed_streams(options):
+        return ExitCode.bad_args
+
     _log, _log_mutex = proxy_logger.make_shared_logger_and_proxy(
         logging_factory, __name__, [None, options.verbose])
     _log.debug('ocrmypdf ' + VERSION)
@@ -522,9 +590,9 @@ def run_pipeline():
                     file."""))
                 return ExitCode.bad_args
         elif not is_file_writable(options.output_file):
-                _log.error(textwrap.dedent("""\
+            _log.error(textwrap.dedent("""\
-                    Cutput file location is not writable."""))
+                Cutput file location is not writable."""))
-                return ExitCode.file_access_error
+            return ExitCode.file_access_error
 
         manager = JobContextManager()
         manager.register('JobContext', JobContext)

ocrmypdf/exec/ghostscript.py

@@ -2,12 +2,32 @@
 # © 2015 James R. Barlow: github.com/jbarlow83
 
 from tempfile import NamedTemporaryFile
-from subprocess import Popen, PIPE, STDOUT, check_call
+from subprocess import Popen, PIPE, STDOUT, check_call, CalledProcessError, \
+    check_output
 from shutil import copy
+from functools import lru_cache
 from . import get_program
 from ..pdfa import SRGB_ICC_PROFILE
 
 
+@lru_cache(maxsize=1)
+def version():
+    args_gs = [
+        get_program('gs'),
+        '--version'
+    ]
+    try:
+        version = check_output(
+                args_gs, close_fds=True, universal_newlines=True,
+                stderr=STDOUT)
+    except CalledProcessError as e:
+        print("Could not find Ghostscript executable on system PATH.",
+              file=sys.stderr)
+        raise MissingDependencyError from e
+
+    return version.strip()
+
+
 def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
                   pageno=1):
     with NamedTemporaryFile(delete=True) as tmp:
@@ -61,7 +81,7 @@ def generate_pdfa(pdf_pages, output_file, log, threads=1):
                   universal_newlines=True)
         stdout, _ = p.communicate()
 
-        if 'error' in stdout:
+        if 'error' in stdout or 'ERROR' in stdout:
             log.error(stdout)
         elif 'overprint mode not set' in stdout:
             # Unless someone is going to print PDF/A documents on a

ocrmypdf/exec/tesseract.py

@@ -11,6 +11,7 @@ from ..helpers import page_number
 from . import get_program
 from collections import namedtuple
 from textwrap import dedent
+import PyPDF2 as pypdf
 
 from subprocess import Popen, PIPE, CalledProcessError, \
     TimeoutExpired, check_output, STDOUT, DEVNULL
@@ -186,6 +187,8 @@ def tesseract_log_output(log, stdout, input_file):
             log.warning(prefix + "unsure about page orientation")
         elif 'error' in line.lower() or 'exception' in line.lower():
             log.error(prefix + line.strip())
+        elif 'warning' in line.lower():
+            log.warning(prefix + line.strip())
         elif 'read_params_file' in line.lower():
             log.error(prefix + line.strip())
         else:
@@ -270,6 +273,24 @@ def generate_hocr(input_file, output_hocr, language: list, engine_mode,
                 f_out.write(line)
 
 
+def use_skip_page(text_only, skip_pdf, output_pdf):
+    if not text_only:
+        os.symlink(skip_pdf, output_pdf)
+        return
+
+    # For text only we must create a blank page with dimensions identical
+    # to the skip page because this is equivalent to a page with no text
+
+    pdf_in = pypdf.PdfFileReader(skip_pdf)
+    page0 = pdf_in.pages[0]
+
+    with open(output_pdf, 'wb') as out:
+        pdf_out = pypdf.PdfFileWriter()
+        w, h = page0.mediaBox.getWidth(), page0.mediaBox.getHeight()
+        pdf_out.addBlankPage(w, h)
+        pdf_out.write(out)
+
+
 def generate_pdf(input_image, skip_pdf, output_pdf, language: list,
                  engine_mode, text_only: bool,
                  tessconfig: list, timeout: float, pagesegmode: int, log):
@@ -307,14 +328,14 @@ def generate_pdf(input_image, skip_pdf, output_pdf, language: list,
             universal_newlines=True, timeout=timeout)
     except TimeoutExpired:
         page_timedout(log, input_image)
-        shutil.copy(skip_pdf, output_pdf)
+        use_skip_page(text_only, skip_pdf, output_pdf)
     except CalledProcessError as e:
         tesseract_log_output(log, e.output, input_image)
         if 'read_params_file: parameter not found' in e.output:
             raise TesseractConfigError() from e
 
         if 'Image too large' in e.output:
-            shutil.copy(skip_pdf, output_pdf)
+            use_skip_page(text_only, skip_pdf, output_pdf)
             return
         raise e from e
     else:

ocrmypdf/exec/unpaper.py

@@ -78,7 +78,8 @@ def run(input_file, output_file, dpi, log, mode_args):
             raise e from e
         else:
             log.debug(stdout)
-            Image.open(output_pnm.name).save(output_file)
+            # unpaper sets dpi to 72
+            Image.open(output_pnm.name).save(output_file, dpi=(dpi, dpi))
 
 
 def deskew(input_file, output_file, dpi, log):

ocrmypdf/pdfa.py

@@ -26,11 +26,7 @@ pdfa_def_template = u"""%!
 /ICCProfile ($icc_profile)
 def
 
-[ /Title <$title>
+[$docinfo
-  /Author <$author>
-  /Subject <$subject>
-  /Keywords <$keywords>
-  /Creator <$creator>
   /DOCINFO pdfmark
 
 % Define an ICC profile :
@@ -89,20 +85,30 @@ def encode_text_string(s: str) -> str:
 
 
 def _get_pdfa_def(icc_profile, icc_identifier, pdfmark):
-    pdfmark_utf16 = {k: encode_text_string(v) for k, v in pdfmark.items()}
+    # Ghostscript <= 9.21 has a bug where null entries in DOCINFO might produce
+    # ERROR: VMerror (-25) on closing pdfwrite device.
+    # https://bugs.ghostscript.com/show_bug.cgi?id=697684
+    # Work around this by only adding keys that have a nontrivial value
+    docinfo_keys = ('/Title', '/Author', '/Subject', '/Creator', '/Keywords')
+    docinfo_line_template = '  {key} <{value}>'
+
+    def docinfo_gen():
+        for key in docinfo_keys:
+            if key in pdfmark and pdfmark[key].strip() != '':
+                line = docinfo_line_template.format(
+                    key=key, value=encode_text_string(pdfmark[key]))
+                yield line
+
+    docinfo = '\n'.join(docinfo_gen())
 
     t = Template(pdfa_def_template)
     result = t.substitute(icc_profile=icc_profile,
                           icc_identifier=icc_identifier,
-                          title=pdfmark_utf16.get('/Title', ''),
+                          docinfo=docinfo)
-                          author=pdfmark_utf16.get('/Author', ''),
-                          subject=pdfmark_utf16.get('/Subject', ''),
-                          creator=pdfmark_utf16.get('/Creator', ''),
-                          keywords=pdfmark_utf16.get('/Keywords', ''))
     return result
 
 
-def generate_pdfa_def(target_filename, pdfmark, icc='sRGB'):
+def generate_pdfa_ps(target_filename, pdfmark, icc='sRGB'):
     if icc == 'sRGB':
         icc_profile = SRGB_ICC_PROFILE
     else:
@@ -116,6 +122,10 @@ def generate_pdfa_def(target_filename, pdfmark, icc='sRGB'):
         f.write(ps)
 
 
+# The old name is generate_pdfa_def -- now deprecated
+generate_pdfa_def = generate_pdfa_ps
+
+
 def file_claims_pdfa(filename):
     """Determines if the file claims to be PDF/A compliant
 

ocrmypdf/pipeline.py

@@ -23,7 +23,7 @@ from ruffus import formatter, regex, Pipeline, suffix
 
 from .hocrtransform import HocrTransform
 from .pageinfo import pdf_get_all_pageinfo
-from .pdfa import generate_pdfa_def, file_claims_pdfa
+from .pdfa import generate_pdfa_ps, file_claims_pdfa
 from .helpers import re_symlink, is_iterable_notstr, page_number
 from .exec import ghostscript, tesseract, qpdf
 from .exceptions import *
@@ -555,16 +555,21 @@ def select_image_layer(
         re_symlink(page_pdf, output_file, log)
     else:
         pageinfo = get_pageinfo(image, context)
-        dpi = get_page_dpi(pageinfo, options)
+
-        dpi = float(dpi[0]), float(dpi[1])
+        # We rasterize a square DPI version of each page because most image
-        layout_fun = img2pdf.get_fixed_dpi_layout_fun(dpi)
+        # processing tools don't support rectangular DPI. Use the square DPI
+        # as it accurately describes the image. It would be possible to
+        # resample the image at this stage back to non-square DPI to more
+        # closely resemble the input, except that the hocr renderer does not
+        # understand non-square DPI. The tess4 renderer would be fine.
+        dpi = get_page_square_dpi(pageinfo, options)
+        layout_fun = img2pdf.get_fixed_dpi_layout_fun((dpi, dpi))
 
         with open(image, 'rb') as imfile, \
                 open(output_file, 'wb') as pdf:
-            rawdata = imfile.read()
             log.debug('{:4d}: convert'.format(page_number(page_pdf)))
             img2pdf.convert(
-                rawdata, with_pdfrw=False,
+                imfile, with_pdfrw=False,
                 layout_fun=layout_fun, outputstream=pdf)
             log.debug('{:4d}: convert done'.format(page_number(page_pdf)))
 
@@ -762,13 +767,14 @@ def generate_postscript_stub(
     options = context.get_options()
     pdf = pypdf.PdfFileReader(input_file)
     pdfmark = get_pdfmark(pdf, options)
-    generate_pdfa_def(output_file, pdfmark)
+    generate_pdfa_ps(output_file, pdfmark)
 
 
 def skip_page(
         input_file,
         output_file,
-        log):
+        log,
+        context):
     # The purpose of this step is its filter to forward only the skipped
     # files (.skip.oriented.pdf) while disregarding the processed ones
     # (.ocr.oriented.pdf).  Alternative would be for merge_pages to filter
@@ -1031,7 +1037,7 @@ def build_pipeline(options, work_folder, log, context):
         task_func=generate_postscript_stub,
         input=task_repair_pdf,
         filter=formatter(r'\.repaired\.pdf'),
-        output=os.path.join(work_folder, 'pdfa_def.ps'),
+        output=os.path.join(work_folder, 'pdfa.ps'),
         extras=[log, context])
     task_generate_postscript_stub.active_if(options.output_type == 'pdfa')
 
@@ -1043,7 +1049,7 @@ def build_pipeline(options, work_folder, log, context):
         filter=suffix('.skip.oriented.pdf'),
         output='.done.pdf',
         output_dir=work_folder,
-        extras=[log])
+        extras=[log, context])
 
     # Merge pages
     task_merge_pages_ghostscript = main_pipeline.merge(

tests/conftest.py

@@ -22,6 +22,11 @@ def is_linux():
     return platform.system() == 'Linux'
 
 
+@pytest.helpers.register
+def is_macos():
+    return platform.system() == 'Darwin'
+
+
 @pytest.helpers.register
 def running_in_docker():
     # Docker creates a file named /.dockerenv (newer versions) or
@@ -29,6 +34,11 @@ def running_in_docker():
     return os.path.exists('/.dockerenv') or os.path.exists('/.dockerinit')
 
 
+@pytest.helpers.register
+def running_in_travis():
+    return os.environ.get('TRAVIS') == 'true'
+
+
 TESTS_ROOT = os.path.abspath(os.path.dirname(__file__))
 SPOOF_PATH = os.path.join(TESTS_ROOT, 'spoof')
 PROJECT_ROOT = os.path.dirname(TESTS_ROOT)
@@ -91,7 +101,7 @@ def check_ocrmypdf(input_file, output_file, *args, env=None):
 
     p, out, err = run_ocrmypdf(input_file, output_file, *args, env=env)
     #print(err)  # ensure py.test collects the output, use -s to view
-    assert p.returncode == 0
+    assert p.returncode == 0, "<stderr>\n" + err + "\n</stderr>"
     assert os.path.exists(str(output_file)), "Output file not created"
     assert os.stat(str(output_file)).st_size > 100, "PDF too small or empty"
     assert out == "", \
@@ -115,3 +125,11 @@ def run_ocrmypdf(input_file, output_file, *args, env=None):
     #print(err)
 
     return p, out, err
+
+
+@pytest.helpers.register
+def first_page_dimensions(pdf):
+    from ocrmypdf import pageinfo
+    info = pageinfo.pdf_get_all_pageinfo(str(pdf))
+    page0 = info[0]
+    return (page0['width_inches'], page0['height_inches'])

tests/test_main.py

@@ -12,6 +12,7 @@ import PyPDF2 as pypdf
 from ocrmypdf.exceptions import ExitCode
 from ocrmypdf import leptonica
 from ocrmypdf.pdfa import file_claims_pdfa
+from ocrmypdf.exec import ghostscript
 
 
 check_ocrmypdf = pytest.helpers.check_ocrmypdf
@@ -166,48 +167,50 @@ def test_preserve_metadata(spoof_tesseract_noop, output_type,
     assert pdfa_info['output'] == output_type
 
 
-@pytest.mark.skipif(
-    pytest.helpers.is_linux() and not pytest.helpers.running_in_docker(),
-    reason="likely to fail if Linux locale is not configured correctly")
 @pytest.mark.parametrize("output_type", [
     'pdfa', 'pdf'
     ])
 def test_override_metadata(spoof_tesseract_noop, output_type, resources,
                            outpdf):
     input_file = resources / 'c02-22.pdf'
-
     german = 'Du siehst den Wald vor lauter Bäumen nicht.'
     chinese = '孔子'
-    high_unicode = 'U+1030C is: 𐌌'
 
     p, out, err = run_ocrmypdf(
         input_file, outpdf,
         '--title', german,
         '--author', chinese,
-        '--subject', high_unicode,
         '--output-type', output_type,
         env=spoof_tesseract_noop)
 
-    assert p.returncode == ExitCode.ok
+    assert p.returncode == ExitCode.ok, err
 
-    pdf = str(outpdf)
+    reader = pypdf.PdfFileReader(outpdf)
 
-    out_pdfinfo = check_output(['pdfinfo', pdf], universal_newlines=True)
+    assert reader.documentInfo['/Title'] == german
-    lines_pdfinfo = out_pdfinfo.splitlines()
+    assert reader.documentInfo['/Author'] == chinese
-    pdfinfo = {}
+    assert reader.documentInfo.get('/Keywords', '') == ''
-    for line in lines_pdfinfo:
-        k, v = line.strip().split(':', maxsplit=1)
-        pdfinfo[k.strip()] = v.strip()
 
-    assert pdfinfo['Title'] == german
+    pdfa_info = file_claims_pdfa(outpdf)
-    assert pdfinfo['Author'] == chinese
-    assert pdfinfo['Subject'] == high_unicode
-    assert pdfinfo.get('Keywords', '') == ''
-
-    pdfa_info = file_claims_pdfa(pdf)
     assert pdfa_info['output'] == output_type
 
 
+def test_high_unicode(spoof_tesseract_noop, resources, no_outpdf):
+
+    # Ghostscript doesn't support high Unicode, so neither do we, to be
+    # safe
+    input_file = resources / 'c02-22.pdf'
+    high_unicode = 'U+1030C is: 𐌌'
+
+    p, out, err = run_ocrmypdf(
+        input_file, no_outpdf,
+        '--subject', high_unicode,
+        '--output-type', 'pdfa',
+        env=spoof_tesseract_noop)
+
+    assert p.returncode == ExitCode.bad_args, err
+
+
 @pytest.mark.parametrize('renderer', [
     'hocr',
     'tesseract',
@@ -409,10 +412,14 @@ def test_force_ocr_on_pdf_with_no_images(spoof_tesseract_crash, resources,
     assert not os.path.exists(no_outpdf)
 
 
+@pytest.mark.skipif(
+    pytest.helpers.is_macos() and pytest.helpers.running_in_travis(),
+    reason="takes too long to install language packs in Travis macOS homebrew")
 def test_french(spoof_tesseract_cache, resources, outpdf):
     p, out, err = run_ocrmypdf(
         resources / 'francais.pdf', outpdf, '-l', 'fra',
         env=spoof_tesseract_cache)
+    print(os.environ)
     assert p.returncode == ExitCode.ok, \
         "This test may fail if Tesseract language packs are missing"
 
@@ -564,6 +571,46 @@ def test_non_square_resolution(renderer, spoof_tesseract_cache,
     assert in_pageinfo[0]['yres'] == out_pageinfo[0]['yres']
 
 
+@pytest.mark.skipif(sys.version_info < (3, 5), reason="needs math.isclose")
+@pytest.mark.parametrize('renderer', [
+    'hocr',
+    'tesseract'
+    ])
+def test_convert_to_square_resolution(renderer, spoof_tesseract_cache,
+                                      resources, outpdf):
+    from math import isclose
+
+    # Confirm input image is non-square resolution
+    in_pageinfo = pdf_get_all_pageinfo(str(resources / 'aspect.pdf'))
+    assert in_pageinfo[0]['xres'] != in_pageinfo[0]['yres']
+
+    # --force-ocr requires means forced conversion to square resolution
+    check_ocrmypdf(
+        resources / 'aspect.pdf', outpdf,
+        '--force-ocr',
+        '--pdf-renderer', renderer, env=spoof_tesseract_cache)
+
+    out_pageinfo = pdf_get_all_pageinfo(str(outpdf))
+
+    in_p0, out_p0 = in_pageinfo[0], out_pageinfo[0]
+
+    # Resolution show now be equal
+    assert out_p0['xres'] == out_p0['yres']
+
+    # Page size should match input page size
+    assert isclose(in_p0['width_inches'],
+                   out_p0['width_inches'])
+    assert isclose(in_p0['height_inches'],
+                   out_p0['height_inches'])
+
+    # Because we rasterized the page to produce a new image, it should occupy
+    # the entire page
+    out_im_w = out_p0['images'][0]['width'] / out_p0['images'][0]['dpi_w']
+    out_im_h = out_p0['images'][0]['height'] / out_p0['images'][0]['dpi_h']
+    assert isclose(out_p0['width_inches'], out_im_w)
+    assert isclose(out_p0['height_inches'], out_im_h)
+
+
 def test_image_to_pdf(spoof_tesseract_noop, resources, outpdf):
     check_ocrmypdf(
         resources / 'LinnSequencer.jpg', outpdf, '--image-dpi', '200',
@@ -614,6 +661,23 @@ def test_stdout(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf):
     assert qpdf.check(output_file, log=None)
 
 
+def test_closed_streams(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf):
+    input_file = str(resources / 'francais.pdf')
+    output_file = str(outpdf)
+
+    def evil_closer():
+        os.close(0)
+        os.close(1)
+
+    p_args = ocrmypdf_exec + [input_file, output_file]
+    p = Popen(
+        p_args, close_fds=True, stdout=None, stderr=PIPE, stdin=None,
+        env=spoof_tesseract_noop, preexec_fn=evil_closer)
+    out, err = p.communicate()
+    print(err.decode())
+    assert p.returncode == ExitCode.ok
+
+
 def test_masks(spoof_tesseract_noop, resources, outpdf):
     check_ocrmypdf(resources / 'masks.pdf', outpdf, env=spoof_tesseract_noop)
 
@@ -748,3 +812,28 @@ def test_form_xobject(spoof_tesseract_noop, resources, outpdf):
     check_ocrmypdf(resources / 'formxobject.pdf', outpdf,
                    '--force-ocr',
                    env=spoof_tesseract_noop)
+
+
+@pytest.mark.skipif(sys.version_info < (3, 5), reason="needs math.isclose")
+@pytest.mark.parametrize('renderer', [
+    'hocr',
+    'tesseract',
+])
+def test_pagesize_consistency(renderer, resources, outpdf):
+    from math import isclose
+
+    first_page_dimensions = pytest.helpers.first_page_dimensions
+
+    infile = resources / 'linn.pdf'
+
+    before_dims = first_page_dimensions(infile)
+
+    check_ocrmypdf(
+        infile,
+        outpdf, '--pdf-renderer', renderer,
+        '--clean', '--deskew', '--remove-background', '--clean-final')
+
+    after_dims = first_page_dimensions(outpdf)
+
+    assert isclose(before_dims[0], after_dims[0])
+    assert isclose(before_dims[1], after_dims[1])

tests/test_tess4.py

@@ -4,19 +4,110 @@
 import pytest
 from ocrmypdf.exceptions import ExitCode
 from ocrmypdf.exec import tesseract
+from ocrmypdf import pageinfo
+import sys
+import os
+import PyPDF2 as pypdf
+
+
+spoof = pytest.helpers.spoof
+
+
+def tess4_possible_location():
+    """The location of tesseract 4 may be OCRMYPDF_TESS4, OCRMYPDF_TESSERACT,
+    or the installed version on PATH."""
+    return os.environ.get('OCRMYPDF_TESS4') or \
+           os.environ.get('OCRMYPDF_TESSERACT') or \
+            'tesseract'
+
+
+@pytest.fixture
+def ensure_tess4():
+    return spoof(tesseract=tess4_possible_location())
+
+
+def tess4_available():
+    """Check if a tesseract 4 binary is available, even if it's not the
+    official "tesseract" on PATH
+
+    """
+    old_environ = os.environ.copy()
+    try:
+        os.environ['OCRMYPDF_TESSERACT'] = tess4_possible_location()
+        return tesseract.v4() and tesseract.has_textonly_pdf()
+    finally:
+        os.environ = old_environ
 
 
 # Skip all tests in this file if not tesseract 4
-pytestmark = pytest.mark.skipif(not tesseract.v4(),
+pytestmark = pytest.mark.skipif(
-                                reason="tesseract 4.0 required")
+    not tess4_available(),
+    reason="tesseract 4.0 with textonly_pdf feature required")
+
+check_ocrmypdf = pytest.helpers.check_ocrmypdf
+run_ocrmypdf = pytest.helpers.run_ocrmypdf
+spoof = pytest.helpers.spoof
 
 
-@pytest.mark.skipif(not tesseract.has_textonly_pdf(),
+def test_textonly_pdf(ensure_tess4, resources, outdir):
-                    reason="requires textonly_pdf feature")
+    check_ocrmypdf(
-def test_textonly_pdf(resources, outdir):
-    pytest.helpers.check_ocrmypdf(
         resources / 'linn.pdf',
-        outdir / 'linn_textonly.pdf', '--pdf-renderer', 'tess4')
+        outdir / 'linn_textonly.pdf', '--pdf-renderer', 'tess4',
+        env=ensure_tess4)
 
 
+@pytest.mark.skipif(sys.version_info < (3, 5), reason="needs math.isclose")
+def test_pagesize_consistency_tess4(ensure_tess4, resources, outpdf):
+    from math import isclose
 
+    infile = resources / 'linn.pdf'
+
+    before_dims = pytest.helpers.first_page_dimensions(infile)
+
+    check_ocrmypdf(
+        infile,
+        outpdf, '--pdf-renderer', 'tess4',
+        '--clean', '--deskew', '--remove-background', '--clean-final',
+        env=ensure_tess4)
+
+    after_dims = pytest.helpers.first_page_dimensions(outpdf)
+
+    assert isclose(before_dims[0], after_dims[0])
+    assert isclose(before_dims[1], after_dims[1])
+
+
+@pytest.mark.parametrize('basename', ['graph_ocred.pdf', 'cardinal.pdf'])
+def test_skip_pages_does_not_replicate(
+        ensure_tess4, resources, basename, outdir):
+    infile = resources / basename
+    outpdf = outdir / basename
+
+    check_ocrmypdf(
+        infile,
+        outpdf, '--pdf-renderer', 'tess4', '--force-ocr',
+        '--tesseract-timeout', '0',
+        env=ensure_tess4
+    )
+
+    info_in = pageinfo.pdf_get_all_pageinfo(str(infile))
+
+    info = pageinfo.pdf_get_all_pageinfo(str(outpdf))
+    for page in info:
+        assert len(page['images']) == 1, "skipped page was replicated"
+
+    for n in range(len(info_in)):
+        assert info[n]['width_inches'] == info_in[n]['width_inches']
+
+
+def test_content_preservation(ensure_tess4, resources, outpdf):
+    infile = resources / 'masks.pdf'
+
+    check_ocrmypdf(
+        infile,
+        outpdf, '--pdf-renderer', 'tess4', '--tesseract-timeout', '0',
+        env=ensure_tess4
+    )
+
+    info = pageinfo.pdf_get_all_pageinfo(str(outpdf))
+    page = info[0]
+    assert len(page['images']) > 1, "masked were rasterized"