v5.4.3 release notes

As reported in https://github.com/jbarlow83/homebrew-ocrmypdf/issues/4#issuecomment-345029899

.travis.yml

@@ -85,6 +85,7 @@ deploy:
   script: /usr/bin/true
   on:
     branch: master
+    tags: true
     condition: $TRAVIS_OS_NAME == "osx"
 
 after_deploy: |

.travis/autobrew.py

@@ -23,6 +23,10 @@ class Ocrmypdf < Formula
   depends_on "unpaper"
   depends_on "qpdf"
 
+  # mactex installs its own ghostscript by default which causes problems
+  # mactex users should use caskroom/cask/mactex-no-ghostscript instead
+  conflicts_with cask: "caskroom/cask/mactex"
+
   # For Pillow source install
   depends_on "openjpeg"
   depends_on "freetype"
@@ -30,6 +34,7 @@ class Ocrmypdf < Formula
   depends_on "libjpeg"
   depends_on "webp"
   depends_on "little-cms2"
+  depends_on "zlib"
 
 ${resources}
   def install

.travis/linux_before_install.sh

@@ -25,14 +25,6 @@ sudo apt-get install -y --no-install-recommends \
 		tesseract-ocr-fra \
 		tesseract-ocr-deu
 
-# Workaround for https://github.com/tesseract-ocr/tesseract/issues/1167
-# Replace tesseract data installed by tesseract-ocr-eng with a known good version
-sudo apt-get install -y wget
-wget -q https://github.com/tesseract-ocr/tessdata/raw/master/eng.traineddata -O packages/eng.traineddata
-sha1sum packages/eng.traineddata
-sha1sum /usr/share/tesseract-ocr/4.00/tessdata/eng.traineddata
-sudo cp packages/eng.traineddata /usr/share/tesseract-ocr/4.00/tessdata/eng.traineddata
-
 pip install --upgrade pip
 mkdir -p packages
 [ -f packages/unpaper_6.1-1.deb ] || wget -q 'https://www.dropbox.com/s/vaq0kbwi6e6au80/unpaper_6.1-1.deb?raw=1' -O packages/unpaper_6.1-1.deb

LICENSE.rst

@@ -1,4 +1,7 @@
-Copyright (c) 2013-2016, The OCRmyPDF Authors
+Copyright (c) 2014-2017, James R. Barlow
+
+Copyright (c) 2013-2014, Julien Pfefferkorn
+Copyright (c) 2013-2017, The OCRmyPDF Authors
 
 Permission is hereby granted, free of charge, to any person obtaining a
 copy of this software and associated documentation files (the

docs/installation.rst

@@ -105,6 +105,12 @@ In this worked example, the current working directory contains an input file cal
 
 Note that ``ocrmypdf`` has its own separate ``-v VERBOSITYLEVEL`` argument to control debug verbosity. All Docker arguments should before the ``ocrmypdf`` image name and all arguments to ``ocrmypdf`` should be listed after.
 
+In some environments the permissions associated with Docker can be complex to configure. The process that executes Docker may end up not having the permissions to write the specified file system. In that case one can stream the file into and out of the Docker process and avoid all permission hassles, using ``-`` as the input and output filename:
+
+.. code-block:: bash
+
+    docker run --rm -i   ocrmypdf <other arguments to ocrmypdf> - - <input.pdf >output.pdf
+
 For convenience, a shell alias can hide the docker command:
 
 .. code-block:: bash

docs/release_notes.rst

@@ -5,9 +5,30 @@ OCRmyPDF uses `semantic versioning <http://semver.org/>`_ for its command line i
 
 The OCRmyPDF package itself does not contain a public API, although it is fairly stable and breaking changes are usually timed with a major release. A future release will clearly define the stable public API.
 
-v5.4
+v5.4.3
 ------
 
+-   If a subprocess fails to report its version when queried, exit cleanly with an error instead of throwing an exception
+-   Added test to confirm that the system locale is Unicode-aware and fail early if it's not
+-   Clarified some copyright information
+-   Updated pinned requirements.txt so the homebrew formula captures more recent versions
+
+
+v5.4.2
+------
+
+-   Fixed a regression from v5.4.1 that caused sidecar files to be created as empty files
+
+
+v5.4.1
+------
+
+-   Add workaround for Tesseract v4.00alpha crash when trying to obtain orientation and the latest language packs are installed
+
+
+v5.4
+----
+
 -   Change wording of a deprecation warning to improve clarity
 -   Added option to generate PDF/A-1b output if desired (``--output-type pdfa-1``); default remains PDF/A-2b generation
 -   Update documentation

ocrmypdf/__init__.py

@@ -4,6 +4,7 @@ import pkg_resources
 
 PROGRAM_NAME = 'ocrmypdf'
 
-VERSION = pkg_resources.get_distribution('ocrmypdf').version
-
+# Official PEP 396
+__version__ = pkg_resources.get_distribution('ocrmypdf').version
 
+VERSION = __version__

ocrmypdf/__main__.py

@@ -28,6 +28,7 @@ from . import PROGRAM_NAME, VERSION
 
 from .exceptions import *
 from . import exceptions as ocrmypdf_exceptions
+from . import _unicodefun
 
 warnings.simplefilter('ignore', pypdf.utils.PdfReadWarning)
 
@@ -49,6 +50,8 @@ def complain(message):
 if 'IDE_PROJECT_ROOTS' in os.environ:
     os.environ['PATH'] = '/usr/local/bin:' + os.environ['PATH']
 
+_unicodefun._verify_python3_env()
+
 if tesseract.version() < MINIMUM_TESS_VERSION:
     complain(
         "Please install tesseract {0} or newer "

ocrmypdf/_unicodefun.py

@@ -0,0 +1,108 @@
+# Copyright (c) 2014, Armin Ronacher
+#
+# Copyright (c) 2017, James R Barlow
+#
+# Some rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#
+#     * Redistributions in binary form must reproduce the above
+#       copyright notice, this list of conditions and the following
+#       disclaimer in the documentation and/or other materials provided
+#       with the distribution.
+#
+#     * The names of the contributors may not be used to endorse or
+#       promote products derived from this software without specific
+#       prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import os
+import sys
+import codecs
+
+
+def _verify_python3_env():
+    """Ensures that the environment is good for unicode on Python 3."""
+    try:
+        import locale
+        fs_enc = codecs.lookup(locale.getpreferredencoding()).name
+    except Exception:
+        fs_enc = 'ascii'
+    if fs_enc != 'ascii':
+        return
+
+    extra = ''
+    if os.name == 'posix':
+        import subprocess
+        rv = subprocess.Popen(['locale', '-a'], stdout=subprocess.PIPE,
+                              stderr=subprocess.PIPE).communicate()[0]
+        good_locales = set()
+        has_c_utf8 = False
+
+        # Make sure we're operating on text here.
+        if isinstance(rv, bytes):
+            rv = rv.decode('ascii', 'replace')
+
+        for line in rv.splitlines():
+            locale = line.strip()
+            if locale.lower().endswith(('.utf-8', '.utf8')):
+                good_locales.add(locale)
+                if locale.lower() in ('c.utf8', 'c.utf-8'):
+                    has_c_utf8 = True
+
+        extra += '\n\n'
+        if not good_locales:
+            extra += (
+                'Additional information: on this system no suitable UTF-8\n'
+                'locales were discovered.  This most likely requires resolving\n'
+                'by reconfiguring the locale system.'
+            )
+        elif has_c_utf8:
+            extra += (
+                'This system supports the C.UTF-8 locale which is recommended.\n'
+                'You might be able to resolve your issue by exporting the\n'
+                'following environment variables:\n\n'
+                '    export LC_ALL=C.UTF-8\n'
+                '    export LANG=C.UTF-8'
+            )
+        else:
+            extra += (
+                'This system lists a couple of UTF-8 supporting locales that\n'
+                'you can pick from.  The following suitable locales were\n'
+                'discovered: %s'
+            ) % ', '.join(sorted(good_locales))
+
+        bad_locale = None
+        for locale in os.environ.get('LC_ALL'), os.environ.get('LANG'):
+            if locale and locale.lower().endswith(('.utf-8', '.utf8')):
+                bad_locale = locale
+            if locale is not None:
+                break
+        if bad_locale is not None:
+            extra += (
+                '\n\ocrmypdf discovered that you exported a UTF-8 locale\n'
+                'but the locale system could not pick up from it because\n'
+                'it does not exist.  The exported locale is "%s" but it\n'
+                'is not supported'
+            ) % bad_locale
+
+    raise RuntimeError('ocrmypdf will abort further execution because Python 3 '
+                       'was configured to use ASCII as encoding for the '
+                       'environment.' + extra)

ocrmypdf/exec/__init__.py

@@ -4,9 +4,40 @@
 """Wrappers to manage subprocess calls"""
 
 import os
+import re
+import sys
+from subprocess import run, STDOUT, PIPE, CalledProcessError
+from ..exceptions import MissingDependencyError
 
 
 def get_program(name):
     "Check environment variables for overrides to this program"
     envvar = 'OCRMYPDF_' + name.upper()
     return os.environ.get(envvar, name)
+
+
+def get_version(program, *, 
+        version_arg='--version', regex=r'(\d+(\.\d+)*)'):
+    "Get the version of the specified program, "
+    args_prog = [
+        get_program(program),
+        version_arg
+    ]
+    try:
+        proc = run(
+            args_prog, close_fds=True, universal_newlines=True,
+            stdout=PIPE, stderr=STDOUT, check=True)
+        output = proc.stdout
+    except CalledProcessError as e:
+        raise MissingDependencyError(
+            "Could not find program '{}' on the PATH".format(program)) from e
+
+    try:
+        version = re.match(regex, output.strip()).group(1)
+    except AttributeError as e:
+        raise MissingDependencyError(
+            ("The program '{}' did not report its version. "
+            "Message was:\n{}").format(program, output)
+        )
+
+    return version

ocrmypdf/exec/ghostscript.py

@@ -8,28 +8,14 @@ from functools import lru_cache
 import re
 import sys
 from PIL import Image
-from . import get_program
+from . import get_program, get_version
 from ..exceptions import SubprocessOutputError, MissingDependencyError
 from ..helpers import fspath
 
 
 @lru_cache(maxsize=1)
 def version():
-    args_gs = [
-        get_program('gs'),
-        '--version'
-    ]
-    try:
-        proc = run(
-                args_gs, close_fds=True, universal_newlines=True,
-                stdout=PIPE, stderr=STDOUT, check=True)
-        ver = proc.stdout
-    except CalledProcessError as e:
-        print("Could not find Ghostscript executable on system PATH.",
-              file=sys.stderr)
-        raise MissingDependencyError from e
-
-    return ver.strip()
+    return get_version('gs')
 
 
 def _gs_error_reported(stream):

ocrmypdf/exec/qpdf.py

@@ -9,25 +9,12 @@ import re
 
 from ..exceptions import InputFileError, SubprocessOutputError, \
     MissingDependencyError, EncryptedPdfError
-from . import get_program
+from . import get_program, get_version
 
 
 @lru_cache(maxsize=1)
 def version():
-    args_qpdf = [
-        get_program('qpdf'),
-        '--version'
-    ]
-    try:
-        p = run(args_qpdf, universal_newlines=True, stderr=STDOUT,
-                stdout=PIPE)
-    except CalledProcessError as e:
-        print("Could not find qpdf executable on system PATH.",
-              file=sys.stderr)
-        raise MissingDependencyError() from e
-
-    qpdf_version = re.match(r'qpdf version (.+)', p.stdout).group(1)
-    return qpdf_version
+    return get_version('qpdf', regex=r'qpdf version (.+)')    
 
 
 def check(input_file, log=None):

ocrmypdf/exec/tesseract.py

@@ -14,7 +14,7 @@ from subprocess import PIPE, CalledProcessError, \
 
 from ..exceptions import MissingDependencyError, TesseractConfigError
 from ..helpers import page_number
-from . import get_program
+from . import get_program, get_version
 
 OrientationConfidence = namedtuple(
     'OrientationConfidence',
@@ -40,21 +40,7 @@ HOCR_TEMPLATE = """<?xml version="1.0" encoding="UTF-8"?>
 
 @lru_cache(maxsize=1)
 def version():
-    args_tess = [
-        get_program('tesseract'),
-        '--version'
-    ]
-    try:
-        versions = check_output(
-                args_tess, close_fds=True, universal_newlines=True,
-                stderr=STDOUT)
-    except CalledProcessError as e:
-        print("Could not find Tesseract executable on system PATH.",
-              file=sys.stderr)
-        raise MissingDependencyError from e
-
-    tesseract_version = re.match(r'tesseract\s(.+)', versions).group(1)
-    return tesseract_version
+    return get_version('tesseract', regex=r'tesseract\s(.+)')
 
 
 def v4():
@@ -126,7 +112,7 @@ def tess_base_args(langs, engine_mode):
 
 def get_orientation(input_file, language: list, engine_mode, timeout: float,
                     log):
-    args_tesseract = tess_base_args(language, engine_mode) + [
+    args_tesseract = tess_base_args(['osd'], engine_mode) + [
         psm(), '0',
         input_file,
         'stdout'

ocrmypdf/exec/unpaper.py

@@ -9,19 +9,7 @@ import sys
 import os
 from functools import lru_cache
 from ..exceptions import MissingDependencyError
-from . import get_program
-
-
-@lru_cache(maxsize=1)
-def version():
-    args_unpaper = [
-        get_program('unpaper'),
-        '--version'
-    ]
-    ver = check_output(
-        args_unpaper, close_fds=True, universal_newlines=True,
-        stderr=STDOUT, timeout=5)
-    return ver.strip()
+from . import get_program, get_version
 
 
 try:
@@ -31,6 +19,11 @@ except ImportError:
     raise
 
 
+@lru_cache(maxsize=1)
+def version():
+    return get_version('unpaper')
+
+
 def run(input_file, output_file, dpi, log, mode_args):
     args_unpaper = [
         get_program('unpaper'),

ocrmypdf/helpers.py

@@ -32,7 +32,7 @@ def re_symlink(input_file, soft_link_name, log=None):
                 "%s exists and is not a link" % soft_link_name)
         try:
             os.unlink(soft_link_name)
-        except:
+        except OSError:
             prdebug("Can't unlink %s" % (soft_link_name))
 
     if not os.path.exists(input_file):

ocrmypdf/pipeline.py

@@ -945,7 +945,7 @@ def merge_sidecars(
             if page_num != 0:
                 stream.write('\f')  # Form feed between pages
             if txt_file:
-                with open(txt_file, 'r') as in_:
+                with open(txt_file, 'r', encoding="utf-8") as in_:
                     txt = in_.read()
                     # Tesseract v4 alpha started adding form feeds in 
                     # commit aa6eb6b
@@ -953,9 +953,9 @@ def merge_sidecars(
                     # for consistency just ignore its form feeds and insert our
                     # own
                     if txt.endswith('\f'):
-                        stream.write(in_.read()[:-1])
+                        stream.write(txt[:-1])
                     else:
-                        stream.write(in_.read())
+                        stream.write(txt)
             else:
                 stream.write('[OCR skipped on page {}]'.format(
                         page_num + 1))
@@ -964,7 +964,7 @@ def merge_sidecars(
         write_pages(sys.stdout)
         sys.stdout.flush()
     else:
-        with open(output_file, 'w', encoding='utf-8') as out:
+        with open(output_file, 'w', encoding="utf-8") as out:
             write_pages(out)
 
 

requirements.txt

@@ -2,8 +2,8 @@
 # setup.py lists a separate set of requirements that are looser to simplify
 # installation
 ruffus == 2.6.3
-Pillow == 4.1.1
+Pillow == 4.3.0
 reportlab == 3.4.0
 PyPDF2 == 1.26.0
-img2pdf == 0.2.3
-cffi == 1.10.0
+img2pdf == 0.2.4
+cffi == 1.11.2

setup.py

@@ -217,6 +217,7 @@ setup(
         "Topic :: Text Processing :: Indexing",
         "Topic :: Text Processing :: Linguistic",
         ],
+    python_requires='>=3.5',
     setup_requires=[
         'setuptools_scm',       # so that version will work
         'cffi>=1.9.1'           # to build the leptonica module

tests/test_main.py

@@ -805,7 +805,6 @@ def test_user_words(resources, outdir):
             '--image-dpi', 150,
             '--sidecar', sidecar_before
         )
-
         assert 'cromulent' not in sidecar_before.open().read()
 
     with word_list.open('w') as f:
@@ -984,6 +983,19 @@ def test_sidecar_pagecount(spoof_tesseract_cache, resources, outpdf):
         "Sidecar page count does not match PDF page count"
 
 
+def test_sidecar_nonempty(spoof_tesseract_cache, resources, outpdf):
+    sidecar = outpdf + '.txt'
+    check_ocrmypdf(
+        resources / 'ccitt.pdf', outpdf,
+        '--sidecar', sidecar,
+        env=spoof_tesseract_cache
+    )
+
+    with open(sidecar, 'r') as f:
+        ocr_text = f.read()
+    assert 'the' in ocr_text
+
+
 def test_pdfa_1(spoof_tesseract_cache, resources, outpdf):
     check_ocrmypdf(
         resources / 'ccitt.pdf', outpdf,
@@ -992,4 +1004,16 @@ def test_pdfa_1(spoof_tesseract_cache, resources, outpdf):
     )
 
     pdfa_info = file_claims_pdfa(outpdf)
-    assert pdfa_info['conformance'] == 'PDF/A-1B'
+    assert pdfa_info['conformance'] == 'PDF/A-1B'
+
+
+def test_bad_locale():
+    env = os.environ.copy()
+    env['LC_ALL'] = 'C'
+
+    p, out, err = run_ocrmypdf(
+        'a', 'b', env=env
+    )
+    assert out == '', "stdout not clean"
+    assert p.returncode != 0
+    assert 'configured to use ASCII as encoding' in err, "should whine"