Travis, true is a program, not a keyword

Seems better to not claim the existence of several entities that don’t
exist as the older one does

.travis.yml

@@ -80,14 +80,14 @@ deploy:
   # we really just want to run after_deploy *after* pypi upload is done, but
   # after_deploy on runs if a given box deployed
 - provider: script
-  script: .travis/null_deploy.sh
+  script: /usr/bin/true
   on:
     branch: master
     tags: false
     condition: $TRAVIS_OS_NAME == "osx"
 
 after_deploy: |
-  if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+  if [[ "$TRAVIS_OS_NAME" == "osx" && "$TRAVIS_BRANCH" == "master" ]]; then
     bash .travis/osx_brew.sh
   elif [[ "$TRAVIS_PYTHON_VERSION" == "3.6" && "$TRAVIS_OS_NAME" == "linux" ]]; then
     curl -H "Content-Type: application/json" --data '{"build": true}' -X POST https://registry.hub.docker.com/u/jbarlow83/ocrmypdf/trigger/$DOCKERHUB_OCRMYPDF_TOKEN/

.travis/autobrew.py

@@ -1,3 +1,6 @@
+#!/usr/bin/env python3
+# © 2016 James R. Barlow: github.com/jbarlow83
+
 from string import Template
 from subprocess import run, PIPE
 import re
@@ -25,7 +28,6 @@ class Ocrmypdf < Formula
   depends_on "libpng"
 
 ${resources}
-
   def install
     ENV.append ["SETUPTOOLS_SCM_PRETEND_VERSION"], "v${ocrmypdf_version}"
     ENV.each do |key, value|

.travis/linux_before_install.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# © 2017 James R. Barlow: github.com/jbarlow83
 set -euo pipefail
 set -x
 

.travis/null_deploy.sh

@@ -1,3 +0,0 @@
-#!/usr/bin/env bash
-
-exit 0

.travis/osx_before_install.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# © 2017 James R. Barlow: github.com/jbarlow83
 set -euo pipefail
 set -x
 

.travis/osx_brew.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# © 2017 James R. Barlow: github.com/jbarlow83
 set -euo pipefail
 set -x
 

docs/cookbook.rst

@@ -95,13 +95,19 @@ Use a program like `img2pdf <https://gitlab.mister-muffin.de/josch/img2pdf>`_ to
 
 	img2pdf my-images*.jpg | ocrmypdf - myfile.pdf
 
-If given a single image as input, OCRmyPDF will try converting it to a PDF on its own.  If the DPI specified in the image is incorrect, it can be overridden with ``--image-dpi``:
+``img2pdf`` also has features to control the position of images on a page, if desired.
+
+For convenience, OCRmyPDF can convert single images to PDFs on its own. If the resolution (dots per inch, DPI) of an image is not set or is incorrect, it can be overridden with ``--image-dpi``. (As 1 inch is 2.54 cm, 1 dpi = 0.39 dpcm).
 
 .. code-block:: bash
 
 	ocrmypdf --image-dpi 300 image.png myfile.pdf
 
-This feature may be removed at some point, because OCRmyPDF does not specialize in converting images to PDFs.
+If you have multiple images, you must use ``img2pdf`` to convert the images to PDF.
+
+.. note::
+
+    ImageMagick ``convert`` can also convert a group of images to PDF, but in the author's experience it takes a long time, transcodes unnecessarily and gives poor results.
 
 You can also use Tesseract 3.04+ directly to convert single page images or multi-page TIFFs to PDF:
 
@@ -149,3 +155,12 @@ Image processing commands can be combined. The order in which options are given
 	ocrmypdf --deskew --clean --rotate-pages input.pdf output.pdf
 
 
+Improving OCR quality
+---------------------
+
+The `Image processing`_ features can improve OCR quality.
+
+Rotating pages and deskewing helps to ensure that the page orientation is correct before OCR begins. Removing the background and/or cleaning the page can also improve results. The ``--oversample DPI`` argument can be specified to resample images to higher resolution before attempting OCR; this can improve results as well.
+
+OCR quality will suffer if the resolution of input images is not correct (since the range of pixel sizes that will be checked for possible fonts will also be incorrect).
+

docs/introduction.rst

@@ -101,4 +101,7 @@ To the author's knowledge, OCRmyPDF is the most feature-rich and thoroughly test
 * pdf2pdfocr
 * pdfsandwich
 * pypdfocr
-* pdfbeads
+* pdfbeads
+
+If you are looking for a micro web-frontend for OCRmyPDF, consider the third-party `OCRmyPDF-web <https://github.com/sseemayer/OCRmyPDF-web>`_. Bear in mind that OCRmyPDF is not designed to be secure against malware-bearing PDFs (see `Using OCRmyPDF online`_).
+

docs/release_notes.rst

@@ -4,6 +4,12 @@ Release notes
 OCRmyPDF uses `semantic versioning <http://semver.org/>`_.
 
 
+v5.0.1
+------
+
+-   Fixed issue #169, exception due to failure to create sidecar text files on some versions of Tesseract 3.04, including the jbarlow83/ocrmypdf image
+
+
 v5.0
 ----
 

ocrmypdf/__init__.py

@@ -1,3 +1,5 @@
+# © 2017 James R. Barlow: github.com/jbarlow83
+
 import pkg_resources
 
 PROGRAM_NAME = 'ocrmypdf'

ocrmypdf/__main__.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# © 2015-16 James R. Barlow: github.com/jbarlow83
+# © 2015-17 James R. Barlow: github.com/jbarlow83
 
 from contextlib import suppress
 from tempfile import mkdtemp

ocrmypdf/exec/ghostscript.py

@@ -6,6 +6,7 @@ from subprocess import run, PIPE, STDOUT, CalledProcessError
 from shutil import copy
 from functools import lru_cache
 import re
+import sys
 from . import get_program
 from ..exceptions import SubprocessOutputError
 

ocrmypdf/exec/tesseract.py

@@ -21,27 +21,22 @@ OrientationConfidence = namedtuple(
     'OrientationConfidence',
     ('angle', 'confidence'))
 
-HOCR_TEMPLATE = '''<?xml version="1.0" encoding="UTF-8"?>
+HOCR_TEMPLATE = """<?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
  <head>
   <title></title>
-  <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
-  <meta name='ocr-system' content='tesseract 3.02.02' />
+<meta http-equiv="Content-Type" content="text/html;charset=utf-8" />
+  <meta name='ocr-system' content='tesseract 3.05.00' />
   <meta name='ocr-capabilities' content='ocr_page ocr_carea ocr_par ocr_line ocrx_word'/>
- </head>
- <body>
-  <div class='ocr_page' id='page_1' title='image "x.tif"; bbox 0 0 {0} {1}; ppageno 0'>
-   <div class='ocr_carea' id='block_1_1' title="bbox 0 1 {0} {1}">
-    <p class='ocr_par' dir='ltr' id='par_1' title="bbox 0 1 {0} {1}">
-     <span class='ocr_line' id='line_1' title="bbox 0 1 {0} {1}"><span class='ocrx_word' id='word_1' title="bbox 0 1 {0} {1}"> </span>
-     </span>
-    </p>
-   </div>
+</head>
+<body>
+  <div class='ocr_page' id='page_1' title='image "_blank.png"; bbox 0 0 {0} {1}; ppageno 0'>
   </div>
  </body>
-</html>'''
+</html>
+"""
 
 
 @lru_cache(maxsize=1)
@@ -229,11 +224,13 @@ def generate_hocr(input_file, output_files, language: list, engine_mode,
 
     # Reminder: test suite tesseract spoofers will break after any changes
     # to the number of order parameters here
+    # Tesseract 3.04 requires the order here to be "hocr txt" and will fail
+    # on "txt hocr"
     args_tesseract.extend([
         input_file,
         prefix,
-        'txt',
-        'hocr'
+        'hocr',
+        'txt'
     ] + tessconfig)
     try:
         log.debug(args_tesseract)
@@ -316,8 +313,8 @@ def generate_pdf(*, input_image, skip_pdf, output_pdf, output_text,
     args_tesseract.extend([
         input_image,
         prefix,
-        'txt',
         'pdf',
+        'txt'
     ] + tessconfig)
 
     try:

ocrmypdf/lib/__init__.py

@@ -1 +1,2 @@
 """Bindings to external libraries"""
+# © 2017 James R. Barlow: github.com/jbarlow83

ocrmypdf/lib/compile_leptonica.py

@@ -1,3 +1,5 @@
+# © 2017 James R. Barlow: github.com/jbarlow83
+
 from cffi import FFI
 
 ffibuilder = FFI()

tests/spoof/gs_feature_elision.py

@@ -1,4 +1,6 @@
 #!/usr/bin/env python3
+# © 2016 James R. Barlow: github.com/jbarlow83
+
 import sys
 import os
 from subprocess import check_call

tests/spoof/gs_pdfa_failure.py

@@ -1,4 +1,6 @@
 #!/usr/bin/env python3
+# © 2016 James R. Barlow: github.com/jbarlow83
+
 import sys
 import os
 

tests/spoof/gs_raster_failure.py

@@ -1,4 +1,6 @@
 #!/usr/bin/env python3
+# © 2016 James R. Barlow: github.com/jbarlow83
+
 import sys
 import os
 

tests/spoof/gs_render_failure.py

@@ -1,4 +1,6 @@
 #!/usr/bin/env python3
+# © 2016 James R. Barlow: github.com/jbarlow83
+
 import sys
 import os
 

tests/spoof/qpdf_dummy_return2.py

@@ -1,4 +1,6 @@
 #!/usr/bin/env python3
+# © 2016 James R. Barlow: github.com/jbarlow83
+
 import sys
 
 

tests/spoof/tesseract_big_image_error.py

@@ -1,4 +1,6 @@
 #!/usr/bin/env python3
+# © 2016 James R. Barlow: github.com/jbarlow83
+
 import sys
 
 
@@ -20,11 +22,11 @@ def main():
     elif sys.argv[1] == '--list-langs':
         print('List of available languages (1):\neng', file=sys.stderr)
         sys.exit(0)
-    elif sys.argv[-1] == 'hocr':
+    elif sys.argv[-2] == 'hocr':
         print("Image too large: (33830, 14959)\n"
               "Error during processing.", file=sys.stderr)
         sys.exit(1)
-    elif sys.argv[-1] == 'pdf':
+    elif sys.argv[-2] == 'pdf':
         print("Image too large: (33830, 14959)\n"
               "Error during processing.", file=sys.stderr)
         sys.exit(1)

tests/spoof/tesseract_cache.py

@@ -1,4 +1,6 @@
 #!/usr/bin/env python3
+# © 2016 James R. Barlow: github.com/jbarlow83
+
 import sys
 import os
 import hashlib
@@ -9,18 +11,27 @@ import subprocess
 """Cache output of tesseract to speed up test suite
 
 The cache is keyed by a hash that includes the tesseract version, some of
-the command line, and the binary dump of the input file. The output file,
-stdout, and stderr are replicated on a cache hit.
+the command line, and the binary dump of the input file, and this file itself.
+ Therefore any updates to this file invalidate cache. Uses SHA-1 because it is
+ fast and defeating a hash collision here is not exactly a priority. :P
+
+The output files, stdout, and stderr are replicated on a cache hit. The output
+files are either a .pdf and .txt or .hocr and .txt.
 
 Page orientation checks are also cached (-psm 0 stdout)
 
-Errors and crashes are not cached.
+Errors and crashes are not cached. If the arguments don't match a known
+caching template then real tesseract is called with the same arguments.
 
 Things not checked:
 -changes to tesseract installation that don't affect --version
 
+Assumes Tesseract 3.04 or higher.
+
 Will fail on Tesseract 3.02.02 in "hocr" mode because it doesn't produce
-the incorrect file extension.
+the incorrect file extension. Will fail on 3.03 because that has no sidecar
+text support. Will fail to replicate a 3.04 bug if wrong parameter order is
+given.
 
 """
 
@@ -35,10 +46,12 @@ def real_tesseract():
     return  # Not reachable
 
 def main():
-    operation = sys.argv[-1]
+    operation = sys.argv[-2]
     sidecar = False
-    if sys.argv[-2] == 'txt':
+    if sys.argv[-1] == 'txt':
         sidecar = True
+    elif sys.argv[-1] == 'stdout':
+        operation = 'stdout'
 
     # For anything unexpected operation, defer to real tesseract binary
     # Currently this includes all use of "--tesseract-config"

tests/spoof/tesseract_crash.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+# © 2016 James R. Barlow: github.com/jbarlow83
 import sys
 import os
 import signal
@@ -28,10 +29,10 @@ def main():
     elif sys.argv[1] == '--list-langs':
         print('List of available languages (1):\neng', file=sys.stderr)
         sys.exit(0)
-    elif sys.argv[-1] == 'hocr':
+    elif sys.argv[-2] == 'hocr':
         print("KABOOM! Tesseract failed for some reason", file=sys.stderr)
         sys.exit(128 + signal.SIGSEGV)
-    elif sys.argv[-1] == 'pdf':
+    elif sys.argv[-2] == 'pdf':
         print("KABOOM! Tesseract failed for some reason", file=sys.stderr)
         sys.exit(128 + signal.SIGSEGV)
     elif sys.argv[-1] == 'stdout':

tests/spoof/tesseract_noop.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+# © 2016 James R. Barlow: github.com/jbarlow83
 import sys
 import img2pdf
 from PIL import Image
@@ -52,7 +53,7 @@ def main():
     elif sys.argv[1] == '--list-langs':
         print('List of available languages (1):\neng', file=sys.stderr)
         sys.exit(0)
-    elif sys.argv[-1] == 'hocr':
+    elif sys.argv[-2] == 'hocr':
         inputf = sys.argv[-4]
         output = sys.argv[-3]
         with Image.open(inputf) as im, \
@@ -61,7 +62,7 @@ def main():
             f.write(HOCR_TEMPLATE.format(str(w), str(h)))
         with open(output + '.txt', 'w') as f:
             f.write('')
-    elif sys.argv[-1] == 'pdf':
+    elif sys.argv[-2] == 'pdf':
         inputf = sys.argv[-4]
         output = sys.argv[-3]
         pdf_bytes = img2pdf.convert([inputf], dpi=300)

tests/spoof/unpaper_oldversion.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+# © 2016 James R. Barlow: github.com/jbarlow83
 import sys
 
 def main():

tests/test_main.py

@@ -4,8 +4,6 @@
 from subprocess import Popen, PIPE, check_output, check_call, DEVNULL
 import os
 import shutil
-from contextlib import suppress
-import sys
 import pytest
 from ocrmypdf.pageinfo import pdf_get_all_pageinfo
 import PyPDF2 as pypdf
@@ -13,6 +11,7 @@ from ocrmypdf.exceptions import ExitCode
 from ocrmypdf import leptonica
 from ocrmypdf.pdfa import file_claims_pdfa
 from ocrmypdf.exec import ghostscript
+import logging
 
 
 check_ocrmypdf = pytest.helpers.check_ocrmypdf
@@ -78,13 +77,11 @@ def test_deskew(spoof_tesseract_noop, resources, outdir):
 
     # Now render as an image again and use Leptonica to find the skew angle
     # to confirm that it was deskewed
-    from ocrmypdf.exec.ghostscript import rasterize_pdf
-    import logging
     log = logging.getLogger()
 
     deskewed_png = outdir / 'deskewed.png'
 
-    rasterize_pdf(
+    ghostscript.rasterize_pdf(
         str(deskewed_pdf),
         str(deskewed_png),
         xres=150,
@@ -119,13 +116,11 @@ def test_remove_background(spoof_tesseract_noop, resources, outdir):
         '--image-dpi', '150',
         env=spoof_tesseract_noop)
 
-    from ocrmypdf.exec.ghostscript import rasterize_pdf
-    import logging
     log = logging.getLogger()
 
     output_png = outdir / 'remove_bg.png'
 
-    rasterize_pdf(
+    ghostscript.rasterize_pdf(
         str(output_pdf),
         str(output_png),
         xres=100,
@@ -271,10 +266,6 @@ def test_argsfile(spoof_tesseract_noop, resources, outdir):
 def check_monochrome_correlation(
         outdir,
         reference_pdf, reference_pageno, test_pdf, test_pageno):
-
-    import ocrmypdf.exec.ghostscript as ghostscript
-    import logging
-
     gslog = logging.getLogger()
 
     reference_png = outdir / '{}.ref{:04d}.png'.format(