v4.5.6 release notes

Past behavior was to continue and let ruffus puke eventually

.gitignore

@@ -6,6 +6,7 @@ pyvenv.cfg
 tasks.py
 .bash_history
 .ruffus_history.sqlite
+.idea/
 
 # Package building
 *.egg-info/

.travis.yml

@@ -21,16 +21,23 @@ matrix:
       sudo: required
       language: python
       python: 3.6
+      env:
+        - secure: "oyX5xesoHD7qcDXKIxMyGZPi+H/WxcvfFkaviEmq84K1DDyHk48+9e92IKgrw8/lcTADnEo/AgVKfnhCPflFimk1xTkgaK4sUg1WLI2YjmaHcwl5SlBHa2rN3uGBwy1hyP92qyv/mMc9R59NtRJ8u76lbn6eN9wi7lkFWdE6BTw="  # DOCKERHUB_OCRMYPDF_TOKEN
     - os: osx
       osx_image: xcode8
       language: generic
+      env:
+        - secure: "hsf6MT+n2x3OiDM2fQyJZdV0/PWYmv81LdVqC6cfnHBE/8N3DloJRqQ7WfO14TxhiK9PEC7MpyCj0lSabUHEO7gSH6Vks6I1asoSkt8S9/bSMlhT4hei+pwVpeGEiU5xHVATNjY+D919VC3IFvc3XmjT74h/2SLhaZ+jhEmDggM="  # HOMEBREW_OCRMYPDF_TOKEN
 
 before_cache:
 - rm -f $HOME/.cache/pip/log/debug.log
 
-before_install:
-- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then bash .travis/linux_before_install.sh ; fi
-- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then bash .travis/osx_before_install.sh ; fi
+before_install: |
+  if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
+    bash .travis/linux_before_install.sh
+  elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+    bash .travis/osx_before_install.sh
+  fi
 
 install:
 - pip3 install .
@@ -41,20 +48,35 @@ script:
 - pytest
 - mv dont_import_this_ocrmypdf ocrmypdf
 
-after_success:
-- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then bash .travis/osx_brew.sh ; fi
-
-# See https://www.appneta.com/blog/pypi-deployment-with-travis-ci/ for
-# steps to set up testpypi deploy for untagged builds if desired
-
 deploy:
-  provider: pypi
+  # release for main pypi
+- provider: pypi
   user: ocrmypdf-travis
   password:
-    secure: DTFOmmNL6olA0+yXvp4u9jXZlZeqrJsJ0526jzqf4a3gZ6jnGTq5UI6WzRsslSyoMMfXKtHQebqHM6ogSgCZinyZ3ufHJo8fn9brxbEc2gsiWkbj5o3bGwdWMT1vNNE7XW0VCpw87rZ1EEwjl4FJHFudMlPR1yfU5+uq0k0PACo=
+    secure: "DTFOmmNL6olA0+yXvp4u9jXZlZeqrJsJ0526jzqf4a3gZ6jnGTq5UI6WzRsslSyoMMfXKtHQebqHM6ogSgCZinyZ3ufHJo8fn9brxbEc2gsiWkbj5o3bGwdWMT1vNNE7XW0VCpw87rZ1EEwjl4FJHFudMlPR1yfU5+uq0k0PACo="
   distributions: "sdist bdist_wheel"
   on:
     branch: master
     tags: true
     condition: $TRAVIS_PYTHON_VERSION == "3.6" && $TRAVIS_OS_NAME == "linux"
   skip_upload_docs: true
+
+  # test pypi
+- provider: pypi
+  server: https://testpypi.python.org/pypi
+  user: ocrmypdf-travis
+  password:
+    secure: "DTFOmmNL6olA0+yXvp4u9jXZlZeqrJsJ0526jzqf4a3gZ6jnGTq5UI6WzRsslSyoMMfXKtHQebqHM6ogSgCZinyZ3ufHJo8fn9brxbEc2gsiWkbj5o3bGwdWMT1vNNE7XW0VCpw87rZ1EEwjl4FJHFudMlPR1yfU5+uq0k0PACo="
+  distributions: "sdist"
+  on:
+    branch: develop
+    tags: false
+    condition: $TRAVIS_OS_NAME == "osx"
+  skip_upload_docs: true
+
+after_deploy: |
+  if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+    bash .travis/osx_brew.sh
+  elif [[ "$TRAVIS_PYTHON_VERSION" == "3.6" && "$TRAVIS_OS_NAME" == "linux" ]]; then
+    curl -H "Content-Type: application/json" --data '{"build": true}' -X POST https://registry.hub.docker.com/u/jbarlow83/ocrmypdf/trigger/$DOCKERHUB_OCRMYPDF_TOKEN/
+  fi

.travis/osx_brew.sh

@@ -6,3 +6,15 @@ pip3 install homebrew-pypi-poet
 python3 .travis/autobrew.py
 cat ocrmypdf.rb
 brew audit ocrmypdf.rb
+
+# Important: disable debug output so token is hidden
+set +x
+git clone https://$HOMEBREW_OCRMYPDF_TOKEN@github.com/jbarlow83/homebrew-ocrmypdf.git
+set -x
+
+pushd homebrew-ocrmypdf
+cp ../ocrmypdf.rb Formula/ocrmypdf.rb
+git add Formula/ocrmypdf.rb
+git commit -m "homebrew-ocrmypdf: automatic release $TRAVIS_BUILD_NUMBER $TRAVIS_TAG"
+git push origin master
+popd

docs/release_notes.rst

@@ -4,6 +4,21 @@ Release notes
 OCRmyPDF uses `semantic versioning <http://semver.org/>`_.
 
 
+v4.5.6
+------
+
+-   Fixed issue #156, 'NoneType' object has no attribute 'getObject' on pages with no optional /Contents record.  This should resolve all issues related to pages with no /Contents record.
+-   Fixed issue #158, ocrmypdf now stops and terminates if Ghostscript fails on an intermediate step, as it is not possible to proceed.
+-   Fixed issue #160, exception thrown on certain invalid arguments instead of error message
+
+
+v4.5.5
+------
+
+-   Automated update of macOS homebrew tap
+-   Fixed issue #154, KeyError '/Contents' when searching for text on blank pages that have no /Contents record.  Note: incomplete fix for this issue.
+
+
 v4.5.4
 ------
 

ocrmypdf/__main__.py

@@ -268,7 +268,7 @@ def check_options_languages(options, _log):
             "data for the following requested languages: \n")
         for lang in (set(options.language) - tesseract.languages()):
             msg += lang + '\n'
-        raise argparse.ArgumentError(msg)
+        raise argparse.ArgumentError(None, msg)
 
 
 def check_options_output(options, log):
@@ -324,6 +324,7 @@ def check_options_preprocessing(options, log):
 def check_options_ocr_behavior(options, log):
     if options.force_ocr and options.skip_text:
         raise argparse.ArgumentError(
+            None,
             "Error: --force-ocr and --skip-text are mutually incompatible.")
 
     if set(options.language) & {'chi_sim', 'chi_tra'} and \
@@ -420,7 +421,7 @@ def available_cpu_count():
 
 
 def cleanup_ruffus_error_message(msg):
-    msg = re.sub(r'\s+', r' ', msg, re.MULTILINE)
+    msg = re.sub(r'\s+', r' ', msg)
     msg = re.sub(r"\((.+?)\)", r'\1', msg)
     msg = msg.strip()
     return msg

ocrmypdf/exceptions.py

@@ -48,7 +48,7 @@ class InputFileError(ExitCodeException):
 
 
 class SubprocessOutputError(ExitCodeException):
-    exit_code = ExitCode.other_error
+    exit_code = ExitCode.child_process_error
 
 
 class EncryptedPdfError(ExitCodeException):

ocrmypdf/exec/ghostscript.py

@@ -6,8 +6,9 @@ from subprocess import Popen, PIPE, STDOUT, check_call, CalledProcessError, \
     check_output
 from shutil import copy
 from functools import lru_cache
+import re
 from . import get_program
-from ..pdfa import SRGB_ICC_PROFILE
+from ..exceptions import SubprocessOutputError
 
 
 @lru_cache(maxsize=1)
@@ -28,6 +29,10 @@ def version():
     return version.strip()
 
 
+def _gs_error_reported(stream):
+    return re.search(r'error', stream, flags=re.IGNORECASE)
+
+
 def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
                   pageno=1):
     with NamedTemporaryFile(delete=True) as tmp:
@@ -48,15 +53,16 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
         p = Popen(args_gs, close_fds=True, stdout=PIPE, stderr=STDOUT,
                   universal_newlines=True)
         stdout, _ = p.communicate()
-        if 'error' in stdout:
-            log.error(stdout)  # Ghostscript puts errors in stdout
+        if _gs_error_reported(stdout):
+            log.error(stdout)
         else:
             log.debug(stdout)
 
         if p.returncode == 0:
             copy(tmp.name, output_file)
         else:
-            log.error('Ghostscript rendering failed')
+            log.error('Ghostscript rasterizing failed')
+            raise SubprocessOutputError()
 
 
 def generate_pdfa(pdf_pages, output_file, log, threads=1):
@@ -81,7 +87,7 @@ def generate_pdfa(pdf_pages, output_file, log, threads=1):
                   universal_newlines=True)
         stdout, _ = p.communicate()
 
-        if 'error' in stdout or 'ERROR' in stdout:
+        if _gs_error_reported(stdout):
             log.error(stdout)
         elif 'overprint mode not set' in stdout:
             # Unless someone is going to print PDF/A documents on a
@@ -99,4 +105,5 @@ def generate_pdfa(pdf_pages, output_file, log, threads=1):
             # PDF/A - check PDF/A status elsewhere
             copy(gs_pdf.name, output_file)
         else:
-            log.error('Ghostscript PDF/A failed')
+            log.error('Ghostscript PDF/A rendering failed')
+            raise SubprocessOutputError()

ocrmypdf/pageinfo.py

@@ -430,7 +430,7 @@ def _find_images(pdf, container, shorthand=None):
 
     """
 
-    if container.get('/Type') == '/Page':
+    if container.get('/Type') == '/Page' and '/Contents' in container:
         # For a /Page the content stream is attached to the page's /Contents
         page = container
         contentstream = pypdf.pdf.ContentStream(page.getContents(), pdf)
@@ -464,6 +464,9 @@ def _find_images(pdf, container, shorthand=None):
 
 
 def _page_has_text(pdf, page):
+    if not '/Contents' in page:
+        return False
+
     # Simple test
     text = page.extractText()
     if text.strip() != '':

tests/resources/README.rst

@@ -85,6 +85,9 @@ under the terms of the license in LICENSE.rst.
     *   - overlay.pdf
         - @maxandersen
         - PDF file generated by PDFPen pro that triggered content stream parse errors
+    *   - no_conentes.pdf
+        - @jbarlow83
+        - synthetic PDF with a blank page that has no /Contents entry
 
 Assemblies
 ==========

tests/resources/no_contents.pdf

@@ -0,0 +1,21 @@
+%PDF-1.3
+%¿÷¢þ
+1 0 obj
+<< /Pages 2 0 R /Type /Catalog >>
+endobj
+2 0 obj
+<< /Count 1 /Kids [ 3 0 R ] /Type /Pages >>
+endobj
+3 0 obj
+<< /MediaBox [ 0 0 720 720 ] /Parent 2 0 R /Resources << >> /Type /Page >>
+endobj
+xref
+0 4
+0000000000 65535 f 
+0000000015 00000 n 
+0000000064 00000 n 
+0000000123 00000 n 
+trailer << /Root 1 0 R /Size 4 /ID [<52bba3c78160d0c6e851b59110e5d076><52bba3c78160d0c6e851b59110e5d076>] >>
+startxref
+213
+%%EOF

tests/spoof/gs_raster_failure.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+import sys
+import os
+
+"""Replicate Ghostscript raster failure while allowing rendering"""
+
+
+def real_ghostscript(argv):
+    gs_args = ['gs'] + argv[1:]
+    os.execvp("gs", gs_args)
+    return  # Not reachable
+
+
+def main():
+    if '--version' in sys.argv:
+        print('9.20')
+        print('SPOOFED: ' + os.path.basename(__filename__))
+        sys.exit(0)
+
+    # For any rendering calls (device == pdfwrite) call real ghostscript
+    if '-sDEVICE=pdfwrite' in sys.argv:
+        real_ghostscript(sys.argv)
+        return
+
+    # Fail
+    print("ERROR: Ghost story archive not found")
+    sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

tests/spoof/gs_render_failure.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+import sys
+import os
+
+"""Replicate Ghostscript render failure while allowing rasterizing"""
+
+
+def real_ghostscript(argv):
+    gs_args = ['gs'] + argv[1:]
+    os.execvp("gs", gs_args)
+    return  # Not reachable
+
+
+def main():
+    if '--version' in sys.argv:
+        print('9.20')
+        print('SPOOFED: ' + os.path.basename(__filename__))
+        sys.exit(0)
+
+    # For any rasterize calls (device != pdfwrite) call real ghostscript
+    if '-sDEVICE=pdfwrite' not in sys.argv:
+        real_ghostscript(sys.argv)
+        return
+
+    # Fail
+    print("ERROR: Casper is not a friendly ghost")
+    sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

tests/test_main.py

@@ -51,6 +51,16 @@ def spoof_no_tess_pdfa_warning():
     return spoof(tesseract='tesseract_noop.py', gs='gs_feature_elision.py')
 
 
+@pytest.fixture
+def spoof_no_tess_gs_render_fail():
+    return spoof(tesseract='tesseract_noop.py', gs='gs_render_failure.py')
+
+
+@pytest.fixture
+def spoof_no_tess_gs_raster_fail():
+    return spoof(tesseract='tesseract_noop.py', gs='gs_raster_failure.py')
+
+
 @pytest.fixture
 def spoof_qpdf_always_error():
     return spoof(qpdf='qpdf_dummy_return2.py')
@@ -843,4 +853,25 @@ def test_skip_big_with_no_images(spoof_tesseract_noop, resources, outpdf):
     check_ocrmypdf(resources / 'blank.pdf', outpdf,
                    '--skip-big', '5',
                    '--force-ocr',
+                   env=spoof_tesseract_noop)
+
+
+def test_gs_render_failure(spoof_no_tess_gs_render_fail, resources, outpdf):
+    p, out, err = run_ocrmypdf(
+        resources / 'blank.pdf', outpdf,
+        env=spoof_no_tess_gs_render_fail)
+    print(err)
+    assert p.returncode == ExitCode.child_process_error
+
+
+def test_gs_raster_failure(spoof_no_tess_gs_raster_fail, resources, outpdf):
+    p, out, err = run_ocrmypdf(
+        resources / 'ccitt.pdf', outpdf,
+        env=spoof_no_tess_gs_raster_fail)
+    print(err)
+    assert p.returncode == ExitCode.child_process_error
+
+
+def test_no_contents(spoof_tesseract_noop, resources, outpdf):
+    check_ocrmypdf(resources / 'no_contents.pdf', outpdf, '--force-ocr',
                    env=spoof_tesseract_noop)

tests/test_pageinfo.py

@@ -108,3 +108,11 @@ def test_form_xobject(resources):
     pdfinfo = pageinfo.pdf_get_all_pageinfo(str(filename))
     pdfimage = pdfinfo[0]['images'][0]
     assert pdfimage['width'] == 50
+
+
+def test_no_contents(resources):
+    filename = resources / 'no_contents.pdf'
+
+    pdfinfo = pageinfo.pdf_get_all_pageinfo(str(filename))
+    assert len(pdfinfo[0]['images']) == 0
+    assert pdfinfo[0]['has_text'] == False