Merge branch 'hotfix/v3.1.1'

Don't exit when qpdf repairs the file successfully but displays warning

.gitignore

@@ -9,9 +9,15 @@ build/
 dist/
 *.egg-info/
 venv/
+venv-3.4/
+venv-3.5/
 */test/output
 bin/
 include/
 lib/
 pip-selfcheck.json
 pyvenv.cfg
+htmlcov/
+.coverage
+.cache/
+.ipynb_checkpoints/

.travis.yml

@@ -0,0 +1,41 @@
+language: python
+
+cache:
+    directories:
+        - $HOME/.cache/pip
+        - $HOME/.ccache
+        - tarballs
+
+python:
+    - 3.4
+
+before_install:
+    # Ubuntu packages
+    - sudo add-apt-repository ppa:evl.ms/precise -y  # for Ghostscript 9.15
+    - sudo add-apt-repository ppa:lyrasis/precise-backports -y  # for Tesseract 3.03
+    - sudo add-apt-repository ppa:b-eltzner/qpdfview-exp -y  # for QPDF 5
+    - sudo add-apt-repository ppa:itachi-san/ffmpeg -y  # for libav 11.2 (for unpaper)
+    - sudo apt-get update -qq  # must go after all add-apt-repo
+    - sudo apt-get install -y ghostscript tesseract-ocr tesseract-ocr-deu tesseract-ocr-eng tesseract-ocr-fra qpdf poppler-utils gcc libavformat-dev libavcodec-dev libavutil-dev automake make pkg-config xsltproc 
+
+    # pip
+    - pip install --upgrade pip
+
+    # Download, make and install unpaper (using ccache)
+    - mkdir -p tarballs
+    - "[ -f tarballs/unpaper-6.1.tar.xz ] || wget -q https://www.flameeyes.eu/files/unpaper-6.1.tar.xz -O tarballs/unpaper-6.1.tar.xz"
+    - tar -xvf tarballs/unpaper-6.1.tar.xz
+    - export PATH="/usr/lib/ccache:$PATH"
+    - pushd unpaper-6.1 && ./configure --prefix=/usr && make -j && sudo make install && popd
+
+install:
+    - pip install -r requirements.txt
+    - pip install -r test_requirements.txt
+
+script:
+    - python setup.py clean
+    - python setup.py install
+    - py.test
+
+os:
+    - linux

Dockerfile

@@ -44,12 +44,13 @@ RUN apt-get install -y \
   xsltproc
 
 WORKDIR /root
-RUN wget https://github.com/Flameeyes/unpaper/archive/unpaper-6.1.tar.gz
+RUN wget -q https://github.com/Flameeyes/unpaper/archive/unpaper-6.1.tar.gz
 RUN tar xf unpaper-6.1.tar.gz
 WORKDIR /root/unpaper-unpaper-6.1
 RUN autoreconf -i
 RUN ./configure CFLAGS="-O2 -march=native -pipe -flto"
 RUN make -j install
+WORKDIR /
 
 RUN apt-get remove -y \
   gcc \
@@ -59,7 +60,7 @@ RUN apt-get remove -y \
   xsltproc \
   make
 RUN apt-get autoremove -y && apt-get clean -y
-RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/*
 
 # Set up a Python virtualenv and take all of the system packages, so we can
 # rely on the platform packages rather than importing GCC and compiling them

README.rst

@@ -10,23 +10,25 @@ Main features
 -  Generates a searchable
    `PDF/A <https://en.wikipedia.org/?title=PDF/A>`__ file from a regular PDF
    only containing images
--  Places OCRed text accurately below the image to ease copy / paste
+-  Places OCR text accurately below the image to ease copy / paste
 -  Keeps the exact resolution of the original embedded images
 
    -  or if requested oversamples the images before OCRing so as to get
       better results
 
--  When possible, copies input images directly to output without transcoding them,
+-  When possible, copies input images directly to output without transcoding,
    to preserve image quality
 -  Keeps file size about the same
 -  If requested deskews and/or cleans the image before performing OCR
 -  Validates input and output files
 -  Provides debug mode to enable easy verification of the OCR results
--  Processes several pages in parallel when more than one CPU core is
+-  Processes pages in parallel when more than one CPU core is
    available
--  Uses Tesseract OCR engine
+-  Uses `Tesseract OCR <https://github.com/tesseract-ocr/tesseract>`__ engine
+-  Supports the `39 languages <https://code.google.com/p/tesseract-ocr/downloads/list>`__ recognized by Tesseract
+-  Battle-tested on thousands of PDFs, a test suite and continuous integration
 
-For details: please consult the `release notes <RELEASE_NOTES.rst>`__
+For details: please consult the `release notes <RELEASE_NOTES.rst>`__.
 
 Motivation
 ----------
@@ -90,16 +92,18 @@ To execute the OCRmyPDF on a local file, you must `provide a writable volume to
 
    docker run -v "$(pwd):/home/docker" <other docker arguments>   ocrmypdf <your arguments to ocrmypdf>
 
-In this worked example, the current working directory contains an input file called `test.pdf` and the output will go to `output.pdf`:: 
+In this worked example, the current working directory contains an input file called ``test.pdf`` and the output will go to ``output.pdf``:: 
 
    docker run -v "$(pwd):/home/docker"   ocrmypdf --skip-text test.pdf output.pdf
 
-Note that `ocrmypdf` has its own separate -v argument to control debug verbosity. All Docker arguments should before the `ocrmypdf` container name and all arguments to `ocrmypdf` should be listed after.
+Note that ``ocrmypdf`` has its own separate ``-v VERBOSITYLEVEL`` argument to control debug verbosity. All Docker arguments should before the ``ocrmypdf`` container name and all arguments to ``ocrmypdf`` should be listed after.
 
-Installing on Mac OS X Yosemite
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Installing on Mac OS X
+~~~~~~~~~~~~~~~~~~~~~~
 
-If it's not already present, `install Homebrew <http://brew.sh/>`__
+These instructions probably work on all Mac OS X versions later than 10.7 (Lion). OCRmyPDF is known to work on Yosemite and El Capitan, and regularly tested on El Capitan.
+
+If it's not already present, `install Homebrew <http://brew.sh/>`__.
 
 Update Homebrew::
 
@@ -120,7 +124,7 @@ It is also recommended that install Pillow and confirm it can read and write JPE
    pip3 install --upgrade pip
    pip3 install --upgrade pillow
 
-To test that your Python imaging library (Pillow) can access JPEG and PNG files, try this command::
+Sometimes, the Python imaging library (Pillow) can end up being compiled and installed without support for JPEG and PNG files. (Arguably, this is an unfixed bug in Pillow's installer.) To confirm that Pillow is compiled correctly and can access JPEG and PNG files, try this command::
 
    python3 -c "from PIL import Image; im = Image.new('1', (1, 1)); im.save('test.png'); im.save('test.jpg')"
 
@@ -137,7 +141,7 @@ The command line program should now be available::
 Installing on Ubuntu 14.04 LTS
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Installing on Ubuntu 14.04 LTS (trusty) is more difficult than other options, because of certain bugs in package installation.
+Installing on Ubuntu 14.04 LTS (trusty) is more difficult than other options, because of certain bugs in Python package installation.
 
 Update apt-get::
 
@@ -176,22 +180,33 @@ package <http://www.thefourtheye.in/2014/12/Python-venv-problem-with-ensurepip-i
    source venv-ocrmypdf/bin/activate
    pip install ocrmypdf
 
-Ubuntu 14.04 only installs `unpaper` version 0.4.2, which is not supported by OCRmyPDF because it is produces invalid output. This program is an optional dependency, and provides page deskewing and cleaning. See `Dockerfile <Dockerfile>`__ for an example of how to building unpaper 6.1 from source. If you choose to install unpaper later, OCRmyPDF will use the foremost version on the system PATH.
+Ubuntu 14.04 only installs ``unpaper`` version 0.4.2, which is not supported by OCRmyPDF because it is produces invalid output. This program is an optional dependency, and provides page deskewing and cleaning. See `Dockerfile <Dockerfile>`__ for an example of how to building unpaper 6.1 from source. If you choose to install unpaper later, OCRmyPDF will use the foremost version on the system PATH.
 
       
 Installing HEAD revision from sources
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-To install the HEAD revision from sources in development mode::
+If you have ``git`` and ``python3.4`` or ``python3.5`` installed, you can install from source. When the ``pip`` installer runs,
+it will alert you if dependencies are missing.
+
+First, clone the HEAD revision::
 
    git clone -b master https://github.com/jbarlow83/OCRmyPDF.git
    cd OCRmyPDF
+
+To install the HEAD revision from sources::
+
+   pip3 install .
+
+Or, to install in `development mode <https://pythonhosted.org/setuptools/setuptools.html#development-mode>`__, 
+allowing customization of OCRmyPDF, use the ``-e`` flag::
+
    pip3 install -e .
    
-On certain Linux/UNIX platforms such as Ubuntu, you may need to use 
+On certain Linux distributions such as Ubuntu, you may need to use 
 run the install command as superuser::
 
-   sudo pip3 install -e .
+   sudo pip3 install [-e] .
    
 Note that this will alter your system's Python distribution. If you prefer 
 to not install as superuser, you can install the package in a Python virtual environment::
@@ -200,10 +215,10 @@ to not install as superuser, you can install the package in a Python virtual env
    pyvenv venv
    source venv/bin/activate
    cd OCRmyPDF
-   pip3 install -e .
+   pip3 install .
 
-If your platform does not have ``pip3``, make sure that Python 3.4+ and the `pip` 
-package are installed.
+However, ``ocrmypdf`` will only be accessible on the system PATH after
+you activate the virtual environment.
 
 To run the program::
    
@@ -221,10 +236,10 @@ In case you detect an issue, please:
 
 -  Check if your issue is already known
 -  If no problem report exists on github, please create one here:
-   https://github.com/fritz-hh/OCRmyPDF/issues
+   https://github.com/jbarlow83/OCRmyPDF/issues
 -  Describe your problem thoroughly
 -  Append the console output of the script when running the debug mode
-   (-v 1 option)
+   (``-v 1`` option)
 -  If possible provide your input PDF file as well as the content of the
    temporary folder (using a file sharing service like
    www.file-upload.net)

RELEASE_NOTES.rst

@@ -5,6 +5,33 @@ Please always read this file before installing the package
 
 Download software here: https://github.com/jbarlow83/OCRmyPDF/tags
 
+
+v3.1.1:
+
+Changes
+-------
+
+-  Fixed bug that caused incorrect page size and DPI calculations on documents with mixed page sizes
+
+
+v3.1:
+=====
+
+Changes
+-------
+
+-  Default output format is now PDF/A-2b instead of PDF/A-1b
+-  Python 3.5 and OS X El Capitan are now supported platforms - no changes were
+   needed to implement support
+-  Improved some error messages related to missing input files
+-  Fixed issue #20 - uppercase .PDF extension not accepted
+-  Fixed an issue where OCRmyPDF failed to text that certain pages contained previously OCR'ed text, 
+   such as OCR text produced by Tesseract 3.04
+-  Inserts /Creator tag into PDFs so that errors can be traced back to this project
+-  Added new option --pdf-renderer=auto, to let OCRmyPDF pick the best PDF renderer. 
+   Currently it always chooses the 'hocrtransform' renderer but that behavior may change.
+-  Set up Travis CI automatic integration testing
+
 v3.0:
 =====
 
@@ -36,6 +63,8 @@ Changes
 
 -  New, robust rewrite in Python 3.4+ with ruffus_ pipelines
 -  Now uses Ghostscript 9.14's improved color conversion model to preserve PDF colors
+-  OCR text is now rendered in the PDF as invisible text. Previous versions of OCRmyPDF
+   incorrectly rendered visible text with an image on top.
 -  All "tasks" in the pipeline can be executed in parallel on any
    available CPUs, increasing performance
 -  The ``-o DPI`` argument has been phased out, in favor of ``--oversample DPI``, in

ocrmypdf/ghostscript.py

@@ -44,7 +44,7 @@ def generate_pdfa(pdf_pages, output_file, threads=1):
             "-sDEVICE=pdfwrite",
             "-sColorConversionStrategy=/RGB",
             "-sProcessColorModel=DeviceRGB",
-            "-dPDFA",
+            "-dPDFA=2",
             "-sPDFACompatibilityPolicy=2",
             "-sOutputICCProfile=srgb.icc",
             "-sOutputFile=" + gs_pdf.name,

ocrmypdf/main.py

@@ -39,9 +39,7 @@ warnings.simplefilter('ignore', pypdf.utils.PdfReadWarning)
 
 
 BASEDIR = os.path.dirname(os.path.realpath(__file__))
-JHOVE_PATH = os.path.realpath(os.path.join(BASEDIR, 'jhove'))
-JHOVE_JAR = os.path.join(JHOVE_PATH, 'bin', 'JhoveApp.jar')
-JHOVE_CFG = os.path.join(JHOVE_PATH, 'conf', 'jhove.conf')
+VERSION = '3.1.1'
 
 
 # -------------
@@ -103,12 +101,12 @@ check_pil_encoder('zlib', 'PNG')
 parser = cmdline.get_argparse(
     prog="ocrmypdf",
     description="Generate searchable PDF file from an image-only PDF file.",
-    version='3.0',
+    version=VERSION,
     fromfile_prefix_chars='@',
     ignored_args=[
         'touch_files_only', 'recreate_database', 'checksum_file_name',
         'key_legend_in_graph', 'draw_graph_horizontally', 'flowchart_format',
-        'forced_tasks', 'target_tasks'])
+        'forced_tasks', 'target_tasks', 'use_threads'])
 
 parser.add_argument(
     'input_file',
@@ -176,7 +174,7 @@ advanced.add_argument(
     '--tesseract-config', default=[], type=list, action='append',
     help="additional Tesseract configuration files")
 advanced.add_argument(
-    '--pdf-renderer', choices=['tesseract', 'hocr'], default='hocr',
+    '--pdf-renderer', choices=['auto', 'tesseract', 'hocr'], default='auto',
     help='choose OCR PDF renderer')
 advanced.add_argument(
     '--tesseract-timeout', default=180.0, type=float,
@@ -218,6 +216,8 @@ if not set(options.language).issubset(tesseract.languages()):
 # ----------
 # Arguments
 
+if options.pdf_renderer == 'auto':
+    options.pdf_renderer = 'hocr'
 
 if any((options.deskew, options.clean, options.clean_final)):
     try:
@@ -341,9 +341,8 @@ def cleanup_working_files(*args):
 
 @transform(
     input=options.input_file,
-    filter=suffix('.pdf'),
-    output='.repaired.pdf',
-    output_dir=work_folder,
+    filter=formatter('(?i)\.pdf'),
+    output=work_folder + '{basename[0]}.repaired.pdf',
     extras=[_log, _pdfinfo, _pdfinfo_lock])
 def repair_pdf(
         input_file,
@@ -357,14 +356,20 @@ def repair_pdf(
     try:
         out = check_output(args_qpdf, stderr=STDOUT, universal_newlines=True)
     except CalledProcessError as e:
+        exit_with_error = True
         if e.returncode == 2:
             print("{0}: not a valid PDF, and could not repair it.".format(
                     options.input_file))
             print("Details:")
             print(e.output)
+        elif e.returncode == 3 and e.output.find("operation succeeded"):
+            exit_with_error = False
+            out = e.output
+            print(e.output)
         else:
             print(e.output)
-        sys.exit(ExitCode.input_file)
+        if exit_with_error:
+            sys.exit(ExitCode.input_file)
 
     log.debug(out)
 
@@ -468,7 +473,6 @@ def rasterize_with_ghostscript(
         log,
         pdfinfo,
         pdfinfo_lock):
-
     pageinfo = get_pageinfo(input_file, pdfinfo, pdfinfo_lock)
 
     device = 'png16m'  # 24-bit
@@ -758,6 +762,11 @@ def generate_postscript_stub(
     if options.subject:
         pdfmark['subject'] = options.subject
 
+    pdfmark['creator'] = '{0} {1} / Tesseract OCR{2} {3}'.format(
+            parser.prog, VERSION,
+            '+PDF' if options.pdf_renderer == 'tesseract' else '',
+            tesseract.version())
+
     generate_pdfa_def(output_file, pdfmark)
 
 
@@ -881,10 +890,16 @@ def available_cpu_count():
     return 1
 
 
+def cleanup_ruffus_error_message(msg):
+    msg = re.sub(r'\s+', r' ', msg, re.MULTILINE)
+    msg = re.sub(r"\((.+?)\)", r'\1', msg)
+    msg = msg.strip()
+    return msg
+
+
 def run_pipeline():
     if not options.jobs or options.jobs == 1:
         options.jobs = available_cpu_count()
-
     try:
         cmdline.run(options)
     except ruffus_exceptions.RethrownJobError as e:
@@ -896,9 +911,21 @@ def run_pipeline():
         for exc in e.args:
             task_name, job_name, exc_name, exc_value, exc_stack = exc
             if exc_name == 'builtins.SystemExit':
-                return eval(
-                    exc_value,
-                    {'ExitCode': ExitCode}, {'exc_value': exc_value})
+                match = re.search(r"\.(.+?)\)", exc_value)
+                exit_code_name = match.groups()[0]
+                exit_code = getattr(ExitCode, exit_code_name, 'other_error')
+                return exit_code
+            elif exc_name == 'ruffus.ruffus_exceptions.MissingInputFileError':
+                print(cleanup_ruffus_error_message(exc_value))
+                return ExitCode.input_file
+            elif exc_name == 'builtins.TypeError':
+                # Even though repair_pdf will fail, ruffus will still try
+                # to call split_pages with no input files, likely due to a bug
+                if task_name == 'split_pages':
+                    print("Input file '{0}' is not a valid PDF".format(
+                        options.input_file))
+                    return ExitCode.input_file
+
         return ExitCode.other_error
 
     if not validate_pdfa(options.output_file, _log):

ocrmypdf/pageinfo.py

@@ -104,16 +104,37 @@ def _find_page_images(page, pageinfo):
         yield image
 
 
-def _pdf_get_pageinfo(infile, page: int):
+def _page_has_text(pdf, page):
+    # Simple test
+    text = page.extractText()
+    if text.strip() != '':
+        return True
+
+    # More nuanced test to deal with quirks of Tesseract PDF generation
+    # Check if there's a Glyphless font
+    try:
+        font = page['/Resources']['/Font']
+    except KeyError:
+        pass
+    else:
+        font_objects = list(font.keys())
+        for font_object in font_objects:
+            basefont = font[font_object]['/BaseFont']
+            if basefont.endswith('GlyphLessFont'):
+                return True
+
+    return False
+
+
+def _pdf_get_pageinfo(infile, pageno: int):
     pageinfo = {}
-    pageinfo['pageno'] = page
+    pageinfo['pageno'] = pageno
     pageinfo['images'] = []
 
     pdf = pypdf.PdfFileReader(infile)
-    page = pdf.pages[page - 1]
+    page = pdf.pages[pageno]
 
-    text = page.extractText()
-    pageinfo['has_text'] = (text.strip() != '')
+    pageinfo['has_text'] = _page_has_text(pdf, page)
 
     width_pt = page['/MediaBox'][2] - page['/MediaBox'][0]
     height_pt = page['/MediaBox'][3] - page['/MediaBox'][1]

ocrmypdf/pdfa.py

@@ -27,6 +27,7 @@ def
   /Author <$author>
   /Subject <$subject>
   /Keywords <$keywords>
+  /Creator <$creator>
   /DOCINFO pdfmark
 
 % Define an ICC profile :
@@ -86,6 +87,7 @@ def _get_pdfa_def(icc_profile, icc_identifier, pdfmark):
                           title=pdfmark_utf16.get('title', ''),
                           author=pdfmark_utf16.get('author', ''),
                           subject=pdfmark_utf16.get('subject', ''),
+                          creator=pdfmark_utf16.get('creator', ''),
                           keywords=pdfmark_utf16.get('keywords', ''))
     return result
 

ocrmypdf/test/test_pageinfo.py

@@ -11,6 +11,7 @@ import shutil
 import pytest
 import img2pdf
 from pkg_resources import Requirement, resource_filename
+import pytest
 
 req = Requirement.parse('ocrmypdf')
 
@@ -99,6 +100,7 @@ def test_single_page_inline_image():
         pageinfo.pdf_get_all_pageinfo(filename)
 
 
+@pytest.mark.skipif(True, reason="need to fix packaging error")
 def test_jpeg():
     filename = resource_filename(req, 'tests/resources/c02-22.pdf')
 

setup.cfg

@@ -1,2 +1,4 @@
 [bdist_wheel]
-python-tag = py34
+python-tag = py34
+[pytest]
+norecursedirs = lib 

setup.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 # © 2015 James R. Barlow: github.com/jbarlow83
 
+from __future__ import print_function, unicode_literals
 from setuptools import setup
 from subprocess import Popen, STDOUT, check_output, CalledProcessError
 from string import Template
@@ -9,6 +10,10 @@ import re
 import sys
 
 
+if sys.version_info < (3, 4):
+    print("Python 3.4 or newer is required")
+    sys.exit(1)
+
 missing_program = '''
 The program '{program}' could not be executed or was not found on your
 system PATH.
@@ -179,7 +184,7 @@ tests_require = open('test_requirements.txt').read().splitlines()
 
 setup(
     name='ocrmypdf',
-    version='3.0',  # also update: release notes, main.py
+    version='3.1.1',  # also update: release notes, main.py
     description='OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched',
     url='https://github.com/jbarlow83/OCRmyPDF',
     author='James R. Barlow',

tests/test_main.py

@@ -328,3 +328,29 @@ def test_missing_docinfo():
         'missing_docinfo.pdf', 'missing_docinfo.pdf', '-l', 'eng', '-c')
     assert p.returncode == ExitCode.ok, err
 
+
+def test_uppercase_extension():
+    shutil.copy(_make_input("skew.pdf"), _make_input("UPPERCASE.PDF"))
+    try:
+        check_ocrmypdf("UPPERCASE.PDF", "UPPERCASE_OUT.PDF")
+    finally:
+        os.unlink(_make_input("UPPERCASE.PDF"))
+
+
+def test_input_file_not_found():
+    input_file = "does not exist.pdf"
+    sh, out, err = run_ocrmypdf_sh(
+        _make_input(input_file),
+        _make_output("will not happen.pdf"))
+    assert sh.returncode == ExitCode.input_file
+    assert (input_file in out or input_file in err)
+
+
+def test_input_file_not_a_pdf():
+    input_file = __file__  # Try to OCR this file
+    sh, out, err = run_ocrmypdf_sh(
+        _make_input(input_file),
+        _make_output("will not happen.pdf"))
+    assert sh.returncode == ExitCode.input_file
+    assert (input_file in out or input_file in err)
+