Add option to produce PDF/A-1B

It's still in beta for a reason apparently

.coveragerc

@@ -0,0 +1,16 @@
+# Coverage isn't really compatible with subprocesses so results are unreliable
+
+[run]
+branch = True
+#concurrency = multiprocessing
+source = ocrmypdf/
+
+[report]
+exclude_lines =
+	pragma: no cover
+	def __repr__
+	raise AssertionError
+	raise NotImplementedError
+	if 0:
+	if False:
+	if __name__ == .__main__.:

.gitignore

@@ -39,3 +39,6 @@ tests/output/
 tests/resources/private/
 tmp/
 pdfbox-app*.jar
+.vscode/
+IDEAS
+_Dockerfile.local

.travis.yml

@@ -43,6 +43,7 @@ install:
 
 script:
 - mv ocrmypdf dont_import_this_ocrmypdf
+- tesseract --version
 - pytest -n auto
 - mv dont_import_this_ocrmypdf ocrmypdf
 

.travis/linux_before_install.sh

@@ -25,6 +25,14 @@ sudo apt-get install -y --no-install-recommends \
 		tesseract-ocr-fra \
 		tesseract-ocr-deu
 
+# Workaround for https://github.com/tesseract-ocr/tesseract/issues/1167
+# Replace tesseract data installed by tesseract-ocr-eng with a known good version
+sudo apt-get install -y wget
+wget -q https://github.com/tesseract-ocr/tessdata/raw/master/eng.traineddata -O packages/eng.traineddata
+sha1sum packages/eng.traineddata
+sha1sum /usr/share/tesseract-ocr/4.00/tessdata/eng.traineddata
+sudo cp packages/eng.traineddata /usr/share/tesseract-ocr/4.00/tessdata/eng.traineddata
+
 pip install --upgrade pip
 mkdir -p packages
 [ -f packages/unpaper_6.1-1.deb ] || wget -q 'https://www.dropbox.com/s/vaq0kbwi6e6au80/unpaper_6.1-1.deb?raw=1' -O packages/unpaper_6.1-1.deb

.travis/osx_brew.sh

@@ -6,7 +6,9 @@ set -x
 pip3 install homebrew-pypi-poet
 python3 .travis/autobrew.py
 cat ocrmypdf.rb
-brew audit ocrmypdf.rb
+
+# brew audit crashes Travis
+#brew audit ocrmypdf.rb
 
 # Important: disable debug output so token is hidden
 set +x

Dockerfile

@@ -1,7 +1,6 @@
 # OCRmyPDF
 #
-# VERSION               4.4.2
+FROM      ubuntu:17.04
-FROM      ubuntu:16.10
 MAINTAINER James R. Barlow <jim@purplerock.ca>
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
@@ -20,6 +19,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   tesseract-ocr-spa \
   tesseract-ocr-deu
 
+ENV LANG=C.UTF-8
+
 RUN python3 -m venv --system-site-packages /appenv
 
 # This installs the latest binary wheel instead of the code in the current

Dockerfile.polyglot

@@ -1,8 +1,6 @@
 # OCRmyPDF polyglot
 #
-# VERSION               4.4.2
 FROM      jbarlow83/ocrmypdf:latest
-MAINTAINER James R. Barlow <jim@purplerock.ca>
 
 USER root
 

Dockerfile.tess4

@@ -1,7 +1,6 @@
 # OCRmyPDF
 #
-# VERSION               4.4.2
+FROM 	ubuntu:17.04
-FROM 	ubuntu:16.10
 MAINTAINER James R. Barlow <jim@purplerock.ca>
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
@@ -30,6 +29,8 @@ RUN apt-get update \
 		tesseract-ocr-rus \
 		tesseract-ocr-chi-sim 
 
+ENV LANG=C.UTF-8
+
 RUN python3 -m venv --system-site-packages /appenv
 
 # This installs the latest binary wheel instead of the code in the current

MANIFEST.in

@@ -7,12 +7,12 @@ include dev_requirements.txt
 include .git_archival.txt
 
 # docker
-include Dockerfile
+include Dockerfile*
-include Dockerfile.polyglot
 include .dockerignore
 
 # tests
 include pytest.ini
+include .coveragerc
 recursive-include tests *.jpg
 recursive-include tests *.png
 recursive-include tests *.pdf
@@ -38,11 +38,14 @@ recursive-exclude docs/_build *
 recursive-include ocrmypdf/data *
 include *.py
 exclude tasks.py
+exclude debug-runner.py
+recursive-exclude .travis *
+exclude .travis*
+
 
 # code
 recursive-include ocrmypdf *.py
 exclude ocrmypdf/lib/_leptonica.py
-include OCRmyPDF.sh
 include docker-wrapper.sh
 
 

README.rst

@@ -4,6 +4,12 @@ OCRmyPDF
 .. image:: https://travis-ci.org/jbarlow83/OCRmyPDF.svg?branch=master
     :target: https://travis-ci.org/jbarlow83/OCRmyPDF
 
+.. image:: https://img.shields.io/pypi/v/ocrmypdf.svg   
+    :target: https://pypi.org/project/ocrmypdf/
+
+.. image:: https://img.shields.io/docker/build/jbarlow83/ocrmypdf.svg
+    :target: https://hub.docker.com/r/jbarlow83/ocrmypdf/
+
 OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to
 be searched or copy-pasted.
 

docs/batch.rst

@@ -90,13 +90,83 @@ This user contributed script also provides an example of batch processing.
 API
 """
 
-OCRmyPDF is currently supported as a command line interface. Due to limitations in one of the libraries OCRmyPDF depends on, it is not yet usable as an API.
+OCRmyPDF is currently supported as a command line interface. This means that even if you are using OCRmyPDF in a Python script, you should run it in a subprocess rather importing the ocrmypdf package.
+
+The reason for this limitation is that the `ruffus <https://github.com/bunbun/ruffus/>`_ library that OCRmyPDF depends on is unfortunately not reentrant. OCRmyPDF works by defining each operation it does as a ruffus task that takes one or more files as input and generates one or more files as output. As such ruffus is fairly fundamental.
+
+(If you find individual functions implemented in OCRmyPDF useful (such as ``ocrmypdf.pdfinfo``), you can use these if you wish to.)
+
+
+Synology DiskStations
+"""""""""""""""""""""
+
+Synology DiskStations (Network Attached Storage devices) can run the Docker image of OCRmyPDF if the Synology `Docker package <https://www.synology.com/en-global/dsm/packages/Docker>`_ is installed. Attached is a script to address particular quirks of using OCRmyPDF on one of these devices.
+
+This is only possible for x86-based Synology products. Some Synology products use ARM or Power processors and do not support Docker. Further adjustments might be needed to deal with the Synology's relatively limited CPU and RAM.
+
+.. code-block:: python
+
+	#!/bin/env python3
+	# Contributed by github.com/Enantiomerie
+
+	# script needs 2 arguments
+	# 1. source dir with *.pdf - default is location of script
+	# 2. move dir where *.pdf and *_OCR.pdf are moved to
+
+	import logging
+	import os
+	import subprocess
+	import sys
+	import time
+	import shutil
+
+	script_dir = os.path.dirname(os.path.realpath(__file__))
+	timestamp = time.strftime("%Y-%m-%d-%H%M_")
+	log_file = script_dir + '/' + timestamp + 'ocrmypdf.log'
+	logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s', filename=log_file, filemode='w')
+
+	if len(sys.argv) > 1:
+	    start_dir = sys.argv[1]
+	else:
+	    start_dir = '.'
+
+	for dir_name, subdirs, file_list in os.walk(start_dir):
+	    logging.info('\n')
+	    logging.info(dir_name + '\n')
+	    os.chdir(dir_name)
+	    for filename in file_list:
+	        file_ext = os.path.splitext(filename)[1]
+	        if file_ext == '.pdf':
+	            full_path = dir_name + '/' + filename
+	            file_noext = os.path.splitext(filename)[0]
+	            timestamp_OCR = time.strftime("%Y-%m-%d-%H%M_OCR_")
+	            filename_OCR = timestamp_OCR + file_noext + '.pdf'
+	            docker_mount = dir_name + ':/home/docker'
+	# create string for pdf processing 
+	# diskstation needs a user:group docker:docker. find uid:gid of your diskstation docker:docker with id docker.
+	# use this uid:gid in -u flag
+	# rw rights for docker:docker at source dir are also necessary
+	# the script is processed as root user via chron 
+	            cmd = ['docker', 'run', '--rm', '-v', docker_mount, '-u="1030:65538"', 'jbarlow83/ocrmypdf', , '--deskew' , filename, filename_OCR]
+	            logging.info(cmd)
+	            proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+	            result = proc.stdout.read()
+	            logging.info(result)
+	            full_path_OCR = dir_name + '/' + filename_OCR
+	            os.chmod(full_path_OCR, 0o666)
+	            os.chmod(full_path, 0o666)
+	            full_path_OCR_archive = sys.argv[2]
+	            full_path_archive = sys.argv[2] + '/no_ocr'
+	            shutil.move(full_path_OCR,full_path_OCR_archive)
+	            shutil.move(full_path, full_path_archive)
+	logging.info('Finished.\n')
+
 
 
 Huge batch jobs
 """""""""""""""
 
-If you have thousands of files to work with, contact the author.
+If you have thousands of files to work with, contact the author. Consulting work related to OCRmyPDF helps fund this open source project and all inquiries are appreciated.
 
 
 Hot (watched) folders

docs/cookbook.rst

@@ -147,6 +147,16 @@ Image processing commands can be combined. The order in which options are given
     ocrmypdf --deskew --clean --rotate-pages input.pdf output.pdf
 
 
+Don't actually OCR my PDF
+"""""""""""""""""""""""""
+
+If you set ``--tesseract-timeout 0`` OCRmyPDF will apply its image processing without performing OCR, if all you want to is to apply image processing or PDF/A conversion.
+
+.. code-block:: bash
+
+    ocrmypdf --tesseract-timeout=0 --remove-background input.pdf output.pdf
+
+
 Improving OCR quality
 ---------------------
 

docs/installation.rst

@@ -214,15 +214,8 @@ follow these steps.
 Installing on Ubuntu 14.04 LTS
 ------------------------------
 
-Installing on Ubuntu 14.04 LTS (trusty) is more difficult than some other options, because of bugs in Python package installation and because OCRmyPDF depends on some packages newer than are available in the main distribution.
+Installing on Ubuntu 14.04 LTS (trusty) is more difficult than some other options,
-
+because it is older.
-Add new "apt" repositories needed for backports of Ghostscript 9.16, libav-11 (for unpaper 6.1) and Tesseract 4.00 (alpha). This will replace Ghostscript and Tesseract 3.x on your system. If you prefer to not modify your system in this matter, consider using a Docker container.
-
-.. code-block:: bash
-
-    sudo add-apt-repository ppa:vshn/ghostscript -y
-    sudo add-apt-repository ppa:heyarje/libav-11 -y
-    sudo add-apt-repository ppa:alex-p/tesseract-ocr
 
 Update apt-get:
 
@@ -239,39 +232,38 @@ Install system dependencies:
         zlib1g-dev \
         libjpeg-dev \
         libffi-dev \
-        libavformat56 libavcodec56 libavutil54 \
+        qpdf
+
+We will need backports of Ghostscript 9.16, libav-11 (for unpaper 6.1),
+Tesseract 4.00 (alpha), and Python 3.6. This will replace Ghostscript and
+Tesseract 3.x on your system. Python 3.6 will be installed alongside the system
+Python 3.
+
+If you prefer to not modify your system in this matter, consider using a Docker container.
+
+.. code-block:: bash
+
+    sudo add-apt-repository ppa:vshn/ghostscript -y
+    sudo add-apt-repository ppa:heyarje/libav-11 -y
+    sudo add-apt-repository ppa:alex-p/tesseract-ocr -y
+    sudo add-apt-repository ppa:jonathonf/python-3.6 -y
+
+    sudo apt-get update
+
+    sudo apt-get install \
+        python3.6 \
         ghostscript \
-        qpdf \
-        python3-pip \
-        python3-pil \
-        python3-pytest \
-        python3-reportlab \
-        python3-wheel \
-        python3-venv \
         tesseract-ocr \
-        tesseract-ocr-eng
+        tesseract-ocr-eng \
+        libavformat56 libavcodec56 libavutil54 \
+        wget
 
-If you wish install OCRmyPDF to the system Python, then install as follows (note this installs new packages
+Now we need to install ``pip`` and let it install ocrmypdf:
-into your system Python, which could interfere with other programs):
 
 .. code-block:: bash
 
-    sudo pip3 install ocrmypdf
+    wget -O - -o /dev/null https://bootstrap.pypa.io/get-pip.py | python3.6
-
+    pip3.6 install ocrmypdf
-If you wish to install OCRmyPDF to a virtual environment to isolate the system Python, you can
-follow these steps.  This includes a workaround `for a known, unresolved issue in Ubuntu 14.04's ensurepip
-package <http://www.thefourtheye.in/2014/12/Python-venv-problem-with-ensurepip-in-Ubuntu.html>`_:
-
-.. code-block:: bash
-
-    sudo apt-get install python3-venv
-    python3 -m venv venv-ocrmypdf --without-pip
-    source venv-ocrmypdf/bin/activate
-    wget -O - -o /dev/null https://bootstrap.pypa.io/get-pip.py | python
-    deactivate
-    python3 -m venv --system-site-packages venv-ocrmypdf
-    source venv-ocrmypdf/bin/activate
-    pip install ocrmypdf
 
 These installation instructions omit the optional dependency ``unpaper``, which is only available at version 0.4.2 in Ubuntu 14.04. The author could not find a backport of ``unpaper``, and created a .deb package to do the job of installing unpaper 6.1 (for x86 64-bit only):
 

docs/languages.rst

@@ -33,6 +33,34 @@ Docker users
 
 Users of the Docker image may use the alternative :ref:`"polyglot" container <docker-polyglot>` which includes all languages.
 
+Adding individual language packs to a Docker image
+""""""""""""""""""""""""""""""""""""""""""""""""""
+
+If you wish to add a single language pack, you could do the following:
+
+* Download the desired ``.trainedata`` file from the `tessdata <https://github.com/tesseract-ocr/tessdata>`_ repository. Let's use Hebrew in this example (``heb.traineddata``)
+
+* Copy the file to ``/home/user/downloads/heb.traineddata``.
+
+* Create a new container based on the ocrmypdf-tess4 image and jump into it with a terminal:
+
+.. code-block:: bash
+
+	host$ docker run  -v /home/user/downloads:/home/docker -it --entrypoint /bin/bash ocrmypdf-tess4
+
+* Put the file where Tesseract expects it:
+
+.. code-block:: bash
+
+	docker$ cp /home/docker/heb.traineddata /usr/share/tesseract-ocr/tessdata
+
+* Note the container id, and save it as a new image (in this example, ``ocrmypdf-tess4-heb``)
+
+.. code-block:: bash
+
+    host$ docker commit <container_id> ocrmypdf-tess4-heb
+
+
 Known limitations
 -----------------
 

docs/release_notes.rst

@@ -5,6 +5,34 @@ OCRmyPDF uses `semantic versioning <http://semver.org/>`_ for its command line i
 
 The OCRmyPDF package itself does not contain a public API, although it is fairly stable and breaking changes are usually timed with a major release. A future release will clearly define the stable public API.
 
+v5.4
+------
+
+-   Change wording of a deprecation warning to improve clarity
+-   Added option to generate PDF/A-1b output if desired (``--output-type pdfa-1``); default remains PDF/A-2b generation
+-   Update documentation
+
+
+v5.3.3
+------
+
+-   Fixed missing error message that should occur when trying to force ``--pdf-renderer sandwich`` on old versions of Tesseract
+-   Update copyright information in test files
+-   Set system ``LANG`` to UTF-8 in Dockerfiles to avoid UTF-8 encoding errors
+
+
+v5.3.2
+------
+
+-   Fixed a broken test case related to language packs
+
+
+v5.3.1
+------
+
+-   Fixed wrong return code given for missing Tesseract language packs
+-   Fixed "brew audit" crashing on Travis when trying to auto-brew
+
 
 v5.3
 ----

ocrmypdf/__main__.py

@@ -1,29 +1,25 @@
 #!/usr/bin/env python3
 # © 2015-17 James R. Barlow: github.com/jbarlow83
 
-from contextlib import suppress
 from tempfile import mkdtemp
 from collections.abc import Sequence
 import sys
 import os
 import re
-import shutil
 import warnings
 import multiprocessing
 import atexit
 import textwrap
-import img2pdf
 import logging
 import argparse
 
 import PyPDF2 as pypdf
-from PIL import Image
 
 import ruffus.ruffus_exceptions as ruffus_exceptions
 import ruffus.cmdline as cmdline
 import ruffus.proxy_logger as proxy_logger
 
-from .pipeline import JobContext, JobContextManager, re_symlink, \
+from .pipeline import JobContext, JobContextManager, \
     cleanup_working_files, build_pipeline
 from .pdfa import file_claims_pdfa
 from .helpers import is_iterable_notstr, re_symlink, is_file_writable
@@ -134,12 +130,15 @@ parser.add_argument(
     '--image-dpi', metavar='DPI', type=int,
     help="For input image instead of PDF, use this DPI instead of file's.")
 parser.add_argument(
-    '--output-type', choices=['pdfa', 'pdf'], default='pdfa',
+    '--output-type', choices=['pdfa', 'pdf', 'pdfa-1', 'pdfa-2'], 
+    default='pdfa',
     help="Choose output type. 'pdfa' creates a PDF/A-2b compliant file for "
          "long term archiving (default, recommended) but may not suitable "
          "for users who want their file altered as little as possible. 'pdfa' "
          "also has problems with full Unicode text. 'pdf' attempts to "
-         "preserve file contents as much as possible.")
+         "preserve file contents as much as possible. 'pdf-a1' creates a "
+         "PDF/A1-b file. 'pdf-a2' is equivalent to 'pdfa'."
+         )
 
 # Use null string '\0' as sentinel to indicate the user supplied no argument,
 # since that is the only invalid character for filepaths on all platforms
@@ -322,7 +321,7 @@ def check_options_languages(options, _log):
             "data for the following requested languages: \n")
         for lang in (set(options.language) - tesseract.languages()):
             msg += lang + '\n'
-        raise argparse.ArgumentError(None, msg)
+        raise MissingDependencyError(msg)
 
 
 def check_options_output(options, log):
@@ -332,19 +331,29 @@ def check_options_output(options, log):
         else:
             options.pdf_renderer = 'hocr'
 
+    if options.pdf_renderer == 'sandwich' and not tesseract.has_textonly_pdf():
+        raise MissingDependencyError(
+            "The 'sandwich' renderer requires Tesseract 3.05.01 or newer; "
+            "or Tesseract 4.00 alpha newer than February 2017.")
+
     if options.pdf_renderer == 'tess4':
         log.warning("The 'tess4' PDF renderer has been renamed to 'sandwich'. "
                     "Please use --pdf-renderer=sandwich.")
         options.pdf_renderer = 'sandwich'
 
     if options.pdf_renderer == 'tesseract':
-        log.warning("The 'tesseract' PDF renderer is deprecated.")
         if tesseract.version() < '3.05' and options.output_type == 'pdfa':
             log.warning(
                 "For best results use --pdf-renderer=tesseract "
                 "--output-type=pdf to disable PDF/A generation via "
                 "Ghostscript, which is known to corrupt the OCR text of "
                 "some PDFs produced your version of Tesseract.")
+        elif tesseract.has_textonly_pdf():
+            log.warning(
+                "The argument --pdf-renderer=tesseract provides support for "
+                "versions of tesseract older than your version. For best "
+                "results omit this argument and let OCRmyPDF choose the "
+                "best available renderer.")
 
     if options.debug_rendering and options.pdf_renderer != 'hocr':
         log.info(
@@ -417,7 +426,7 @@ def check_options_ocr_behavior(options, log):
 def check_options_advanced(options, log):
     if tesseract.v4():
         log.info(
-            "Tesseract v4.x.alpha found. OCRmyPDF support is experimental.")
+            "Tesseract v4.x.alpha found.")
     if options.tesseract_oem and not tesseract.v4():
         log.warning(
             "--tesseract-oem requires Tesseract 4.x -- argument ignored")
@@ -435,9 +444,9 @@ def check_options_advanced(options, log):
 
 def check_options_metadata(options, log):
     import unicodedata
-    metadata = [options.title, options.author, options.keywords,
+    docinfo = [options.title, options.author, options.keywords,
                options.subject]
-    for s in (m for m in metadata if m):
+    for s in (m for m in docinfo if m):
         for c in s:
             if unicodedata.category(c) == 'Co' or ord(c) >= 0x10000:
                 raise ValueError(
@@ -519,31 +528,31 @@ def cleanup_ruffus_error_message(msg):
 def do_ruffus_exception(ruffus_five_tuple, options, log):
     """Replace the elaborate ruffus stack trace with a user friendly
     description of the error message that occurred."""
+    exit_code = None
 
     task_name, job_name, exc_name, exc_value, exc_stack = ruffus_five_tuple
     if exc_name == 'builtins.SystemExit':
         match = re.search(r"\.(.+?)\)", exc_value)
         exit_code_name = match.groups()[0]
         exit_code = getattr(ExitCode, exit_code_name, 'other_error')        
-        return exit_code
     elif exc_name == 'ruffus.ruffus_exceptions.MissingInputFileError':
         log.error(cleanup_ruffus_error_message(exc_value))
-        return ExitCode.input_file
+        exit_code = ExitCode.input_file
     elif exc_name == 'builtins.TypeError':
         # Even though repair_pdf will fail, ruffus will still try
         # to call split_pages with no input files, likely due to a bug
         if task_name == 'split_pages':
             log.error("Input file '{0}' is not a valid PDF".format(
                 options.input_file))
-            return ExitCode.input_file
+            exit_code = ExitCode.input_file
     elif exc_name == 'builtins.KeyboardInterrupt':
         log.error("Interrupted by user")
-        return ExitCode.ctrl_c
+        exit_code = ExitCode.ctrl_c
     elif exc_name == 'subprocess.CalledProcessError':
         # It's up to the subprocess handler to report something useful
         msg = "Error occurred while running this command:"
         log.error(msg + '\n' + exc_value)
-        return ExitCode.child_process_error
+        exit_code = ExitCode.child_process_error
     elif exc_name == 'ocrmypdf.exceptions.PdfMergeFailedError':
         log.error(textwrap.dedent("""\
             Failed to merge PDF image layer with OCR layer
@@ -554,11 +563,11 @@ def do_ruffus_exception(ruffus_five_tuple, options, log):
             Try using
                 ocrmypdf --pdf-renderer tesseract  [..other args..]
             """))
-        return ExitCode.input_file
+        exit_code = ExitCode.input_file
     elif exc_name.startswith('ocrmypdf.exceptions.'):
         base_exc_name = exc_name.replace('ocrmypdf.exceptions.', '')
         exc_class = getattr(ocrmypdf_exceptions, base_exc_name)
-        return exc_class.exit_code
+        exit_code = exc_class.exit_code
     elif exc_name == 'PyPDF2.utils.PdfReadError' and \
             'not been decrypted' in exc_value:
         log.error(textwrap.dedent("""\
@@ -571,7 +580,10 @@ def do_ruffus_exception(ruffus_five_tuple, options, log):
             (Only algorithms "R = 1" and "R = 2" are supported.)
 
             """))
-        return ExitCode.encrypted_pdf
+        exit_code = ExitCode.encrypted_pdf
+
+    if exit_code is not None:
+        return exit_code
 
     if not options.verbose:
         log.error(exc_stack)
@@ -637,6 +649,20 @@ def check_closed_streams(options):
     return True
 
 
+def log_page_orientations(pdfinfo, _log):
+    direction = {0: 'n', 90: 'e',
+                180: 's', 270: 'w'}
+    orientations = []
+    for n, page in enumerate(pdfinfo):
+        angle = pdfinfo[n].rotation or 0
+        if angle != 0:
+            orientations.append('{0}{1}'.format(
+                n + 1,
+                direction.get(angle, '')))
+    if orientations:
+        _log.info('Page orientations detected: ' + ' '.join(orientations))
+
+
 def run_pipeline():
     options = parser.parse_args()
     options.verbose_abbreviated_path = 1
@@ -726,7 +752,6 @@ def run_pipeline():
             _log.error("Unexpected ruffus exception: " + str(e))
             _log.error(repr(e))
             return ExitCode.other_error
-        else:
         return exitcode
     except ExitCodeException as e:
         return e.exit_code
@@ -737,7 +762,7 @@ def run_pipeline():
     if options.flowchart:
         _log.info("Flowchart saved to {}".format(options.flowchart))
     elif options.output_file != '-':
-        if options.output_type == 'pdfa':
+        if options.output_type.startswith('pdfa'):
             pdfa_info = file_claims_pdfa(options.output_file)
             if pdfa_info['pass']:
                 msg = 'Output file is a {} (as expected)'
@@ -745,7 +770,6 @@ def run_pipeline():
             else:
                 msg = 'Output file is okay but is not PDF/A (seems to be {})'
                 _log.warning(msg.format(pdfa_info['conformance']))
-
                 return ExitCode.invalid_output_pdf
         if not qpdf.check(options.output_file, _log):
             _log.warning('Output file: The generated PDF is INVALID')
@@ -757,17 +781,8 @@ def run_pipeline():
     if options.verbose:
         from pprint import pformat
         _log.debug(pformat(pdfinfo))
-    direction = {0: 'n', 90: 'e',
+
-                 180: 's', 270: 'w'}
+    log_page_orientations(pdfinfo, _log)
-    orientations = []
-    for n, page in enumerate(pdfinfo):
-        angle = pdfinfo[n].rotation or 0
-        if angle != 0:
-            orientations.append('{0}{1}'.format(
-                n + 1,
-                direction.get(angle, '')))
-    if orientations:
-        _log.info('Page orientations detected: ' + ' '.join(orientations))
 
     return ExitCode.ok
 

ocrmypdf/exceptions.py

@@ -20,7 +20,7 @@ class ExitCode(IntEnum):
 
 
 class ExitCodeException(Exception):
-    pass
+    exit_code = ExitCode.other_error
 
 
 class PdfMergeFailedError(ExitCodeException):

ocrmypdf/exec/__init__.py

@@ -7,5 +7,6 @@ import os
 
 
 def get_program(name):
+    "Check environment variables for overrides to this program"
     envvar = 'OCRMYPDF_' + name.upper()
     return os.environ.get(envvar, name)

ocrmypdf/exec/ghostscript.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# © 2015 James R. Barlow: github.com/jbarlow83
+# © 2017 James R. Barlow: github.com/jbarlow83
 
 from tempfile import NamedTemporaryFile
 from subprocess import run, PIPE, STDOUT, CalledProcessError
@@ -7,9 +7,9 @@ from shutil import copy
 from functools import lru_cache
 import re
 import sys
-from . import get_program
-from ..exceptions import SubprocessOutputError
 from PIL import Image
+from . import get_program
+from ..exceptions import SubprocessOutputError, MissingDependencyError
 from ..helpers import fspath
 
 
@@ -20,15 +20,16 @@ def version():
         '--version'
     ]
     try:
-        version = check_output(
+        proc = run(
                 args_gs, close_fds=True, universal_newlines=True,
-                stderr=STDOUT)
+                stdout=PIPE, stderr=STDOUT, check=True)
+        ver = proc.stdout
     except CalledProcessError as e:
         print("Could not find Ghostscript executable on system PATH.",
               file=sys.stderr)
         raise MissingDependencyError from e
 
-    return version.strip()
+    return ver.strip()
 
 
 def _gs_error_reported(stream):
@@ -103,7 +104,7 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
 
 
 def generate_pdfa(pdf_pages, output_file, compression, log,
-                  threads=1, pdf_version='1.5'):
+                  threads=1, pdf_version='1.5', pdfa_part='2'):
     compression_args = []
     if compression == 'jpeg':
         compression_args = [
@@ -139,7 +140,7 @@ def generate_pdfa(pdf_pages, output_file, compression, log,
             "-sProcessColorModel=DeviceRGB"
         ] + compression_args + [
             "-dJPEGQ=95",
-            "-dPDFA=2",
+            "-dPDFA=" + pdfa_part,
             "-dPDFACompatibilityPolicy=1",
             "-sOutputFile=" + gs_pdf.name,
         ]

ocrmypdf/exec/qpdf.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# © 2015 James R. Barlow: github.com/jbarlow83
+# © 2017 James R. Barlow: github.com/jbarlow83
 
 from subprocess import CalledProcessError, STDOUT, PIPE, run, check_output
 from functools import lru_cache

ocrmypdf/exec/tesseract.py

@@ -1,21 +1,20 @@
 #!/usr/bin/env python3
-# © 2015 James R. Barlow: github.com/jbarlow83
+# © 2017 James R. Barlow: github.com/jbarlow83
 
 import sys
 import os
 import re
 import shutil
 from functools import lru_cache
-from ..exceptions import MissingDependencyError, TesseractConfigError
-from ..helpers import page_number
-from . import get_program
 from collections import namedtuple
 from textwrap import dedent
 import PyPDF2 as pypdf
+from subprocess import PIPE, CalledProcessError, \
+    TimeoutExpired, check_output, STDOUT
 
-from subprocess import Popen, PIPE, CalledProcessError, \
+from ..exceptions import MissingDependencyError, TesseractConfigError
-    TimeoutExpired, check_output, STDOUT, DEVNULL
+from ..helpers import page_number
-
+from . import get_program
 
 OrientationConfidence = namedtuple(
     'OrientationConfidence',
@@ -60,7 +59,7 @@ def version():
 
 def v4():
     "Is this Tesseract v4.0?"
-    return (version() >= '4')
+    return version() >= '4'
 
 
 @lru_cache(maxsize=1)
@@ -74,6 +73,7 @@ def has_textonly_pdf():
         get_program('tesseract'),
         '--print-parameters'
     ]
+    params = ''
     try:
         params = check_output(
                 args_tess, close_fds=True, universal_newlines=True,
@@ -113,12 +113,12 @@ def languages():
     return set(lang.strip() for lang in langs.splitlines()[1:])
 
 
-def tess_base_args(languages, engine_mode):
+def tess_base_args(langs, engine_mode):
     args = [
         get_program('tesseract'),
     ]
-    if languages:
+    if langs:
-        args.extend(['-l', '+'.join(languages)])
+        args.extend(['-l', '+'.join(langs)])
     if engine_mode is not None and v4():
         args.extend(['--oem', str(engine_mode)])
     return args

ocrmypdf/exec/unpaper.py

@@ -3,7 +3,7 @@
 # unpaper documentation:
 # https://github.com/Flameeyes/unpaper/blob/master/doc/basic-concepts.md
 
-from subprocess import CalledProcessError, STDOUT, check_output, check_call
+from subprocess import CalledProcessError, STDOUT, check_output
 from tempfile import NamedTemporaryFile
 import sys
 import os
@@ -18,10 +18,10 @@ def version():
         get_program('unpaper'),
         '--version'
     ]
-    version = check_output(
+    ver = check_output(
         args_unpaper, close_fds=True, universal_newlines=True,
         stderr=STDOUT, timeout=5)
-    return version.strip()
+    return ver.strip()
 
 
 try:

ocrmypdf/lib/compile_leptonica.py

@@ -183,10 +183,39 @@ pixColorMagnitude(PIX     *pixs,
                   l_int32  bwhite,
                   l_int32  type);
 
+PIX *
+pixMaskOverColorPixels(PIX     *pixs,
+                       l_int32  threshdiff,
+                       l_int32  mindist);
+
+l_int32
+pixGetAverageMaskedRGB(PIX        *pixs,
+                       PIX        *pixm,
+                       l_int32     x,
+                       l_int32     y,
+                       l_int32     factor,
+                       l_int32     type,
+                       l_float32  *prval,
+                       l_float32  *pgval,
+                       l_float32  *pbval);
+
+PIX * 
+pixGlobalNormRGB(PIX * 	pixd,
+                 PIX * 	pixs,
+                 l_int32 	rval,
+                 l_int32 	gval,
+                 l_int32 	bval,
+                 l_int32 	mapval); 
+
+PIX *
+pixInvert(PIX * pixd,
+          PIX * pixs);
+
 void                 
 boxDestroy(BOX  **pbox);
 
-void lept_free(void *ptr);
+void
+lept_free(void *ptr);  
 """)
 
 

ocrmypdf/pipeline.py

@@ -2,19 +2,10 @@
 # © 2016 James R. Barlow: github.com/jbarlow83
 
 from contextlib import suppress
-from tempfile import mkdtemp
-from functools import partial
 import sys
 import os
-import re
 import shutil
-import warnings
-import multiprocessing
-import atexit
-import textwrap
 import img2pdf
-import logging
-import argparse
 
 import PyPDF2 as pypdf
 from PIL import Image
@@ -23,7 +14,7 @@ from ruffus import formatter, regex, Pipeline, suffix
 
 from .hocrtransform import HocrTransform
 from .pdfinfo import PdfInfo, Encoding, Colorspace
-from .pdfa import generate_pdfa_ps, file_claims_pdfa
+from .pdfa import generate_pdfa_ps
 from .helpers import re_symlink, is_iterable_notstr, page_number
 from .exec import ghostscript, tesseract, qpdf
 from .exceptions import *
@@ -51,6 +42,8 @@ class JobContext:
 
     def __init__(self):
         self.pdfinfo = None
+        self.options = None
+        self.work_folder = None
 
     def generate_pdfinfo(self, infile):
         self.pdfinfo = PdfInfo(infile)
@@ -189,7 +182,8 @@ def repair_pdf(
     pdfinfo = PdfInfo(output_file)
 
     if pdfinfo.has_userunit and options.output_type == 'pdfa':
-        log.error("This input file uses a PDF feature that is not supported "
+        log.error(
+            "This input file uses a PDF feature that is not supported "
             "by Ghostscript, so you cannot use --output-type=pdfa for this "
             "file. (Specifically, it uses the PDF-1.6 /UserUnit feature to "
             "support very large or small page sizes, and Ghostscript cannot "
@@ -706,7 +700,7 @@ def combine_layers(
 
     if rotation != 0:
         log.info("{0:4d}: rotating image layer {1} degrees".format(
-            page_number(image), rotation, tx, ty))
+            page_number(image), rotation))
 
     try:
         page_text.mergeRotatedScaledTranslatedPage(
@@ -885,7 +879,8 @@ def merge_pages_ghostscript(
         output_file=output_file,
         compression=options.pdfa_image_compression,
         log=log,
-        threads=options.jobs or 1)
+        threads=options.jobs or 1,
+        pdfa_part=('1' if options.output_type == 'pdfa-1' else '2'))
 
 
 def merge_pages_qpdf(
@@ -936,7 +931,6 @@ def merge_sidecars(
         output_file,
         log,
         context):
-    options = context.get_options()
     pdfinfo = context.get_pdfinfo()
 
     txt_files = [None] * len(pdfinfo)
@@ -947,15 +941,24 @@ def merge_sidecars(
             txt_files[idx] = infile
 
     def write_pages(stream):
-        for page_number, txt_file in enumerate(txt_files):
+        for page_num, txt_file in enumerate(txt_files):
-            if page_number != 0:
+            if page_num != 0:
                 stream.write('\f')  # Form feed between pages
             if txt_file:
                 with open(txt_file, 'r') as in_:
+                    txt = in_.read()
+                    # Tesseract v4 alpha started adding form feeds in 
+                    # commit aa6eb6b
+                    # No obvious way to detect what binaries will do this, so
+                    # for consistency just ignore its form feeds and insert our
+                    # own
+                    if txt.endswith('\f'):
+                        stream.write(in_.read()[:-1])
+                    else:
                         stream.write(in_.read())
             else:
                 stream.write('[OCR skipped on page {}]'.format(
-                        page_number + 1))
+                        page_num + 1))
 
     if output_file == '-':
         write_pages(sys.stdout)
@@ -1139,7 +1142,8 @@ def build_pipeline(options, work_folder, log, context):
         output=os.path.join(work_folder, r'\1.rendered.pdf'),
         extras=[log, context])
     task_combine_layers.graphviz(fillcolor='"#00cc66"')
-    task_combine_layers.active_if(options.pdf_renderer == 'hocr' or options.pdf_renderer == 'sandwich')
+    task_combine_layers.active_if(options.pdf_renderer == 'hocr' or 
+                                  options.pdf_renderer == 'sandwich')
 
     # Tesseract OCR+PDF
     task_ocr_tesseract_and_render_pdf = main_pipeline.collate(
@@ -1161,7 +1165,7 @@ def build_pipeline(options, work_folder, log, context):
         filter=formatter(r'\.repaired\.pdf'),
         output=os.path.join(work_folder, 'pdfa.ps'),
         extras=[log, context])
-    task_generate_postscript_stub.active_if(options.output_type == 'pdfa')
+    task_generate_postscript_stub.active_if(options.output_type.startswith('pdfa'))
 
 
     # Bypass valve
@@ -1183,7 +1187,7 @@ def build_pipeline(options, work_folder, log, context):
                task_generate_postscript_stub],
         output=os.path.join(work_folder, 'merged.pdf'),
         extras=[log, context])
-    task_merge_pages_ghostscript.active_if(options.output_type == 'pdfa')
+    task_merge_pages_ghostscript.active_if(options.output_type.startswith('pdfa'))
 
     task_merge_pages_qpdf = main_pipeline.merge(
         task_func=merge_pages_qpdf,

setup.py

@@ -185,9 +185,15 @@ if 'upload' in sys.argv[1:]:
 
 tests_require = open('test_requirements.txt').read().splitlines()
 
+
+def readme():
+    with open('README.rst') as f:
+        return f.read()
+
 setup(
     name='ocrmypdf',
     description='OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched',
+    long_description=readme(),
     url='https://github.com/jbarlow83/OCRmyPDF',
     author='James R. Barlow',
     author_email='jim@purplerock.ca',

tests/resources/README.rst

@@ -57,7 +57,7 @@ under the terms of the license in LICENSE.rst.
         - test image with 200 x 100 DPI resolution
     *   - blank.pdf
         - @jbarlow83
-        - blank PDF
+        - blank PDF generated by Adobe Illustrator CC 17, containing a lot of application-specific metadata/bloat
     *   - cmyk.pdf
         - @jbarlow83
         - a CMYK image created in Photoshop
@@ -88,14 +88,18 @@ under the terms of the license in LICENSE.rst.
     *   - overlay.pdf
         - @maxandersen
         - PDF file generated by PDFPen pro that triggered content stream parse errors
-    *   - no_conentes.pdf
+    *   - no_contents.pdf
         - @jbarlow83
         - synthetic PDF with a blank page that has no /Contents entry
+    *   - crom.png
+        - @jbarlow83
+        - test for non-dictionary words
+
 
 Assemblies
 ==========
 
-These test resources are assemblies from other previously mentioned files, released under the same license terms as their input files.
+These test resources are assemblies or derivatives from other previously mentioned files, released under the same license terms as their input files.
 
 - cardinal.pdf (four cardinal directions, baked-in rotated copies of LinnSequencer.jpg)
 - ccitt.pdf (LinnSequencer.jpg, converted to CCITT encoding)
@@ -107,6 +111,9 @@ These test resources are assemblies from other previously mentioned files, relea
 - rotated_skew.pdf (a /Rotate'd and skewed document from LinnSequencer.jpg)
 - skew.pdf (from LinnSequencer.jpg, skew simulated by adjusting the transformation matrix)
 - skew-encrypted.pdf (skew.pdf with encryption - access supported by PyPDF2)
+- baiona_gray.png (from baiona.png)
+- poster.pdf (from LinnSequencer.jpg)
+
 
 
 .. _`Wikimedia: LinnSequencer`: https://upload.wikimedia.org/wikipedia/en/b/b7/LinnSequencer_hardware_MIDI_sequencer_brochure_page_2_300dpi.jpg

tests/spoof/tesseract_big_image_error.py

@@ -4,9 +4,9 @@
 import sys
 
 
-VERSION_STRING = '''tesseract 3.04.00
+VERSION_STRING = '''tesseract 3.05.01
- leptonica-1.72
+ leptonica-1.74.4
-  libjpeg 8d : libpng 1.6.19 : libtiff 4.0.6 : zlib 1.2.5
+  libjpeg 9b : libpng 1.6.32 : libtiff 4.0.8 : zlib 1.2.8
 SPOOFED: return error claiming image too big
 '''
 
@@ -20,10 +20,10 @@ def main():
         print(VERSION_STRING, file=sys.stderr)
         sys.exit(0)
     elif sys.argv[1] == '--list-langs':
-        print('List of available languages (1):\neng', file=sys.stderr)
+        print('List of available languages (1):\neng\n', file=sys.stderr)
         sys.exit(0)
     elif sys.argv[1] == '--print-parameters':
-        print('A parameter list would go here', file=sys.stderr)
+        print('A parameter list would go here\ntextonly_pdf 0\n', file=sys.stderr)
         sys.exit(0)
     elif sys.argv[-2] == 'hocr':
         print("Image too large: (33830, 14959)\n"

tests/spoof/tesseract_crash.py

@@ -5,9 +5,9 @@ import os
 import signal
 
 
-VERSION_STRING = '''tesseract 3.04.00
+VERSION_STRING = '''tesseract 3.05.01
- leptonica-1.72
+ leptonica-1.74.4
-  libjpeg 8d : libpng 1.6.19 : libtiff 4.0.6 : zlib 1.2.5
+  libjpeg 9b : libpng 1.6.32 : libtiff 4.0.8 : zlib 1.2.8
 SPOOFED: CRASH ON OCR or -psm 0
 '''
 
@@ -30,7 +30,8 @@ def main():
         print('List of available languages (1):\neng', file=sys.stderr)
         sys.exit(0)
     elif sys.argv[1] == '--print-parameters':
-        print('List of parameters would go here', file=sys.stderr)
+        print('A parameter list would go here\ntextonly_pdf 0\n',
+              file=sys.stderr)
         sys.exit(0)
     elif sys.argv[-2] == 'hocr':
         print("KABOOM! Tesseract failed for some reason", file=sys.stderr)

tests/test_main.py

@@ -404,9 +404,13 @@ def test_force_ocr_on_pdf_with_no_images(spoof_tesseract_crash, resources,
 @pytest.mark.skipif(
     pytest.helpers.is_macos() and pytest.helpers.running_in_travis(),
     reason="takes too long to install language packs in Travis macOS homebrew")
-def test_french(spoof_tesseract_cache, resources, outpdf):
+def test_french(spoof_tesseract_cache, resources, outdir):
+    # Produce a sidecar too - implicit test that system locale is set up
+    # properly
+    sidecar = outdir / 'francais.txt'
     p, out, err = run_ocrmypdf(
-        resources / 'francais.pdf', outpdf, '-l', 'fra',
+        resources / 'francais.pdf', outdir / 'francais.pdf', '-l', 'fra',
+        '--sidecar', sidecar,
         env=spoof_tesseract_cache)
     print(os.environ)
     assert p.returncode == ExitCode.ok, \
@@ -416,7 +420,7 @@ def test_french(spoof_tesseract_cache, resources, outpdf):
 def test_klingon(resources, outpdf):
     p, out, err = run_ocrmypdf(
         resources / 'francais.pdf', outpdf, '-l', 'klz')
-    assert p.returncode == ExitCode.bad_args
+    assert p.returncode == ExitCode.missing_dependency
 
 
 def test_missing_docinfo(spoof_tesseract_noop, resources, outpdf):
@@ -978,3 +982,14 @@ def test_sidecar_pagecount(spoof_tesseract_cache, resources, outpdf):
     # formfeeds is the page count less one
     assert ocr_text.count('\f') == num_pages - 1, \
         "Sidecar page count does not match PDF page count"
+
+
+def test_pdfa_1(spoof_tesseract_cache, resources, outpdf):
+    check_ocrmypdf(
+        resources / 'ccitt.pdf', outpdf,
+        '--output-type', 'pdfa-1',
+        env=spoof_tesseract_cache
+    )
+
+    pdfa_info = file_claims_pdfa(outpdf)
+    assert pdfa_info['conformance'] == 'PDF/A-1B'