v5.4.1 release notes

Suggested by amitdo in https://github.com/tesseract-ocr/tesseract/issues/1167

.coveragerc

@@ -0,0 +1,16 @@
+# Coverage isn't really compatible with subprocesses so results are unreliable
+
+[run]
+branch = True
+#concurrency = multiprocessing
+source = ocrmypdf/
+
+[report]
+exclude_lines =
+	pragma: no cover
+	def __repr__
+	raise AssertionError
+	raise NotImplementedError
+	if 0:
+	if False:
+	if __name__ == .__main__.:

.gitignore

@@ -38,4 +38,7 @@ tests/cache/
 tests/output/
 tests/resources/private/
 tmp/
-pdfbox-app*.jar
+pdfbox-app*.jar
+.vscode/
+IDEAS
+_Dockerfile.local

.travis.yml

@@ -85,6 +85,7 @@ deploy:
   script: /usr/bin/true
   on:
     branch: master
+    tags: true
     condition: $TRAVIS_OS_NAME == "osx"
 
 after_deploy: |

Dockerfile

@@ -1,7 +1,6 @@
 # OCRmyPDF
 #
-# VERSION               4.4.2
-FROM      ubuntu:16.10
+FROM      ubuntu:17.04
 MAINTAINER James R. Barlow <jim@purplerock.ca>
 
 RUN apt-get update && apt-get install -y --no-install-recommends \

Dockerfile.polyglot

@@ -1,8 +1,6 @@
 # OCRmyPDF polyglot
 #
-# VERSION               4.4.2
 FROM      jbarlow83/ocrmypdf:latest
-MAINTAINER James R. Barlow <jim@purplerock.ca>
 
 USER root
 

Dockerfile.tess4

@@ -1,7 +1,6 @@
 # OCRmyPDF
 #
-# VERSION               4.4.2
-FROM 	ubuntu:16.10
+FROM 	ubuntu:17.04
 MAINTAINER James R. Barlow <jim@purplerock.ca>
 
 RUN apt-get update && apt-get install -y --no-install-recommends \

MANIFEST.in

@@ -7,12 +7,12 @@ include dev_requirements.txt
 include .git_archival.txt
 
 # docker
-include Dockerfile
-include Dockerfile.polyglot
+include Dockerfile*
 include .dockerignore
 
 # tests
 include pytest.ini
+include .coveragerc
 recursive-include tests *.jpg
 recursive-include tests *.png
 recursive-include tests *.pdf
@@ -38,11 +38,14 @@ recursive-exclude docs/_build *
 recursive-include ocrmypdf/data *
 include *.py
 exclude tasks.py
+exclude debug-runner.py
+recursive-exclude .travis *
+exclude .travis*
+
 
 # code
 recursive-include ocrmypdf *.py
 exclude ocrmypdf/lib/_leptonica.py
-include OCRmyPDF.sh
 include docker-wrapper.sh
 
 

docs/batch.rst

@@ -90,13 +90,83 @@ This user contributed script also provides an example of batch processing.
 API
 """
 
-OCRmyPDF is currently supported as a command line interface. Due to limitations in one of the libraries OCRmyPDF depends on, it is not yet usable as an API.
+OCRmyPDF is currently supported as a command line interface. This means that even if you are using OCRmyPDF in a Python script, you should run it in a subprocess rather importing the ocrmypdf package.
+
+The reason for this limitation is that the `ruffus <https://github.com/bunbun/ruffus/>`_ library that OCRmyPDF depends on is unfortunately not reentrant. OCRmyPDF works by defining each operation it does as a ruffus task that takes one or more files as input and generates one or more files as output. As such ruffus is fairly fundamental.
+
+(If you find individual functions implemented in OCRmyPDF useful (such as ``ocrmypdf.pdfinfo``), you can use these if you wish to.)
+
+
+Synology DiskStations
+"""""""""""""""""""""
+
+Synology DiskStations (Network Attached Storage devices) can run the Docker image of OCRmyPDF if the Synology `Docker package <https://www.synology.com/en-global/dsm/packages/Docker>`_ is installed. Attached is a script to address particular quirks of using OCRmyPDF on one of these devices.
+
+This is only possible for x86-based Synology products. Some Synology products use ARM or Power processors and do not support Docker. Further adjustments might be needed to deal with the Synology's relatively limited CPU and RAM.
+
+.. code-block:: python
+
+	#!/bin/env python3
+	# Contributed by github.com/Enantiomerie
+
+	# script needs 2 arguments
+	# 1. source dir with *.pdf - default is location of script
+	# 2. move dir where *.pdf and *_OCR.pdf are moved to
+
+	import logging
+	import os
+	import subprocess
+	import sys
+	import time
+	import shutil
+
+	script_dir = os.path.dirname(os.path.realpath(__file__))
+	timestamp = time.strftime("%Y-%m-%d-%H%M_")
+	log_file = script_dir + '/' + timestamp + 'ocrmypdf.log'
+	logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s', filename=log_file, filemode='w')
+
+	if len(sys.argv) > 1:
+	    start_dir = sys.argv[1]
+	else:
+	    start_dir = '.'
+
+	for dir_name, subdirs, file_list in os.walk(start_dir):
+	    logging.info('\n')
+	    logging.info(dir_name + '\n')
+	    os.chdir(dir_name)
+	    for filename in file_list:
+	        file_ext = os.path.splitext(filename)[1]
+	        if file_ext == '.pdf':
+	            full_path = dir_name + '/' + filename
+	            file_noext = os.path.splitext(filename)[0]
+	            timestamp_OCR = time.strftime("%Y-%m-%d-%H%M_OCR_")
+	            filename_OCR = timestamp_OCR + file_noext + '.pdf'
+	            docker_mount = dir_name + ':/home/docker'
+	# create string for pdf processing 
+	# diskstation needs a user:group docker:docker. find uid:gid of your diskstation docker:docker with id docker.
+	# use this uid:gid in -u flag
+	# rw rights for docker:docker at source dir are also necessary
+	# the script is processed as root user via chron 
+	            cmd = ['docker', 'run', '--rm', '-v', docker_mount, '-u="1030:65538"', 'jbarlow83/ocrmypdf', , '--deskew' , filename, filename_OCR]
+	            logging.info(cmd)
+	            proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+	            result = proc.stdout.read()
+	            logging.info(result)
+	            full_path_OCR = dir_name + '/' + filename_OCR
+	            os.chmod(full_path_OCR, 0o666)
+	            os.chmod(full_path, 0o666)
+	            full_path_OCR_archive = sys.argv[2]
+	            full_path_archive = sys.argv[2] + '/no_ocr'
+	            shutil.move(full_path_OCR,full_path_OCR_archive)
+	            shutil.move(full_path, full_path_archive)
+	logging.info('Finished.\n')
+
 
 
 Huge batch jobs
 """""""""""""""
 
-If you have thousands of files to work with, contact the author.
+If you have thousands of files to work with, contact the author. Consulting work related to OCRmyPDF helps fund this open source project and all inquiries are appreciated.
 
 
 Hot (watched) folders

docs/languages.rst

@@ -33,6 +33,34 @@ Docker users
 
 Users of the Docker image may use the alternative :ref:`"polyglot" container <docker-polyglot>` which includes all languages.
 
+Adding individual language packs to a Docker image
+""""""""""""""""""""""""""""""""""""""""""""""""""
+
+If you wish to add a single language pack, you could do the following:
+
+* Download the desired ``.trainedata`` file from the `tessdata <https://github.com/tesseract-ocr/tessdata>`_ repository. Let's use Hebrew in this example (``heb.traineddata``)
+
+* Copy the file to ``/home/user/downloads/heb.traineddata``.
+
+* Create a new container based on the ocrmypdf-tess4 image and jump into it with a terminal:
+
+.. code-block:: bash
+
+	host$ docker run  -v /home/user/downloads:/home/docker -it --entrypoint /bin/bash ocrmypdf-tess4
+
+* Put the file where Tesseract expects it:
+
+.. code-block:: bash
+
+	docker$ cp /home/docker/heb.traineddata /usr/share/tesseract-ocr/tessdata
+
+* Note the container id, and save it as a new image (in this example, ``ocrmypdf-tess4-heb``)
+
+.. code-block:: bash
+
+    host$ docker commit <container_id> ocrmypdf-tess4-heb
+
+
 Known limitations
 -----------------
 

docs/release_notes.rst

@@ -5,6 +5,19 @@ OCRmyPDF uses `semantic versioning <http://semver.org/>`_ for its command line i
 
 The OCRmyPDF package itself does not contain a public API, although it is fairly stable and breaking changes are usually timed with a major release. A future release will clearly define the stable public API.
 
+v5.4.1
+------
+
+-   Add workaround for Tesseract v4.00alpha crash when trying to obtain orientation and the latest language packs are installed
+
+
+v5.4
+----
+
+-   Change wording of a deprecation warning to improve clarity
+-   Added option to generate PDF/A-1b output if desired (``--output-type pdfa-1``); default remains PDF/A-2b generation
+-   Update documentation
+
 
 v5.3.3
 ------

ocrmypdf/__main__.py

@@ -1,29 +1,25 @@
 #!/usr/bin/env python3
 # © 2015-17 James R. Barlow: github.com/jbarlow83
 
-from contextlib import suppress
 from tempfile import mkdtemp
 from collections.abc import Sequence
 import sys
 import os
 import re
-import shutil
 import warnings
 import multiprocessing
 import atexit
 import textwrap
-import img2pdf
 import logging
 import argparse
 
 import PyPDF2 as pypdf
-from PIL import Image
 
 import ruffus.ruffus_exceptions as ruffus_exceptions
 import ruffus.cmdline as cmdline
 import ruffus.proxy_logger as proxy_logger
 
-from .pipeline import JobContext, JobContextManager, re_symlink, \
+from .pipeline import JobContext, JobContextManager, \
     cleanup_working_files, build_pipeline
 from .pdfa import file_claims_pdfa
 from .helpers import is_iterable_notstr, re_symlink, is_file_writable
@@ -134,12 +130,15 @@ parser.add_argument(
     '--image-dpi', metavar='DPI', type=int,
     help="For input image instead of PDF, use this DPI instead of file's.")
 parser.add_argument(
-    '--output-type', choices=['pdfa', 'pdf'], default='pdfa',
+    '--output-type', choices=['pdfa', 'pdf', 'pdfa-1', 'pdfa-2'], 
+    default='pdfa',
     help="Choose output type. 'pdfa' creates a PDF/A-2b compliant file for "
          "long term archiving (default, recommended) but may not suitable "
          "for users who want their file altered as little as possible. 'pdfa' "
          "also has problems with full Unicode text. 'pdf' attempts to "
-         "preserve file contents as much as possible.")
+         "preserve file contents as much as possible. 'pdf-a1' creates a "
+         "PDF/A1-b file. 'pdf-a2' is equivalent to 'pdfa'."
+         )
 
 # Use null string '\0' as sentinel to indicate the user supplied no argument,
 # since that is the only invalid character for filepaths on all platforms
@@ -343,13 +342,18 @@ def check_options_output(options, log):
         options.pdf_renderer = 'sandwich'
 
     if options.pdf_renderer == 'tesseract':
-        log.warning("The 'tesseract' PDF renderer is deprecated.")
         if tesseract.version() < '3.05' and options.output_type == 'pdfa':
             log.warning(
                 "For best results use --pdf-renderer=tesseract "
                 "--output-type=pdf to disable PDF/A generation via "
                 "Ghostscript, which is known to corrupt the OCR text of "
                 "some PDFs produced your version of Tesseract.")
+        elif tesseract.has_textonly_pdf():
+            log.warning(
+                "The argument --pdf-renderer=tesseract provides support for "
+                "versions of tesseract older than your version. For best "
+                "results omit this argument and let OCRmyPDF choose the "
+                "best available renderer.")
 
     if options.debug_rendering and options.pdf_renderer != 'hocr':
         log.info(
@@ -422,7 +426,7 @@ def check_options_ocr_behavior(options, log):
 def check_options_advanced(options, log):
     if tesseract.v4():
         log.info(
-            "Tesseract v4.x.alpha found. OCRmyPDF support is experimental.")
+            "Tesseract v4.x.alpha found.")
     if options.tesseract_oem and not tesseract.v4():
         log.warning(
             "--tesseract-oem requires Tesseract 4.x -- argument ignored")
@@ -440,9 +444,9 @@ def check_options_advanced(options, log):
 
 def check_options_metadata(options, log):
     import unicodedata
-    metadata = [options.title, options.author, options.keywords,
-                options.subject]
-    for s in (m for m in metadata if m):
+    docinfo = [options.title, options.author, options.keywords,
+               options.subject]
+    for s in (m for m in docinfo if m):
         for c in s:
             if unicodedata.category(c) == 'Co' or ord(c) >= 0x10000:
                 raise ValueError(
@@ -524,31 +528,31 @@ def cleanup_ruffus_error_message(msg):
 def do_ruffus_exception(ruffus_five_tuple, options, log):
     """Replace the elaborate ruffus stack trace with a user friendly
     description of the error message that occurred."""
+    exit_code = None
 
     task_name, job_name, exc_name, exc_value, exc_stack = ruffus_five_tuple
     if exc_name == 'builtins.SystemExit':
         match = re.search(r"\.(.+?)\)", exc_value)
         exit_code_name = match.groups()[0]
-        exit_code = getattr(ExitCode, exit_code_name, 'other_error')
-        return exit_code
+        exit_code = getattr(ExitCode, exit_code_name, 'other_error')        
     elif exc_name == 'ruffus.ruffus_exceptions.MissingInputFileError':
         log.error(cleanup_ruffus_error_message(exc_value))
-        return ExitCode.input_file
+        exit_code = ExitCode.input_file
     elif exc_name == 'builtins.TypeError':
         # Even though repair_pdf will fail, ruffus will still try
         # to call split_pages with no input files, likely due to a bug
         if task_name == 'split_pages':
             log.error("Input file '{0}' is not a valid PDF".format(
                 options.input_file))
-            return ExitCode.input_file
+            exit_code = ExitCode.input_file
     elif exc_name == 'builtins.KeyboardInterrupt':
         log.error("Interrupted by user")
-        return ExitCode.ctrl_c
+        exit_code = ExitCode.ctrl_c
     elif exc_name == 'subprocess.CalledProcessError':
         # It's up to the subprocess handler to report something useful
         msg = "Error occurred while running this command:"
         log.error(msg + '\n' + exc_value)
-        return ExitCode.child_process_error
+        exit_code = ExitCode.child_process_error
     elif exc_name == 'ocrmypdf.exceptions.PdfMergeFailedError':
         log.error(textwrap.dedent("""\
             Failed to merge PDF image layer with OCR layer
@@ -559,11 +563,11 @@ def do_ruffus_exception(ruffus_five_tuple, options, log):
             Try using
                 ocrmypdf --pdf-renderer tesseract  [..other args..]
             """))
-        return ExitCode.input_file
+        exit_code = ExitCode.input_file
     elif exc_name.startswith('ocrmypdf.exceptions.'):
         base_exc_name = exc_name.replace('ocrmypdf.exceptions.', '')
         exc_class = getattr(ocrmypdf_exceptions, base_exc_name)
-        return exc_class.exit_code
+        exit_code = exc_class.exit_code
     elif exc_name == 'PyPDF2.utils.PdfReadError' and \
             'not been decrypted' in exc_value:
         log.error(textwrap.dedent("""\
@@ -576,7 +580,10 @@ def do_ruffus_exception(ruffus_five_tuple, options, log):
             (Only algorithms "R = 1" and "R = 2" are supported.)
 
             """))
-        return ExitCode.encrypted_pdf
+        exit_code = ExitCode.encrypted_pdf
+
+    if exit_code is not None:
+        return exit_code
 
     if not options.verbose:
         log.error(exc_stack)
@@ -642,6 +649,20 @@ def check_closed_streams(options):
     return True
 
 
+def log_page_orientations(pdfinfo, _log):
+    direction = {0: 'n', 90: 'e',
+                180: 's', 270: 'w'}
+    orientations = []
+    for n, page in enumerate(pdfinfo):
+        angle = pdfinfo[n].rotation or 0
+        if angle != 0:
+            orientations.append('{0}{1}'.format(
+                n + 1,
+                direction.get(angle, '')))
+    if orientations:
+        _log.info('Page orientations detected: ' + ' '.join(orientations))
+
+
 def run_pipeline():
     options = parser.parse_args()
     options.verbose_abbreviated_path = 1
@@ -731,8 +752,7 @@ def run_pipeline():
             _log.error("Unexpected ruffus exception: " + str(e))
             _log.error(repr(e))
             return ExitCode.other_error
-        else:
-            return exitcode
+        return exitcode
     except ExitCodeException as e:
         return e.exit_code
     except Exception as e:
@@ -742,7 +762,7 @@ def run_pipeline():
     if options.flowchart:
         _log.info("Flowchart saved to {}".format(options.flowchart))
     elif options.output_file != '-':
-        if options.output_type == 'pdfa':
+        if options.output_type.startswith('pdfa'):
             pdfa_info = file_claims_pdfa(options.output_file)
             if pdfa_info['pass']:
                 msg = 'Output file is a {} (as expected)'
@@ -750,7 +770,6 @@ def run_pipeline():
             else:
                 msg = 'Output file is okay but is not PDF/A (seems to be {})'
                 _log.warning(msg.format(pdfa_info['conformance']))
-
                 return ExitCode.invalid_output_pdf
         if not qpdf.check(options.output_file, _log):
             _log.warning('Output file: The generated PDF is INVALID')
@@ -762,17 +781,8 @@ def run_pipeline():
     if options.verbose:
         from pprint import pformat
         _log.debug(pformat(pdfinfo))
-    direction = {0: 'n', 90: 'e',
-                 180: 's', 270: 'w'}
-    orientations = []
-    for n, page in enumerate(pdfinfo):
-        angle = pdfinfo[n].rotation or 0
-        if angle != 0:
-            orientations.append('{0}{1}'.format(
-                n + 1,
-                direction.get(angle, '')))
-    if orientations:
-        _log.info('Page orientations detected: ' + ' '.join(orientations))
+
+    log_page_orientations(pdfinfo, _log)
 
     return ExitCode.ok
 

ocrmypdf/exceptions.py

@@ -20,7 +20,7 @@ class ExitCode(IntEnum):
 
 
 class ExitCodeException(Exception):
-    pass
+    exit_code = ExitCode.other_error
 
 
 class PdfMergeFailedError(ExitCodeException):

ocrmypdf/exec/__init__.py

@@ -7,5 +7,6 @@ import os
 
 
 def get_program(name):
+    "Check environment variables for overrides to this program"
     envvar = 'OCRMYPDF_' + name.upper()
     return os.environ.get(envvar, name)

ocrmypdf/exec/ghostscript.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# © 2015 James R. Barlow: github.com/jbarlow83
+# © 2017 James R. Barlow: github.com/jbarlow83
 
 from tempfile import NamedTemporaryFile
 from subprocess import run, PIPE, STDOUT, CalledProcessError
@@ -7,9 +7,9 @@ from shutil import copy
 from functools import lru_cache
 import re
 import sys
-from . import get_program
-from ..exceptions import SubprocessOutputError
 from PIL import Image
+from . import get_program
+from ..exceptions import SubprocessOutputError, MissingDependencyError
 from ..helpers import fspath
 
 
@@ -20,15 +20,16 @@ def version():
         '--version'
     ]
     try:
-        version = check_output(
+        proc = run(
                 args_gs, close_fds=True, universal_newlines=True,
-                stderr=STDOUT)
+                stdout=PIPE, stderr=STDOUT, check=True)
+        ver = proc.stdout
     except CalledProcessError as e:
         print("Could not find Ghostscript executable on system PATH.",
               file=sys.stderr)
         raise MissingDependencyError from e
 
-    return version.strip()
+    return ver.strip()
 
 
 def _gs_error_reported(stream):
@@ -103,7 +104,7 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
 
 
 def generate_pdfa(pdf_pages, output_file, compression, log,
-                  threads=1, pdf_version='1.5'):
+                  threads=1, pdf_version='1.5', pdfa_part='2'):
     compression_args = []
     if compression == 'jpeg':
         compression_args = [
@@ -139,7 +140,7 @@ def generate_pdfa(pdf_pages, output_file, compression, log,
             "-sProcessColorModel=DeviceRGB"
         ] + compression_args + [
             "-dJPEGQ=95",
-            "-dPDFA=2",
+            "-dPDFA=" + pdfa_part,
             "-dPDFACompatibilityPolicy=1",
             "-sOutputFile=" + gs_pdf.name,
         ]
@@ -166,4 +167,4 @@ def generate_pdfa(pdf_pages, output_file, compression, log,
             copy(gs_pdf.name, output_file)
         else:
             log.error('Ghostscript PDF/A rendering failed')
-            raise SubprocessOutputError()
+            raise SubprocessOutputError()

ocrmypdf/exec/qpdf.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# © 2015 James R. Barlow: github.com/jbarlow83
+# © 2017 James R. Barlow: github.com/jbarlow83
 
 from subprocess import CalledProcessError, STDOUT, PIPE, run, check_output
 from functools import lru_cache
@@ -134,4 +134,4 @@ def merge(input_files, output_file, min_version=None):
         input_files[0], '--pages'
     ] + input_files + ['--', output_file]
     run(args_qpdf, check=True)
-
+    

ocrmypdf/exec/tesseract.py

@@ -1,21 +1,20 @@
 #!/usr/bin/env python3
-# © 2015 James R. Barlow: github.com/jbarlow83
+# © 2017 James R. Barlow: github.com/jbarlow83
 
 import sys
 import os
 import re
 import shutil
 from functools import lru_cache
-from ..exceptions import MissingDependencyError, TesseractConfigError
-from ..helpers import page_number
-from . import get_program
 from collections import namedtuple
 from textwrap import dedent
 import PyPDF2 as pypdf
+from subprocess import PIPE, CalledProcessError, \
+    TimeoutExpired, check_output, STDOUT
 
-from subprocess import Popen, PIPE, CalledProcessError, \
-    TimeoutExpired, check_output, STDOUT, DEVNULL
-
+from ..exceptions import MissingDependencyError, TesseractConfigError
+from ..helpers import page_number
+from . import get_program
 
 OrientationConfidence = namedtuple(
     'OrientationConfidence',
@@ -60,7 +59,7 @@ def version():
 
 def v4():
     "Is this Tesseract v4.0?"
-    return (version() >= '4')
+    return version() >= '4'
 
 
 @lru_cache(maxsize=1)
@@ -74,6 +73,7 @@ def has_textonly_pdf():
         get_program('tesseract'),
         '--print-parameters'
     ]
+    params = ''
     try:
         params = check_output(
                 args_tess, close_fds=True, universal_newlines=True,
@@ -113,12 +113,12 @@ def languages():
     return set(lang.strip() for lang in langs.splitlines()[1:])
 
 
-def tess_base_args(languages, engine_mode):
+def tess_base_args(langs, engine_mode):
     args = [
         get_program('tesseract'),
     ]
-    if languages:
-        args.extend(['-l', '+'.join(languages)])
+    if langs:
+        args.extend(['-l', '+'.join(langs)])
     if engine_mode is not None and v4():
         args.extend(['--oem', str(engine_mode)])
     return args
@@ -126,7 +126,7 @@ def tess_base_args(languages, engine_mode):
 
 def get_orientation(input_file, language: list, engine_mode, timeout: float,
                     log):
-    args_tesseract = tess_base_args(language, engine_mode) + [
+    args_tesseract = tess_base_args(['osd'], engine_mode) + [
         psm(), '0',
         input_file,
         'stdout'

ocrmypdf/exec/unpaper.py

@@ -3,7 +3,7 @@
 # unpaper documentation:
 # https://github.com/Flameeyes/unpaper/blob/master/doc/basic-concepts.md
 
-from subprocess import CalledProcessError, STDOUT, check_output, check_call
+from subprocess import CalledProcessError, STDOUT, check_output
 from tempfile import NamedTemporaryFile
 import sys
 import os
@@ -18,10 +18,10 @@ def version():
         get_program('unpaper'),
         '--version'
     ]
-    version = check_output(
+    ver = check_output(
         args_unpaper, close_fds=True, universal_newlines=True,
         stderr=STDOUT, timeout=5)
-    return version.strip()
+    return ver.strip()
 
 
 try:

ocrmypdf/lib/compile_leptonica.py

@@ -183,10 +183,39 @@ pixColorMagnitude(PIX     *pixs,
                   l_int32  bwhite,
                   l_int32  type);
 
-void
+PIX *
+pixMaskOverColorPixels(PIX     *pixs,
+                       l_int32  threshdiff,
+                       l_int32  mindist);
+
+l_int32
+pixGetAverageMaskedRGB(PIX        *pixs,
+                       PIX        *pixm,
+                       l_int32     x,
+                       l_int32     y,
+                       l_int32     factor,
+                       l_int32     type,
+                       l_float32  *prval,
+                       l_float32  *pgval,
+                       l_float32  *pbval);
+
+PIX * 
+pixGlobalNormRGB(PIX * 	pixd,
+                 PIX * 	pixs,
+                 l_int32 	rval,
+                 l_int32 	gval,
+                 l_int32 	bval,
+                 l_int32 	mapval); 
+
+PIX *
+pixInvert(PIX * pixd,
+          PIX * pixs);
+
+void                 
 boxDestroy(BOX  **pbox);
 
-void lept_free(void *ptr);
+void
+lept_free(void *ptr);  
 """)
 
 

ocrmypdf/pipeline.py

@@ -2,19 +2,10 @@
 # © 2016 James R. Barlow: github.com/jbarlow83
 
 from contextlib import suppress
-from tempfile import mkdtemp
-from functools import partial
 import sys
 import os
-import re
 import shutil
-import warnings
-import multiprocessing
-import atexit
-import textwrap
 import img2pdf
-import logging
-import argparse
 
 import PyPDF2 as pypdf
 from PIL import Image
@@ -23,7 +14,7 @@ from ruffus import formatter, regex, Pipeline, suffix
 
 from .hocrtransform import HocrTransform
 from .pdfinfo import PdfInfo, Encoding, Colorspace
-from .pdfa import generate_pdfa_ps, file_claims_pdfa
+from .pdfa import generate_pdfa_ps
 from .helpers import re_symlink, is_iterable_notstr, page_number
 from .exec import ghostscript, tesseract, qpdf
 from .exceptions import *
@@ -51,6 +42,8 @@ class JobContext:
 
     def __init__(self):
         self.pdfinfo = None
+        self.options = None
+        self.work_folder = None
 
     def generate_pdfinfo(self, infile):
         self.pdfinfo = PdfInfo(infile)
@@ -189,7 +182,8 @@ def repair_pdf(
     pdfinfo = PdfInfo(output_file)
 
     if pdfinfo.has_userunit and options.output_type == 'pdfa':
-        log.error("This input file uses a PDF feature that is not supported "
+        log.error(
+            "This input file uses a PDF feature that is not supported "
             "by Ghostscript, so you cannot use --output-type=pdfa for this "
             "file. (Specifically, it uses the PDF-1.6 /UserUnit feature to "
             "support very large or small page sizes, and Ghostscript cannot "
@@ -706,7 +700,7 @@ def combine_layers(
 
     if rotation != 0:
         log.info("{0:4d}: rotating image layer {1} degrees".format(
-            page_number(image), rotation, tx, ty))
+            page_number(image), rotation))
 
     try:
         page_text.mergeRotatedScaledTranslatedPage(
@@ -826,9 +820,9 @@ def get_pdfmark(base_pdf, options):
         renderer_tag = 'OCR'
 
     pdfmark['/Creator'] = '{0} {1} / Tesseract {2} {3}'.format(
-            PROGRAM_NAME, VERSION,
-            renderer_tag,
-            tesseract.version())
+        PROGRAM_NAME, VERSION,
+        renderer_tag,
+        tesseract.version())
     return pdfmark
 
 
@@ -885,7 +879,8 @@ def merge_pages_ghostscript(
         output_file=output_file,
         compression=options.pdfa_image_compression,
         log=log,
-        threads=options.jobs or 1)
+        threads=options.jobs or 1,
+        pdfa_part=('1' if options.output_type == 'pdfa-1' else '2'))
 
 
 def merge_pages_qpdf(
@@ -936,7 +931,6 @@ def merge_sidecars(
         output_file,
         log,
         context):
-    options = context.get_options()
     pdfinfo = context.get_pdfinfo()
 
     txt_files = [None] * len(pdfinfo)
@@ -947,15 +941,24 @@ def merge_sidecars(
             txt_files[idx] = infile
 
     def write_pages(stream):
-        for page_number, txt_file in enumerate(txt_files):
-            if page_number != 0:
+        for page_num, txt_file in enumerate(txt_files):
+            if page_num != 0:
                 stream.write('\f')  # Form feed between pages
             if txt_file:
                 with open(txt_file, 'r') as in_:
-                    stream.write(in_.read())
+                    txt = in_.read()
+                    # Tesseract v4 alpha started adding form feeds in 
+                    # commit aa6eb6b
+                    # No obvious way to detect what binaries will do this, so
+                    # for consistency just ignore its form feeds and insert our
+                    # own
+                    if txt.endswith('\f'):
+                        stream.write(in_.read()[:-1])
+                    else:
+                        stream.write(in_.read())
             else:
                 stream.write('[OCR skipped on page {}]'.format(
-                        page_number + 1))
+                        page_num + 1))
 
     if output_file == '-':
         write_pages(sys.stdout)
@@ -1139,7 +1142,8 @@ def build_pipeline(options, work_folder, log, context):
         output=os.path.join(work_folder, r'\1.rendered.pdf'),
         extras=[log, context])
     task_combine_layers.graphviz(fillcolor='"#00cc66"')
-    task_combine_layers.active_if(options.pdf_renderer == 'hocr' or options.pdf_renderer == 'sandwich')
+    task_combine_layers.active_if(options.pdf_renderer == 'hocr' or 
+                                  options.pdf_renderer == 'sandwich')
 
     # Tesseract OCR+PDF
     task_ocr_tesseract_and_render_pdf = main_pipeline.collate(
@@ -1161,7 +1165,7 @@ def build_pipeline(options, work_folder, log, context):
         filter=formatter(r'\.repaired\.pdf'),
         output=os.path.join(work_folder, 'pdfa.ps'),
         extras=[log, context])
-    task_generate_postscript_stub.active_if(options.output_type == 'pdfa')
+    task_generate_postscript_stub.active_if(options.output_type.startswith('pdfa'))
 
 
     # Bypass valve
@@ -1183,7 +1187,7 @@ def build_pipeline(options, work_folder, log, context):
                task_generate_postscript_stub],
         output=os.path.join(work_folder, 'merged.pdf'),
         extras=[log, context])
-    task_merge_pages_ghostscript.active_if(options.output_type == 'pdfa')
+    task_merge_pages_ghostscript.active_if(options.output_type.startswith('pdfa'))
 
     task_merge_pages_qpdf = main_pipeline.merge(
         task_func=merge_pages_qpdf,

tests/resources/README.rst

@@ -57,7 +57,7 @@ under the terms of the license in LICENSE.rst.
         - test image with 200 x 100 DPI resolution
     *   - blank.pdf
         - @jbarlow83
-        - blank PDF
+        - blank PDF generated by Adobe Illustrator CC 17, containing a lot of application-specific metadata/bloat
     *   - cmyk.pdf
         - @jbarlow83
         - a CMYK image created in Photoshop

tests/test_main.py

@@ -982,3 +982,14 @@ def test_sidecar_pagecount(spoof_tesseract_cache, resources, outpdf):
     # formfeeds is the page count less one
     assert ocr_text.count('\f') == num_pages - 1, \
         "Sidecar page count does not match PDF page count"
+
+
+def test_pdfa_1(spoof_tesseract_cache, resources, outpdf):
+    check_ocrmypdf(
+        resources / 'ccitt.pdf', outpdf,
+        '--output-type', 'pdfa-1',
+        env=spoof_tesseract_cache
+    )
+
+    pdfa_info = file_claims_pdfa(outpdf)
+    assert pdfa_info['conformance'] == 'PDF/A-1B'