Help text: example of shell pipeline with img2pdf

RELEASE_NOTES.rst

@@ -2,6 +2,19 @@ RELEASE NOTES
 =============
 
 
+v4.2.2:
+=======
+
+-  Improvements to documentation
+
+
+v4.2.1:
+=======
+
+-  Fixed an issue where PDF pages that contained stencil masks would report an incorrect DPI and cause Ghostscript to abort
+-  Implemented stdin streaming
+
+
 v4.2:
 =====
 

ocrmypdf/main.py

@@ -13,6 +13,7 @@ import atexit
 import textwrap
 import img2pdf
 import logging
+import argparse
 
 import PyPDF2 as pypdf
 from PIL import Image
@@ -103,23 +104,70 @@ check_pil_encoder('zlib', 'PNG')
 
 parser = cmdline.get_argparse(
     prog="ocrmypdf",
-    description="Generate searchable PDF file from an image-only PDF file.",
     version=VERSION,
     fromfile_prefix_chars='@',
     ignored_args=[
         'touch_files_only', 'recreate_database', 'checksum_file_name',
         'key_legend_in_graph', 'draw_graph_horizontally', 'flowchart_format',
-        'forced_tasks', 'target_tasks', 'use_threads', 'jobs', 'log_file'])
+        'forced_tasks', 'target_tasks', 'use_threads', 'jobs', 'log_file'],
+    formatter_class=argparse.RawDescriptionHelpFormatter,
+    description="""\
+Generates a searchable PDF or PDF/A from a regular PDF.
+
+OCRmyPDF rasterizes each page of the input PDF, optionally corrects page
+rotation and performs image processing, runs the Tesseract OCR engine on the
+image, and then creates a PDF from the OCR information.
+""",
+    epilog="""\
+OCRmyPDF attempts to keep the output file at about the same size.  If a file
+contains losslessly compressed images, and output file will be losslessly
+compressed as well.
+
+PDF is a page description file that attempts to preserve a layout exactly.
+A PDF can contain vector objects (such as text or lines) and raster objects
+(images).  A page might have multiple images.  OCRmyPDF is prepared to deal
+with the wide variety of PDFs that exist in the wild.
+
+When a PDF page contains text, OCRmyPDF assumes that the page has already
+been OCRed or is a "born digital" page that should not be OCRed.  The default
+behavior is to exit in this case without producing a file.  You can use the
+option --skip-text to ignore pages with text, or --force-ocr to rasterize
+all objects on the page and produce an image-only PDF as output.
+
+    ocrmypdf --skip-text file_with_some_text_pages.pdf output.pdf
+
+    ocrmypdf --force-ocr word_document.pdf output.pdf
+
+If you are concerned about long-term archiving of PDFs, use the default option
+--output-type pdfa which converts the PDF to a standardized PDF/A-2b.  This
+converts images to sRGB colorspace, removes some features from the PDF such
+as Javascript or forms. If you want to minimize the number of changes made to
+your PDF, use --output-type pdf.
+
+If OCRmyPDF is given an image file as input, it will attempt to convert the
+image to a PDF before processing.  For more control over the conversion of
+images to PDF, use the Python package img2pdf or other image to PDF software.
+
+For example, this command uses img2pdf to convert all .png files beginning
+with the 'page' prefix to a PDF, fitting each image on A4-sized paper, and
+sending the result to OCRmyPDF through a pipe.  img2pdf is a dependency of
+ocrmypdf so it is already installed.
+
+    img2pdf --pagesize A4 page*.png | ocrmypdf - myfile.pdf
+
+""")
 
 parser.add_argument(
     'input_file',
-    help="PDF file containing the images to be OCRed")
+    help="PDF file containing the images to be OCRed (or '-' to read from "
+         "standard input)")
 parser.add_argument(
     'output_file',
     help="output searchable PDF file")
 parser.add_argument(
     '-l', '--language', action='append',
-    help="languages of the file to be OCRed")
+    help="languages of the file to be OCRed (see tesseract --list-langs for "
+         "all language packs installed in your system)")
 parser.add_argument(
     '-j', '--jobs', metavar='N', type=int,
     help="Use up to N CPU cores simultaneously (default: use all)")
@@ -151,8 +199,8 @@ metadata.add_argument(
     help="set document keywords")
 
 preprocessing = parser.add_argument_group(
-    "Preprocessing options",
-    "Improve OCR quality and final image")
+    "Image preprocessing options",
+    "Options to improve the quality of the final PDF and OCR")
 preprocessing.add_argument(
     '-r', '--rotate-pages', action='store_true',
     help="automatically rotate pages based on detected text orientation")
@@ -161,10 +209,13 @@ preprocessing.add_argument(
     help="deskew each page before performing OCR")
 preprocessing.add_argument(
     '-c', '--clean', action='store_true',
-    help="clean pages from scanning artifacts before performing OCR")
+    help="clean pages from scanning artifacts before performing OCR, and send "
+         "the cleaned page to OCR, but do not include the cleaned page in "
+         "the output ")
 preprocessing.add_argument(
     '-i', '--clean-final', action='store_true',
-    help="incorporate the cleaned image in the final PDF file")
+    help="clean page as above, and incorporate the cleaned image in the final "
+         "PDF")
 preprocessing.add_argument(
     '--oversample', metavar='DPI', type=int, default=0,
     help="oversample images to at least the specified DPI, to improve OCR "
@@ -175,11 +226,13 @@ ocrsettings = parser.add_argument_group(
     "Control how OCR is applied")
 ocrsettings.add_argument(
     '-f', '--force-ocr', action='store_true',
-    help="rasterize any fonts or vector images on each page and apply OCR")
+    help="rasterize any fonts or vector objects on each page, apply OCR, and "
+         "save the rastered output (this rewrites the PDF)")
 ocrsettings.add_argument(
     '-s', '--skip-text', action='store_true',
-    help="skip OCR on any pages that already contain text, but include the"
-         " page in final output")
+    help="skip OCR on any pages that already contain text, but include the "
+         "page in final output; useful for PDFs that contain a mix of "
+         "images, text pages, and/or previously OCRed pages")
 ocrsettings.add_argument(
     '--skip-big', type=float, metavar='MPixels',
     help="skip OCR on pages larger than the specified amount of megapixels, "
@@ -196,7 +249,13 @@ advanced.add_argument(
     help="set Tesseract page segmentation mode (see tesseract --help)")
 advanced.add_argument(
     '--pdf-renderer', choices=['auto', 'tesseract', 'hocr'], default='auto',
-    help='choose OCR PDF renderer')
+    help="choose OCR PDF renderer - the default option is to let OCRmyPDF "
+         "choose.  The 'tesseract' PDF renderer is more accurate and does a "
+         "better job and document structure such as recognizing columns. It "
+         "also does a better job on non-Latin languages. However, it does "
+         "not work as well when older versions of Tesseract or Ghostscript "
+         "are installed, and some combinations of arguments to do not work "
+         "with --pdf-renderer tesseract.")
 advanced.add_argument(
     '--tesseract-timeout', default=180.0, type=float, metavar='SECONDS',
     help='give up on OCR after the timeout, but copy the preprocessed page '
@@ -1330,6 +1389,7 @@ def run_pipeline():
 
         if options.input_file == '-':
             # stdin
+            _log.info('reading file from standard input')
             with open(start_input_file, 'wb') as stream_buffer:
                 from shutil import copyfileobj
                 copyfileobj(sys.stdin.buffer, stream_buffer)
@@ -1374,9 +1434,12 @@ def run_pipeline():
     if options.output_type == 'pdfa':
         pdfa_info = file_claims_pdfa(options.output_file)
         if pdfa_info['pass']:
-            _log.info(pdfa_info['message'])
+            msg = 'Output file is a {} (as expected)'
+            _log.info(msg.format(pdfa_info['conformance']))
         else:
-            _log.warning(pdfa_info['message'])
+            msg = 'Output file was generated but is not PDF/A (seems to be {})'
+            _log.warning(msg.format(pdfa_info['conformance']))
+
             return ExitCode.invalid_output_pdf
 
     if not qpdf.check(options.output_file, _log):

ocrmypdf/pdfa.py

@@ -135,7 +135,8 @@ def file_claims_pdfa(filename):
             aboutUri='',
             namespace='http://www.aiim.org/pdfa/ns/id/')
     except AttributeError:
-        return {'pass': False, 'output': 'pdf', 'message': 'No XMP metadata'}
+        return {'pass': False, 'output': 'pdf',
+                'conformance': 'No XMP metadata'}
 
     pdfa_dict = {attr.localName: attr.value for attr in pdfa_nodes}
     pdfa_dict['pass'] = False
@@ -144,15 +145,15 @@ def file_claims_pdfa(filename):
         part_conformance = pdfa_dict['part'] + pdfa_dict['conformance']
         valid_part_conforms = {'1A', '1B', '2A', '2B', '2U', '3A', '3B', '3U'}
 
-        message = 'File claims to be PDF/A-{}'.format(
+        conformance = 'PDF/A-{}'.format(
             part_conformance)
 
         if part_conformance in valid_part_conforms:
             pdfa_dict['pass'] = True
             pdfa_dict['output'] = 'pdfa'
-        pdfa_dict['message'] = message
+        pdfa_dict['conformance'] = conformance
     else:
-        pdfa_dict['message'] = 'File is a regular PDF'
+        pdfa_dict['conformance'] = 'PDF'
 
     return pdfa_dict
 

setup.py

@@ -6,7 +6,7 @@ from __future__ import print_function, unicode_literals
 
 import sys
 if sys.version_info < (3, 4):
-    print("Python 3.4 or newer is required")
+    print("Python 3.4 or newer is required", file=sys.stderr)
     sys.exit(1)
 
 from setuptools import setup  # nopep8

tests/test_main.py

@@ -2,7 +2,7 @@
 # © 2015 James R. Barlow: github.com/jbarlow83
 
 from __future__ import print_function
-from subprocess import Popen, PIPE, check_output, check_call
+from subprocess import Popen, PIPE, check_output, check_call, DEVNULL
 import os
 import shutil
 from contextlib import suppress
@@ -609,3 +609,20 @@ def test_jbig2_passthrough(spoof_tesseract_cache):
     out_pageinfo = pdf_get_all_pageinfo(out)
     assert out_pageinfo[0]['images'][0]['enc'] == 'jbig2'
 
+
+def test_stdin(spoof_tesseract_noop):
+    input_file = _infile('francais.pdf')
+    output_file = _outfile('test_stdin.pdf')
+
+    p1_args = ['cat', input_file]
+    p1 = Popen(p1_args, close_fds=True, stdin=DEVNULL, stdout=PIPE)
+
+    p2_args = ['ocrmypdf', '-', output_file]
+    p2 = Popen(
+        p2_args, close_fds=True, stdout=PIPE, stderr=PIPE,
+        stdin=p1.stdout, env=spoof_tesseract_noop)
+    p1.stdout.close()
+    out, err = p2.communicate()
+
+    assert p2.returncode == ExitCode.ok
+